reeval 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
- package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
- package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
- package/dist/default-effort-DGhQmu95.js.map +1 -0
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
- package/dist/eval-source-D3sbAzt-.js.map +1 -0
- package/dist/evaluate-COdzf7Hl.d.ts +92 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
- package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
- package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/harness-C2berMiM.js +105 -0
- package/dist/harness-C2berMiM.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/ink-list-BpfidQ1h.js +274 -0
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
- package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-CzgpsTqv.js +246 -0
- package/dist/judgments-CzgpsTqv.js.map +1 -0
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
- package/dist/live-CgYbO4Te.js.map +1 -0
- package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
- package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
- package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
- package/dist/questions-CWlR9ffu.js.map +1 -0
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/remarks-DPCONFxZ.js +138 -0
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/resume-AW5oXzUc.js +691 -0
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
- package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/suite-COglUhPd.js +1272 -0
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
- package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
- package/dist/vertex-Eb2wqahk.js.map +1 -0
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +238 -90
- package/dist/view/index.js +11 -9
- package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
- package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +26 -11
- package/skills/eval-rubric/SKILL.md +31 -4
- package/dist/adapter-DyvDj4ZN.js.map +0 -1
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/command-TLmY-rYp.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-DtSX8E_V.js +0 -101
- package/dist/contrasts-DtSX8E_V.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker--2ytOHcu.js.map +0 -1
- package/dist/doctor-eE65YpEv.js.map +0 -1
- package/dist/environment-Cy3FVLR5.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-BoDKeuJu.js.map +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-BVZHDf_z.js.map +0 -1
- package/dist/init-BWnMbUnc.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/judge-ByEUlzOM.js.map +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/node-CwP1G178.js.map +0 -1
- package/dist/opencode-providers-yNKyXD49.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/plugins-BEs2Am3L.js.map +0 -1
- package/dist/preflight-50Qw5e5K.js.map +0 -1
- package/dist/provenance-Cc8fFE7Z.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-CP_bBtYm.js +0 -1
- package/dist/providers-DZSTlC9R.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-DeZZT8PK.js +0 -116
- package/dist/remarks-DeZZT8PK.js.map +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resolve-B_JlQ0La.js.map +0 -1
- package/dist/resume-CexsWuGK.js +0 -434
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/runtime-CSZ1hjlN.js.map +0 -1
- package/dist/sandbox-B5W8lLL1.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BKDZO3ra.js +0 -2
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/src-C9elednc.js +0 -2
- package/dist/src-CM7xdJFA.js +0 -2
- package/dist/src-CWEGknZo.js +0 -2
- package/dist/src-Cpy9MNS3.js.map +0 -1
- package/dist/src-DVaAKCsH.js +0 -1
- package/dist/src-DurP--_D.js +0 -2
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CKoN5NAx.js.map +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -1,32 +1,33 @@
|
|
|
1
|
-
import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
3
|
-
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { a as
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { r as
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
2
|
+
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-Cfi45Bvy.js";
|
|
3
|
+
import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
4
|
+
import { _ as snapshotProblem, a as egressPolicySchema, c as gatePolicySchema, g as snapshotOptionSchema, l as gateProblem, s as envVarsSchema } from "./plain-BYUf_fUV.js";
|
|
5
|
+
import { a as isSandboxKind, d as sandboxKind, f as suggestedSandbox, i as idPart, n as capabilitiesOf, o as maxConcurrentProblem, t as SANDBOX_KINDS, u as sandboxHelpers } from "./kinds-C1oY3qyC.js";
|
|
6
|
+
import { C as plainCredentialHeaders, D as sandboxOf, E as sameWorkspaceOnly, F as environmentDataSchema, I as sandboxDescriptorSchema, L as traitsSchema, M as unbrokeredSecrets, N as workspaceLabel, O as sandboxPaths, P as workspaceOf, _ as isMovingRef, a as checkEgress, c as environmentDataProblem, h as isEnvironment, j as traitsOf, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isSecret, w as plannedSecrets, z as sandboxProblem } from "./environment-DB8o3hGi.js";
|
|
7
|
+
import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DXv2gnaT.js";
|
|
8
|
+
import { c as sha256, i as fromCwd } from "./ids-C_advdJA.js";
|
|
9
|
+
import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-BGiXt8_B.js";
|
|
10
|
+
import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-Eb2wqahk.js";
|
|
11
|
+
import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
|
|
12
|
+
import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-COMXT1Hx.js";
|
|
13
|
+
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
14
14
|
import { u as ownFileURL } from "./root-Co0jhOT5.js";
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import { n as
|
|
18
|
-
import "./
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
15
|
+
import { B as PACKAGE_HOSTS, z as HARNESS_HOSTS } from "./src-Ck2r_EMT.js";
|
|
16
|
+
import { y as localPluginServers } from "./plugins-BkmPxDiY.js";
|
|
17
|
+
import { a as agentFor, b as isRoutingGateway, f as isAgentKind, n as DEFAULT_HARNESS, r as HARNESS_EXAMPLE, t as AGENTS, u as credentialOf, y as ROUTING_GATEWAYS } from "./agents-C6zhkxp3.js";
|
|
18
|
+
import { i as pairFactors } from "./pairs-BGb134Vu.js";
|
|
19
|
+
import { u as opencodeModelError } from "./harness-C2berMiM.js";
|
|
20
|
+
import { r as tartTagWarning } from "./tart-ref-DYZijjrU.js";
|
|
21
|
+
import "./stats-C18W4xPX.js";
|
|
22
|
+
import { n as gateLabel } from "./gate-CgYdaprG.js";
|
|
23
|
+
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-C8xPuXYq.js";
|
|
24
|
+
import { t as docker } from "./sandbox-kSgr7vo5.js";
|
|
25
|
+
import { z } from "zod";
|
|
21
26
|
import { existsSync, readFileSync } from "node:fs";
|
|
22
|
-
import { availableParallelism,
|
|
23
|
-
import { dirname, join, resolve
|
|
27
|
+
import { availableParallelism, totalmem } from "node:os";
|
|
28
|
+
import { dirname, join, resolve } from "node:path";
|
|
24
29
|
import { parseDocument } from "yaml";
|
|
25
|
-
import {
|
|
26
|
-
import { connect, isIP } from "node:net";
|
|
27
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
28
|
-
import { spawn } from "node:child_process";
|
|
29
|
-
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
30
|
+
import { isIP } from "node:net";
|
|
30
31
|
// src/judge/artifacts.ts
|
|
31
32
|
/** A capture handle: a name plus the file the attempt wrote. */
|
|
32
33
|
function isCaptureHandle(value) {
|
|
@@ -88,20 +89,22 @@ const sharedInputFields = {
|
|
|
88
89
|
task: z.boolean().optional(),
|
|
89
90
|
output: z.boolean().optional(),
|
|
90
91
|
files: z.boolean().optional(),
|
|
91
|
-
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
92
|
+
diff: z.union([z.boolean(), z.array(nonEmpty$1()).min(1)]).optional(),
|
|
92
93
|
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
93
|
-
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
94
|
+
checks: z.union([z.boolean(), z.array(nonEmpty$1())]).optional(),
|
|
95
|
+
observations: z.union([z.boolean(), z.array(nonEmpty$1())]).optional()
|
|
94
96
|
};
|
|
95
97
|
const turnsSchema = z.enum(["last", "all"]);
|
|
96
|
-
const { task, output, files, diff, transcript, checks } = sharedInputFields;
|
|
98
|
+
const { task, output, files, diff, transcript, checks, observations } = sharedInputFields;
|
|
97
99
|
const judgeInputsSchema = z.strictObject({
|
|
98
100
|
task,
|
|
99
101
|
output,
|
|
100
102
|
files,
|
|
101
103
|
diff,
|
|
102
|
-
contents: z.array(nonEmpty()).optional(),
|
|
104
|
+
contents: z.array(nonEmpty$1()).optional(),
|
|
103
105
|
transcript,
|
|
104
106
|
checks,
|
|
107
|
+
observations,
|
|
105
108
|
artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
|
|
106
109
|
});
|
|
107
110
|
const judgeSubjectSchema = z.strictObject({
|
|
@@ -149,7 +152,8 @@ const VALUES$1 = {
|
|
|
149
152
|
contents: "a list of paths or patterns",
|
|
150
153
|
transcript: "true, false, 'tools' or 'full'",
|
|
151
154
|
turns: "'last' or 'all'",
|
|
152
|
-
checks: "true, false or a list of commands"
|
|
155
|
+
checks: "true, false or a list of commands",
|
|
156
|
+
observations: "true, false or a list of names"
|
|
153
157
|
};
|
|
154
158
|
function distance(a, b) {
|
|
155
159
|
const row = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
@@ -239,7 +243,7 @@ function judgeProblem(judge, site) {
|
|
|
239
243
|
const pairFactorSchema = z.enum(["environment", "harness"]);
|
|
240
244
|
/** Evaluation models score yes/no questions only, so they can't answer a question. */
|
|
241
245
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
242
|
-
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
|
|
246
|
+
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty$1()).min(1)]);
|
|
243
247
|
const questionInputFields = {
|
|
244
248
|
...sharedInputFields,
|
|
245
249
|
artifacts: questionArtifactsSchema.optional()
|
|
@@ -257,9 +261,9 @@ const unique = (list, ctx) => {
|
|
|
257
261
|
});
|
|
258
262
|
};
|
|
259
263
|
const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
|
|
260
|
-
const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
|
|
264
|
+
const choicesSchema = z.array(nonEmpty$1()).min(2).superRefine(unique);
|
|
261
265
|
const questionSchema = z.strictObject({
|
|
262
|
-
question: nonEmpty(),
|
|
266
|
+
question: nonEmpty$1(),
|
|
263
267
|
model: questionModelSchema.optional(),
|
|
264
268
|
across: acrossSchema.optional(),
|
|
265
269
|
choices: choicesSchema.optional(),
|
|
@@ -296,7 +300,8 @@ const VALUES = {
|
|
|
296
300
|
diff: "true, false or a non-empty list of patterns",
|
|
297
301
|
transcript: "true, false, 'tools' or 'full'",
|
|
298
302
|
turns: "'last' or 'all'",
|
|
299
|
-
checks: "true, false or a list of commands"
|
|
303
|
+
checks: "true, false or a list of commands",
|
|
304
|
+
observations: "true, false or a list of names"
|
|
300
305
|
};
|
|
301
306
|
const describe$2 = (value) => {
|
|
302
307
|
if (Array.isArray(value)) return `a list of ${value.length}`;
|
|
@@ -344,8 +349,8 @@ function acrossProblem(question, across, where) {
|
|
|
344
349
|
function choicesProblem(question, choices, where) {
|
|
345
350
|
if (choicesSchema.safeParse(choices).success) return void 0;
|
|
346
351
|
const at = `questions "${question}" choices in ${where}`;
|
|
347
|
-
if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
|
|
348
|
-
const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty().safeParse(c).success);
|
|
352
|
+
if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty$1().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
|
|
353
|
+
const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty$1().safeParse(c).success);
|
|
349
354
|
if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
|
|
350
355
|
return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
|
|
351
356
|
}
|
|
@@ -359,7 +364,7 @@ function questionProblem(item, where) {
|
|
|
359
364
|
return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
|
|
360
365
|
}
|
|
361
366
|
const { question, model, across, choices } = item;
|
|
362
|
-
if (!nonEmpty().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
|
|
367
|
+
if (!nonEmpty$1().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
|
|
363
368
|
const text = question;
|
|
364
369
|
if (across !== void 0) {
|
|
365
370
|
const problem = acrossProblem(text, across, where);
|
|
@@ -398,6 +403,7 @@ const captureOptionsSchema = z.strictObject({
|
|
|
398
403
|
model: z.string().includes("/").optional()
|
|
399
404
|
});
|
|
400
405
|
const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
|
|
406
|
+
const environmentList = z.union([environmentDataSchema, z.array(environmentDataSchema).min(1)]);
|
|
401
407
|
const budgetSchema = z.strictObject({
|
|
402
408
|
attempts: count().optional(),
|
|
403
409
|
duration: durationSchema.optional()
|
|
@@ -416,9 +422,9 @@ const recordMode = z.enum([
|
|
|
416
422
|
"always"
|
|
417
423
|
]);
|
|
418
424
|
const profileDefaultsDataSchema = z.strictObject({
|
|
419
|
-
|
|
425
|
+
harnesses: harnessList.optional(),
|
|
420
426
|
sandbox: sandboxDescriptorSchema.optional(),
|
|
421
|
-
|
|
427
|
+
environments: environmentList.optional(),
|
|
422
428
|
runs: runs().optional(),
|
|
423
429
|
gate: gatePolicySchema.optional(),
|
|
424
430
|
env: envVarsSchema.optional(),
|
|
@@ -432,7 +438,8 @@ const profileDefaultsDataSchema = z.strictObject({
|
|
|
432
438
|
const profileLimitSchema = z.strictObject({
|
|
433
439
|
runs: count().optional(),
|
|
434
440
|
compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional(),
|
|
435
|
-
|
|
441
|
+
harnesses: z.array(z.string()).optional(),
|
|
442
|
+
traits: z.record(z.string(), z.union([z.string(), z.array(z.string()).min(1)])).optional()
|
|
436
443
|
});
|
|
437
444
|
const profileDataSchema = z.strictObject({
|
|
438
445
|
defaults: profileDefaultsDataSchema.optional(),
|
|
@@ -477,9 +484,9 @@ const analysisConfigDataSchema = z.strictObject({
|
|
|
477
484
|
});
|
|
478
485
|
const reevalConfigDataSchema = z.strictObject({
|
|
479
486
|
include: z.array(z.string()).optional(),
|
|
480
|
-
|
|
487
|
+
harnesses: harnessList.optional(),
|
|
481
488
|
sandbox: sandboxDescriptorSchema.optional(),
|
|
482
|
-
|
|
489
|
+
environments: environmentList.optional(),
|
|
483
490
|
runs: runs().optional(),
|
|
484
491
|
gate: gatePolicySchema.optional(),
|
|
485
492
|
env: envVarsSchema.optional(),
|
|
@@ -559,6 +566,7 @@ function configDataProblem(value, file) {
|
|
|
559
566
|
}
|
|
560
567
|
if (issue.code === "invalid_union") {
|
|
561
568
|
const values = "options" in issue ? issue.options : void 0;
|
|
569
|
+
if (!values && issue.path.at(-1) === "environments") return `${at} must be an environment or a list of them (got ${got(v)})`;
|
|
562
570
|
return values ? `${at} is ${oneOf(values)} (got ${got(v)})` : `${at} matches none of its forms (got ${got(v)})`;
|
|
563
571
|
}
|
|
564
572
|
if (issue.code === "invalid_type" && v === void 0) return `${at} is missing`;
|
|
@@ -573,1126 +581,6 @@ function configDataProblem(value, file) {
|
|
|
573
581
|
path: []
|
|
574
582
|
}, v)}`;
|
|
575
583
|
}
|
|
576
|
-
// src/gateway/routes.ts
|
|
577
|
-
/** Every gateway that brokers a key, by name. */
|
|
578
|
-
const ROUTING_GATEWAYS = {
|
|
579
|
-
vercel: VERCEL_GATEWAY,
|
|
580
|
-
openrouter: OPENROUTER_GATEWAY
|
|
581
|
-
};
|
|
582
|
-
/** Whether a harness's `gateway` names one of {@link ROUTING_GATEWAYS}. */
|
|
583
|
-
const isRoutingGateway = (name) => name !== void 0 && Object.hasOwn(ROUTING_GATEWAYS, name);
|
|
584
|
-
// src/harness/agents.ts
|
|
585
|
-
const WEB_EVENTS = {
|
|
586
|
-
webSearch: "web-search",
|
|
587
|
-
webFetch: "web-fetch"
|
|
588
|
-
};
|
|
589
|
-
/** `harnessMetadata.cursorSdk.unreportedTools` as transcript event types. */
|
|
590
|
-
function cursorUnreported(harnessMetadata) {
|
|
591
|
-
const tools = harnessMetadata?.cursorSdk?.unreportedTools;
|
|
592
|
-
return Array.isArray(tools) ? tools.flatMap((t) => WEB_EVENTS[t] ? [WEB_EVENTS[t]] : []) : [];
|
|
593
|
-
}
|
|
594
|
-
/** A Gemini API key's only request: `POST /v1beta/models/<model>:streamGenerateContent`. */
|
|
595
|
-
const GEMINI_ENDPOINTS = [{
|
|
596
|
-
host: "generativelanguage.googleapis.com",
|
|
597
|
-
path: { regex: "^/v1beta/models/[^/]+:streamGenerateContent$" },
|
|
598
|
-
method: ["POST"]
|
|
599
|
-
}];
|
|
600
|
-
/** Antigravity's: its model calls, and the `:generateContent` call its `search_web` tool makes. */
|
|
601
|
-
const ANTIGRAVITY_ENDPOINTS = [{
|
|
602
|
-
host: "generativelanguage.googleapis.com",
|
|
603
|
-
path: { regex: "^/v1beta/models/[\\w.-]+:(?:streamGenerateContent|generateContent)$" },
|
|
604
|
-
method: ["POST"]
|
|
605
|
-
}];
|
|
606
|
-
/** Goose's: the model call, and the model list it fetches with the key at the start of a turn. */
|
|
607
|
-
const GOOSE_ENDPOINTS = [...GEMINI_ENDPOINTS, {
|
|
608
|
-
host: "generativelanguage.googleapis.com",
|
|
609
|
-
path: { regex: "^/v1beta/models$" },
|
|
610
|
-
method: ["GET"]
|
|
611
|
-
}];
|
|
612
|
-
/**
|
|
613
|
-
* What Grok Build sends the key on: the model list and the key's status as a session starts, and
|
|
614
|
-
* its model calls (Chat Completions, and the Responses API for a model or tool that takes it).
|
|
615
|
-
*/
|
|
616
|
-
const XAI_ENDPOINTS = [
|
|
617
|
-
{
|
|
618
|
-
host: "api.x.ai",
|
|
619
|
-
path: { exact: "/v1/models" },
|
|
620
|
-
method: ["GET"]
|
|
621
|
-
},
|
|
622
|
-
{
|
|
623
|
-
host: "api.x.ai",
|
|
624
|
-
path: { exact: "/v1/api-key" },
|
|
625
|
-
method: ["GET"]
|
|
626
|
-
},
|
|
627
|
-
{
|
|
628
|
-
host: "api.x.ai",
|
|
629
|
-
path: { exact: "/v1/chat/completions" },
|
|
630
|
-
method: ["POST"]
|
|
631
|
-
},
|
|
632
|
-
{
|
|
633
|
-
host: "api.x.ai",
|
|
634
|
-
path: { exact: "/v1/responses" },
|
|
635
|
-
method: ["POST"]
|
|
636
|
-
}
|
|
637
|
-
];
|
|
638
|
-
/** xAI's model calls, whose responses name the model that served them. */
|
|
639
|
-
const XAI_MODEL_RESPONSES = XAI_ENDPOINTS.filter((e) => e.path.exact === "/v1/chat/completions" || e.path.exact === "/v1/responses");
|
|
640
|
-
/** The model an xAI response says served it. */
|
|
641
|
-
const xaiServedModel = ({ host, responseModel }) => host === "api.x.ai" ? responseModel : void 0;
|
|
642
|
-
/** The model a Gemini API request names in its URL. */
|
|
643
|
-
const geminiServedModel = ({ host, path }) => host === "generativelanguage.googleapis.com" ? /^\/v1beta\/models\/([^/:]+):streamGenerateContent$/.exec(path)?.[1] : void 0;
|
|
644
|
-
/** Cursor's key exchange: the only request the Cursor SDK sends the API key on. */
|
|
645
|
-
const CURSOR_EXCHANGE = {
|
|
646
|
-
host: "api2.cursor.sh",
|
|
647
|
-
path: { exact: "/auth/exchange_user_api_key" },
|
|
648
|
-
method: ["POST"]
|
|
649
|
-
};
|
|
650
|
-
/** Codex's model calls: the Responses API, whose response names the served model. */
|
|
651
|
-
const OPENAI_MODEL_ENDPOINTS = [{
|
|
652
|
-
host: "api.openai.com",
|
|
653
|
-
path: { exact: "/v1/responses" },
|
|
654
|
-
method: ["POST"]
|
|
655
|
-
}];
|
|
656
|
-
/** What Codex sends the key on: its model calls, and its web search tool's own endpoint. */
|
|
657
|
-
const OPENAI_ENDPOINTS = [...OPENAI_MODEL_ENDPOINTS, {
|
|
658
|
-
host: "api.openai.com",
|
|
659
|
-
path: { exact: "/v1/alpha/search" },
|
|
660
|
-
method: ["POST"]
|
|
661
|
-
}];
|
|
662
|
-
/**
|
|
663
|
-
* The model the Responses API says served a request, from its response (a dated snapshot for some
|
|
664
|
-
* models), and the reasoning effort Codex resolved for it, e.g. `gpt-5.6-luna[effort=medium]`.
|
|
665
|
-
*/
|
|
666
|
-
const openaiServedModel = ({ host, path, responseModel, responseEffort }) => host === "api.openai.com" && path === "/v1/responses" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
|
|
667
|
-
/**
|
|
668
|
-
* OpenCode on the model's provider (`anthropic/claude-haiku-4-5` is Anthropic's): its key, the
|
|
669
|
-
* requests OpenCode sends it on, its hosts plus Exa's for web search, and the served model from
|
|
670
|
-
* the URL (Gemini's API, Bedrock's) or the response body. Bedrock's hosts are its `AWS_REGION`'s.
|
|
671
|
-
*/
|
|
672
|
-
function opencodeForModel(model, env = process.env) {
|
|
673
|
-
const resolved = resolveOpenCodeProvider(model, env);
|
|
674
|
-
if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
|
|
675
|
-
const { provider } = resolved;
|
|
676
|
-
const modelResponses = providerModelResponses(provider);
|
|
677
|
-
return {
|
|
678
|
-
credential: provider.env,
|
|
679
|
-
endpoints: providerEndpoints(provider),
|
|
680
|
-
...modelResponses.length ? { modelResponses } : {},
|
|
681
|
-
servedModel: (request) => providerServedModel(provider, request),
|
|
682
|
-
hosts: [...providerHosts(provider), "mcp.exa.ai"]
|
|
683
|
-
};
|
|
684
|
-
}
|
|
685
|
-
/**
|
|
686
|
-
* The variables besides its key that OpenCode's provider takes from the host, as they are:
|
|
687
|
-
* Bedrock's `AWS_REGION` (default `us-east-1`); none for most providers.
|
|
688
|
-
*/
|
|
689
|
-
function opencodeVars(model, env = process.env) {
|
|
690
|
-
const resolved = resolveOpenCodeProvider(model, env);
|
|
691
|
-
if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
|
|
692
|
-
return resolved.vars;
|
|
693
|
-
}
|
|
694
|
-
/** Codex's `config.toml` `[mcp_servers]` shape. */
|
|
695
|
-
function codexMcpServers(servers) {
|
|
696
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
697
|
-
if (s.transport === "sse") throw new InfraError(`Codex has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
698
|
-
return s.transport === "stdio" ? [name, {
|
|
699
|
-
command: s.command,
|
|
700
|
-
args: s.args,
|
|
701
|
-
env: s.env ?? {}
|
|
702
|
-
}] : [name, {
|
|
703
|
-
url: s.url,
|
|
704
|
-
http_headers: s.headers ?? {}
|
|
705
|
-
}];
|
|
706
|
-
}));
|
|
707
|
-
}
|
|
708
|
-
/** The `.mcp.json` shape. */
|
|
709
|
-
function claudeMcpServers(servers) {
|
|
710
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
711
|
-
type: "stdio",
|
|
712
|
-
command: s.command,
|
|
713
|
-
args: s.args,
|
|
714
|
-
env: s.env
|
|
715
|
-
}] : [name, {
|
|
716
|
-
type: s.transport,
|
|
717
|
-
url: s.url,
|
|
718
|
-
headers: s.headers
|
|
719
|
-
}]));
|
|
720
|
-
}
|
|
721
|
-
/** The Antigravity CLI's `mcp_config.json` shape: remote servers take `serverUrl`, not `url`. */
|
|
722
|
-
function antigravityMcpServers(servers) {
|
|
723
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
724
|
-
command: s.command,
|
|
725
|
-
args: s.args,
|
|
726
|
-
env: s.env
|
|
727
|
-
}] : [name, {
|
|
728
|
-
serverUrl: s.url,
|
|
729
|
-
headers: s.headers
|
|
730
|
-
}]));
|
|
731
|
-
}
|
|
732
|
-
/**
|
|
733
|
-
* A `{env:…}` or `{file:…}` OpenCode would substitute. It runs `\{env:[^}]+\}`, then
|
|
734
|
-
* `\{file:[^}]+\}`, over its whole config text with no escape, so a reference left open at the end
|
|
735
|
-
* of a value runs on into the JSON after it, and only `{env:}` and `{file:}` are left alone.
|
|
736
|
-
*/
|
|
737
|
-
const OPENCODE_REFERENCE = /\{(?:env|file):(?!\})/;
|
|
738
|
-
const fieldKey = (key) => /^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
|
|
739
|
-
/**
|
|
740
|
-
* The first part of a server OpenCode would substitute, as what the server wouldn't get: its name,
|
|
741
|
-
* a value as `mcp.<name>.<field>`, or an env or header name. Names are in the same config text as
|
|
742
|
-
* values, so they're substituted too; a secret's value is never named.
|
|
743
|
-
*/
|
|
744
|
-
function openCodeReference(name, s) {
|
|
745
|
-
if (OPENCODE_REFERENCE.test(name)) return "its name";
|
|
746
|
-
const server = `mcp${fieldKey(name)}`;
|
|
747
|
-
const [map, kind, values] = s.transport === "stdio" ? [
|
|
748
|
-
"env",
|
|
749
|
-
"env",
|
|
750
|
-
s.env ?? {}
|
|
751
|
-
] : [
|
|
752
|
-
"headers",
|
|
753
|
-
"header",
|
|
754
|
-
s.headers ?? {}
|
|
755
|
-
];
|
|
756
|
-
return [...s.transport === "stdio" ? [[`${server}.command`, s.command], ...s.args.map((a, i) => [`${server}.args[${i}]`, a])] : [[`${server}.url`, s.url]], ...Object.entries(values).flatMap(([k, v]) => [[`the ${kind} name ${k} in ${server}.${map}`, k], [`${server}.${map}${fieldKey(k)}`, v]])].find(([, text]) => OPENCODE_REFERENCE.test(text))?.[0];
|
|
757
|
-
}
|
|
758
|
-
/**
|
|
759
|
-
* OpenCode's config shape (its `mcp.servers`), which the adapter sets `codemode: false` on.
|
|
760
|
-
* OpenCode has no SSE transport, and refuses a value it would substitute.
|
|
761
|
-
*/
|
|
762
|
-
function opencodeMcpServers(servers) {
|
|
763
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
764
|
-
if (s.transport === "sse") throw new InfraError(`OpenCode has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
765
|
-
const field = openCodeReference(name, s);
|
|
766
|
-
if (field) throw new InfraError(`OpenCode substitutes {env:…} and {file:…} anywhere in its config, with no escape, so the ${name} server wouldn't get ${field} as written: take the reference out, or leave OpenCode out of this cell`);
|
|
767
|
-
return s.transport === "stdio" ? [name, {
|
|
768
|
-
type: "local",
|
|
769
|
-
command: [s.command, ...s.args],
|
|
770
|
-
environment: s.env ?? {}
|
|
771
|
-
}] : [name, {
|
|
772
|
-
type: "remote",
|
|
773
|
-
url: s.url,
|
|
774
|
-
headers: s.headers ?? {},
|
|
775
|
-
oauth: false
|
|
776
|
-
}];
|
|
777
|
-
}));
|
|
778
|
-
}
|
|
779
|
-
/**
|
|
780
|
-
* The key Goose keeps an extension by (its `name_to_key`): whitespace dropped, anything but ASCII
|
|
781
|
-
* letters, digits, `_` and `-` made `_`, lower-cased. Of two extensions with one key it keeps one.
|
|
782
|
-
*/
|
|
783
|
-
const gooseKey = (name) => name.replace(/\s/g, "").replace(/[^A-Za-z0-9_-]/g, "_").toLowerCase();
|
|
784
|
-
/** Goose's extension shape (its `config.yaml`); it has no SSE transport. */
|
|
785
|
-
function gooseMcpServers(servers) {
|
|
786
|
-
const byKey = /* @__PURE__ */ new Map();
|
|
787
|
-
for (const name of Object.keys(servers)) {
|
|
788
|
-
const key = gooseKey(name);
|
|
789
|
-
const other = byKey.get(key);
|
|
790
|
-
if (other !== void 0) throw new InfraError(`Goose keys the ${other} and ${name} MCP servers both as ${key}, and keeps only one: rename one`);
|
|
791
|
-
byKey.set(key, name);
|
|
792
|
-
}
|
|
793
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
794
|
-
if (s.transport === "sse") throw new InfraError(`Goose has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
795
|
-
return s.transport === "stdio" ? [name, {
|
|
796
|
-
type: "stdio",
|
|
797
|
-
cmd: s.command,
|
|
798
|
-
args: s.args,
|
|
799
|
-
envs: s.env ?? {}
|
|
800
|
-
}] : [name, {
|
|
801
|
-
type: "streamable_http",
|
|
802
|
-
uri: s.url,
|
|
803
|
-
headers: s.headers ?? {}
|
|
804
|
-
}];
|
|
805
|
-
}));
|
|
806
|
-
}
|
|
807
|
-
/**
|
|
808
|
-
* pi's `mcp.json` shape, which the adapter escapes and sets `"exposure": "direct"` on. pi
|
|
809
|
-
* skips a server whose name isn't letters, digits, `_` and `-`, so the plan refuses it instead.
|
|
810
|
-
* A plugin's server, `<plugin>:<server>`, is named `<plugin>_<server>`, so its tools are
|
|
811
|
-
* `mcp__<plugin>_<server>__<tool>`.
|
|
812
|
-
*/
|
|
813
|
-
function piMcpServers(servers) {
|
|
814
|
-
const named = /* @__PURE__ */ new Map();
|
|
815
|
-
return Object.fromEntries(Object.entries(servers).map(([key, s]) => {
|
|
816
|
-
const name = /^[a-z0-9-]+:[^:]+$/.test(key) ? key.replace(":", "_") : key;
|
|
817
|
-
if (!/^[A-Za-z0-9_-]+$/.test(name)) throw new InfraError(`pi takes MCP server names of letters, digits, "_" and "-" only, so it can't use the ${key} server: rename it`);
|
|
818
|
-
const other = named.get(name);
|
|
819
|
-
if (other) throw new InfraError(`pi names the ${key} MCP server ${name}, as it does the ${other} server, and keeps only one: rename one`);
|
|
820
|
-
named.set(name, key);
|
|
821
|
-
if (s.transport === "sse") throw new InfraError(`pi has no SSE MCP transport, so it can't use the ${key} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
822
|
-
return s.transport === "stdio" ? [name, {
|
|
823
|
-
command: s.command,
|
|
824
|
-
args: s.args,
|
|
825
|
-
env: s.env ?? {}
|
|
826
|
-
}] : [name, {
|
|
827
|
-
url: s.url,
|
|
828
|
-
headers: s.headers ?? {}
|
|
829
|
-
}];
|
|
830
|
-
}));
|
|
831
|
-
}
|
|
832
|
-
/**
|
|
833
|
-
* ACP's `McpServer` shape, which fx and Grok Build take on `session/new`: `env` and `headers` as
|
|
834
|
-
* name-value lists.
|
|
835
|
-
*/
|
|
836
|
-
function acpMcpServers(servers) {
|
|
837
|
-
const pairs = (values = {}) => Object.entries(values).map(([name, value]) => ({
|
|
838
|
-
name,
|
|
839
|
-
value
|
|
840
|
-
}));
|
|
841
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, s.command.startsWith("/") ? {
|
|
842
|
-
command: s.command,
|
|
843
|
-
args: s.args,
|
|
844
|
-
env: pairs(s.env)
|
|
845
|
-
} : {
|
|
846
|
-
command: "/usr/bin/env",
|
|
847
|
-
args: [s.command, ...s.args],
|
|
848
|
-
env: pairs(s.env)
|
|
849
|
-
}] : [name, {
|
|
850
|
-
type: s.transport,
|
|
851
|
-
url: s.url,
|
|
852
|
-
headers: pairs(s.headers)
|
|
853
|
-
}]));
|
|
854
|
-
}
|
|
855
|
-
/**
|
|
856
|
-
* fx on the model's provider: OpenAI's Chat Completions API for a bare OpenAI id, and Vercel AI
|
|
857
|
-
* Gateway for a gateway id, which only `vercel()` in the gateway list serves. Each response's
|
|
858
|
-
* first `model` names the model that served it.
|
|
859
|
-
*/
|
|
860
|
-
function fxForModel(model) {
|
|
861
|
-
if (!model.includes("/")) {
|
|
862
|
-
const endpoints = [{
|
|
863
|
-
host: "api.openai.com",
|
|
864
|
-
path: { exact: "/v1/chat/completions" },
|
|
865
|
-
method: ["POST"]
|
|
866
|
-
}];
|
|
867
|
-
return {
|
|
868
|
-
credential: "OPENAI_API_KEY",
|
|
869
|
-
endpoints,
|
|
870
|
-
modelResponses: endpoints,
|
|
871
|
-
servedModel: ({ host, responseModel }) => host === "api.openai.com" ? responseModel : void 0,
|
|
872
|
-
hosts: ["api.openai.com"]
|
|
873
|
-
};
|
|
874
|
-
}
|
|
875
|
-
return {
|
|
876
|
-
credential: VERCEL_KEY,
|
|
877
|
-
endpoints: FX_VERCEL_ENDPOINTS,
|
|
878
|
-
modelResponses: [FX_VERCEL_CALL],
|
|
879
|
-
servedModel: servedOnVercel,
|
|
880
|
-
askedModel: fxAskedModel,
|
|
881
|
-
hosts: [VERCEL_HOST]
|
|
882
|
-
};
|
|
883
|
-
}
|
|
884
|
-
/** The Cursor SDK's `McpServerConfig` shape. */
|
|
885
|
-
function cursorMcpServers(servers) {
|
|
886
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
887
|
-
type: "stdio",
|
|
888
|
-
command: s.command,
|
|
889
|
-
args: s.args,
|
|
890
|
-
env: s.env
|
|
891
|
-
}] : [name, {
|
|
892
|
-
type: s.transport,
|
|
893
|
-
url: s.url,
|
|
894
|
-
headers: s.headers
|
|
895
|
-
}]));
|
|
896
|
-
}
|
|
897
|
-
/** Claude Code's hook events as Cursor names them, as the Cursor SDK converts Claude Code's own. */
|
|
898
|
-
const CURSOR_HOOK_EVENTS = {
|
|
899
|
-
SessionStart: "sessionStart",
|
|
900
|
-
SessionEnd: "sessionEnd",
|
|
901
|
-
UserPromptSubmit: "beforeSubmitPrompt",
|
|
902
|
-
PreToolUse: "preToolUse",
|
|
903
|
-
PostToolUse: "postToolUse",
|
|
904
|
-
SubagentStop: "subagentStop",
|
|
905
|
-
PreCompact: "preCompact",
|
|
906
|
-
Stop: "stop"
|
|
907
|
-
};
|
|
908
|
-
/** Claude Code's tool names as Cursor's hooks match them; null for one Cursor has no tool for. */
|
|
909
|
-
const CURSOR_HOOK_TOOLS = {
|
|
910
|
-
Bash: "Shell",
|
|
911
|
-
Read: "Read",
|
|
912
|
-
Write: "Write",
|
|
913
|
-
Edit: "Write",
|
|
914
|
-
Glob: null,
|
|
915
|
-
Grep: "Grep",
|
|
916
|
-
WebFetch: "WebFetch",
|
|
917
|
-
WebSearch: "WebSearch",
|
|
918
|
-
Task: "Task"
|
|
919
|
-
};
|
|
920
|
-
/**
|
|
921
|
-
* A Claude Code hook group's tool matcher as Cursor's, on the tool events only: each name mapped,
|
|
922
|
-
* `mcp__<server>__<tool>` as `MCP:<tool>`; null when it names only tools Cursor lacks.
|
|
923
|
-
*/
|
|
924
|
-
function cursorHookMatcher(event, matcher) {
|
|
925
|
-
if (event !== "PreToolUse" && event !== "PostToolUse") return void 0;
|
|
926
|
-
if (!matcher || matcher === "*") return void 0;
|
|
927
|
-
const out = [];
|
|
928
|
-
for (const raw of matcher.split("|")) {
|
|
929
|
-
const name = raw.trim();
|
|
930
|
-
const parts = name.split("__");
|
|
931
|
-
const tool = name.startsWith("mcp__") && parts.length >= 3 ? `MCP:${parts.slice(2).join("__")}` : CURSOR_HOOK_TOOLS[name] === void 0 ? name : CURSOR_HOOK_TOOLS[name];
|
|
932
|
-
if (tool && !out.includes(tool)) out.push(tool);
|
|
933
|
-
}
|
|
934
|
-
return out.length ? out.join("|") : null;
|
|
935
|
-
}
|
|
936
|
-
/** Where Cursor reads what reeval unpacks from a plugin, in the workspace (settingSources `project`). */
|
|
937
|
-
const CURSOR_PLUGIN_UNPACK = {
|
|
938
|
-
rules: ".cursor/rules",
|
|
939
|
-
agents: ".cursor/agents",
|
|
940
|
-
commands: ".cursor/commands",
|
|
941
|
-
hooks: {
|
|
942
|
-
file: ".cursor/hooks.json",
|
|
943
|
-
events: CURSOR_HOOK_EVENTS,
|
|
944
|
-
matcher: cursorHookMatcher
|
|
945
|
-
}
|
|
946
|
-
};
|
|
947
|
-
/**
|
|
948
|
-
* A plugin subagent's or command's frontmatter as OpenCode 2 takes it: the description, and a
|
|
949
|
-
* subagent's `mode`. OpenCode skips a file whose frontmatter its schema refuses, without saying,
|
|
950
|
-
* and Claude Code's `tools: Read, Grep`, `model: sonnet` and `allowed-tools` are such keys.
|
|
951
|
-
*/
|
|
952
|
-
function opencodeFrontmatter(part, front) {
|
|
953
|
-
const d = front.description;
|
|
954
|
-
const description = typeof d === "string" || typeof d === "number" || typeof d === "boolean" ? String(d) : void 0;
|
|
955
|
-
return {
|
|
956
|
-
...description ? { description } : {},
|
|
957
|
-
...part === "agents" ? { mode: "subagent" } : {}
|
|
958
|
-
};
|
|
959
|
-
}
|
|
960
|
-
/** Where OpenCode reads plugin subagents and commands, in the workspace's config directory. */
|
|
961
|
-
const OPENCODE_PLUGIN_UNPACK = {
|
|
962
|
-
agents: ".opencode/agents",
|
|
963
|
-
commands: ".opencode/commands",
|
|
964
|
-
frontmatter: opencodeFrontmatter
|
|
965
|
-
};
|
|
966
|
-
/**
|
|
967
|
-
* Where pi reads plugin commands, as prompt templates (`/<name>`), in the workspace, which its
|
|
968
|
-
* `--approve` trusts. Its frontmatter takes `description` and `argument-hint`, and ignores the
|
|
969
|
-
* rest of Claude Code's.
|
|
970
|
-
*/
|
|
971
|
-
const PI_PLUGIN_UNPACK = { commands: ".pi/prompts" };
|
|
972
|
-
/**
|
|
973
|
-
* What Claude Code's plugin format bundles, which Grok Build reads too: every part but rules,
|
|
974
|
-
* which neither loads from a plugin.
|
|
975
|
-
*/
|
|
976
|
-
const CLAUDE_FORMAT_PARTS = PLUGIN_PARTS.filter((p) => p !== "rules");
|
|
977
|
-
/** The marketplace Claude Code's settings name for the session's plugins. */
|
|
978
|
-
const CLAUDE_PLUGIN_MARKETPLACE = "reeval";
|
|
979
|
-
/**
|
|
980
|
-
* Claude Code's settings for the session's plugins: a `directory` marketplace at the directory
|
|
981
|
-
* they were copied to, listing each, and each enabled as `<plugin>@reeval`. Claude Code installs
|
|
982
|
-
* and loads them as it starts, since the bridge leaves every settings source on.
|
|
983
|
-
*/
|
|
984
|
-
function claudePluginFiles(plugins) {
|
|
985
|
-
const [first] = plugins;
|
|
986
|
-
if (!first) return [];
|
|
987
|
-
const json = (value) => `${JSON.stringify(value, null, 2)}\n`;
|
|
988
|
-
const marketplace = {
|
|
989
|
-
name: CLAUDE_PLUGIN_MARKETPLACE,
|
|
990
|
-
owner: { name: "reeval" },
|
|
991
|
-
plugins: plugins.map((p) => ({
|
|
992
|
-
name: p.name,
|
|
993
|
-
source: `./${p.name}`
|
|
994
|
-
}))
|
|
995
|
-
};
|
|
996
|
-
const settings = {
|
|
997
|
-
extraKnownMarketplaces: { [CLAUDE_PLUGIN_MARKETPLACE]: { source: {
|
|
998
|
-
source: "directory",
|
|
999
|
-
path: first.dir.slice(0, first.dir.lastIndexOf("/"))
|
|
1000
|
-
} } },
|
|
1001
|
-
enabledPlugins: Object.fromEntries(plugins.map((p) => [`${p.name}@${CLAUDE_PLUGIN_MARKETPLACE}`, true]))
|
|
1002
|
-
};
|
|
1003
|
-
return [{
|
|
1004
|
-
path: `${PLUGIN_DIR}/.claude-plugin/marketplace.json`,
|
|
1005
|
-
content: json(marketplace)
|
|
1006
|
-
}, {
|
|
1007
|
-
path: "~/.claude/settings.json",
|
|
1008
|
-
content: json(settings),
|
|
1009
|
-
merge: true
|
|
1010
|
-
}];
|
|
1011
|
-
}
|
|
1012
|
-
const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
1013
|
-
/**
|
|
1014
|
-
* Writes the layout's plugin files into the sandbox and returns their paths. A `merge` file's keys
|
|
1015
|
-
* go into the JSON object already at its path (an object-valued key merged one level down), so
|
|
1016
|
-
* settings an environment's `files` wrote there stay.
|
|
1017
|
-
*/
|
|
1018
|
-
async function writePluginFiles(sandbox, layout, plugins) {
|
|
1019
|
-
const written = [];
|
|
1020
|
-
for (const f of layout.pluginFiles?.(plugins) ?? []) {
|
|
1021
|
-
const existing = f.merge ? await sandbox.readFile(f.path) : null;
|
|
1022
|
-
let content = f.content;
|
|
1023
|
-
if (existing?.trim()) {
|
|
1024
|
-
let base;
|
|
1025
|
-
try {
|
|
1026
|
-
base = JSON.parse(withoutBom(existing));
|
|
1027
|
-
} catch {}
|
|
1028
|
-
if (!isObject(base)) throw new InfraError(`${f.path} isn't a JSON object, so reeval can't enable the session's plugins in it: write it as one in files, or leave it out`);
|
|
1029
|
-
const ours = JSON.parse(f.content);
|
|
1030
|
-
const merged = { ...base };
|
|
1031
|
-
for (const [k, v] of Object.entries(ours)) {
|
|
1032
|
-
const was = base[k];
|
|
1033
|
-
merged[k] = isObject(v) && isObject(was) ? {
|
|
1034
|
-
...was,
|
|
1035
|
-
...v
|
|
1036
|
-
} : v;
|
|
1037
|
-
}
|
|
1038
|
-
content = `${JSON.stringify(merged, null, 2)}\n`;
|
|
1039
|
-
}
|
|
1040
|
-
await sandbox.writeFile(f.path, content);
|
|
1041
|
-
written.push(f.path);
|
|
1042
|
-
}
|
|
1043
|
-
return written;
|
|
1044
|
-
}
|
|
1045
|
-
/**
|
|
1046
|
-
* The MCP servers the agent is given: all of them, less the plugins' own where it takes plugin
|
|
1047
|
-
* items namespaced, since it loads those from the plugin and would otherwise start each twice.
|
|
1048
|
-
*/
|
|
1049
|
-
function agentMcp(layout, mcp, plugins) {
|
|
1050
|
-
if (layout.pluginPlacement !== "namespaced") return mcp;
|
|
1051
|
-
const own = new Set(plugins.flatMap((p) => p.servers));
|
|
1052
|
-
return Object.fromEntries(Object.entries(mcp).filter(([name]) => !own.has(name)));
|
|
1053
|
-
}
|
|
1054
|
-
/** Where each agent with an image reads rules and skills. */
|
|
1055
|
-
const LAYOUTS = {
|
|
1056
|
-
cursor: {
|
|
1057
|
-
rulesFile: "AGENTS.md",
|
|
1058
|
-
workspaceSkillDir: ".cursor/skills",
|
|
1059
|
-
harnessSkillDir: ".agents/skills",
|
|
1060
|
-
pluginPlacement: "flat",
|
|
1061
|
-
pluginUnpack: CURSOR_PLUGIN_UNPACK,
|
|
1062
|
-
pluginHookEvents: Object.keys(CURSOR_HOOK_EVENTS)
|
|
1063
|
-
},
|
|
1064
|
-
"claude-code": {
|
|
1065
|
-
rulesFile: "CLAUDE.md",
|
|
1066
|
-
workspaceSkillDir: ".claude/skills",
|
|
1067
|
-
harnessSkillDir: ".claude/skills",
|
|
1068
|
-
pluginPlacement: "namespaced",
|
|
1069
|
-
pluginFiles: claudePluginFiles
|
|
1070
|
-
},
|
|
1071
|
-
antigravity: {
|
|
1072
|
-
rulesFile: "AGENTS.md",
|
|
1073
|
-
workspaceSkillDir: ".agents/skills",
|
|
1074
|
-
harnessSkillDir: ".gemini/config/skills",
|
|
1075
|
-
pluginPlacement: "namespaced",
|
|
1076
|
-
pluginHookEvents: [
|
|
1077
|
-
"SessionStart",
|
|
1078
|
-
"PreToolUse",
|
|
1079
|
-
"PostToolUse",
|
|
1080
|
-
"Stop"
|
|
1081
|
-
]
|
|
1082
|
-
},
|
|
1083
|
-
opencode: {
|
|
1084
|
-
rulesFile: "AGENTS.md",
|
|
1085
|
-
workspaceSkillDir: ".opencode/skills",
|
|
1086
|
-
harnessSkillDir: ".config/opencode/skills",
|
|
1087
|
-
pluginPlacement: "flat",
|
|
1088
|
-
pluginUnpack: OPENCODE_PLUGIN_UNPACK
|
|
1089
|
-
},
|
|
1090
|
-
goose: {
|
|
1091
|
-
rulesFile: "AGENTS.md",
|
|
1092
|
-
workspaceSkillDir: ".goose/skills",
|
|
1093
|
-
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/goose/home/.config/goose/skills",
|
|
1094
|
-
pluginPlacement: "flat"
|
|
1095
|
-
},
|
|
1096
|
-
pi: {
|
|
1097
|
-
rulesFile: "AGENTS.md",
|
|
1098
|
-
workspaceSkillDir: ".pi/skills",
|
|
1099
|
-
harnessSkillDir: ".pi/agent/skills",
|
|
1100
|
-
pluginPlacement: "flat",
|
|
1101
|
-
pluginUnpack: PI_PLUGIN_UNPACK
|
|
1102
|
-
},
|
|
1103
|
-
codex: {
|
|
1104
|
-
rulesFile: "AGENTS.md",
|
|
1105
|
-
workspaceSkillDir: ".agents/skills",
|
|
1106
|
-
harnessSkillDir: ".agents/skills",
|
|
1107
|
-
pluginPlacement: "namespaced"
|
|
1108
|
-
},
|
|
1109
|
-
fx: {
|
|
1110
|
-
rulesFile: "AGENTS.md",
|
|
1111
|
-
workspaceSkillDir: ".fx/skills",
|
|
1112
|
-
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/fx/home/.fx/skills",
|
|
1113
|
-
pluginPlacement: "flat"
|
|
1114
|
-
},
|
|
1115
|
-
"grok-build": {
|
|
1116
|
-
rulesFile: "AGENTS.md",
|
|
1117
|
-
workspaceSkillDir: ".grok/skills",
|
|
1118
|
-
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/grok-build/home/.grok/skills",
|
|
1119
|
-
pluginPlacement: "namespaced"
|
|
1120
|
-
}
|
|
1121
|
-
};
|
|
1122
|
-
/** Every real agent reeval can drive, by harness kind. */
|
|
1123
|
-
const AGENTS = {
|
|
1124
|
-
cursor: {
|
|
1125
|
-
...LAYOUTS.cursor,
|
|
1126
|
-
pluginParts: PLUGIN_PARTS,
|
|
1127
|
-
name: "Cursor",
|
|
1128
|
-
factory: "cursor",
|
|
1129
|
-
reasoning: "streamed",
|
|
1130
|
-
reasoningTokens: false,
|
|
1131
|
-
credential: "CURSOR_API_KEY",
|
|
1132
|
-
endpoints: [CURSOR_EXCHANGE],
|
|
1133
|
-
tokens: {
|
|
1134
|
-
endpoint: CURSOR_EXCHANGE,
|
|
1135
|
-
fields: ["accessToken", "refreshToken"],
|
|
1136
|
-
hosts: ["cursor.sh", "*.cursor.sh"]
|
|
1137
|
-
},
|
|
1138
|
-
unreported: cursorUnreported,
|
|
1139
|
-
mcpServers: cursorMcpServers,
|
|
1140
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-DurP--_D.js")).createCursorSdk({
|
|
1141
|
-
auth,
|
|
1142
|
-
...mcpServers ? { mcpServers } : {},
|
|
1143
|
-
...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
|
|
1144
|
-
...descriptor.kind === "cursor" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1145
|
-
requireCredentialBrokering: true
|
|
1146
|
-
})
|
|
1147
|
-
},
|
|
1148
|
-
"claude-code": {
|
|
1149
|
-
...LAYOUTS["claude-code"],
|
|
1150
|
-
pluginParts: CLAUDE_FORMAT_PARTS,
|
|
1151
|
-
name: "Claude Code",
|
|
1152
|
-
factory: "claudeCode",
|
|
1153
|
-
reasoning: "streamed",
|
|
1154
|
-
reasoningTokens: false,
|
|
1155
|
-
subagentCalls: true,
|
|
1156
|
-
credential: "ANTHROPIC_API_KEY",
|
|
1157
|
-
endpoints: [
|
|
1158
|
-
{
|
|
1159
|
-
host: "api.anthropic.com",
|
|
1160
|
-
path: { exact: "/v1/messages" },
|
|
1161
|
-
method: ["POST"]
|
|
1162
|
-
},
|
|
1163
|
-
{
|
|
1164
|
-
host: "api.anthropic.com",
|
|
1165
|
-
path: { exact: "/v1/messages/count_tokens" },
|
|
1166
|
-
method: ["POST"]
|
|
1167
|
-
},
|
|
1168
|
-
{
|
|
1169
|
-
host: "api.anthropic.com",
|
|
1170
|
-
path: { exact: "/api/claude_code/settings" },
|
|
1171
|
-
method: ["GET"]
|
|
1172
|
-
},
|
|
1173
|
-
{
|
|
1174
|
-
host: "api.anthropic.com",
|
|
1175
|
-
path: { exact: "/api/claude_code/policy_limits" },
|
|
1176
|
-
method: ["GET"]
|
|
1177
|
-
}
|
|
1178
|
-
],
|
|
1179
|
-
mcpServers: claudeMcpServers,
|
|
1180
|
-
adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-ByYk_wAs.js")).createClaudeCode({
|
|
1181
|
-
auth,
|
|
1182
|
-
...mcpServers ? { mcpServers } : {},
|
|
1183
|
-
...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1184
|
-
env: {
|
|
1185
|
-
IS_SANDBOX: "1",
|
|
1186
|
-
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
|
|
1187
|
-
...env
|
|
1188
|
-
}
|
|
1189
|
-
})
|
|
1190
|
-
},
|
|
1191
|
-
antigravity: {
|
|
1192
|
-
...LAYOUTS.antigravity,
|
|
1193
|
-
pluginParts: PLUGIN_PARTS,
|
|
1194
|
-
pluginManifestsFirst: [
|
|
1195
|
-
"plugin.json",
|
|
1196
|
-
".antigravity-plugin/plugin.json",
|
|
1197
|
-
".gemini-plugin/plugin.json",
|
|
1198
|
-
"gemini-extension.json"
|
|
1199
|
-
],
|
|
1200
|
-
name: "Antigravity",
|
|
1201
|
-
factory: "antigravity",
|
|
1202
|
-
reasoning: "none",
|
|
1203
|
-
reasoningTokens: true,
|
|
1204
|
-
credential: "GEMINI_API_KEY",
|
|
1205
|
-
endpoints: ANTIGRAVITY_ENDPOINTS,
|
|
1206
|
-
expectedBlocked: {
|
|
1207
|
-
hosts: ["play.googleapis.com", "antigravity-unleash.goog"],
|
|
1208
|
-
why: "Antigravity's telemetry and feature flags, which its CLI sends even with telemetry off"
|
|
1209
|
-
},
|
|
1210
|
-
servedModel: geminiServedModel,
|
|
1211
|
-
mcpServers: antigravityMcpServers,
|
|
1212
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-CWEGknZo.js")).createAntigravity({
|
|
1213
|
-
auth,
|
|
1214
|
-
...vertex ? { vertex } : {},
|
|
1215
|
-
...mcpServers ? { mcpServers } : {},
|
|
1216
|
-
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
1217
|
-
name,
|
|
1218
|
-
dir
|
|
1219
|
-
})) } : {},
|
|
1220
|
-
...descriptor.kind === "antigravity" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1221
|
-
requireCredentialBrokering: true
|
|
1222
|
-
})
|
|
1223
|
-
},
|
|
1224
|
-
opencode: {
|
|
1225
|
-
...LAYOUTS.opencode,
|
|
1226
|
-
pluginParts: [
|
|
1227
|
-
"skills",
|
|
1228
|
-
"agents",
|
|
1229
|
-
"commands",
|
|
1230
|
-
"mcp"
|
|
1231
|
-
],
|
|
1232
|
-
name: "OpenCode",
|
|
1233
|
-
factory: "opencode",
|
|
1234
|
-
configuredModelOnStart: true,
|
|
1235
|
-
reasoning: "streamed",
|
|
1236
|
-
reasoningTokens: "provider",
|
|
1237
|
-
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1238
|
-
endpoints: GEMINI_ENDPOINTS,
|
|
1239
|
-
servedModel: geminiServedModel,
|
|
1240
|
-
forModel: opencodeForModel,
|
|
1241
|
-
mcpServers: opencodeMcpServers,
|
|
1242
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BxrHipV0.js")).createOpenCode({
|
|
1243
|
-
auth,
|
|
1244
|
-
model,
|
|
1245
|
-
...vertex ? { vertex } : { vars: opencodeVars(model) },
|
|
1246
|
-
...descriptor.kind === "opencode" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1247
|
-
...mcpServers ? { mcpServers } : {},
|
|
1248
|
-
requireCredentialBrokering: true
|
|
1249
|
-
})
|
|
1250
|
-
},
|
|
1251
|
-
goose: {
|
|
1252
|
-
...LAYOUTS.goose,
|
|
1253
|
-
pluginParts: ["skills", "mcp"],
|
|
1254
|
-
name: "Goose",
|
|
1255
|
-
factory: "goose",
|
|
1256
|
-
configuredModelOnStart: true,
|
|
1257
|
-
reasoning: "streamed",
|
|
1258
|
-
reasoningTokens: false,
|
|
1259
|
-
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1260
|
-
endpoints: GOOSE_ENDPOINTS,
|
|
1261
|
-
expectedBlocked: {
|
|
1262
|
-
hosts: ["models.dev"],
|
|
1263
|
-
why: "Goose's model catalog refresh, which no setting turns off"
|
|
1264
|
-
},
|
|
1265
|
-
servedModel: geminiServedModel,
|
|
1266
|
-
mcpServers: gooseMcpServers,
|
|
1267
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-CM7xdJFA.js")).createGoose({
|
|
1268
|
-
...vertex ? { vertex } : {},
|
|
1269
|
-
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1270
|
-
auth,
|
|
1271
|
-
provider: (await import("./src-CM7xdJFA.js")).AI_GATEWAY_PROVIDER
|
|
1272
|
-
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1273
|
-
auth,
|
|
1274
|
-
provider: (await import("./src-CM7xdJFA.js")).OPENROUTER_PROVIDER
|
|
1275
|
-
} : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
|
|
1276
|
-
model,
|
|
1277
|
-
...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1278
|
-
...mcpServers ? { mcpServers } : {},
|
|
1279
|
-
requireCredentialBrokering: true
|
|
1280
|
-
})
|
|
1281
|
-
},
|
|
1282
|
-
pi: {
|
|
1283
|
-
...LAYOUTS.pi,
|
|
1284
|
-
pluginParts: [
|
|
1285
|
-
"skills",
|
|
1286
|
-
"commands",
|
|
1287
|
-
"mcp"
|
|
1288
|
-
],
|
|
1289
|
-
name: "pi",
|
|
1290
|
-
factory: "pi",
|
|
1291
|
-
reasoning: "streamed",
|
|
1292
|
-
reasoningTokens: true,
|
|
1293
|
-
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1294
|
-
endpoints: GEMINI_ENDPOINTS,
|
|
1295
|
-
servedModel: geminiServedModel,
|
|
1296
|
-
mcpServers: piMcpServers,
|
|
1297
|
-
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-BC2WXULN.js")).createPi({
|
|
1298
|
-
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1299
|
-
auth,
|
|
1300
|
-
gateway: (await import("./src-BC2WXULN.js")).PI_AI_GATEWAY
|
|
1301
|
-
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1302
|
-
auth,
|
|
1303
|
-
gateway: (await import("./src-BC2WXULN.js")).PI_OPENROUTER
|
|
1304
|
-
} : {
|
|
1305
|
-
auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
|
|
1306
|
-
...vertex ? { vertex } : {}
|
|
1307
|
-
},
|
|
1308
|
-
...descriptor.kind === "pi" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1309
|
-
...mcpServers ? { mcpServers } : {},
|
|
1310
|
-
requireCredentialBrokering: true
|
|
1311
|
-
})
|
|
1312
|
-
},
|
|
1313
|
-
codex: {
|
|
1314
|
-
...LAYOUTS.codex,
|
|
1315
|
-
pluginParts: ["skills", "mcp"],
|
|
1316
|
-
pluginApps: "they need a ChatGPT login",
|
|
1317
|
-
pluginManifestsFirst: ["plugin.json", ".codex-plugin/plugin.json"],
|
|
1318
|
-
pluginRootManifestNeedsSchema: true,
|
|
1319
|
-
pluginServersByName: true,
|
|
1320
|
-
name: "Codex",
|
|
1321
|
-
factory: "codex",
|
|
1322
|
-
reasoning: "whole",
|
|
1323
|
-
reasoningTokens: true,
|
|
1324
|
-
credential: "OPENAI_API_KEY",
|
|
1325
|
-
endpoints: OPENAI_ENDPOINTS,
|
|
1326
|
-
modelResponses: OPENAI_MODEL_ENDPOINTS,
|
|
1327
|
-
expectedBlocked: {
|
|
1328
|
-
hosts: [
|
|
1329
|
-
"github.com",
|
|
1330
|
-
"api.github.com",
|
|
1331
|
-
"chatgpt.com"
|
|
1332
|
-
],
|
|
1333
|
-
why: "Codex's plugin sync, which it runs on every turn once plugins are on",
|
|
1334
|
-
withPlugins: true
|
|
1335
|
-
},
|
|
1336
|
-
servedModel: openaiServedModel,
|
|
1337
|
-
mcpServers: codexMcpServers,
|
|
1338
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-C9elednc.js")).createCodex({
|
|
1339
|
-
auth,
|
|
1340
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-C9elednc.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-C9elednc.js")).OPENROUTER_PROVIDER } : {},
|
|
1341
|
-
...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1342
|
-
...mcpServers ? { mcpServers } : {},
|
|
1343
|
-
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
1344
|
-
name,
|
|
1345
|
-
dir
|
|
1346
|
-
})) } : {},
|
|
1347
|
-
requireCredentialBrokering: true
|
|
1348
|
-
})
|
|
1349
|
-
},
|
|
1350
|
-
fx: {
|
|
1351
|
-
...LAYOUTS.fx,
|
|
1352
|
-
pluginParts: ["skills", "mcp"],
|
|
1353
|
-
name: "fx",
|
|
1354
|
-
factory: "fx",
|
|
1355
|
-
configuredModelOnStart: true,
|
|
1356
|
-
reasoning: "streamed",
|
|
1357
|
-
reasoningTokens: "provider",
|
|
1358
|
-
...fxForModel("provider/model"),
|
|
1359
|
-
forModel: fxForModel,
|
|
1360
|
-
mcpServers: acpMcpServers,
|
|
1361
|
-
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-BKDZO3ra.js")).createFx({
|
|
1362
|
-
auth,
|
|
1363
|
-
model,
|
|
1364
|
-
...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
|
|
1365
|
-
...env?.FX_VERTEX_PROJECT ? { vertex: {
|
|
1366
|
-
project: env.FX_VERTEX_PROJECT,
|
|
1367
|
-
location: env.FX_VERTEX_LOCATION ?? ""
|
|
1368
|
-
} } : {},
|
|
1369
|
-
...descriptor.kind === "fx" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1370
|
-
...mcpServers ? { mcpServers } : {},
|
|
1371
|
-
requireCredentialBrokering: true
|
|
1372
|
-
})
|
|
1373
|
-
},
|
|
1374
|
-
"grok-build": {
|
|
1375
|
-
...LAYOUTS["grok-build"],
|
|
1376
|
-
pluginParts: CLAUDE_FORMAT_PARTS,
|
|
1377
|
-
pluginServersByName: true,
|
|
1378
|
-
name: "Grok Build",
|
|
1379
|
-
factory: "grokBuild",
|
|
1380
|
-
configuredModelOnStart: true,
|
|
1381
|
-
reasoning: "streamed",
|
|
1382
|
-
reasoningTokens: true,
|
|
1383
|
-
credential: "XAI_API_KEY",
|
|
1384
|
-
endpoints: XAI_ENDPOINTS,
|
|
1385
|
-
modelResponses: XAI_MODEL_RESPONSES,
|
|
1386
|
-
servedModel: xaiServedModel,
|
|
1387
|
-
mcpServers: acpMcpServers,
|
|
1388
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
|
|
1389
|
-
const grok = await import("./src-Cx0HJIJ-.js");
|
|
1390
|
-
return grok.createGrokBuild({
|
|
1391
|
-
...vertexChat ? {
|
|
1392
|
-
auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
|
|
1393
|
-
provider: grok.vertexProvider(vertexChat.project, vertexChat.region)
|
|
1394
|
-
} : {
|
|
1395
|
-
auth,
|
|
1396
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: grok.AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: grok.OPENROUTER_PROVIDER } : {}
|
|
1397
|
-
},
|
|
1398
|
-
model,
|
|
1399
|
-
...plugins.length ? {
|
|
1400
|
-
pluginDirs: plugins.map((p) => p.dir),
|
|
1401
|
-
pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
|
|
1402
|
-
} : {},
|
|
1403
|
-
...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1404
|
-
...mcpServers ? { mcpServers } : {},
|
|
1405
|
-
requireCredentialBrokering: true
|
|
1406
|
-
});
|
|
1407
|
-
}
|
|
1408
|
-
}
|
|
1409
|
-
};
|
|
1410
|
-
/** The harness of a config that names none: Cursor, on Cursor's own model. */
|
|
1411
|
-
const DEFAULT_HARNESS = {
|
|
1412
|
-
kind: "cursor",
|
|
1413
|
-
model: "composer-2.5"
|
|
1414
|
-
};
|
|
1415
|
-
/** A harness to name in messages that ask for one. */
|
|
1416
|
-
const HARNESS_EXAMPLE = "cursor({ model: 'composer-2.5' })";
|
|
1417
|
-
const isAgentKind = (kind) => typeof kind === "string" && Object.hasOwn(AGENTS, kind);
|
|
1418
|
-
/**
|
|
1419
|
-
* A server as the plan knows it: its names, literal values and each `secret.template`'s text. A
|
|
1420
|
-
* host secret's value is read when the attempt starts, so the plan has it as `''`.
|
|
1421
|
-
*/
|
|
1422
|
-
function plannedMcpServer(s) {
|
|
1423
|
-
const literal = (values) => Object.fromEntries(Object.entries(values).map(([k, v]) => [k, typeof v === "string" ? v : v.kind === "secret-template" ? v.template : ""]));
|
|
1424
|
-
return s.transport === "stdio" ? {
|
|
1425
|
-
transport: "stdio",
|
|
1426
|
-
command: s.command,
|
|
1427
|
-
args: s.args,
|
|
1428
|
-
env: literal(s.env)
|
|
1429
|
-
} : {
|
|
1430
|
-
transport: s.transport,
|
|
1431
|
-
url: s.url,
|
|
1432
|
-
headers: literal(s.headers)
|
|
1433
|
-
};
|
|
1434
|
-
}
|
|
1435
|
-
/**
|
|
1436
|
-
* Why a harness's agent can't take the environment's MCP servers (Goose, OpenCode, pi and Codex
|
|
1437
|
-
* take no `mcp.sse`; OpenCode would substitute a `{env:…}` or `{file:…}` in a name or value), in the
|
|
1438
|
-
* agent's own words, so the plan refuses them before any attempt starts; undefined when it can.
|
|
1439
|
-
*/
|
|
1440
|
-
function refusedMcp(harness, layers) {
|
|
1441
|
-
if (!isAgentKind(harness.kind)) return void 0;
|
|
1442
|
-
const servers = /* @__PURE__ */ new Map();
|
|
1443
|
-
for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, plannedMcpServer(server));
|
|
1444
|
-
if (servers.size === 0) return void 0;
|
|
1445
|
-
try {
|
|
1446
|
-
AGENTS[harness.kind].mcpServers(Object.fromEntries(servers));
|
|
1447
|
-
return;
|
|
1448
|
-
} catch (error) {
|
|
1449
|
-
if (error instanceof InfraError) return error.message;
|
|
1450
|
-
throw error;
|
|
1451
|
-
}
|
|
1452
|
-
}
|
|
1453
|
-
/** The first host variable a plugin MCP server reads (`${VAR}`), as `VAR`; undefined for none. */
|
|
1454
|
-
function hostVariable(s) {
|
|
1455
|
-
const values = Object.entries(s.transport === "stdio" ? s.env : s.headers);
|
|
1456
|
-
for (const [key, v] of values) {
|
|
1457
|
-
if (typeof v === "string") continue;
|
|
1458
|
-
if (v.kind === "secret") return v.from ?? key;
|
|
1459
|
-
return /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/.exec(v.template)?.[1] ?? key;
|
|
1460
|
-
}
|
|
1461
|
-
}
|
|
1462
|
-
const quoted = (parts) => parts.map((p) => `'${p}'`).join(", ");
|
|
1463
|
-
/**
|
|
1464
|
-
* Why a harness's agent can't take the plugins, in the agent's own words: a part it doesn't
|
|
1465
|
-
* load, apps it can't take (`pluginApps`), a hook event or type it doesn't run, a plugin MCP
|
|
1466
|
-
* server it refuses (named `<plugin>:<server>`) or whose `<plugin>:<server>` key `mcp` already has,
|
|
1467
|
-
* or, where it takes plugin items flat, a skill or MCP server whose name another already has, or a
|
|
1468
|
-
* subagent, command or rule another unpacks to the same file. `own` names the
|
|
1469
|
-
* environment's own skills and MCP servers, and `unread` the plugins not fetched yet (git and
|
|
1470
|
-
* npm ones, at plan time) by their call. Undefined when it can take them.
|
|
1471
|
-
*/
|
|
1472
|
-
function refusedPlugins(harness, plugins, own, unread = []) {
|
|
1473
|
-
if (plugins.length === 0 && unread.length === 0) return void 0;
|
|
1474
|
-
const agent = isAgentKind(harness.kind) ? AGENTS[harness.kind] : void 0;
|
|
1475
|
-
const mimics = harness.kind === "fake" && harness.layout ? AGENTS[harness.layout] : void 0;
|
|
1476
|
-
const loads = (agent ?? mimics)?.pluginParts ?? PLUGIN_PARTS;
|
|
1477
|
-
const name = agent?.name ?? (mimics ? `The fake agent with ${mimics.name}'s layout` : "The fake agent");
|
|
1478
|
-
if (loads.length === 0) return `${name} can't load plugins yet: leave ${plugins[0] ? `the ${plugins[0].name} plugin` : unread[0]} out of this cell`;
|
|
1479
|
-
for (const p of plugins) {
|
|
1480
|
-
const refused = p.kept.filter((part) => !loads.includes(part));
|
|
1481
|
-
if (refused.length === 0) continue;
|
|
1482
|
-
const can = p.kept.filter((part) => loads.includes(part));
|
|
1483
|
-
const what = refused.map((part) => `${part} (${PART_PATHS[part]})`).join(" and ");
|
|
1484
|
-
const fix = can.length ? `use only: [${quoted(can)}], or leave` : "leave";
|
|
1485
|
-
return `${name} can't load the ${p.name} plugin's ${what}: ${fix} the plugin out of this cell`;
|
|
1486
|
-
}
|
|
1487
|
-
const reads = agent ?? mimics;
|
|
1488
|
-
const first = reads?.pluginManifestsFirst ?? [];
|
|
1489
|
-
for (const p of plugins) {
|
|
1490
|
-
const other = p.manifests.find((m) => first.includes(m) && (m !== "plugin.json" || !reads?.pluginRootManifestNeedsSchema || p.rootSchema));
|
|
1491
|
-
if (other) return `${name} reads the ${p.name} plugin's ${other} in place of its ${PLUGIN_MANIFEST}, so reeval can't hold it to the parts it places: remove ${other} from the plugin, or leave the plugin out of this cell`;
|
|
1492
|
-
}
|
|
1493
|
-
const apps = (agent ?? mimics)?.pluginApps;
|
|
1494
|
-
const withApps = apps ? plugins.find((p) => p.apps) : void 0;
|
|
1495
|
-
if (withApps) return `${name} can't load the ${withApps.name} plugin's apps: ${apps}`;
|
|
1496
|
-
const layout = layoutOf(harness);
|
|
1497
|
-
const runs = layout.pluginHookEvents;
|
|
1498
|
-
for (const p of runs ? plugins : []) {
|
|
1499
|
-
const event = p.hooks.events.find((e) => !runs?.includes(e));
|
|
1500
|
-
const type = p.hooks.otherTypes[0];
|
|
1501
|
-
if (!event && !type) continue;
|
|
1502
|
-
const rest = p.kept.filter((part) => part !== "hooks");
|
|
1503
|
-
const fix = rest.length ? `use only: [${quoted(rest)}], or leave` : "leave";
|
|
1504
|
-
return event ? `${name} can't run the ${p.name} plugin's ${event} hooks (hooks/hooks.json): it runs ${runs?.join(", ")} hooks; ${fix} the plugin out of this cell` : `${name} can't run the ${p.name} plugin's ${type} hooks (hooks/hooks.json): it runs only command hooks; ${fix} the plugin out of this cell`;
|
|
1505
|
-
}
|
|
1506
|
-
const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), plannedMcpServer(s)])));
|
|
1507
|
-
if (agent && Object.keys(servers).length) {
|
|
1508
|
-
const alongside = agent.pluginPlacement === "flat" ? Object.fromEntries(own.mcp.map((n) => [n, {
|
|
1509
|
-
transport: "stdio",
|
|
1510
|
-
command: "true",
|
|
1511
|
-
args: [],
|
|
1512
|
-
env: {}
|
|
1513
|
-
}])) : {};
|
|
1514
|
-
try {
|
|
1515
|
-
agent.mcpServers({
|
|
1516
|
-
...alongside,
|
|
1517
|
-
...servers
|
|
1518
|
-
});
|
|
1519
|
-
} catch (error) {
|
|
1520
|
-
if (error instanceof InfraError) return error.message;
|
|
1521
|
-
throw error;
|
|
1522
|
-
}
|
|
1523
|
-
}
|
|
1524
|
-
for (const p of plugins) for (const server of Object.keys(p.mcp)) {
|
|
1525
|
-
const key = pluginServerName(p.name, server);
|
|
1526
|
-
if (own.mcp.includes(key)) return `the ${p.name} plugin's MCP server "${server}" would join mcp as ${key}, which mcp already names: rename mcp["${key}"]`;
|
|
1527
|
-
}
|
|
1528
|
-
if (layout.pluginPlacement === "namespaced") {
|
|
1529
|
-
for (const p of plugins) {
|
|
1530
|
-
if (!p.kept.includes("mcp")) continue;
|
|
1531
|
-
for (const [server, s] of Object.entries(p.mcp)) {
|
|
1532
|
-
const secret = hostVariable(s);
|
|
1533
|
-
if (!secret) continue;
|
|
1534
|
-
const rest = p.kept.filter((part) => part !== "mcp");
|
|
1535
|
-
return `${name} loads the ${p.name} plugin's MCP server "${server}" itself, so reeval can't give it ${secret} from the host: use only: [${quoted(rest)}] and declare the server in mcp, where reeval fills it in`;
|
|
1536
|
-
}
|
|
1537
|
-
}
|
|
1538
|
-
if (!(agent ?? mimics)?.pluginServersByName) return void 0;
|
|
1539
|
-
const byName = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
|
|
1540
|
-
for (const p of plugins) for (const server of Object.keys(p.mcp)) {
|
|
1541
|
-
const other = byName.get(server);
|
|
1542
|
-
if (other) return `${name} names the ${p.name} plugin's MCP server "${server}" by its own name, as it does ${other}, and keeps only one: rename one, or leave mcp out of the plugin with only`;
|
|
1543
|
-
byName.set(server, `the ${p.name} plugin's MCP server "${server}"`);
|
|
1544
|
-
}
|
|
1545
|
-
return;
|
|
1546
|
-
}
|
|
1547
|
-
const serverOwner = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
|
|
1548
|
-
for (const p of plugins) for (const server of Object.keys(p.mcp)) {
|
|
1549
|
-
const other = serverOwner.get(server);
|
|
1550
|
-
if (other) return `the ${p.name} plugin's MCP server "${server}" clashes with ${other}`;
|
|
1551
|
-
serverOwner.set(server, `the ${p.name} plugin's MCP server "${server}"`);
|
|
1552
|
-
}
|
|
1553
|
-
const skillOwner = new Map(own.skills.map((s) => [s.name, `${s.call}'s "${s.name}"`]));
|
|
1554
|
-
for (const p of plugins) for (const { name: skill } of p.skills) {
|
|
1555
|
-
const other = skillOwner.get(skill);
|
|
1556
|
-
if (other) return `the ${p.name} plugin's skill "${skill}" and ${other} would both be placed at ~/${layout.harnessSkillDir}/${skill}: rename one, or narrow it with only or exclude`;
|
|
1557
|
-
skillOwner.set(skill, `the ${p.name} plugin's skill "${skill}"`);
|
|
1558
|
-
}
|
|
1559
|
-
for (const part of ["agents", "commands"]) {
|
|
1560
|
-
const dir = layout.pluginUnpack?.[part];
|
|
1561
|
-
if (!dir) continue;
|
|
1562
|
-
const owner = /* @__PURE__ */ new Map();
|
|
1563
|
-
for (const p of plugins) for (const item of p[part]) {
|
|
1564
|
-
const other = owner.get(item);
|
|
1565
|
-
const what = part === "agents" ? "subagent" : "command";
|
|
1566
|
-
if (other) return `the ${p.name} plugin's ${what} "${item}" and the ${other} plugin's would both be placed at ${dir}/${item}.md: rename one, or narrow one with only`;
|
|
1567
|
-
owner.set(item, p.name);
|
|
1568
|
-
}
|
|
1569
|
-
}
|
|
1570
|
-
const rules = layout.pluginUnpack?.rules;
|
|
1571
|
-
const ruleOwner = /* @__PURE__ */ new Map();
|
|
1572
|
-
for (const p of rules ? plugins : []) for (const rule of p.rules) {
|
|
1573
|
-
const file = `${p.name}-${rule.replace(/\.mdc?$/, "")}.mdc`;
|
|
1574
|
-
const what = `the ${p.name} plugin's rule "${rule}"`;
|
|
1575
|
-
const other = ruleOwner.get(file);
|
|
1576
|
-
if (other) return `${what} and ${other} would both be placed at ${rules}/${file}: rename one, or narrow one with only`;
|
|
1577
|
-
ruleOwner.set(file, what);
|
|
1578
|
-
}
|
|
1579
|
-
}
|
|
1580
|
-
/**
|
|
1581
|
-
* A real agent's catalog entry, with what its model's provider decides, or its gateway's route:
|
|
1582
|
-
* on Vertex, the attempt's `vertex()` settings decide the project and region.
|
|
1583
|
-
*/
|
|
1584
|
-
function agentFor(h, vertex) {
|
|
1585
|
-
const agent = AGENTS[h.kind];
|
|
1586
|
-
if (h.gateway === "vertex") {
|
|
1587
|
-
const route = VERTEX_ROUTES[h.kind];
|
|
1588
|
-
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vertex AI`);
|
|
1589
|
-
if (!vertex) throw new InfraError(`${agent.name} on Vertex AI has no vertex() settings`);
|
|
1590
|
-
return {
|
|
1591
|
-
...agent,
|
|
1592
|
-
...route(vertex)
|
|
1593
|
-
};
|
|
1594
|
-
}
|
|
1595
|
-
if (isRoutingGateway(h.gateway)) {
|
|
1596
|
-
const gateway = ROUTING_GATEWAYS[h.gateway];
|
|
1597
|
-
const route = gateway.routes[h.kind];
|
|
1598
|
-
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on ${gateway.name}`);
|
|
1599
|
-
return {
|
|
1600
|
-
...agent,
|
|
1601
|
-
...route(process.env)
|
|
1602
|
-
};
|
|
1603
|
-
}
|
|
1604
|
-
return agent.forModel ? {
|
|
1605
|
-
...agent,
|
|
1606
|
-
...agent.forModel(h.model)
|
|
1607
|
-
} : agent;
|
|
1608
|
-
}
|
|
1609
|
-
/** The catalog entry for a harness's agent; undefined for the fake harness. */
|
|
1610
|
-
const agentOf = (h, vertex) => h.kind === "fake" ? void 0 : agentFor(h, vertex);
|
|
1611
|
-
/**
|
|
1612
|
-
* The host variable a harness authenticates with; undefined for the fake harness, and on Vertex,
|
|
1613
|
-
* whose token comes from Application Default Credentials.
|
|
1614
|
-
*/
|
|
1615
|
-
const credentialOf = (h) => h.kind === "fake" || h.gateway === "vertex" ? void 0 : agentFor(h).credential;
|
|
1616
|
-
/** The fake agent's own layout. */
|
|
1617
|
-
const FAKE_LAYOUT = {
|
|
1618
|
-
rulesFile: "AGENTS.md",
|
|
1619
|
-
workspaceSkillDir: ".agents/skills",
|
|
1620
|
-
harnessSkillDir: ".agents/skills",
|
|
1621
|
-
pluginPlacement: "flat"
|
|
1622
|
-
};
|
|
1623
|
-
/** Where a harness's agent reads rules and skills; a fake one with `layout` mimics that agent. */
|
|
1624
|
-
function layoutOf(descriptor) {
|
|
1625
|
-
if (descriptor.kind !== "fake") return LAYOUTS[descriptor.kind];
|
|
1626
|
-
return descriptor.layout ? LAYOUTS[descriptor.layout] : FAKE_LAYOUT;
|
|
1627
|
-
}
|
|
1628
|
-
/**
|
|
1629
|
-
* The hosts a harness's agent is known to try in a session with or without plugins; a fake agent
|
|
1630
|
-
* with `layout` expects that agent's. On Vertex, the attempt's `vertex()` settings pick the route.
|
|
1631
|
-
*/
|
|
1632
|
-
function expectedBlockedFor(descriptor, plugins, vertex) {
|
|
1633
|
-
const expected = descriptor.kind !== "fake" ? agentFor(descriptor, vertex).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
|
|
1634
|
-
return expected?.withPlugins && !plugins ? void 0 : expected;
|
|
1635
|
-
}
|
|
1636
|
-
/**
|
|
1637
|
-
* How the APIs behind the harnesses say they refused a key: an HTTP 401, Anthropic's and OpenAI's
|
|
1638
|
-
* wording, Google's (which answers 400), and OpenCode's for any provider's 401 or 403. A bare 403 isn't here: the gateway and MCP servers send
|
|
1639
|
-
* those for reasons that have nothing to do with the harness's key. Grok Build checks its key as
|
|
1640
|
-
* it starts and, when xAI refuses it (a 400 or 401, not an outage), offers no `xai.api_key`
|
|
1641
|
-
* sign-in, which is all `@ai-sdk/harness-acp` reports.
|
|
1642
|
-
*/
|
|
1643
|
-
const REFUSED_KEY = /\bHTTP 401\b|\b401\b.{0,40}\bunauthori[sz]ed\b|\bunauthori[sz]ed\b.{0,40}\b401\b|\b(?:invalid|incorrect) (?:x-)?api[ _-]?key\b|\bapi key not valid\b|\bAPI_KEY_INVALID\b|\bauthentication_error\b|\bprovider authentication required\b|\bauthentication method "xai\.api_key" is not advertised\b/i;
|
|
1644
|
-
/**
|
|
1645
|
-
* What to do when the harness's API refused its key, naming the variable; else undefined. The
|
|
1646
|
-
* error's wording says so for some CLIs; `refused` is the egress record's word for every one: a
|
|
1647
|
-
* brokered request the API answered with 401, whatever the CLI made of it.
|
|
1648
|
-
*/
|
|
1649
|
-
function authHint(agent, message, refused = false) {
|
|
1650
|
-
if (!refused && !REFUSED_KEY.test(message)) return void 0;
|
|
1651
|
-
if (agent.token) return `The API rejected the access token reeval minted from ${agent.credential}: check gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS) and the project's Vertex AI access`;
|
|
1652
|
-
return keyRejected(agent.credential);
|
|
1653
|
-
}
|
|
1654
|
-
/** What to do when the API refused the key in `credential`, naming it. */
|
|
1655
|
-
const keyRejected = (credential) => `The API rejected ${credential}: set a valid key in the environment or .env.local`;
|
|
1656
|
-
/**
|
|
1657
|
-
* The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. Empty for
|
|
1658
|
-
* a brokered token, which the adapter never sees.
|
|
1659
|
-
*/
|
|
1660
|
-
function agentAuth(agent, env = process.env) {
|
|
1661
|
-
if (agent.token) return {};
|
|
1662
|
-
const value = env[agent.credential];
|
|
1663
|
-
if (!value) throw new InfraError(`${agent.credential} isn't set, so the ${agent.name} harness can't authenticate. Set it in the environment or .env.local.`);
|
|
1664
|
-
return {
|
|
1665
|
-
...agent.authEnv,
|
|
1666
|
-
[agent.authAs ?? agent.credential]: value
|
|
1667
|
-
};
|
|
1668
|
-
}
|
|
1669
|
-
function brokeringFor(h, vertex) {
|
|
1670
|
-
if (h.kind === "fake") {
|
|
1671
|
-
if (!h.brokers) return void 0;
|
|
1672
|
-
const { credential, endpoints } = AGENTS[h.layout ?? "cursor"];
|
|
1673
|
-
return {
|
|
1674
|
-
credential,
|
|
1675
|
-
endpoints
|
|
1676
|
-
};
|
|
1677
|
-
}
|
|
1678
|
-
const agent = agentOf(h, vertex);
|
|
1679
|
-
if (!agent) return void 0;
|
|
1680
|
-
const { credential, endpoints, tokens, servedModel, askedModel, modelResponses, upstream } = agent;
|
|
1681
|
-
const { localResponses } = agent;
|
|
1682
|
-
return {
|
|
1683
|
-
credential,
|
|
1684
|
-
endpoints,
|
|
1685
|
-
...tokens ? { tokens: {
|
|
1686
|
-
name: credential,
|
|
1687
|
-
...tokens
|
|
1688
|
-
} } : {},
|
|
1689
|
-
...servedModel ? { servedModel } : {},
|
|
1690
|
-
...askedModel ? { askedModel } : {},
|
|
1691
|
-
...modelResponses ? { modelResponses } : {},
|
|
1692
|
-
...upstream ? { upstream } : {},
|
|
1693
|
-
...localResponses?.length ? { localResponses } : {}
|
|
1694
|
-
};
|
|
1695
|
-
}
|
|
1696
584
|
// src/harness/effort.ts
|
|
1697
585
|
/**
|
|
1698
586
|
* The reasoning effort a harness asked for. Same user-facing name on every factory; the CLI's
|
|
@@ -1742,6 +630,638 @@ function resolvedEffortOf(models) {
|
|
|
1742
630
|
}
|
|
1743
631
|
return out;
|
|
1744
632
|
}
|
|
633
|
+
// src/contract/attempt.ts
|
|
634
|
+
const nullableNumber = z.number().nullable();
|
|
635
|
+
const fileChangeSchema = z.strictObject({
|
|
636
|
+
path: z.string(),
|
|
637
|
+
status: z.enum([
|
|
638
|
+
"added",
|
|
639
|
+
"modified",
|
|
640
|
+
"deleted"
|
|
641
|
+
])
|
|
642
|
+
});
|
|
643
|
+
const diffSchema = z.strictObject({
|
|
644
|
+
patch: z.string(),
|
|
645
|
+
files: z.array(fileChangeSchema)
|
|
646
|
+
});
|
|
647
|
+
const turnStatusSchema = z.enum([
|
|
648
|
+
"completed",
|
|
649
|
+
"aborted",
|
|
650
|
+
"error"
|
|
651
|
+
]);
|
|
652
|
+
const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
|
|
653
|
+
const at$1 = {
|
|
654
|
+
ts: z.string(),
|
|
655
|
+
atMs: z.number()
|
|
656
|
+
};
|
|
657
|
+
const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
658
|
+
z.strictObject({
|
|
659
|
+
...at$1,
|
|
660
|
+
type: z.enum(["text", "reasoning"]),
|
|
661
|
+
text: z.string(),
|
|
662
|
+
durationMs: nullableNumber
|
|
663
|
+
}),
|
|
664
|
+
z.strictObject({
|
|
665
|
+
...at$1,
|
|
666
|
+
type: z.literal("file-edit"),
|
|
667
|
+
toolCallId: z.string(),
|
|
668
|
+
path: z.string(),
|
|
669
|
+
kind: z.enum([
|
|
670
|
+
"create",
|
|
671
|
+
"modify",
|
|
672
|
+
"write",
|
|
673
|
+
"delete"
|
|
674
|
+
]),
|
|
675
|
+
added: nullableNumber,
|
|
676
|
+
removed: nullableNumber,
|
|
677
|
+
durationMs: nullableNumber,
|
|
678
|
+
stalled: z.boolean().optional(),
|
|
679
|
+
subagent: z.string().optional()
|
|
680
|
+
}),
|
|
681
|
+
z.strictObject({
|
|
682
|
+
...at$1,
|
|
683
|
+
type: z.literal("web-search"),
|
|
684
|
+
toolCallId: z.string(),
|
|
685
|
+
query: z.string(),
|
|
686
|
+
references: nullableNumber,
|
|
687
|
+
ok: z.boolean().nullable().optional(),
|
|
688
|
+
durationMs: nullableNumber,
|
|
689
|
+
stalled: z.boolean().optional(),
|
|
690
|
+
subagent: z.string().optional()
|
|
691
|
+
}),
|
|
692
|
+
z.strictObject({
|
|
693
|
+
...at$1,
|
|
694
|
+
type: z.literal("web-fetch"),
|
|
695
|
+
toolCallId: z.string(),
|
|
696
|
+
url: z.string(),
|
|
697
|
+
ok: z.boolean().nullable(),
|
|
698
|
+
durationMs: nullableNumber,
|
|
699
|
+
stalled: z.boolean().optional(),
|
|
700
|
+
subagent: z.string().optional()
|
|
701
|
+
}),
|
|
702
|
+
z.strictObject({
|
|
703
|
+
...at$1,
|
|
704
|
+
type: z.literal("tool"),
|
|
705
|
+
toolCallId: z.string(),
|
|
706
|
+
tool: z.string(),
|
|
707
|
+
input: z.unknown(),
|
|
708
|
+
output: z.unknown(),
|
|
709
|
+
ok: z.boolean().nullable(),
|
|
710
|
+
durationMs: nullableNumber,
|
|
711
|
+
stalled: z.boolean().optional(),
|
|
712
|
+
subagent: z.string().optional()
|
|
713
|
+
}),
|
|
714
|
+
z.strictObject({
|
|
715
|
+
...at$1,
|
|
716
|
+
type: z.literal("error"),
|
|
717
|
+
message: z.string()
|
|
718
|
+
}),
|
|
719
|
+
z.strictObject({
|
|
720
|
+
...at$1,
|
|
721
|
+
type: z.literal("retry"),
|
|
722
|
+
message: z.string()
|
|
723
|
+
})
|
|
724
|
+
]);
|
|
725
|
+
const turnDataSchema = z.strictObject({
|
|
726
|
+
prompt: z.string(),
|
|
727
|
+
text: z.string(),
|
|
728
|
+
status: turnStatusSchema,
|
|
729
|
+
startedAt: z.string(),
|
|
730
|
+
durationMs: z.number(),
|
|
731
|
+
transcript: z.array(transcriptEventSchema),
|
|
732
|
+
usage: tokenUsageSchema.nullable(),
|
|
733
|
+
model: z.string().nullable(),
|
|
734
|
+
unreported: z.array(webEventTypeSchema),
|
|
735
|
+
diff: diffSchema
|
|
736
|
+
});
|
|
737
|
+
const execResultSchema = z.strictObject({
|
|
738
|
+
exitCode: z.number().int(),
|
|
739
|
+
stdout: z.string(),
|
|
740
|
+
stderr: z.string()
|
|
741
|
+
});
|
|
742
|
+
const execRetrySchema = z.strictObject({
|
|
743
|
+
retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
|
|
744
|
+
interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
|
|
745
|
+
});
|
|
746
|
+
/** A background command has no exit for `retries` to wait on. */
|
|
747
|
+
const noBackgroundRetry = (options, ctx) => {
|
|
748
|
+
if (!options.background) return;
|
|
749
|
+
for (const key of ["retries", "interval"]) {
|
|
750
|
+
if (options[key] === void 0) continue;
|
|
751
|
+
ctx.addIssue({
|
|
752
|
+
code: "custom",
|
|
753
|
+
path: [key],
|
|
754
|
+
message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
|
|
755
|
+
});
|
|
756
|
+
}
|
|
757
|
+
};
|
|
758
|
+
const execOptionsSchema = z.strictObject({
|
|
759
|
+
background: z.boolean().optional(),
|
|
760
|
+
...execRetrySchema.shape
|
|
761
|
+
}).superRefine(noBackgroundRetry);
|
|
762
|
+
/** What's wrong with `c.exec()`'s options, or undefined. */
|
|
763
|
+
const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
|
|
764
|
+
const assertionRecordSchema = z.strictObject({
|
|
765
|
+
label: z.string(),
|
|
766
|
+
labelled: z.boolean(),
|
|
767
|
+
matcher: z.string(),
|
|
768
|
+
status: z.enum([
|
|
769
|
+
"pass",
|
|
770
|
+
"fail",
|
|
771
|
+
"error"
|
|
772
|
+
]),
|
|
773
|
+
soft: z.boolean(),
|
|
774
|
+
at: z.string().optional(),
|
|
775
|
+
message: z.string().optional(),
|
|
776
|
+
durationMs: z.number().optional()
|
|
777
|
+
});
|
|
778
|
+
const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
|
|
779
|
+
const captureConsoleMessageSchema = z.strictObject({
|
|
780
|
+
level: z.enum([
|
|
781
|
+
"error",
|
|
782
|
+
"warning",
|
|
783
|
+
"info",
|
|
784
|
+
"log",
|
|
785
|
+
"debug"
|
|
786
|
+
]),
|
|
787
|
+
text: z.string(),
|
|
788
|
+
source: z.string(),
|
|
789
|
+
url: z.string().optional()
|
|
790
|
+
});
|
|
791
|
+
const captureFailedRequestSchema = z.strictObject({
|
|
792
|
+
method: z.string(),
|
|
793
|
+
url: z.string(),
|
|
794
|
+
status: z.number().optional(),
|
|
795
|
+
error: z.string().optional()
|
|
796
|
+
});
|
|
797
|
+
const capturePageFields = {
|
|
798
|
+
status: z.number().optional(),
|
|
799
|
+
finalUrl: z.string(),
|
|
800
|
+
console: z.array(captureConsoleMessageSchema),
|
|
801
|
+
errors: z.array(z.string()),
|
|
802
|
+
failedRequests: z.array(captureFailedRequestSchema),
|
|
803
|
+
dropped: z.number().optional()
|
|
804
|
+
};
|
|
805
|
+
z.strictObject(capturePageFields);
|
|
806
|
+
const captureSchema = z.strictObject({
|
|
807
|
+
name: z.string(),
|
|
808
|
+
port: z.number(),
|
|
809
|
+
path: z.string(),
|
|
810
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]),
|
|
811
|
+
file: z.strictObject({
|
|
812
|
+
path: z.string(),
|
|
813
|
+
sha256: z.string(),
|
|
814
|
+
bytes: z.number()
|
|
815
|
+
}),
|
|
816
|
+
...capturePageFields
|
|
817
|
+
});
|
|
818
|
+
const judgeArtifactRefSchema = z.strictObject({
|
|
819
|
+
name: z.string(),
|
|
820
|
+
path: z.string(),
|
|
821
|
+
sha256: z.string(),
|
|
822
|
+
bytes: z.number(),
|
|
823
|
+
mediaType: z.literal("image/png")
|
|
824
|
+
});
|
|
825
|
+
const verdictFields = {
|
|
826
|
+
threshold: z.number(),
|
|
827
|
+
question: z.string(),
|
|
828
|
+
model: z.string(),
|
|
829
|
+
gateway: gatewayNameSchema.optional(),
|
|
830
|
+
artifacts: z.array(judgeArtifactRefSchema).optional()
|
|
831
|
+
};
|
|
832
|
+
const verdictSchema = z.strictObject({
|
|
833
|
+
verdict: z.enum(["pass", "fail"]),
|
|
834
|
+
score: z.number(),
|
|
835
|
+
...verdictFields,
|
|
836
|
+
resolvedModel: z.string().optional(),
|
|
837
|
+
upstream: z.string().optional(),
|
|
838
|
+
costUsd: z.number().optional()
|
|
839
|
+
});
|
|
840
|
+
const judgeFailureSchema = z.strictObject({
|
|
841
|
+
verdict: z.literal("error"),
|
|
842
|
+
score: z.null(),
|
|
843
|
+
...verdictFields,
|
|
844
|
+
error: z.string()
|
|
845
|
+
});
|
|
846
|
+
/** A pattern, for the matchers that take a string or a `RegExp`. */
|
|
847
|
+
const patternSchema = z.strictObject({
|
|
848
|
+
regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
|
|
849
|
+
flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
|
|
850
|
+
}).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
|
|
851
|
+
/** A pattern in the runner's own language, which reeval never compiles. */
|
|
852
|
+
function hasAnsweredPattern(value) {
|
|
853
|
+
if (Array.isArray(value)) return value.some(hasAnsweredPattern);
|
|
854
|
+
if (typeof value !== "object" || value === null) return false;
|
|
855
|
+
return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
|
|
856
|
+
}
|
|
857
|
+
const answeredPatternSchema = z.strictObject({
|
|
858
|
+
matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
|
|
859
|
+
answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
|
|
860
|
+
}).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
|
|
861
|
+
const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
|
|
862
|
+
const matcherArgSchema = z.union([
|
|
863
|
+
patternSchema,
|
|
864
|
+
answeredPatternSchema,
|
|
865
|
+
jsonValueSchema
|
|
866
|
+
]).superRefine((value, ctx) => {
|
|
867
|
+
if (!isPatternLike(value)) return;
|
|
868
|
+
for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
|
|
869
|
+
}).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
|
|
870
|
+
const matcherSubjectSchema = z.union([
|
|
871
|
+
z.strictObject({ turn: turnIndex }),
|
|
872
|
+
z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
|
|
873
|
+
z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
|
|
874
|
+
z.strictObject({
|
|
875
|
+
exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
|
|
876
|
+
result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
|
|
877
|
+
}),
|
|
878
|
+
z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
|
|
879
|
+
]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
|
|
880
|
+
const nonEmpty = z.string().regex(/\S/, "must not be empty");
|
|
881
|
+
const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
|
|
882
|
+
const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
|
|
883
|
+
const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
|
|
884
|
+
const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
|
|
885
|
+
const captureWaitSchema = z.enum(["load", "idle"]);
|
|
886
|
+
const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
|
|
887
|
+
const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
|
|
888
|
+
const captureCallDataSchema = z.strictObject({
|
|
889
|
+
name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
|
|
890
|
+
path: z.string().optional(),
|
|
891
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
892
|
+
timeout: durationSchema.optional(),
|
|
893
|
+
wait: waitSchema.optional(),
|
|
894
|
+
instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
|
|
895
|
+
model: z.string().optional()
|
|
896
|
+
}).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
|
|
897
|
+
const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
|
|
898
|
+
const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
|
|
899
|
+
const judgeCallDataSchema = z.strictObject({
|
|
900
|
+
model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
|
|
901
|
+
...judgeFields,
|
|
902
|
+
...sharedInputFields,
|
|
903
|
+
contents: z.array(nonEmpty).optional(),
|
|
904
|
+
turns: turnsSchema.optional(),
|
|
905
|
+
on: judgeOnSchema.optional().describe("The turn `task`, `output`, `transcript` and `diff` read instead of the latest: an earlier turn, or a value graded as the output."),
|
|
906
|
+
artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
|
|
907
|
+
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, the inputs it shows the judge (at least one turned on) and `on`.");
|
|
908
|
+
/** An observation's name: one a query, a tally line and a judge's input can carry as it is. */
|
|
909
|
+
const OBSERVE_NAME = /^[A-Za-z0-9_.:/-]{1,64}$/;
|
|
910
|
+
const callSiteSchema = z.strictObject({
|
|
911
|
+
file: nonEmpty.describe("The eval file: absolute, or relative to the project root."),
|
|
912
|
+
line: z.number().int().min(1).describe("From 1.")
|
|
913
|
+
}).describe("Where the eval made the call, which the record names beside the check's outcome.");
|
|
914
|
+
const attemptCallSchema = z.discriminatedUnion("op", [
|
|
915
|
+
z.strictObject({
|
|
916
|
+
op: z.literal("send"),
|
|
917
|
+
prompt: z.string().describe("The turn's prompt.")
|
|
918
|
+
}).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
|
|
919
|
+
z.strictObject({
|
|
920
|
+
op: z.literal("exec"),
|
|
921
|
+
command: nonEmpty.describe("A shell command, run in the workspace."),
|
|
922
|
+
background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
|
|
923
|
+
...execRetrySchema.shape
|
|
924
|
+
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
925
|
+
z.strictObject({
|
|
926
|
+
op: z.literal("serve"),
|
|
927
|
+
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
928
|
+
port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
|
|
929
|
+
}).describe("`c.serve(command, port?)`: a background process, once it listens, and its port."),
|
|
930
|
+
z.strictObject({
|
|
931
|
+
op: z.literal("kill"),
|
|
932
|
+
handle: handleId
|
|
933
|
+
}).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
|
|
934
|
+
z.strictObject({
|
|
935
|
+
op: z.literal("capture"),
|
|
936
|
+
target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
|
|
937
|
+
options: captureCallDataSchema.optional()
|
|
938
|
+
}).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
|
|
939
|
+
z.strictObject({
|
|
940
|
+
op: z.literal("readFile"),
|
|
941
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
|
|
942
|
+
}).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
|
|
943
|
+
z.strictObject({
|
|
944
|
+
op: z.literal("writeFile"),
|
|
945
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
|
|
946
|
+
contents: z.string().describe("The file's contents, in `encoding`."),
|
|
947
|
+
encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
|
|
948
|
+
}).describe("`c.writeFile(path, contents)`, creating its directories."),
|
|
949
|
+
z.strictObject({
|
|
950
|
+
op: z.literal("observe"),
|
|
951
|
+
name: z.string().regex(OBSERVE_NAME, "must be 1 to 64 letters, digits, _ . : / or -").describe("What the value is of, e.g. `installed`."),
|
|
952
|
+
value: jsonValueSchema.describe("Any JSON, at most 4 KB once serialized.")
|
|
953
|
+
}).describe("`c.observe(name, value)`: records a fact about the attempt that never affects its outcome. A name observed again keeps the last value. Fails after the body's `done`."),
|
|
954
|
+
z.strictObject({
|
|
955
|
+
op: z.literal("expect"),
|
|
956
|
+
subject: matcherSubjectSchema,
|
|
957
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
|
|
958
|
+
args: z.array(matcherArgSchema).describe("The matcher's arguments."),
|
|
959
|
+
not: z.boolean().describe("Called as `expect(…).not`."),
|
|
960
|
+
soft: z.boolean().describe("Called as `expect`: a failure fails the attempt at its end and the body carries on. False for `expect.require`, whose failure the runner raises, ending the body."),
|
|
961
|
+
label: z.string().optional().describe("`expect`'s second argument."),
|
|
962
|
+
at: callSiteSchema.optional(),
|
|
963
|
+
poll: z.strictObject({
|
|
964
|
+
interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
|
|
965
|
+
timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
|
|
966
|
+
}).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
|
|
967
|
+
}).superRefine((call, ctx) => {
|
|
968
|
+
if (call.poll && "value" in call.subject) ctx.addIssue({
|
|
969
|
+
code: "custom",
|
|
970
|
+
path: ["subject"],
|
|
971
|
+
message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
|
|
972
|
+
});
|
|
973
|
+
if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
|
|
974
|
+
code: "custom",
|
|
975
|
+
path: ["subject", "result"],
|
|
976
|
+
message: "can't be polled: drop result so reeval runs the command on each try"
|
|
977
|
+
});
|
|
978
|
+
if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
|
|
979
|
+
code: "custom",
|
|
980
|
+
path: ["args"],
|
|
981
|
+
message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
|
|
982
|
+
});
|
|
983
|
+
}).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
|
|
984
|
+
z.strictObject({
|
|
985
|
+
op: z.literal("record"),
|
|
986
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
|
|
987
|
+
subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
|
|
988
|
+
label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
|
|
989
|
+
status: z.enum(["pass", "fail"]),
|
|
990
|
+
message: z.string().optional().describe("Why it failed."),
|
|
991
|
+
soft: z.boolean().optional().describe("The body carried on after a failure, as after an `expect`. Default false: the check raised, as Python's `assert` does."),
|
|
992
|
+
at: callSiteSchema.optional()
|
|
993
|
+
}).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
|
|
994
|
+
z.strictObject({
|
|
995
|
+
op: z.literal("judge"),
|
|
996
|
+
question: nonEmpty.describe("A yes/no question about the inputs `options` turns on."),
|
|
997
|
+
options: judgeCallDataSchema
|
|
998
|
+
}).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
|
|
999
|
+
]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
|
|
1000
|
+
const attemptErrorSchema = z.looseObject({
|
|
1001
|
+
name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
|
|
1002
|
+
message: z.string()
|
|
1003
|
+
}).describe("Why a call failed, in reeval's words.");
|
|
1004
|
+
const id = nonEmpty.describe("Pairs a reply with its request.");
|
|
1005
|
+
const attemptRequestSchema = z.strictObject({
|
|
1006
|
+
id,
|
|
1007
|
+
call: attemptCallSchema
|
|
1008
|
+
}).describe("A call, numbered so its reply can find it.");
|
|
1009
|
+
z.union([
|
|
1010
|
+
z.discriminatedUnion("op", [
|
|
1011
|
+
z.strictObject({
|
|
1012
|
+
id,
|
|
1013
|
+
ok: z.literal(true),
|
|
1014
|
+
op: z.literal("send"),
|
|
1015
|
+
turn: turnDataSchema
|
|
1016
|
+
}),
|
|
1017
|
+
z.strictObject({
|
|
1018
|
+
id,
|
|
1019
|
+
ok: z.literal(true),
|
|
1020
|
+
op: z.literal("exec"),
|
|
1021
|
+
result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
|
|
1022
|
+
}),
|
|
1023
|
+
z.strictObject({
|
|
1024
|
+
id,
|
|
1025
|
+
ok: z.literal(true),
|
|
1026
|
+
op: z.literal("serve"),
|
|
1027
|
+
handle: handleId,
|
|
1028
|
+
port: z.number().describe("The container port it listens on.")
|
|
1029
|
+
}),
|
|
1030
|
+
z.strictObject({
|
|
1031
|
+
id,
|
|
1032
|
+
ok: z.literal(true),
|
|
1033
|
+
op: z.literal("kill")
|
|
1034
|
+
}),
|
|
1035
|
+
z.strictObject({
|
|
1036
|
+
id,
|
|
1037
|
+
ok: z.literal(true),
|
|
1038
|
+
op: z.literal("capture"),
|
|
1039
|
+
capture: captureSchema
|
|
1040
|
+
}),
|
|
1041
|
+
z.strictObject({
|
|
1042
|
+
id,
|
|
1043
|
+
ok: z.literal(true),
|
|
1044
|
+
op: z.literal("readFile"),
|
|
1045
|
+
contents: z.string().nullable().describe("Null when there is no such file.")
|
|
1046
|
+
}),
|
|
1047
|
+
z.strictObject({
|
|
1048
|
+
id,
|
|
1049
|
+
ok: z.literal(true),
|
|
1050
|
+
op: z.literal("writeFile")
|
|
1051
|
+
}),
|
|
1052
|
+
z.strictObject({
|
|
1053
|
+
id,
|
|
1054
|
+
ok: z.literal(true),
|
|
1055
|
+
op: z.literal("observe")
|
|
1056
|
+
}),
|
|
1057
|
+
z.strictObject({
|
|
1058
|
+
id,
|
|
1059
|
+
ok: z.literal(true),
|
|
1060
|
+
op: z.literal("expect"),
|
|
1061
|
+
assertion: assertionRecordSchema
|
|
1062
|
+
}),
|
|
1063
|
+
z.strictObject({
|
|
1064
|
+
id,
|
|
1065
|
+
ok: z.literal(true),
|
|
1066
|
+
op: z.literal("record"),
|
|
1067
|
+
assertion: assertionRecordSchema
|
|
1068
|
+
}),
|
|
1069
|
+
z.strictObject({
|
|
1070
|
+
id,
|
|
1071
|
+
ok: z.literal(true),
|
|
1072
|
+
op: z.literal("judge"),
|
|
1073
|
+
verdict: z.union([verdictSchema, judgeFailureSchema])
|
|
1074
|
+
})
|
|
1075
|
+
]),
|
|
1076
|
+
z.strictObject({
|
|
1077
|
+
id,
|
|
1078
|
+
ok: z.literal(true),
|
|
1079
|
+
op: z.literal("expect"),
|
|
1080
|
+
untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
|
|
1081
|
+
}),
|
|
1082
|
+
z.strictObject({
|
|
1083
|
+
id,
|
|
1084
|
+
ok: z.literal(false),
|
|
1085
|
+
op: z.enum([
|
|
1086
|
+
"send",
|
|
1087
|
+
"exec",
|
|
1088
|
+
"readFile",
|
|
1089
|
+
"writeFile",
|
|
1090
|
+
"observe",
|
|
1091
|
+
"expect",
|
|
1092
|
+
"record",
|
|
1093
|
+
"judge",
|
|
1094
|
+
"serve",
|
|
1095
|
+
"kill",
|
|
1096
|
+
"capture"
|
|
1097
|
+
]),
|
|
1098
|
+
error: attemptErrorSchema
|
|
1099
|
+
})
|
|
1100
|
+
]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
|
|
1101
|
+
const cellRoleSchema = z.enum(["control", "variant"]);
|
|
1102
|
+
const cellSchema = z.strictObject({
|
|
1103
|
+
key: z.string(),
|
|
1104
|
+
harness: z.string(),
|
|
1105
|
+
environment: z.string(),
|
|
1106
|
+
sandbox: z.string(),
|
|
1107
|
+
role: cellRoleSchema,
|
|
1108
|
+
variant: z.string().optional(),
|
|
1109
|
+
traits: traitsSchema.optional()
|
|
1110
|
+
});
|
|
1111
|
+
z.strictObject({
|
|
1112
|
+
type: z.literal("start"),
|
|
1113
|
+
attempt: z.strictObject({
|
|
1114
|
+
id: nonEmpty,
|
|
1115
|
+
cell: cellSchema,
|
|
1116
|
+
n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
|
|
1117
|
+
}).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
|
|
1118
|
+
file: nonEmpty.describe("The eval file, relative to the project."),
|
|
1119
|
+
body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
|
|
1120
|
+
}).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
|
|
1121
|
+
z.strictObject({
|
|
1122
|
+
type: z.literal("abort"),
|
|
1123
|
+
reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
|
|
1124
|
+
message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
|
|
1125
|
+
}).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
|
|
1126
|
+
z.strictObject({
|
|
1127
|
+
type: z.literal("done"),
|
|
1128
|
+
error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect.require` is one.")
|
|
1129
|
+
}).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
|
|
1130
|
+
const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
|
|
1131
|
+
const OTHER_KEYS = [
|
|
1132
|
+
"id",
|
|
1133
|
+
"call",
|
|
1134
|
+
"turn",
|
|
1135
|
+
"attempt",
|
|
1136
|
+
"value",
|
|
1137
|
+
"regexp",
|
|
1138
|
+
"flags",
|
|
1139
|
+
"matchedBy",
|
|
1140
|
+
"answers",
|
|
1141
|
+
...Object.keys(judgeCallDataSchema.shape),
|
|
1142
|
+
...Object.keys(captureSchema.shape),
|
|
1143
|
+
...Object.keys(captureCallDataSchema.shape),
|
|
1144
|
+
"port",
|
|
1145
|
+
"handle",
|
|
1146
|
+
"exec",
|
|
1147
|
+
"result",
|
|
1148
|
+
"readFile",
|
|
1149
|
+
"interval",
|
|
1150
|
+
"timeout",
|
|
1151
|
+
"ms"
|
|
1152
|
+
];
|
|
1153
|
+
/** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
|
|
1154
|
+
const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
|
|
1155
|
+
// src/attempt/observe.ts
|
|
1156
|
+
/** The most JSON one value may serialize to, masked. */
|
|
1157
|
+
const MAX_OBSERVED_BYTES = 4096;
|
|
1158
|
+
/** The deepest a value may nest arrays and objects, so a check of it can't run out of stack. */
|
|
1159
|
+
const MAX_DEPTH = 64;
|
|
1160
|
+
const at = (path) => path ? ` at ${path}` : "";
|
|
1161
|
+
/** Why `value` isn't JSON, naming where, or undefined. */
|
|
1162
|
+
function notJson(value, path, seen, depth = 0) {
|
|
1163
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return void 0;
|
|
1164
|
+
if (typeof value === "number") return Number.isFinite(value) ? void 0 : `${value}${at(path)} isn't a finite number`;
|
|
1165
|
+
if (typeof value !== "object") return `a ${typeof value}${at(path)} isn't JSON`;
|
|
1166
|
+
if (seen.has(value)) return `the value refers to itself${at(path)}`;
|
|
1167
|
+
if (depth >= MAX_DEPTH) return `it nests deeper than ${MAX_DEPTH} levels${at(path)}`;
|
|
1168
|
+
const proto = Object.getPrototypeOf(value);
|
|
1169
|
+
if (!Array.isArray(value) && proto !== Object.prototype && proto !== null) return `a ${value.constructor?.name ?? "class instance"}${at(path)} isn't plain JSON`;
|
|
1170
|
+
seen.add(value);
|
|
1171
|
+
const entries = Array.isArray(value) ? Array.from(value, (v, i) => [`${path}[${i}]`, v]) : Object.entries(value).map(([k, v]) => [path ? `${path}.${k}` : k, v]);
|
|
1172
|
+
for (const [p, v] of entries) {
|
|
1173
|
+
const problem = notJson(v, p, seen, depth + 1);
|
|
1174
|
+
if (problem) return problem;
|
|
1175
|
+
}
|
|
1176
|
+
seen.delete(value);
|
|
1177
|
+
}
|
|
1178
|
+
/**
|
|
1179
|
+
* `value` with each number whose digits are a secret's masked too: `redactValue` leaves numbers
|
|
1180
|
+
* as they are, but an observed value goes to a judge's or question's model as it's recorded.
|
|
1181
|
+
*/
|
|
1182
|
+
function maskNumbers(value) {
|
|
1183
|
+
if (typeof value === "number") return redactSecrets(String(value)) === String(value) ? value : REDACTED;
|
|
1184
|
+
if (Array.isArray(value)) return value.map(maskNumbers);
|
|
1185
|
+
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, maskNumbers(v)]));
|
|
1186
|
+
return value;
|
|
1187
|
+
}
|
|
1188
|
+
/**
|
|
1189
|
+
* An observation's name as it's recorded and shown: masked as its value is. A name that holds a
|
|
1190
|
+
* secret is a value too, and a judge's or question's list of names is matched against this.
|
|
1191
|
+
*/
|
|
1192
|
+
const observedName = (name) => redactSecrets(name);
|
|
1193
|
+
/** Each attempt's names as its body gave them, by the name they're recorded as. */
|
|
1194
|
+
const givenNames = /* @__PURE__ */ new WeakMap();
|
|
1195
|
+
/**
|
|
1196
|
+
* Whether this attempt observed `name` as given: its masked name alone may be another secret's.
|
|
1197
|
+
* An attempt that never called `c.observe` has observed nothing.
|
|
1198
|
+
*/
|
|
1199
|
+
const observedHere = (a, name) => givenNames.get(a)?.get(observedName(name)) === name;
|
|
1200
|
+
/**
|
|
1201
|
+
* `c.observe`: keeps the last value per name, masked, with the turn it came after. Two names that
|
|
1202
|
+
* mask to the same name are refused rather than renamed: a name is the key that tallies, compare
|
|
1203
|
+
* and `attempt_values` group by across attempts, and a numbered stand-in would depend on the
|
|
1204
|
+
* order of each attempt's calls, joining values that don't belong together.
|
|
1205
|
+
*/
|
|
1206
|
+
function observer(a) {
|
|
1207
|
+
const raw = /* @__PURE__ */ new Map();
|
|
1208
|
+
givenNames.set(a, raw);
|
|
1209
|
+
return (given, value) => {
|
|
1210
|
+
if (typeof given !== "string" || !OBSERVE_NAME.test(given)) throw new UsageError(`c.observe() needs a name of 1 to 64 letters, digits, _ . : / or - (got ${JSON.stringify(typeof given === "string" ? observedName(given) : given)})`);
|
|
1211
|
+
const name = observedName(given);
|
|
1212
|
+
const earlier = raw.get(name);
|
|
1213
|
+
if (earlier !== void 0 && earlier !== given) throw new UsageError(`c.observe("${name}") names a secret, and its masked name is another observation's: name the value without the secret`);
|
|
1214
|
+
if (a.ended) throw new UsageError(`c.observe("${name}") came after the eval's body returned`);
|
|
1215
|
+
if (a.signal.aborted) throw new UsageError(`c.observe("${name}") came after the attempt timed out or was canceled`);
|
|
1216
|
+
const problem = notJson(value, "", /* @__PURE__ */ new Set());
|
|
1217
|
+
if (problem) throw new UsageError(`c.observe("${name}") takes JSON: ${problem}`);
|
|
1218
|
+
const masked = maskNumbers(redactValue(value));
|
|
1219
|
+
const bytes = Buffer.byteLength(JSON.stringify(masked));
|
|
1220
|
+
if (bytes > 4096) throw new UsageError(`c.observe("${name}") takes at most ${MAX_OBSERVED_BYTES} bytes of JSON (got ${bytes}): observe a count or a summary instead`);
|
|
1221
|
+
a.record.observations ??= [];
|
|
1222
|
+
const list = a.record.observations;
|
|
1223
|
+
const same = list.find((o) => o.name === name);
|
|
1224
|
+
if (!same && list.length >= 50) throw new UsageError(`c.observe("${name}") is one name too many: an attempt observes at most 50`);
|
|
1225
|
+
raw.set(name, given);
|
|
1226
|
+
const observation = {
|
|
1227
|
+
name,
|
|
1228
|
+
value: masked,
|
|
1229
|
+
turn: a.turns.length
|
|
1230
|
+
};
|
|
1231
|
+
if (same) Object.assign(same, observation);
|
|
1232
|
+
else list.push(observation);
|
|
1233
|
+
};
|
|
1234
|
+
}
|
|
1235
|
+
// src/judge/state-options.ts
|
|
1236
|
+
/**
|
|
1237
|
+
* What one call turned on, every key set (`false` when off), in the record's key order, so equal
|
|
1238
|
+
* selections serialize the same.
|
|
1239
|
+
*/
|
|
1240
|
+
function resolveStateOptions(call) {
|
|
1241
|
+
const observations = observedNames(call.observations);
|
|
1242
|
+
return {
|
|
1243
|
+
task: call.task ?? false,
|
|
1244
|
+
output: call.output ?? false,
|
|
1245
|
+
files: call.files ?? false,
|
|
1246
|
+
diff: call.diff ?? false,
|
|
1247
|
+
contents: call.contents ?? [],
|
|
1248
|
+
transcript: call.transcript ?? false,
|
|
1249
|
+
turns: call.turns ?? "last",
|
|
1250
|
+
checks: call.checks ?? false,
|
|
1251
|
+
...observationsOn(observations) ? { observations } : {}
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
/** A list of observation names as they're recorded, so a secret in one is masked. */
|
|
1255
|
+
const observedNames = (value) => Array.isArray(value) ? value.map(observedName) : value;
|
|
1256
|
+
/** `observations` when it turns the input on: `true` or a non-empty list. */
|
|
1257
|
+
const observationsOn = (value) => value === true || Array.isArray(value) && value.length > 0;
|
|
1258
|
+
/** `inner` over `outer`, field by field. */
|
|
1259
|
+
function mergeJudge(outer, inner) {
|
|
1260
|
+
return inner ? {
|
|
1261
|
+
...outer,
|
|
1262
|
+
...inner
|
|
1263
|
+
} : outer;
|
|
1264
|
+
}
|
|
1745
1265
|
// src/compare/state.ts
|
|
1746
1266
|
/** The error a per-attempt answer carries when the run stopped before a model answered it. */
|
|
1747
1267
|
const STOPPED_FIRST = "the run stopped before it was answered";
|
|
@@ -1764,6 +1284,7 @@ const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
|
|
|
1764
1284
|
* question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
|
|
1765
1285
|
*/
|
|
1766
1286
|
function resolveQuestionState(state) {
|
|
1287
|
+
const observations = observedNames(state?.observations);
|
|
1767
1288
|
return {
|
|
1768
1289
|
task: state?.task ?? false,
|
|
1769
1290
|
output: state?.output ?? false,
|
|
@@ -1771,7 +1292,8 @@ function resolveQuestionState(state) {
|
|
|
1771
1292
|
diff: state?.diff ?? false,
|
|
1772
1293
|
transcript: state?.transcript ?? false,
|
|
1773
1294
|
turns: state?.turns ?? "all",
|
|
1774
|
-
checks: state?.checks ?? false
|
|
1295
|
+
checks: state?.checks ?? false,
|
|
1296
|
+
...observationsOn(observations) ? { observations } : {}
|
|
1775
1297
|
};
|
|
1776
1298
|
}
|
|
1777
1299
|
/** `inner` over `outer`, field by field. */
|
|
@@ -1792,22 +1314,22 @@ function withQuestionDefaults(q, defaults) {
|
|
|
1792
1314
|
const isOff = (value) => value === false || Array.isArray(value) && !value.length;
|
|
1793
1315
|
/** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1794
1316
|
function questionShows(state, artifacts) {
|
|
1795
|
-
const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
|
|
1317
|
+
const on = [...KEYS.filter((k) => k !== "turns" && !isOff(state[k])), ...state.observations ? ["observations"] : []];
|
|
1796
1318
|
return artifacts === void 0 ? on : [...on, "artifacts"];
|
|
1797
1319
|
}
|
|
1798
1320
|
// src/contract/compare.ts
|
|
1799
1321
|
const compareGateSchema = z.enum([
|
|
1800
1322
|
"variants",
|
|
1801
1323
|
"all",
|
|
1802
|
-
"
|
|
1324
|
+
"observe"
|
|
1803
1325
|
]);
|
|
1804
1326
|
/** The compare object, with environments checked by `environment`. */
|
|
1805
1327
|
const compareSchema = (environment) => z.strictObject({
|
|
1806
|
-
|
|
1328
|
+
control: environment,
|
|
1807
1329
|
variants: z.array(environment).min(1),
|
|
1808
1330
|
gate: compareGateSchema.optional(),
|
|
1809
1331
|
minLift: z.number().min(-1).max(1).optional(),
|
|
1810
|
-
|
|
1332
|
+
harnesses: z.array(z.union([z.string(), harnessDescriptorSchema])).optional()
|
|
1811
1333
|
});
|
|
1812
1334
|
compareSchema(z.custom());
|
|
1813
1335
|
const describe$1 = (value) => {
|
|
@@ -1825,14 +1347,17 @@ function explain(where) {
|
|
|
1825
1347
|
const [key, index] = issue.path;
|
|
1826
1348
|
const v = spec[key];
|
|
1827
1349
|
switch (key) {
|
|
1828
|
-
case "
|
|
1350
|
+
case "control":
|
|
1351
|
+
if (v === void 0) return noControl(where);
|
|
1352
|
+
if (Array.isArray(v)) return `compare.control in ${where} takes one environment (got ${v.map((e) => e?.name ?? describe$1(e)).join(", ")}). For a grid without a compare, use environments.`;
|
|
1353
|
+
return `compare.control in ${where} must be an environment from defineEnvironment (got ${describe$1(v)}). It is what each variant is measured against; variants are not layered on it.`;
|
|
1829
1354
|
case "gate": return `compare.gate in ${where} is ${oneOf(compareGateSchema.options)} (got ${describe$1(v)})`;
|
|
1830
1355
|
case "minLift": return `compare.minLift in ${where} is points from -1 to 1, e.g. minLift: 0.1 (got ${describe$1(v)})`;
|
|
1831
|
-
case "
|
|
1356
|
+
case "harnesses": {
|
|
1832
1357
|
const item = index === void 0 ? void 0 : v[index];
|
|
1833
1358
|
const kind = item?.kind;
|
|
1834
1359
|
const problem = kind && kind in FACTORY ? harnessProblem(item) : void 0;
|
|
1835
|
-
return problem ? `compare.
|
|
1360
|
+
return problem ? `compare.harnesses in ${where}: ${problem}` : `compare.harnesses in ${where} must be a list of harness ids, e.g. harnesses: ['cursor:composer-2.5'] (got ${describe$1(index === void 0 ? v : item)})`;
|
|
1836
1361
|
}
|
|
1837
1362
|
default: return index === void 0 ? `compare has no variants in ${where}. Give at least one environment.` : `compare.variants in ${where} must be environments from defineEnvironment (got ${describe$1(v[index])}). Variants share the eval's prompt and checks; if a variant needs different checks, write a separate eval.`;
|
|
1838
1363
|
}
|
|
@@ -1840,375 +1365,23 @@ function explain(where) {
|
|
|
1840
1365
|
}
|
|
1841
1366
|
/** The order the hand-written checks ran in, so the first refusal stays the same. */
|
|
1842
1367
|
const ORDER = [
|
|
1843
|
-
"
|
|
1368
|
+
"control",
|
|
1844
1369
|
"gate",
|
|
1845
1370
|
"minLift",
|
|
1846
|
-
"
|
|
1371
|
+
"harnesses",
|
|
1847
1372
|
"variants"
|
|
1848
1373
|
];
|
|
1374
|
+
const noControl = (where) => `compare in ${where} has no control. Name it: compare: { control: app, variants: [...] }.`;
|
|
1849
1375
|
/**
|
|
1850
1376
|
* What's wrong with a compare object whose environments `environment` checks, or undefined.
|
|
1851
1377
|
* `where` names the group or eval it's set on.
|
|
1852
1378
|
*/
|
|
1853
1379
|
function compareProblem(value, where, environment) {
|
|
1854
|
-
if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: {
|
|
1855
|
-
if (
|
|
1380
|
+
if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: { control, variants: [...] }.`;
|
|
1381
|
+
if (isObjectLike(value) && "baseline" in value) return `Unknown compare option "baseline" in ${where}. Did you mean "control"?`;
|
|
1382
|
+
if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { control, variants } (got ${describe$1(value)}). Use compare: { control: a, variants: [b, c], gate?, minLift?, harnesses? }.`;
|
|
1856
1383
|
return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
|
|
1857
1384
|
}
|
|
1858
|
-
const POLL_MS = 200;
|
|
1859
|
-
/**
|
|
1860
|
-
* Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
|
|
1861
|
-
* (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
|
|
1862
|
-
*/
|
|
1863
|
-
function probeScript(pid, tables = true) {
|
|
1864
|
-
return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
|
|
1865
|
-
}
|
|
1866
|
-
/** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
|
|
1867
|
-
function procAddress(hex) {
|
|
1868
|
-
const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
|
|
1869
|
-
if (bytes.length === 4) return bytes.join(".");
|
|
1870
|
-
if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
|
|
1871
|
-
const groups = [];
|
|
1872
|
-
for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
|
|
1873
|
-
const full = groups.join(":");
|
|
1874
|
-
const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
|
|
1875
|
-
return longest ? full.replace(longest, "::") : full;
|
|
1876
|
-
}
|
|
1877
|
-
/** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
|
|
1878
|
-
const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
|
|
1879
|
-
function parseProbe(stdout) {
|
|
1880
|
-
const lines = stdout.split("\n");
|
|
1881
|
-
const alive = lines.includes("alive");
|
|
1882
|
-
if (lines.includes("noproc")) return {
|
|
1883
|
-
proc: false,
|
|
1884
|
-
alive
|
|
1885
|
-
};
|
|
1886
|
-
const listeners = [];
|
|
1887
|
-
for (const l of lines) {
|
|
1888
|
-
const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
|
|
1889
|
-
const address = m ? procAddress(m[1]) : "";
|
|
1890
|
-
if (m && reachable(address)) listeners.push({
|
|
1891
|
-
port: Number.parseInt(m[2], 16),
|
|
1892
|
-
address
|
|
1893
|
-
});
|
|
1894
|
-
}
|
|
1895
|
-
return {
|
|
1896
|
-
proc: true,
|
|
1897
|
-
alive,
|
|
1898
|
-
listeners
|
|
1899
|
-
};
|
|
1900
|
-
}
|
|
1901
|
-
/** The ports listening now that weren't before the command started, ascending. */
|
|
1902
|
-
const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
|
|
1903
|
-
/** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
|
|
1904
|
-
const addressFor = (listeners, port) => {
|
|
1905
|
-
const all = listeners.filter((l) => l.port === port).map((l) => l.address);
|
|
1906
|
-
return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
|
|
1907
|
-
};
|
|
1908
|
-
/** `want` is taken whoever listens on it: reeval allocated it, or the eval named it. */
|
|
1909
|
-
function decide(s) {
|
|
1910
|
-
const { listeners, alive } = s.probe;
|
|
1911
|
-
const ports = newPorts(listeners, s.before);
|
|
1912
|
-
if (listeners.some((l) => l.port === s.want)) return {
|
|
1913
|
-
kind: "port",
|
|
1914
|
-
port: s.want,
|
|
1915
|
-
address: addressFor(listeners, s.want)
|
|
1916
|
-
};
|
|
1917
|
-
const late = s.elapsedMs >= s.timeoutMs;
|
|
1918
|
-
if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
|
|
1919
|
-
if (ports.length === 1) {
|
|
1920
|
-
const port = ports[0];
|
|
1921
|
-
return {
|
|
1922
|
-
kind: "port",
|
|
1923
|
-
port,
|
|
1924
|
-
address: addressFor(listeners, port)
|
|
1925
|
-
};
|
|
1926
|
-
}
|
|
1927
|
-
return {
|
|
1928
|
-
kind: "several",
|
|
1929
|
-
ports
|
|
1930
|
-
};
|
|
1931
|
-
}
|
|
1932
|
-
if (!alive) return { kind: "exited" };
|
|
1933
|
-
return late ? {
|
|
1934
|
-
kind: "timeout",
|
|
1935
|
-
ports
|
|
1936
|
-
} : { kind: "wait" };
|
|
1937
|
-
}
|
|
1938
|
-
const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
|
|
1939
|
-
/** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
|
|
1940
|
-
function printedUrls(output, ports) {
|
|
1941
|
-
const found = /* @__PURE__ */ new Map();
|
|
1942
|
-
for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
|
|
1943
|
-
const port = Number(m[1]);
|
|
1944
|
-
if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
|
|
1945
|
-
}
|
|
1946
|
-
return [...found].map(([url, port]) => ({
|
|
1947
|
-
url,
|
|
1948
|
-
port
|
|
1949
|
-
}));
|
|
1950
|
-
}
|
|
1951
|
-
/** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
|
|
1952
|
-
async function hostListening(port) {
|
|
1953
|
-
for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
|
|
1954
|
-
const socket = connect({
|
|
1955
|
-
port,
|
|
1956
|
-
host
|
|
1957
|
-
});
|
|
1958
|
-
socket.once("connect", () => {
|
|
1959
|
-
socket.destroy();
|
|
1960
|
-
resolve(true);
|
|
1961
|
-
});
|
|
1962
|
-
socket.once("error", () => resolve(false));
|
|
1963
|
-
})) return host;
|
|
1964
|
-
}
|
|
1965
|
-
const sleep = (ms, signal) => new Promise((resolve) => {
|
|
1966
|
-
const done = () => {
|
|
1967
|
-
clearTimeout(timer);
|
|
1968
|
-
signal?.removeEventListener("abort", done);
|
|
1969
|
-
resolve();
|
|
1970
|
-
};
|
|
1971
|
-
const timer = setTimeout(done, ms);
|
|
1972
|
-
signal?.addEventListener("abort", done, { once: true });
|
|
1973
|
-
});
|
|
1974
|
-
const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
|
|
1975
|
-
/**
|
|
1976
|
-
* Polls until something listens on `want`, or, without a given port, until the ports that are
|
|
1977
|
-
* new since the command started have settled on one. Throws, with the command's last output,
|
|
1978
|
-
* when they settle on several, it exits, or nothing listens in time.
|
|
1979
|
-
*/
|
|
1980
|
-
async function awaitListening(command, w) {
|
|
1981
|
-
const timeoutMs = w.timeoutMs ?? 12e4;
|
|
1982
|
-
const started = Date.now();
|
|
1983
|
-
let last = "";
|
|
1984
|
-
let changed = started;
|
|
1985
|
-
const output = async () => plain(await w.output().catch(() => ""));
|
|
1986
|
-
const fail = async (why, out) => {
|
|
1987
|
-
const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
|
|
1988
|
-
return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
|
|
1989
|
-
};
|
|
1990
|
-
for (;;) {
|
|
1991
|
-
w.signal?.throwIfAborted();
|
|
1992
|
-
const probe = await w.probe();
|
|
1993
|
-
const now = Date.now();
|
|
1994
|
-
if (!probe.proc) {
|
|
1995
|
-
const address = await (w.hostListening ?? hostListening)(w.want);
|
|
1996
|
-
if (address !== void 0) return {
|
|
1997
|
-
port: w.want,
|
|
1998
|
-
address,
|
|
1999
|
-
detected: false,
|
|
2000
|
-
readyMs: now - started
|
|
2001
|
-
};
|
|
2002
|
-
if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
|
|
2003
|
-
if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
|
|
2004
|
-
} else {
|
|
2005
|
-
const key = newPorts(probe.listeners, w.before).join(",");
|
|
2006
|
-
if (key !== last) {
|
|
2007
|
-
last = key;
|
|
2008
|
-
changed = now;
|
|
2009
|
-
}
|
|
2010
|
-
const d = decide({
|
|
2011
|
-
want: w.want,
|
|
2012
|
-
given: w.given,
|
|
2013
|
-
probe,
|
|
2014
|
-
before: w.before,
|
|
2015
|
-
stableMs: now - changed,
|
|
2016
|
-
elapsedMs: now - started,
|
|
2017
|
-
timeoutMs
|
|
2018
|
-
});
|
|
2019
|
-
if (d.kind === "port") return {
|
|
2020
|
-
port: d.port,
|
|
2021
|
-
address: d.address,
|
|
2022
|
-
detected: d.port !== w.want,
|
|
2023
|
-
readyMs: now - started
|
|
2024
|
-
};
|
|
2025
|
-
if (d.kind === "several") {
|
|
2026
|
-
const out = await output();
|
|
2027
|
-
const urls = printedUrls(out, d.ports);
|
|
2028
|
-
const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
|
|
2029
|
-
const hint = urls[0]?.port ?? d.ports[0];
|
|
2030
|
-
throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
|
|
2031
|
-
}
|
|
2032
|
-
if (d.kind === "exited") throw await fail("exited before it listened on a port");
|
|
2033
|
-
if (d.kind === "timeout") {
|
|
2034
|
-
const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
|
|
2035
|
-
throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
|
|
2036
|
-
}
|
|
2037
|
-
}
|
|
2038
|
-
await sleep(POLL_MS, w.signal);
|
|
2039
|
-
}
|
|
2040
|
-
}
|
|
2041
|
-
// src/sandbox/fake.ts
|
|
2042
|
-
/** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
|
|
2043
|
-
const FAKE_EGRESS = {
|
|
2044
|
-
enforce: "none",
|
|
2045
|
-
record: false
|
|
2046
|
-
};
|
|
2047
|
-
/**
|
|
2048
|
-
* The POSIX shell commands run in: `sh`, which Windows lacks, so there the `sh.exe` on PATH or
|
|
2049
|
-
* Git for Windows'.
|
|
2050
|
-
*/
|
|
2051
|
-
function posixShell(platform = process.platform, env = process.env, exists = existsSync) {
|
|
2052
|
-
if (platform !== "win32") return "sh";
|
|
2053
|
-
const onPath = (env.PATH ?? env.Path ?? "").split(win32.delimiter).filter(Boolean);
|
|
2054
|
-
const git = [
|
|
2055
|
-
env.ProgramFiles,
|
|
2056
|
-
env["ProgramFiles(x86)"],
|
|
2057
|
-
env.LOCALAPPDATA && win32.join(env.LOCALAPPDATA, "Programs")
|
|
2058
|
-
].filter((dir) => !!dir).map((dir) => win32.join(dir, "Git", "bin"));
|
|
2059
|
-
const found = [...onPath, ...git].map((dir) => win32.join(dir, "sh.exe")).find(exists);
|
|
2060
|
-
if (found) return found;
|
|
2061
|
-
throw new UsageError("The fake sandbox runs commands with a POSIX sh, and there's none on this machine. Install Git for Windows (https://git-scm.com/download/win), which brings sh.exe, or run the evals in docker().");
|
|
2062
|
-
}
|
|
2063
|
-
const EXEC_TAG = "REEVAL_FAKE_EXEC";
|
|
2064
|
-
const KILL_TAGGED = `for pass in 1 2 3 4 5; do
|
|
2065
|
-
found=
|
|
2066
|
-
for f in $(grep -lzx "${EXEC_TAG}=$1" /proc/[0-9]*/environ 2>/dev/null); do
|
|
2067
|
-
p=\${f#/proc/}; kill -9 "\${p%/environ}" 2>/dev/null; found=1
|
|
2068
|
-
done
|
|
2069
|
-
[ -n "$found" ] || break
|
|
2070
|
-
done`;
|
|
2071
|
-
/**
|
|
2072
|
-
* Stops a command's MSYS processes on Windows. `taskkill /t` doesn't reach them: Git's `sh.exe`
|
|
2073
|
-
* starts bash, whose fork and exec leave each new process's Windows parent gone, so a timed-out
|
|
2074
|
-
* `sleep 30` would run on, holding the command's pipes and working directory.
|
|
2075
|
-
*/
|
|
2076
|
-
function killTagged(sh, tag) {
|
|
2077
|
-
runSh(sh, KILL_TAGGED, tag);
|
|
2078
|
-
}
|
|
2079
|
-
/**
|
|
2080
|
-
* Kills host processes whose cwd is the sandbox, including native `node.exe` `c.serve` starts.
|
|
2081
|
-
* Match only the unique `reeval-sandbox-*` folder name: Git Bash `pwd` and node.exe's
|
|
2082
|
-
* `/proc/cwd` disagree (`/tmp/...` vs `D:\\Users\\...\\Temp\\...`), and an empty
|
|
2083
|
-
* `"$prefix"/*` is `/*`, which SIGKILLs the Windows runner (exit 2304).
|
|
2084
|
-
*/
|
|
2085
|
-
const KILL_UNDER = `cd "$1" || exit 0
|
|
2086
|
-
base=$(pwd -P 2>/dev/null || pwd)
|
|
2087
|
-
name=\${base##*/}
|
|
2088
|
-
case "$name" in
|
|
2089
|
-
reeval-sandbox-?*) ;;
|
|
2090
|
-
*) exit 0 ;;
|
|
2091
|
-
esac
|
|
2092
|
-
for f in /proc/[0-9]*/cwd; do
|
|
2093
|
-
t=$(readlink "$f" 2>/dev/null) || continue
|
|
2094
|
-
case "$t" in
|
|
2095
|
-
*"$name"*) ;;
|
|
2096
|
-
*) continue ;;
|
|
2097
|
-
esac
|
|
2098
|
-
p=\${f#/proc/}; p=\${p%/cwd}
|
|
2099
|
-
[ -n "$p" ] || continue
|
|
2100
|
-
kill -9 "$p" 2>/dev/null
|
|
2101
|
-
wp=$(tr -d '\\r\\n' < /proc/$p/winpid 2>/dev/null)
|
|
2102
|
-
[ -n "$wp" ] && taskkill /F /PID "$wp" >/dev/null 2>&1
|
|
2103
|
-
done`;
|
|
2104
|
-
function runSh(sh, script, arg) {
|
|
2105
|
-
return new Promise((resolve) => {
|
|
2106
|
-
const child = spawn(sh, [
|
|
2107
|
-
"-c",
|
|
2108
|
-
script,
|
|
2109
|
-
"sh",
|
|
2110
|
-
arg
|
|
2111
|
-
], {
|
|
2112
|
-
stdio: "ignore",
|
|
2113
|
-
windowsHide: true
|
|
2114
|
-
});
|
|
2115
|
-
const done = () => resolve();
|
|
2116
|
-
child.on("error", done);
|
|
2117
|
-
child.on("close", done);
|
|
2118
|
-
});
|
|
2119
|
-
}
|
|
2120
|
-
/** Windows refuses to delete a directory a live process still has as cwd; retry after kills. */
|
|
2121
|
-
async function removeTree(path) {
|
|
2122
|
-
const deadline = Date.now() + 8e3;
|
|
2123
|
-
for (;;) {
|
|
2124
|
-
try {
|
|
2125
|
-
await rm(path, {
|
|
2126
|
-
recursive: true,
|
|
2127
|
-
force: true
|
|
2128
|
-
});
|
|
2129
|
-
if (!existsSync(path)) return;
|
|
2130
|
-
} catch {}
|
|
2131
|
-
if (Date.now() >= deadline) {
|
|
2132
|
-
await rm(path, {
|
|
2133
|
-
recursive: true,
|
|
2134
|
-
force: true
|
|
2135
|
-
}).catch(() => {});
|
|
2136
|
-
return;
|
|
2137
|
-
}
|
|
2138
|
-
await new Promise((r) => setTimeout(r, 50));
|
|
2139
|
-
}
|
|
2140
|
-
}
|
|
2141
|
-
/**
|
|
2142
|
-
* A temp directory on the host. Commands run unisolated on the host shell, so this is
|
|
2143
|
-
* only meant for tests and for the fake harness.
|
|
2144
|
-
*/
|
|
2145
|
-
async function createFakeSandbox(descriptor, env = {}) {
|
|
2146
|
-
const sh = posixShell();
|
|
2147
|
-
const parent = descriptor.dir ?? tmpdir();
|
|
2148
|
-
await mkdir(parent, { recursive: true });
|
|
2149
|
-
const base = await mkdtemp(join(parent, "reeval-sandbox-"));
|
|
2150
|
-
const workdir = join(base, "work");
|
|
2151
|
-
await mkdir(workdir);
|
|
2152
|
-
const stateDir = `${base}.state`;
|
|
2153
|
-
await mkdir(stateDir);
|
|
2154
|
-
let destroyed;
|
|
2155
|
-
const resolve = (path) => path.startsWith("~/") ? join(base, path.slice(2)) : join(workdir, path);
|
|
2156
|
-
return {
|
|
2157
|
-
workdir,
|
|
2158
|
-
stateDir,
|
|
2159
|
-
execOwn: (command, { signal } = {}) => run(sh, ["-c", command], {
|
|
2160
|
-
cwd: workdir,
|
|
2161
|
-
env: {
|
|
2162
|
-
HOME: base,
|
|
2163
|
-
...env
|
|
2164
|
-
},
|
|
2165
|
-
signal
|
|
2166
|
-
}),
|
|
2167
|
-
exec: (command, { cwd, env: extra, signal } = {}) => {
|
|
2168
|
-
const tag = process.platform === "win32" && signal ? randomUUID() : void 0;
|
|
2169
|
-
const stop = () => killTagged(sh, tag);
|
|
2170
|
-
if (tag) signal?.addEventListener("abort", stop, { once: true });
|
|
2171
|
-
return run(sh, ["-c", cwd ? `cd ${shellQuote(cwd)} && ${command}` : command], {
|
|
2172
|
-
cwd: workdir,
|
|
2173
|
-
env: {
|
|
2174
|
-
HOME: base,
|
|
2175
|
-
...env,
|
|
2176
|
-
...extra,
|
|
2177
|
-
...tag ? { [EXEC_TAG]: tag } : {}
|
|
2178
|
-
},
|
|
2179
|
-
signal
|
|
2180
|
-
}).finally(() => signal?.removeEventListener("abort", stop));
|
|
2181
|
-
},
|
|
2182
|
-
home: base,
|
|
2183
|
-
readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
|
|
2184
|
-
getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
|
|
2185
|
-
async writeFile(path, content) {
|
|
2186
|
-
const target = resolve(path);
|
|
2187
|
-
await mkdir(dirname(target), { recursive: true });
|
|
2188
|
-
await writeFile(target, content);
|
|
2189
|
-
},
|
|
2190
|
-
destroy: () => {
|
|
2191
|
-
destroyed ??= (async () => {
|
|
2192
|
-
await rm(stateDir, {
|
|
2193
|
-
recursive: true,
|
|
2194
|
-
force: true
|
|
2195
|
-
}).catch(() => {});
|
|
2196
|
-
if (process.platform === "win32") {
|
|
2197
|
-
if (existsSync(base)) await runSh(sh, KILL_UNDER, base);
|
|
2198
|
-
await removeTree(base);
|
|
2199
|
-
return;
|
|
2200
|
-
}
|
|
2201
|
-
await rm(base, {
|
|
2202
|
-
recursive: true,
|
|
2203
|
-
force: true
|
|
2204
|
-
}).catch(() => {});
|
|
2205
|
-
})().finally(() => {
|
|
2206
|
-
destroyed = void 0;
|
|
2207
|
-
});
|
|
2208
|
-
return destroyed;
|
|
2209
|
-
}
|
|
2210
|
-
};
|
|
2211
|
-
}
|
|
2212
1385
|
// src/ui/mode.ts
|
|
2213
1386
|
const set = (env, name) => {
|
|
2214
1387
|
const value = env[name];
|
|
@@ -2261,16 +1434,16 @@ function resolveOutput(input) {
|
|
|
2261
1434
|
"errors-only",
|
|
2262
1435
|
"none"
|
|
2263
1436
|
]);
|
|
2264
|
-
const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "
|
|
1437
|
+
const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "none";
|
|
2265
1438
|
const color = env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions;
|
|
2266
1439
|
return {
|
|
2267
1440
|
mode,
|
|
2268
1441
|
statusLine,
|
|
1442
|
+
interactive: statusLine && !!input.stdinTTY && outputLogs === "none",
|
|
2269
1443
|
logOrder,
|
|
2270
1444
|
logPrefix,
|
|
2271
1445
|
outputLogs,
|
|
2272
1446
|
color,
|
|
2273
|
-
rich: statusLine && color,
|
|
2274
1447
|
githubActions,
|
|
2275
1448
|
heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
|
|
2276
1449
|
};
|
|
@@ -2281,11 +1454,7 @@ function resolveOutput(input) {
|
|
|
2281
1454
|
* `egress: { allow: [...DEFAULT_EGRESS_ALLOW, 'api.example.com'] }`.
|
|
2282
1455
|
*/
|
|
2283
1456
|
const DEFAULT_EGRESS_ALLOW = PACKAGE_HOSTS;
|
|
2284
|
-
|
|
2285
|
-
const SANDBOX_EGRESS = {
|
|
2286
|
-
docker: DOCKER_EGRESS,
|
|
2287
|
-
fake: FAKE_EGRESS
|
|
2288
|
-
};
|
|
1457
|
+
Object.fromEntries(Object.entries(SANDBOX_KINDS).map(([kind, k]) => [kind, k.capabilities.egress]));
|
|
2289
1458
|
/** The policy an attempt runs under when nothing sets one. */
|
|
2290
1459
|
const DEFAULT_EGRESS = "proxy";
|
|
2291
1460
|
/** Whether the policy allows only some hosts: `deny-all` and allowlists. */
|
|
@@ -2368,13 +1537,16 @@ const LEVELS = [
|
|
|
2368
1537
|
];
|
|
2369
1538
|
/** Why the sandbox can't enforce the attempt's policy, or undefined when it can. */
|
|
2370
1539
|
function unenforceable(planned, sandbox, where) {
|
|
1540
|
+
const capabilities = capabilitiesOf(sandbox);
|
|
1541
|
+
const can = capabilities.egress.enforce;
|
|
1542
|
+
if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under egress 'proxy', where the egress gateway is the only way out, but the ${sandbox.kind} sandbox enforces no egress policy: its traffic that ignores the proxy goes out directly. Set egress: 'allow-all' to run it there, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.enforce !== "none")}.`;
|
|
2371
1543
|
const needs = egressNeeds(planned);
|
|
2372
|
-
const can = SANDBOX_EGRESS[sandbox.kind].enforce;
|
|
2373
1544
|
if (LEVELS.indexOf(can) >= LEVELS.indexOf(needs)) return void 0;
|
|
2374
1545
|
const hosts = allowedHosts(planned)?.join(", ");
|
|
2375
1546
|
const what = needs !== "allowlist" ? "turning egress off" : planned.policy === "deny-all" ? `a host allowlist for its automatic hosts (${hosts})` : `a host allowlist (${hosts})`;
|
|
2376
1547
|
const enforces = can === "none" ? "enforces no egress policy" : `only enforces ${can === "all-or-nothing" ? "'allow-all' or 'deny-all'" : can}`;
|
|
2377
|
-
const
|
|
1548
|
+
const enforcing = suggestedSandbox((c) => LEVELS.indexOf(c.egress.enforce) >= LEVELS.indexOf(needs));
|
|
1549
|
+
const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"} egress: 'allow-all'`;
|
|
2378
1550
|
return `${where} runs under egress ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
|
|
2379
1551
|
}
|
|
2380
1552
|
const describeEgress = (policy) => typeof policy === "string" ? `'${policy}'` : `{ allow: [${policy.allow.map((h) => `'${h}'`).join(", ")}] }`;
|
|
@@ -2398,34 +1570,12 @@ function egressLabel({ policy, auto }) {
|
|
|
2398
1570
|
const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
|
|
2399
1571
|
return auto.length ? `${own} + auto: ${auto.join(", ")}` : own;
|
|
2400
1572
|
}
|
|
2401
|
-
// src/
|
|
2402
|
-
/**
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
*/
|
|
2406
|
-
function resolveStateOptions(call) {
|
|
2407
|
-
return {
|
|
2408
|
-
task: call.task ?? false,
|
|
2409
|
-
output: call.output ?? false,
|
|
2410
|
-
files: call.files ?? false,
|
|
2411
|
-
diff: call.diff ?? false,
|
|
2412
|
-
contents: call.contents ?? [],
|
|
2413
|
-
transcript: call.transcript ?? false,
|
|
2414
|
-
turns: call.turns ?? "last",
|
|
2415
|
-
checks: call.checks ?? false
|
|
2416
|
-
};
|
|
2417
|
-
}
|
|
2418
|
-
/** `inner` over `outer`, field by field. */
|
|
2419
|
-
function mergeJudge(outer, inner) {
|
|
2420
|
-
return inner ? {
|
|
2421
|
-
...outer,
|
|
2422
|
-
...inner
|
|
2423
|
-
} : outer;
|
|
2424
|
-
}
|
|
2425
|
-
const KNOWN = [...[
|
|
2426
|
-
"harness",
|
|
1573
|
+
// src/options.ts
|
|
1574
|
+
/** Every eval/group option key `checkOptions` accepts. The feature matrix holds each to an inventory entry. */
|
|
1575
|
+
const EVAL_OPTION_KEYS = [
|
|
1576
|
+
"harnesses",
|
|
2427
1577
|
"sandbox",
|
|
2428
|
-
"
|
|
1578
|
+
"environments",
|
|
2429
1579
|
"compare",
|
|
2430
1580
|
"runs",
|
|
2431
1581
|
"gate",
|
|
@@ -2440,12 +1590,14 @@ const KNOWN = [...[
|
|
|
2440
1590
|
"tags",
|
|
2441
1591
|
"revision",
|
|
2442
1592
|
"pin"
|
|
2443
|
-
]
|
|
1593
|
+
];
|
|
1594
|
+
const KNOWN = [...EVAL_OPTION_KEYS];
|
|
2444
1595
|
const SUGGEST = {
|
|
2445
|
-
|
|
1596
|
+
harness: "harnesses",
|
|
2446
1597
|
sandboxes: "sandbox",
|
|
2447
|
-
|
|
2448
|
-
baseline: "
|
|
1598
|
+
environment: "environments",
|
|
1599
|
+
baseline: "compare",
|
|
1600
|
+
control: "compare",
|
|
2449
1601
|
variants: "compare",
|
|
2450
1602
|
repeat: "runs",
|
|
2451
1603
|
repeats: "runs",
|
|
@@ -2480,47 +1632,27 @@ function mergeSnapshot(outer, inner) {
|
|
|
2480
1632
|
compress: inner.compress ?? outer.compress
|
|
2481
1633
|
};
|
|
2482
1634
|
}
|
|
2483
|
-
/** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
|
|
1635
|
+
/** Throws unless `value` is `'all'`, `'observe'` or `{ passRate }` with a rate in (0, 1]. */
|
|
2484
1636
|
function checkGate(value, where) {
|
|
2485
1637
|
const problem = gateProblem(value, where);
|
|
2486
1638
|
if (problem) throw new UsageError(problem);
|
|
2487
1639
|
}
|
|
2488
1640
|
const PINNABLE = [
|
|
2489
|
-
"
|
|
1641
|
+
"harnesses",
|
|
2490
1642
|
"runs",
|
|
2491
1643
|
"compare"
|
|
2492
1644
|
];
|
|
2493
1645
|
const ID = /^[A-Za-z0-9._:@+\-=[\]]+$/;
|
|
2494
|
-
const sanitize = (text) => (text.split("/").at(-1) ?? text).replace(/[^A-Za-z0-9._:@-]+/g, "-");
|
|
2495
1646
|
function harnessId(h) {
|
|
2496
1647
|
if (h.id) return h.id;
|
|
2497
1648
|
const effort = harnessEffort(h);
|
|
2498
|
-
const suffix = effort ? `[effort=${
|
|
1649
|
+
const suffix = effort ? `[effort=${idPart(effort)}]` : "";
|
|
2499
1650
|
if (h.kind === "fake") return `fake${suffix}`;
|
|
2500
|
-
return `${h.model ? `${h.kind}:${
|
|
2501
|
-
}
|
|
2502
|
-
function installId(install) {
|
|
2503
|
-
if (!install?.length) return "";
|
|
2504
|
-
const key = JSON.stringify(install.map((step) => typeof step === "string" ? step : {
|
|
2505
|
-
run: step.run,
|
|
2506
|
-
timeout: step.timeout
|
|
2507
|
-
}));
|
|
2508
|
-
return `+${createHash("sha256").update(key).digest("hex").slice(0, 8)}`;
|
|
1651
|
+
return `${h.model ? `${h.kind}:${idPart(h.model)}` : h.kind}${suffix}`;
|
|
2509
1652
|
}
|
|
2510
1653
|
/** The provider a sandbox runs on: its kind, or `podman` for a Docker sandbox on Podman. */
|
|
2511
|
-
const sandboxProvider = (s) => s.
|
|
2512
|
-
|
|
2513
|
-
if (s.id) return s.id;
|
|
2514
|
-
if (s.kind === "fake") return "fake";
|
|
2515
|
-
const install = s.kind === "docker" ? installId(s.install) : "";
|
|
2516
|
-
const engine = s.engine ?? "docker";
|
|
2517
|
-
if (s.dockerfile) {
|
|
2518
|
-
const [file = "", dir = ""] = s.dockerfile.split(/[\\/]/).reverse();
|
|
2519
|
-
return `${engine}:${sanitize(file === "Dockerfile" ? dir || file : file)}${install}`;
|
|
2520
|
-
}
|
|
2521
|
-
const image = s.image ?? s.from;
|
|
2522
|
-
return image ? `${engine}:${sanitize(image)}${install}` : `${engine}${install}`;
|
|
2523
|
-
}
|
|
1654
|
+
const sandboxProvider = (s) => sandboxKind(s).provider(s);
|
|
1655
|
+
const sandboxId = (s) => s.id ?? sandboxKind(s).id(s);
|
|
2524
1656
|
/** The planned attempts' sandboxes by id, once each, or `fallback`'s when none is planned. */
|
|
2525
1657
|
function plannedSandboxIds(planned, fallback) {
|
|
2526
1658
|
const ids = new Set(planned.map(sandboxId));
|
|
@@ -2542,11 +1674,14 @@ const describe = (value) => {
|
|
|
2542
1674
|
}
|
|
2543
1675
|
return JSON.stringify(value) ?? String(value);
|
|
2544
1676
|
};
|
|
2545
|
-
/** `compare: false` and `
|
|
1677
|
+
/** `compare: false` and `environments` beside `compare` are never legal options. */
|
|
2546
1678
|
function refuseIllegalCompare(options, where) {
|
|
2547
|
-
if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the
|
|
2548
|
-
|
|
2549
|
-
|
|
1679
|
+
if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the control and strips the variants. On a group or eval, omit compare or pass compare: { control, variants }.`);
|
|
1680
|
+
if (options.environments !== void 0 && options.compare !== void 0) {
|
|
1681
|
+
const first = Array.isArray(options.environments) ? options.environments[0] : options.environments;
|
|
1682
|
+
const name = isObjectLike(first) && typeof first.name === "string" ? first.name : "app";
|
|
1683
|
+
throw new UsageError(`${where} sets both environments and compare. With compare, the control is compare.control: move it there, as compare: { control: ${name}, variants: [...] }.`);
|
|
1684
|
+
}
|
|
2550
1685
|
}
|
|
2551
1686
|
const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
|
|
2552
1687
|
/** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
|
|
@@ -2577,7 +1712,7 @@ function checkOptions(options, where, allowed = KNOWN) {
|
|
|
2577
1712
|
if (allowed.includes(key)) continue;
|
|
2578
1713
|
const near = SUGGEST[key] ?? closest(key, allowed);
|
|
2579
1714
|
const value = options[key];
|
|
2580
|
-
const hint = key === "
|
|
1715
|
+
const hint = key === "cases" || key === "caseKey" ? ` ${key} belongs to one eval: evaluate(name, { cases: [...] }, async (c, item) => { … }).` : key === "where" ? " where is an option of evaluate() only, beside cases: evaluate(name, { cases: [...], where: ({ case, environment }) => … }, …)." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
|
|
2581
1716
|
throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
|
|
2582
1717
|
}
|
|
2583
1718
|
const o = options;
|
|
@@ -2595,32 +1730,26 @@ function checkOptions(options, where, allowed = KNOWN) {
|
|
|
2595
1730
|
});
|
|
2596
1731
|
if (judgeIssue) throw new UsageError(judgeIssue);
|
|
2597
1732
|
if (o.sandbox !== void 0) {
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
if (max !== void 0 && !(Number.isInteger(max) && max >= 1)) throw new UsageError(`sandbox.maxConcurrent in ${where} must be a whole number of at least 1 (got ${String(max)})`);
|
|
1733
|
+
if (!isSandboxKind(o.sandbox.kind)) throw new UsageError(`sandbox in ${where} must come from ${sandboxHelpers()} (got ${describe(o.sandbox)})`);
|
|
1734
|
+
const limit = maxConcurrentProblem(o.sandbox, where);
|
|
1735
|
+
if (limit) throw new UsageError(limit);
|
|
2602
1736
|
const problem = sandboxProblem(o.sandbox);
|
|
2603
1737
|
if (problem) throw new UsageError(`sandbox in ${where}: ${problem}`);
|
|
2604
1738
|
}
|
|
2605
|
-
if (o.
|
|
2606
|
-
if (Array.isArray(o.environment)) throw new UsageError(`environment takes one environment, not a list (${where}). To compare environments, use compare: { baseline, variants: [a, b] }. baseline is the control; variants are not layered on it.`);
|
|
2607
|
-
if (!isEnvironment(o.environment)) throw new UsageError(`environment in ${where} must come from defineEnvironment(…) (got ${describe(o.environment)})`);
|
|
2608
|
-
const problem = environmentDataProblem(o.environment, "environment", where);
|
|
2609
|
-
if (problem) throw new UsageError(problem);
|
|
2610
|
-
}
|
|
1739
|
+
if (o.environments !== void 0) checkEnvironments(o.environments, where);
|
|
2611
1740
|
refuseIllegalCompare(options, where);
|
|
2612
1741
|
if (o.compare !== void 0) {
|
|
2613
1742
|
const problem = compareProblem(o.compare, where, z.custom(isEnvironment));
|
|
2614
1743
|
if (problem) throw new UsageError(problem);
|
|
2615
|
-
const {
|
|
2616
|
-
for (const [label, env] of [["compare.
|
|
1744
|
+
const { control, variants } = o.compare;
|
|
1745
|
+
for (const [label, env] of [["compare.control", control], ...variants.map((v, i) => [`compare.variants[${i}]`, v])]) {
|
|
2617
1746
|
const bad = env && environmentDataProblem(env, label, where);
|
|
2618
1747
|
if (bad) throw new UsageError(bad);
|
|
2619
1748
|
}
|
|
2620
1749
|
}
|
|
2621
1750
|
if (o.pin !== void 0) {
|
|
2622
1751
|
const bad = Array.isArray(o.pin) ? o.pin.filter((k) => !PINNABLE.includes(k)) : [o.pin];
|
|
2623
|
-
if (bad.length) throw new UsageError(`pin in ${where} lists
|
|
1752
|
+
if (bad.length) throw new UsageError(`pin in ${where} lists harnesses, runs and/or compare (got ${bad.map((k) => JSON.stringify(k)).join(", ")})`);
|
|
2624
1753
|
}
|
|
2625
1754
|
if (o.capture !== void 0) checkCapture(o.capture, where);
|
|
2626
1755
|
const questionIssue = o.question !== void 0 && questionOptionsProblem(o.question, where);
|
|
@@ -2670,42 +1799,82 @@ function checkCaptureModel(value, where) {
|
|
|
2670
1799
|
if (typeof value !== "string" || !value.includes("/")) throw new UsageError(`capture.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(value)})`);
|
|
2671
1800
|
if (isEvaluationModel(value)) throw new UsageError(`capture.model "${value}" in ${where} is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
|
|
2672
1801
|
}
|
|
1802
|
+
/** Each environment `environments` names, as one, a list or an edit, comes from defineEnvironment. */
|
|
1803
|
+
function checkEnvironments(value, where) {
|
|
1804
|
+
if (typeof value === "function") return;
|
|
1805
|
+
let list;
|
|
1806
|
+
let label = "environments";
|
|
1807
|
+
if (isEnvironmentEdit(value)) {
|
|
1808
|
+
if ("add" in value && "only" in value) throw new UsageError(`environments in ${where} takes { add } or { only }, not both. Use add to append to the inherited environments, only to keep some of them.`);
|
|
1809
|
+
if ("only" in value) {
|
|
1810
|
+
const names = [value.only].flat();
|
|
1811
|
+
if (!names.length || names.some((n) => typeof n !== "string")) throw new UsageError(`environments { only } in ${where} lists environment names, e.g. { only: ['app'] } (got ${describe(value.only)})`);
|
|
1812
|
+
return;
|
|
1813
|
+
}
|
|
1814
|
+
list = [value.add].flat();
|
|
1815
|
+
label = "environments.add";
|
|
1816
|
+
} else list = [value].flat();
|
|
1817
|
+
if (list.length === 0) throw new UsageError(`${label} is an empty list in ${where}. Give at least one, or leave environments out.`);
|
|
1818
|
+
list.forEach((env, i) => {
|
|
1819
|
+
const at = Array.isArray(value) || label !== "environments" ? `${label}[${i}]` : label;
|
|
1820
|
+
if (!isEnvironment(env)) throw new UsageError(`${at} in ${where} must come from defineEnvironment(…) (got ${describe(env)})`);
|
|
1821
|
+
const problem = environmentDataProblem(env, at, where);
|
|
1822
|
+
if (problem) throw new UsageError(problem);
|
|
1823
|
+
});
|
|
1824
|
+
}
|
|
1825
|
+
const isEnvironmentEdit = (v) => isObjectLike(v) && !Array.isArray(v) && !isEnvironment(v) && ("add" in v || "only" in v);
|
|
1826
|
+
function applyEnvironments(inherited, value, where) {
|
|
1827
|
+
if (value === void 0) return inherited;
|
|
1828
|
+
const list = [typeof value === "function" ? value(inherited) : isEnvironmentEdit(value) ? editEnvironments(inherited, value, where) : value].flat();
|
|
1829
|
+
if (list.length === 0) throw new UsageError(`environments is an empty list in ${where}. Give at least one, or leave environments out.`);
|
|
1830
|
+
if (typeof value === "function") checkEnvironments(list, where);
|
|
1831
|
+
return list;
|
|
1832
|
+
}
|
|
1833
|
+
/** `{ add }` appends to the inherited environments; `{ only }` keeps those with these names. */
|
|
1834
|
+
function editEnvironments(inherited, edit, where) {
|
|
1835
|
+
if ("add" in edit) return [...inherited, ...[edit.add].flat()];
|
|
1836
|
+
const names = [edit.only].flat();
|
|
1837
|
+
const have = inherited.map((e) => e.name);
|
|
1838
|
+
const missing = names.filter((n) => !have.includes(n));
|
|
1839
|
+
if (missing.length) throw new UsageError(`environments { only: ${missing.map((n) => JSON.stringify(n)).join(", ")} } in ${where} names no inherited environment (inherited: ${have.join(", ") || "none"})`);
|
|
1840
|
+
return inherited.filter((e) => names.includes(e.name));
|
|
1841
|
+
}
|
|
2673
1842
|
function applyHarness(inherited, value, where) {
|
|
2674
1843
|
if (value === void 0) return inherited;
|
|
2675
1844
|
const next = typeof value === "function" ? value(inherited) : isHarnessEdit(value) ? editHarness(inherited, value, where) : value;
|
|
2676
1845
|
const list = Array.isArray(next) ? next : [next];
|
|
2677
|
-
if (list.length === 0) throw new UsageError(`
|
|
1846
|
+
if (list.length === 0) throw new UsageError(`harnesses is an empty list in ${where}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
|
|
2678
1847
|
checkHarnesses(list, where);
|
|
2679
1848
|
return list;
|
|
2680
1849
|
}
|
|
2681
1850
|
const isHarnessEdit = (v) => typeof v === "object" && v !== null && !Array.isArray(v) && !("kind" in v) && ("add" in v || "only" in v);
|
|
2682
1851
|
/** `{ add }` appends to the inherited harnesses; `{ only }` keeps those with these ids. */
|
|
2683
1852
|
function editHarness(inherited, edit, where) {
|
|
2684
|
-
if ("add" in edit && "only" in edit) throw new UsageError(`
|
|
1853
|
+
if ("add" in edit && "only" in edit) throw new UsageError(`harnesses in ${where} takes { add } or { only }, not both. Use add to append to the inherited harnesses, only to keep some of them.`);
|
|
2685
1854
|
if ("add" in edit) return [...inherited, ...Array.isArray(edit.add) ? edit.add : [edit.add]];
|
|
2686
1855
|
const ids = Array.isArray(edit.only) ? edit.only : [edit.only];
|
|
2687
1856
|
const have = inherited.map(harnessId);
|
|
2688
1857
|
const missing = ids.filter((id) => typeof id !== "string" || !have.includes(id));
|
|
2689
|
-
if (missing.length) throw new UsageError(`
|
|
1858
|
+
if (missing.length) throw new UsageError(`harnesses { only: ${missing.map((id) => JSON.stringify(id)).join(", ")} } in ${where} names no inherited harness (inherited: ${have.join(", ")})`);
|
|
2690
1859
|
return inherited.filter((h) => ids.includes(harnessId(h)));
|
|
2691
1860
|
}
|
|
2692
1861
|
/** Each harness is one, with a valid `maxConcurrent`. */
|
|
2693
1862
|
function checkHarnesses(list, where) {
|
|
2694
1863
|
for (const item of list) {
|
|
2695
|
-
if (!isHarness(item)) throw new UsageError(`
|
|
1864
|
+
if (!isHarness(item)) throw new UsageError(`harnesses in ${where} must be like ${HARNESS_EXAMPLE} (got ${describe(item)})`);
|
|
2696
1865
|
if (item.kind === "opencode") {
|
|
2697
1866
|
const why = opencodeModelError(item.model);
|
|
2698
1867
|
if (why) throw new UsageError(`opencode() in ${where}: ${why}`);
|
|
2699
1868
|
}
|
|
2700
|
-
checkMaxConcurrent(item.maxConcurrent, `
|
|
1869
|
+
checkMaxConcurrent(item.maxConcurrent, `harnesses.maxConcurrent in ${where}`);
|
|
2701
1870
|
const { startTimeout } = item;
|
|
2702
1871
|
if (startTimeout !== void 0) try {
|
|
2703
1872
|
parseDuration(startTimeout);
|
|
2704
1873
|
} catch (error) {
|
|
2705
|
-
throw new UsageError(`
|
|
1874
|
+
throw new UsageError(`harnesses.startTimeout in ${where}: ${error.message}`);
|
|
2706
1875
|
}
|
|
2707
1876
|
const problem = harnessProblem(item);
|
|
2708
|
-
if (problem) throw new UsageError(`
|
|
1877
|
+
if (problem) throw new UsageError(`harnesses in ${where}: ${problem}`);
|
|
2709
1878
|
}
|
|
2710
1879
|
}
|
|
2711
1880
|
function checkUnique(list, where) {
|
|
@@ -2732,31 +1901,27 @@ const plannedQuestions = (value) => value.map((q, i) => {
|
|
|
2732
1901
|
});
|
|
2733
1902
|
/** Whether the eval's matrix varies each factor a pairwise question can vary. */
|
|
2734
1903
|
const variesBy = (r) => ({
|
|
2735
|
-
environment:
|
|
1904
|
+
environment: r.compare ? r.compare.variants.length > 0 : r.environments.length > 1,
|
|
2736
1905
|
harness: r.harness.length > 1
|
|
2737
1906
|
});
|
|
2738
1907
|
const FACTOR_FIX = {
|
|
2739
|
-
environment: (_, where) => `varies the environment (across: 'environment'), but ${where} runs one environment, so it has no pairs to ask.
|
|
2740
|
-
harness: (r, where) => `varies the harness (across: 'harness'), but ${where} runs one harness (${r.harness.map(harnessId).join(", ")}), so it has no pairs to ask. Run it on two or more,
|
|
1908
|
+
environment: (_, where) => `varies the environment (across: 'environment'), but ${where} runs one environment, so it has no pairs to ask. Run it in two or more, environments: [a, b], or compare: { control, variants: [...] }`,
|
|
1909
|
+
harness: (r, where) => `varies the harness (across: 'harness'), but ${where} runs one harness (${r.harness.map(harnessId).join(", ")}), so it has no pairs to ask. Run it on two or more, harnesses: [a, b]`
|
|
2741
1910
|
};
|
|
2742
1911
|
function normalizeCompare(value) {
|
|
2743
1912
|
if (value === void 0) return void 0;
|
|
2744
1913
|
return {
|
|
2745
|
-
|
|
2746
|
-
baseline: emptyEnvironment(EMPTY_BASELINE),
|
|
2747
|
-
from: "empty"
|
|
2748
|
-
},
|
|
1914
|
+
control: value.control,
|
|
2749
1915
|
variants: value.variants,
|
|
2750
1916
|
gate: value.gate ?? "variants",
|
|
2751
1917
|
...value.minLift !== void 0 ? { minLift: value.minLift } : {},
|
|
2752
|
-
...value.
|
|
1918
|
+
...value.harnesses ? { harnesses: value.harnesses.map((h) => typeof h === "string" ? h : harnessId(h)) } : {},
|
|
2753
1919
|
of: value.variants.length
|
|
2754
1920
|
};
|
|
2755
1921
|
}
|
|
2756
1922
|
/**
|
|
2757
|
-
* Runs per cell when no level sets `runs`: a compare with variants runs `
|
|
2758
|
-
*
|
|
2759
|
-
* included, runs once.
|
|
1923
|
+
* Runs per cell when no level sets `runs`: a compare with variants runs `COMPARE_RUNS` a side;
|
|
1924
|
+
* anything else, a pooled family's cases included, runs once.
|
|
2760
1925
|
*/
|
|
2761
1926
|
const COMPARE_DEFAULT_RUNS = 4;
|
|
2762
1927
|
function applyLevel(out, options, where, egress = "own") {
|
|
@@ -2765,9 +1930,9 @@ function applyLevel(out, options, where, egress = "own") {
|
|
|
2765
1930
|
[key]: where
|
|
2766
1931
|
};
|
|
2767
1932
|
if (options.egress !== void 0) out.egress[egress] = options.egress;
|
|
2768
|
-
out.harness = applyHarness(out.harness, options.
|
|
1933
|
+
out.harness = applyHarness(out.harness, options.harnesses, where);
|
|
2769
1934
|
if (options.sandbox !== void 0) out.sandbox = options.sandbox;
|
|
2770
|
-
|
|
1935
|
+
out.environments = applyEnvironments(out.environments, options.environments, where);
|
|
2771
1936
|
if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
|
|
2772
1937
|
if (options.runs !== void 0) {
|
|
2773
1938
|
const { runs, bestOf } = runsOf(options.runs);
|
|
@@ -2808,6 +1973,7 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2808
1973
|
capture: { ...base.capture },
|
|
2809
1974
|
question: { ...base.question },
|
|
2810
1975
|
questions: [...base.questions],
|
|
1976
|
+
environments: [...base.environments],
|
|
2811
1977
|
tags: [...base.tags],
|
|
2812
1978
|
pin: [...base.pin]
|
|
2813
1979
|
};
|
|
@@ -2817,28 +1983,33 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2817
1983
|
...profile.defaults,
|
|
2818
1984
|
judge: profileJudge(profile)
|
|
2819
1985
|
}, `profiles.${profile.name}.defaults`, "inherited");
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
}
|
|
1986
|
+
let compareAt;
|
|
1987
|
+
for (const { options: own, where: at } of levels) {
|
|
1988
|
+
if (compareAt && own.environments !== void 0) throw new UsageError(`${at} sets environments, but ${compareAt} sets compare, which runs its control in their place. Set compare: { control, variants: [...] } on ${at} instead, or move ${at} out of ${compareAt}.`);
|
|
1989
|
+
applyLevel(out, own, at);
|
|
1990
|
+
if (own.compare !== void 0) compareAt = at;
|
|
1991
|
+
}
|
|
2826
1992
|
if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
|
|
2827
1993
|
checkUnique(out.harness, where);
|
|
2828
|
-
|
|
1994
|
+
if (!out.compare) {
|
|
1995
|
+
const names = out.environments.map((e) => e.name);
|
|
1996
|
+
const dupe = names.find((n, i) => names.indexOf(n) !== i);
|
|
1997
|
+
if (dupe) throw new UsageError(`environments in ${where} lists two environments named ${dupe}. Give each its own name.`);
|
|
1998
|
+
}
|
|
1999
|
+
const harnessFromProfile = profile?.defaults.harnesses !== void 0 && !levels.some((l) => l.options.harnesses);
|
|
2829
2000
|
const authored = variesBy(out);
|
|
2830
2001
|
if (harnessFromProfile) authored.harness ||= base.harness.length > 1;
|
|
2831
2002
|
for (const q of options.skipped ? [] : out.questions) for (const factor of pairFactors(q.across)) {
|
|
2832
2003
|
if (authored[factor]) continue;
|
|
2833
2004
|
throw new UsageError(`questions "${q.question}" in ${where} ${FACTOR_FIX[factor](out, where)}, or leave out across to ask each attempt on its own.`);
|
|
2834
2005
|
}
|
|
2835
|
-
if (profile?.defaults.
|
|
2006
|
+
if (profile?.defaults.harnesses !== void 0 && levels.some((l) => l.options.harnesses)) notes.push("harnesses set by the eval");
|
|
2836
2007
|
const ids = out.harness.map(harnessId);
|
|
2837
|
-
const missing = (out.compare?.
|
|
2838
|
-
if (missing.length) throw new UsageError(`compare.
|
|
2008
|
+
const missing = (out.compare?.harnesses)?.filter((id) => !ids.includes(id)) ?? [];
|
|
2009
|
+
if (missing.length) throw new UsageError(`compare.harnesses lists ${missing.join(", ")}, which ${where} doesn't run (it runs ${ids.join(", ")}).`);
|
|
2839
2010
|
for (const v of out.compare?.variants ?? []) {
|
|
2840
|
-
if (v.name === out.compare?.
|
|
2841
|
-
const control = out.compare?.
|
|
2011
|
+
if (v.name === out.compare?.control.name) throw new UsageError(`compare in ${where} lists ${v.name}, which is the control. List only the other environments.`);
|
|
2012
|
+
const control = out.compare?.control;
|
|
2842
2013
|
const shared = control && sameWorkspaceOnly(control, v);
|
|
2843
2014
|
if (control && shared) {
|
|
2844
2015
|
const at = shared.ref ? `at ref "${shared.ref}"` : "with no ref (the repository HEAD)";
|
|
@@ -2855,13 +2026,13 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2855
2026
|
const tag = `profile ${profile.name}`;
|
|
2856
2027
|
const pinned = (key) => out.pin.includes(key);
|
|
2857
2028
|
let comparedLeft = true;
|
|
2858
|
-
if (limit.
|
|
2859
|
-
const kept = out.harness.filter((h) => limit.
|
|
2860
|
-
if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.
|
|
2029
|
+
if (limit.harnesses && !pinned("harnesses")) {
|
|
2030
|
+
const kept = out.harness.filter((h) => limit.harnesses?.includes(harnessId(h)));
|
|
2031
|
+
if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harnesses.join(", ") || "none"}`;
|
|
2861
2032
|
else if (kept.length < out.harness.length) {
|
|
2862
2033
|
notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
|
|
2863
2034
|
out.harness = kept;
|
|
2864
|
-
const scope = out.compare?.
|
|
2035
|
+
const scope = out.compare?.harnesses;
|
|
2865
2036
|
if (out.compare?.variants.length && scope && !kept.some((h) => scope.includes(harnessId(h)))) {
|
|
2866
2037
|
notes.push(`compare off: no compared harness left (${tag})`);
|
|
2867
2038
|
out.compare = {
|
|
@@ -2927,6 +2098,8 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2927
2098
|
const set = from === where ? "" : ` from ${from}`;
|
|
2928
2099
|
throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
|
|
2929
2100
|
}
|
|
2101
|
+
if (out.gate === "observe" && out.compare?.minLift !== void 0 && out.compare.variants.length) throw new UsageError(`${where} has gate: 'observe', so its compare can't take minLift: ${out.compare.minLift}: minLift gates the lift, and nothing in it gates. Drop minLift, or gate the eval.`);
|
|
2102
|
+
if (out.gate === "observe" && out.bestOf !== void 0) throw new UsageError(`${where} has gate: 'observe', so it can't take runs: { bestOf: ${out.bestOf} }: best-of decides each cell by its majority. Set a fixed count, e.g. runs: 3.`);
|
|
2930
2103
|
if (out.bestOf !== void 0) {
|
|
2931
2104
|
if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
|
|
2932
2105
|
out.gate = MAJORITY_GATE;
|
|
@@ -2957,14 +2130,14 @@ function defaultRuns(out, notes, options) {
|
|
|
2957
2130
|
notes.push(`runs ${COMPARE_DEFAULT_RUNS} (compare default)`);
|
|
2958
2131
|
}
|
|
2959
2132
|
/**
|
|
2960
|
-
* Refuses a profile's `limit.
|
|
2133
|
+
* Refuses a profile's `limit.harnesses` id that no collected eval runs on: it keeps nothing, and a
|
|
2961
2134
|
* limit that keeps nothing skips every eval. `harnesses` are the ids every eval in the loaded
|
|
2962
2135
|
* files runs on before the limit, whatever the tag and name filters pick. When a path filter
|
|
2963
2136
|
* loaded only some files (`partial`), an id may name a harness of the others, so only a limit
|
|
2964
2137
|
* none of whose ids match is refused.
|
|
2965
2138
|
*/
|
|
2966
2139
|
function checkLimitHarness(profile, configFile, collected) {
|
|
2967
|
-
const ids = profile?.limit.
|
|
2140
|
+
const ids = profile?.limit.harnesses;
|
|
2968
2141
|
if (!profile || !ids?.length) return;
|
|
2969
2142
|
const unknown = ids.filter((id) => !collected.harnesses.includes(id));
|
|
2970
2143
|
if (!unknown.length || collected.partial && unknown.length < ids.length) return;
|
|
@@ -2972,71 +2145,59 @@ function checkLimitHarness(profile, configFile, collected) {
|
|
|
2972
2145
|
const have = [...collected.harnesses].sort();
|
|
2973
2146
|
const evals = collected.partial ? "eval in the selected files" : "eval";
|
|
2974
2147
|
const the = collected.partial ? "the evals in the selected files" : "the evals";
|
|
2975
|
-
throw new UsageError(`profiles.${profile.name}.limit.
|
|
2148
|
+
throw new UsageError(`profiles.${profile.name}.limit.harnesses in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
|
|
2976
2149
|
}
|
|
2977
2150
|
/**
|
|
2978
2151
|
* The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
|
|
2979
|
-
* parent's), else the
|
|
2980
|
-
*
|
|
2981
|
-
*
|
|
2152
|
+
* parent's), else the cascaded option (`defineConfig` / profile defaults / group / eval), else
|
|
2153
|
+
* implicit Docker. A named environment is a total override: the config environment's sandbox is
|
|
2154
|
+
* not inherited.
|
|
2982
2155
|
*/
|
|
2983
|
-
const attemptSandbox = (resolved, env) =>
|
|
2984
|
-
const own = env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox;
|
|
2985
|
-
if (own) return own;
|
|
2986
|
-
const variant = resolved.compare?.from === "empty" && env.name === "baseline" ? resolved.compare.variants[0] : void 0;
|
|
2987
|
-
return sandboxOf(variant?.layers ?? [], resolved.sandbox);
|
|
2988
|
-
};
|
|
2156
|
+
const attemptSandbox = (resolved, env) => env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox ?? sandboxOf([], resolved.sandbox);
|
|
2989
2157
|
/**
|
|
2990
2158
|
* Which level set this attempt's sandbox: the cascade source of the environment that named one
|
|
2991
|
-
* (an `.extend()` child still names that environment), else the
|
|
2992
|
-
*
|
|
2993
|
-
* Docker).
|
|
2159
|
+
* (an `.extend()` child still names that environment), else the cascaded `sandbox` option, else
|
|
2160
|
+
* the config (implicit Docker).
|
|
2994
2161
|
*/
|
|
2995
2162
|
function sandboxSetBy(planned, resolved, configFile) {
|
|
2996
|
-
if (!planned.environment.layers.some((l) => l.spec.sandbox))
|
|
2997
|
-
|
|
2998
|
-
return resolved.sources?.sandbox ?? configFile;
|
|
2999
|
-
}
|
|
3000
|
-
if (resolved.compare) {
|
|
3001
|
-
if (planned.cell.role === "control") {
|
|
3002
|
-
if (resolved.compare.from === "environment") return resolved.sources?.environment ?? configFile;
|
|
3003
|
-
if (resolved.compare.from === "empty") return configFile;
|
|
3004
|
-
return resolved.sources?.compare ?? configFile;
|
|
3005
|
-
}
|
|
3006
|
-
return resolved.sources?.compare ?? configFile;
|
|
3007
|
-
}
|
|
3008
|
-
return resolved.sources?.environment ?? configFile;
|
|
2163
|
+
if (!planned.environment.layers.some((l) => l.spec.sandbox)) return resolved.sources?.sandbox ?? configFile;
|
|
2164
|
+
return (resolved.compare ? resolved.sources?.compare : resolved.sources?.environments) ?? configFile;
|
|
3009
2165
|
}
|
|
3010
2166
|
/**
|
|
3011
2167
|
* Per harness: the control, then each variant it compares on; then run numbers. `vertex` is the
|
|
3012
2168
|
* run's `vertex()` settings, for each harness that resolved to Vertex AI.
|
|
3013
2169
|
*/
|
|
2170
|
+
const cellTraits = (env) => {
|
|
2171
|
+
const traits = traitsOf(env.layers);
|
|
2172
|
+
return Object.keys(traits).length ? { traits } : {};
|
|
2173
|
+
};
|
|
3014
2174
|
function expand(resolved, vertex) {
|
|
3015
2175
|
const { compare } = resolved;
|
|
3016
|
-
const
|
|
3017
|
-
const policy = resolved.gate ?? "all";
|
|
2176
|
+
const controls = (compare ? [compare.control] : resolved.environments.length ? resolved.environments : [void 0]).map((e) => applied(e));
|
|
2177
|
+
const policy = resolved.gate === "observe" ? "none" : resolved.gate ?? "all";
|
|
3018
2178
|
const gateIf = (gated) => gated ? policy : "none";
|
|
3019
2179
|
const attempts = [];
|
|
3020
2180
|
for (const harness of resolved.harness) {
|
|
3021
2181
|
const hid = harnessId(harness);
|
|
3022
2182
|
const onVertex = harness.gateway === "vertex" && harness.kind !== "fake" ? vertex : void 0;
|
|
3023
|
-
const cells =
|
|
3024
|
-
const sandbox = attemptSandbox(resolved,
|
|
2183
|
+
const cells = controls.map((env) => {
|
|
2184
|
+
const sandbox = attemptSandbox(resolved, env);
|
|
3025
2185
|
const sid = sandboxId(sandbox);
|
|
3026
2186
|
return {
|
|
3027
|
-
env
|
|
2187
|
+
env,
|
|
3028
2188
|
sandbox,
|
|
3029
2189
|
cell: {
|
|
3030
|
-
key: cellKey(hid,
|
|
2190
|
+
key: cellKey(hid, env.name, sid),
|
|
3031
2191
|
harness: hid,
|
|
3032
|
-
environment:
|
|
2192
|
+
environment: env.name,
|
|
3033
2193
|
sandbox: sid,
|
|
3034
|
-
role: "control"
|
|
2194
|
+
role: "control",
|
|
2195
|
+
...cellTraits(env)
|
|
3035
2196
|
},
|
|
3036
2197
|
gate: gateIf(!compare || compare.variants.length === 0 || compare.gate === "all")
|
|
3037
2198
|
};
|
|
3038
|
-
})
|
|
3039
|
-
if (compare && (!compare.
|
|
2199
|
+
});
|
|
2200
|
+
if (compare && (!compare.harnesses || compare.harnesses.includes(hid))) for (const v of compare.variants) {
|
|
3040
2201
|
const env = applied(v);
|
|
3041
2202
|
const sandbox = attemptSandbox(resolved, env);
|
|
3042
2203
|
const sid = sandboxId(sandbox);
|
|
@@ -3049,13 +2210,14 @@ function expand(resolved, vertex) {
|
|
|
3049
2210
|
environment: env.name,
|
|
3050
2211
|
sandbox: sid,
|
|
3051
2212
|
role: "variant",
|
|
3052
|
-
variant: v.name
|
|
2213
|
+
variant: v.name,
|
|
2214
|
+
...cellTraits(env)
|
|
3053
2215
|
},
|
|
3054
|
-
gate: gateIf(compare.gate !== "
|
|
2216
|
+
gate: gateIf(compare.gate !== "observe")
|
|
3055
2217
|
});
|
|
3056
2218
|
}
|
|
3057
2219
|
for (const { env, sandbox, cell, gate } of cells) {
|
|
3058
|
-
const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : ""].filter(Boolean);
|
|
2220
|
+
const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : controls.length > 1 ? env.name : ""].filter(Boolean);
|
|
3059
2221
|
for (let n = 1; n <= resolved.runs; n++) {
|
|
3060
2222
|
const label = [axes.join(" · "), resolved.runs > 1 ? `#${n}` : ""].filter(Boolean).join(" ");
|
|
3061
2223
|
attempts.push({
|
|
@@ -3092,14 +2254,16 @@ function environmentWarnings(attempts) {
|
|
|
3092
2254
|
}
|
|
3093
2255
|
if (s.source === "registry" && !s.version) out.add(`skill.registry(${s.id}) has no version: results aren't comparable over time. Pin one (the resolved commit is recorded either way).`);
|
|
3094
2256
|
}
|
|
2257
|
+
const tag = a.sandbox.kind === "tart" ? tartTagWarning(a.sandbox.image) : null;
|
|
2258
|
+
if (tag) out.add(tag);
|
|
3095
2259
|
}
|
|
3096
2260
|
return [...out];
|
|
3097
2261
|
}
|
|
3098
|
-
/** Secrets
|
|
2262
|
+
/** Secrets an isolated sandbox's attempt would get as real values, because they have no `hosts`. */
|
|
3099
2263
|
function unbrokered(env, attempts) {
|
|
3100
2264
|
const out = /* @__PURE__ */ new Map();
|
|
3101
2265
|
for (const a of attempts) {
|
|
3102
|
-
if (a.sandbox.
|
|
2266
|
+
if (!capabilitiesOf(a.sandbox).isolated) continue;
|
|
3103
2267
|
for (const s of unbrokeredSecrets(env, a.environment.layers)) out.set(s.label, s);
|
|
3104
2268
|
}
|
|
3105
2269
|
return [...out.values()];
|
|
@@ -3108,12 +2272,21 @@ function unbrokered(env, attempts) {
|
|
|
3108
2272
|
const reachesSandbox = (s) => !s.optional || Boolean(process.env[s.from]);
|
|
3109
2273
|
/** `GEMINI_API_KEY → generativelanguage.googleapis.com` for each secret with `hosts`. */
|
|
3110
2274
|
const withHosts = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).flatMap((s) => s.hosts ? [`${s.label} → ${s.hosts.join(", ")}`] : []))];
|
|
3111
|
-
/** The secrets an attempt brokers: only
|
|
3112
|
-
const brokeredIn = (env, attempt) => attempt.sandbox.
|
|
3113
|
-
/** The plan's note for secrets with `hosts` on
|
|
3114
|
-
function
|
|
3115
|
-
const
|
|
3116
|
-
|
|
2275
|
+
/** The secrets an attempt brokers: only a sandbox that brokers swaps a placeholder for its value. */
|
|
2276
|
+
const brokeredIn = (env, attempt) => capabilitiesOf(attempt.sandbox).brokers ? withHosts(env, attempt) : [];
|
|
2277
|
+
/** The plan's note for secrets with `hosts` on a sandbox that can't broker them, which only ever has their placeholder. */
|
|
2278
|
+
function unbrokeredHere(env, attempts) {
|
|
2279
|
+
const byProvider = /* @__PURE__ */ new Map();
|
|
2280
|
+
for (const a of attempts) {
|
|
2281
|
+
if (capabilitiesOf(a.sandbox).brokers) continue;
|
|
2282
|
+
const names = withHosts(env, a);
|
|
2283
|
+
if (!names.length) continue;
|
|
2284
|
+
const provider = sandboxProvider(a.sandbox);
|
|
2285
|
+
const seen = byProvider.get(provider) ?? /* @__PURE__ */ new Set();
|
|
2286
|
+
for (const name of names) seen.add(name);
|
|
2287
|
+
byProvider.set(provider, seen);
|
|
2288
|
+
}
|
|
2289
|
+
return [...byProvider].map(([provider, names]) => `not brokered (${provider} sandbox, placeholder only): ${[...names].join("; ")}`);
|
|
3117
2290
|
}
|
|
3118
2291
|
/** The host variables an attempt's secrets read, brokered or not. */
|
|
3119
2292
|
const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
|
|
@@ -3138,11 +2311,12 @@ function sandboxWarnings(attempts, resolved, origin) {
|
|
|
3138
2311
|
const out = [];
|
|
3139
2312
|
const seen = /* @__PURE__ */ new Set();
|
|
3140
2313
|
for (const a of attempts) {
|
|
3141
|
-
|
|
2314
|
+
const privileges = sandboxKind(a.sandbox).privileges(a.sandbox);
|
|
2315
|
+
if (!privileges.root && !privileges.capabilities.length) continue;
|
|
3142
2316
|
const where = sandboxSetBy(a, resolved, origin.configFile);
|
|
3143
2317
|
const level = levelOf(where);
|
|
3144
2318
|
const file = level === "eval" || level === "group" ? origin.file : null;
|
|
3145
|
-
if (
|
|
2319
|
+
if (privileges.root) {
|
|
3146
2320
|
const message = `running as root in the sandbox: ${where} (docker({ root: true }))`;
|
|
3147
2321
|
const key = `${file}\0${message}`;
|
|
3148
2322
|
if (!seen.has(key)) {
|
|
@@ -3156,8 +2330,8 @@ function sandboxWarnings(attempts, resolved, origin) {
|
|
|
3156
2330
|
});
|
|
3157
2331
|
}
|
|
3158
2332
|
}
|
|
3159
|
-
if (
|
|
3160
|
-
const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${
|
|
2333
|
+
if (privileges.capabilities.length) {
|
|
2334
|
+
const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${privileges.capabilities.map((c) => `'${c}'`).join(", ")}] }))`;
|
|
3161
2335
|
const key = `${file}\0${message}`;
|
|
3162
2336
|
if (!seen.has(key)) {
|
|
3163
2337
|
seen.add(key);
|
|
@@ -3166,7 +2340,7 @@ function sandboxWarnings(attempts, resolved, origin) {
|
|
|
3166
2340
|
level,
|
|
3167
2341
|
where,
|
|
3168
2342
|
file,
|
|
3169
|
-
capabilities: [...
|
|
2343
|
+
capabilities: [...privileges.capabilities],
|
|
3170
2344
|
message
|
|
3171
2345
|
});
|
|
3172
2346
|
}
|
|
@@ -3181,7 +2355,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
3181
2355
|
const notes = [
|
|
3182
2356
|
...r.notes,
|
|
3183
2357
|
...capture ? [capture] : [],
|
|
3184
|
-
...
|
|
2358
|
+
...unbrokeredHere(resolved.env, attempts)
|
|
3185
2359
|
];
|
|
3186
2360
|
return {
|
|
3187
2361
|
harnesses: skip ? 0 : resolved.harness.length,
|
|
@@ -3189,10 +2363,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
3189
2363
|
run: resolved.compare.variants.length,
|
|
3190
2364
|
of: resolved.compare.of
|
|
3191
2365
|
} : null,
|
|
3192
|
-
...resolved.compare ? { compare: {
|
|
3193
|
-
baseline: resolved.compare.baseline.name,
|
|
3194
|
-
...resolved.compare.from ? { from: resolved.compare.from } : {}
|
|
3195
|
-
} } : {},
|
|
2366
|
+
...resolved.compare ? { compare: { baseline: resolved.compare.control.name } } : {},
|
|
3196
2367
|
...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
|
|
3197
2368
|
runs: resolved.runs,
|
|
3198
2369
|
...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
|
|
@@ -3257,7 +2428,11 @@ const GATEWAYS = {
|
|
|
3257
2428
|
vertex: { cantServe: (h) => {
|
|
3258
2429
|
if (h.kind === "fake") return void 0;
|
|
3259
2430
|
if (!VERTEX_ROUTES[h.kind]) return VERTEX_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vertex AI yet`;
|
|
3260
|
-
|
|
2431
|
+
const effort = harnessEffort(h);
|
|
2432
|
+
const refused = vertexModelRefusalFor(h);
|
|
2433
|
+
if (refused) return refused;
|
|
2434
|
+
if (h.kind === "claude-code" && effort !== void 0) return claudeCodeTakesEffort(h.model, "vertex") ? void 0 : `Claude Code sends no effort to ${claudeCodeModelName(h.model, "vertex")} on Vertex AI, where it sends one only to the models its catalog lists: drop effort, or run one of them, e.g. claude-sonnet-5-5`;
|
|
2435
|
+
return h.kind === "opencode" ? openCodeEffortOn(h.model, "vertex", effort) : void 0;
|
|
3261
2436
|
} }
|
|
3262
2437
|
};
|
|
3263
2438
|
/** OpenCode's id for `model` on `gateway`, as that gateway's route writes it. */
|
|
@@ -3423,9 +2598,9 @@ function modelGatewayOrNone(model, pin, scope) {
|
|
|
3423
2598
|
const CONFIG_KEYS = [
|
|
3424
2599
|
"include",
|
|
3425
2600
|
"gateway",
|
|
3426
|
-
"
|
|
2601
|
+
"harnesses",
|
|
3427
2602
|
"sandbox",
|
|
3428
|
-
"
|
|
2603
|
+
"environments",
|
|
3429
2604
|
"runs",
|
|
3430
2605
|
"gate",
|
|
3431
2606
|
"env",
|
|
@@ -3455,9 +2630,9 @@ const PROFILE_KEYS = [
|
|
|
3455
2630
|
"gateway"
|
|
3456
2631
|
];
|
|
3457
2632
|
const DEFAULTS_KEYS = [
|
|
3458
|
-
"
|
|
2633
|
+
"harnesses",
|
|
3459
2634
|
"sandbox",
|
|
3460
|
-
"
|
|
2635
|
+
"environments",
|
|
3461
2636
|
"runs",
|
|
3462
2637
|
"gate",
|
|
3463
2638
|
"env",
|
|
@@ -3471,7 +2646,8 @@ const DEFAULTS_KEYS = [
|
|
|
3471
2646
|
const LIMIT_KEYS = [
|
|
3472
2647
|
"runs",
|
|
3473
2648
|
"compare",
|
|
3474
|
-
"
|
|
2649
|
+
"harnesses",
|
|
2650
|
+
"traits"
|
|
3475
2651
|
];
|
|
3476
2652
|
/**
|
|
3477
2653
|
* The config: a plain object, or a callback that destructures the helpers it needs, e.g.
|
|
@@ -3565,7 +2741,7 @@ function parseYamlConfig(text, file) {
|
|
|
3565
2741
|
return doc.toJS() ?? {};
|
|
3566
2742
|
}
|
|
3567
2743
|
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3568
|
-
/** A config's Dockerfile that isn't there, named by its field. */
|
|
2744
|
+
/** A config's host file (a Dockerfile, an SSH key) that isn't there, named by its field. */
|
|
3569
2745
|
const missingIn = (file) => (field, full) => {
|
|
3570
2746
|
throw new UsageError(`${field} in ${file}: ${full} doesn't exist. Paths in ${file} are relative to its directory.`);
|
|
3571
2747
|
};
|
|
@@ -3581,7 +2757,10 @@ function withHostPaths(config, root, file, { environments = false } = {}) {
|
|
|
3581
2757
|
if (!isRecord(value)) return value;
|
|
3582
2758
|
const out = { ...value };
|
|
3583
2759
|
if ("sandbox" in value) out.sandbox = sandboxPaths(value.sandbox, root, `${prefix}sandbox`, missing);
|
|
3584
|
-
if (environments && "
|
|
2760
|
+
if (environments && "environments" in value) {
|
|
2761
|
+
const field = `${prefix}environments`;
|
|
2762
|
+
out.environments = Array.isArray(value.environments) ? value.environments.map((e, i) => environmentPaths(e, root, `${field}[${i}]`, missing)) : environmentPaths(value.environments, root, field, missing);
|
|
2763
|
+
}
|
|
3585
2764
|
return out;
|
|
3586
2765
|
};
|
|
3587
2766
|
const out = options(config, "");
|
|
@@ -3600,10 +2779,15 @@ function withHostPaths(config, root, file, { environments = false } = {}) {
|
|
|
3600
2779
|
function configFromData(data) {
|
|
3601
2780
|
const withEnvironment = (options) => {
|
|
3602
2781
|
if (!isObjectLike(options)) return options;
|
|
3603
|
-
const {
|
|
3604
|
-
|
|
2782
|
+
const { environments } = options;
|
|
2783
|
+
const from = (e) => isEnvironment(e) ? environmentFrom(e) : e;
|
|
2784
|
+
if (Array.isArray(environments)) return {
|
|
2785
|
+
...options,
|
|
2786
|
+
environments: environments.map(from)
|
|
2787
|
+
};
|
|
2788
|
+
return isEnvironment(environments) ? {
|
|
3605
2789
|
...options,
|
|
3606
|
-
|
|
2790
|
+
environments: from(environments)
|
|
3607
2791
|
} : options;
|
|
3608
2792
|
};
|
|
3609
2793
|
const config = withEnvironment(data);
|
|
@@ -3629,10 +2813,10 @@ function configFromData(data) {
|
|
|
3629
2813
|
* let through, such as a `null` or a misspelled nested key.
|
|
3630
2814
|
*/
|
|
3631
2815
|
function resolveConfigData(input, root, file) {
|
|
3632
|
-
if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like
|
|
2816
|
+
if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harnesses: and judge: (got ${JSON.stringify(input)})`);
|
|
3633
2817
|
const data = withHostPaths(input, root, file, { environments: true });
|
|
3634
|
-
const { environment, profiles } = data;
|
|
3635
|
-
if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.
|
|
2818
|
+
const { environments: environment, profiles } = data;
|
|
2819
|
+
if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.environments)].flat().some((e) => e !== void 0 && !isEnvironment(e))) {
|
|
3636
2820
|
const problem = configDataProblem(data, file);
|
|
3637
2821
|
if (problem) throw new UsageError(problem);
|
|
3638
2822
|
}
|
|
@@ -3712,12 +2896,18 @@ function checkProfile(name, profile, file) {
|
|
|
3712
2896
|
}
|
|
3713
2897
|
if (p.limit !== void 0) {
|
|
3714
2898
|
unknownKey(p.limit, LIMIT_KEYS, `${where}.limit`, file);
|
|
3715
|
-
const { runs, compare, harness } = p.limit;
|
|
2899
|
+
const { runs, compare, harnesses: harness, traits } = p.limit;
|
|
3716
2900
|
if (runs !== void 0 && !(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`${where}.limit.runs in ${file} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
|
|
3717
2901
|
if (compare !== void 0 && compare !== false && !(Number.isInteger(compare?.maxVariants) && compare.maxVariants >= 0)) throw new UsageError(`${where}.limit.compare in ${file} must be false or { maxVariants: n }, e.g. { maxVariants: 1 } (got ${JSON.stringify(compare)})`);
|
|
3718
2902
|
if (harness !== void 0) {
|
|
3719
2903
|
const bad = Array.isArray(harness) ? harness.find((h) => typeof h !== "string") : harness;
|
|
3720
|
-
if (bad !== void 0) throw new UsageError(`${where}.limit.
|
|
2904
|
+
if (bad !== void 0) throw new UsageError(`${where}.limit.harnesses in ${file} can only list harness ids to allow (got ${describeValue(bad)}). To add harnesses, use ${where}.defaults.harnesses.`);
|
|
2905
|
+
}
|
|
2906
|
+
if (traits !== void 0) {
|
|
2907
|
+
const ok = (v) => typeof v === "string" || Array.isArray(v) && v.length > 0 && v.every((s) => typeof s === "string");
|
|
2908
|
+
const entries = typeof traits === "object" && traits !== null && !Array.isArray(traits) ? Object.entries(traits) : null;
|
|
2909
|
+
const bad = entries?.find(([, v]) => !ok(v));
|
|
2910
|
+
if (!entries || bad) throw new UsageError(`${where}.limit.traits in ${file} must map each trait key to a value or a list of values a cell's environment has, e.g. { lang: 'ts' } or { lang: ['ts', 'py'] } (got ${bad ? `${bad[0]}: ${JSON.stringify(bad[1])}` : JSON.stringify(traits)})`);
|
|
3721
2911
|
}
|
|
3722
2912
|
}
|
|
3723
2913
|
if (p.tags !== void 0 && (!Array.isArray(p.tags) || p.tags.some((t) => typeof t !== "string"))) throw new UsageError(`${where}.tags in ${file} must be a list of tag filters, e.g. tags: ['smoke'] (got ${JSON.stringify(p.tags)})`);
|
|
@@ -3831,8 +3021,8 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3831
3021
|
const artifacts = user.history?.artifacts ?? "all";
|
|
3832
3022
|
if (!ARTIFACTS.includes(artifacts)) throw new UsageError(`history.artifacts in ${file} is ${oneOf(ARTIFACTS)} (got ${JSON.stringify(artifacts)})`);
|
|
3833
3023
|
const historyDir = resolve(root, user.history?.dir ?? join(evalRoot(include), ".history"));
|
|
3834
|
-
const harness = user.
|
|
3835
|
-
if (harness.length === 0) throw new UsageError(`
|
|
3024
|
+
const harness = user.harnesses === void 0 ? [DEFAULT_HARNESS] : [user.harnesses].flat();
|
|
3025
|
+
if (harness.length === 0) throw new UsageError(`harnesses is an empty list in ${file}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
|
|
3836
3026
|
checkHarnesses(harness, file);
|
|
3837
3027
|
const profiles = {};
|
|
3838
3028
|
if (user.profiles !== void 0) {
|
|
@@ -3855,7 +3045,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3855
3045
|
include,
|
|
3856
3046
|
harness,
|
|
3857
3047
|
sandbox: user.sandbox ?? docker(),
|
|
3858
|
-
|
|
3048
|
+
environments: user.environments === void 0 ? [] : [user.environments].flat(),
|
|
3859
3049
|
runs: user.runs,
|
|
3860
3050
|
gate: user.gate,
|
|
3861
3051
|
env: user.env ?? {},
|
|
@@ -3886,7 +3076,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3886
3076
|
}
|
|
3887
3077
|
const ANALYSIS_DEFAULTS = {
|
|
3888
3078
|
maxInputTokens: 2e4,
|
|
3889
|
-
maxOutputTokens:
|
|
3079
|
+
maxOutputTokens: 32e3
|
|
3890
3080
|
};
|
|
3891
3081
|
function checkAnalysis(analysis, file) {
|
|
3892
3082
|
if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
|
|
@@ -3997,6 +3187,11 @@ function runModelGateway(config, profile, model, judge = false) {
|
|
|
3997
3187
|
*/
|
|
3998
3188
|
function missingCredentials(config, env = process.env, options = {}) {
|
|
3999
3189
|
const missing = [];
|
|
3190
|
+
for (const s of options.sandboxes ?? []) {
|
|
3191
|
+
if (s.kind !== "boat" || env[s.apiKey.from]) continue;
|
|
3192
|
+
const line = `${s.apiKey.from} (for boat())`;
|
|
3193
|
+
if (!missing.includes(line)) missing.push(line);
|
|
3194
|
+
}
|
|
4000
3195
|
const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
|
|
4001
3196
|
for (const key of keys) if (!env[key]) missing.push(key);
|
|
4002
3197
|
const judge = runJudgeModel(config, options.profile);
|
|
@@ -4054,6 +3249,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
|
|
|
4054
3249
|
return at ? [`${name} (character ${at})`] : [];
|
|
4055
3250
|
});
|
|
4056
3251
|
}
|
|
4057
|
-
export {
|
|
3252
|
+
export { runsOf as $, brokeredIn as A, execOptionsProblem as At, expand as B, decideState as Bt, openCodeModelOn as C, resolveStateOptions as Ct, EVAL_OPTION_KEYS as D, CAPTURE_NAME as Dt, COMPARE_DEFAULT_RUNS as E, observer as Et, checkHarnesses as F, NO_JUDGE_INPUT as Ft, mergeSnapshot as G, levelOf as H, resolveJudgeArtifacts as Ht, checkLimitHarness as I, inputsOn as It, plainHeaders as J, plainHeaderError as K, checkOptions as L, judgeProblem as Lt, cellKey as M, resolvedEffortOf as Mt, checkCapture as N, servedEffortOf as Nt, MAJORITY_GATE as O, CAPTURE_NAME_RULE as Ot, checkGate as P, INPUT_KEYS as Pt, refuseIllegalCompare as Q, environmentWarnings as R, artifactRefsOf as Rt, fallbackHint as S, mergeJudge as St, routedHarness as T, observedName as Tt, majorityOf as U, harnessId as V, filePartsOf as Vt, matchesTags as W, plannedSandboxes as X, plannedSandboxIds as Y, profileJudge as Z, fellBackToDirect as _, resolveOutput as _t, configPathFor as a, unbrokered as at, resolveModelGateway as b, questionShows as bt, findConfig as c, DEFAULT_EGRESS as ct, malformedCredentials as d, ciRunner as dt, sandboxId as et, missingCredentials as f, directOnCloud as ft, selectProfile as g, unenforceable as gt, runJudgeModel as h, isDirect as ht, configLabel as i, secretVarsIn as it, cascade as j, harnessEffort as jt, attemptSandbox as k, attemptRequestProblem as kt, hostCapacity as l, DEFAULT_EGRESS_ALLOW as lt, resolveConcurrency as m, gatewayPolicy as mt, analysisModelOf as n, sandboxSetBy as nt, defaultConcurrency as o, unbrokeredError as ot, requireConfig as p, egressLabel as pt, plainHeaderWarning as q, configFilesPhrase as r, sandboxWarnings as rt, defineConfig as s, unbrokeredWarning as st, CONFIG_FILES as t, sandboxProvider as tt, loadConfig as u, allowedHosts as ut, gatewayScope as v, STOPPED_FIRST as vt, resolveGateway as w, observedHere as wt, effortRefusedOn as x, resolveQuestionState as xt, modelGatewayOrNone as y, UNRECORDED_QUESTION_STATE as yt, evalPlan as z, badJudgeArtifacts as zt };
|
|
4058
3253
|
|
|
4059
|
-
//# sourceMappingURL=config-
|
|
3254
|
+
//# sourceMappingURL=config-B9mB1kF5.js.map
|