reeval 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +803 -0
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C8xPuXYq.js → adapter-BdzhdRYo.js} +8 -6
- package/dist/adapter-BdzhdRYo.js.map +1 -0
- package/dist/{agent-CBoLndM7.js → agent-Dx-bDmlV.js} +179 -31
- package/dist/agent-Dx-bDmlV.js.map +1 -0
- package/dist/{agents-C6zhkxp3.js → agents-l0-k7vw3.js} +362 -102
- package/dist/agents-l0-k7vw3.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
- package/dist/app-files-B-CQYB0c.js.map +1 -0
- package/dist/{assemble-UPCRrtOz.js → assemble-CYMpN8LC.js} +1 -1
- package/dist/{assemble-t6h018An.js → assemble-CsZIcJo8.js} +171 -18
- package/dist/assemble-CsZIcJo8.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
- package/dist/{attempt-host-CIm1-11m.js → attempt-host-BwSmNYkX.js} +17 -8
- package/dist/attempt-host-BwSmNYkX.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
- package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
- package/dist/{boat-CJCtzfB1.js → boat-su1Y7LPd.js} +17 -14
- package/dist/boat-su1Y7LPd.js.map +1 -0
- package/dist/bridge/index.mjs +113 -8
- package/dist/{capture-iBzFiC0j.js → capture-BqIAeFyC.js} +55 -29
- package/dist/{capture-iBzFiC0j.js.map → capture-BqIAeFyC.js.map} +1 -1
- package/dist/capture-D0Ln5aCP.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-20Fpvs5H.js → cli-s_3opn_I.js} +219 -109
- package/dist/cli-s_3opn_I.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
- package/dist/cli-spec-DxzIjmq_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-C5fIzHQa.js +803 -0
- package/dist/client-C5fIzHQa.js.map +1 -0
- package/dist/{command-CRh-cb6Z.js → command-B5OAIjp7.js} +25 -21
- package/dist/command-B5OAIjp7.js.map +1 -0
- package/dist/{command-BLNVDiIg.js → command-BJ7KqczP.js} +10 -10
- package/dist/{command-BLNVDiIg.js.map → command-BJ7KqczP.js.map} +1 -1
- package/dist/{command-aUJvENwy.js → command-BXLZsEwQ.js} +16 -16
- package/dist/{command-aUJvENwy.js.map → command-BXLZsEwQ.js.map} +1 -1
- package/dist/{command-B5OyHpOu.js → command-DA_SHQUQ.js} +16 -16
- package/dist/{command-B5OyHpOu.js.map → command-DA_SHQUQ.js.map} +1 -1
- package/dist/{commands-CWy-WndG.js → commands-BKBamOcX.js} +174 -40
- package/dist/commands-BKBamOcX.js.map +1 -0
- package/dist/{commands-CfdBX-mc.js → commands-BPvAOEmb.js} +1 -1
- package/dist/{config-B9mB1kF5.js → config-C0URar0t.js} +255 -89
- package/dist/config-C0URar0t.js.map +1 -0
- package/dist/{context-Bkv3ndjq.js → context-Sj58aJEu.js} +2 -2
- package/dist/{context-Bkv3ndjq.js.map → context-Sj58aJEu.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-BSn9xjmo.js → cursor-catalog-FWgSo-T1.js} +4 -4
- package/dist/{cursor-catalog-BSn9xjmo.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
- package/dist/db-Bc4L-Wcr.js +2 -0
- package/dist/{db-zgC1tvKB.js → db-BjAWxVAG.js} +142 -248
- package/dist/db-BjAWxVAG.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BAy8ixd1.js → default-effort-CWlhZ6U1.js} +1 -1
- package/dist/{default-effort-DGhQmu95.js → default-effort-Ck2GYIxX.js} +4 -4
- package/dist/{default-effort-DGhQmu95.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
- package/dist/{dist-CAFD1yfv.js → dist-DeqeCUZe.js} +15 -8
- package/dist/dist-DeqeCUZe.js.map +1 -0
- package/dist/{docker-DAMnm3L9.js → docker-CDdK6Ity.js} +49 -22
- package/dist/docker-CDdK6Ity.js.map +1 -0
- package/dist/{docker-Cqw-CyWz.js → docker-D74l67k8.js} +4 -4
- package/dist/{docker-Cqw-CyWz.js.map → docker-D74l67k8.js.map} +1 -1
- package/dist/{docker-Dr4XeDc8.js → docker-DMug3l-a.js} +1 -1
- package/dist/{doctor-B2PohneB.js → doctor-eFxyn6at.js} +85 -46
- package/dist/doctor-eFxyn6at.js.map +1 -0
- package/dist/{environment-DB8o3hGi.js → environment-CJIKz-Ud.js} +49 -11
- package/dist/environment-CJIKz-Ud.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-D3sbAzt-.js → eval-source-CXIRJ91v.js} +24 -12
- package/dist/eval-source-CXIRJ91v.js.map +1 -0
- package/dist/{evaluate-CwKeN_fo.js → evaluate-DA4QsrHO.js} +792 -281
- package/dist/evaluate-DA4QsrHO.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
- package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
- package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
- package/dist/{files-BiFU16BW.js → files-BVVWAS0A.js} +6 -6
- package/dist/{files-BiFU16BW.js.map → files-BVVWAS0A.js.map} +1 -1
- package/dist/files-BaqXvB88.js +1 -0
- package/dist/{files-BEwfjD8J.js → files-Bey1VAex.js} +3 -3
- package/dist/{files-BEwfjD8J.js.map → files-Bey1VAex.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/files-CNHhSyb3.js +1 -0
- package/dist/files-CW1lChNh.js +1 -0
- package/dist/{files-BRzV7aPh.js → files-D6j-XpaK.js} +5 -5
- package/dist/{files-BRzV7aPh.js.map → files-D6j-XpaK.js.map} +1 -1
- package/dist/{files-Cl7BQ5gE.js → files-F44NRCN0.js} +143 -20
- package/dist/files-F44NRCN0.js.map +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-fDiJ8UIV.js → git-Jqkf9SBa.js} +3 -3
- package/dist/{git-fDiJ8UIV.js.map → git-Jqkf9SBa.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-BNgcR77K.js → images-Dsc4Wi-F.js} +65 -30
- package/dist/images-Dsc4Wi-F.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/init-C0oUG3jh.js +1 -0
- package/dist/{init-DbuZVRrW.js → init-QB5ecPDA.js} +33 -14
- package/dist/init-QB5ecPDA.js.map +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
- package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
- package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
- package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
- package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
- package/dist/layout-DfQa4mJs.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
- package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
- package/dist/live-DwjWJTdn.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
- package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
- package/dist/{live-CnGrRM6j.js → live-y1p3oNXH.js} +9 -9
- package/dist/{live-CnGrRM6j.js.map → live-y1p3oNXH.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
- package/dist/model-3pZZzloU.js.map +1 -0
- package/dist/{model-047Dcd-a.js → model-DEjwetkJ.js} +1 -1
- package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{model-DJjLw-0r.js → model-jRKSQBg6.js} +5 -5
- package/dist/{model-DJjLw-0r.js.map → model-jRKSQBg6.js.map} +1 -1
- package/dist/model-nfewGTzB.js +1 -0
- package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
- package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
- package/dist/options-DpsJ_5CU.js +1 -0
- package/dist/{patch-DH4e_Gkd.js → patch-Bx-qp8Li.js} +2 -2
- package/dist/{patch-DH4e_Gkd.js.map → patch-Bx-qp8Li.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-Cgq9NCEX.js → plan-ChO0A-xn.js} +63 -12
- package/dist/plan-ChO0A-xn.js.map +1 -0
- package/dist/{plugins-BkmPxDiY.js → plugins-BioMWncI.js} +636 -9
- package/dist/plugins-BioMWncI.js.map +1 -0
- package/dist/{preflight-R-InZCQb.js → preflight-BUjk_6wT.js} +1 -1
- package/dist/{preflight-CtlloeOQ.js → preflight-CJSGzMzc.js} +8 -9
- package/dist/preflight-CJSGzMzc.js.map +1 -0
- package/dist/preflight-DD3fI4Hz.js +1 -0
- package/dist/{preflight-BElE_q0u.js → preflight-qFCXmxha.js} +4 -4
- package/dist/{preflight-BElE_q0u.js.map → preflight-qFCXmxha.js.map} +1 -1
- package/dist/{prepare-C7f5mPl5.js → prepare-D7oSSFNm.js} +9 -9
- package/dist/{prepare-C7f5mPl5.js.map → prepare-D7oSSFNm.js.map} +1 -1
- package/dist/{prepare-CzkAdbXm.js → prepare-xHU2BpN-.js} +2 -2
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-zsc4--dP.js → provenance-B-flBAzR.js} +5 -5
- package/dist/{provenance-zsc4--dP.js.map → provenance-B-flBAzR.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/{providers-B8xYMeah.js → providers-B6StVyfG.js} +215 -48
- package/dist/providers-B6StVyfG.js.map +1 -0
- package/dist/providers-Dljf6al6.js +1 -0
- package/dist/query-BAc2tQZj.js +1 -0
- package/dist/{query-Bb3SYICk.js → query-CkLmJv7R.js} +2 -2
- package/dist/{query-Bb3SYICk.js.map → query-CkLmJv7R.js.map} +1 -1
- package/dist/{questions-CWlR9ffu.js → questions-CJIh2FSX.js} +7 -7
- package/dist/{questions-CWlR9ffu.js.map → questions-CJIh2FSX.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
- package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
- package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
- package/dist/redact-DL2BTykw.js.map +1 -0
- package/dist/{remarks-vtkPmQb9.js → remarks-B3wjKscW.js} +46 -11
- package/dist/remarks-B3wjKscW.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-BgYWqogI.js → remote-C88amFRN.js} +6 -6
- package/dist/{remote-BgYWqogI.js.map → remote-C88amFRN.js.map} +1 -1
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/resolve-DW92wpIZ.js +1 -0
- package/dist/{resolve-Do0ZaYH6.js → resolve-G576PiFJ.js} +14 -11
- package/dist/resolve-G576PiFJ.js.map +1 -0
- package/dist/{results-JyuM64gy.js → results-zVHIW6xw.js} +9 -22
- package/dist/results-zVHIW6xw.js.map +1 -0
- package/dist/{resume-AW5oXzUc.js → resume-D_PcUXfG.js} +94 -16
- package/dist/resume-D_PcUXfG.js.map +1 -0
- package/dist/{retry-ChzISxmy.js → retry-BvTzCxdc.js} +4 -4
- package/dist/{retry-ChzISxmy.js.map → retry-BvTzCxdc.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
- package/dist/routes-B0F2R-K7.js.map +1 -0
- package/dist/{runtime-BpW1f1Mx.js → runtime-B3r_pAq6.js} +4 -4
- package/dist/{runtime-BpW1f1Mx.js.map → runtime-B3r_pAq6.js.map} +1 -1
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-kSgr7vo5.js → sandbox-Duh3z1TV.js} +3 -3
- package/dist/{sandbox-kSgr7vo5.js.map → sandbox-Duh3z1TV.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CGEsvVRG.js +1 -0
- package/dist/{server-Cn8VqO9S.js → server-DUWMRKik.js} +8 -8
- package/dist/{server-Cn8VqO9S.js.map → server-DUWMRKik.js.map} +1 -1
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-CYOhR98B.js → source-CHuN4Nfg.js} +2 -2
- package/dist/{source-CYOhR98B.js.map → source-CHuN4Nfg.js.map} +1 -1
- package/dist/source-Wz1Nrfue.js +1 -0
- package/dist/src--yl1OypK.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
- package/dist/src-B-HuDqpH.js.map +1 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Ck2r_EMT.js → src-BHJ4MWvY.js} +504 -119
- package/dist/src-BHJ4MWvY.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
- package/dist/src-BI0v8qlZ.js.map +1 -0
- package/dist/src-BL-_0QoE.js +1 -0
- package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
- package/dist/src-BqNzHHWn.js.map +1 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C-mOSczH.js +2 -0
- package/dist/src-CZaYu6ER.js +3 -0
- package/dist/src-CeBhwD8k.js +2 -0
- package/dist/src-CkEiiyON.js +2 -0
- package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
- package/dist/src-CmA5Yev6.js.map +1 -0
- package/dist/src-CwH_nEaw.js +2 -0
- package/dist/src-DA95rM6v.js +4 -0
- package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
- package/dist/src-DL-EE4DU.js.map +1 -0
- package/dist/src-De0KY5S_.js +2 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/state-B3jex5KU.js +1 -0
- package/dist/{state-BOLt8pJg.js → state-DUBRDyMk.js} +22 -22
- package/dist/{state-BOLt8pJg.js.map → state-DUBRDyMk.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
- package/dist/suite-Bhs9-Z7O.js +2 -0
- package/dist/{summary-DU5zaibx.js → summary-CSp4cfnr.js} +25 -7
- package/dist/{summary-DU5zaibx.js.map → summary-CSp4cfnr.js.map} +1 -1
- package/dist/sync-CxAY6Dki.js +559 -0
- package/dist/sync-CxAY6Dki.js.map +1 -0
- package/dist/tart-CcrrtSrp.js +704 -0
- package/dist/tart-CcrrtSrp.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-F-6seRC9.js +192 -0
- package/dist/tart-bake-F-6seRC9.js.map +1 -0
- package/dist/tart-bake-RK7RaT9E.js +1 -0
- package/dist/tart-cli-B86Jqo1t.js +89 -0
- package/dist/tart-cli-B86Jqo1t.js.map +1 -0
- package/dist/tart-cli-ClvRniLQ.js +1 -0
- package/dist/tart-hosts-Brii7zHM.js +1 -0
- package/dist/tart-hosts-DPyZsg-1.js +313 -0
- package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
- package/dist/tart-images-CWSf03w8.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-CVDdJy0H.js +433 -0
- package/dist/tool-class-CVDdJy0H.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
- package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +27 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-DAIiLiV3.js → views-DRxXCyIH.js} +6 -6
- package/dist/{views-DAIiLiV3.js.map → views-DRxXCyIH.js.map} +1 -1
- package/dist/views-i67V1ly3.js +2 -0
- package/dist/{worker-entry-BIROTPX1.js → worker-entry-C3Q4U63F.js} +12 -9
- package/dist/worker-entry-C3Q4U63F.js.map +1 -0
- package/dist/workspace-DhUOW8if.js +1 -0
- package/dist/{workspace-uuE51x35.js → workspace-P72SXLXr.js} +7 -7
- package/dist/{workspace-uuE51x35.js.map → workspace-P72SXLXr.js.map} +1 -1
- package/package.json +21 -10
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C8xPuXYq.js.map +0 -1
- package/dist/agent-CBoLndM7.js.map +0 -1
- package/dist/agents-C6zhkxp3.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-t6h018An.js.map +0 -1
- package/dist/attempt-host-CIm1-11m.js.map +0 -1
- package/dist/boat-CJCtzfB1.js.map +0 -1
- package/dist/capture-DoxFUN3J.js +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-20Fpvs5H.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-CRh-cb6Z.js.map +0 -1
- package/dist/commands-CWy-WndG.js.map +0 -1
- package/dist/config-B9mB1kF5.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-P2fXw4nx.js +0 -2
- package/dist/db-zgC1tvKB.js.map +0 -1
- package/dist/dist-CAFD1yfv.js.map +0 -1
- package/dist/docker-DAMnm3L9.js.map +0 -1
- package/dist/doctor-B2PohneB.js.map +0 -1
- package/dist/environment-DB8o3hGi.js.map +0 -1
- package/dist/eval-source-D3sbAzt-.js.map +0 -1
- package/dist/evaluate-CwKeN_fo.js.map +0 -1
- package/dist/files-B4KQUiM7.js +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-C3mCC8Mn.js +0 -1
- package/dist/files-Cl7BQ5gE.js.map +0 -1
- package/dist/files-DnlA1A7N.js +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-BNgcR77K.js.map +0 -1
- package/dist/init-4RPeD-K-.js +0 -1
- package/dist/init-DbuZVRrW.js.map +0 -1
- package/dist/kinds-C1oY3qyC.js +0 -697
- package/dist/kinds-C1oY3qyC.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CJCK11nh.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/options-CHubslQf.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-Cgq9NCEX.js.map +0 -1
- package/dist/plugins-BkmPxDiY.js.map +0 -1
- package/dist/preflight-4gWSy0Pe.js +0 -1
- package/dist/preflight-CtlloeOQ.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-B8xYMeah.js.map +0 -1
- package/dist/providers-CVu8e0xS.js +0 -1
- package/dist/query-DIfkUcPh.js +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/remarks-vtkPmQb9.js.map +0 -1
- package/dist/resolve-Do0ZaYH6.js.map +0 -1
- package/dist/resolve-amw_2Odz.js +0 -1
- package/dist/results-JyuM64gy.js.map +0 -1
- package/dist/resume-AW5oXzUc.js.map +0 -1
- package/dist/server-2_gUnVvf.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-CY3etKzw.js +0 -1
- package/dist/src-6Zzyg4Qd.js +0 -2
- package/dist/src-BDhcPtnI.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BRMsk9nh.js +0 -2
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-Ck2r_EMT.js.map +0 -1
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DB7N8TYJ.js +0 -2
- package/dist/src-DPDaAYwh.js +0 -2
- package/dist/src-Dbk19BiP.js +0 -2
- package/dist/src-DtodMT59.js +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-T-OeuJLr.js +0 -2
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-zntEDlnx.js +0 -3
- package/dist/state-kimtIhBP.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/tart-BTVXp8rz.js +0 -373
- package/dist/tart-BTVXp8rz.js.map +0 -1
- package/dist/tart-cli-BIC9HqPg.js +0 -37
- package/dist/tart-cli-BIC9HqPg.js.map +0 -1
- package/dist/tart-cli-BMrkmBjI.js +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-DtRhj2Tn.js +0 -2
- package/dist/worker-entry-BIROTPX1.js.map +0 -1
- package/dist/workspace-CcQRT31C.js +0 -1
|
@@ -1,33 +1,34 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-
|
|
2
|
+
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-DtZwIAfy.js";
|
|
3
|
+
import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-ChJTQRmz.js";
|
|
4
|
+
import { u as ownFileURL } from "./root-BzeoyfGd.js";
|
|
5
|
+
import { X as HARNESS_HOSTS, Z as PACKAGE_HOSTS } from "./src-BHJ4MWvY.js";
|
|
6
|
+
import { _ as snapshotProblem, c as gatePolicySchema, g as snapshotOptionSchema, l as gateProblem, o as egressSchema, s as envVarsSchema, t as VAR } from "./plain-Ctn8oUdp.js";
|
|
7
|
+
import { i as fromCwd, l as sha256 } from "./ids-B-JKTCP8.js";
|
|
8
|
+
import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-DL2BTykw.js";
|
|
9
|
+
import { H as sandboxHelpers, I as idPart, L as isSandboxKind, N as SANDBOX_KINDS, P as capabilitiesOf, R as maxConcurrentProblem, U as sandboxKind, W as suggestedSandbox, y as localPluginServers } from "./plugins-BioMWncI.js";
|
|
10
|
+
import { n as isRoutingGateway, t as ROUTING_GATEWAYS } from "./routes-B0F2R-K7.js";
|
|
11
|
+
import { c as GATEWAY_NAMES, d as gatewayDescriptorSchema, f as gatewayEntryDataSchema, i as claudeCodeTakesEffort, o as harnessDescriptorSchema, p as gatewayNameSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as directOffPin } from "./harness-FcTyyPco.js";
|
|
12
|
+
import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-BEWxHoxH.js";
|
|
13
|
+
import { _ as isAgentKind, g as harnessCredentials, i as HARNESS_EXAMPLE, p as directProviderRefusal, r as DEFAULT_HARNESS, s as agentFor, t as AGENTS } from "./agents-l0-k7vw3.js";
|
|
3
14
|
import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
4
|
-
import { _ as
|
|
5
|
-
import { a as
|
|
6
|
-
import { C as plainCredentialHeaders, D as sandboxOf, E as sameWorkspaceOnly, F as environmentDataSchema, I as sandboxDescriptorSchema, L as traitsSchema, M as unbrokeredSecrets, N as workspaceLabel, O as sandboxPaths, P as workspaceOf, _ as isMovingRef, a as checkEgress, c as environmentDataProblem, h as isEnvironment, j as traitsOf, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isSecret, w as plannedSecrets, z as sandboxProblem } from "./environment-DB8o3hGi.js";
|
|
7
|
-
import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DXv2gnaT.js";
|
|
8
|
-
import { c as sha256, i as fromCwd } from "./ids-C_advdJA.js";
|
|
9
|
-
import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-BGiXt8_B.js";
|
|
10
|
-
import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-Eb2wqahk.js";
|
|
11
|
-
import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
|
|
12
|
-
import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-COMXT1Hx.js";
|
|
15
|
+
import { B as sandboxProblem, C as plainCredentialEnv, D as sameWorkspaceOnly, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, M as traitsOf, N as unbrokeredSecrets, O as sandboxOf, P as workspaceLabel, R as traitsSchema, T as plannedSecrets, _ as isMovingRef, a as checkEgress, c as environmentDataProblem, h as isEnvironment, k as sandboxPaths, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isSecret, w as plainCredentialHeaders } from "./environment-CJIKz-Ud.js";
|
|
16
|
+
import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-eGH39PaP.js";
|
|
13
17
|
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
14
|
-
import { u as ownFileURL } from "./root-Co0jhOT5.js";
|
|
15
|
-
import { B as PACKAGE_HOSTS, z as HARNESS_HOSTS } from "./src-Ck2r_EMT.js";
|
|
16
|
-
import { y as localPluginServers } from "./plugins-BkmPxDiY.js";
|
|
17
|
-
import { a as agentFor, b as isRoutingGateway, f as isAgentKind, n as DEFAULT_HARNESS, r as HARNESS_EXAMPLE, t as AGENTS, u as credentialOf, y as ROUTING_GATEWAYS } from "./agents-C6zhkxp3.js";
|
|
18
18
|
import { i as pairFactors } from "./pairs-BGb134Vu.js";
|
|
19
|
-
import { u as opencodeModelError } from "./harness-
|
|
19
|
+
import { u as opencodeModelError } from "./harness-nwnyghJZ.js";
|
|
20
20
|
import { r as tartTagWarning } from "./tart-ref-DYZijjrU.js";
|
|
21
21
|
import "./stats-C18W4xPX.js";
|
|
22
22
|
import { n as gateLabel } from "./gate-CgYdaprG.js";
|
|
23
|
-
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-
|
|
24
|
-
import { t as docker } from "./sandbox-
|
|
23
|
+
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-BdzhdRYo.js";
|
|
24
|
+
import { t as docker } from "./sandbox-Duh3z1TV.js";
|
|
25
|
+
import { i as resolveHistory } from "./resolve-Bul8lHaa.js";
|
|
25
26
|
import { z } from "zod";
|
|
26
27
|
import { existsSync, readFileSync } from "node:fs";
|
|
27
|
-
import { availableParallelism, totalmem } from "node:os";
|
|
28
28
|
import { dirname, join, resolve } from "node:path";
|
|
29
|
-
import {
|
|
29
|
+
import { availableParallelism, totalmem } from "node:os";
|
|
30
30
|
import { isIP } from "node:net";
|
|
31
|
+
import { parseDocument } from "yaml";
|
|
31
32
|
// src/judge/artifacts.ts
|
|
32
33
|
/** A capture handle: a name plus the file the attempt wrote. */
|
|
33
34
|
function isCaptureHandle(value) {
|
|
@@ -408,12 +409,38 @@ const budgetSchema = z.strictObject({
|
|
|
408
409
|
attempts: count().optional(),
|
|
409
410
|
duration: durationSchema.optional()
|
|
410
411
|
});
|
|
411
|
-
const
|
|
412
|
-
|
|
412
|
+
const localStoreDescriptorSchema = z.strictObject({
|
|
413
|
+
kind: z.literal("local"),
|
|
414
|
+
dir: z.string().optional()
|
|
415
|
+
});
|
|
416
|
+
const s3StoreEnvSchema = z.strictObject({
|
|
417
|
+
accessKeyId: z.string().optional(),
|
|
418
|
+
secretAccessKey: z.string().optional(),
|
|
419
|
+
sessionToken: z.string().optional(),
|
|
420
|
+
endpoint: z.string().optional(),
|
|
421
|
+
bucket: z.string().optional(),
|
|
422
|
+
region: z.string().optional()
|
|
423
|
+
});
|
|
424
|
+
const s3StoreDescriptorSchema = z.strictObject({
|
|
425
|
+
kind: z.literal("s3"),
|
|
426
|
+
bucket: z.string().optional(),
|
|
427
|
+
endpoint: z.string().optional(),
|
|
428
|
+
region: z.string().optional(),
|
|
429
|
+
prefix: z.string().optional(),
|
|
430
|
+
env: s3StoreEnvSchema.optional()
|
|
431
|
+
});
|
|
432
|
+
const historyStoreDescriptorSchema = z.discriminatedUnion("kind", [localStoreDescriptorSchema, s3StoreDescriptorSchema]);
|
|
433
|
+
const historyConfigDataSchema = z.strictObject({
|
|
434
|
+
store: historyStoreDescriptorSchema.optional(),
|
|
413
435
|
artifacts: z.enum([
|
|
414
436
|
"all",
|
|
415
437
|
"failures",
|
|
416
438
|
"none"
|
|
439
|
+
]).optional(),
|
|
440
|
+
debug: z.enum([
|
|
441
|
+
"none",
|
|
442
|
+
"requests",
|
|
443
|
+
"all"
|
|
417
444
|
]).optional()
|
|
418
445
|
});
|
|
419
446
|
const recordMode = z.enum([
|
|
@@ -428,7 +455,7 @@ const profileDefaultsDataSchema = z.strictObject({
|
|
|
428
455
|
runs: runs().optional(),
|
|
429
456
|
gate: gatePolicySchema.optional(),
|
|
430
457
|
env: envVarsSchema.optional(),
|
|
431
|
-
egress:
|
|
458
|
+
egress: egressSchema.optional(),
|
|
432
459
|
judge: configJudgeDataSchema.optional(),
|
|
433
460
|
timeout: durationSchema.optional(),
|
|
434
461
|
capture: captureOptionsSchema.optional(),
|
|
@@ -448,7 +475,8 @@ const profileDataSchema = z.strictObject({
|
|
|
448
475
|
budget: budgetSchema.optional(),
|
|
449
476
|
record: recordMode.optional(),
|
|
450
477
|
tags: z.array(z.string()).optional(),
|
|
451
|
-
gateway: z.array(gatewayEntryDataSchema).min(1).optional()
|
|
478
|
+
gateway: z.array(gatewayEntryDataSchema).min(1).optional(),
|
|
479
|
+
history: historyConfigDataSchema.optional()
|
|
452
480
|
});
|
|
453
481
|
const fakeVote = z.enum([
|
|
454
482
|
"A",
|
|
@@ -490,7 +518,7 @@ const reevalConfigDataSchema = z.strictObject({
|
|
|
490
518
|
runs: runs().optional(),
|
|
491
519
|
gate: gatePolicySchema.optional(),
|
|
492
520
|
env: envVarsSchema.optional(),
|
|
493
|
-
egress:
|
|
521
|
+
egress: egressSchema.optional(),
|
|
494
522
|
judge: configJudgeDataSchema.optional(),
|
|
495
523
|
timeout: durationSchema.optional(),
|
|
496
524
|
capture: captureOptionsSchema.optional(),
|
|
@@ -498,7 +526,7 @@ const reevalConfigDataSchema = z.strictObject({
|
|
|
498
526
|
snapshot: snapshotOptionSchema.optional(),
|
|
499
527
|
concurrency: count().optional(),
|
|
500
528
|
record: recordMode.optional(),
|
|
501
|
-
history:
|
|
529
|
+
history: historyConfigDataSchema.optional(),
|
|
502
530
|
budget: budgetSchema.optional(),
|
|
503
531
|
profiles: z.record(z.string(), profileDataSchema).optional(),
|
|
504
532
|
defaultProfile: z.strictObject({
|
|
@@ -676,7 +704,8 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
|
676
704
|
removed: nullableNumber,
|
|
677
705
|
durationMs: nullableNumber,
|
|
678
706
|
stalled: z.boolean().optional(),
|
|
679
|
-
subagent: z.string().optional()
|
|
707
|
+
subagent: z.string().optional(),
|
|
708
|
+
durationInferred: z.literal(true).optional()
|
|
680
709
|
}),
|
|
681
710
|
z.strictObject({
|
|
682
711
|
...at$1,
|
|
@@ -687,7 +716,8 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
|
687
716
|
ok: z.boolean().nullable().optional(),
|
|
688
717
|
durationMs: nullableNumber,
|
|
689
718
|
stalled: z.boolean().optional(),
|
|
690
|
-
subagent: z.string().optional()
|
|
719
|
+
subagent: z.string().optional(),
|
|
720
|
+
durationInferred: z.literal(true).optional()
|
|
691
721
|
}),
|
|
692
722
|
z.strictObject({
|
|
693
723
|
...at$1,
|
|
@@ -697,7 +727,8 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
|
697
727
|
ok: z.boolean().nullable(),
|
|
698
728
|
durationMs: nullableNumber,
|
|
699
729
|
stalled: z.boolean().optional(),
|
|
700
|
-
subagent: z.string().optional()
|
|
730
|
+
subagent: z.string().optional(),
|
|
731
|
+
durationInferred: z.literal(true).optional()
|
|
701
732
|
}),
|
|
702
733
|
z.strictObject({
|
|
703
734
|
...at$1,
|
|
@@ -706,10 +737,13 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
|
706
737
|
tool: z.string(),
|
|
707
738
|
input: z.unknown(),
|
|
708
739
|
output: z.unknown(),
|
|
740
|
+
outputView: z.enum(["model", "full"]).optional(),
|
|
741
|
+
outputTruncated: z.strictObject({ bytes: z.number() }).optional(),
|
|
709
742
|
ok: z.boolean().nullable(),
|
|
710
743
|
durationMs: nullableNumber,
|
|
711
744
|
stalled: z.boolean().optional(),
|
|
712
|
-
subagent: z.string().optional()
|
|
745
|
+
subagent: z.string().optional(),
|
|
746
|
+
durationInferred: z.literal(true).optional()
|
|
713
747
|
}),
|
|
714
748
|
z.strictObject({
|
|
715
749
|
...at$1,
|
|
@@ -720,6 +754,18 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
|
720
754
|
...at$1,
|
|
721
755
|
type: z.literal("retry"),
|
|
722
756
|
message: z.string()
|
|
757
|
+
}),
|
|
758
|
+
z.strictObject({
|
|
759
|
+
...at$1,
|
|
760
|
+
type: z.literal("model-call"),
|
|
761
|
+
request: z.string(),
|
|
762
|
+
status: nullableNumber,
|
|
763
|
+
durationMs: z.number(),
|
|
764
|
+
firstByteMs: nullableNumber,
|
|
765
|
+
usage: tokenUsageSchema.nullable(),
|
|
766
|
+
model: z.string().nullable(),
|
|
767
|
+
stream: z.literal(true).optional(),
|
|
768
|
+
role: z.enum(["main", "side"]).optional()
|
|
723
769
|
})
|
|
724
770
|
]);
|
|
725
771
|
const turnDataSchema = z.strictObject({
|
|
@@ -743,6 +789,7 @@ const execRetrySchema = z.strictObject({
|
|
|
743
789
|
retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
|
|
744
790
|
interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
|
|
745
791
|
});
|
|
792
|
+
const execEnvSchema = z.record(z.string().regex(VAR, "must be a variable name: letters, digits and _, not starting with a digit"), z.string().refine((v) => !v.includes("\0"), "must not hold a NUL character"));
|
|
746
793
|
/** A background command has no exit for `retries` to wait on. */
|
|
747
794
|
const noBackgroundRetry = (options, ctx) => {
|
|
748
795
|
if (!options.background) return;
|
|
@@ -757,6 +804,7 @@ const noBackgroundRetry = (options, ctx) => {
|
|
|
757
804
|
};
|
|
758
805
|
const execOptionsSchema = z.strictObject({
|
|
759
806
|
background: z.boolean().optional(),
|
|
807
|
+
env: execEnvSchema.optional(),
|
|
760
808
|
...execRetrySchema.shape
|
|
761
809
|
}).superRefine(noBackgroundRetry);
|
|
762
810
|
/** What's wrong with `c.exec()`'s options, or undefined. */
|
|
@@ -873,6 +921,7 @@ const matcherSubjectSchema = z.union([
|
|
|
873
921
|
z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
|
|
874
922
|
z.strictObject({
|
|
875
923
|
exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
|
|
924
|
+
env: execEnvSchema.optional(),
|
|
876
925
|
result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
|
|
877
926
|
}),
|
|
878
927
|
z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
|
|
@@ -920,8 +969,9 @@ const attemptCallSchema = z.discriminatedUnion("op", [
|
|
|
920
969
|
op: z.literal("exec"),
|
|
921
970
|
command: nonEmpty.describe("A shell command, run in the workspace."),
|
|
922
971
|
background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
|
|
972
|
+
env: execEnvSchema.optional(),
|
|
923
973
|
...execRetrySchema.shape
|
|
924
|
-
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
974
|
+
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, env?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
925
975
|
z.strictObject({
|
|
926
976
|
op: z.literal("serve"),
|
|
927
977
|
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
@@ -1457,6 +1507,47 @@ const DEFAULT_EGRESS_ALLOW = PACKAGE_HOSTS;
|
|
|
1457
1507
|
Object.fromEntries(Object.entries(SANDBOX_KINDS).map(([kind, k]) => [kind, k.capabilities.egress]));
|
|
1458
1508
|
/** The policy an attempt runs under when nothing sets one. */
|
|
1459
1509
|
const DEFAULT_EGRESS = "proxy";
|
|
1510
|
+
/** Whether `egress` sets a policy per phase, `{ setup, agent }`, rather than one for both. */
|
|
1511
|
+
const isPhased = (egress) => typeof egress === "object" && !("allow" in egress);
|
|
1512
|
+
/** The policy `egress` sets for `phase`: a plain policy sets both; undefined when it sets none. */
|
|
1513
|
+
const phaseOf = (egress, phase) => egress === void 0 ? void 0 : isPhased(egress) ? egress[phase] : egress;
|
|
1514
|
+
/**
|
|
1515
|
+
* `over` set at the same level as `under`, later: a plain policy replaces both phases,
|
|
1516
|
+
* `{ setup, agent }` only the phases it names.
|
|
1517
|
+
*/
|
|
1518
|
+
function layerEgress(under, over) {
|
|
1519
|
+
if (!isPhased(over)) return over;
|
|
1520
|
+
const setup = over.setup ?? phaseOf(under, "setup");
|
|
1521
|
+
const agent = over.agent ?? phaseOf(under, "agent");
|
|
1522
|
+
return {
|
|
1523
|
+
...setup !== void 0 ? { setup } : {},
|
|
1524
|
+
...agent !== void 0 ? { agent } : {}
|
|
1525
|
+
};
|
|
1526
|
+
}
|
|
1527
|
+
/** Each phase's policy `egress` sets, the default where it sets none. */
|
|
1528
|
+
const phasePolicies = (egress) => ({
|
|
1529
|
+
setup: phaseOf(egress, "setup") ?? "proxy",
|
|
1530
|
+
agent: phaseOf(egress, "agent") ?? "proxy"
|
|
1531
|
+
});
|
|
1532
|
+
/** `egress` planned with no automatic hosts: what a config-level check can tell of it. */
|
|
1533
|
+
function bareEgress(egress) {
|
|
1534
|
+
const { setup, agent } = phasePolicies(egress);
|
|
1535
|
+
return samePolicy(setup, agent) ? {
|
|
1536
|
+
policy: agent,
|
|
1537
|
+
auto: []
|
|
1538
|
+
} : {
|
|
1539
|
+
policy: agent,
|
|
1540
|
+
auto: [],
|
|
1541
|
+
setup: {
|
|
1542
|
+
policy: setup,
|
|
1543
|
+
auto: []
|
|
1544
|
+
}
|
|
1545
|
+
};
|
|
1546
|
+
}
|
|
1547
|
+
const policyKey = (p) => typeof p === "string" ? p : JSON.stringify([...new Set(p.allow)].sort());
|
|
1548
|
+
const samePolicy = (a, b) => policyKey(a) === policyKey(b);
|
|
1549
|
+
/** The setup phase's planned policy: its own when it differs from the agent's. */
|
|
1550
|
+
const setupEgress = (planned) => planned.setup ?? planned;
|
|
1460
1551
|
/** Whether the policy allows only some hosts: `deny-all` and allowlists. */
|
|
1461
1552
|
const isRestricted = (policy) => policy !== "allow-all" && policy !== "proxy";
|
|
1462
1553
|
/**
|
|
@@ -1465,7 +1556,8 @@ const isRestricted = (policy) => policy !== "allow-all" && policy !== "proxy";
|
|
|
1465
1556
|
*/
|
|
1466
1557
|
const isDirect = (policy) => policy === "allow-all";
|
|
1467
1558
|
/**
|
|
1468
|
-
* The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode
|
|
1559
|
+
* The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode, Goose
|
|
1560
|
+
* and pi), or
|
|
1469
1561
|
* a fake agent's with `layout` (it behaves like that agent). A plain fake agent talks to nothing.
|
|
1470
1562
|
*/
|
|
1471
1563
|
function harnessHosts(harness, vertex) {
|
|
@@ -1481,26 +1573,34 @@ function mcpServerHosts(layers) {
|
|
|
1481
1573
|
return [...urls.values()].flatMap((url) => URL.parse(url)?.hostname ?? []);
|
|
1482
1574
|
}
|
|
1483
1575
|
/**
|
|
1484
|
-
* The policy an attempt runs under, and the hosts a restricted one allows on its
|
|
1485
|
-
* harness's, the brokered secrets' `hosts` and the MCP servers'.
|
|
1486
|
-
* profile defaults (`inherited`), then the environment's layers, then groups
|
|
1576
|
+
* The policy each phase of an attempt runs under, and the hosts a restricted one allows on its
|
|
1577
|
+
* own: the harness's, the brokered secrets' `hosts` and the MCP servers'. Each phase cascades on
|
|
1578
|
+
* its own: config and profile defaults (`inherited`), then the environment's layers, then groups
|
|
1579
|
+
* and the eval (`own`). `setup` is set only when the setup phase's policy differs from the agent's.
|
|
1487
1580
|
*/
|
|
1488
1581
|
function plannedEgress(egress, harness, env, layers, vertex) {
|
|
1489
|
-
const
|
|
1490
|
-
const
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1582
|
+
const pick = (phase) => phaseOf(egress.own, phase) ?? layers.map((l) => phaseOf(l.spec.egress, phase)).findLast((p) => p !== void 0) ?? phaseOf(egress.inherited, phase) ?? "proxy";
|
|
1583
|
+
const phase = (policy) => {
|
|
1584
|
+
if (!isRestricted(policy)) return {
|
|
1585
|
+
policy,
|
|
1586
|
+
auto: []
|
|
1587
|
+
};
|
|
1588
|
+
const own = policy === "deny-all" ? [] : policy.allow;
|
|
1589
|
+
const auto = [
|
|
1590
|
+
...harnessHosts(harness, vertex),
|
|
1591
|
+
...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
|
|
1592
|
+
...mcpServerHosts(layers)
|
|
1593
|
+
];
|
|
1594
|
+
return {
|
|
1595
|
+
policy,
|
|
1596
|
+
auto: [...new Set(auto)].filter((h) => !own.includes(h))
|
|
1597
|
+
};
|
|
1494
1598
|
};
|
|
1495
|
-
const
|
|
1496
|
-
const
|
|
1497
|
-
|
|
1498
|
-
...
|
|
1499
|
-
|
|
1500
|
-
];
|
|
1501
|
-
return {
|
|
1502
|
-
policy,
|
|
1503
|
-
auto: [...new Set(auto)].filter((h) => !own.includes(h))
|
|
1599
|
+
const agent = phase(pick("agent"));
|
|
1600
|
+
const setup = pick("setup");
|
|
1601
|
+
return samePolicy(setup, agent.policy) ? agent : {
|
|
1602
|
+
...agent,
|
|
1603
|
+
setup: phase(setup)
|
|
1504
1604
|
};
|
|
1505
1605
|
}
|
|
1506
1606
|
/** Every host a restricted policy lets through, or null under `proxy` and `allow-all`. */
|
|
@@ -1535,19 +1635,24 @@ const LEVELS = [
|
|
|
1535
1635
|
"all-or-nothing",
|
|
1536
1636
|
"allowlist"
|
|
1537
1637
|
];
|
|
1538
|
-
/** Why the sandbox can't enforce the attempt's
|
|
1638
|
+
/** Why the sandbox can't enforce the attempt's policies, or undefined when it can. */
|
|
1539
1639
|
function unenforceable(planned, sandbox, where) {
|
|
1640
|
+
if (!planned.setup) return unenforceablePhase(planned, sandbox, where, "egress");
|
|
1641
|
+
if (!capabilitiesOf(sandbox).egress.switches) return `${where} runs under egress ${describeEgress(planned.setup.policy)} for setup and ${describeEgress(planned.policy)} for the agent, but the ${sandbox.kind} sandbox can't change its egress policy during an attempt. Give both phases one policy, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.switches)}.`;
|
|
1642
|
+
return unenforceablePhase(planned.setup, sandbox, where, "egress.setup") ?? unenforceablePhase(planned, sandbox, where, "egress.agent");
|
|
1643
|
+
}
|
|
1644
|
+
function unenforceablePhase(planned, sandbox, where, key) {
|
|
1540
1645
|
const capabilities = capabilitiesOf(sandbox);
|
|
1541
1646
|
const can = capabilities.egress.enforce;
|
|
1542
|
-
if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under
|
|
1647
|
+
if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under ${key} 'proxy', where the egress gateway is the only way out, but the ${sandbox.kind} sandbox enforces no egress policy: its traffic that ignores the proxy goes out directly. Set ${key}: 'allow-all' to run it there, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.enforce !== "none")}.`;
|
|
1543
1648
|
const needs = egressNeeds(planned);
|
|
1544
1649
|
if (LEVELS.indexOf(can) >= LEVELS.indexOf(needs)) return void 0;
|
|
1545
1650
|
const hosts = allowedHosts(planned)?.join(", ");
|
|
1546
1651
|
const what = needs !== "allowlist" ? "turning egress off" : planned.policy === "deny-all" ? `a host allowlist for its automatic hosts (${hosts})` : `a host allowlist (${hosts})`;
|
|
1547
1652
|
const enforces = can === "none" ? "enforces no egress policy" : `only enforces ${can === "all-or-nothing" ? "'allow-all' or 'deny-all'" : can}`;
|
|
1548
1653
|
const enforcing = suggestedSandbox((c) => LEVELS.indexOf(c.egress.enforce) >= LEVELS.indexOf(needs));
|
|
1549
|
-
const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"}
|
|
1550
|
-
return `${where} runs under
|
|
1654
|
+
const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"} ${key}: 'allow-all'`;
|
|
1655
|
+
return `${where} runs under ${key} ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
|
|
1551
1656
|
}
|
|
1552
1657
|
const describeEgress = (policy) => typeof policy === "string" ? `'${policy}'` : `{ allow: [${policy.allow.map((h) => `'${h}'`).join(", ")}] }`;
|
|
1553
1658
|
/** A CI runner as the environment names it, which usually runs on a cloud VM; null elsewhere. */
|
|
@@ -1563,8 +1668,35 @@ function ciRunner(env) {
|
|
|
1563
1668
|
* so anything in it reaches cloud metadata and the host's network around the gateway.
|
|
1564
1669
|
*/
|
|
1565
1670
|
const directOnCloud = (where) => `egress 'allow-all' on ${where}: anything in the sandbox can reach the cloud metadata service (169.254.169.254, which hands out the machine's credentials), the host's network and its services around the egress gateway, unfiltered and unrecorded`;
|
|
1566
|
-
/**
|
|
1567
|
-
|
|
1671
|
+
/**
|
|
1672
|
+
* What running attempts with a setup policy of their own (`ids`) side by side on rootless Podman
|
|
1673
|
+
* risks: a container leaving Podman's bridge, as one does at the switch, can cut the others on it
|
|
1674
|
+
* off from the host for seconds.
|
|
1675
|
+
*/
|
|
1676
|
+
const rootlessPodmanSwitch = (ids) => `phased egress on rootless Podman: at the switch to the agent's policy an attempt's container leaves Podman's bridge network, which can cut the other containers on it off from the host for seconds, failing their direct connections (allow-all, and setup steps under it). Under it: ${ids.join(", ")}. Run with --concurrency 1, or on docker(), when those connections matter`;
|
|
1677
|
+
/**
|
|
1678
|
+
* `deny-all + auto: api2.cursor.sh, …` or `allow-all` for the plan; '' under `proxy`, the default.
|
|
1679
|
+
* With a setup policy of its own, `setup allow-all · agent deny-all + auto: …`.
|
|
1680
|
+
*/
|
|
1681
|
+
function egressLabel(planned) {
|
|
1682
|
+
if (!planned.setup) return phaseLabel(planned);
|
|
1683
|
+
return `setup ${phaseLabel(planned.setup) || "proxy"} · agent ${phaseLabel(planned) || "proxy"}`;
|
|
1684
|
+
}
|
|
1685
|
+
/** {@link egressLabel} without the automatic hosts, which {@link autoHosts} lists. */
|
|
1686
|
+
function ownEgressLabel(planned) {
|
|
1687
|
+
const own = (p) => ({
|
|
1688
|
+
...p,
|
|
1689
|
+
auto: []
|
|
1690
|
+
});
|
|
1691
|
+
return egressLabel({
|
|
1692
|
+
...own(planned),
|
|
1693
|
+
...planned.setup ? { setup: own(planned.setup) } : {}
|
|
1694
|
+
});
|
|
1695
|
+
}
|
|
1696
|
+
/** The hosts reeval allows on top of the policy, in either phase. */
|
|
1697
|
+
const autoHosts = (planned) => [.../* @__PURE__ */ new Set([...planned.setup?.auto ?? [], ...planned.auto])];
|
|
1698
|
+
/** One phase's policy and automatic hosts, or `''` for the default, `proxy`. */
|
|
1699
|
+
function phaseLabel({ policy, auto }) {
|
|
1568
1700
|
if (policy === "proxy") return "";
|
|
1569
1701
|
if (policy === "allow-all") return "allow-all";
|
|
1570
1702
|
const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
|
|
@@ -1921,7 +2053,7 @@ function normalizeCompare(value) {
|
|
|
1921
2053
|
}
|
|
1922
2054
|
/**
|
|
1923
2055
|
* Runs per cell when no level sets `runs`: a compare with variants runs `COMPARE_RUNS` a side;
|
|
1924
|
-
* anything else,
|
|
2056
|
+
* anything else runs once, two or more cases with a compare included, whose lift pairs cases.
|
|
1925
2057
|
*/
|
|
1926
2058
|
const COMPARE_DEFAULT_RUNS = 4;
|
|
1927
2059
|
function applyLevel(out, options, where, egress = "own") {
|
|
@@ -1929,7 +2061,7 @@ function applyLevel(out, options, where, egress = "own") {
|
|
|
1929
2061
|
...out.sources,
|
|
1930
2062
|
[key]: where
|
|
1931
2063
|
};
|
|
1932
|
-
if (options.egress !== void 0) out.egress[egress] = options.egress;
|
|
2064
|
+
if (options.egress !== void 0) out.egress[egress] = layerEgress(out.egress[egress], options.egress);
|
|
1933
2065
|
out.harness = applyHarness(out.harness, options.harnesses, where);
|
|
1934
2066
|
if (options.sandbox !== void 0) out.sandbox = options.sandbox;
|
|
1935
2067
|
out.environments = applyEnvironments(out.environments, options.environments, where);
|
|
@@ -2118,10 +2250,15 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2118
2250
|
/** Gives `out` its default runs when no level set them and `--runs` doesn't replace them. */
|
|
2119
2251
|
function defaultRuns(out, notes, options) {
|
|
2120
2252
|
if (out.runs !== void 0 || options.runs !== void 0) return;
|
|
2121
|
-
if (!out.compare?.variants.length
|
|
2253
|
+
if (!out.compare?.variants.length) {
|
|
2122
2254
|
out.runs = 1;
|
|
2123
2255
|
return;
|
|
2124
2256
|
}
|
|
2257
|
+
if (options.pooled) {
|
|
2258
|
+
out.runs = 1;
|
|
2259
|
+
notes.push("runs 1 a case (compare with cases: the lift pairs cases)");
|
|
2260
|
+
return;
|
|
2261
|
+
}
|
|
2125
2262
|
out.runs = COMPARE_DEFAULT_RUNS;
|
|
2126
2263
|
out.sources = {
|
|
2127
2264
|
...out.sources,
|
|
@@ -2302,6 +2439,18 @@ function plainHeaders(attempts) {
|
|
|
2302
2439
|
const templateHint = (h) => `read it from a host variable with secret.template, e.g. secret.template('Bearer \${VAR}')${h.https ? "" : " and give the MCP server an https:// URL"}`;
|
|
2303
2440
|
const plainHeaderWarning = (h) => `${h.label} is a plain string, so it's in the sandbox as its real value. To give the agent only a placeholder, ${templateHint(h)}.`;
|
|
2304
2441
|
const plainHeaderError = (h, where, configFile) => `${h.label} in ${where} is a plain string, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${templateHint(h)}.`;
|
|
2442
|
+
/** The attempts' `env` strings that look like credentials, once each. */
|
|
2443
|
+
function plainEnv(env, attempts) {
|
|
2444
|
+
const out = /* @__PURE__ */ new Map();
|
|
2445
|
+
for (const a of attempts) for (const e of plainCredentialEnv(env, a.environment.layers)) out.set(e.label, e);
|
|
2446
|
+
return [...out.values()];
|
|
2447
|
+
}
|
|
2448
|
+
/** A placeholder only works for a value sent to an HTTPS API: the gateway swaps it in nowhere else. */
|
|
2449
|
+
const hostsExample = (e) => `secret.fromEnv('${e.name}', { hosts: ['api.example.com'] })`;
|
|
2450
|
+
const maskHint = (e) => e.plugin ? `write it as "\${${e.name}}" in the plugin's .mcp.json, so reeval reads it from the host variable ${e.name}` : `read it from a host variable with secret.fromEnv('${e.name}'); to give the agent only a placeholder too, add the hosts it's sent to if that's an HTTPS API, e.g. ${hostsExample(e)}`;
|
|
2451
|
+
const plainEnvWarning = (e) => `${e.label} is a plain string that looks like a credential, so it's in the sandbox as its real value. To mask it in everything reeval writes, ${maskHint(e)}.`;
|
|
2452
|
+
const brokerEnvHint = (e) => e.plugin ? `declare the server with mcp.stdio() instead of the plugin's .mcp.json, whose host variables can't set hosts, and read it with ${hostsExample(e)}, naming the HTTPS hosts it's sent to` : `read it from a host variable with ${hostsExample(e)}, naming the HTTPS hosts it's sent to`;
|
|
2453
|
+
const plainEnvError = (e, where, configFile) => `${e.label} in ${where} is a plain string that looks like a credential, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${brokerEnvHint(e)}.`;
|
|
2305
2454
|
const levelOf = (where) => where.startsWith("evaluate(") ? "eval" : where.startsWith("group(") ? "group" : where.startsWith("profiles.") ? "profile" : "config";
|
|
2306
2455
|
/**
|
|
2307
2456
|
* A warning per privilege an attempt's Docker sandbox adds, naming the level that set the
|
|
@@ -2375,11 +2524,20 @@ function evalPlan(r, attempts, origin) {
|
|
|
2375
2524
|
...environmentWarnings(attempts),
|
|
2376
2525
|
...skip ? [] : questionWarnings(resolved),
|
|
2377
2526
|
...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning),
|
|
2378
|
-
...plainHeaders(attempts).map(plainHeaderWarning)
|
|
2527
|
+
...plainHeaders(attempts).map(plainHeaderWarning),
|
|
2528
|
+
...plainEnv(r.resolved.env, attempts).map(plainEnvWarning)
|
|
2379
2529
|
],
|
|
2380
2530
|
sandboxWarnings: skip ? [] : sandboxWarnings(attempts, resolved, origin),
|
|
2381
2531
|
brokered: [...new Set(attempts.flatMap((a) => brokeredIn(r.resolved.env, a)))],
|
|
2382
|
-
egress: [...new
|
|
2532
|
+
egress: [...new Map(attempts.map((a) => {
|
|
2533
|
+
const e = {
|
|
2534
|
+
environment: a.environment.name,
|
|
2535
|
+
harness: harnessId(a.harness),
|
|
2536
|
+
policy: ownEgressLabel(a.egress) || "proxy",
|
|
2537
|
+
auto: autoHosts(a.egress)
|
|
2538
|
+
};
|
|
2539
|
+
return [JSON.stringify(e), e];
|
|
2540
|
+
})).values()],
|
|
2383
2541
|
workspaces: [...new Set(attempts.flatMap((a) => {
|
|
2384
2542
|
const w = workspaceOf(a.environment.layers);
|
|
2385
2543
|
return w ? [workspaceLabel(w)] : [];
|
|
@@ -2414,6 +2572,7 @@ const GATEWAYS = {
|
|
|
2414
2572
|
const fxEffort = fxEffortRefusal(h, "direct()");
|
|
2415
2573
|
if (fxEffort) return fxEffort;
|
|
2416
2574
|
if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway or OpenRouter`;
|
|
2575
|
+
if (h.kind === "pi" || h.kind === "goose") return directProviderRefusal(h.kind, h.model);
|
|
2417
2576
|
if (h.kind === "opencode") {
|
|
2418
2577
|
const resolved = resolveOpenCodeProvider(h.model, process.env);
|
|
2419
2578
|
if ("error" in resolved) return resolved.error;
|
|
@@ -2438,7 +2597,7 @@ const GATEWAYS = {
|
|
|
2438
2597
|
/** OpenCode's id for `model` on `gateway`, as that gateway's route writes it. */
|
|
2439
2598
|
function openCodeModelOn(model, gateway) {
|
|
2440
2599
|
if (gateway === "vertex") return openCodeVertexId(model);
|
|
2441
|
-
return gateway
|
|
2600
|
+
return isRoutingGateway(gateway) ? `${gateway}/${model}` : model;
|
|
2442
2601
|
}
|
|
2443
2602
|
/** Why the API behind `gateway` answers OpenCode's `effort` on `model` with a 400, or undefined. */
|
|
2444
2603
|
function openCodeEffortOn(model, gateway, effort) {
|
|
@@ -2627,7 +2786,8 @@ const PROFILE_KEYS = [
|
|
|
2627
2786
|
"budget",
|
|
2628
2787
|
"record",
|
|
2629
2788
|
"tags",
|
|
2630
|
-
"gateway"
|
|
2789
|
+
"gateway",
|
|
2790
|
+
"history"
|
|
2631
2791
|
];
|
|
2632
2792
|
const DEFAULTS_KEYS = [
|
|
2633
2793
|
"harnesses",
|
|
@@ -2649,11 +2809,9 @@ const LIMIT_KEYS = [
|
|
|
2649
2809
|
"harnesses",
|
|
2650
2810
|
"traits"
|
|
2651
2811
|
];
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
*/
|
|
2656
|
-
const defineConfig = (config) => config;
|
|
2812
|
+
function defineConfig(config) {
|
|
2813
|
+
return config;
|
|
2814
|
+
}
|
|
2657
2815
|
/** The script configs `reeval` looks for in each directory, first match first, as Vitest orders its own. */
|
|
2658
2816
|
const SCRIPT_FILES = [
|
|
2659
2817
|
"ts",
|
|
@@ -2845,11 +3003,6 @@ function evalRoot(include) {
|
|
|
2845
3003
|
for (let i = 0; dirs.every((d) => d[i] !== void 0 && d[i] === dirs[0]?.[i]); i++) common.push(dirs[0]?.[i]);
|
|
2846
3004
|
return common.join("/");
|
|
2847
3005
|
}
|
|
2848
|
-
const ARTIFACTS = [
|
|
2849
|
-
"all",
|
|
2850
|
-
"failures",
|
|
2851
|
-
"none"
|
|
2852
|
-
];
|
|
2853
3006
|
const RECORD = [
|
|
2854
3007
|
"manual",
|
|
2855
3008
|
"ci",
|
|
@@ -2869,7 +3022,7 @@ const unknownKey = (value, allowed, where, file) => {
|
|
|
2869
3022
|
function checkBudget(budget, where, file) {
|
|
2870
3023
|
if (budget === void 0) return {};
|
|
2871
3024
|
if (typeof budget !== "object" || budget === null) throw new UsageError(`${where} in ${file} must be an object like { attempts: 20, duration: '1h' } (got ${JSON.stringify(budget)})`);
|
|
2872
|
-
if ("usd" in budget || "tokens" in budget) throw new UsageError(`${where}.${"usd" in budget ? "usd" : "tokens"} in ${file} isn't supported yet: reeval records the tokens
|
|
3025
|
+
if ("usd" in budget || "tokens" in budget) throw new UsageError(`${where}.${"usd" in budget ? "usd" : "tokens"} in ${file} isn't supported yet: reeval records the tokens and cost of each attempt, but enforces no cost or token budget. Cap the run with attempts or duration.`);
|
|
2873
3026
|
unknownKey(budget, ["attempts", "duration"], where, file);
|
|
2874
3027
|
const { attempts, duration } = budget;
|
|
2875
3028
|
if (attempts !== void 0 && !(Number.isInteger(attempts) && attempts >= 1)) throw new UsageError(`${where}.attempts in ${file} must be a whole number of at least 1 (got ${JSON.stringify(attempts)})`);
|
|
@@ -3002,9 +3155,17 @@ function selectProfile(config, flag, env = process.env) {
|
|
|
3002
3155
|
...profile.budget
|
|
3003
3156
|
},
|
|
3004
3157
|
...profile.record ? { record: profile.record } : {},
|
|
3005
|
-
...profile.gateway ? { gateway: profile.gateway } : {}
|
|
3158
|
+
...profile.gateway ? { gateway: profile.gateway } : {},
|
|
3159
|
+
...config.profileHistory[name] ? { history: config.profileHistory[name] } : {}
|
|
3006
3160
|
};
|
|
3007
3161
|
}
|
|
3162
|
+
/** The config with the profile's `history` in place of its own, when the profile sets one. */
|
|
3163
|
+
function withProfileHistory(config, profile) {
|
|
3164
|
+
return profile?.history ? {
|
|
3165
|
+
...config,
|
|
3166
|
+
history: profile.history
|
|
3167
|
+
} : config;
|
|
3168
|
+
}
|
|
3008
3169
|
/**
|
|
3009
3170
|
* The config with its defaults filled in and checked; `root` is its directory, which relative
|
|
3010
3171
|
* host paths written as data resolve against.
|
|
@@ -3018,16 +3179,23 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3018
3179
|
if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
|
|
3019
3180
|
const record = user.record ?? "ci";
|
|
3020
3181
|
if (!RECORD.includes(record)) throw new UsageError(`record in ${file} is ${oneOf(RECORD)} (got ${JSON.stringify(record)})`);
|
|
3021
|
-
const
|
|
3022
|
-
|
|
3023
|
-
const historyDir = resolve(root, user.history?.dir ?? join(evalRoot(include), ".history"));
|
|
3182
|
+
const historyDefault = join(evalRoot(include), ".history");
|
|
3183
|
+
const history = resolveHistory(user.history, "history", file, root, historyDefault);
|
|
3024
3184
|
const harness = user.harnesses === void 0 ? [DEFAULT_HARNESS] : [user.harnesses].flat();
|
|
3025
3185
|
if (harness.length === 0) throw new UsageError(`harnesses is an empty list in ${file}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
|
|
3026
3186
|
checkHarnesses(harness, file);
|
|
3027
3187
|
const profiles = {};
|
|
3188
|
+
const profileHistory = {};
|
|
3028
3189
|
if (user.profiles !== void 0) {
|
|
3029
3190
|
if (typeof user.profiles !== "object" || user.profiles === null) throw new UsageError(`profiles in ${file} must be an object of named profiles, e.g. profiles: { pr: { limit: { runs: 1 } } } (got ${JSON.stringify(user.profiles)})`);
|
|
3030
|
-
for (const [name, profile] of Object.entries(user.profiles))
|
|
3191
|
+
for (const [name, profile] of Object.entries(user.profiles)) {
|
|
3192
|
+
const { history: own, ...checked } = checkProfile(name, profile, file);
|
|
3193
|
+
profiles[name] = checked;
|
|
3194
|
+
if (own !== void 0) {
|
|
3195
|
+
const where = `profiles.${name}.history`;
|
|
3196
|
+
profileHistory[name] = resolveHistory(own, where, file, root, historyDefault, history);
|
|
3197
|
+
}
|
|
3198
|
+
}
|
|
3031
3199
|
}
|
|
3032
3200
|
const defaultProfile = user.defaultProfile ?? {};
|
|
3033
3201
|
unknownKey(defaultProfile, ["ci", "local"], "defaultProfile", file);
|
|
@@ -3063,10 +3231,8 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3063
3231
|
}, user.snapshot),
|
|
3064
3232
|
concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
|
|
3065
3233
|
record,
|
|
3066
|
-
history
|
|
3067
|
-
|
|
3068
|
-
artifacts
|
|
3069
|
-
},
|
|
3234
|
+
history,
|
|
3235
|
+
profileHistory,
|
|
3070
3236
|
secrets: checkSecrets(user.secrets, file),
|
|
3071
3237
|
debug: checkDebug(user.debug, file),
|
|
3072
3238
|
analysis: checkAnalysis(user.analysis, file),
|
|
@@ -3146,11 +3312,11 @@ function checkConcurrency(value, where) {
|
|
|
3146
3312
|
if (typeof n !== "number" || !Number.isInteger(n) || n < 1) throw new UsageError(`${where} must be a whole number of at least 1 (got ${String(value)})`);
|
|
3147
3313
|
return n;
|
|
3148
3314
|
}
|
|
3149
|
-
/** `--concurrency`, then `REEVAL_CONCURRENCY`, then `concurrency` in the config,
|
|
3150
|
-
|
|
3315
|
+
/** `--concurrency`, then `REEVAL_CONCURRENCY`, then `concurrency` in the config, or undefined. */
|
|
3316
|
+
function givenConcurrency(config, flag, env = process.env) {
|
|
3151
3317
|
if (flag !== void 0) return checkConcurrency(flag, "--concurrency");
|
|
3152
3318
|
if (env.REEVAL_CONCURRENCY) return checkConcurrency(env.REEVAL_CONCURRENCY, "REEVAL_CONCURRENCY");
|
|
3153
|
-
return config.concurrency
|
|
3319
|
+
return config.concurrency;
|
|
3154
3320
|
}
|
|
3155
3321
|
function resolveJudge(judge, file, path = "judge") {
|
|
3156
3322
|
judge ??= {};
|
|
@@ -3192,8 +3358,8 @@ function missingCredentials(config, env = process.env, options = {}) {
|
|
|
3192
3358
|
const line = `${s.apiKey.from} (for boat())`;
|
|
3193
3359
|
if (!missing.includes(line)) missing.push(line);
|
|
3194
3360
|
}
|
|
3195
|
-
const keys = new
|
|
3196
|
-
for (const
|
|
3361
|
+
const keys = new Map((options.harnesses ?? config.harness).map(harnessCredentials).filter((names) => names.length > 0).map((names) => [names.join(" or "), names]));
|
|
3362
|
+
for (const [line, names] of keys) if (!names.some((k) => env[k])) missing.push(line);
|
|
3197
3363
|
const judge = runJudgeModel(config, options.profile);
|
|
3198
3364
|
const models = [
|
|
3199
3365
|
...typeof judge === "string" ? [{
|
|
@@ -3239,7 +3405,7 @@ function missingCredentials(config, env = process.env, options = {}) {
|
|
|
3239
3405
|
* then hangs on a request it can't send rather than failing to authenticate.
|
|
3240
3406
|
*/
|
|
3241
3407
|
function malformedCredentials(config, env = process.env, options = {}) {
|
|
3242
|
-
const names = (options.harnesses ?? config.harness).flatMap((h) =>
|
|
3408
|
+
const names = (options.harnesses ?? config.harness).flatMap((h) => harnessCredentials(h).find((name) => env[name]) ?? []);
|
|
3243
3409
|
const judge = runJudgeModel(config, options.profile);
|
|
3244
3410
|
if (judge !== void 0) names.push(...judgeCredentials(judge, runModelGateway(config, options.profile, judge, true)));
|
|
3245
3411
|
for (const { model, gateway } of options.judges ?? []) names.push(...judgeCredentials(model, gateway));
|
|
@@ -3249,6 +3415,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
|
|
|
3249
3415
|
return at ? [`${name} (character ${at})`] : [];
|
|
3250
3416
|
});
|
|
3251
3417
|
}
|
|
3252
|
-
export {
|
|
3418
|
+
export { plannedSandboxIds as $, resolveJudgeArtifacts as $t, attemptSandbox as A, UNRECORDED_QUESTION_STATE as At, evalPlan as B, attemptRequestProblem as Bt, fallbackHint as C, phaseLabel as Ct, COMPARE_DEFAULT_RUNS as D, unenforceable as Dt, routedHarness as E, setupEgress as Et, checkGate as F, observedHere as Ft, matchesTags as G, INPUT_KEYS as Gt, harnessId as H, harnessEffort as Ht, checkHarnesses as I, observedName as It, plainEnvError as J, judgeProblem as Jt, mergeSnapshot as K, NO_JUDGE_INPUT as Kt, checkLimitHarness as L, observer as Lt, cascade as M, resolveQuestionState as Mt, cellKey as N, mergeJudge as Nt, EVAL_OPTION_KEYS as O, resolveOutput as Ot, checkCapture as P, resolveStateOptions as Pt, plainHeaders as Q, filePartsOf as Qt, checkOptions as R, CAPTURE_NAME as Rt, effortRefusedOn as S, layerEgress as St, resolveGateway as T, rootlessPodmanSwitch as Tt, levelOf as U, resolvedEffortOf as Ut, expand as V, execOptionsProblem as Vt, majorityOf as W, servedEffortOf as Wt, plainHeaderError as X, badJudgeArtifacts as Xt, plainEnvWarning as Y, artifactRefsOf as Yt, plainHeaderWarning as Z, decideState as Zt, withProfileHistory as _, directOnCloud as _t, configPathFor as a, sandboxProvider as at, modelGatewayOrNone as b, isDirect as bt, findConfig as c, secretVarsIn as ct, loadConfig as d, unbrokeredWarning as dt, plannedSandboxes as et, malformedCredentials as f, DEFAULT_EGRESS as ft, selectProfile as g, ciRunner as gt, runJudgeModel as h, bareEgress as ht, configLabel as i, sandboxId as it, brokeredIn as j, questionShows as jt, MAJORITY_GATE as k, STOPPED_FIRST as kt, givenConcurrency as l, unbrokered as lt, requireConfig as m, allowedHosts as mt, analysisModelOf as n, refuseIllegalCompare as nt, defaultConcurrency as o, sandboxSetBy as ot, missingCredentials as p, DEFAULT_EGRESS_ALLOW as pt, plainEnv as q, inputsOn as qt, configFilesPhrase as r, runsOf as rt, defineConfig as s, sandboxWarnings as st, CONFIG_FILES as t, profileJudge as tt, hostCapacity as u, unbrokeredError as ut, fellBackToDirect as v, egressLabel as vt, openCodeModelOn as w, phasePolicies as wt, resolveModelGateway as x, isPhased as xt, gatewayScope as y, gatewayPolicy as yt, environmentWarnings as z, CAPTURE_NAME_RULE as zt };
|
|
3253
3419
|
|
|
3254
|
-
//# sourceMappingURL=config-
|
|
3420
|
+
//# sourceMappingURL=config-C0URar0t.js.map
|