reeval 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/README.md +15 -15
- package/dist/{adapter-BdzhdRYo.js → adapter-Dlq3-4OS.js} +7 -50
- package/dist/adapter-Dlq3-4OS.js.map +1 -0
- package/dist/{agent-Dx-bDmlV.js → agent-OsAfayRf.js} +11 -12
- package/dist/agent-OsAfayRf.js.map +1 -0
- package/dist/{agents-l0-k7vw3.js → agents-BztvTkfM.js} +46 -37
- package/dist/agents-BztvTkfM.js.map +1 -0
- package/dist/{app-files-B-CQYB0c.js → app-files-6NHzghym.js} +3 -3
- package/dist/app-files-6NHzghym.js.map +1 -0
- package/dist/{assemble-CsZIcJo8.js → assemble-Doa1m-HW.js} +17 -15
- package/dist/{assemble-CsZIcJo8.js.map → assemble-Doa1m-HW.js.map} +1 -1
- package/dist/{assemble-CYMpN8LC.js → assemble-XrZskSNj.js} +1 -1
- package/dist/attach-BH2HdwH_.js +219 -0
- package/dist/attach-BH2HdwH_.js.map +1 -0
- package/dist/{attempt-CKfKsqqw.js → attempt-_dUsblVx.js} +2 -2
- package/dist/{attempt-CKfKsqqw.js.map → attempt-_dUsblVx.js.map} +1 -1
- package/dist/{attempt-host-BwSmNYkX.js → attempt-host-8mQV8Uee.js} +7 -7
- package/dist/{attempt-host-BwSmNYkX.js.map → attempt-host-8mQV8Uee.js.map} +1 -1
- package/dist/boat-DKRH0Gv7.js +1 -0
- package/dist/boat-DKjqo_ie.js +704 -0
- package/dist/boat-DKjqo_ie.js.map +1 -0
- package/dist/boat-images-CRN_U_eG.js +366 -0
- package/dist/boat-images-CRN_U_eG.js.map +1 -0
- package/dist/bundle-DU2aQVeq.js +780 -0
- package/dist/bundle-DU2aQVeq.js.map +1 -0
- package/dist/{capture-BqIAeFyC.js → capture-By1byHaX.js} +52 -27
- package/dist/capture-By1byHaX.js.map +1 -0
- package/dist/capture-BzjNv0qP.js +1 -0
- package/dist/{cli-s_3opn_I.js → cli-DMHOwdVr.js} +560 -1049
- package/dist/cli-DMHOwdVr.js.map +1 -0
- package/dist/{cli-spec-DxzIjmq_.js → cli-spec-D0wuUaj9.js} +179 -42
- package/dist/cli-spec-D0wuUaj9.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-C5fIzHQa.js → client-B41UK150.js} +8 -4
- package/dist/client-B41UK150.js.map +1 -0
- package/dist/{command-BJ7KqczP.js → command-8GIHYGZf.js} +9 -9
- package/dist/{command-BJ7KqczP.js.map → command-8GIHYGZf.js.map} +1 -1
- package/dist/{command-B5OAIjp7.js → command-CAmnWLFL.js} +14 -14
- package/dist/{command-B5OAIjp7.js.map → command-CAmnWLFL.js.map} +1 -1
- package/dist/{command-BXLZsEwQ.js → command-CXxckfsq.js} +13 -13
- package/dist/{command-BXLZsEwQ.js.map → command-CXxckfsq.js.map} +1 -1
- package/dist/{command-DA_SHQUQ.js → command-ao0NZ8h0.js} +16 -15
- package/dist/command-ao0NZ8h0.js.map +1 -0
- package/dist/{commands-BPvAOEmb.js → commands-B3lnIPj5.js} +1 -1
- package/dist/{commands-BKBamOcX.js → commands-ttChCsAy.js} +15 -15
- package/dist/{commands-BKBamOcX.js.map → commands-ttChCsAy.js.map} +1 -1
- package/dist/{config-C0URar0t.js → config-myJT1Qjc.js} +62 -37
- package/dist/config-myJT1Qjc.js.map +1 -0
- package/dist/{context-Sj58aJEu.js → context-DnzR9TNK.js} +2 -2
- package/dist/{context-Sj58aJEu.js.map → context-DnzR9TNK.js.map} +1 -1
- package/dist/cost-CdRX1s2p.js +1 -0
- package/dist/{cursor-catalog-FWgSo-T1.js → cursor-catalog-CUQrabJ_.js} +2 -2
- package/dist/{cursor-catalog-FWgSo-T1.js.map → cursor-catalog-CUQrabJ_.js.map} +1 -1
- package/dist/daytona-Dfhwtr9P.js +677 -0
- package/dist/daytona-Dfhwtr9P.js.map +1 -0
- package/dist/daytona-image-CnG6G00x.js +183 -0
- package/dist/daytona-image-CnG6G00x.js.map +1 -0
- package/dist/daytona-image-yJK3c4EV.js +1 -0
- package/dist/{db-BjAWxVAG.js → db-5H95sFii.js} +35 -9
- package/dist/{db-BjAWxVAG.js.map → db-5H95sFii.js.map} +1 -1
- package/dist/db-Bbg1n_2M.js +2 -0
- package/dist/{default-effort-CWlhZ6U1.js → default-effort-CtNY9CGW.js} +1 -1
- package/dist/{default-effort-Ck2GYIxX.js → default-effort-DYpCkJRk.js} +2 -2
- package/dist/{default-effort-Ck2GYIxX.js.map → default-effort-DYpCkJRk.js.map} +1 -1
- package/dist/{docker-D74l67k8.js → docker-CR0JZ1FE.js} +4 -4
- package/dist/{docker-D74l67k8.js.map → docker-CR0JZ1FE.js.map} +1 -1
- package/dist/docker-kpudeSxV.js +1 -0
- package/dist/{doctor-eFxyn6at.js → doctor-aE-RMbB_.js} +63 -76
- package/dist/doctor-aE-RMbB_.js.map +1 -0
- package/dist/{environment-CJIKz-Ud.js → environment-C8pUCD_A.js} +138 -53
- package/dist/environment-C8pUCD_A.js.map +1 -0
- package/dist/{errors-C1GBcWE2.js → errors-DPUoUTdK.js} +2 -2
- package/dist/{errors-C1GBcWE2.js.map → errors-DPUoUTdK.js.map} +1 -1
- package/dist/{eval-source-CXIRJ91v.js → eval-source-CaLIGH3U.js} +6 -6
- package/dist/{eval-source-CXIRJ91v.js.map → eval-source-CaLIGH3U.js.map} +1 -1
- package/dist/{evaluate-DA4QsrHO.js → evaluate-4kBVtwZZ.js} +250 -215
- package/dist/evaluate-4kBVtwZZ.js.map +1 -0
- package/dist/{evaluate-DHrU4Wld.d.ts → evaluate-DHaNtgVf.d.ts} +2 -2
- package/dist/events-CXxFvVMK.js +475 -0
- package/dist/events-CXxFvVMK.js.map +1 -0
- package/dist/{fake-bootstrap-CnwWG3kP.js → fake-bootstrap-CzxrCqO3.js} +2 -2
- package/dist/{fake-bootstrap-CnwWG3kP.js.map → fake-bootstrap-CzxrCqO3.js.map} +1 -1
- package/dist/files-BFnkOAhn.js +1 -0
- package/dist/files-BgQxoDlN.js +1 -0
- package/dist/{files-D6j-XpaK.js → files-BkvxIqfI.js} +4 -4
- package/dist/{files-D6j-XpaK.js.map → files-BkvxIqfI.js.map} +1 -1
- package/dist/{files-g1watgRC.js → files-CDfG8swM.js} +2 -2
- package/dist/{files-g1watgRC.js.map → files-CDfG8swM.js.map} +1 -1
- package/dist/files-CSe4VaMj.js +1 -0
- package/dist/{files-F44NRCN0.js → files-CX_mmvCc.js} +8 -8
- package/dist/{files-F44NRCN0.js.map → files-CX_mmvCc.js.map} +1 -1
- package/dist/{files-BrGVs4gY.js → files-Cs2dtb7-.js} +2 -2
- package/dist/{files-BrGVs4gY.js.map → files-Cs2dtb7-.js.map} +1 -1
- package/dist/{files-BaqXvB88.js → files-D6M7J3iX.js} +1 -1
- package/dist/{files-BVVWAS0A.js → files-DgHeTdCZ.js} +6 -6
- package/dist/{files-BVVWAS0A.js.map → files-DgHeTdCZ.js.map} +1 -1
- package/dist/{files-Bey1VAex.js → files-DpwJW3w6.js} +3 -3
- package/dist/{files-Bey1VAex.js.map → files-DpwJW3w6.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-Jqkf9SBa.js → git-BnnNEXqh.js} +3 -3
- package/dist/{git-Jqkf9SBa.js.map → git-BnnNEXqh.js.map} +1 -1
- package/dist/harness-install-BfMBYDfP.js +53 -0
- package/dist/harness-install-BfMBYDfP.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/hosted-5Zz2peTR.js +260 -0
- package/dist/hosted-5Zz2peTR.js.map +1 -0
- package/dist/hosted-Co-waHe0.js +131 -0
- package/dist/hosted-Co-waHe0.js.map +1 -0
- package/dist/hosted-run-202QicZ-.js +1 -0
- package/dist/hosted-run-BfAPIMyT.js +230 -0
- package/dist/hosted-run-BfAPIMyT.js.map +1 -0
- package/dist/{ids-B-JKTCP8.js → ids-a6cr243_.js} +17 -12
- package/dist/ids-a6cr243_.js.map +1 -0
- package/dist/{images-Dsc4Wi-F.js → images-DtXwj2wL.js} +70 -55
- package/dist/images-DtXwj2wL.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +5 -5
- package/dist/init-BiL6B3S4.js +1 -0
- package/dist/{init-QB5ecPDA.js → init-Ci_IAp8p.js} +5 -5
- package/dist/{init-QB5ecPDA.js.map → init-Ci_IAp8p.js.map} +1 -1
- package/dist/{ink-list-mCJypk3d.js → ink-list-Dnh6XD3C.js} +54 -15
- package/dist/ink-list-Dnh6XD3C.js.map +1 -0
- package/dist/inputs-load-DPUP_IFL.js +95 -0
- package/dist/inputs-load-DPUP_IFL.js.map +1 -0
- package/dist/inputs-load-M-BGe7gd.js +1 -0
- package/dist/inputs-z4JqnQ5g.js +190 -0
- package/dist/inputs-z4JqnQ5g.js.map +1 -0
- package/dist/internal/vm.d.ts +16 -0
- package/dist/internal/vm.js +22 -0
- package/dist/internal/vm.js.map +1 -0
- package/dist/{invocations-BSr4lRzI.js → invocations-C1gY3E0J.js} +4 -4
- package/dist/{invocations-BSr4lRzI.js.map → invocations-C1gY3E0J.js.map} +1 -1
- package/dist/jiti-ILwyxt93.js +54 -0
- package/dist/jiti-ILwyxt93.js.map +1 -0
- package/dist/{judge-eGH39PaP.js → judge-DwzqucLS.js} +9 -6
- package/dist/{judge-eGH39PaP.js.map → judge-DwzqucLS.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/kinds-DEKXOWVJ.js +592 -0
- package/dist/kinds-DEKXOWVJ.js.map +1 -0
- package/dist/{layout-DfQa4mJs.js → layout-nCnU9Vu2.js} +2 -2
- package/dist/{layout-DfQa4mJs.js.map → layout-nCnU9Vu2.js.map} +1 -1
- package/dist/{list-Dh6T0kdr.js → list-MK4L8tfW.js} +14 -11
- package/dist/{list-Dh6T0kdr.js.map → list-MK4L8tfW.js.map} +1 -1
- package/dist/{live-y1p3oNXH.js → live-BkPNXVmN.js} +8 -8
- package/dist/{live-y1p3oNXH.js.map → live-BkPNXVmN.js.map} +1 -1
- package/dist/{live-xTZs0c61.js → live-Dlq-S2yy.js} +7 -5
- package/dist/live-Dlq-S2yy.js.map +1 -0
- package/dist/live-gZtkBwuo.js +1 -0
- package/dist/machine-service-Byl0BlcT.js +1 -0
- package/dist/machine-service-CZiXiVsS.js +709 -0
- package/dist/machine-service-CZiXiVsS.js.map +1 -0
- package/dist/machines-Ch5bhgvk.js +279 -0
- package/dist/machines-Ch5bhgvk.js.map +1 -0
- package/dist/model-C6aoJLg_.js +1 -0
- package/dist/{model-jRKSQBg6.js → model-CRh8aiYS.js} +6 -5
- package/dist/model-CRh8aiYS.js.map +1 -0
- package/dist/{model-3pZZzloU.js → model-DRa7yv6K.js} +6 -5
- package/dist/model-DRa7yv6K.js.map +1 -0
- package/dist/{model-DNpKiHqO.js → model-LoLnq9y1.js} +2 -2
- package/dist/{model-DNpKiHqO.js.map → model-LoLnq9y1.js.map} +1 -1
- package/dist/{model-DEjwetkJ.js → model-lLaUxxno.js} +1 -1
- package/dist/{node-DM-mLGdX.js → node-BQjSQw0d.js} +19 -8
- package/dist/node-BQjSQw0d.js.map +1 -0
- package/dist/{openrouter-models-C0aRo633.js → openrouter-models-B1sIOTD2.js} +2 -2
- package/dist/{openrouter-models-C0aRo633.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
- package/dist/options-I9I3FwMl.js +1 -0
- package/dist/own-reads-CYmBlrHK.js +15 -0
- package/dist/own-reads-CYmBlrHK.js.map +1 -0
- package/dist/{patch-Bx-qp8Li.js → patch-D8X6VB-e.js} +2 -2
- package/dist/{patch-Bx-qp8Li.js.map → patch-D8X6VB-e.js.map} +1 -1
- package/dist/{plan-ChO0A-xn.js → plan-DgIVIrXJ.js} +47 -9
- package/dist/plan-DgIVIrXJ.js.map +1 -0
- package/dist/{plugins-BioMWncI.js → plugins-DvEbow6a.js} +17 -637
- package/dist/plugins-DvEbow6a.js.map +1 -0
- package/dist/{preflight-CJSGzMzc.js → preflight-BKGbxN67.js} +9 -8
- package/dist/preflight-BKGbxN67.js.map +1 -0
- package/dist/{preflight-qFCXmxha.js → preflight-Qm4wE094.js} +9 -4
- package/dist/preflight-Qm4wE094.js.map +1 -0
- package/dist/preflight-dJqs5x3r.js +1 -0
- package/dist/preflight-fyV5nnea.js +1 -0
- package/dist/prepare-BAjRSoku.js +389 -0
- package/dist/prepare-BAjRSoku.js.map +1 -0
- package/dist/prepare-CRPPfRJf.js +2 -0
- package/dist/prices-C0gKS_8G.js +1 -0
- package/dist/{prices-BrY_ihLs.js → prices-DCokI_Ka.js} +3 -3
- package/dist/{prices-BrY_ihLs.js.map → prices-DCokI_Ka.js.map} +1 -1
- package/dist/{provenance-B-flBAzR.js → provenance-CSyRCc9B.js} +4 -4
- package/dist/{provenance-B-flBAzR.js.map → provenance-CSyRCc9B.js.map} +1 -1
- package/dist/{providers-B6StVyfG.js → providers-DkaEJvF5.js} +185 -35
- package/dist/providers-DkaEJvF5.js.map +1 -0
- package/dist/providers-DxZ26YoG.js +1 -0
- package/dist/{query-CkLmJv7R.js → query-BL66BOqL.js} +2 -2
- package/dist/{query-CkLmJv7R.js.map → query-BL66BOqL.js.map} +1 -1
- package/dist/query-C9OKU1Vz.js +1 -0
- package/dist/{questions-CJIh2FSX.js → questions-boVPzGoC.js} +7 -7
- package/dist/{questions-CJIh2FSX.js.map → questions-boVPzGoC.js.map} +1 -1
- package/dist/recipe-qyIpP96r.js +128 -0
- package/dist/recipe-qyIpP96r.js.map +1 -0
- package/dist/{record-nVVujCW7.js → record-D1_D1KFY.js} +12 -4
- package/dist/record-D1_D1KFY.js.map +1 -0
- package/dist/{redact-DL2BTykw.js → redact-t-L3VGy8.js} +91 -2
- package/dist/redact-t-L3VGy8.js.map +1 -0
- package/dist/{remarks-B3wjKscW.js → remarks-DEvk5f8v.js} +7 -7
- package/dist/{remarks-B3wjKscW.js.map → remarks-DEvk5f8v.js.map} +1 -1
- package/dist/{remote-C88amFRN.js → remote-CQoBRpOv.js} +5 -5
- package/dist/{remote-C88amFRN.js.map → remote-CQoBRpOv.js.map} +1 -1
- package/dist/resolve-BdpO8BN1.js +1 -0
- package/dist/{resolve-Bul8lHaa.js → resolve-Bfo8sjd-.js} +2 -2
- package/dist/{resolve-Bul8lHaa.js.map → resolve-Bfo8sjd-.js.map} +1 -1
- package/dist/{resolve-G576PiFJ.js → resolve-D2RqmybX.js} +9 -9
- package/dist/{resolve-G576PiFJ.js.map → resolve-D2RqmybX.js.map} +1 -1
- package/dist/resources-DqdwihhT.js +62 -0
- package/dist/resources-DqdwihhT.js.map +1 -0
- package/dist/{results-zVHIW6xw.js → results-BpxzyoMd.js} +5 -5
- package/dist/{results-zVHIW6xw.js.map → results-BpxzyoMd.js.map} +1 -1
- package/dist/{resume-D_PcUXfG.js → resume-2ekm_9OB.js} +78 -7
- package/dist/resume-2ekm_9OB.js.map +1 -0
- package/dist/{retry-BvTzCxdc.js → retry-CAhpW1hH.js} +4 -4
- package/dist/{retry-BvTzCxdc.js.map → retry-CAhpW1hH.js.map} +1 -1
- package/dist/{routes-B0F2R-K7.js → routes-Cp4gAjBZ.js} +118 -70
- package/dist/routes-Cp4gAjBZ.js.map +1 -0
- package/dist/{runtime-B3r_pAq6.js → runtime-C5fKSFCz.js} +46 -5
- package/dist/runtime-C5fKSFCz.js.map +1 -0
- package/dist/sandbox/boat.d.ts +11 -3
- package/dist/sandbox/boat.js +18 -4
- package/dist/sandbox/boat.js.map +1 -1
- package/dist/sandbox/daytona.d.ts +30 -0
- package/dist/sandbox/daytona.js +49 -0
- package/dist/sandbox/daytona.js.map +1 -0
- package/dist/{sandbox-Duh3z1TV.js → sandbox-t-Q-8lA_.js} +2 -2
- package/dist/{sandbox-Duh3z1TV.js.map → sandbox-t-Q-8lA_.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-DUWMRKik.js → server-BcLg9WzE.js} +10 -10
- package/dist/{server-DUWMRKik.js.map → server-BcLg9WzE.js.map} +1 -1
- package/dist/server-Buav2-CG.js +1 -0
- package/dist/services-BG8xuU82.js +39 -0
- package/dist/services-BG8xuU82.js.map +1 -0
- package/dist/{source-CHuN4Nfg.js → source-CrXM19HD.js} +2 -2
- package/dist/{source-CHuN4Nfg.js.map → source-CrXM19HD.js.map} +1 -1
- package/dist/source-dvxkqYvk.js +1 -0
- package/dist/src-5dCrqbFd.js +2 -0
- package/dist/{src-BI0v8qlZ.js → src-Bb3k9Kn_.js} +8 -5
- package/dist/{src-BI0v8qlZ.js.map → src-Bb3k9Kn_.js.map} +1 -1
- package/dist/{src-boB6D-5T.js → src-Bf5RZlPi.js} +2 -2
- package/dist/{src-boB6D-5T.js.map → src-Bf5RZlPi.js.map} +1 -1
- package/dist/{src-BqNzHHWn.js → src-C3OhA5pw.js} +2 -2
- package/dist/{src-BqNzHHWn.js.map → src-C3OhA5pw.js.map} +1 -1
- package/dist/{src-BHJ4MWvY.js → src-C5rrQlt8.js} +2199 -771
- package/dist/src-C5rrQlt8.js.map +1 -0
- package/dist/{src-CeBhwD8k.js → src-C6MsGyD6.js} +1 -1
- package/dist/{src-De0KY5S_.js → src-Cf4vRyea.js} +2 -2
- package/dist/src-DEbNp-rx.js +1 -0
- package/dist/{src-B-HuDqpH.js → src-DNWiU__5.js} +2 -2
- package/dist/{src-B-HuDqpH.js.map → src-DNWiU__5.js.map} +1 -1
- package/dist/src-DrLbG1HM.js +2 -0
- package/dist/{src--yl1OypK.js → src-Ds1pmXTg.js} +2 -2
- package/dist/{src-DA95rM6v.js → src-DwcCKsV0.js} +2 -2
- package/dist/{src-CmA5Yev6.js → src-DwcTn4JY.js} +2 -2
- package/dist/{src-CmA5Yev6.js.map → src-DwcTn4JY.js.map} +1 -1
- package/dist/{src-CZaYu6ER.js → src-VG2IRoT4.js} +1 -1
- package/dist/{src-DL-EE4DU.js → src-c3ZQu7Uu.js} +2 -2
- package/dist/{src-DL-EE4DU.js.map → src-c3ZQu7Uu.js.map} +1 -1
- package/dist/{src-CkEiiyON.js → src-io_SKoZ5.js} +2 -2
- package/dist/ssh-pool-BorCizYw.js +1 -0
- package/dist/ssh-pool-Dvfx5GuV.js +174 -0
- package/dist/ssh-pool-Dvfx5GuV.js.map +1 -0
- package/dist/start-COB07NpH.js +718 -0
- package/dist/start-COB07NpH.js.map +1 -0
- package/dist/{state-DUBRDyMk.js → state-B319V1p8.js} +16 -16
- package/dist/{state-DUBRDyMk.js.map → state-B319V1p8.js.map} +1 -1
- package/dist/state-C5oBvmoZ.js +1 -0
- package/dist/store/local.d.ts +1 -1
- package/dist/store/s3.d.ts +1 -1
- package/dist/suite-BwJGOhCZ.js +2 -0
- package/dist/{suite-B9J6Afg6.js → suite-D1tgXZxM.js} +2 -2
- package/dist/{suite-B9J6Afg6.js.map → suite-D1tgXZxM.js.map} +1 -1
- package/dist/{summary-CSp4cfnr.js → summary-ClLns_wv.js} +8 -5
- package/dist/summary-ClLns_wv.js.map +1 -0
- package/dist/{sync-CxAY6Dki.js → sync-mOG74RXH.js} +5 -5
- package/dist/{sync-CxAY6Dki.js.map → sync-mOG74RXH.js.map} +1 -1
- package/dist/{tart-CcrrtSrp.js → tart-DMppfbEm.js} +100 -295
- package/dist/tart-DMppfbEm.js.map +1 -0
- package/dist/{tart-bake-F-6seRC9.js → tart-bake-Cva_qdir.js} +18 -62
- package/dist/tart-bake-Cva_qdir.js.map +1 -0
- package/dist/{tart-bake-RK7RaT9E.js → tart-bake-DVIoXWr_.js} +1 -1
- package/dist/{tart-cli-ClvRniLQ.js → tart-cli-D88c5YYJ.js} +1 -1
- package/dist/{tart-cli-B86Jqo1t.js → tart-cli-LzDud2Z6.js} +2 -2
- package/dist/{tart-cli-B86Jqo1t.js.map → tart-cli-LzDud2Z6.js.map} +1 -1
- package/dist/{tart-hosts-Brii7zHM.js → tart-hosts-8N6Tt7Yb.js} +1 -1
- package/dist/{tart-hosts-DPyZsg-1.js → tart-hosts-CTRY5kQ2.js} +3 -3
- package/dist/{tart-hosts-DPyZsg-1.js.map → tart-hosts-CTRY5kQ2.js.map} +1 -1
- package/dist/{tart-images-CWSf03w8.js → tart-images-JfHirU8W.js} +2 -2
- package/dist/{tart-images-CWSf03w8.js.map → tart-images-JfHirU8W.js.map} +1 -1
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +4 -4
- package/dist/{tool-class-CVDdJy0H.js → tool-class-CncfCw03.js} +2 -2
- package/dist/{tool-class-CVDdJy0H.js.map → tool-class-CncfCw03.js.map} +1 -1
- package/dist/{transcript-BP_2hji-.js → transcript-BXWZiMuC.js} +2 -2
- package/dist/{transcript-BP_2hji-.js.map → transcript-BXWZiMuC.js.map} +1 -1
- package/dist/{types-BJ0wh8wC.d.ts → types-BDo8ttQo.d.ts} +253 -5
- package/dist/types-CbcnJxCy.js.map +1 -1
- package/dist/ui-DUiVPbQ2.js +1024 -0
- package/dist/ui-DUiVPbQ2.js.map +1 -0
- package/dist/vercel-models-Ci33llXz.js +21 -0
- package/dist/vercel-models-Ci33llXz.js.map +1 -0
- package/dist/view/index.d.ts +6 -1
- package/dist/view/index.js +4 -4
- package/dist/views-B0qZR5Ef.js +2 -0
- package/dist/{views-DRxXCyIH.js → views-CiZGh3ws.js} +6 -6
- package/dist/{views-DRxXCyIH.js.map → views-CiZGh3ws.js.map} +1 -1
- package/dist/vm-container-BVn7pXfD.js +148 -0
- package/dist/vm-container-BVn7pXfD.js.map +1 -0
- package/dist/vm-images-CNk-Zl2a.js +375 -0
- package/dist/vm-images-CNk-Zl2a.js.map +1 -0
- package/dist/{worker-entry-C3Q4U63F.js → worker-entry-DmXQ2Znu.js} +31 -11
- package/dist/worker-entry-DmXQ2Znu.js.map +1 -0
- package/dist/workspace-CRgFdmgW.js +1 -0
- package/dist/{workspace-P72SXLXr.js → workspace-wwvwEHzP.js} +6 -6
- package/dist/{workspace-P72SXLXr.js.map → workspace-wwvwEHzP.js.map} +1 -1
- package/docker/Dockerfile +1 -1
- package/package.json +17 -2
- package/skills/reeval/references/environments.md +4 -3
- package/dist/adapter-BdzhdRYo.js.map +0 -1
- package/dist/agent-Dx-bDmlV.js.map +0 -1
- package/dist/agents-l0-k7vw3.js.map +0 -1
- package/dist/app-files-B-CQYB0c.js.map +0 -1
- package/dist/boat-CZpAreTB.js +0 -108
- package/dist/boat-CZpAreTB.js.map +0 -1
- package/dist/boat-su1Y7LPd.js +0 -325
- package/dist/boat-su1Y7LPd.js.map +0 -1
- package/dist/capture-BqIAeFyC.js.map +0 -1
- package/dist/capture-D0Ln5aCP.js +0 -1
- package/dist/cli-s_3opn_I.js.map +0 -1
- package/dist/cli-spec-DxzIjmq_.js.map +0 -1
- package/dist/client-C5fIzHQa.js.map +0 -1
- package/dist/command-DA_SHQUQ.js.map +0 -1
- package/dist/config-C0URar0t.js.map +0 -1
- package/dist/cost-IM80LyGD.js +0 -1
- package/dist/db-Bc4L-Wcr.js +0 -2
- package/dist/docker-CDdK6Ity.js +0 -271
- package/dist/docker-CDdK6Ity.js.map +0 -1
- package/dist/docker-DMug3l-a.js +0 -1
- package/dist/doctor-eFxyn6at.js.map +0 -1
- package/dist/environment-CJIKz-Ud.js.map +0 -1
- package/dist/evaluate-DA4QsrHO.js.map +0 -1
- package/dist/files-BjOJw4sn.js +0 -1
- package/dist/files-CNHhSyb3.js +0 -1
- package/dist/files-CW1lChNh.js +0 -1
- package/dist/ids-B-JKTCP8.js.map +0 -1
- package/dist/images-Dsc4Wi-F.js.map +0 -1
- package/dist/init-C0oUG3jh.js +0 -1
- package/dist/ink-list-mCJypk3d.js.map +0 -1
- package/dist/live-DwjWJTdn.js +0 -1
- package/dist/live-xTZs0c61.js.map +0 -1
- package/dist/model-3pZZzloU.js.map +0 -1
- package/dist/model-jRKSQBg6.js.map +0 -1
- package/dist/model-nfewGTzB.js +0 -1
- package/dist/node-DM-mLGdX.js.map +0 -1
- package/dist/options-DpsJ_5CU.js +0 -1
- package/dist/plan-ChO0A-xn.js.map +0 -1
- package/dist/plugins-BioMWncI.js.map +0 -1
- package/dist/preflight-BUjk_6wT.js +0 -1
- package/dist/preflight-CJSGzMzc.js.map +0 -1
- package/dist/preflight-DD3fI4Hz.js +0 -1
- package/dist/preflight-qFCXmxha.js.map +0 -1
- package/dist/prepare-D7oSSFNm.js +0 -268
- package/dist/prepare-D7oSSFNm.js.map +0 -1
- package/dist/prepare-xHU2BpN-.js +0 -2
- package/dist/prices-B1vm_lPk.js +0 -1
- package/dist/providers-B6StVyfG.js.map +0 -1
- package/dist/providers-Dljf6al6.js +0 -1
- package/dist/query-BAc2tQZj.js +0 -1
- package/dist/record-nVVujCW7.js.map +0 -1
- package/dist/redact-DL2BTykw.js.map +0 -1
- package/dist/resolve-DW92wpIZ.js +0 -1
- package/dist/resume-D_PcUXfG.js.map +0 -1
- package/dist/routes-B0F2R-K7.js.map +0 -1
- package/dist/runtime-B3r_pAq6.js.map +0 -1
- package/dist/server-CGEsvVRG.js +0 -1
- package/dist/source-Wz1Nrfue.js +0 -1
- package/dist/src-BHJ4MWvY.js.map +0 -1
- package/dist/src-BL-_0QoE.js +0 -1
- package/dist/src-C-mOSczH.js +0 -2
- package/dist/src-CwH_nEaw.js +0 -2
- package/dist/state-B3jex5KU.js +0 -1
- package/dist/suite-Bhs9-Z7O.js +0 -2
- package/dist/summary-CSp4cfnr.js.map +0 -1
- package/dist/tarball-CESxtK4X.js +0 -257
- package/dist/tarball-CESxtK4X.js.map +0 -1
- package/dist/tart-CcrrtSrp.js.map +0 -1
- package/dist/tart-bake-F-6seRC9.js.map +0 -1
- package/dist/views-i67V1ly3.js +0 -2
- package/dist/worker-entry-C3Q4U63F.js.map +0 -1
- package/dist/workspace-DhUOW8if.js +0 -1
|
@@ -1,49 +1,217 @@
|
|
|
1
1
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
|
|
2
2
|
import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-DtZwIAfy.js";
|
|
3
|
-
import {
|
|
3
|
+
import { Xt as OPAQUE_OUTPUT, dn as shellQuote$1, in as readTar, p as MCP_TIMER, rn as extractTar, sn as writeTar } from "./src-C5rrQlt8.js";
|
|
4
4
|
import { t as evalSlug } from "./slug-IHh3f770.js";
|
|
5
|
-
import { c as
|
|
6
|
-
import { _ as
|
|
7
|
-
import {
|
|
5
|
+
import { c as recordFileName, i as evalId, l as relativeFile, n as attemptKey, t as attemptId } from "./ids-a6cr243_.js";
|
|
6
|
+
import { _ as redactSplitSecrets, d as maskedVariables, f as redactFields, g as redactSecrets, o as findSecrets, p as redactJson, y as redactValue } from "./redact-t-L3VGy8.js";
|
|
7
|
+
import { n as capabilitiesOf, o as providerLimit, u as sandboxKind } from "./kinds-DEKXOWVJ.js";
|
|
8
8
|
import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
|
|
9
|
-
import { d as
|
|
9
|
+
import { S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, k as pluginSourceLabel, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-DvEbow6a.js";
|
|
10
|
+
import { h as vercelCredentialProblem, m as vercelBaseUrlProblem, p as vercelBaseUrl } from "./routes-Cp4gAjBZ.js";
|
|
10
11
|
import { v as vertexSettings } from "./vertex-BEWxHoxH.js";
|
|
11
12
|
import { n as hasNetworkSession, t as InfraError } from "./types-CbcnJxCy.js";
|
|
12
|
-
import { C as writePluginFiles, S as
|
|
13
|
+
import { C as writePluginFiles, S as refusedSharedMachine, b as refusedMcp, c as agentMcp, d as brokeringFor, f as claudeMcpServers, h as fakeBrokering, l as agentOf, m as expectedBlockedFor, t as AGENTS, v as keyRejected, x as refusedPlugins, y as layoutOf } from "./agents-BztvTkfM.js";
|
|
13
14
|
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
14
|
-
import { d as limitsLabel, f as runMatchLabel, r as withLease, s as judgeFailedMessage, u as TURN_EDITED } from "./client-
|
|
15
|
-
import {
|
|
16
|
-
import { $
|
|
17
|
-
import { f as runJudge, g as isTransient, h as isApiError, l as judgeModelName } from "./judge-
|
|
15
|
+
import { d as limitsLabel, f as runMatchLabel, r as withLease, s as judgeFailedMessage, u as TURN_EDITED } from "./client-B41UK150.js";
|
|
16
|
+
import { A as sandboxPaths, I as workspaceOf, N as traitsOf, f as filesOf, i as callerDir, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-C8pUCD_A.js";
|
|
17
|
+
import { $ as plainHeaders, $t as decideState, E as resolveGateway, Ft as mergeJudge, G as majorityOf, H as expand, It as resolveStateOptions, J as plainEnv, Jt as NO_JUDGE_INPUT, K as matchesTags, Lt as observedHere, M as brokeredIn, N as cascade, Qt as badJudgeArtifacts, Rt as observedName, S as resolveModelGateway, Tt as phaseLabel, U as harnessId, V as evalPlan, Xt as judgeProblem, Y as plainEnvError, Yt as inputsOn, Z as plainHeaderError, Zt as artifactRefsOf, at as sandboxId, b as gatewayScope, bt as gatewayPolicy, dt as unbrokeredError, en as filePartsOf, ht as allowedHosts, it as runsOf, kt as unenforceable, lt as secretVarsIn, ot as sandboxProvider, qt as INPUT_KEYS, rt as refuseIllegalCompare, st as sandboxSetBy, tn as resolveJudgeArtifacts, ut as unbrokered, xt as isDirect, y as fellBackToDirect, yt as egressLabel, z as checkOptions, zt as observer } from "./config-myJT1Qjc.js";
|
|
18
|
+
import { f as runJudge, g as isTransient, h as isApiError, l as judgeModelName } from "./judge-DwzqucLS.js";
|
|
18
19
|
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
19
|
-
import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-
|
|
20
|
+
import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-C5fKSFCz.js";
|
|
20
21
|
import { c as formatBytes } from "./debug-CIXzB4wK.js";
|
|
21
|
-
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
22
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-_dUsblVx.js";
|
|
23
|
+
import { t as attemptDebugDir } from "./files-CDfG8swM.js";
|
|
24
|
+
import { r as openRouterFallbacks } from "./openrouter-models-B1sIOTD2.js";
|
|
25
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-By1byHaX.js";
|
|
26
|
+
import { t as contextTaker } from "./context-DnzR9TNK.js";
|
|
27
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-D2RqmybX.js";
|
|
28
|
+
import { n as fakeBootstrapScript } from "./fake-bootstrap-CzxrCqO3.js";
|
|
24
29
|
import { n as providerStop, t as ProviderStopError } from "./provider-stop-CxzWW7Ft.js";
|
|
25
30
|
import { t as PrepareError } from "./provider-ieWOn4YS.js";
|
|
26
|
-
import { i as preparedWorkspace } from "./workspace-
|
|
27
|
-
import { t as attemptDebugDir } from "./files-g1watgRC.js";
|
|
28
|
-
import { r as openRouterFallbacks } from "./openrouter-models-C0aRo633.js";
|
|
29
|
-
import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-BqIAeFyC.js";
|
|
30
|
-
import { t as contextTaker } from "./context-Sj58aJEu.js";
|
|
31
|
+
import { i as preparedWorkspace } from "./workspace-wwvwEHzP.js";
|
|
31
32
|
import { t as patchPath } from "./patch-C7yBztZj.js";
|
|
32
|
-
import { n as judgeDiff } from "./patch-
|
|
33
|
-
import { t as transcriptForJudge } from "./transcript-
|
|
33
|
+
import { n as judgeDiff } from "./patch-D8X6VB-e.js";
|
|
34
|
+
import { t as transcriptForJudge } from "./transcript-BXWZiMuC.js";
|
|
34
35
|
import { n as modelCallEvents } from "./model-calls-B0BoQfJ4.js";
|
|
35
|
-
import {
|
|
36
|
-
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-
|
|
37
|
-
import {
|
|
36
|
+
import { t as resourcesLine } from "./resources-DqdwihhT.js";
|
|
37
|
+
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-CaLIGH3U.js";
|
|
38
|
+
import { t as environmentInputs } from "./inputs-z4JqnQ5g.js";
|
|
39
|
+
import { i as withDefaultEffort } from "./default-effort-DYpCkJRk.js";
|
|
38
40
|
import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-CCD8XsmI.js";
|
|
39
41
|
import { z } from "zod";
|
|
40
42
|
import { dirname, join } from "node:path";
|
|
41
43
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
44
|
+
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
42
45
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
43
46
|
import { parse, stringify } from "yaml";
|
|
44
|
-
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
45
47
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
46
48
|
import { resolveBridgePort } from "@ai-sdk/harness/utils";
|
|
49
|
+
// src/attempt/assertions.ts
|
|
50
|
+
const MESSAGE_LIMIT = 1024;
|
|
51
|
+
const LABEL_LIMIT = 60;
|
|
52
|
+
/** Masked first, so a secret the limit would cut through is masked whole. */
|
|
53
|
+
const clipped = (text, limit) => {
|
|
54
|
+
const s = redactSecrets(text);
|
|
55
|
+
return s.length > limit ? `${s.slice(0, limit)}…` : s;
|
|
56
|
+
};
|
|
57
|
+
const text = (error) => error instanceof Error ? error.message : String(error);
|
|
58
|
+
const shown = (value) => {
|
|
59
|
+
if (typeof value === "string") return value;
|
|
60
|
+
if (value instanceof RegExp) return String(value);
|
|
61
|
+
try {
|
|
62
|
+
return JSON.stringify(value) ?? String(value);
|
|
63
|
+
} catch {
|
|
64
|
+
return String(value);
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
|
|
68
|
+
/**
|
|
69
|
+
* The records of an attempt's matcher calls, in call order. Without a label, one is
|
|
70
|
+
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
|
|
71
|
+
* any label already recorded, so each record's label is unique (a given `build #2` after two
|
|
72
|
+
* `build`s is `build #2 #2`).
|
|
73
|
+
*/
|
|
74
|
+
function assertionRecords(raw) {
|
|
75
|
+
const seen = /* @__PURE__ */ new Map();
|
|
76
|
+
const used = /* @__PURE__ */ new Set();
|
|
77
|
+
return raw.map((r) => {
|
|
78
|
+
const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.matcher === "toHaveRun" && runMatchLabel(r.args[0]) !== void 0 ? runMatchLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
79
|
+
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
80
|
+
let n = seen.get(base) ?? 0;
|
|
81
|
+
let label;
|
|
82
|
+
do {
|
|
83
|
+
n += 1;
|
|
84
|
+
label = n > 1 ? `${base} #${n}` : base;
|
|
85
|
+
} while (used.has(label));
|
|
86
|
+
seen.set(base, n);
|
|
87
|
+
used.add(label);
|
|
88
|
+
return {
|
|
89
|
+
label,
|
|
90
|
+
labelled: r.label !== void 0,
|
|
91
|
+
matcher: r.matcher,
|
|
92
|
+
status: r.status,
|
|
93
|
+
soft: r.soft,
|
|
94
|
+
...r.at ? { at: r.at } : {},
|
|
95
|
+
...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
|
|
96
|
+
...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
|
|
97
|
+
};
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
101
|
+
/**
|
|
102
|
+
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
103
|
+
* that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
|
|
104
|
+
* `CAUGHT_HINT` after its message.
|
|
105
|
+
*/
|
|
106
|
+
function checkFailure(raw) {
|
|
107
|
+
const failed = raw.find((r) => r.status !== "pass");
|
|
108
|
+
if (!failed) return void 0;
|
|
109
|
+
const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
|
|
110
|
+
if (failed.soft) return error;
|
|
111
|
+
const message = `${error.message} ${CAUGHT_HINT}`;
|
|
112
|
+
const stack = error.stack?.replace(error.message, () => message);
|
|
113
|
+
if (error instanceof DOMException) return Object.defineProperties(new Error(message, { cause: error }), {
|
|
114
|
+
name: own(error.name),
|
|
115
|
+
stack: own(stack)
|
|
116
|
+
});
|
|
117
|
+
return Object.setPrototypeOf(Object.defineProperties(/* @__PURE__ */ new Error(), {
|
|
118
|
+
...Object.getOwnPropertyDescriptors(error),
|
|
119
|
+
name: own(error.name),
|
|
120
|
+
message: own(message),
|
|
121
|
+
stack: own(stack)
|
|
122
|
+
}), Object.getPrototypeOf(error));
|
|
123
|
+
}
|
|
124
|
+
const own = (value) => ({
|
|
125
|
+
value,
|
|
126
|
+
writable: true,
|
|
127
|
+
configurable: true,
|
|
128
|
+
enumerable: false
|
|
129
|
+
});
|
|
130
|
+
// src/attempt/debug.ts
|
|
131
|
+
/** Which config and profile the attempt runs under, and where each resolved option came from. */
|
|
132
|
+
function tracePlan(debug, spec, ctx) {
|
|
133
|
+
const config = ctx.configFile ?? "none (no config file found)";
|
|
134
|
+
const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
|
|
135
|
+
const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
|
|
136
|
+
debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
|
|
137
|
+
debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
|
|
138
|
+
debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
|
|
139
|
+
const env = spec.environment;
|
|
140
|
+
const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
|
|
141
|
+
const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
|
|
142
|
+
debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
|
|
143
|
+
debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
|
|
144
|
+
const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
|
|
145
|
+
debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
|
|
146
|
+
const judge = spec.judge;
|
|
147
|
+
const model = typeof judge.model === "string" ? judge.model : judge.model && "kind" in judge.model ? "fake judge" : judge.model ? "a model object" : "none";
|
|
148
|
+
debug.log("plan", `judge ${model}${judge.threshold !== void 0 ? ` · threshold ${judge.threshold}` : ""} ${from("judge")}`);
|
|
149
|
+
for (const note of spec.notes) debug.log("plan", `profile: ${note}`);
|
|
150
|
+
debug.trace("plan", "config", {
|
|
151
|
+
config: ctx.config,
|
|
152
|
+
profile: ctx.profile
|
|
153
|
+
}, "resolved config");
|
|
154
|
+
debug.trace("plan", "attempt", {
|
|
155
|
+
cell: spec.cell,
|
|
156
|
+
n: spec.n,
|
|
157
|
+
harness: spec.harness,
|
|
158
|
+
sandbox: spec.sandbox,
|
|
159
|
+
egress: spec.egress,
|
|
160
|
+
gate: spec.gate,
|
|
161
|
+
timeoutMs: spec.timeoutMs,
|
|
162
|
+
judge: spec.judge,
|
|
163
|
+
sources: spec.sources,
|
|
164
|
+
env: Object.keys(spec.env).sort()
|
|
165
|
+
}, "resolved attempt");
|
|
166
|
+
}
|
|
167
|
+
/** `POST api2.cursor.sh/auth/exchange`, `GET host/v1/*` or `POST host (path ~ /re/)`. */
|
|
168
|
+
function endpointLabel({ method, host, path }) {
|
|
169
|
+
const where = !path ? host : "exact" in path ? `${host}${path.exact}` : "startsWith" in path ? `${host}${path.startsWith}*` : `${host} (path ~ /${path.regex}/)`;
|
|
170
|
+
return `${method?.join("|") ?? "*"} ${where}`;
|
|
171
|
+
}
|
|
172
|
+
/** The resolved environment and what the gateway brokers, by name. */
|
|
173
|
+
function traceEnvironment(debug, spec, env) {
|
|
174
|
+
const r = env.record;
|
|
175
|
+
debug.log("env", `environment ${r.name} (digest ${r.digest.replace(/^sha256:/, "").slice(0, 12)})${r.parent ? ` · extends ${r.parent}` : ""}${r.overrides.length ? ` · overrides ${r.overrides.join(", ")}` : ""}`);
|
|
176
|
+
const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
|
|
177
|
+
debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
|
|
178
|
+
for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
|
|
179
|
+
const brokering = brokeringFor(spec.harness, spec.vertex);
|
|
180
|
+
if (brokering) {
|
|
181
|
+
const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
|
|
182
|
+
debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
|
|
183
|
+
}
|
|
184
|
+
debug.trace("env", "environment", {
|
|
185
|
+
record: r,
|
|
186
|
+
vars: Object.keys(env.vars).sort(),
|
|
187
|
+
checkVars: Object.keys(env.checkVars).sort(),
|
|
188
|
+
files: env.files.map((f) => f.path),
|
|
189
|
+
setup: env.setup,
|
|
190
|
+
skills: env.skills.map((s) => s.name),
|
|
191
|
+
workspaceSkills: env.workspaceSkills.map((s) => s.name),
|
|
192
|
+
plugins: env.plugins.map((p) => ({
|
|
193
|
+
name: p.name,
|
|
194
|
+
parts: p.parts,
|
|
195
|
+
files: p.files.length
|
|
196
|
+
})),
|
|
197
|
+
mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
|
|
198
|
+
transport: m.transport,
|
|
199
|
+
command: m.command,
|
|
200
|
+
args: m.args,
|
|
201
|
+
env: Object.keys(m.env)
|
|
202
|
+
} : {
|
|
203
|
+
transport: m.transport,
|
|
204
|
+
url: m.url,
|
|
205
|
+
headers: Object.keys(m.headers)
|
|
206
|
+
}])),
|
|
207
|
+
brokered: env.substitutions.map(({ name, hosts, in: where, phase }) => ({
|
|
208
|
+
name,
|
|
209
|
+
hosts,
|
|
210
|
+
in: where,
|
|
211
|
+
phase
|
|
212
|
+
}))
|
|
213
|
+
}, "resolved environment");
|
|
214
|
+
}
|
|
47
215
|
// src/harness/fake.ts
|
|
48
216
|
const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n") ? 1 : 0) : 0;
|
|
49
217
|
/**
|
|
@@ -62,7 +230,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
62
230
|
if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
|
|
63
231
|
if (descriptor.bootstrap) {
|
|
64
232
|
const r = await sandbox.exec(fakeBootstrapScript(descriptor.bootstrap), { signal });
|
|
65
|
-
if (r.exitCode !== 0) throw new InfraError(`The fake agent's bootstrap exited ${r.exitCode}: ${(r.stderr || r.stdout).trim().slice(-500)}`);
|
|
233
|
+
if (r.exitCode !== 0) throw new InfraError(`The fake agent's bootstrap exited ${r.exitCode}: ${redactSecrets(r.stderr || r.stdout).trim().slice(-500)}`);
|
|
66
234
|
}
|
|
67
235
|
const layout = layoutOf(descriptor);
|
|
68
236
|
const brokered = descriptor.brokers ? await brokerKey(descriptor, sandbox) : void 0;
|
|
@@ -402,118 +570,13 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
402
570
|
/** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
|
|
403
571
|
async function createHarness(descriptor, sandbox, vertex) {
|
|
404
572
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
405
|
-
const { createAgentHarness } = await import("./agent-
|
|
573
|
+
const { createAgentHarness } = await import("./agent-OsAfayRf.js");
|
|
406
574
|
return createAgentHarness(descriptor, sandbox, vertex);
|
|
407
575
|
}
|
|
408
576
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
409
577
|
const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
|
|
410
|
-
// src/attempt/debug.ts
|
|
411
|
-
/** Which config and profile the attempt runs under, and where each resolved option came from. */
|
|
412
|
-
function tracePlan(debug, spec, ctx) {
|
|
413
|
-
const config = ctx.configFile ?? "none (no config file found)";
|
|
414
|
-
const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
|
|
415
|
-
const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
|
|
416
|
-
debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
|
|
417
|
-
debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
|
|
418
|
-
debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
|
|
419
|
-
const env = spec.environment;
|
|
420
|
-
const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
|
|
421
|
-
const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
|
|
422
|
-
debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
|
|
423
|
-
debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
|
|
424
|
-
const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
|
|
425
|
-
debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
|
|
426
|
-
const judge = spec.judge;
|
|
427
|
-
const model = typeof judge.model === "string" ? judge.model : judge.model && "kind" in judge.model ? "fake judge" : judge.model ? "a model object" : "none";
|
|
428
|
-
debug.log("plan", `judge ${model}${judge.threshold !== void 0 ? ` · threshold ${judge.threshold}` : ""} ${from("judge")}`);
|
|
429
|
-
for (const note of spec.notes) debug.log("plan", `profile: ${note}`);
|
|
430
|
-
debug.trace("plan", "config", {
|
|
431
|
-
config: ctx.config,
|
|
432
|
-
profile: ctx.profile
|
|
433
|
-
}, "resolved config");
|
|
434
|
-
debug.trace("plan", "attempt", {
|
|
435
|
-
cell: spec.cell,
|
|
436
|
-
n: spec.n,
|
|
437
|
-
harness: spec.harness,
|
|
438
|
-
sandbox: spec.sandbox,
|
|
439
|
-
egress: spec.egress,
|
|
440
|
-
gate: spec.gate,
|
|
441
|
-
timeoutMs: spec.timeoutMs,
|
|
442
|
-
judge: spec.judge,
|
|
443
|
-
sources: spec.sources,
|
|
444
|
-
env: Object.keys(spec.env).sort()
|
|
445
|
-
}, "resolved attempt");
|
|
446
|
-
}
|
|
447
|
-
/** `POST api2.cursor.sh/auth/exchange`, `GET host/v1/*` or `POST host (path ~ /re/)`. */
|
|
448
|
-
function endpointLabel({ method, host, path }) {
|
|
449
|
-
const where = !path ? host : "exact" in path ? `${host}${path.exact}` : "startsWith" in path ? `${host}${path.startsWith}*` : `${host} (path ~ /${path.regex}/)`;
|
|
450
|
-
return `${method?.join("|") ?? "*"} ${where}`;
|
|
451
|
-
}
|
|
452
|
-
/** The resolved environment and what the gateway brokers, by name. */
|
|
453
|
-
function traceEnvironment(debug, spec, env) {
|
|
454
|
-
const r = env.record;
|
|
455
|
-
debug.log("env", `environment ${r.name} (digest ${r.digest.replace(/^sha256:/, "").slice(0, 12)})${r.parent ? ` · extends ${r.parent}` : ""}${r.overrides.length ? ` · overrides ${r.overrides.join(", ")}` : ""}`);
|
|
456
|
-
const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
|
|
457
|
-
debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
|
|
458
|
-
for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
|
|
459
|
-
const brokering = brokeringFor(spec.harness, spec.vertex);
|
|
460
|
-
if (brokering) {
|
|
461
|
-
const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
|
|
462
|
-
debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
|
|
463
|
-
}
|
|
464
|
-
debug.trace("env", "environment", {
|
|
465
|
-
record: r,
|
|
466
|
-
vars: Object.keys(env.vars).sort(),
|
|
467
|
-
checkVars: Object.keys(env.checkVars).sort(),
|
|
468
|
-
files: env.files.map((f) => f.path),
|
|
469
|
-
setup: env.setup,
|
|
470
|
-
skills: env.skills.map((s) => s.name),
|
|
471
|
-
workspaceSkills: env.workspaceSkills.map((s) => s.name),
|
|
472
|
-
plugins: env.plugins.map((p) => ({
|
|
473
|
-
name: p.name,
|
|
474
|
-
parts: p.parts,
|
|
475
|
-
files: p.files.length
|
|
476
|
-
})),
|
|
477
|
-
mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
|
|
478
|
-
transport: m.transport,
|
|
479
|
-
command: m.command,
|
|
480
|
-
args: m.args,
|
|
481
|
-
env: Object.keys(m.env)
|
|
482
|
-
} : {
|
|
483
|
-
transport: m.transport,
|
|
484
|
-
url: m.url,
|
|
485
|
-
headers: Object.keys(m.headers)
|
|
486
|
-
}])),
|
|
487
|
-
brokered: env.substitutions.map(({ name, hosts, in: where, phase }) => ({
|
|
488
|
-
name,
|
|
489
|
-
hosts,
|
|
490
|
-
in: where,
|
|
491
|
-
phase
|
|
492
|
-
}))
|
|
493
|
-
}, "resolved environment");
|
|
494
|
-
}
|
|
495
578
|
// src/attempt/environment.ts
|
|
496
579
|
/**
|
|
497
|
-
* How many VMs the hosts preflight kept for `spec`'s sandbox run at once, or null when it runs
|
|
498
|
-
* on no hosts or preflight left nothing to read. The plan's cap counts every host it names at its
|
|
499
|
-
* `vms`; a host skipped, or with memory for fewer, runs fewer.
|
|
500
|
-
*/
|
|
501
|
-
async function preparedVmCap(spec, root, runId) {
|
|
502
|
-
const { sandbox, harness } = spec;
|
|
503
|
-
if (sandbox.kind !== "tart" || !sandbox.hosts) return null;
|
|
504
|
-
const { preparedFor } = await import("./preflight-BUjk_6wT.js");
|
|
505
|
-
const { createProviders, preparedHosts } = await import("./providers-Dljf6al6.js");
|
|
506
|
-
const provider = createProviders().tart;
|
|
507
|
-
if (!provider) return null;
|
|
508
|
-
try {
|
|
509
|
-
const prepared = preparedFor(root, runId, provider, sandbox, harness);
|
|
510
|
-
const hosts = prepared ? preparedHosts(prepared) : null;
|
|
511
|
-
return hosts ? hosts.reduce((sum, h) => sum + h.vms, 0) : null;
|
|
512
|
-
} catch {
|
|
513
|
-
return null;
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
/**
|
|
517
580
|
* Resolves the attempt's environment (files, skills, secrets and their placeholders)
|
|
518
581
|
* and finds the image its sandbox starts from, when its kind has one, which preflight prepared.
|
|
519
582
|
* Runs on the test's signal, before the eval timer starts.
|
|
@@ -538,8 +601,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
538
601
|
record.image = null;
|
|
539
602
|
const kind = sandboxKind(spec.sandbox);
|
|
540
603
|
if (kind.capabilities.images) {
|
|
541
|
-
const { preparedFor } = await import("./preflight-
|
|
542
|
-
const { attemptImage, createProviders, preparedImage } = await import("./providers-
|
|
604
|
+
const { preparedFor } = await import("./preflight-dJqs5x3r.js");
|
|
605
|
+
const { attemptImage, createProviders, preparedImage } = await import("./providers-DxZ26YoG.js");
|
|
543
606
|
signal.throwIfAborted();
|
|
544
607
|
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
545
608
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -564,8 +627,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
564
627
|
const { ref, id, base, install, agentCli } = record.image;
|
|
565
628
|
debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
|
|
566
629
|
} else if (kind.capabilities.isolated) {
|
|
567
|
-
const { preparedFor } = await import("./preflight-
|
|
568
|
-
const { bakedImage, createProviders, preparedHosts, vmImage } = await import("./providers-
|
|
630
|
+
const { preparedFor } = await import("./preflight-dJqs5x3r.js");
|
|
631
|
+
const { bakedImage, createProviders, preparedHosts, sandboxBase, vmImage } = await import("./providers-DxZ26YoG.js");
|
|
569
632
|
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
570
633
|
const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
|
|
571
634
|
const vm = prepared ? vmImage(prepared) : null;
|
|
@@ -574,78 +637,16 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
574
637
|
if (baked) a.image = baked;
|
|
575
638
|
const hosts = prepared ? preparedHosts(prepared) : null;
|
|
576
639
|
if (hosts) a.hosts = hosts;
|
|
640
|
+
const base = prepared ? sandboxBase(prepared) : null;
|
|
641
|
+
if (base) {
|
|
642
|
+
record.sandboxBase = base;
|
|
643
|
+
a.base = base;
|
|
644
|
+
debugLog("image", `${base.image} in ${base.provider} snapshot ${base.snapshot} · ${base.built ? "built by this run" : "saved before"}${base.pinned ? "" : " · a base tag that could not be resolved to a digest"}`);
|
|
645
|
+
}
|
|
577
646
|
const from = vm?.ref ?? ("image" in spec.sandbox ? spec.sandbox.image : void 0);
|
|
578
|
-
debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from ${baked ? `${baked}, which has the harness baked in` : "it"}`);
|
|
647
|
+
if (!base) debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from ${baked ? `${baked}, which has the harness baked in` : "it"}`);
|
|
579
648
|
} else debugLog("image", kind.capabilities.isolated ? `none: the ${spec.sandbox.kind} sandbox starts from its provider's base` : `none: the ${spec.sandbox.kind} sandbox runs on the host`);
|
|
580
649
|
}
|
|
581
|
-
// src/attempt/assertions.ts
|
|
582
|
-
const MESSAGE_LIMIT = 1024;
|
|
583
|
-
const LABEL_LIMIT = 60;
|
|
584
|
-
/** Masked first, so a secret the limit would cut through is masked whole. */
|
|
585
|
-
const clipped = (text, limit) => {
|
|
586
|
-
const s = redactSecrets(text);
|
|
587
|
-
return s.length > limit ? `${s.slice(0, limit)}…` : s;
|
|
588
|
-
};
|
|
589
|
-
const text = (error) => error instanceof Error ? error.message : String(error);
|
|
590
|
-
const shown = (value) => {
|
|
591
|
-
if (typeof value === "string") return value;
|
|
592
|
-
if (value instanceof RegExp) return String(value);
|
|
593
|
-
try {
|
|
594
|
-
return JSON.stringify(value) ?? String(value);
|
|
595
|
-
} catch {
|
|
596
|
-
return String(value);
|
|
597
|
-
}
|
|
598
|
-
};
|
|
599
|
-
const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
|
|
600
|
-
/**
|
|
601
|
-
* The records of an attempt's matcher calls, in call order. Without a label, one is
|
|
602
|
-
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
|
|
603
|
-
* any label already recorded, so each record's label is unique (a given `build #2` after two
|
|
604
|
-
* `build`s is `build #2 #2`).
|
|
605
|
-
*/
|
|
606
|
-
function assertionRecords(raw) {
|
|
607
|
-
const seen = /* @__PURE__ */ new Map();
|
|
608
|
-
const used = /* @__PURE__ */ new Set();
|
|
609
|
-
return raw.map((r) => {
|
|
610
|
-
const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.matcher === "toHaveRun" && runMatchLabel(r.args[0]) !== void 0 ? runMatchLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
611
|
-
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
612
|
-
let n = seen.get(base) ?? 0;
|
|
613
|
-
let label;
|
|
614
|
-
do {
|
|
615
|
-
n += 1;
|
|
616
|
-
label = n > 1 ? `${base} #${n}` : base;
|
|
617
|
-
} while (used.has(label));
|
|
618
|
-
seen.set(base, n);
|
|
619
|
-
used.add(label);
|
|
620
|
-
return {
|
|
621
|
-
label,
|
|
622
|
-
labelled: r.label !== void 0,
|
|
623
|
-
matcher: r.matcher,
|
|
624
|
-
status: r.status,
|
|
625
|
-
soft: r.soft,
|
|
626
|
-
...r.at ? { at: r.at } : {},
|
|
627
|
-
...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
|
|
628
|
-
...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
|
|
629
|
-
};
|
|
630
|
-
});
|
|
631
|
-
}
|
|
632
|
-
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
633
|
-
/**
|
|
634
|
-
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
635
|
-
* that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
|
|
636
|
-
* `CAUGHT_HINT` after its message.
|
|
637
|
-
*/
|
|
638
|
-
function checkFailure(raw) {
|
|
639
|
-
const failed = raw.find((r) => r.status !== "pass");
|
|
640
|
-
if (!failed) return void 0;
|
|
641
|
-
const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
|
|
642
|
-
if (failed.soft) return error;
|
|
643
|
-
const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
|
|
644
|
-
caught.name = error.name;
|
|
645
|
-
caught.message = `${error.message} ${CAUGHT_HINT}`;
|
|
646
|
-
caught.stack = error.stack?.replace(error.message, caught.message);
|
|
647
|
-
return caught;
|
|
648
|
-
}
|
|
649
650
|
// src/judge/size.ts
|
|
650
651
|
/** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
|
|
651
652
|
const CHARS_PER_TOKEN = 4;
|
|
@@ -1274,6 +1275,13 @@ function recordFailure(a, error, timedOut, raw) {
|
|
|
1274
1275
|
if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = judgeOutage(a, error, raw) ? "infra" : "judge";
|
|
1275
1276
|
record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
|
|
1276
1277
|
}
|
|
1278
|
+
/** What the sandbox gave the attempt and what it used, into the record, before the sandbox goes. */
|
|
1279
|
+
async function recordResources(a) {
|
|
1280
|
+
const resources = await a.meter?.stop({ gone: a.signal.aborted }).catch(() => void 0);
|
|
1281
|
+
if (!resources) return;
|
|
1282
|
+
a.record.resources = resources;
|
|
1283
|
+
debugLog("teardown", `resources ${resourcesLine(resources)}`);
|
|
1284
|
+
}
|
|
1277
1285
|
/**
|
|
1278
1286
|
* A process the kernel killed at the sandbox's memory limit makes the attempt an infra error,
|
|
1279
1287
|
* whatever its checks said: the limit, not the agent, decided it. Returns the error to fail it with.
|
|
@@ -1385,7 +1393,7 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
1385
1393
|
verdict: j.verdict,
|
|
1386
1394
|
score: j.score,
|
|
1387
1395
|
threshold: j.threshold,
|
|
1388
|
-
question: clip(j.question, 200)
|
|
1396
|
+
question: clip(redactSecrets(j.question), 200)
|
|
1389
1397
|
})),
|
|
1390
1398
|
turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
|
|
1391
1399
|
...failedChecks(record),
|
|
@@ -1672,7 +1680,7 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
|
1672
1680
|
},
|
|
1673
1681
|
report: (update) => progress({
|
|
1674
1682
|
at: Date.now(),
|
|
1675
|
-
...update
|
|
1683
|
+
...redactFields(update, ["kind"])
|
|
1676
1684
|
}),
|
|
1677
1685
|
async timed(phase, fn) {
|
|
1678
1686
|
const start = performance.now();
|
|
@@ -2119,6 +2127,8 @@ async function startSession(a, runId) {
|
|
|
2119
2127
|
env: env.vars,
|
|
2120
2128
|
image: a.image,
|
|
2121
2129
|
...a.hosts && { hosts: a.hosts },
|
|
2130
|
+
...a.base && { base: a.base },
|
|
2131
|
+
harness: spec.harness,
|
|
2122
2132
|
brokering: brokeringFor(spec.harness, spec.vertex),
|
|
2123
2133
|
...agentOf(spec.harness, spec.vertex)?.keychainTrust && { keychainTrust: true },
|
|
2124
2134
|
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
|
|
@@ -2130,6 +2140,7 @@ async function startSession(a, runId) {
|
|
|
2130
2140
|
})
|
|
2131
2141
|
});
|
|
2132
2142
|
a.sandbox = box;
|
|
2143
|
+
a.meter = box.meter?.();
|
|
2133
2144
|
if (!capabilitiesOf(spec.sandbox).isolated) debugLog("container", `${sandboxKind(spec.sandbox).provider(spec.sandbox)} sandbox at ${box.workdir}`);
|
|
2134
2145
|
a.phases = brokerPhases(box, env.substitutions, spec.egress);
|
|
2135
2146
|
if (a.signal.aborted) box.destroy();
|
|
@@ -2194,6 +2205,7 @@ async function startSession(a, runId) {
|
|
|
2194
2205
|
});
|
|
2195
2206
|
debugLog("harness", `${harnessId(spec.harness)} session ready in ${formatDuration(performance.now() - harnessStart)}`);
|
|
2196
2207
|
a.session = session;
|
|
2208
|
+
await a.meter?.ready(a.signal);
|
|
2197
2209
|
const startMs = performance.now() - startedAt;
|
|
2198
2210
|
debugLog("container", `sandbox and session started in ${formatDuration(startMs)}`);
|
|
2199
2211
|
a.report({
|
|
@@ -2739,6 +2751,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2739
2751
|
clock?.clear();
|
|
2740
2752
|
await a.endStarts();
|
|
2741
2753
|
for (const bg of a.backgrounds) await bg.kill().catch(() => {});
|
|
2754
|
+
await recordResources(a);
|
|
2742
2755
|
const { sandbox } = a;
|
|
2743
2756
|
if (sandbox && !a.signal.aborted) await takeContext(sandbox, AbortSignal.timeout(3e4));
|
|
2744
2757
|
if (sandbox) a.report({ kind: "cleanup" });
|
|
@@ -3097,12 +3110,13 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
|
|
|
3097
3110
|
const hid = harnessId(harness);
|
|
3098
3111
|
if (gatewaySet || harness.gateway !== void 0) {
|
|
3099
3112
|
routes.set(hid, route);
|
|
3100
|
-
|
|
3113
|
+
const at = route.name === "vercel" && !vercelBaseUrlProblem() ? vercelBaseUrl() : void 0;
|
|
3114
|
+
gatewayNotes.push(`gateway: ${hid} → ${route.name}${at ? ` at ${at}` : ""}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
|
|
3101
3115
|
}
|
|
3102
3116
|
if (route.name === "direct") return harness;
|
|
3103
3117
|
if (harness.kind !== "fake") try {
|
|
3104
3118
|
if (route.name === "vercel") {
|
|
3105
|
-
const problem = vercelCredentialProblem();
|
|
3119
|
+
const problem = vercelCredentialProblem() ?? vercelBaseUrlProblem();
|
|
3106
3120
|
if (problem) throw new UsageError(problem);
|
|
3107
3121
|
}
|
|
3108
3122
|
if (route.name === "vertex") {
|
|
@@ -3155,10 +3169,12 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
|
|
|
3155
3169
|
const why = unenforceable(planned.egress, planned.sandbox, where);
|
|
3156
3170
|
if (why) throw new UsageError(why);
|
|
3157
3171
|
const on = `${where} on ${harnessId(planned.harness)}`;
|
|
3158
|
-
const unverified = unverifiedHarness(planned.harness, planned.sandbox);
|
|
3159
|
-
if (unverified) throw new UsageError(`${on}: ${unverified}`);
|
|
3160
3172
|
const mcp = refusedMcp(planned.harness, planned.environment.layers);
|
|
3161
3173
|
if (mcp) throw new UsageError(`${on}: ${mcp}`);
|
|
3174
|
+
const kind = sandboxKind(planned.sandbox);
|
|
3175
|
+
const slots = kind.slotNetworks?.(planned.sandbox) ? 1 : kind.slotsPerMachine?.(planned.sandbox) ?? 1;
|
|
3176
|
+
const shared = refusedSharedMachine(planned.harness, slots);
|
|
3177
|
+
if (shared) throw new UsageError(`${on}: ${shared}`);
|
|
3162
3178
|
const plugins = planPlugins(planned, ctx.config.root);
|
|
3163
3179
|
if (plugins.refusal) throw new UsageError(`${on}: ${plugins.refusal}`);
|
|
3164
3180
|
for (const note of plugins.notes) if (!pluginNotes.includes(note)) pluginNotes.push(note);
|
|
@@ -3583,7 +3599,22 @@ function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
|
|
|
3583
3599
|
...options,
|
|
3584
3600
|
...hooks
|
|
3585
3601
|
};
|
|
3586
|
-
if (spec &&
|
|
3602
|
+
if (spec && getRunContext().inputs) {
|
|
3603
|
+
const { root } = getRunContext().config;
|
|
3604
|
+
const found = environmentInputs(spec.environment.layers, spec.sandbox, root);
|
|
3605
|
+
const { attempt } = options;
|
|
3606
|
+
const warnings = [.../* @__PURE__ */ new Set([...attempt.plan.warnings, ...found.warnings])];
|
|
3607
|
+
test.attempt = {
|
|
3608
|
+
...attempt,
|
|
3609
|
+
plan: {
|
|
3610
|
+
...attempt.plan,
|
|
3611
|
+
warnings
|
|
3612
|
+
},
|
|
3613
|
+
inputs: found.inputs
|
|
3614
|
+
};
|
|
3615
|
+
}
|
|
3616
|
+
const readCap = spec && sandboxKind(spec.sandbox).attemptCap?.(spec.sandbox);
|
|
3617
|
+
if (spec && test.gate && readCap) {
|
|
3587
3618
|
const gate = {
|
|
3588
3619
|
...test.gate,
|
|
3589
3620
|
limits: { ...test.gate.limits }
|
|
@@ -3592,8 +3623,12 @@ function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
|
|
|
3592
3623
|
test.gate = gate;
|
|
3593
3624
|
test.ready = async () => {
|
|
3594
3625
|
const { config, runId } = getRunContext();
|
|
3595
|
-
const cap = await
|
|
3596
|
-
|
|
3626
|
+
const cap = await readCap({
|
|
3627
|
+
harness: spec.harness,
|
|
3628
|
+
root: config.root,
|
|
3629
|
+
runId
|
|
3630
|
+
});
|
|
3631
|
+
const key = `sandbox:${sandboxProvider(spec.sandbox)}`;
|
|
3597
3632
|
if (cap && gate.limits[key]) gate.limits[key] = Math.min(gate.limits[key], cap);
|
|
3598
3633
|
return before ? before() : true;
|
|
3599
3634
|
};
|
|
@@ -3752,4 +3787,4 @@ function optionsFrom(written, where, base) {
|
|
|
3752
3787
|
}
|
|
3753
3788
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
3754
3789
|
|
|
3755
|
-
//# sourceMappingURL=evaluate-
|
|
3790
|
+
//# sourceMappingURL=evaluate-4kBVtwZZ.js.map
|