reeval 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +803 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-BdzhdRYo.js} +8 -6
- package/dist/adapter-BdzhdRYo.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-Dx-bDmlV.js} +179 -31
- package/dist/agent-Dx-bDmlV.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-l0-k7vw3.js} +361 -101
- package/dist/agents-l0-k7vw3.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
- package/dist/app-files-B-CQYB0c.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CYMpN8LC.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-CsZIcJo8.js} +171 -18
- package/dist/assemble-CsZIcJo8.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-BwSmNYkX.js} +17 -8
- package/dist/attempt-host-BwSmNYkX.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
- package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
- package/dist/{boat-DSNgZt-t.js → boat-su1Y7LPd.js} +17 -14
- package/dist/boat-su1Y7LPd.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/{capture-Ccf0Hmyx.js → capture-BqIAeFyC.js} +55 -29
- package/dist/{capture-Ccf0Hmyx.js.map → capture-BqIAeFyC.js.map} +1 -1
- package/dist/capture-D0Ln5aCP.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-s_3opn_I.js} +219 -109
- package/dist/cli-s_3opn_I.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
- package/dist/cli-spec-DxzIjmq_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-C5fIzHQa.js +803 -0
- package/dist/client-C5fIzHQa.js.map +1 -0
- package/dist/{command-DSL7I-SI.js → command-B5OAIjp7.js} +25 -21
- package/dist/command-B5OAIjp7.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BJ7KqczP.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BJ7KqczP.js.map} +1 -1
- package/dist/{command-D8oMB2fa.js → command-BXLZsEwQ.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-BXLZsEwQ.js.map} +1 -1
- package/dist/{command-Ff0T-1Hi.js → command-DA_SHQUQ.js} +16 -16
- package/dist/{command-Ff0T-1Hi.js.map → command-DA_SHQUQ.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-BKBamOcX.js} +174 -40
- package/dist/commands-BKBamOcX.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-BPvAOEmb.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0URar0t.js} +255 -89
- package/dist/config-C0URar0t.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-Sj58aJEu.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-Sj58aJEu.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-FWgSo-T1.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
- package/dist/db-Bc4L-Wcr.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-BjAWxVAG.js} +142 -248
- package/dist/db-BjAWxVAG.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-CWlhZ6U1.js} +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-Ck2GYIxX.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DG_uFgn2.js → docker-CDdK6Ity.js} +41 -16
- package/dist/docker-CDdK6Ity.js.map +1 -0
- package/dist/{docker-DFrjoNs-.js → docker-D74l67k8.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-D74l67k8.js.map} +1 -1
- package/dist/{docker--mukta3L.js → docker-DMug3l-a.js} +1 -1
- package/dist/{doctor-DWK4ttex.js → doctor-eFxyn6at.js} +85 -46
- package/dist/doctor-eFxyn6at.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-CJIKz-Ud.js} +49 -11
- package/dist/environment-CJIKz-Ud.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CXIRJ91v.js} +24 -12
- package/dist/eval-source-CXIRJ91v.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-DA4QsrHO.js} +792 -281
- package/dist/evaluate-DA4QsrHO.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
- package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
- package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
- package/dist/{files-BOBdC0uI.js → files-BVVWAS0A.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BVVWAS0A.js.map} +1 -1
- package/dist/files-BaqXvB88.js +1 -0
- package/dist/{files-tiNWe8Ih.js → files-Bey1VAex.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-Bey1VAex.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/files-CNHhSyb3.js +1 -0
- package/dist/files-CW1lChNh.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-D6j-XpaK.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-D6j-XpaK.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-F44NRCN0.js} +143 -20
- package/dist/files-F44NRCN0.js.map +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-Jqkf9SBa.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-Jqkf9SBa.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Dsc4Wi-F.js} +65 -30
- package/dist/images-Dsc4Wi-F.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/init-C0oUG3jh.js +1 -0
- package/dist/{init-Bi1Q3lRi.js → init-QB5ecPDA.js} +33 -14
- package/dist/init-QB5ecPDA.js.map +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
- package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
- package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
- package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
- package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
- package/dist/layout-DfQa4mJs.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
- package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
- package/dist/live-DwjWJTdn.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
- package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
- package/dist/{live-DL0se4Zd.js → live-y1p3oNXH.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-y1p3oNXH.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
- package/dist/model-3pZZzloU.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-DEjwetkJ.js} +1 -1
- package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-jRKSQBg6.js} +5 -5
- package/dist/{model-cgt5uvu8.js.map → model-jRKSQBg6.js.map} +1 -1
- package/dist/model-nfewGTzB.js +1 -0
- package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
- package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
- package/dist/options-DpsJ_5CU.js +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-Bx-qp8Li.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-Bx-qp8Li.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-ChO0A-xn.js} +63 -12
- package/dist/plan-ChO0A-xn.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-BioMWncI.js} +636 -9
- package/dist/plugins-BioMWncI.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-BUjk_6wT.js} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-CJSGzMzc.js} +8 -9
- package/dist/preflight-CJSGzMzc.js.map +1 -0
- package/dist/preflight-DD3fI4Hz.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-qFCXmxha.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-qFCXmxha.js.map} +1 -1
- package/dist/{prepare-DQEDXoDp.js → prepare-D7oSSFNm.js} +9 -9
- package/dist/{prepare-DQEDXoDp.js.map → prepare-D7oSSFNm.js.map} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-xHU2BpN-.js} +2 -2
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-B-flBAzR.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-B-flBAzR.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-B6StVyfG.js} +215 -48
- package/dist/providers-B6StVyfG.js.map +1 -0
- package/dist/providers-Dljf6al6.js +1 -0
- package/dist/query-BAc2tQZj.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-CkLmJv7R.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-CkLmJv7R.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-CJIh2FSX.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-CJIh2FSX.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
- package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
- package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
- package/dist/redact-DL2BTykw.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-B3wjKscW.js} +46 -11
- package/dist/remarks-B3wjKscW.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-C88amFRN.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-C88amFRN.js.map} +1 -1
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/resolve-DW92wpIZ.js +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-G576PiFJ.js} +14 -11
- package/dist/resolve-G576PiFJ.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-zVHIW6xw.js} +9 -22
- package/dist/results-zVHIW6xw.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-D_PcUXfG.js} +94 -16
- package/dist/resume-D_PcUXfG.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-BvTzCxdc.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-BvTzCxdc.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
- package/dist/routes-B0F2R-K7.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-B3r_pAq6.js} +4 -4
- package/dist/{runtime-Xm9oeC7o.js.map → runtime-B3r_pAq6.js.map} +1 -1
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-Duh3z1TV.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-Duh3z1TV.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CGEsvVRG.js +1 -0
- package/dist/{server-C00zd6zl.js → server-DUWMRKik.js} +8 -8
- package/dist/{server-C00zd6zl.js.map → server-DUWMRKik.js.map} +1 -1
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-CHuN4Nfg.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-CHuN4Nfg.js.map} +1 -1
- package/dist/source-Wz1Nrfue.js +1 -0
- package/dist/src--yl1OypK.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
- package/dist/src-B-HuDqpH.js.map +1 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-y0nOeRRE.js → src-BHJ4MWvY.js} +494 -112
- package/dist/src-BHJ4MWvY.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
- package/dist/src-BI0v8qlZ.js.map +1 -0
- package/dist/src-BL-_0QoE.js +1 -0
- package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
- package/dist/src-BqNzHHWn.js.map +1 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C-mOSczH.js +2 -0
- package/dist/src-CZaYu6ER.js +3 -0
- package/dist/src-CeBhwD8k.js +2 -0
- package/dist/src-CkEiiyON.js +2 -0
- package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
- package/dist/src-CmA5Yev6.js.map +1 -0
- package/dist/src-CwH_nEaw.js +2 -0
- package/dist/src-DA95rM6v.js +4 -0
- package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
- package/dist/src-DL-EE4DU.js.map +1 -0
- package/dist/src-De0KY5S_.js +2 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/state-B3jex5KU.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUBRDyMk.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUBRDyMk.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
- package/dist/suite-Bhs9-Z7O.js +2 -0
- package/dist/{summary-DDy9DfyJ.js → summary-CSp4cfnr.js} +25 -7
- package/dist/{summary-DDy9DfyJ.js.map → summary-CSp4cfnr.js.map} +1 -1
- package/dist/sync-CxAY6Dki.js +559 -0
- package/dist/sync-CxAY6Dki.js.map +1 -0
- package/dist/{tart-dWoJvWHc.js → tart-CcrrtSrp.js} +179 -61
- package/dist/tart-CcrrtSrp.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-F-6seRC9.js +192 -0
- package/dist/tart-bake-F-6seRC9.js.map +1 -0
- package/dist/tart-bake-RK7RaT9E.js +1 -0
- package/dist/tart-cli-B86Jqo1t.js +89 -0
- package/dist/tart-cli-B86Jqo1t.js.map +1 -0
- package/dist/tart-cli-ClvRniLQ.js +1 -0
- package/dist/tart-hosts-Brii7zHM.js +1 -0
- package/dist/tart-hosts-DPyZsg-1.js +313 -0
- package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
- package/dist/tart-images-CWSf03w8.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-CVDdJy0H.js +433 -0
- package/dist/tool-class-CVDdJy0H.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
- package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +27 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-DRxXCyIH.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-DRxXCyIH.js.map} +1 -1
- package/dist/views-i67V1ly3.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-C3Q4U63F.js} +12 -9
- package/dist/worker-entry-C3Q4U63F.js.map +1 -0
- package/dist/workspace-DhUOW8if.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-P72SXLXr.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-P72SXLXr.js.map} +1 -1
- package/package.json +12 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -1,199 +1,49 @@
|
|
|
1
1
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-
|
|
3
|
-
import {
|
|
4
|
-
import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
|
|
5
|
-
import { a as TURN_EDITED, f as limitsLabel, r as withLease } from "./client-KM4C5r5_.js";
|
|
6
|
-
import { d as sandboxKind, n as capabilitiesOf, s as providerLimit } from "./kinds-Bh2I35id.js";
|
|
7
|
-
import { O as sandboxPaths, P as workspaceOf, f as filesOf, i as callerDir, j as traitsOf, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-cPMCOk-h.js";
|
|
2
|
+
import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-DtZwIAfy.js";
|
|
3
|
+
import { Dt as OPAQUE_OUTPUT, jt as shellQuote$1, n as MCP_TIMER } from "./src-BHJ4MWvY.js";
|
|
8
4
|
import { t as evalSlug } from "./slug-IHh3f770.js";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { l as
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
5
|
+
import { c as relativeFile, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-B-JKTCP8.js";
|
|
6
|
+
import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-DL2BTykw.js";
|
|
7
|
+
import { O as pluginSourceLabel, P as capabilitiesOf, S as pluginBuildId, U as sandboxKind, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall, z as providerLimit } from "./plugins-BioMWncI.js";
|
|
8
|
+
import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
|
|
9
|
+
import { d as vercelCredentialProblem } from "./routes-B0F2R-K7.js";
|
|
10
|
+
import { v as vertexSettings } from "./vertex-BEWxHoxH.js";
|
|
11
|
+
import { n as hasNetworkSession, t as InfraError } from "./types-CbcnJxCy.js";
|
|
12
|
+
import { C as writePluginFiles, S as unverifiedHarness, b as refusedMcp, c as agentMcp, d as brokeringFor, f as claudeMcpServers, h as fakeBrokering, l as agentOf, m as expectedBlockedFor, t as AGENTS, v as keyRejected, x as refusedPlugins, y as layoutOf } from "./agents-l0-k7vw3.js";
|
|
13
|
+
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
14
|
+
import { d as limitsLabel, f as runMatchLabel, r as withLease, s as judgeFailedMessage, u as TURN_EDITED } from "./client-C5fIzHQa.js";
|
|
15
|
+
import { F as workspaceOf, M as traitsOf, f as filesOf, i as callerDir, k as sandboxPaths, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-CJIKz-Ud.js";
|
|
16
|
+
import { $t as resolveJudgeArtifacts, B as evalPlan, Ct as phaseLabel, Dt as unenforceable, Ft as observedHere, G as matchesTags, Gt as INPUT_KEYS, H as harnessId, It as observedName, J as plainEnvError, Jt as judgeProblem, Kt as NO_JUDGE_INPUT, Lt as observer, M as cascade, Nt as mergeJudge, Pt as resolveStateOptions, Q as plainHeaders, Qt as filePartsOf, R as checkOptions, T as resolveGateway, V as expand, W as majorityOf, X as plainHeaderError, Xt as badJudgeArtifacts, Yt as artifactRefsOf, Zt as decideState, at as sandboxProvider, bt as isDirect, ct as secretVarsIn, it as sandboxId, j as brokeredIn, lt as unbrokered, mt as allowedHosts, nt as refuseIllegalCompare, ot as sandboxSetBy, q as plainEnv, qt as inputsOn, rt as runsOf, ut as unbrokeredError, v as fellBackToDirect, vt as egressLabel, x as resolveModelGateway, y as gatewayScope, yt as gatewayPolicy } from "./config-C0URar0t.js";
|
|
17
|
+
import { f as runJudge, g as isTransient, h as isApiError, l as judgeModelName } from "./judge-eGH39PaP.js";
|
|
15
18
|
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
16
|
-
import {
|
|
17
|
-
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BfnhazOv.js";
|
|
18
|
-
import { _ as unverifiedHarness, c as brokeringFor, d as expectedBlockedFor, g as refusedPlugins, h as refusedMcp, l as claudeMcpServers, m as layoutOf, o as agentMcp, p as keyRejected, t as AGENTS, v as writePluginFiles } from "./agents-DXuFxtGC.js";
|
|
19
|
-
import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
|
|
20
|
-
import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-Xm9oeC7o.js";
|
|
19
|
+
import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-B3r_pAq6.js";
|
|
21
20
|
import { c as formatBytes } from "./debug-CIXzB4wK.js";
|
|
22
|
-
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import { t as
|
|
27
|
-
import {
|
|
28
|
-
import { t as
|
|
29
|
-
import {
|
|
21
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CKfKsqqw.js";
|
|
22
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-G576PiFJ.js";
|
|
23
|
+
import { n as fakeBootstrapScript } from "./fake-bootstrap-CnwWG3kP.js";
|
|
24
|
+
import { n as providerStop, t as ProviderStopError } from "./provider-stop-CxzWW7Ft.js";
|
|
25
|
+
import { t as PrepareError } from "./provider-ieWOn4YS.js";
|
|
26
|
+
import { i as preparedWorkspace } from "./workspace-P72SXLXr.js";
|
|
27
|
+
import { t as attemptDebugDir } from "./files-g1watgRC.js";
|
|
28
|
+
import { r as openRouterFallbacks } from "./openrouter-models-C0aRo633.js";
|
|
29
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-BqIAeFyC.js";
|
|
30
|
+
import { t as contextTaker } from "./context-Sj58aJEu.js";
|
|
30
31
|
import { t as patchPath } from "./patch-C7yBztZj.js";
|
|
31
|
-
import { n as judgeDiff } from "./patch-
|
|
32
|
-
import { t as transcriptForJudge } from "./transcript-
|
|
32
|
+
import { n as judgeDiff } from "./patch-Bx-qp8Li.js";
|
|
33
|
+
import { t as transcriptForJudge } from "./transcript-BP_2hji-.js";
|
|
34
|
+
import { n as modelCallEvents } from "./model-calls-B0BoQfJ4.js";
|
|
33
35
|
import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CESxtK4X.js";
|
|
34
|
-
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-
|
|
35
|
-
import { i as withDefaultEffort } from "./default-effort-
|
|
36
|
-
import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-
|
|
36
|
+
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-CXIRJ91v.js";
|
|
37
|
+
import { i as withDefaultEffort } from "./default-effort-Ck2GYIxX.js";
|
|
38
|
+
import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-CCD8XsmI.js";
|
|
37
39
|
import { z } from "zod";
|
|
38
40
|
import { dirname, join } from "node:path";
|
|
39
|
-
import {
|
|
40
|
-
import { randomBytes } from "node:crypto";
|
|
41
|
-
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
41
|
+
import { randomBytes, randomUUID } from "node:crypto";
|
|
42
42
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
43
|
+
import { parse, stringify } from "yaml";
|
|
43
44
|
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
const LABEL_LIMIT = 60;
|
|
47
|
-
/** Masked first, so a secret the limit would cut through is masked whole. */
|
|
48
|
-
const clipped = (text, limit) => {
|
|
49
|
-
const s = redactSecrets(text);
|
|
50
|
-
return s.length > limit ? `${s.slice(0, limit)}…` : s;
|
|
51
|
-
};
|
|
52
|
-
const text = (error) => error instanceof Error ? error.message : String(error);
|
|
53
|
-
const shown = (value) => {
|
|
54
|
-
if (typeof value === "string") return value;
|
|
55
|
-
if (value instanceof RegExp) return String(value);
|
|
56
|
-
try {
|
|
57
|
-
return JSON.stringify(value) ?? String(value);
|
|
58
|
-
} catch {
|
|
59
|
-
return String(value);
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
|
-
const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
|
|
63
|
-
/**
|
|
64
|
-
* The records of an attempt's matcher calls, in call order. Without a label, one is
|
|
65
|
-
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
|
|
66
|
-
* any label already recorded, so each record's label is unique (a given `build #2` after two
|
|
67
|
-
* `build`s is `build #2 #2`).
|
|
68
|
-
*/
|
|
69
|
-
function assertionRecords(raw) {
|
|
70
|
-
const seen = /* @__PURE__ */ new Map();
|
|
71
|
-
const used = /* @__PURE__ */ new Set();
|
|
72
|
-
return raw.map((r) => {
|
|
73
|
-
const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
74
|
-
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
75
|
-
let n = seen.get(base) ?? 0;
|
|
76
|
-
let label;
|
|
77
|
-
do {
|
|
78
|
-
n += 1;
|
|
79
|
-
label = n > 1 ? `${base} #${n}` : base;
|
|
80
|
-
} while (used.has(label));
|
|
81
|
-
seen.set(base, n);
|
|
82
|
-
used.add(label);
|
|
83
|
-
return {
|
|
84
|
-
label,
|
|
85
|
-
labelled: r.label !== void 0,
|
|
86
|
-
matcher: r.matcher,
|
|
87
|
-
status: r.status,
|
|
88
|
-
soft: r.soft,
|
|
89
|
-
...r.at ? { at: r.at } : {},
|
|
90
|
-
...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
|
|
91
|
-
...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
|
|
92
|
-
};
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
96
|
-
/**
|
|
97
|
-
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
98
|
-
* that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
|
|
99
|
-
* `CAUGHT_HINT` after its message.
|
|
100
|
-
*/
|
|
101
|
-
function checkFailure(raw) {
|
|
102
|
-
const failed = raw.find((r) => r.status !== "pass");
|
|
103
|
-
if (!failed) return void 0;
|
|
104
|
-
const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
|
|
105
|
-
if (failed.soft) return error;
|
|
106
|
-
const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
|
|
107
|
-
caught.name = error.name;
|
|
108
|
-
caught.message = `${error.message} ${CAUGHT_HINT}`;
|
|
109
|
-
caught.stack = error.stack?.replace(error.message, caught.message);
|
|
110
|
-
return caught;
|
|
111
|
-
}
|
|
112
|
-
// src/attempt/debug.ts
|
|
113
|
-
/** Which config and profile the attempt runs under, and where each resolved option came from. */
|
|
114
|
-
function tracePlan(debug, spec, ctx) {
|
|
115
|
-
const config = ctx.configFile ?? "none (no config file found)";
|
|
116
|
-
const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
|
|
117
|
-
const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
|
|
118
|
-
debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
|
|
119
|
-
debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
|
|
120
|
-
debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
|
|
121
|
-
const env = spec.environment;
|
|
122
|
-
const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
|
|
123
|
-
const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
|
|
124
|
-
debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
|
|
125
|
-
debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
|
|
126
|
-
const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
|
|
127
|
-
debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
|
|
128
|
-
const judge = spec.judge;
|
|
129
|
-
const model = typeof judge.model === "string" ? judge.model : judge.model && "kind" in judge.model ? "fake judge" : judge.model ? "a model object" : "none";
|
|
130
|
-
debug.log("plan", `judge ${model}${judge.threshold !== void 0 ? ` · threshold ${judge.threshold}` : ""} ${from("judge")}`);
|
|
131
|
-
for (const note of spec.notes) debug.log("plan", `profile: ${note}`);
|
|
132
|
-
debug.trace("plan", "config", {
|
|
133
|
-
config: ctx.config,
|
|
134
|
-
profile: ctx.profile
|
|
135
|
-
}, "resolved config");
|
|
136
|
-
debug.trace("plan", "attempt", {
|
|
137
|
-
cell: spec.cell,
|
|
138
|
-
n: spec.n,
|
|
139
|
-
harness: spec.harness,
|
|
140
|
-
sandbox: spec.sandbox,
|
|
141
|
-
egress: spec.egress,
|
|
142
|
-
gate: spec.gate,
|
|
143
|
-
timeoutMs: spec.timeoutMs,
|
|
144
|
-
judge: spec.judge,
|
|
145
|
-
sources: spec.sources,
|
|
146
|
-
env: Object.keys(spec.env).sort()
|
|
147
|
-
}, "resolved attempt");
|
|
148
|
-
}
|
|
149
|
-
/** `POST api2.cursor.sh/auth/exchange`, `GET host/v1/*` or `POST host (path ~ /re/)`. */
|
|
150
|
-
function endpointLabel({ method, host, path }) {
|
|
151
|
-
const where = !path ? host : "exact" in path ? `${host}${path.exact}` : "startsWith" in path ? `${host}${path.startsWith}*` : `${host} (path ~ /${path.regex}/)`;
|
|
152
|
-
return `${method?.join("|") ?? "*"} ${where}`;
|
|
153
|
-
}
|
|
154
|
-
/** The resolved environment and what the gateway brokers, by name. */
|
|
155
|
-
function traceEnvironment(debug, spec, env) {
|
|
156
|
-
const r = env.record;
|
|
157
|
-
debug.log("env", `environment ${r.name} (digest ${r.digest.replace(/^sha256:/, "").slice(0, 12)})${r.parent ? ` · extends ${r.parent}` : ""}${r.overrides.length ? ` · overrides ${r.overrides.join(", ")}` : ""}`);
|
|
158
|
-
const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
|
|
159
|
-
debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
|
|
160
|
-
for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
|
|
161
|
-
const brokering = brokeringFor(spec.harness, spec.vertex);
|
|
162
|
-
if (brokering) {
|
|
163
|
-
const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
|
|
164
|
-
debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
|
|
165
|
-
}
|
|
166
|
-
debug.trace("env", "environment", {
|
|
167
|
-
record: r,
|
|
168
|
-
vars: Object.keys(env.vars).sort(),
|
|
169
|
-
checkVars: Object.keys(env.checkVars).sort(),
|
|
170
|
-
files: env.files.map((f) => f.path),
|
|
171
|
-
setup: env.setup,
|
|
172
|
-
skills: env.skills.map((s) => s.name),
|
|
173
|
-
workspaceSkills: env.workspaceSkills.map((s) => s.name),
|
|
174
|
-
plugins: env.plugins.map((p) => ({
|
|
175
|
-
name: p.name,
|
|
176
|
-
parts: p.parts,
|
|
177
|
-
files: p.files.length
|
|
178
|
-
})),
|
|
179
|
-
mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
|
|
180
|
-
transport: m.transport,
|
|
181
|
-
command: m.command,
|
|
182
|
-
args: m.args,
|
|
183
|
-
env: Object.keys(m.env)
|
|
184
|
-
} : {
|
|
185
|
-
transport: m.transport,
|
|
186
|
-
url: m.url,
|
|
187
|
-
headers: Object.keys(m.headers)
|
|
188
|
-
}])),
|
|
189
|
-
brokered: env.substitutions.map(({ name, hosts, in: where, phase }) => ({
|
|
190
|
-
name,
|
|
191
|
-
hosts,
|
|
192
|
-
in: where,
|
|
193
|
-
phase
|
|
194
|
-
}))
|
|
195
|
-
}, "resolved environment");
|
|
196
|
-
}
|
|
45
|
+
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
46
|
+
import { resolveBridgePort } from "@ai-sdk/harness/utils";
|
|
197
47
|
// src/harness/fake.ts
|
|
198
48
|
const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n") ? 1 : 0) : 0;
|
|
199
49
|
/**
|
|
@@ -206,12 +56,17 @@ const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n"
|
|
|
206
56
|
*/
|
|
207
57
|
const createFakeHarness = (descriptor) => {
|
|
208
58
|
let starts = 0;
|
|
209
|
-
return { async start(sandbox,
|
|
59
|
+
return { async start(sandbox, signal, { skills, workspaceSkills, mcp: all, plugins }) {
|
|
210
60
|
if (descriptor.failWith) throw new InfraError(descriptor.failWith);
|
|
211
61
|
const hangs = descriptor.startHangs;
|
|
212
62
|
if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
|
|
63
|
+
if (descriptor.bootstrap) {
|
|
64
|
+
const r = await sandbox.exec(fakeBootstrapScript(descriptor.bootstrap), { signal });
|
|
65
|
+
if (r.exitCode !== 0) throw new InfraError(`The fake agent's bootstrap exited ${r.exitCode}: ${(r.stderr || r.stdout).trim().slice(-500)}`);
|
|
66
|
+
}
|
|
213
67
|
const layout = layoutOf(descriptor);
|
|
214
68
|
const brokered = descriptor.brokers ? await brokerKey(descriptor, sandbox) : void 0;
|
|
69
|
+
if (descriptor.brokers && AGENTS[descriptor.layout ?? "cursor"].bridgePort) await reachBridge(sandbox, signal);
|
|
215
70
|
const mcp = agentMcp(layout, all, plugins);
|
|
216
71
|
for (const path of await writePluginFiles(sandbox, layout, plugins)) debugLog("env", `plugin settings → ${path} by the fake agent`);
|
|
217
72
|
const dir = `~/${layout.harnessSkillDir}`;
|
|
@@ -231,9 +86,11 @@ const createFakeHarness = (descriptor) => {
|
|
|
231
86
|
const files = (skilled ? descriptor.withSkill?.files : void 0) ?? descriptor.files ?? {};
|
|
232
87
|
const reply = (skilled ? descriptor.withSkill?.text : void 0) ?? descriptor.text ?? "Done.";
|
|
233
88
|
const flaked = Math.random() < (descriptor.flakeRate ?? 0);
|
|
89
|
+
let running;
|
|
234
90
|
return {
|
|
235
91
|
async send(prompt, signal) {
|
|
236
92
|
const startedAt = Date.now();
|
|
93
|
+
const brokeredBefore = sandbox.brokeredStatuses?.().length ?? 0;
|
|
237
94
|
const script = descriptor.script ?? [];
|
|
238
95
|
const steps = script[Math.min(turns++, script.length - 1)] ?? [];
|
|
239
96
|
const at = () => {
|
|
@@ -261,10 +118,19 @@ const createFakeHarness = (descriptor) => {
|
|
|
261
118
|
durationMs: descriptor.turnMs ?? Date.now() - startedAt,
|
|
262
119
|
transcript,
|
|
263
120
|
usage: descriptor.usage ?? null,
|
|
121
|
+
...!descriptor.costUsd ? {} : { cost: {
|
|
122
|
+
usd: descriptor.costUsd,
|
|
123
|
+
basis: "reported",
|
|
124
|
+
source: "fake"
|
|
125
|
+
} },
|
|
264
126
|
model: descriptor.servedModel ?? null,
|
|
265
127
|
unreported: [],
|
|
266
128
|
...descriptor.turnMs === void 0 ? {} : { agentMs: descriptor.turnMs }
|
|
267
129
|
});
|
|
130
|
+
running = (note) => ({
|
|
131
|
+
...done("", "aborted"),
|
|
132
|
+
transcript: [...transcript, note]
|
|
133
|
+
});
|
|
268
134
|
const write = async (path, content) => {
|
|
269
135
|
const time = at();
|
|
270
136
|
const before = await sandbox.readFile(path);
|
|
@@ -294,8 +160,9 @@ const createFakeHarness = (descriptor) => {
|
|
|
294
160
|
tool: tool.tool,
|
|
295
161
|
input: tool.input,
|
|
296
162
|
output: tool.output,
|
|
163
|
+
outputView: "model",
|
|
297
164
|
ok: tool.ok === void 0 ? true : tool.ok,
|
|
298
|
-
durationMs: 0
|
|
165
|
+
durationMs: tool.durationMs === void 0 ? 0 : tool.durationMs
|
|
299
166
|
});
|
|
300
167
|
if (flaked) {
|
|
301
168
|
const text = "I ran out of ideas and left the workspace unchanged.";
|
|
@@ -323,6 +190,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
323
190
|
tool: "shell",
|
|
324
191
|
input: { command: step.exec },
|
|
325
192
|
output: r.stdout,
|
|
193
|
+
outputView: "model",
|
|
326
194
|
ok: r.exitCode === 0,
|
|
327
195
|
durationMs: Date.now() - startedAt - time.atMs
|
|
328
196
|
});
|
|
@@ -340,6 +208,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
340
208
|
tool: `mcp__${server}__${tool}`,
|
|
341
209
|
input: {},
|
|
342
210
|
output: r.output,
|
|
211
|
+
outputView: "model",
|
|
343
212
|
ok: r.ok,
|
|
344
213
|
durationMs: Date.now() - startedAt - time.atMs
|
|
345
214
|
});
|
|
@@ -350,7 +219,16 @@ const createFakeHarness = (descriptor) => {
|
|
|
350
219
|
type: "error",
|
|
351
220
|
message: step.error
|
|
352
221
|
});
|
|
353
|
-
|
|
222
|
+
const statuses = sandbox.brokeredStatuses?.().slice(brokeredBefore) ?? [];
|
|
223
|
+
const stopped = providerStop({
|
|
224
|
+
errors: [step.error],
|
|
225
|
+
status: "error",
|
|
226
|
+
statuses
|
|
227
|
+
});
|
|
228
|
+
return {
|
|
229
|
+
...done("", "error"),
|
|
230
|
+
...stopped && { providerStop: stopped }
|
|
231
|
+
};
|
|
354
232
|
} else if ("refuseKey" in step) {
|
|
355
233
|
const message = `${keyRejected(step.refuseKey)}. 401 Unauthorized (the fake agent's)`;
|
|
356
234
|
debugLog("harness", `turn failed: ${message}`);
|
|
@@ -379,6 +257,10 @@ const createFakeHarness = (descriptor) => {
|
|
|
379
257
|
});
|
|
380
258
|
return done(text, "completed");
|
|
381
259
|
},
|
|
260
|
+
stopped: async (note) => {
|
|
261
|
+
if (!running) throw new Error("No turn is running");
|
|
262
|
+
return running(note);
|
|
263
|
+
},
|
|
382
264
|
async close() {}
|
|
383
265
|
};
|
|
384
266
|
} };
|
|
@@ -388,7 +270,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
388
270
|
* endpoints, as a real adapter does, and returns the variable its `exec` steps get it in.
|
|
389
271
|
*/
|
|
390
272
|
async function brokerKey(descriptor, sandbox) {
|
|
391
|
-
const { credential, endpoints } =
|
|
273
|
+
const { credential, endpoints } = fakeBrokering(descriptor.layout);
|
|
392
274
|
const key = process.env[credential];
|
|
393
275
|
if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
|
|
394
276
|
const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
|
|
@@ -408,6 +290,40 @@ async function brokerKey(descriptor, sandbox) {
|
|
|
408
290
|
return { [credential]: placeholder };
|
|
409
291
|
}
|
|
410
292
|
/**
|
|
293
|
+
* Serves on the port the layout's adapter would put its bridge on, in the sandbox, and reaches it
|
|
294
|
+
* from the host through the session's endpoint, as that adapter's start does.
|
|
295
|
+
*/
|
|
296
|
+
async function reachBridge(sandbox, signal) {
|
|
297
|
+
const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
|
|
298
|
+
if (!network) throw new InfraError("The fake agent's bridge needs a sandbox session");
|
|
299
|
+
let port;
|
|
300
|
+
try {
|
|
301
|
+
port = resolveBridgePort({
|
|
302
|
+
sandboxSession: network,
|
|
303
|
+
portOverride: void 0
|
|
304
|
+
});
|
|
305
|
+
} catch (error) {
|
|
306
|
+
throw new InfraError(`Fake harness failed to start: ${error.message}`);
|
|
307
|
+
}
|
|
308
|
+
const server = await network.spawn({
|
|
309
|
+
command: `node -e ${shellQuote$1(`require('http').createServer((q, s) => s.end('bridge')).listen(${port}, '0.0.0.0', () => console.log('up'))`)}`,
|
|
310
|
+
abortSignal: signal
|
|
311
|
+
});
|
|
312
|
+
try {
|
|
313
|
+
const up = await server.stdout.getReader().read();
|
|
314
|
+
if (!new TextDecoder().decode(up.value).includes("up")) throw new InfraError(`The fake agent's bridge didn't listen on ${port}`);
|
|
315
|
+
const { url } = await network.getPortEndpoint({
|
|
316
|
+
port,
|
|
317
|
+
protocol: "http"
|
|
318
|
+
});
|
|
319
|
+
const reply = await (await fetch(url, { signal })).text();
|
|
320
|
+
if (reply !== "bridge") throw new InfraError(`${url} answered ${JSON.stringify(reply)}`);
|
|
321
|
+
debugLog("harness", `bridge port ${port} reached through ${url}`);
|
|
322
|
+
} finally {
|
|
323
|
+
await server.kill();
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
411
327
|
* Starts a stdio MCP server in the sandbox, as an agent would, and calls one tool with no
|
|
412
328
|
* arguments: initialize, then the call.
|
|
413
329
|
*/
|
|
@@ -445,8 +361,8 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
445
361
|
arguments: {}
|
|
446
362
|
}
|
|
447
363
|
}
|
|
448
|
-
].map((m) => shellQuote(JSON.stringify(m)));
|
|
449
|
-
const command = [server.command, ...server.args].map(shellQuote).join(" ");
|
|
364
|
+
].map((m) => shellQuote$1(JSON.stringify(m)));
|
|
365
|
+
const command = [server.command, ...server.args].map(shellQuote$1).join(" ");
|
|
450
366
|
const script = [
|
|
451
367
|
"d=$(mktemp -d) && : > \"$d/out\" && : > \"$d/err\" || exit 1",
|
|
452
368
|
"{",
|
|
@@ -486,13 +402,118 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
486
402
|
/** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
|
|
487
403
|
async function createHarness(descriptor, sandbox, vertex) {
|
|
488
404
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
489
|
-
const { createAgentHarness } = await import("./agent-
|
|
405
|
+
const { createAgentHarness } = await import("./agent-Dx-bDmlV.js");
|
|
490
406
|
return createAgentHarness(descriptor, sandbox, vertex);
|
|
491
407
|
}
|
|
492
408
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
493
409
|
const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
|
|
410
|
+
// src/attempt/debug.ts
|
|
411
|
+
/** Which config and profile the attempt runs under, and where each resolved option came from. */
|
|
412
|
+
function tracePlan(debug, spec, ctx) {
|
|
413
|
+
const config = ctx.configFile ?? "none (no config file found)";
|
|
414
|
+
const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
|
|
415
|
+
const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
|
|
416
|
+
debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
|
|
417
|
+
debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
|
|
418
|
+
debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
|
|
419
|
+
const env = spec.environment;
|
|
420
|
+
const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
|
|
421
|
+
const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
|
|
422
|
+
debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
|
|
423
|
+
debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
|
|
424
|
+
const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
|
|
425
|
+
debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
|
|
426
|
+
const judge = spec.judge;
|
|
427
|
+
const model = typeof judge.model === "string" ? judge.model : judge.model && "kind" in judge.model ? "fake judge" : judge.model ? "a model object" : "none";
|
|
428
|
+
debug.log("plan", `judge ${model}${judge.threshold !== void 0 ? ` · threshold ${judge.threshold}` : ""} ${from("judge")}`);
|
|
429
|
+
for (const note of spec.notes) debug.log("plan", `profile: ${note}`);
|
|
430
|
+
debug.trace("plan", "config", {
|
|
431
|
+
config: ctx.config,
|
|
432
|
+
profile: ctx.profile
|
|
433
|
+
}, "resolved config");
|
|
434
|
+
debug.trace("plan", "attempt", {
|
|
435
|
+
cell: spec.cell,
|
|
436
|
+
n: spec.n,
|
|
437
|
+
harness: spec.harness,
|
|
438
|
+
sandbox: spec.sandbox,
|
|
439
|
+
egress: spec.egress,
|
|
440
|
+
gate: spec.gate,
|
|
441
|
+
timeoutMs: spec.timeoutMs,
|
|
442
|
+
judge: spec.judge,
|
|
443
|
+
sources: spec.sources,
|
|
444
|
+
env: Object.keys(spec.env).sort()
|
|
445
|
+
}, "resolved attempt");
|
|
446
|
+
}
|
|
447
|
+
/** `POST api2.cursor.sh/auth/exchange`, `GET host/v1/*` or `POST host (path ~ /re/)`. */
|
|
448
|
+
function endpointLabel({ method, host, path }) {
|
|
449
|
+
const where = !path ? host : "exact" in path ? `${host}${path.exact}` : "startsWith" in path ? `${host}${path.startsWith}*` : `${host} (path ~ /${path.regex}/)`;
|
|
450
|
+
return `${method?.join("|") ?? "*"} ${where}`;
|
|
451
|
+
}
|
|
452
|
+
/** The resolved environment and what the gateway brokers, by name. */
|
|
453
|
+
function traceEnvironment(debug, spec, env) {
|
|
454
|
+
const r = env.record;
|
|
455
|
+
debug.log("env", `environment ${r.name} (digest ${r.digest.replace(/^sha256:/, "").slice(0, 12)})${r.parent ? ` · extends ${r.parent}` : ""}${r.overrides.length ? ` · overrides ${r.overrides.join(", ")}` : ""}`);
|
|
456
|
+
const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
|
|
457
|
+
debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
|
|
458
|
+
for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
|
|
459
|
+
const brokering = brokeringFor(spec.harness, spec.vertex);
|
|
460
|
+
if (brokering) {
|
|
461
|
+
const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
|
|
462
|
+
debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
|
|
463
|
+
}
|
|
464
|
+
debug.trace("env", "environment", {
|
|
465
|
+
record: r,
|
|
466
|
+
vars: Object.keys(env.vars).sort(),
|
|
467
|
+
checkVars: Object.keys(env.checkVars).sort(),
|
|
468
|
+
files: env.files.map((f) => f.path),
|
|
469
|
+
setup: env.setup,
|
|
470
|
+
skills: env.skills.map((s) => s.name),
|
|
471
|
+
workspaceSkills: env.workspaceSkills.map((s) => s.name),
|
|
472
|
+
plugins: env.plugins.map((p) => ({
|
|
473
|
+
name: p.name,
|
|
474
|
+
parts: p.parts,
|
|
475
|
+
files: p.files.length
|
|
476
|
+
})),
|
|
477
|
+
mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
|
|
478
|
+
transport: m.transport,
|
|
479
|
+
command: m.command,
|
|
480
|
+
args: m.args,
|
|
481
|
+
env: Object.keys(m.env)
|
|
482
|
+
} : {
|
|
483
|
+
transport: m.transport,
|
|
484
|
+
url: m.url,
|
|
485
|
+
headers: Object.keys(m.headers)
|
|
486
|
+
}])),
|
|
487
|
+
brokered: env.substitutions.map(({ name, hosts, in: where, phase }) => ({
|
|
488
|
+
name,
|
|
489
|
+
hosts,
|
|
490
|
+
in: where,
|
|
491
|
+
phase
|
|
492
|
+
}))
|
|
493
|
+
}, "resolved environment");
|
|
494
|
+
}
|
|
494
495
|
// src/attempt/environment.ts
|
|
495
496
|
/**
|
|
497
|
+
* How many VMs the hosts preflight kept for `spec`'s sandbox run at once, or null when it runs
|
|
498
|
+
* on no hosts or preflight left nothing to read. The plan's cap counts every host it names at its
|
|
499
|
+
* `vms`; a host skipped, or with memory for fewer, runs fewer.
|
|
500
|
+
*/
|
|
501
|
+
async function preparedVmCap(spec, root, runId) {
|
|
502
|
+
const { sandbox, harness } = spec;
|
|
503
|
+
if (sandbox.kind !== "tart" || !sandbox.hosts) return null;
|
|
504
|
+
const { preparedFor } = await import("./preflight-BUjk_6wT.js");
|
|
505
|
+
const { createProviders, preparedHosts } = await import("./providers-Dljf6al6.js");
|
|
506
|
+
const provider = createProviders().tart;
|
|
507
|
+
if (!provider) return null;
|
|
508
|
+
try {
|
|
509
|
+
const prepared = preparedFor(root, runId, provider, sandbox, harness);
|
|
510
|
+
const hosts = prepared ? preparedHosts(prepared) : null;
|
|
511
|
+
return hosts ? hosts.reduce((sum, h) => sum + h.vms, 0) : null;
|
|
512
|
+
} catch {
|
|
513
|
+
return null;
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
496
517
|
* Resolves the attempt's environment (files, skills, secrets and their placeholders)
|
|
497
518
|
* and finds the image its sandbox starts from, when its kind has one, which preflight prepared.
|
|
498
519
|
* Runs on the test's signal, before the eval timer starts.
|
|
@@ -501,7 +522,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
501
522
|
const { spec, record } = a;
|
|
502
523
|
const env = await resolveEnvironment(spec.environment, spec.env, root, where, layoutOf(spec.harness).rulesFile, spec.egress.policy, (w) => preparedWorkspace(root, runId, w), {
|
|
503
524
|
harness: spec.harness,
|
|
504
|
-
allowed: allowedHosts(spec.egress)
|
|
525
|
+
allowed: allowedHosts(spec.egress),
|
|
526
|
+
...spec.egress.setup ? { setupEgress: spec.egress.setup.policy } : {}
|
|
505
527
|
});
|
|
506
528
|
a.env = env;
|
|
507
529
|
record.environment = env.record;
|
|
@@ -516,8 +538,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
516
538
|
record.image = null;
|
|
517
539
|
const kind = sandboxKind(spec.sandbox);
|
|
518
540
|
if (kind.capabilities.images) {
|
|
519
|
-
const { preparedFor } = await import("./preflight-
|
|
520
|
-
const { attemptImage, createProviders, preparedImage } = await import("./providers-
|
|
541
|
+
const { preparedFor } = await import("./preflight-BUjk_6wT.js");
|
|
542
|
+
const { attemptImage, createProviders, preparedImage } = await import("./providers-Dljf6al6.js");
|
|
521
543
|
signal.throwIfAborted();
|
|
522
544
|
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
523
545
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -542,16 +564,88 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
542
564
|
const { ref, id, base, install, agentCli } = record.image;
|
|
543
565
|
debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
|
|
544
566
|
} else if (kind.capabilities.isolated) {
|
|
545
|
-
const { preparedFor } = await import("./preflight-
|
|
546
|
-
const { createProviders, vmImage } = await import("./providers-
|
|
567
|
+
const { preparedFor } = await import("./preflight-BUjk_6wT.js");
|
|
568
|
+
const { bakedImage, createProviders, preparedHosts, vmImage } = await import("./providers-Dljf6al6.js");
|
|
547
569
|
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
548
570
|
const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
|
|
549
571
|
const vm = prepared ? vmImage(prepared) : null;
|
|
550
572
|
if (vm) record.vmImage = vm;
|
|
573
|
+
const baked = prepared ? bakedImage(prepared) : null;
|
|
574
|
+
if (baked) a.image = baked;
|
|
575
|
+
const hosts = prepared ? preparedHosts(prepared) : null;
|
|
576
|
+
if (hosts) a.hosts = hosts;
|
|
551
577
|
const from = vm?.ref ?? ("image" in spec.sandbox ? spec.sandbox.image : void 0);
|
|
552
|
-
debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from it`);
|
|
578
|
+
debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from ${baked ? `${baked}, which has the harness baked in` : "it"}`);
|
|
553
579
|
} else debugLog("image", kind.capabilities.isolated ? `none: the ${spec.sandbox.kind} sandbox starts from its provider's base` : `none: the ${spec.sandbox.kind} sandbox runs on the host`);
|
|
554
580
|
}
|
|
581
|
+
// src/attempt/assertions.ts
|
|
582
|
+
const MESSAGE_LIMIT = 1024;
|
|
583
|
+
const LABEL_LIMIT = 60;
|
|
584
|
+
/** Masked first, so a secret the limit would cut through is masked whole. */
|
|
585
|
+
const clipped = (text, limit) => {
|
|
586
|
+
const s = redactSecrets(text);
|
|
587
|
+
return s.length > limit ? `${s.slice(0, limit)}…` : s;
|
|
588
|
+
};
|
|
589
|
+
const text = (error) => error instanceof Error ? error.message : String(error);
|
|
590
|
+
const shown = (value) => {
|
|
591
|
+
if (typeof value === "string") return value;
|
|
592
|
+
if (value instanceof RegExp) return String(value);
|
|
593
|
+
try {
|
|
594
|
+
return JSON.stringify(value) ?? String(value);
|
|
595
|
+
} catch {
|
|
596
|
+
return String(value);
|
|
597
|
+
}
|
|
598
|
+
};
|
|
599
|
+
const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
|
|
600
|
+
/**
|
|
601
|
+
* The records of an attempt's matcher calls, in call order. Without a label, one is
|
|
602
|
+
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
|
|
603
|
+
* any label already recorded, so each record's label is unique (a given `build #2` after two
|
|
604
|
+
* `build`s is `build #2 #2`).
|
|
605
|
+
*/
|
|
606
|
+
function assertionRecords(raw) {
|
|
607
|
+
const seen = /* @__PURE__ */ new Map();
|
|
608
|
+
const used = /* @__PURE__ */ new Set();
|
|
609
|
+
return raw.map((r) => {
|
|
610
|
+
const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.matcher === "toHaveRun" && runMatchLabel(r.args[0]) !== void 0 ? runMatchLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
611
|
+
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
612
|
+
let n = seen.get(base) ?? 0;
|
|
613
|
+
let label;
|
|
614
|
+
do {
|
|
615
|
+
n += 1;
|
|
616
|
+
label = n > 1 ? `${base} #${n}` : base;
|
|
617
|
+
} while (used.has(label));
|
|
618
|
+
seen.set(base, n);
|
|
619
|
+
used.add(label);
|
|
620
|
+
return {
|
|
621
|
+
label,
|
|
622
|
+
labelled: r.label !== void 0,
|
|
623
|
+
matcher: r.matcher,
|
|
624
|
+
status: r.status,
|
|
625
|
+
soft: r.soft,
|
|
626
|
+
...r.at ? { at: r.at } : {},
|
|
627
|
+
...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
|
|
628
|
+
...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
|
|
629
|
+
};
|
|
630
|
+
});
|
|
631
|
+
}
|
|
632
|
+
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
633
|
+
/**
|
|
634
|
+
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
635
|
+
* that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
|
|
636
|
+
* `CAUGHT_HINT` after its message.
|
|
637
|
+
*/
|
|
638
|
+
function checkFailure(raw) {
|
|
639
|
+
const failed = raw.find((r) => r.status !== "pass");
|
|
640
|
+
if (!failed) return void 0;
|
|
641
|
+
const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
|
|
642
|
+
if (failed.soft) return error;
|
|
643
|
+
const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
|
|
644
|
+
caught.name = error.name;
|
|
645
|
+
caught.message = `${error.message} ${CAUGHT_HINT}`;
|
|
646
|
+
caught.stack = error.stack?.replace(error.message, caught.message);
|
|
647
|
+
return caught;
|
|
648
|
+
}
|
|
555
649
|
// src/judge/size.ts
|
|
556
650
|
/** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
|
|
557
651
|
const CHARS_PER_TOKEN = 4;
|
|
@@ -607,6 +701,19 @@ function stateHint(state, question) {
|
|
|
607
701
|
* no model's limit, so this follows every provider error.
|
|
608
702
|
*/
|
|
609
703
|
const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
|
|
704
|
+
const count = (digits) => Number(digits.replaceAll(",", ""));
|
|
705
|
+
/**
|
|
706
|
+
* Whether the provider refused the call on a token limit this input alone is over, which no
|
|
707
|
+
* retry fixes: a request too large, OpenAI's `Limit 30000, Requested 45000`, or Anthropic's
|
|
708
|
+
* `of 30,000 input tokens per minute` against the input's estimated size.
|
|
709
|
+
*/
|
|
710
|
+
function overTokenLimit(message, state, question) {
|
|
711
|
+
if (/request too large/i.test(message)) return true;
|
|
712
|
+
const asked = /Limit:? ([\d,]+), Requested:? ([\d,]+)/i.exec(message);
|
|
713
|
+
if (asked) return count(asked[2]) > count(asked[1]);
|
|
714
|
+
const perMinute = /([\d,]+) (?:input )?tokens per minute/i.exec(message);
|
|
715
|
+
return !!perMinute && inputTokens(state, question) >= count(perMinute[1]);
|
|
716
|
+
}
|
|
610
717
|
// src/judge/state.ts
|
|
611
718
|
function turnState(t) {
|
|
612
719
|
return {
|
|
@@ -778,10 +885,16 @@ function judgeCaller(a, readFile) {
|
|
|
778
885
|
debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
|
|
779
886
|
if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
|
|
780
887
|
debugTrace("judge", "prompt", input, "judge prompt");
|
|
781
|
-
const failed = (message, retries) => {
|
|
888
|
+
const failed = (message, transient, retries) => {
|
|
782
889
|
a.judgeErrors.push({
|
|
783
890
|
call: order,
|
|
784
|
-
message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}
|
|
891
|
+
message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`,
|
|
892
|
+
transient,
|
|
893
|
+
matcher: judgeFailedMessage({
|
|
894
|
+
question,
|
|
895
|
+
model: name,
|
|
896
|
+
error: message
|
|
897
|
+
})
|
|
785
898
|
});
|
|
786
899
|
input.error = message;
|
|
787
900
|
if (retries?.length) input.retries = retries;
|
|
@@ -819,7 +932,7 @@ function judgeCaller(a, readFile) {
|
|
|
819
932
|
...unpinned,
|
|
820
933
|
gateway: entry || gateway
|
|
821
934
|
} : unpinned;
|
|
822
|
-
const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
|
|
935
|
+
const { verdict, retries, usage } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
|
|
823
936
|
...routed,
|
|
824
937
|
model,
|
|
825
938
|
threshold: atLeast
|
|
@@ -832,6 +945,7 @@ function judgeCaller(a, readFile) {
|
|
|
832
945
|
judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
|
|
833
946
|
...recorded,
|
|
834
947
|
...retries.length ? { retries } : {},
|
|
948
|
+
...usage ? { usage } : {},
|
|
835
949
|
call: order
|
|
836
950
|
});
|
|
837
951
|
const { score } = verdict;
|
|
@@ -853,7 +967,8 @@ function judgeCaller(a, readFile) {
|
|
|
853
967
|
if (a.signal.aborted) throw error;
|
|
854
968
|
const { retries } = error;
|
|
855
969
|
const message = error instanceof Error ? error.message : String(error);
|
|
856
|
-
|
|
970
|
+
const transient = isTransient(error) && !overTokenLimit(message, state, question);
|
|
971
|
+
return failed(isApiError(error) && !transient ? withInputSize(message, state, question) : message, transient, retries);
|
|
857
972
|
}
|
|
858
973
|
};
|
|
859
974
|
return async (question, given) => {
|
|
@@ -878,6 +993,191 @@ async function createSandbox(descriptor, options) {
|
|
|
878
993
|
throw new InfraError(`Failed to create ${kind.provider(descriptor)} sandbox: ${error.message}`);
|
|
879
994
|
}
|
|
880
995
|
}
|
|
996
|
+
// src/view/timeline.ts
|
|
997
|
+
const isToolLike = (e) => e.type === "file-edit" || e.type === "web-search" || e.type === "web-fetch" || e.type === "tool";
|
|
998
|
+
/**
|
|
999
|
+
* The events with each tool call's missing `durationMs` run to the next event's `atMs` (the
|
|
1000
|
+
* turn's end for the last), flagged `durationInferred`. A main model call ends it too: the agent
|
|
1001
|
+
* only calls the model again once the tool's result is back. A side call or a call that carries
|
|
1002
|
+
* the whole turn doesn't.
|
|
1003
|
+
*/
|
|
1004
|
+
function inferDurations(events, turnMs) {
|
|
1005
|
+
return events.map((e, i) => {
|
|
1006
|
+
if (!isToolLike(e) || e.durationMs !== null) return e;
|
|
1007
|
+
const next = events.slice(i + 1).find((n) => n.atMs > e.atMs && (n.type !== "model-call" || !n.stream && n.role !== "side"));
|
|
1008
|
+
const end = Math.max(e.atMs, next ? next.atMs : turnMs);
|
|
1009
|
+
return {
|
|
1010
|
+
...e,
|
|
1011
|
+
durationMs: end - e.atMs,
|
|
1012
|
+
durationInferred: true
|
|
1013
|
+
};
|
|
1014
|
+
});
|
|
1015
|
+
}
|
|
1016
|
+
const SHELL_TOOL = /shell|bash|terminal|command|exec/i;
|
|
1017
|
+
const OUTPUT_TOOL = /^(bash_?output|task_?output|await_?shell)$/i;
|
|
1018
|
+
const CURSOR_TERMINAL = /\.cursor\/projects\/[^"]*\/terminals\/[^"/]+\.txt/;
|
|
1019
|
+
const TERMINAL_COMMAND = /command:\s*("(?:[^"\\]|\\.)*")/;
|
|
1020
|
+
const record = (v) => v && typeof v === "object" ? v : {};
|
|
1021
|
+
/** A shell call's command, or null for another tool. */
|
|
1022
|
+
function commandOf(e) {
|
|
1023
|
+
if (e.type !== "tool" || !SHELL_TOOL.test(e.tool) || OUTPUT_TOOL.test(e.tool)) return null;
|
|
1024
|
+
const input = record(e.input);
|
|
1025
|
+
const command = input.command ?? input.cmd;
|
|
1026
|
+
return typeof command === "string" ? command.trim() : null;
|
|
1027
|
+
}
|
|
1028
|
+
const strings = (v) => typeof v === "string" ? [v] : v && typeof v === "object" ? Object.values(v).flatMap(strings) : [];
|
|
1029
|
+
const textOf = (v) => typeof v === "string" ? v : JSON.stringify(v) ?? "";
|
|
1030
|
+
/** The command a Cursor terminal file's header names, from a read of it. */
|
|
1031
|
+
function terminalCommand(e) {
|
|
1032
|
+
if (e.type !== "tool") return null;
|
|
1033
|
+
const match = strings(e.output).map((text) => TERMINAL_COMMAND.exec(text)).find(Boolean);
|
|
1034
|
+
if (!match?.[1]) return null;
|
|
1035
|
+
try {
|
|
1036
|
+
return String(JSON.parse(match[1])).trim();
|
|
1037
|
+
} catch {
|
|
1038
|
+
return null;
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
function backgrounded(e, named) {
|
|
1042
|
+
if (e.type !== "tool") return false;
|
|
1043
|
+
const input = record(e.input);
|
|
1044
|
+
if (input.run_in_background === true || input.is_background === true) return true;
|
|
1045
|
+
const command = commandOf(e);
|
|
1046
|
+
if (command === null) return false;
|
|
1047
|
+
return /(^|[^&])&$/.test(command) || /(^|[\s;&|])(nohup|setsid)\s/.test(command) || named.has(command);
|
|
1048
|
+
}
|
|
1049
|
+
function polls(e) {
|
|
1050
|
+
if (e.type !== "tool") return false;
|
|
1051
|
+
if (OUTPUT_TOOL.test(e.tool)) return true;
|
|
1052
|
+
const command = commandOf(e);
|
|
1053
|
+
if (command !== null) return /^(sleep|wait)(\s|$)/.test(command);
|
|
1054
|
+
return CURSOR_TERMINAL.test(textOf(e.input));
|
|
1055
|
+
}
|
|
1056
|
+
const endOf = (e) => e.atMs + (e.durationMs ?? 0);
|
|
1057
|
+
/** Work that ends a wait no poll has checked yet: an edit, or another command. */
|
|
1058
|
+
const works = (e) => e.type === "file-edit" || commandOf(e) !== null;
|
|
1059
|
+
/**
|
|
1060
|
+
* The background jobs the agent waited on, a heuristic over the transcript: a span opens at a
|
|
1061
|
+
* call run in the background or a poll and takes in each poll after it. Until its first poll,
|
|
1062
|
+
* lookups (reads, searches) leave it open and an edit or another command drops it; after it,
|
|
1063
|
+
* the next call that isn't a poll closes it. A background call no poll followed isn't waited on.
|
|
1064
|
+
*/
|
|
1065
|
+
function waitSpans(events) {
|
|
1066
|
+
const named = new Set(events.map(terminalCommand).filter((c) => c !== null));
|
|
1067
|
+
const spans = [];
|
|
1068
|
+
let open = null;
|
|
1069
|
+
const close = () => {
|
|
1070
|
+
if (open?.polls.length) spans.push(open);
|
|
1071
|
+
open = null;
|
|
1072
|
+
};
|
|
1073
|
+
events.forEach((e, i) => {
|
|
1074
|
+
if (!isToolLike(e) || e.subagent) return;
|
|
1075
|
+
if (backgrounded(e, named)) {
|
|
1076
|
+
if (open?.polls.length) close();
|
|
1077
|
+
open ??= {
|
|
1078
|
+
startMs: e.atMs,
|
|
1079
|
+
endMs: endOf(e),
|
|
1080
|
+
background: i,
|
|
1081
|
+
polls: []
|
|
1082
|
+
};
|
|
1083
|
+
} else if (polls(e)) {
|
|
1084
|
+
open ??= {
|
|
1085
|
+
startMs: e.atMs,
|
|
1086
|
+
endMs: e.atMs,
|
|
1087
|
+
background: null,
|
|
1088
|
+
polls: []
|
|
1089
|
+
};
|
|
1090
|
+
open.polls.push(i);
|
|
1091
|
+
open.endMs = Math.max(open.endMs, endOf(e));
|
|
1092
|
+
} else if (open?.polls.length || works(e)) close();
|
|
1093
|
+
});
|
|
1094
|
+
close();
|
|
1095
|
+
return spans;
|
|
1096
|
+
}
|
|
1097
|
+
const PRIORITY = [
|
|
1098
|
+
"waiting",
|
|
1099
|
+
"tool",
|
|
1100
|
+
"model"
|
|
1101
|
+
];
|
|
1102
|
+
/** The turn from 0 to `turnMs` as segments, each moment the first of waiting, tool, model and other. */
|
|
1103
|
+
function segments(events, turnMs) {
|
|
1104
|
+
const marks = [];
|
|
1105
|
+
const add = (kind, start, end) => {
|
|
1106
|
+
const s = Math.max(0, Math.min(turnMs, start));
|
|
1107
|
+
const e = Math.max(0, Math.min(turnMs, end));
|
|
1108
|
+
if (e <= s) return;
|
|
1109
|
+
marks.push({
|
|
1110
|
+
at: s,
|
|
1111
|
+
kind,
|
|
1112
|
+
delta: 1
|
|
1113
|
+
}, {
|
|
1114
|
+
at: e,
|
|
1115
|
+
kind,
|
|
1116
|
+
delta: -1
|
|
1117
|
+
});
|
|
1118
|
+
};
|
|
1119
|
+
for (const span of waitSpans(events)) add(0, span.startMs, span.endMs);
|
|
1120
|
+
for (const e of events) if (isToolLike(e)) add(1, e.atMs, endOf(e));
|
|
1121
|
+
else if (e.type === "model-call" && !e.stream) add(2, e.atMs, e.atMs + e.durationMs);
|
|
1122
|
+
marks.sort((a, b) => a.at - b.at);
|
|
1123
|
+
const depth = [
|
|
1124
|
+
0,
|
|
1125
|
+
0,
|
|
1126
|
+
0
|
|
1127
|
+
];
|
|
1128
|
+
const out = [];
|
|
1129
|
+
let at = 0;
|
|
1130
|
+
const push = (kind, end) => {
|
|
1131
|
+
if (end <= at) return;
|
|
1132
|
+
const last = out.at(-1);
|
|
1133
|
+
if (last?.kind === kind) last.endMs = end;
|
|
1134
|
+
else out.push({
|
|
1135
|
+
kind,
|
|
1136
|
+
startMs: at,
|
|
1137
|
+
endMs: end
|
|
1138
|
+
});
|
|
1139
|
+
at = end;
|
|
1140
|
+
};
|
|
1141
|
+
const current = () => PRIORITY[depth.findIndex((d) => d > 0)] ?? "other";
|
|
1142
|
+
for (const m of marks) {
|
|
1143
|
+
push(current(), m.at);
|
|
1144
|
+
depth[m.kind] = (depth[m.kind] ?? 0) + m.delta;
|
|
1145
|
+
}
|
|
1146
|
+
push(current(), turnMs);
|
|
1147
|
+
return out;
|
|
1148
|
+
}
|
|
1149
|
+
/** A turn's `timeSplit`, from its transcript (durations inferred) and length. */
|
|
1150
|
+
function timeSplit(events, turnMs) {
|
|
1151
|
+
const total = {
|
|
1152
|
+
waiting: 0,
|
|
1153
|
+
tool: 0,
|
|
1154
|
+
model: 0,
|
|
1155
|
+
other: 0
|
|
1156
|
+
};
|
|
1157
|
+
for (const s of segments(events, turnMs)) total[s.kind] += s.endMs - s.startMs;
|
|
1158
|
+
const seen = events.some((e) => e.type === "model-call" && !e.stream);
|
|
1159
|
+
const waitingMs = Math.round(total.waiting);
|
|
1160
|
+
const toolMs = Math.round(total.tool);
|
|
1161
|
+
const modelMs = Math.round(total.model);
|
|
1162
|
+
return {
|
|
1163
|
+
modelMs: seen ? modelMs : null,
|
|
1164
|
+
toolMs,
|
|
1165
|
+
waitingMs,
|
|
1166
|
+
otherMs: Math.max(0, Math.round(turnMs) - waitingMs - toolMs - modelMs)
|
|
1167
|
+
};
|
|
1168
|
+
}
|
|
1169
|
+
/** The turns' splits summed; `modelMs` null when any turn's is, undefined with none. */
|
|
1170
|
+
function sumSplits(splits) {
|
|
1171
|
+
const known = splits.filter((s) => s !== void 0);
|
|
1172
|
+
if (known.length === 0) return void 0;
|
|
1173
|
+
const sum = (f) => known.reduce((n, s) => n + f(s), 0);
|
|
1174
|
+
return {
|
|
1175
|
+
modelMs: known.every((s) => s.modelMs !== null) ? sum((s) => s.modelMs ?? 0) : null,
|
|
1176
|
+
toolMs: sum((s) => s.toolMs),
|
|
1177
|
+
waitingMs: sum((s) => s.waitingMs),
|
|
1178
|
+
otherMs: sum((s) => s.otherMs)
|
|
1179
|
+
};
|
|
1180
|
+
}
|
|
881
1181
|
// src/attempt/record.ts
|
|
882
1182
|
function classify(error, signal) {
|
|
883
1183
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -909,6 +1209,34 @@ function isJudges(a, error, raw) {
|
|
|
909
1209
|
if (a.judgeThrown.has(message)) return true;
|
|
910
1210
|
return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
|
|
911
1211
|
}
|
|
1212
|
+
/** The error the attempt's failed judge calls fail it with once its body is done, if any failed. */
|
|
1213
|
+
function judgeErrorsMessage(a) {
|
|
1214
|
+
const messages = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
|
|
1215
|
+
const [only, ...more] = messages;
|
|
1216
|
+
if (!only) return void 0;
|
|
1217
|
+
return more.length ? `${more.length + 1} judge calls failed:\n${messages.join("\n")}` : only;
|
|
1218
|
+
}
|
|
1219
|
+
/** A matcher's message without the `label: ` its `expect` puts first. */
|
|
1220
|
+
const unlabeled = (r) => {
|
|
1221
|
+
const message = messageOf(r.error);
|
|
1222
|
+
const prefix = r.label === void 0 ? "" : `${r.label}: `;
|
|
1223
|
+
return message.startsWith(prefix) ? message.slice(prefix.length) : message;
|
|
1224
|
+
};
|
|
1225
|
+
/**
|
|
1226
|
+
* Whether judge calls the provider couldn't answer (a rate limit, a 5xx, overload, the network,
|
|
1227
|
+
* after the retries) are all that failed the attempt: every failed call was transient, and every
|
|
1228
|
+
* check that didn't hold is a `toPass()` on one of them, not polled. A verdict that answered, a
|
|
1229
|
+
* non-transient judge error (a bad key or model, an input over the provider's limit) or any other
|
|
1230
|
+
* check makes it the attempt's own failure. By message, since another language's runner hands back
|
|
1231
|
+
* its own copy of the error.
|
|
1232
|
+
*/
|
|
1233
|
+
function judgeOutage(a, error, raw) {
|
|
1234
|
+
if (!a.judgeErrors.length || !a.judgeErrors.every((e) => e.transient)) return false;
|
|
1235
|
+
const message = messageOf(error);
|
|
1236
|
+
if (a.judgeThrown.has(message) && message !== judgeErrorsMessage(a)) return false;
|
|
1237
|
+
const unanswered = new Set(a.judgeErrors.map((e) => e.matcher));
|
|
1238
|
+
return raw.every((r) => r.status === "pass" || judged(r) && !r.matcher.startsWith("poll.") && unanswered.has(unlabeled(r)));
|
|
1239
|
+
}
|
|
912
1240
|
/**
|
|
913
1241
|
* The error that fails the attempt: a judge's gives way to the first other check that didn't hold
|
|
914
1242
|
* (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
|
|
@@ -921,7 +1249,8 @@ function attemptError(a, error, raw) {
|
|
|
921
1249
|
}
|
|
922
1250
|
/**
|
|
923
1251
|
* Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
|
|
924
|
-
* assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
|
|
1252
|
+
* assertion, which is the judge's when `error`, from `attemptError`, is a judge's. A judge's that
|
|
1253
|
+
* only a provider outage caused is an infra error, so `--retry-errors` and `--resume` run it again.
|
|
925
1254
|
*/
|
|
926
1255
|
function recordFailure(a, error, timedOut, raw) {
|
|
927
1256
|
const { record } = a;
|
|
@@ -941,7 +1270,8 @@ function recordFailure(a, error, timedOut, raw) {
|
|
|
941
1270
|
setBy: setBy ?? []
|
|
942
1271
|
};
|
|
943
1272
|
}
|
|
944
|
-
if (
|
|
1273
|
+
if (error instanceof ProviderStopError && !timedOut) record.error.provider = { ...error.stop };
|
|
1274
|
+
if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = judgeOutage(a, error, raw) ? "infra" : "judge";
|
|
945
1275
|
record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
|
|
946
1276
|
}
|
|
947
1277
|
/**
|
|
@@ -965,6 +1295,8 @@ async function recordOom(a, sandbox) {
|
|
|
965
1295
|
a.record.outcome = "infra_error";
|
|
966
1296
|
return new InfraError(message);
|
|
967
1297
|
}
|
|
1298
|
+
/** The gateway tags entries with the phase it was switched to, which is always one of the attempt's. */
|
|
1299
|
+
const phased = (entry) => entry;
|
|
968
1300
|
/**
|
|
969
1301
|
* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
|
|
970
1302
|
* Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
|
|
@@ -974,17 +1306,23 @@ function recordEgress(a, sandbox) {
|
|
|
974
1306
|
const traffic = sandbox?.traffic?.();
|
|
975
1307
|
const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
|
|
976
1308
|
const refused = traffic?.blocked ?? [];
|
|
977
|
-
const isExpected = (h) => !!expected?.hosts.includes(h.host);
|
|
1309
|
+
const isExpected = (h) => h.phase !== "setup" && !!expected?.hosts.includes(h.host);
|
|
978
1310
|
const expectedBlocked = refused.filter(isExpected);
|
|
1311
|
+
const { setup } = a.spec.egress;
|
|
1312
|
+
const direct = isDirect(a.spec.egress.policy) || !!setup && isDirect(setup.policy);
|
|
979
1313
|
a.record.egress = {
|
|
980
1314
|
policy: a.spec.egress.policy,
|
|
981
1315
|
allowed: allowedHosts(a.spec.egress),
|
|
1316
|
+
...setup ? { setup: {
|
|
1317
|
+
policy: setup.policy,
|
|
1318
|
+
allowed: allowedHosts(setup)
|
|
1319
|
+
} } : {},
|
|
982
1320
|
recorded: !!traffic,
|
|
983
|
-
...traffic &&
|
|
984
|
-
contacted: traffic?.contacted ?? [],
|
|
985
|
-
blocked: refused.filter((h) => !isExpected(h)),
|
|
986
|
-
expectedBlocked,
|
|
987
|
-
unresolved: traffic?.unresolved ?? []
|
|
1321
|
+
...traffic && direct ? { partial: true } : {},
|
|
1322
|
+
contacted: (traffic?.contacted ?? []).map(phased),
|
|
1323
|
+
blocked: refused.filter((h) => !isExpected(h)).map(phased),
|
|
1324
|
+
expectedBlocked: expectedBlocked.map(phased),
|
|
1325
|
+
unresolved: (traffic?.unresolved ?? []).map(phased)
|
|
988
1326
|
};
|
|
989
1327
|
if (expected && expectedBlocked.length) {
|
|
990
1328
|
const hosts = expectedBlocked.map((h) => `${h.host}:${h.port} ×${h.connections}`).join(", ");
|
|
@@ -999,6 +1337,25 @@ function recordEgress(a, sandbox) {
|
|
|
999
1337
|
if (cost !== void 0) a.record.reportedCostUsd = cost;
|
|
1000
1338
|
}
|
|
1001
1339
|
/**
|
|
1340
|
+
* Each turn's model calls and inferred tool durations in its transcript, and where its time
|
|
1341
|
+
* went. After teardown, so a stream that outlived its turn (Cursor's) has ended.
|
|
1342
|
+
*/
|
|
1343
|
+
function recordTimeline(a, sandbox) {
|
|
1344
|
+
const calls = sandbox?.modelCalls?.() ?? [];
|
|
1345
|
+
for (const turn of a.record.turns) {
|
|
1346
|
+
const events = inferDurations([...turn.transcript, ...modelCallEvents(calls, {
|
|
1347
|
+
startedAt: Date.parse(turn.startedAt),
|
|
1348
|
+
durationMs: turn.durationMs,
|
|
1349
|
+
model: turn.model,
|
|
1350
|
+
steps: a.turnSteps.get(turn) ?? []
|
|
1351
|
+
})].sort((x, y) => x.atMs - y.atMs), turn.durationMs);
|
|
1352
|
+
turn.transcript = events;
|
|
1353
|
+
turn.timeSplit = timeSplit(events, turn.durationMs);
|
|
1354
|
+
}
|
|
1355
|
+
const split = sumSplits(a.record.turns.map((t) => t.timeSplit));
|
|
1356
|
+
if (split) a.record.timeSplit = split;
|
|
1357
|
+
}
|
|
1358
|
+
/**
|
|
1002
1359
|
* The record as its staging file holds it: redacted, with a secret split across a turn's raw
|
|
1003
1360
|
* stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
|
|
1004
1361
|
*/
|
|
@@ -1169,18 +1526,32 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
|
|
|
1169
1526
|
// src/sandbox/phases.ts
|
|
1170
1527
|
const strip = ({ phase: _, ...sub }) => sub;
|
|
1171
1528
|
/**
|
|
1172
|
-
* Which brokered secrets the gateway swaps in when
|
|
1173
|
-
* at least one check or setup step is running, so a
|
|
1174
|
-
* the placeholder.
|
|
1529
|
+
* Which brokered secrets the gateway swaps in when, and which egress policy it enforces when.
|
|
1530
|
+
* `to: 'checks'` secrets are active only while at least one check or setup step is running, so a
|
|
1531
|
+
* request the agent makes between them gets the placeholder. A setup policy of its own holds from
|
|
1532
|
+
* the sandbox's start until `agent()`; every check runs under the agent's.
|
|
1175
1533
|
*/
|
|
1176
|
-
function brokerPhases(box, subs) {
|
|
1534
|
+
function brokerPhases(box, subs, egress) {
|
|
1177
1535
|
const agent = subs.filter((s) => s.phase === "agent").map(strip);
|
|
1178
1536
|
const all = subs.map(strip);
|
|
1179
1537
|
const perPhase = box.setSubstitutions && all.length > agent.length;
|
|
1180
1538
|
let running = 0;
|
|
1539
|
+
const enter = async (phase, planned) => {
|
|
1540
|
+
if (!box.setEgress) throw new InfraError(`Can't switch the sandbox's egress between the setup and agent phases: its egress gateway isn't running`);
|
|
1541
|
+
debugLog("container", `egress → ${phase}: ${phaseLabel(planned) || "proxy"}`);
|
|
1542
|
+
await box.setEgress({
|
|
1543
|
+
policy: gatewayPolicy(planned),
|
|
1544
|
+
direct: isDirect(planned.policy),
|
|
1545
|
+
phase
|
|
1546
|
+
});
|
|
1547
|
+
};
|
|
1181
1548
|
return {
|
|
1182
1549
|
async start() {
|
|
1183
1550
|
if (all.length > 0) await box.setSubstitutions?.(agent);
|
|
1551
|
+
if (egress?.setup) await enter("setup", egress.setup);
|
|
1552
|
+
},
|
|
1553
|
+
async agent() {
|
|
1554
|
+
if (egress?.setup) await enter("agent", egress);
|
|
1184
1555
|
},
|
|
1185
1556
|
async checks(fn) {
|
|
1186
1557
|
if (!perPhase) return fn();
|
|
@@ -1321,7 +1692,8 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
|
1321
1692
|
artifactBlobs: /* @__PURE__ */ new Map(),
|
|
1322
1693
|
capturePages: /* @__PURE__ */ new Map(),
|
|
1323
1694
|
stepBlobs: /* @__PURE__ */ new Map(),
|
|
1324
|
-
snapshotDir
|
|
1695
|
+
snapshotDir,
|
|
1696
|
+
turnSteps: /* @__PURE__ */ new Map()
|
|
1325
1697
|
};
|
|
1326
1698
|
return a;
|
|
1327
1699
|
}
|
|
@@ -1456,7 +1828,7 @@ async function copyWorkspace(box, workspace, signal) {
|
|
|
1456
1828
|
if (r.exitCode !== 0) throw new InfraError(`Couldn't unpack the workspace ${workspace.label} into the sandbox (the image needs tar): ${tailOf(r.stderr || r.stdout)}`);
|
|
1457
1829
|
if (workspace.init) {
|
|
1458
1830
|
const at = "2000-01-01T00:00:00Z";
|
|
1459
|
-
const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote(`${workspace.label} at ${workspace.commit}`)}`, {
|
|
1831
|
+
const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote$1(`${workspace.label} at ${workspace.commit}`)}`, {
|
|
1460
1832
|
signal,
|
|
1461
1833
|
env: {
|
|
1462
1834
|
GIT_AUTHOR_NAME: "reeval",
|
|
@@ -1471,13 +1843,87 @@ async function copyWorkspace(box, workspace, signal) {
|
|
|
1471
1843
|
}
|
|
1472
1844
|
debugLog("env", `workspace ${workspace.label} at ${workspace.commit.slice(0, 12)} (${tar.byteLength} bytes of tar${workspace.init ? ", with a .git" : ""}) in ${formatDuration(performance.now() - start)}`);
|
|
1473
1845
|
}
|
|
1846
|
+
/** Bytes per tar: each is built in memory and piped in whole. */
|
|
1847
|
+
const TAR_BATCH_BYTES = 67108864;
|
|
1848
|
+
/**
|
|
1849
|
+
* Writes `files` (workspace-relative, or `~/` for the home directory) with one tar per
|
|
1850
|
+
* destination: on a container or VM each `writeFile` is a process of its own, which a vendored
|
|
1851
|
+
* directory of thousands of files turns into minutes. A lone file is written as it is, and a
|
|
1852
|
+
* sandbox where the tar doesn't unpack gets a write per file.
|
|
1853
|
+
*/
|
|
1854
|
+
async function writeFiles(box, files, signal) {
|
|
1855
|
+
const home = files.flatMap((f) => f.path.startsWith("~/") ? [{
|
|
1856
|
+
...f,
|
|
1857
|
+
path: f.path.slice(2)
|
|
1858
|
+
}] : []);
|
|
1859
|
+
await unpackFiles(box, "", files.filter((f) => !f.path.startsWith("~/")), signal);
|
|
1860
|
+
await unpackFiles(box, "~/", home, signal);
|
|
1861
|
+
}
|
|
1862
|
+
async function unpackFiles(box, prefix, files, signal) {
|
|
1863
|
+
const one = async () => {
|
|
1864
|
+
for (const f of files) await box.writeFile(`${prefix}${f.path}`, f.content);
|
|
1865
|
+
};
|
|
1866
|
+
if (files.length < 2) return one();
|
|
1867
|
+
const entries = files.map((f) => ({
|
|
1868
|
+
type: "file",
|
|
1869
|
+
path: f.path,
|
|
1870
|
+
mode: 420,
|
|
1871
|
+
mtime: 0,
|
|
1872
|
+
data: typeof f.content === "string" ? Buffer.from(f.content, "utf8") : Buffer.from(f.content.buffer, f.content.byteOffset, f.content.byteLength)
|
|
1873
|
+
}));
|
|
1874
|
+
const batches = [[]];
|
|
1875
|
+
let size = 0;
|
|
1876
|
+
for (const e of entries) {
|
|
1877
|
+
if (size + e.data.length > TAR_BATCH_BYTES && batches[batches.length - 1]?.length) {
|
|
1878
|
+
batches.push([]);
|
|
1879
|
+
size = 0;
|
|
1880
|
+
}
|
|
1881
|
+
batches[batches.length - 1]?.push(e);
|
|
1882
|
+
size += e.data.length;
|
|
1883
|
+
}
|
|
1884
|
+
const dir = shellQuote$1(prefix ? box.home : box.workdir);
|
|
1885
|
+
const name = `.reeval-files-${randomUUID()}.tar`;
|
|
1886
|
+
const unpack = [
|
|
1887
|
+
"unset TAR_OPTIONS",
|
|
1888
|
+
`cd ${dir} || exit 1`,
|
|
1889
|
+
"d=$PWD",
|
|
1890
|
+
`t=$(mktemp -d) || { rm -f ${name}; exit 1; }`,
|
|
1891
|
+
"(",
|
|
1892
|
+
` tar -xmf "$d/${name}" -C "$t" && cd "$t" && find . -type f > "$t.all" || exit $?`,
|
|
1893
|
+
" : > \"$t.new\"",
|
|
1894
|
+
" while IFS= read -r f; do",
|
|
1895
|
+
" if [ -e \"$d/$f\" ] || [ -L \"$d/$f\" ]; then",
|
|
1896
|
+
" cat \"$f\" > \"$d/$f\" || exit 1",
|
|
1897
|
+
" continue",
|
|
1898
|
+
" fi",
|
|
1899
|
+
` p=\${f%/*}`,
|
|
1900
|
+
` while [ "$p" != . ] && [ ! -L "$d/$p" ]; do p=\${p%/*}; done`,
|
|
1901
|
+
" if [ \"$p\" = . ]; then printf '%s\\n' \"$f\" >> \"$t.new\"",
|
|
1902
|
+
` else mkdir -p "$d/\${f%/*}" && cat "$f" > "$d/$f" || exit 1`,
|
|
1903
|
+
" fi",
|
|
1904
|
+
" done < \"$t.all\"",
|
|
1905
|
+
" [ ! -s \"$t.new\" ] || { tar -cf \"$t.tar\" -T \"$t.new\" && cd \"$d\" && tar -xf \"$t.tar\"; }",
|
|
1906
|
+
")",
|
|
1907
|
+
"s=$?",
|
|
1908
|
+
`cd "$d"; rm -rf "$t" "$t.all" "$t.new" "$t.tar" ${name}`,
|
|
1909
|
+
"exit $s"
|
|
1910
|
+
].join("\n");
|
|
1911
|
+
for (const batch of batches) {
|
|
1912
|
+
await box.writeFile(`${prefix}${name}`, writeTar(batch));
|
|
1913
|
+
const r = await box.exec(unpack, { signal });
|
|
1914
|
+
if (r.exitCode !== 0) {
|
|
1915
|
+
debugLog("env", `the files' tar didn't unpack (exit ${r.exitCode}: ${tailOf(r.stderr || r.stdout)}): writing ${files.length} files one at a time`);
|
|
1916
|
+
return one();
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1474
1920
|
/**
|
|
1475
1921
|
* A built plugin's directory, copied from where its layer left it in the image, less the parts
|
|
1476
1922
|
* `only` leaves out, with the manifest as reeval read it.
|
|
1477
1923
|
*/
|
|
1478
1924
|
async function copyBuiltPlugin(a, box, plugin, dir) {
|
|
1479
|
-
const dropped = droppedPaths(plugin.parts).map((p) => shellQuote(`${dir}/${p}`));
|
|
1480
|
-
const r = await box.exec(`mkdir -p ${shellQuote(dir)} && cp -a ${shellQuote(plugin.built.dir)}/. ${shellQuote(dir)}/${dropped.length ? ` && rm -rf ${dropped.join(" ")}` : ""}`, { signal: a.signal });
|
|
1925
|
+
const dropped = droppedPaths(plugin.parts).map((p) => shellQuote$1(`${dir}/${p}`));
|
|
1926
|
+
const r = await box.exec(`mkdir -p ${shellQuote$1(dir)} && cp -a ${shellQuote$1(plugin.built.dir)}/. ${shellQuote$1(dir)}/${dropped.length ? ` && rm -rf ${dropped.join(" ")}` : ""}`, { signal: a.signal });
|
|
1481
1927
|
if (r.exitCode !== 0) throw new InfraError(`Couldn't copy the ${plugin.name} plugin's build from ${plugin.built.tag}: ${tailOf(r.stderr || r.stdout)}`);
|
|
1482
1928
|
const manifest = plugin.files.find((f) => f.path === PLUGIN_MANIFEST);
|
|
1483
1929
|
if (manifest) await box.writeFile(`${dir}/${PLUGIN_MANIFEST}`, manifest.content);
|
|
@@ -1500,7 +1946,7 @@ async function copyPlugins(a, box, env) {
|
|
|
1500
1946
|
else for (const f of plugin.files) await box.writeFile(`${at}/${f.path}`, f.content);
|
|
1501
1947
|
const executable = built ? [] : plugin.files.flatMap((f) => f.executable ? [`${dir}/${f.path}`] : []);
|
|
1502
1948
|
if (executable.length) {
|
|
1503
|
-
const r = await box.exec(`chmod +x ${executable.map(shellQuote).join(" ")}`, { signal: a.signal });
|
|
1949
|
+
const r = await box.exec(`chmod +x ${executable.map(shellQuote$1).join(" ")}`, { signal: a.signal });
|
|
1504
1950
|
if (r.exitCode !== 0) throw new InfraError(`Couldn't make the ${plugin.name} plugin's ${executable.length} executable files executable: ${tailOf(r.stderr || r.stdout)}`);
|
|
1505
1951
|
}
|
|
1506
1952
|
const root = (v) => v.replaceAll(PLUGIN_ROOT, dir);
|
|
@@ -1533,7 +1979,7 @@ async function copyPlugins(a, box, env) {
|
|
|
1533
1979
|
* only, so the sandbox has `find`.
|
|
1534
1980
|
*/
|
|
1535
1981
|
async function unpackedFromCopy(a, box, dir) {
|
|
1536
|
-
const r = await box.exec(`cd ${shellQuote(dir)} && find rules agents commands hooks -maxdepth 1 -type f 2>/dev/null; true`, { signal: a.signal });
|
|
1982
|
+
const r = await box.exec(`cd ${shellQuote$1(dir)} && find rules agents commands hooks -maxdepth 1 -type f 2>/dev/null; true`, { signal: a.signal });
|
|
1537
1983
|
const out = [];
|
|
1538
1984
|
for (const path of r.stdout.split("\n").filter(Boolean)) {
|
|
1539
1985
|
const content = await box.readFile(`${dir}/${path}`);
|
|
@@ -1562,7 +2008,7 @@ async function writeUnpackedPlugins(a, box, env, plugins, unpack) {
|
|
|
1562
2008
|
const files = unpackPlugins(placed, unpack, unpack.hooks ? await box.readFile(unpack.hooks.file) : null);
|
|
1563
2009
|
const fresh = files.filter((f) => f.path !== unpack.hooks?.file);
|
|
1564
2010
|
if (fresh.length) {
|
|
1565
|
-
const taken = (await box.exec(`for p in ${fresh.map((f) => shellQuote(f.path)).join(" ")}; do { [ -e "$p" ] || [ -L "$p" ]; } && echo "$p"; done; true`, { signal: a.signal })).stdout.trim().split("\n").filter(Boolean);
|
|
2011
|
+
const taken = (await box.exec(`for p in ${fresh.map((f) => shellQuote$1(f.path)).join(" ")}; do { [ -e "$p" ] || [ -L "$p" ]; } && echo "$p"; done; true`, { signal: a.signal })).stdout.trim().split("\n").filter(Boolean);
|
|
1566
2012
|
if (taken.length) throw new InfraError(`The workspace already has ${taken.join(", ")}, which a plugin would replace: rename the plugin's file, or leave the plugin out of this cell`);
|
|
1567
2013
|
}
|
|
1568
2014
|
for (const f of files) {
|
|
@@ -1596,40 +2042,71 @@ async function writePluginMcp(box, env, plugins, servers) {
|
|
|
1596
2042
|
debugLog("env", `plugin ${placed.name}'s MCP servers → ${at}/.mcp.json, through the timer`);
|
|
1597
2043
|
}
|
|
1598
2044
|
}
|
|
2045
|
+
/** How long a start that failed on a transient error waits before it's tried again. */
|
|
2046
|
+
const TRANSIENT_START_DELAY_MS = 2e3;
|
|
1599
2047
|
/**
|
|
1600
|
-
* Starts
|
|
1601
|
-
* is aborted (and closed, should it finish later) and started once more
|
|
1602
|
-
*
|
|
1603
|
-
*
|
|
2048
|
+
* Starts a harness session within `timeoutMs`, at most twice. A start that doesn't become ready in
|
|
2049
|
+
* time is aborted (and closed, should it finish later) and started once more, and so is one that
|
|
2050
|
+
* fails on a transient {@link InfraError} (a 5xx or a network error): nothing has run yet, so the
|
|
2051
|
+
* retry can't change what the attempt measures. Each retry is passed to `onRetry` to record. A
|
|
2052
|
+
* second timeout or transient failure, or any other failure, is the attempt's infra error.
|
|
1604
2053
|
*/
|
|
1605
|
-
async function
|
|
1606
|
-
const {
|
|
1607
|
-
const ms = startTimeoutMs(descriptor);
|
|
1608
|
-
const late = `${descriptor.kind}: session did not become ready within ${formatDuration(ms)}`;
|
|
2054
|
+
async function startWithRetry(start, { kind, timeoutMs, signal, onRetry, transientDelayMs = TRANSIENT_START_DELAY_MS }) {
|
|
2055
|
+
const late = `${kind}: session did not become ready within ${formatDuration(timeoutMs)}`;
|
|
1609
2056
|
for (let tries = 1;; tries++) {
|
|
1610
2057
|
const stop = new AbortController();
|
|
1611
2058
|
let timer;
|
|
1612
2059
|
const timedOut = new Promise((resolve) => {
|
|
1613
|
-
timer = setTimeout(() => resolve(null),
|
|
2060
|
+
timer = setTimeout(() => resolve(null), timeoutMs);
|
|
1614
2061
|
});
|
|
1615
|
-
const
|
|
1616
|
-
const
|
|
2062
|
+
const startedAt = Date.now();
|
|
2063
|
+
const starting = start(AbortSignal.any([signal, stop.signal]));
|
|
2064
|
+
let session;
|
|
2065
|
+
try {
|
|
2066
|
+
session = await Promise.race([starting, timedOut]);
|
|
2067
|
+
} catch (error) {
|
|
2068
|
+
signal.throwIfAborted();
|
|
2069
|
+
if (!(error instanceof InfraError) || !error.transient) throw error;
|
|
2070
|
+
if (tries === 2) throw new InfraError(`${error.message} (tried twice)`);
|
|
2071
|
+
onRetry({
|
|
2072
|
+
error: error.message,
|
|
2073
|
+
waitedMs: Date.now() - startedAt
|
|
2074
|
+
});
|
|
2075
|
+
debugLog("harness", `${error.message}; starting it once more`);
|
|
2076
|
+
await new Promise((resolve) => setTimeout(resolve, transientDelayMs));
|
|
2077
|
+
signal.throwIfAborted();
|
|
2078
|
+
continue;
|
|
2079
|
+
} finally {
|
|
2080
|
+
clearTimeout(timer);
|
|
2081
|
+
}
|
|
1617
2082
|
if (session) return session;
|
|
1618
2083
|
stop.abort(new InfraError(late));
|
|
1619
2084
|
starting.then((s) => s.close().catch(() => {}), () => {});
|
|
1620
|
-
|
|
2085
|
+
signal.throwIfAborted();
|
|
1621
2086
|
if (tries === 2) throw new InfraError(`${late} (tried twice)`);
|
|
1622
|
-
|
|
2087
|
+
onRetry({
|
|
1623
2088
|
error: late,
|
|
1624
|
-
waitedMs:
|
|
1625
|
-
}
|
|
2089
|
+
waitedMs: timeoutMs
|
|
2090
|
+
});
|
|
1626
2091
|
debugLog("harness", `${late}; starting it once more`);
|
|
1627
2092
|
}
|
|
1628
2093
|
}
|
|
2094
|
+
function startHarness(a, harness, box, options) {
|
|
2095
|
+
const { harness: descriptor } = a.spec;
|
|
2096
|
+
return startWithRetry((signal) => harness.start(box, signal, options), {
|
|
2097
|
+
kind: descriptor.kind,
|
|
2098
|
+
timeoutMs: startTimeoutMs(descriptor),
|
|
2099
|
+
signal: a.signal,
|
|
2100
|
+
onRetry: (retry) => {
|
|
2101
|
+
a.record.startRetries = [...a.record.startRetries ?? [], retry];
|
|
2102
|
+
}
|
|
2103
|
+
});
|
|
2104
|
+
}
|
|
1629
2105
|
/**
|
|
1630
2106
|
* Starts the sandbox (with the brokered secrets its gateway swaps in), writes the
|
|
1631
2107
|
* environment's files, workspace skills and plugins, runs its setup steps, snapshots the workspace for
|
|
1632
|
-
* the diff, and starts the harness session with the
|
|
2108
|
+
* the diff, switches to the agent's egress, and starts the harness session with the
|
|
2109
|
+
* harness-placed skills and MCP servers.
|
|
1633
2110
|
*/
|
|
1634
2111
|
async function startSession(a, runId) {
|
|
1635
2112
|
const startedAt = performance.now();
|
|
@@ -1641,7 +2118,9 @@ async function startSession(a, runId) {
|
|
|
1641
2118
|
signal: a.signal,
|
|
1642
2119
|
env: env.vars,
|
|
1643
2120
|
image: a.image,
|
|
2121
|
+
...a.hosts && { hosts: a.hosts },
|
|
1644
2122
|
brokering: brokeringFor(spec.harness, spec.vertex),
|
|
2123
|
+
...agentOf(spec.harness, spec.vertex)?.keychainTrust && { keychainTrust: true },
|
|
1645
2124
|
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
|
|
1646
2125
|
substitutions: env.substitutions,
|
|
1647
2126
|
egress: spec.egress,
|
|
@@ -1652,7 +2131,7 @@ async function startSession(a, runId) {
|
|
|
1652
2131
|
});
|
|
1653
2132
|
a.sandbox = box;
|
|
1654
2133
|
if (!capabilitiesOf(spec.sandbox).isolated) debugLog("container", `${sandboxKind(spec.sandbox).provider(spec.sandbox)} sandbox at ${box.workdir}`);
|
|
1655
|
-
a.phases = brokerPhases(box, env.substitutions);
|
|
2134
|
+
a.phases = brokerPhases(box, env.substitutions, spec.egress);
|
|
1656
2135
|
if (a.signal.aborted) box.destroy();
|
|
1657
2136
|
a.signal.throwIfAborted();
|
|
1658
2137
|
await a.phases.start();
|
|
@@ -1661,23 +2140,29 @@ async function startSession(a, runId) {
|
|
|
1661
2140
|
debugLog("broker", `gateway swaps in ${agent.join(", ") || "nothing"} for the agent`);
|
|
1662
2141
|
}
|
|
1663
2142
|
if (env.workspace) await copyWorkspace(box, env.workspace, a.signal);
|
|
2143
|
+
const skillDir = workspaceSkillDir(spec.harness);
|
|
2144
|
+
const skillFileList = env.workspaceSkills.map((skill) => ({
|
|
2145
|
+
skill,
|
|
2146
|
+
files: skillFiles(skill).map((f) => ({
|
|
2147
|
+
path: `${skillDir}/${skill.name}/${f.path}`,
|
|
2148
|
+
content: f.content
|
|
2149
|
+
}))
|
|
2150
|
+
}));
|
|
2151
|
+
const toWrite = [...env.files, ...skillFileList.flatMap((s) => s.files)];
|
|
2152
|
+
const filesStart = performance.now();
|
|
2153
|
+
await writeFiles(box, toWrite, a.signal);
|
|
2154
|
+
if (toWrite.length) debugLog("env", `${toWrite.length} files written in ${formatDuration(performance.now() - filesStart)}`);
|
|
1664
2155
|
for (const f of env.files) {
|
|
1665
|
-
await box.writeFile(f.path, f.content);
|
|
1666
2156
|
const size = typeof f.content === "string" ? Buffer.byteLength(f.content) : f.content.byteLength;
|
|
1667
2157
|
debugLog("env", `file ${f.path} (${size} bytes${f.executable ? ", executable" : ""})`);
|
|
1668
2158
|
}
|
|
1669
2159
|
const executable = env.files.flatMap((f) => f.executable ? [f.path.startsWith("~/") ? `${box.home}/${f.path.slice(2)}` : f.path] : []);
|
|
1670
2160
|
if (executable.length) {
|
|
1671
|
-
const paths = executable.map((p) => shellQuote(p.startsWith("-") ? `./${p}` : p));
|
|
2161
|
+
const paths = executable.map((p) => shellQuote$1(p.startsWith("-") ? `./${p}` : p));
|
|
1672
2162
|
const r = await box.exec(`chmod +x ${paths.join(" ")}`, { signal: a.signal });
|
|
1673
2163
|
if (r.exitCode !== 0) throw new InfraError(`Couldn't make ${executable.join(", ")} executable: ${tailOf(r.stderr || r.stdout)}`);
|
|
1674
2164
|
}
|
|
1675
|
-
const
|
|
1676
|
-
for (const skill of env.workspaceSkills) {
|
|
1677
|
-
const files = skillFiles(skill);
|
|
1678
|
-
for (const f of files) await box.writeFile(`${skillDir}/${skill.name}/${f.path}`, f.content);
|
|
1679
|
-
debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
|
|
1680
|
-
}
|
|
2165
|
+
for (const { skill, files } of skillFileList) debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
|
|
1681
2166
|
const plugins = await copyPlugins(a, box, env);
|
|
1682
2167
|
const layout = layoutOf(spec.harness);
|
|
1683
2168
|
if (layout.pluginUnpack && plugins.length) await writeUnpackedPlugins(a, box, env, plugins, layout.pluginUnpack);
|
|
@@ -1691,6 +2176,7 @@ async function startSession(a, runId) {
|
|
|
1691
2176
|
}
|
|
1692
2177
|
await snapshot(box, a.signal);
|
|
1693
2178
|
debugLog("env", "workspace snapshot taken: the diff starts here");
|
|
2179
|
+
await a.phases.agent();
|
|
1694
2180
|
const timed = await timeMcpServers(box, env.mcp, a.signal);
|
|
1695
2181
|
if (timed.log) {
|
|
1696
2182
|
a.mcpStartup = timed.log;
|
|
@@ -1803,8 +2289,8 @@ function cleanArchive(tar) {
|
|
|
1803
2289
|
};
|
|
1804
2290
|
}
|
|
1805
2291
|
const tarCommand = (workdir) => {
|
|
1806
|
-
const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
|
|
1807
|
-
return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
|
|
2292
|
+
const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote$1(d)}`).join(" ");
|
|
2293
|
+
return `cd ${shellQuote$1(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
|
|
1808
2294
|
};
|
|
1809
2295
|
const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
|
|
1810
2296
|
const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
|
|
@@ -1833,9 +2319,9 @@ async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
|
|
|
1833
2319
|
},
|
|
1834
2320
|
entries: null
|
|
1835
2321
|
});
|
|
1836
|
-
const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
|
|
2322
|
+
const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote$1(d)}`).join(" -o ");
|
|
1837
2323
|
const tar = tarCommand(box.workdir);
|
|
1838
|
-
const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
|
|
2324
|
+
const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote$1(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
|
|
1839
2325
|
const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
|
|
1840
2326
|
const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
|
|
1841
2327
|
const rawBytes = Number(raw.trim());
|
|
@@ -1950,10 +2436,10 @@ async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SN
|
|
|
1950
2436
|
/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
|
|
1951
2437
|
const TURN_STOP_GRACE_MS = 3e3;
|
|
1952
2438
|
/**
|
|
1953
|
-
* Resolves `grace` ms after `signal` aborts, with the turn as cut off
|
|
1954
|
-
* once the turn settles on its own.
|
|
2439
|
+
* Resolves `grace` ms after `signal` aborts, with the turn as cut off: what the session says it
|
|
2440
|
+
* got to, or an empty turn; `cancel` drops the timer once the turn settles on its own.
|
|
1955
2441
|
*/
|
|
1956
|
-
function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
|
|
2442
|
+
function cutOff(signal, startedAt, session, grace = TURN_STOP_GRACE_MS) {
|
|
1957
2443
|
let timer;
|
|
1958
2444
|
let onAbort = () => {};
|
|
1959
2445
|
const cutoff = new Promise((resolve) => {
|
|
@@ -1961,22 +2447,25 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
|
|
|
1961
2447
|
timer = setTimeout(() => {
|
|
1962
2448
|
const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
|
|
1963
2449
|
const at = Date.now() - Date.parse(startedAt);
|
|
1964
|
-
|
|
2450
|
+
const note = {
|
|
2451
|
+
type: "error",
|
|
2452
|
+
message,
|
|
2453
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2454
|
+
atMs: at
|
|
2455
|
+
};
|
|
2456
|
+
const empty = {
|
|
1965
2457
|
text: "",
|
|
1966
2458
|
status: "aborted",
|
|
1967
|
-
transcript: [
|
|
1968
|
-
type: "error",
|
|
1969
|
-
message,
|
|
1970
|
-
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1971
|
-
atMs: at
|
|
1972
|
-
}],
|
|
2459
|
+
transcript: [note],
|
|
1973
2460
|
usage: null,
|
|
1974
2461
|
model: null,
|
|
1975
2462
|
unreported: [],
|
|
1976
2463
|
reasoningMs: null,
|
|
1977
2464
|
startedAt,
|
|
1978
2465
|
durationMs: at
|
|
1979
|
-
}
|
|
2466
|
+
};
|
|
2467
|
+
if (!session.stopped) return resolve(empty);
|
|
2468
|
+
session.stopped(note).then(resolve, () => resolve(empty));
|
|
1980
2469
|
}, grace);
|
|
1981
2470
|
};
|
|
1982
2471
|
if (signal.aborted) onAbort();
|
|
@@ -2064,7 +2553,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2064
2553
|
}, `turn ${turnNumber} prompt`);
|
|
2065
2554
|
const watch = a.mcpStartup && watchMcpStartup(a.mcpStartup);
|
|
2066
2555
|
const signal = watch ? AbortSignal.any([a.signal, watch.signal]) : a.signal;
|
|
2067
|
-
const { cutoff, cancel } = cutOff(signal, entry.startedAt);
|
|
2556
|
+
const { cutoff, cancel } = cutOff(signal, entry.startedAt, agent);
|
|
2068
2557
|
const sendStarted = Date.now();
|
|
2069
2558
|
const sending = agent.send(prompt, signal).catch((error) => {
|
|
2070
2559
|
if (!watch?.signal.aborted || a.signal.aborted) throw error;
|
|
@@ -2081,7 +2570,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2081
2570
|
};
|
|
2082
2571
|
});
|
|
2083
2572
|
const timedBefore = a.record.timings.agentMs;
|
|
2084
|
-
const { raw, rejectedKey, reasoningMs, agentMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2573
|
+
const { raw, rejectedKey, providerStop, reasoningMs, agentMs, steps, cost, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2085
2574
|
entry.reasoningMs = null;
|
|
2086
2575
|
if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
|
|
2087
2576
|
throw error;
|
|
@@ -2089,6 +2578,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2089
2578
|
cancel();
|
|
2090
2579
|
watch?.close();
|
|
2091
2580
|
});
|
|
2581
|
+
if (steps) a.turnSteps.set(entry, steps);
|
|
2092
2582
|
const sendWallMs = agentMs ?? Date.now() - sendStarted;
|
|
2093
2583
|
if (agentMs !== void 0) a.record.timings.agentMs = timedBefore + agentMs;
|
|
2094
2584
|
const startup = await watch?.end();
|
|
@@ -2104,7 +2594,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2104
2594
|
atMs: Math.max(0, seenAt - Date.parse(sent.startedAt))
|
|
2105
2595
|
}].sort((x, y) => x.atMs - y.atMs)
|
|
2106
2596
|
} : sent;
|
|
2107
|
-
Object.assign(entry, result, raw ? { raw } : {}, reasoningMs !== void 0 ? { reasoningMs } : {}, {
|
|
2597
|
+
Object.assign(entry, result, raw ? { raw } : {}, reasoningMs !== void 0 ? { reasoningMs } : {}, cost ? { cost } : {}, {
|
|
2108
2598
|
durationMs: Math.max(result.durationMs, sendWallMs),
|
|
2109
2599
|
...startup ? { mcpStartup: mcpStartupOf(startup.entries) } : {}
|
|
2110
2600
|
});
|
|
@@ -2115,7 +2605,8 @@ function turnSender(a, takeContext, checks) {
|
|
|
2115
2605
|
await takeContext(box, a.signal);
|
|
2116
2606
|
const count = (...types) => result.transcript.filter((e) => types.includes(e.type)).length;
|
|
2117
2607
|
const failure = result.transcript.find((e) => e.type === "error");
|
|
2118
|
-
const
|
|
2608
|
+
const stopped = result.status === "aborted" ? void 0 : providerStop;
|
|
2609
|
+
const infra = timedOut || result.status !== "aborted" && rejectedKey || stopped?.message;
|
|
2119
2610
|
let changes;
|
|
2120
2611
|
let diffError;
|
|
2121
2612
|
if (!infra && !a.signal.aborted) try {
|
|
@@ -2138,7 +2629,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2138
2629
|
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${filesChanged === void 0 ? "" : `${plural(filesChanged, "file")} changed · `}${plural(count("file-edit"), "edit tool call")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
2139
2630
|
for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
|
|
2140
2631
|
if (infra) {
|
|
2141
|
-
const error = new InfraError(infra);
|
|
2632
|
+
const error = !timedOut && stopped ? new ProviderStopError(stopped) : new InfraError(infra);
|
|
2142
2633
|
a.startFailure ??= { error };
|
|
2143
2634
|
throw error;
|
|
2144
2635
|
}
|
|
@@ -2190,7 +2681,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2190
2681
|
clock?.resume();
|
|
2191
2682
|
}
|
|
2192
2683
|
}, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
|
|
2193
|
-
const destroy = () => void a.sandbox?.destroy();
|
|
2684
|
+
const destroy = () => void Promise.resolve(a.session?.salvage?.()).catch(() => {}).finally(() => a.sandbox?.destroy());
|
|
2194
2685
|
testSignal.addEventListener("abort", destroy, { once: true });
|
|
2195
2686
|
const takeContext = contextTaker(a);
|
|
2196
2687
|
const { exec, readFile, writeFile } = createChecks(a);
|
|
@@ -2234,12 +2725,10 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2234
2725
|
const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
|
|
2235
2726
|
if (a.startFailure) throw a.startFailure.error;
|
|
2236
2727
|
if (missed) throw missed;
|
|
2237
|
-
const judgeErrors = a
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
a.judgeThrown.add(message);
|
|
2242
|
-
throw new Error(message);
|
|
2728
|
+
const judgeErrors = judgeErrorsMessage(a);
|
|
2729
|
+
if (judgeErrors) {
|
|
2730
|
+
a.judgeThrown.add(judgeErrors);
|
|
2731
|
+
throw new Error(judgeErrors);
|
|
2243
2732
|
}
|
|
2244
2733
|
} catch (thrown) {
|
|
2245
2734
|
const error = attemptError(a, thrown, assertions());
|
|
@@ -2261,6 +2750,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2261
2750
|
oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
|
|
2262
2751
|
if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
|
|
2263
2752
|
await sandbox?.destroy();
|
|
2753
|
+
recordTimeline(a, sandbox);
|
|
2264
2754
|
if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
|
|
2265
2755
|
const { record } = a;
|
|
2266
2756
|
const checked = assertionRecords(assertions());
|
|
@@ -2303,8 +2793,11 @@ function bestOfCell(bestOf) {
|
|
|
2303
2793
|
settled: (settled) => settle[n - 1]?.(sideOf(settled))
|
|
2304
2794
|
});
|
|
2305
2795
|
}
|
|
2306
|
-
/**
|
|
2307
|
-
|
|
2796
|
+
/**
|
|
2797
|
+
* Characters a case key can't hold: attempt ids split on `#` and eval ids on `›`, and control
|
|
2798
|
+
* characters. A `/` is allowed: a key is its own path segment wherever it's in one.
|
|
2799
|
+
*/
|
|
2800
|
+
const BAD_KEY = /[#›\u0000-\u001f\u007f]/;
|
|
2308
2801
|
/** The name a case's eval plans under: `adds a health check [nextjs]`. */
|
|
2309
2802
|
const caseEvalName = (name, key) => `${name} [${key}]`;
|
|
2310
2803
|
/** The key a case gets without `caseKey`: a string, a finite number, or an object's `id` or `name`. */
|
|
@@ -2318,7 +2811,7 @@ function defaultKey(item) {
|
|
|
2318
2811
|
}
|
|
2319
2812
|
/**
|
|
2320
2813
|
* Each case's key, in list order: `caseKey(item)` when given, else its default. Trimmed, 1–80
|
|
2321
|
-
* characters, with no `#`,
|
|
2814
|
+
* characters, with no `#`, `›` or control characters, and unique within the eval.
|
|
2322
2815
|
*/
|
|
2323
2816
|
function caseKeys(cases, caseKey, where) {
|
|
2324
2817
|
const seen = /* @__PURE__ */ new Map();
|
|
@@ -2327,7 +2820,7 @@ function caseKeys(cases, caseKey, where) {
|
|
|
2327
2820
|
if (raw === void 0) throw new UsageError(`cases[${i}] in ${where} has no key: a case is its own key when it's a string or a number, or an object's id or name. Give it an id, or pass caseKey: (item) => item.slug.`);
|
|
2328
2821
|
if (typeof raw !== "string") throw new UsageError(`caseKey in ${where} must return a string (got ${redactSecrets(JSON.stringify(raw) ?? String(raw))} for cases[${i}])`);
|
|
2329
2822
|
const key = raw.trim();
|
|
2330
|
-
if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #,
|
|
2823
|
+
if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
|
|
2331
2824
|
const earlier = seen.get(key);
|
|
2332
2825
|
if (earlier !== void 0) throw new UsageError(`cases[${earlier}] and cases[${i}] in ${where} have the same key "${redactSecrets(key)}". Each case needs its own key${caseKey ? "" : ": give them distinct ids or names, or pass caseKey"}.`);
|
|
2333
2826
|
seen.set(key, i);
|
|
@@ -2653,6 +3146,8 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
|
|
|
2653
3146
|
if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
|
|
2654
3147
|
const [plain] = plainHeaders(plannedAttempts);
|
|
2655
3148
|
if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
|
|
3149
|
+
const [plainVar] = plainEnv(resolved.env, plannedAttempts);
|
|
3150
|
+
if (plainVar) throw new UsageError(plainEnvError(plainVar, where, ctx.config.file));
|
|
2656
3151
|
}
|
|
2657
3152
|
const pluginNotes = [];
|
|
2658
3153
|
const pluginLabels = /* @__PURE__ */ new Map();
|
|
@@ -3084,10 +3579,26 @@ function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
|
|
|
3084
3579
|
bestOfCells.set(cell.key, of);
|
|
3085
3580
|
hooks = of(n);
|
|
3086
3581
|
}
|
|
3087
|
-
|
|
3582
|
+
const test = {
|
|
3088
3583
|
...options,
|
|
3089
3584
|
...hooks
|
|
3090
|
-
}
|
|
3585
|
+
};
|
|
3586
|
+
if (spec && test.gate && spec.sandbox.kind === "tart" && spec.sandbox.hosts) {
|
|
3587
|
+
const gate = {
|
|
3588
|
+
...test.gate,
|
|
3589
|
+
limits: { ...test.gate.limits }
|
|
3590
|
+
};
|
|
3591
|
+
const before = test.ready;
|
|
3592
|
+
test.gate = gate;
|
|
3593
|
+
test.ready = async () => {
|
|
3594
|
+
const { config, runId } = getRunContext();
|
|
3595
|
+
const cap = await preparedVmCap(spec, config.root, runId);
|
|
3596
|
+
const key = "sandbox:tart";
|
|
3597
|
+
if (cap && gate.limits[key]) gate.limits[key] = Math.min(gate.limits[key], cap);
|
|
3598
|
+
return before ? before() : true;
|
|
3599
|
+
};
|
|
3600
|
+
}
|
|
3601
|
+
registerTest(testName, test, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
3091
3602
|
}
|
|
3092
3603
|
};
|
|
3093
3604
|
return {
|
|
@@ -3241,4 +3752,4 @@ function optionsFrom(written, where, base) {
|
|
|
3241
3752
|
}
|
|
3242
3753
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
3243
3754
|
|
|
3244
|
-
//# sourceMappingURL=evaluate-
|
|
3755
|
+
//# sourceMappingURL=evaluate-DA4QsrHO.js.map
|