reeval 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +16 -16
- package/THIRD_PARTY_NOTICES.md +0 -2068
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +65 -50
- package/dist/{adapter-dHdepWKb.js → adapter-BHMbIeH9.js} +7 -7
- package/dist/{adapter-dHdepWKb.js.map → adapter-BHMbIeH9.js.map} +1 -1
- package/dist/{agent-CZpiwA_r.js → agent-D67VCd6Q.js} +8 -8
- package/dist/{agent-CZpiwA_r.js.map → agent-D67VCd6Q.js.map} +1 -1
- package/dist/{agents-CEapsHV-.js → agents-Dnv1MmgY.js} +40 -59
- package/dist/agents-Dnv1MmgY.js.map +1 -0
- package/dist/{analysis-BnG_1x_O.js → analysis-8vvMnUJB.js} +4 -5
- package/dist/analysis-8vvMnUJB.js.map +1 -0
- package/dist/api-ZeGnedB3.js +141 -0
- package/dist/api-ZeGnedB3.js.map +1 -0
- package/dist/assemble-BlA7FWRn.js +1 -0
- package/dist/{assemble-Dw1bF13I.js → assemble-CuvlsOaE.js} +117 -66
- package/dist/assemble-CuvlsOaE.js.map +1 -0
- package/dist/{attach-DvYiMu6w.js → attach-6av4DmO3.js} +24 -6
- package/dist/attach-6av4DmO3.js.map +1 -0
- package/dist/{attempt-DxR1agFo.js → attempt-D5WH0yFa.js} +3 -3
- package/dist/{attempt-DxR1agFo.js.map → attempt-D5WH0yFa.js.map} +1 -1
- package/dist/{attempt-host-DgeJFSZw.js → attempt-host-Be_-uDKJ.js} +37 -14
- package/dist/attempt-host-Be_-uDKJ.js.map +1 -0
- package/dist/boat-BCNNEJni.js +298 -0
- package/dist/boat-BCNNEJni.js.map +1 -0
- package/dist/boat-CwBW2sbp.js +1 -0
- package/dist/{boat-CUNgSc75.js → boat-CygRcaM0.js} +115 -196
- package/dist/boat-CygRcaM0.js.map +1 -0
- package/dist/{boat-images-BuFigiEb.js → boat-images-asGLQXD5.js} +19 -23
- package/dist/boat-images-asGLQXD5.js.map +1 -0
- package/dist/{bundle-yEgI3RsI.js → bundle-B7P-Awnm.js} +105 -17
- package/dist/bundle-B7P-Awnm.js.map +1 -0
- package/dist/{capture-DdEKVrwk.js → capture-BwLa-4s-.js} +20 -19
- package/dist/{capture-DdEKVrwk.js.map → capture-BwLa-4s-.js.map} +1 -1
- package/dist/capture-CG-J-1-j.js +1 -0
- package/dist/cases-Bsp-G-ZL.js +1 -0
- package/dist/{cases-B7QkvMGE.js → cases-C6vnV-gn.js} +60 -42
- package/dist/cases-C6vnV-gn.js.map +1 -0
- package/dist/classify-BIJQ2Xqo.js +163 -0
- package/dist/classify-BIJQ2Xqo.js.map +1 -0
- package/dist/{cli-Z7aw8k25.js → cli-BoCPYLUQ.js} +305 -124
- package/dist/cli-BoCPYLUQ.js.map +1 -0
- package/dist/{cli-spec-3zvLC9YW.js → cli-spec-DgEWHOt3.js} +164 -50
- package/dist/cli-spec-DgEWHOt3.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-Dj65gXG3.js +105 -0
- package/dist/client-Dj65gXG3.js.map +1 -0
- package/dist/{cloud-owVBaSRt.js → cloud-TuNUlZ0L.js} +3 -3
- package/dist/{cloud-owVBaSRt.js.map → cloud-TuNUlZ0L.js.map} +1 -1
- package/dist/{cloud-files-BjinrTTe.js → cloud-files-CBovrfZA.js} +8 -8
- package/dist/{cloud-files-BjinrTTe.js.map → cloud-files-CBovrfZA.js.map} +1 -1
- package/dist/{cloud-DuHcNQVV.js → cloud-igGIS4Bm.js} +43 -20
- package/dist/cloud-igGIS4Bm.js.map +1 -0
- package/dist/cloud-project-Ch7jjlwh.js +110 -0
- package/dist/cloud-project-Ch7jjlwh.js.map +1 -0
- package/dist/cloud-project-DdUVB4EE.js +1 -0
- package/dist/cloud-push-DvnqOgRd.js +817 -0
- package/dist/cloud-push-DvnqOgRd.js.map +1 -0
- package/dist/cloud-push-MXKreNyq.js +1 -0
- package/dist/{cloud-run-DqMkSGYB.js → cloud-run-CbGF8Lwt.js} +285 -48
- package/dist/cloud-run-CbGF8Lwt.js.map +1 -0
- package/dist/cloud-run-CghE2dG6.js +1 -0
- package/dist/{cloud-secrets-Bak83tik.js → cloud-secrets-BWoV4ddM.js} +43 -35
- package/dist/cloud-secrets-BWoV4ddM.js.map +1 -0
- package/dist/cloud-secrets-DlPRfS2l.js +1 -0
- package/dist/command-Bps8kBML.js +141 -0
- package/dist/command-Bps8kBML.js.map +1 -0
- package/dist/{command-CtmSJzPC.js → command-ByUmSm2D.js} +29 -23
- package/dist/command-ByUmSm2D.js.map +1 -0
- package/dist/{command-CeygdoVe.js → command-CQk7FCI1.js} +92 -26
- package/dist/command-CQk7FCI1.js.map +1 -0
- package/dist/{command-CoHN1ibe.js → command-CxuGky7A.js} +44 -20
- package/dist/command-CxuGky7A.js.map +1 -0
- package/dist/{command-efTGfUYb.js → command-pxfK2bkV.js} +26 -16
- package/dist/command-pxfK2bkV.js.map +1 -0
- package/dist/commands-Cl3Y_UXg.js +1 -0
- package/dist/{commands-DR-NolV2.js → commands-DGWNNyRD.js} +54 -38
- package/dist/commands-DGWNNyRD.js.map +1 -0
- package/dist/{config-Bn2OhCN_.js → config-mU7wFvQq.js} +46 -32
- package/dist/config-mU7wFvQq.js.map +1 -0
- package/dist/core-BXYzOEyQ.js +2058 -0
- package/dist/core-BXYzOEyQ.js.map +1 -0
- package/dist/{slug-IHh3f770.js → cost-DQ0ayy6h.js} +37 -2
- package/dist/cost-DQ0ayy6h.js.map +1 -0
- package/dist/{cursor-catalog-BYIDSsLn.js → cursor-catalog-EOFft6LJ.js} +4 -4
- package/dist/{cursor-catalog-BYIDSsLn.js.map → cursor-catalog-EOFft6LJ.js.map} +1 -1
- package/dist/{daytona-CWs9j9mt.js → daytona-DtCjv8rH.js} +22 -17
- package/dist/daytona-DtCjv8rH.js.map +1 -0
- package/dist/daytona-image-Xy5o53Ji.js +1 -0
- package/dist/{daytona-image-DmGlFAyq.js → daytona-image-w4uTRZ0z.js} +6 -6
- package/dist/{daytona-image-DmGlFAyq.js.map → daytona-image-w4uTRZ0z.js.map} +1 -1
- package/dist/db-BcplCdzu.js +1 -0
- package/dist/{db-D6yQib7M.js → db-yigMuHM2.js} +31 -119
- package/dist/db-yigMuHM2.js.map +1 -0
- package/dist/debug-CGg7KHKq.js +1 -0
- package/dist/{debug-CIXzB4wK.js → debug-T29p6HOX.js} +2 -10
- package/dist/debug-T29p6HOX.js.map +1 -0
- package/dist/{default-effort-Ck6ViSaI.js → default-effort-B1b8t81b.js} +1 -1
- package/dist/{default-effort-Bpz5ULfC.js → default-effort-BDYXBzu5.js} +3 -3
- package/dist/{default-effort-Bpz5ULfC.js.map → default-effort-BDYXBzu5.js.map} +1 -1
- package/dist/{dist-CZtW7eI5.js → dist-DCG7vZGQ.js} +3 -3
- package/dist/{dist-CZtW7eI5.js.map → dist-DCG7vZGQ.js.map} +1 -1
- package/dist/{docker-Dgw4aaRk.js → docker-DcnX2WpM.js} +4 -4
- package/dist/{docker-Dgw4aaRk.js.map → docker-DcnX2WpM.js.map} +1 -1
- package/dist/{docker-Bbi4kmFj.js → docker-XCyDUd7_.js} +1 -1
- package/dist/{doctor-BfS6jswW.js → doctor-DTKa8rrq.js} +27 -27
- package/dist/{doctor-BfS6jswW.js.map → doctor-DTKa8rrq.js.map} +1 -1
- package/dist/duration-Bz2H7pXb.js +34 -0
- package/dist/duration-Bz2H7pXb.js.map +1 -0
- package/dist/{environment-BY8lVFFK.js → environment-BCBg6c0p.js} +38 -12
- package/dist/environment-BCBg6c0p.js.map +1 -0
- package/dist/errors-BjzEWESY.js +93 -0
- package/dist/errors-BjzEWESY.js.map +1 -0
- package/dist/{errors-P8oHhNzy.js → errors-Cda7UzRA.js} +2 -2
- package/dist/{errors-P8oHhNzy.js.map → errors-Cda7UzRA.js.map} +1 -1
- package/dist/{eval-source-BB8In3xo.js → eval-source-CSPyKuV6.js} +6 -6
- package/dist/{eval-source-BB8In3xo.js.map → eval-source-CSPyKuV6.js.map} +1 -1
- package/dist/{evaluate-B9_nukrc.d.ts → evaluate-Nnh0iPuo.d.ts} +1 -1
- package/dist/{evaluate-B8I9AKgw.js → evaluate-wgW1lGQj.js} +280 -60
- package/dist/evaluate-wgW1lGQj.js.map +1 -0
- package/dist/{events-D97ZzAeq.js → events-DG_Cq5Pn.js} +154 -33
- package/dist/events-DG_Cq5Pn.js.map +1 -0
- package/dist/events-RaK1pDmT.js +1 -0
- package/dist/{fake-bootstrap-4DFtzib0.js → fake-bootstrap-uCyHhnMO.js} +2 -2
- package/dist/{fake-bootstrap-4DFtzib0.js.map → fake-bootstrap-uCyHhnMO.js.map} +1 -1
- package/dist/files-BnsRehge.js +1 -0
- package/dist/{files-B0DbptiO.js → files-Bv6ipFjc.js} +4 -4
- package/dist/{files-B0DbptiO.js.map → files-Bv6ipFjc.js.map} +1 -1
- package/dist/{files-GBTUBAek.js → files-CcnSILye.js} +3 -3
- package/dist/{files-GBTUBAek.js.map → files-CcnSILye.js.map} +1 -1
- package/dist/{files-BesexM88.js → files-CjPm3yyj.js} +173 -41
- package/dist/files-CjPm3yyj.js.map +1 -0
- package/dist/files-CkmCV5rH.js +1 -0
- package/dist/{files-DE13BqBC.js → files-ClUbQPj9.js} +2 -2
- package/dist/{files-DE13BqBC.js.map → files-ClUbQPj9.js.map} +1 -1
- package/dist/{files-B4K1k__s.js → files-CqfVnRXw.js} +5 -5
- package/dist/{files-B4K1k__s.js.map → files-CqfVnRXw.js.map} +1 -1
- package/dist/files-D4Hb9HZS.js +1 -0
- package/dist/files-DCcRkJN4.js +1 -0
- package/dist/{files-TwSVeWwM.js → files-GAocj_Z1.js} +27 -41
- package/dist/files-GAocj_Z1.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-DatgUcnf.js → git-CRPr6I2T.js} +3 -3
- package/dist/{git-DatgUcnf.js.map → git-CRPr6I2T.js.map} +1 -1
- package/dist/{harness-FcTyyPco.js → harness-Dcd3ffkY.js} +3 -2
- package/dist/harness-Dcd3ffkY.js.map +1 -0
- package/dist/{harness-EZNoeUBf.js → harness-DvqNGwl6.js} +2 -2
- package/dist/{harness-EZNoeUBf.js.map → harness-DvqNGwl6.js.map} +1 -1
- package/dist/{harness-install-HgR73tpF.js → harness-install-B0Y2GjnO.js} +4 -4
- package/dist/{harness-install-HgR73tpF.js.map → harness-install-B0Y2GjnO.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/http-DtlSzw4A.js +96 -0
- package/dist/http-DtlSzw4A.js.map +1 -0
- package/dist/{ids-DqEvrBFZ.js → ids-CtxRn13a.js} +4 -3
- package/dist/ids-CtxRn13a.js.map +1 -0
- package/dist/{images-SRE4f3SM.js → images-JspTw1vt.js} +13 -13
- package/dist/{images-SRE4f3SM.js.map → images-JspTw1vt.js.map} +1 -1
- package/dist/index.d.ts +7 -2
- package/dist/index.js +4 -5
- package/dist/init-BHIWfE2L.js +1 -0
- package/dist/{init-D5F9MYPC.js → init-wmMbauok.js} +6 -6
- package/dist/{init-D5F9MYPC.js.map → init-wmMbauok.js.map} +1 -1
- package/dist/{ink-list-CLq6YVB8.js → ink-list-B5Gul5iZ.js} +7 -6
- package/dist/ink-list-B5Gul5iZ.js.map +1 -0
- package/dist/{inputs-B_FsOD2N.js → inputs-BWTtz1-v.js} +6 -6
- package/dist/{inputs-B_FsOD2N.js.map → inputs-BWTtz1-v.js.map} +1 -1
- package/dist/inputs-load-C8HEtAej.js +1 -0
- package/dist/{inputs-load-B4MFkp0Q.js → inputs-load-DQRC3dUA.js} +3 -3
- package/dist/{inputs-load-B4MFkp0Q.js.map → inputs-load-DQRC3dUA.js.map} +1 -1
- package/dist/internal/vm-api/boat.d.ts +46 -0
- package/dist/internal/vm-api/boat.js +1 -0
- package/dist/internal/vm-api/daytona.d.ts +26 -0
- package/dist/internal/vm-api/daytona.js +220 -0
- package/dist/internal/vm-api/daytona.js.map +1 -0
- package/dist/internal/vm-api.d.ts +86 -0
- package/dist/internal/vm-api.js +3 -0
- package/dist/internal/vm.d.ts +1 -1
- package/dist/internal/vm.js +1 -1
- package/dist/{invocations-BxQqe5GU.js → invocations-CSA9OKFn.js} +5 -5
- package/dist/{invocations-BxQqe5GU.js.map → invocations-CSA9OKFn.js.map} +1 -1
- package/dist/issue-ipG9Yy1h.js +1 -0
- package/dist/{issue-iErT9KkK.js → issue-lKqtL3iX.js} +4 -4
- package/dist/{issue-iErT9KkK.js.map → issue-lKqtL3iX.js.map} +1 -1
- package/dist/{jiti-CtahaHaY.js → jiti-aNfS_lSj.js} +2 -2
- package/dist/{jiti-CtahaHaY.js.map → jiti-aNfS_lSj.js.map} +1 -1
- package/dist/{judge-MOKPsr6R.js → judge-etHC4K02.js} +5 -5
- package/dist/{judge-MOKPsr6R.js.map → judge-etHC4K02.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-Dur4D2pj.js → judgments-CVCUWT-g.js} +7 -4
- package/dist/{judgments-Dur4D2pj.js.map → judgments-CVCUWT-g.js.map} +1 -1
- package/dist/{kinds-Dohijzi-.js → kinds-BQX5ZxpK.js} +37 -11
- package/dist/{kinds-Dohijzi-.js.map → kinds-BQX5ZxpK.js.map} +1 -1
- package/dist/labels-Dtyf5-pd.js +251 -0
- package/dist/labels-Dtyf5-pd.js.map +1 -0
- package/dist/{layout-BhhJAEgf.js → layout-Co4tPHWI.js} +3 -3
- package/dist/{layout-BhhJAEgf.js.map → layout-Co4tPHWI.js.map} +1 -1
- package/dist/link-Cqs2DhxR.js +127 -0
- package/dist/link-Cqs2DhxR.js.map +1 -0
- package/dist/{list-ClDb8qiX.js → list-CjcNH-5e.js} +10 -7
- package/dist/list-CjcNH-5e.js.map +1 -0
- package/dist/{live-QFILMmql.js → live-BRF0ogR6.js} +3 -5
- package/dist/live-BRF0ogR6.js.map +1 -0
- package/dist/live-DTpM9b4D.js +56 -0
- package/dist/live-DTpM9b4D.js.map +1 -0
- package/dist/{machine-service-xFhakTs5.js → machine-service-BetvBWHz.js} +146 -13
- package/dist/machine-service-BetvBWHz.js.map +1 -0
- package/dist/{machine-service-DpJScNtA.js → machine-service-XFymGCHj.js} +1 -1
- package/dist/{machines-Cm-rU7jp.js → machines-CkDTbMZ6.js} +25 -25
- package/dist/{machines-Cm-rU7jp.js.map → machines-CkDTbMZ6.js.map} +1 -1
- package/dist/markdown-COts-jxf.js +653 -0
- package/dist/markdown-COts-jxf.js.map +1 -0
- package/dist/{model-Rmv-L9Jb.js → model-BYHXnYLe.js} +1 -1
- package/dist/model-CrsA2Evf.js +1 -0
- package/dist/{model-CcvSrrq9.js → model-DNC53cPL.js} +4 -4
- package/dist/{model-CcvSrrq9.js.map → model-DNC53cPL.js.map} +1 -1
- package/dist/{model-D9WMMEWD.js → model-F4UH_YG6.js} +4 -4
- package/dist/{model-D9WMMEWD.js.map → model-F4UH_YG6.js.map} +1 -1
- package/dist/{model-mTaaEkeQ.js → model-xemZQ7DK.js} +251 -65
- package/dist/model-xemZQ7DK.js.map +1 -0
- package/dist/{node-Bl2CtSjB.js → node-gPT36n_n.js} +19 -8
- package/dist/{node-Bl2CtSjB.js.map → node-gPT36n_n.js.map} +1 -1
- package/dist/options-CAZfxE0h.js +1 -0
- package/dist/otlp-BxPnO99e.js +1 -0
- package/dist/otlp-DbNz-occ.js +25 -0
- package/dist/otlp-DbNz-occ.js.map +1 -0
- package/dist/outcome-1LBk2e3w.js +38 -0
- package/dist/outcome-1LBk2e3w.js.map +1 -0
- package/dist/{patch-DhJDHRFs.js → patch-Dxzv8msl.js} +3 -3
- package/dist/{patch-DhJDHRFs.js.map → patch-Dxzv8msl.js.map} +1 -1
- package/dist/{patch-C7yBztZj.js → patch-uwo6t4Zy.js} +26 -2
- package/dist/patch-uwo6t4Zy.js.map +1 -0
- package/dist/{pi-providers-HygmL1y-.js → pi-providers-BjcQv2A1.js} +23 -9
- package/dist/{pi-providers-HygmL1y-.js.map → pi-providers-BjcQv2A1.js.map} +1 -1
- package/dist/{plain-Ctn8oUdp.js → plain-DXUNwqpO.js} +2 -2
- package/dist/{plain-Ctn8oUdp.js.map → plain-DXUNwqpO.js.map} +1 -1
- package/dist/{plan-CC0bKbm9.js → plan-DSJX7Oyq.js} +9 -9
- package/dist/{plan-CC0bKbm9.js.map → plan-DSJX7Oyq.js.map} +1 -1
- package/dist/{plugins-PwEtW8GH.js → plugins-DVEOMsYw.js} +7 -7
- package/dist/{plugins-PwEtW8GH.js.map → plugins-DVEOMsYw.js.map} +1 -1
- package/dist/{preflight-BHUEfA4m.js → preflight-B3oLZttl.js} +4 -4
- package/dist/{preflight-BHUEfA4m.js.map → preflight-B3oLZttl.js.map} +1 -1
- package/dist/preflight-CLYRuQ3r.js +1 -0
- package/dist/{preflight-DEV_vrdB.js → preflight-DXMD6jou.js} +1 -1
- package/dist/{preflight-BzvLYQ70.js → preflight-dUe0QiPK.js} +9 -9
- package/dist/{preflight-BzvLYQ70.js.map → preflight-dUe0QiPK.js.map} +1 -1
- package/dist/{prepare-D7hPrh7Q.js → prepare-BTaX7vcC.js} +8 -8
- package/dist/{prepare-D7hPrh7Q.js.map → prepare-BTaX7vcC.js.map} +1 -1
- package/dist/{prepare-DS42k409.js → prepare-QdiYPOwR.js} +2 -2
- package/dist/prices-CfnkJOth.js +1 -0
- package/dist/{prices-DXJsP2Ha.js → prices-CoLymMRd.js} +3 -3
- package/dist/{prices-DXJsP2Ha.js.map → prices-CoLymMRd.js.map} +1 -1
- package/dist/{process-RPlEhpHz.js → process-C_nh1H7X.js} +9 -3
- package/dist/process-C_nh1H7X.js.map +1 -0
- package/dist/provenance-Cylt_YxT.d.ts +222 -0
- package/dist/{provenance-DgQJ-dq7.js → provenance-Di42xQIS.js} +5 -5
- package/dist/{provenance-DgQJ-dq7.js.map → provenance-Di42xQIS.js.map} +1 -1
- package/dist/provider-B24hZX95.js +1 -0
- package/dist/{provider-ieWOn4YS.js → provider-Cn7lmECy.js} +2 -2
- package/dist/{provider-ieWOn4YS.js.map → provider-Cn7lmECy.js.map} +1 -1
- package/dist/{provider-stop-CxzWW7Ft.js → provider-stop-DjBMfZW5.js} +2 -2
- package/dist/{provider-stop-CxzWW7Ft.js.map → provider-stop-DjBMfZW5.js.map} +1 -1
- package/dist/{providers-N4HDMeeo.js → providers-Bm5NLaRh.js} +1 -1
- package/dist/{providers-B5Vr0hKH.js → providers-CTTfYbm0.js} +28 -28
- package/dist/{providers-B5Vr0hKH.js.map → providers-CTTfYbm0.js.map} +1 -1
- package/dist/query-CTMdWXbx.js +1 -0
- package/dist/{query-CmTjYbqh.js → query-TzNtTvwf.js} +2 -2
- package/dist/{query-CmTjYbqh.js.map → query-TzNtTvwf.js.map} +1 -1
- package/dist/{questions-DnqkJga0.js → questions-BqisU7A0.js} +19 -13
- package/dist/questions-BqisU7A0.js.map +1 -0
- package/dist/{recipe-B0NeT_Sn.js → recipe-DzjXne35.js} +4 -4
- package/dist/{recipe-B0NeT_Sn.js.map → recipe-DzjXne35.js.map} +1 -1
- package/dist/{record-2liAelrB.js → record-Cj3yWR0E.js} +43 -12
- package/dist/record-Cj3yWR0E.js.map +1 -0
- package/dist/{redact-DMueYS-v.js → redact-CfTHYWLv.js} +30 -1
- package/dist/redact-CfTHYWLv.js.map +1 -0
- package/dist/{remarks-D5qOZzcX.js → remarks-BPKGorBW.js} +4 -2
- package/dist/remarks-BPKGorBW.js.map +1 -0
- package/dist/{remarks-D-9f1y1B.js → remarks-CYXYiB_O.js} +21 -11
- package/dist/{remarks-D-9f1y1B.js.map → remarks-CYXYiB_O.js.map} +1 -1
- package/dist/{remote-B7BIRUTK.js → remote-D4UEa7Rs.js} +10 -8
- package/dist/{remote-B7BIRUTK.js.map → remote-D4UEa7Rs.js.map} +1 -1
- package/dist/renderer/ascii.d.ts +6 -0
- package/dist/renderer/ascii.js +7 -0
- package/dist/renderer/ascii.js.map +1 -0
- package/dist/renderer/markdown.d.ts +31 -0
- package/dist/renderer/markdown.js +1 -0
- package/dist/report-BSi-R8hC.js +941 -0
- package/dist/report-BSi-R8hC.js.map +1 -0
- package/dist/report-inputs-5XwZBIRK.js +113 -0
- package/dist/report-inputs-5XwZBIRK.js.map +1 -0
- package/dist/{resolve-B60w6DcB.js → resolve-BEltmYhD.js} +6 -5
- package/dist/resolve-BEltmYhD.js.map +1 -0
- package/dist/resolve-C7A5Q2Jj.js +1 -0
- package/dist/{resolve-t9sOkGGS.js → resolve-nRzGt4Ff.js} +30 -13
- package/dist/resolve-nRzGt4Ff.js.map +1 -0
- package/dist/{results-Df910ONm.js → results-DXBaxfIM.js} +11 -7
- package/dist/{results-Df910ONm.js.map → results-DXBaxfIM.js.map} +1 -1
- package/dist/{resume-4Q9q8AcE.js → resume-DZPjjToJ.js} +46 -21
- package/dist/resume-DZPjjToJ.js.map +1 -0
- package/dist/{retry-C15f5yyy.js → retry-rLdXCinf.js} +5 -5
- package/dist/{retry-C15f5yyy.js.map → retry-rLdXCinf.js.map} +1 -1
- package/dist/{root-Cm93-6ix.js → root-Co0jhOT5.js} +2 -2
- package/dist/{root-Cm93-6ix.js.map → root-Co0jhOT5.js.map} +1 -1
- package/dist/run-D_o_8Fkv.js +128 -0
- package/dist/run-D_o_8Fkv.js.map +1 -0
- package/dist/{client-kUwtq-vt.js → runtime-DsRjnydE.js} +525 -105
- package/dist/runtime-DsRjnydE.js.map +1 -0
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +2 -2
- package/dist/sandbox/daytona.d.ts +1 -1
- package/dist/sandbox/daytona.js +1 -1
- package/dist/{sandbox-BffNervX.js → sandbox-B8gDRs5E.js} +3 -3
- package/dist/{sandbox-BffNervX.js.map → sandbox-B8gDRs5E.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{secrets-file-Dtd3h3kY.js → secrets-file-B8rUJV2L.js} +4 -4
- package/dist/{secrets-file-Dtd3h3kY.js.map → secrets-file-B8rUJV2L.js.map} +1 -1
- package/dist/server-eUWbjZoi.js +1 -0
- package/dist/server-mJzPXBMM.js +204 -0
- package/dist/server-mJzPXBMM.js.map +1 -0
- package/dist/{services-BG8xuU82.js → services-CHh83UDC.js} +4 -2
- package/dist/services-CHh83UDC.js.map +1 -0
- package/dist/{shared-DtZwIAfy.js → shared-bXaqVxk-.js} +5 -34
- package/dist/shared-bXaqVxk-.js.map +1 -0
- package/dist/skill-dirs-B7o83G5I.js +63 -0
- package/dist/skill-dirs-B7o83G5I.js.map +1 -0
- package/dist/source-C6JgdJdJ.js +1 -0
- package/dist/{source-zoRWRMql.js → source-DxgCge_u.js} +33 -18
- package/dist/source-DxgCge_u.js.map +1 -0
- package/dist/spans-B4MwgEyy.js +1 -0
- package/dist/spans-Dq3fo6HM.js +6 -0
- package/dist/spans-Dq3fo6HM.js.map +1 -0
- package/dist/spans-tf3biLpT.js +72 -0
- package/dist/spans-tf3biLpT.js.map +1 -0
- package/dist/src-B2psni1B.js +1 -0
- package/dist/{src-DiS13D_q.js → src-BI35cTig.js} +2 -2
- package/dist/src-BLePHsjw.js +2 -0
- package/dist/{src-Bj3WJs3t.js → src-CftydE9Q.js} +2 -2
- package/dist/{src-CDoAEO4u.js → src-ClMhIgi3.js} +2 -2
- package/dist/{src-CDoAEO4u.js.map → src-ClMhIgi3.js.map} +1 -1
- package/dist/{src-CUrcaZ91.js → src-CnOj4_n4.js} +1 -1
- package/dist/src-Cs3nmbrr.js +2 -0
- package/dist/{src-TMKittwH.js → src-D0GezGkJ.js} +2 -2
- package/dist/{src-OC89dAjO.js → src-D1Q6ElaF.js} +182 -36
- package/dist/src-D1Q6ElaF.js.map +1 -0
- package/dist/{src-CFBg2ecq.js → src-D_9qqp-Y.js} +2 -2
- package/dist/{src-DwcTn4JY.js → src-DhZuOcgq.js} +5 -5
- package/dist/{src-DwcTn4JY.js.map → src-DhZuOcgq.js.map} +1 -1
- package/dist/{src-CGBwMZgM.js → src-Dm2YIeXn.js} +8 -8
- package/dist/{src-CGBwMZgM.js.map → src-Dm2YIeXn.js.map} +1 -1
- package/dist/{src-CcbV6VpV.js → src-Dpzfe-vl.js} +3 -3
- package/dist/{src-c3ZQu7Uu.js → src-NYxoDS3b.js} +5 -5
- package/dist/{src-c3ZQu7Uu.js.map → src-NYxoDS3b.js.map} +1 -1
- package/dist/{src-C3OhA5pw.js → src-Y4AiZxLt.js} +5 -5
- package/dist/{src-C3OhA5pw.js.map → src-Y4AiZxLt.js.map} +1 -1
- package/dist/{src-BQWAxhKD.js → src-YpmRtJmP.js} +3 -3
- package/dist/{src-BQWAxhKD.js.map → src-YpmRtJmP.js.map} +1 -1
- package/dist/{src-Bb3k9Kn_.js → src-nIPDQ297.js} +66 -11
- package/dist/{src-Bb3k9Kn_.js.map → src-nIPDQ297.js.map} +1 -1
- package/dist/ssh-pool-B9bnSQIj.js +1 -0
- package/dist/{ssh-pool-DwboLN9O.js → ssh-pool-Gv8gAsgZ.js} +19 -16
- package/dist/ssh-pool-Gv8gAsgZ.js.map +1 -0
- package/dist/{start-BwMyDneA.js → start-YOYgbaNU.js} +45 -13
- package/dist/start-YOYgbaNU.js.map +1 -0
- package/dist/state-Bvsi2WgZ.js +1 -0
- package/dist/{state-CXTaT0t8.js → state-CJlYQtdl.js} +21 -38
- package/dist/state-CJlYQtdl.js.map +1 -0
- package/dist/store/local.d.ts +1 -1
- package/dist/store/s3.d.ts +1 -1
- package/dist/studio-share-4T6GYO7l.js +209 -0
- package/dist/studio-share-4T6GYO7l.js.map +1 -0
- package/dist/suite-fO-u8jlS.js +2 -0
- package/dist/{suite-C4XSg6Ls.js → suite-oH7uiCRq.js} +107 -106
- package/dist/suite-oH7uiCRq.js.map +1 -0
- package/dist/summary-CTMWfnOF.js +75 -0
- package/dist/summary-CTMWfnOF.js.map +1 -0
- package/dist/{sync-bnI6WWaL.js → sync-BNduS38K.js} +6 -6
- package/dist/{sync-bnI6WWaL.js.map → sync-BNduS38K.js.map} +1 -1
- package/dist/{tart-CSP5Q0qV.js → tart-B6Uz4S3I.js} +43 -13
- package/dist/tart-B6Uz4S3I.js.map +1 -0
- package/dist/{tart-DmVuVsmX.js → tart-DNrumWZu.js} +1 -1
- package/dist/{tart-CzViW3WZ.js → tart-Duqp33Zh.js} +2 -2
- package/dist/{tart-CzViW3WZ.js.map → tart-Duqp33Zh.js.map} +1 -1
- package/dist/{tart-bake-d9UvIfe3.js → tart-bake-B4QyNb7f.js} +1 -1
- package/dist/{tart-bake-BCKRfyKm.js → tart-bake-s99wBzxO.js} +9 -9
- package/dist/{tart-bake-BCKRfyKm.js.map → tart-bake-s99wBzxO.js.map} +1 -1
- package/dist/{tart-cli-CqSpUYGV.js → tart-cli-B0xVYBti.js} +6 -4
- package/dist/tart-cli-B0xVYBti.js.map +1 -0
- package/dist/{tart-cli-C4d5FXG1.js → tart-cli-DfteJCN9.js} +1 -1
- package/dist/{tart-hosts-Du7SZM4w.js → tart-hosts-B3Ze8aMp.js} +1 -1
- package/dist/{tart-hosts-G0EUDt07.js → tart-hosts-DfVUcZqA.js} +6 -6
- package/dist/{tart-hosts-G0EUDt07.js.map → tart-hosts-DfVUcZqA.js.map} +1 -1
- package/dist/{tart-images-De0-ib7R.js → tart-images-yzIyb0bQ.js} +8 -3
- package/dist/tart-images-yzIyb0bQ.js.map +1 -0
- package/dist/telemetry-bBhK3DSs.js +399 -0
- package/dist/telemetry-bBhK3DSs.js.map +1 -0
- package/dist/testing.d.ts +11 -2
- package/dist/testing.js +4 -4
- package/dist/{tool-class-B3ZFTSOw.js → tool-class-CB-m4PiF.js} +2 -2
- package/dist/{tool-class-B3ZFTSOw.js.map → tool-class-CB-m4PiF.js.map} +1 -1
- package/dist/{trace-CfppUzD9.js → trace-CJQLlAbB.js} +2 -2
- package/dist/{trace-CfppUzD9.js.map → trace-CJQLlAbB.js.map} +1 -1
- package/dist/{traits-Bj1PM5F7.js → traits-DPIffsN8.js} +2 -2
- package/dist/{traits-Bj1PM5F7.js.map → traits-DPIffsN8.js.map} +1 -1
- package/dist/{transcript-CW02A4s9.js → transcript-TWiaAuoL.js} +2 -2
- package/dist/{transcript-CW02A4s9.js.map → transcript-TWiaAuoL.js.map} +1 -1
- package/dist/{resources-DqdwihhT.js → turn-error-Dmv_Muj7.js} +15 -2
- package/dist/turn-error-Dmv_Muj7.js.map +1 -0
- package/dist/{types-D4WEnzhY.d.ts → types-BZaItiui.d.ts} +159 -13
- package/dist/types-Buut5yGe.js +29 -0
- package/dist/{types-CbcnJxCy.js.map → types-Buut5yGe.js.map} +1 -1
- package/dist/types-C48zuZbT.d.ts +156 -0
- package/dist/types-CYz4dqAd.d.ts +1366 -0
- package/dist/{ui-DfvPNmPP.js → ui-BIYUGm8D.js} +21 -17
- package/dist/ui-BIYUGm8D.js.map +1 -0
- package/dist/{vertex-BLxYasfw.js → vertex-DQQGhUn7.js} +2 -2
- package/dist/{vertex-BLxYasfw.js.map → vertex-DQQGhUn7.js.map} +1 -1
- package/dist/vertex-DtWe0Xf6.js +1 -0
- package/dist/view/index.d.ts +142 -809
- package/dist/view/index.js +17 -12
- package/dist/view/report/index.d.ts +217 -0
- package/dist/view/report/index.js +4 -0
- package/dist/views-DOYgIWRs.js +2 -0
- package/dist/{views-ChoSQxfF.js → views-DPsEJBBP.js} +6 -6
- package/dist/{views-ChoSQxfF.js.map → views-DPsEJBBP.js.map} +1 -1
- package/dist/{vm-container-DOtIi_rd.js → vm-container-rHUmgBRf.js} +13 -11
- package/dist/vm-container-rHUmgBRf.js.map +1 -0
- package/dist/{vm-images-CHutQVml.js → vm-images-BC2ZjZl5.js} +19 -21
- package/dist/vm-images-BC2ZjZl5.js.map +1 -0
- package/dist/wait-CixM6W0G.js +65 -0
- package/dist/wait-CixM6W0G.js.map +1 -0
- package/dist/{worker-entry-CKBfEt0h.js → worker-entry-CN1eVLzz.js} +7 -7
- package/dist/{worker-entry-CKBfEt0h.js.map → worker-entry-CN1eVLzz.js.map} +1 -1
- package/dist/{workspace-Cip2dVwU.js → workspace-BS-FUp5I.js} +9 -9
- package/dist/{workspace-Cip2dVwU.js.map → workspace-BS-FUp5I.js.map} +1 -1
- package/dist/workspace-D8HqjfsQ.js +1 -0
- package/package.json +38 -1
- package/skills/reeval/references/cli.md +1 -1
- package/skills/reeval/references/config.md +1 -1
- package/skills/reeval/references/environments.md +1 -1
- package/dist/agents-CEapsHV-.js.map +0 -1
- package/dist/analysis-BnG_1x_O.js.map +0 -1
- package/dist/app-files-Dsr3U1Q7.js +0 -10
- package/dist/app-files-Dsr3U1Q7.js.map +0 -1
- package/dist/assemble-DUtbByvj.js +0 -1
- package/dist/assemble-Dw1bF13I.js.map +0 -1
- package/dist/attach-DvYiMu6w.js.map +0 -1
- package/dist/attempt-host-DgeJFSZw.js.map +0 -1
- package/dist/boat-BBPOc_bs.js +0 -1
- package/dist/boat-CUNgSc75.js.map +0 -1
- package/dist/boat-images-BuFigiEb.js.map +0 -1
- package/dist/bundle-yEgI3RsI.js.map +0 -1
- package/dist/capture-DxxB3d7p.js +0 -1
- package/dist/cases-B7QkvMGE.js.map +0 -1
- package/dist/cases-CA7gxxdr.js +0 -1
- package/dist/cli-Z7aw8k25.js.map +0 -1
- package/dist/cli-spec-3zvLC9YW.js.map +0 -1
- package/dist/client-kUwtq-vt.js.map +0 -1
- package/dist/cloud-DuHcNQVV.js.map +0 -1
- package/dist/cloud-run-C08P_Uap.js +0 -1
- package/dist/cloud-run-DqMkSGYB.js.map +0 -1
- package/dist/cloud-secrets-Bak83tik.js.map +0 -1
- package/dist/command-Bs6Knfm4.js +0 -298
- package/dist/command-Bs6Knfm4.js.map +0 -1
- package/dist/command-CeygdoVe.js.map +0 -1
- package/dist/command-CoHN1ibe.js.map +0 -1
- package/dist/command-CtmSJzPC.js.map +0 -1
- package/dist/command-efTGfUYb.js.map +0 -1
- package/dist/commands-B_cAs8nG.js +0 -1
- package/dist/commands-DR-NolV2.js.map +0 -1
- package/dist/config-Bn2OhCN_.js.map +0 -1
- package/dist/context-Ba7PNMMe.js +0 -120
- package/dist/context-Ba7PNMMe.js.map +0 -1
- package/dist/daytona-CWs9j9mt.js.map +0 -1
- package/dist/daytona-image-BjVpi3KS.js +0 -1
- package/dist/db-Bjq6qr6c.js +0 -2
- package/dist/db-D6yQib7M.js.map +0 -1
- package/dist/debug-CF4p4Lp_.js +0 -1
- package/dist/debug-CIXzB4wK.js.map +0 -1
- package/dist/environment-BY8lVFFK.js.map +0 -1
- package/dist/evaluate-B8I9AKgw.js.map +0 -1
- package/dist/events-D97ZzAeq.js.map +0 -1
- package/dist/files-5HsFIpxw.js +0 -1
- package/dist/files-BNg3wgw_.js +0 -1
- package/dist/files-BesexM88.js.map +0 -1
- package/dist/files-DNgQwgA0.js +0 -1
- package/dist/files-DvDUSrWL.js +0 -1
- package/dist/files-TwSVeWwM.js.map +0 -1
- package/dist/gate-CgYdaprG.js +0 -49
- package/dist/gate-CgYdaprG.js.map +0 -1
- package/dist/harness-FcTyyPco.js.map +0 -1
- package/dist/ids-DqEvrBFZ.js.map +0 -1
- package/dist/init-CS5kEaVg.js +0 -1
- package/dist/ink-list-CLq6YVB8.js.map +0 -1
- package/dist/inputs-load-D4IsEjq3.js +0 -1
- package/dist/issue-DmSorsTF.js +0 -1
- package/dist/list-ClDb8qiX.js.map +0 -1
- package/dist/live-C7iGaDCA.js +0 -120
- package/dist/live-C7iGaDCA.js.map +0 -1
- package/dist/live-C7kN5v2P.js +0 -1
- package/dist/live-QFILMmql.js.map +0 -1
- package/dist/machine-service-xFhakTs5.js.map +0 -1
- package/dist/model-Colu5mEp.js +0 -1
- package/dist/model-mTaaEkeQ.js.map +0 -1
- package/dist/options-CmRy82gl.js +0 -1
- package/dist/pairs-BGb134Vu.js +0 -230
- package/dist/pairs-BGb134Vu.js.map +0 -1
- package/dist/patch-C7yBztZj.js.map +0 -1
- package/dist/paths-Dwmi7L4X.js +0 -75
- package/dist/paths-Dwmi7L4X.js.map +0 -1
- package/dist/preflight-xa8_eGOW.js +0 -1
- package/dist/prices-C_mbZa5a.js +0 -1
- package/dist/process-RPlEhpHz.js.map +0 -1
- package/dist/provider-DHT20Gl8.js +0 -1
- package/dist/query-BXV6pRyI.js +0 -1
- package/dist/questions-DnqkJga0.js.map +0 -1
- package/dist/record-2liAelrB.js.map +0 -1
- package/dist/redact-DMueYS-v.js.map +0 -1
- package/dist/remarks-D5qOZzcX.js.map +0 -1
- package/dist/resolve-B60w6DcB.js.map +0 -1
- package/dist/resolve-CCLiJX4i.js +0 -1
- package/dist/resolve-t9sOkGGS.js.map +0 -1
- package/dist/resources-DqdwihhT.js.map +0 -1
- package/dist/resume-4Q9q8AcE.js.map +0 -1
- package/dist/runtime-CIsljVBy.js +0 -457
- package/dist/runtime-CIsljVBy.js.map +0 -1
- package/dist/server-CeYUmuuu.js +0 -1
- package/dist/server-fSNuT3Z4.js +0 -284
- package/dist/server-fSNuT3Z4.js.map +0 -1
- package/dist/services-BG8xuU82.js.map +0 -1
- package/dist/shared-DtZwIAfy.js.map +0 -1
- package/dist/slug-IHh3f770.js.map +0 -1
- package/dist/source-n7nUVm9p.js +0 -1
- package/dist/source-zoRWRMql.js.map +0 -1
- package/dist/src-9V3C6A7C.js +0 -2
- package/dist/src-CB5cZhaN.js +0 -2
- package/dist/src-DB0vml0O.js +0 -1
- package/dist/src-OC89dAjO.js.map +0 -1
- package/dist/ssh-pool-ChpT-fDU.js +0 -1
- package/dist/ssh-pool-DwboLN9O.js.map +0 -1
- package/dist/start-BwMyDneA.js.map +0 -1
- package/dist/state-CXTaT0t8.js.map +0 -1
- package/dist/state-CltRWlCu.js +0 -1
- package/dist/suite-C4XSg6Ls.js.map +0 -1
- package/dist/suite-CIsyPqPA.js +0 -2
- package/dist/summary-DeeQJAR5.js +0 -169
- package/dist/summary-DeeQJAR5.js.map +0 -1
- package/dist/tart-CSP5Q0qV.js.map +0 -1
- package/dist/tart-cli-CqSpUYGV.js.map +0 -1
- package/dist/tart-images-De0-ib7R.js.map +0 -1
- package/dist/types-CbcnJxCy.js +0 -15
- package/dist/ui-DfvPNmPP.js.map +0 -1
- package/dist/vertex-DZDUpp0W.js +0 -1
- package/dist/views-DU0PM0UW.js +0 -2
- package/dist/vm-container-DOtIi_rd.js.map +0 -1
- package/dist/vm-images-CHutQVml.js.map +0 -1
- package/dist/workspace-BKcNJAPb.js +0 -1
|
@@ -1,51 +1,53 @@
|
|
|
1
|
-
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-
|
|
2
|
-
import { t as attemptDebugDir } from "./files-
|
|
3
|
-
import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-
|
|
1
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-D5WH0yFa.js";
|
|
2
|
+
import { t as attemptDebugDir } from "./files-Bv6ipFjc.js";
|
|
3
|
+
import { D as runMatchLabel, E as limitsLabel, T as TURN_STATUS, a as registerTest, b as judgeFailedMessage, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile, w as TURN_EDITED } from "./runtime-DsRjnydE.js";
|
|
4
4
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
|
|
5
|
-
import { t as
|
|
6
|
-
import {
|
|
7
|
-
import { _ as redactSecrets, b as redactValue, f as maskedVariables, m as redactJson, o as findSecrets, p as redactFields, v as redactSplitSecrets } from "./redact-DMueYS-v.js";
|
|
5
|
+
import { c as recordFileName, i as evalId, l as relativeFile, n as attemptKey, t as attemptId } from "./ids-CtxRn13a.js";
|
|
6
|
+
import { _ as redactSecrets, b as redactValue, f as maskedVariables, m as redactJson, o as findSecrets, p as redactFields, v as redactSplitSecrets } from "./redact-CfTHYWLv.js";
|
|
8
7
|
import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { t as
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import { n as
|
|
18
|
-
import {
|
|
19
|
-
import { n as
|
|
20
|
-
import { n as
|
|
21
|
-
import { n as
|
|
22
|
-
import { i as preparedWorkspace } from "./workspace-
|
|
23
|
-
import { A as sandboxPaths, I as workspaceOf, N as traitsOf, f as filesOf, i as callerDir, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-
|
|
24
|
-
import { f as runJudge, g as isTransient, h as isApiError, l as judgeModelName } from "./judge-
|
|
25
|
-
import { n as judgeDiff } from "./patch-
|
|
26
|
-
import {
|
|
27
|
-
import { t as transcriptForJudge } from "./transcript-
|
|
8
|
+
import { a as evalSlug } from "./cost-DQ0ayy6h.js";
|
|
9
|
+
import { c as withAttemptSpan, n as attemptSpan, o as openSpan, r as childSpan } from "./spans-tf3biLpT.js";
|
|
10
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-BwLa-4s-.js";
|
|
11
|
+
import { d as sandboxKind, n as capabilitiesOf, o as providerLimit } from "./kinds-BQX5ZxpK.js";
|
|
12
|
+
import { $ as plainHeaders, $t as artifactRefsOf, Bt as observer, E as resolveGateway, G as majorityOf, H as expand, It as mergeJudge, J as plainEnv, K as matchesTags, Lt as resolveStateOptions, M as brokeredIn, N as cascade, Qt as judgeProblem, Rt as observedHere, S as resolveModelGateway, Tt as phaseLabel, U as harnessId, V as evalPlan, Xt as NO_JUDGE_INPUT, Y as plainEnvError, Yt as INPUT_KEYS, Z as plainHeaderError, Zt as inputsOn, at as sandboxId, b as gatewayScope, bt as gatewayPolicy, dt as unbrokeredError, en as badJudgeArtifacts, ht as allowedHosts, it as runsOf, kt as unenforceable, lt as secretVarsIn, nn as filePartsOf, ot as sandboxProvider, rn as resolveJudgeArtifacts, rt as refuseIllegalCompare, st as sandboxSetBy, tn as decideState, ut as unbrokered, xt as isDirect, y as fellBackToDirect, yt as egressLabel, z as checkOptions, zt as observedName } from "./config-mU7wFvQq.js";
|
|
13
|
+
import { _ as layoutOf, b as refusedSharedMachine, f as expectedBlockedFor, g as keyRejected, l as brokeringFor, o as agentMcp, p as fakeBrokering, s as agentOf, t as AGENTS, u as claudeMcpServers, v as refusedMcp, x as writePluginFiles, y as refusedPlugins } from "./agents-Dnv1MmgY.js";
|
|
14
|
+
import { S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, k as pluginSourceLabel, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-DVEOMsYw.js";
|
|
15
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-nRzGt4Ff.js";
|
|
16
|
+
import { n as shellQuote$1 } from "./process-C_nh1H7X.js";
|
|
17
|
+
import { i as hasNetworkSession, r as RunStopped, t as InfraError } from "./types-Buut5yGe.js";
|
|
18
|
+
import { a as hostEnv, n as fetchHeldSecrets } from "./secrets-file-B8rUJV2L.js";
|
|
19
|
+
import { n as fakeBootstrapScript } from "./fake-bootstrap-uCyHhnMO.js";
|
|
20
|
+
import { n as providerStop, t as ProviderStopError } from "./provider-stop-DjBMfZW5.js";
|
|
21
|
+
import { i as preparedWorkspace } from "./workspace-BS-FUp5I.js";
|
|
22
|
+
import { A as sandboxPaths, I as workspaceOf, N as traitsOf, f as filesOf, i as callerDir, l as environmentFrom, p as globFilter, u as environmentPaths, z as withoutOtel } from "./environment-BCBg6c0p.js";
|
|
23
|
+
import { f as runJudge, g as isTransient, h as isApiError, l as judgeModelName } from "./judge-etHC4K02.js";
|
|
24
|
+
import { n as judgeDiff } from "./patch-Dxzv8msl.js";
|
|
25
|
+
import { r as patchPath } from "./patch-uwo6t4Zy.js";
|
|
26
|
+
import { t as transcriptForJudge } from "./transcript-TWiaAuoL.js";
|
|
27
|
+
import { r as withLease } from "./client-Dj65gXG3.js";
|
|
28
28
|
import { n as modelCallEvents } from "./model-calls-B0BoQfJ4.js";
|
|
29
|
-
import { t as PrepareError } from "./provider-
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
29
|
+
import { t as PrepareError } from "./provider-Cn7lmECy.js";
|
|
30
|
+
import { n as resourcesLine, t as turnErrorKind } from "./turn-error-Dmv_Muj7.js";
|
|
31
|
+
import { l as messageLine } from "./suite-oH7uiCRq.js";
|
|
32
|
+
import { Qt as OPAQUE_OUTPUT, an as extractTar, ln as writeTar, on as readTar, p as MCP_TIMER } from "./src-D1Q6ElaF.js";
|
|
33
|
+
import { s as formatBytes } from "./debug-T29p6HOX.js";
|
|
33
34
|
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
34
|
-
import {
|
|
35
|
+
import { n as dataProblem, t as closest } from "./shared-bXaqVxk-.js";
|
|
35
36
|
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
36
|
-
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-
|
|
37
|
-
import { t as environmentInputs } from "./inputs-
|
|
37
|
+
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-CSPyKuV6.js";
|
|
38
|
+
import { t as environmentInputs } from "./inputs-BWTtz1-v.js";
|
|
39
|
+
import { r as parseDuration, t as MAX_TIMER_MS } from "./duration-Bz2H7pXb.js";
|
|
38
40
|
import { r as openRouterFallbacks } from "./openrouter-models-B1sIOTD2.js";
|
|
39
41
|
import { h as vercelCredentialProblem, m as vercelBaseUrlProblem, p as vercelBaseUrl } from "./routes-Cp4gAjBZ.js";
|
|
40
|
-
import { v as vertexSettings } from "./vertex-
|
|
41
|
-
import { i as withDefaultEffort } from "./default-effort-
|
|
42
|
-
import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-
|
|
42
|
+
import { v as vertexSettings } from "./vertex-DQQGhUn7.js";
|
|
43
|
+
import { i as withDefaultEffort } from "./default-effort-BDYXBzu5.js";
|
|
44
|
+
import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-DPIffsN8.js";
|
|
43
45
|
import { z } from "zod";
|
|
44
|
-
import { randomBytes, randomUUID } from "node:crypto";
|
|
45
46
|
import { dirname, join } from "node:path";
|
|
47
|
+
import { parse, stringify } from "yaml";
|
|
48
|
+
import { randomBytes, randomUUID } from "node:crypto";
|
|
46
49
|
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
47
50
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
48
|
-
import { parse, stringify } from "yaml";
|
|
49
51
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
50
52
|
import { resolveBridgePort } from "@ai-sdk/harness/utils";
|
|
51
53
|
// src/attempt/assertions.ts
|
|
@@ -87,18 +89,39 @@ function assertionRecords(raw) {
|
|
|
87
89
|
} while (used.has(label));
|
|
88
90
|
seen.set(base, n);
|
|
89
91
|
used.add(label);
|
|
92
|
+
const reason = r.matcher === "skip" && typeof r.args[0] === "string" ? clipped(r.args[0], MESSAGE_LIMIT) : void 0;
|
|
93
|
+
const fileMatcher = /(^|\.)toHaveFile$/.test(r.matcher);
|
|
94
|
+
const target = r.target ?? (fileMatcher && typeof r.args[0] === "string" ? r.args[1] === void 0 ? r.args[0] : `${r.args[0]} ${r.args[1] instanceof RegExp ? r.args[1].source : shown(r.args[1])}` : void 0);
|
|
90
95
|
return {
|
|
91
96
|
label,
|
|
92
|
-
labelled: r.label !== void 0,
|
|
97
|
+
labelled: r.label !== void 0 || r.matcher === "skip",
|
|
93
98
|
matcher: r.matcher,
|
|
94
99
|
status: r.status,
|
|
95
100
|
soft: r.soft,
|
|
96
101
|
...r.at ? { at: r.at } : {},
|
|
97
|
-
...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
|
|
102
|
+
...r.status === "pass" || r.status === "not_applicable" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
|
|
103
|
+
...reason !== void 0 ? { reason } : {},
|
|
104
|
+
...target !== void 0 ? { target: clipped(target, LABEL_LIMIT) } : {},
|
|
105
|
+
...r.preSatisfied ? { preSatisfied: true } : {},
|
|
98
106
|
...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
|
|
99
107
|
};
|
|
100
108
|
});
|
|
101
109
|
}
|
|
110
|
+
const JUDGE_ERROR = /^judge failed for /;
|
|
111
|
+
/**
|
|
112
|
+
* Writes each judgment's `assertion` from the matching `toPass` label when counts align, and
|
|
113
|
+
* copies `JudgeOptions.label` already on the judgment.
|
|
114
|
+
*/
|
|
115
|
+
function linkJudgmentAssertions(judgments, assertions) {
|
|
116
|
+
const passes = assertions.filter((x) => /(^|\.)toPass$/.test(x.matcher) && !JUDGE_ERROR.test(x.message ?? ""));
|
|
117
|
+
if (passes.length !== judgments.length) return;
|
|
118
|
+
for (let i = 0; i < judgments.length; i++) {
|
|
119
|
+
const j = judgments[i];
|
|
120
|
+
const x = passes[i];
|
|
121
|
+
if (!j || !x) continue;
|
|
122
|
+
j.assertion = x.label;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
102
125
|
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
103
126
|
/**
|
|
104
127
|
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
@@ -106,7 +129,7 @@ const CAUGHT_HINT = "(the eval caught this failed check, which still fails the a
|
|
|
106
129
|
* `CAUGHT_HINT` after its message.
|
|
107
130
|
*/
|
|
108
131
|
function checkFailure(raw) {
|
|
109
|
-
const failed = raw.find((r) => r.status !== "pass");
|
|
132
|
+
const failed = raw.find((r) => r.status !== "pass" && r.status !== "not_applicable");
|
|
110
133
|
if (!failed) return void 0;
|
|
111
134
|
const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
|
|
112
135
|
if (failed.soft) return error;
|
|
@@ -129,6 +152,115 @@ const own = (value) => ({
|
|
|
129
152
|
configurable: true,
|
|
130
153
|
enumerable: false
|
|
131
154
|
});
|
|
155
|
+
// src/attempt/context.ts
|
|
156
|
+
/** Under the home directory, and under each adapter's own home, which an installed CLI runs with. */
|
|
157
|
+
const HOME_CONTEXT_PATHS = [
|
|
158
|
+
".cursor/mcp.json",
|
|
159
|
+
".cursor/cli-config.json",
|
|
160
|
+
".cursor/rules",
|
|
161
|
+
".cursor/skills",
|
|
162
|
+
".cursor/skills-cursor",
|
|
163
|
+
".cursor/plugins",
|
|
164
|
+
".agents/skills",
|
|
165
|
+
".agents/plugins",
|
|
166
|
+
".claude/CLAUDE.md",
|
|
167
|
+
".claude/settings.json",
|
|
168
|
+
".claude/skills",
|
|
169
|
+
".gemini/GEMINI.md",
|
|
170
|
+
".gemini/AGENTS.md",
|
|
171
|
+
".gemini/config/skills",
|
|
172
|
+
".gemini/config/rules",
|
|
173
|
+
".gemini/config/mcp_config.json",
|
|
174
|
+
".gemini/config/plugins",
|
|
175
|
+
".gemini/antigravity-cli/settings.json",
|
|
176
|
+
".config/opencode/skills",
|
|
177
|
+
".config/opencode/opencode.json",
|
|
178
|
+
".config/opencode/opencode.jsonc",
|
|
179
|
+
".config/opencode/AGENTS.md",
|
|
180
|
+
".opencode/skills",
|
|
181
|
+
".config/goose/skills",
|
|
182
|
+
".config/goose/config.yaml",
|
|
183
|
+
".config/goose/.goosehints",
|
|
184
|
+
".pi/agent/skills",
|
|
185
|
+
".pi/agent/AGENTS.md",
|
|
186
|
+
".pi/agent/settings.json",
|
|
187
|
+
".codex/AGENTS.md",
|
|
188
|
+
".codex/config.toml",
|
|
189
|
+
".codex/skills",
|
|
190
|
+
".codex/plugins",
|
|
191
|
+
".fx/skills",
|
|
192
|
+
".fx/settings.json",
|
|
193
|
+
".fx/mcp.json",
|
|
194
|
+
".grok/skills",
|
|
195
|
+
".grok/config.toml",
|
|
196
|
+
".reeval/plugins"
|
|
197
|
+
];
|
|
198
|
+
/** Where the AI SDK adapters install their CLI; the CLI's `$HOME` is `home` under it. */
|
|
199
|
+
const ADAPTER_HOMES = "\"$HOME\"/.ai-sdk-harness/.harness-bootstrap/*/home";
|
|
200
|
+
/**
|
|
201
|
+
* Skills, rules and MCP config the agent could load, with content hashes (never contents).
|
|
202
|
+
* Paths under the home directory are shown as `~/…`; others are relative to the workspace.
|
|
203
|
+
*/
|
|
204
|
+
const CONTEXT_PATHS = [
|
|
205
|
+
...HOME_CONTEXT_PATHS.map((p) => `$HOME/${p}`),
|
|
206
|
+
"AGENTS.md",
|
|
207
|
+
"CLAUDE.md",
|
|
208
|
+
"GEMINI.md",
|
|
209
|
+
".cursorrules",
|
|
210
|
+
".cursor/rules",
|
|
211
|
+
".cursor/skills",
|
|
212
|
+
".cursor/agents",
|
|
213
|
+
".cursor/commands",
|
|
214
|
+
".cursor/hooks.json",
|
|
215
|
+
".cursor/mcp.json",
|
|
216
|
+
".claude/settings.json",
|
|
217
|
+
".claude/skills",
|
|
218
|
+
".agents/skills",
|
|
219
|
+
".agents/rules",
|
|
220
|
+
".agents/mcp_config.json",
|
|
221
|
+
".mcp.json",
|
|
222
|
+
".opencode/skills",
|
|
223
|
+
".opencode/agents",
|
|
224
|
+
".opencode/commands",
|
|
225
|
+
"opencode.json",
|
|
226
|
+
"opencode.jsonc",
|
|
227
|
+
".goosehints",
|
|
228
|
+
".goose/skills",
|
|
229
|
+
".pi/skills",
|
|
230
|
+
".pi/prompts",
|
|
231
|
+
".pi/settings.json",
|
|
232
|
+
".codex/config.toml",
|
|
233
|
+
".fx/skills",
|
|
234
|
+
".grok/skills",
|
|
235
|
+
".grok/config.toml"
|
|
236
|
+
];
|
|
237
|
+
async function agentContext(box, signal) {
|
|
238
|
+
const adapterPaths = `for h in ${ADAPTER_HOMES}; do for p in ${HOME_CONTEXT_PATHS.join(" ")}; do echo "$h/$p"; done; done`;
|
|
239
|
+
const script = `{ for p in ${CONTEXT_PATHS.map((p) => `"${p}"`).join(" ")}; do echo "$p"; done; ${adapterPaths}; } | while IFS= read -r p; do [ -e "$p" ] && find "$p" -type f 2>/dev/null; done | sort | head -500 | while IFS= read -r f; do printf '%s\t%s\n' "$(sha256sum "$f" | cut -d' ' -f1)" "$f"; done`;
|
|
240
|
+
const r = await box.exec(script, { signal }).catch(() => null);
|
|
241
|
+
if (r?.exitCode !== 0) return [];
|
|
242
|
+
const home = (await box.exec("printf %s \"$HOME\"", { signal }).catch(() => null))?.stdout ?? "";
|
|
243
|
+
return r.stdout.split("\n").filter(Boolean).map((line) => {
|
|
244
|
+
const [sha, ...rest] = line.split(" ");
|
|
245
|
+
const path = rest.join(" ");
|
|
246
|
+
return {
|
|
247
|
+
path: home && path.startsWith(`${home}/`) ? `~${path.slice(home.length)}` : path,
|
|
248
|
+
sha256: sha ?? ""
|
|
249
|
+
};
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Records the agent's context once, after the first turn (or at teardown when there was none):
|
|
254
|
+
* adapters write harness skills when a turn starts, so before it the record would miss them.
|
|
255
|
+
*/
|
|
256
|
+
function contextTaker(a) {
|
|
257
|
+
let taken = false;
|
|
258
|
+
return async (box, signal) => {
|
|
259
|
+
if (taken || !capabilitiesOf(a.spec.sandbox).isolated) return;
|
|
260
|
+
taken = true;
|
|
261
|
+
a.record.context = await agentContext(box, signal);
|
|
262
|
+
};
|
|
263
|
+
}
|
|
132
264
|
// src/attempt/debug.ts
|
|
133
265
|
/** Which config and profile the attempt runs under, and where each resolved option came from. */
|
|
134
266
|
function tracePlan(debug, spec, ctx) {
|
|
@@ -572,7 +704,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
572
704
|
/** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
|
|
573
705
|
async function createHarness(descriptor, sandbox, vertex) {
|
|
574
706
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
575
|
-
const { createAgentHarness } = await import("./agent-
|
|
707
|
+
const { createAgentHarness } = await import("./agent-D67VCd6Q.js");
|
|
576
708
|
return createAgentHarness(descriptor, sandbox, vertex);
|
|
577
709
|
}
|
|
578
710
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
@@ -605,8 +737,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
605
737
|
record.image = null;
|
|
606
738
|
const kind = sandboxKind(spec.sandbox);
|
|
607
739
|
if (kind.capabilities.images) {
|
|
608
|
-
const { preparedFor } = await import("./preflight-
|
|
609
|
-
const { attemptImage, createProviders, preparedImage } = await import("./providers-
|
|
740
|
+
const { preparedFor } = await import("./preflight-DXMD6jou.js");
|
|
741
|
+
const { attemptImage, createProviders, preparedImage } = await import("./providers-Bm5NLaRh.js");
|
|
610
742
|
signal.throwIfAborted();
|
|
611
743
|
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
612
744
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -631,8 +763,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
631
763
|
const { ref, id, base, install, agentCli } = record.image;
|
|
632
764
|
debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
|
|
633
765
|
} else if (kind.capabilities.isolated) {
|
|
634
|
-
const { preparedFor } = await import("./preflight-
|
|
635
|
-
const { bakedImage, createProviders, preparedHosts, sandboxBase, vmImage } = await import("./providers-
|
|
766
|
+
const { preparedFor } = await import("./preflight-DXMD6jou.js");
|
|
767
|
+
const { bakedImage, createProviders, preparedHosts, sandboxBase, vmImage } = await import("./providers-Bm5NLaRh.js");
|
|
636
768
|
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
637
769
|
const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
|
|
638
770
|
const vm = prepared ? vmImage(prepared) : null;
|
|
@@ -949,6 +1081,7 @@ function judgeCaller(a, readFile) {
|
|
|
949
1081
|
const { judgements: judgments } = a.record;
|
|
950
1082
|
judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
|
|
951
1083
|
...recorded,
|
|
1084
|
+
...options.label ? { label: options.label } : {},
|
|
952
1085
|
...retries.length ? { retries } : {},
|
|
953
1086
|
...usage ? { usage } : {},
|
|
954
1087
|
call: order
|
|
@@ -992,9 +1125,18 @@ function judgeCaller(a, readFile) {
|
|
|
992
1125
|
async function createSandbox(descriptor, options) {
|
|
993
1126
|
const kind = sandboxKind(descriptor);
|
|
994
1127
|
try {
|
|
995
|
-
|
|
1128
|
+
const box = await kind.create(descriptor, {
|
|
1129
|
+
...options,
|
|
1130
|
+
env: withoutOtel(options.env)
|
|
1131
|
+
});
|
|
1132
|
+
const exec = box.exec.bind(box);
|
|
1133
|
+
box.exec = (command, opts) => exec(command, opts?.env ? {
|
|
1134
|
+
...opts,
|
|
1135
|
+
env: withoutOtel(opts.env)
|
|
1136
|
+
} : opts);
|
|
1137
|
+
return box;
|
|
996
1138
|
} catch (error) {
|
|
997
|
-
if (error instanceof InfraError || options.signal.aborted) throw error;
|
|
1139
|
+
if (error instanceof InfraError || error instanceof RunStopped || options.signal.aborted) throw error;
|
|
998
1140
|
throw new InfraError(`Failed to create ${kind.provider(descriptor)} sandbox: ${error.message}`);
|
|
999
1141
|
}
|
|
1000
1142
|
}
|
|
@@ -1184,6 +1326,36 @@ function sumSplits(splits) {
|
|
|
1184
1326
|
};
|
|
1185
1327
|
}
|
|
1186
1328
|
// src/attempt/record.ts
|
|
1329
|
+
/**
|
|
1330
|
+
* An unhandled mid-turn `status === 'error'` (body never read `turn.status`) becomes an
|
|
1331
|
+
* infra error so crashed attempts leave the counted pass/fail denominator.
|
|
1332
|
+
*/
|
|
1333
|
+
function reclassifyTurnErrors(a) {
|
|
1334
|
+
const { record } = a;
|
|
1335
|
+
if (record.outcome !== "pass" && record.outcome !== "fail") return;
|
|
1336
|
+
const index = record.turns.findIndex((t) => t.status === "error");
|
|
1337
|
+
if (index < 0) return;
|
|
1338
|
+
if (a.turnStatusRead) {
|
|
1339
|
+
record.turnStatusRead = true;
|
|
1340
|
+
return;
|
|
1341
|
+
}
|
|
1342
|
+
record.turnStatusRead = false;
|
|
1343
|
+
const turn = record.turns[index];
|
|
1344
|
+
if (!turn) return;
|
|
1345
|
+
const errEvent = turn.transcript.find((e) => e.type === "error");
|
|
1346
|
+
const errText = errEvent && errEvent.type === "error" ? errEvent.message : void 0;
|
|
1347
|
+
const kind = turnErrorKind(errText);
|
|
1348
|
+
const detail = messageLine(errText ?? "unknown error");
|
|
1349
|
+
record.outcome = "infra_error";
|
|
1350
|
+
record.error = {
|
|
1351
|
+
kind: "infra",
|
|
1352
|
+
message: `Turn ${index} ended with an error: ${detail}`,
|
|
1353
|
+
turn: {
|
|
1354
|
+
index,
|
|
1355
|
+
kind
|
|
1356
|
+
}
|
|
1357
|
+
};
|
|
1358
|
+
}
|
|
1187
1359
|
function classify(error, signal) {
|
|
1188
1360
|
const message = error instanceof Error ? error.message : String(error);
|
|
1189
1361
|
if (signal.aborted) return isTimeout(signal.reason) ? {
|
|
@@ -1193,6 +1365,10 @@ function classify(error, signal) {
|
|
|
1193
1365
|
kind: "aborted",
|
|
1194
1366
|
message: "The run was canceled"
|
|
1195
1367
|
};
|
|
1368
|
+
if (error instanceof RunStopped) return {
|
|
1369
|
+
kind: "aborted",
|
|
1370
|
+
message
|
|
1371
|
+
};
|
|
1196
1372
|
if (error instanceof InfraError) return {
|
|
1197
1373
|
kind: "infra",
|
|
1198
1374
|
message
|
|
@@ -1411,12 +1587,12 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
1411
1587
|
}
|
|
1412
1588
|
/** The record's checks that didn't pass, for the terminal; unset when all did. */
|
|
1413
1589
|
function failedChecks(record) {
|
|
1414
|
-
const failed = (record.assertions ?? []).filter((x) => x.status !== "pass");
|
|
1590
|
+
const failed = (record.assertions ?? []).filter((x) => x.status !== "pass" && x.status !== "not_applicable");
|
|
1415
1591
|
if (!failed.length) return {};
|
|
1416
1592
|
return { failedChecks: failed.map((x) => ({
|
|
1417
1593
|
label: x.label,
|
|
1418
1594
|
...x.at ? { at: x.at } : {},
|
|
1419
|
-
message: clip(x.message?.split("\n")[0] ?? "", 500)
|
|
1595
|
+
message: clip(x.message?.split("\n")[0] ?? x.reason?.split("\n")[0] ?? "", 500)
|
|
1420
1596
|
})) };
|
|
1421
1597
|
}
|
|
1422
1598
|
/** `turn N error: <message>` for each turn with an error event. */
|
|
@@ -1645,6 +1821,14 @@ const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
|
|
|
1645
1821
|
// src/attempt/state.ts
|
|
1646
1822
|
/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
|
|
1647
1823
|
const SETUP_BUDGET_MS = 12e5;
|
|
1824
|
+
/** Lets the language-runner host find the live Attempt from the body's `AttemptContext`. */
|
|
1825
|
+
const attemptByContext = /* @__PURE__ */ new WeakMap();
|
|
1826
|
+
function bindAttemptContext(c, a) {
|
|
1827
|
+
attemptByContext.set(c, a);
|
|
1828
|
+
}
|
|
1829
|
+
function attemptForContext(c) {
|
|
1830
|
+
return attemptByContext.get(c);
|
|
1831
|
+
}
|
|
1648
1832
|
function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
1649
1833
|
const record = {
|
|
1650
1834
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -1688,11 +1872,16 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
|
1688
1872
|
at: Date.now(),
|
|
1689
1873
|
...redactFields(update, ["kind"])
|
|
1690
1874
|
}),
|
|
1691
|
-
async timed(phase, fn) {
|
|
1875
|
+
async timed(phase, fn, name) {
|
|
1692
1876
|
const start = performance.now();
|
|
1877
|
+
const span = childSpan(name ?? phase.replace(/Ms$/, ""));
|
|
1693
1878
|
try {
|
|
1694
1879
|
return await fn();
|
|
1880
|
+
} catch (error) {
|
|
1881
|
+
span.end(error);
|
|
1882
|
+
throw error;
|
|
1695
1883
|
} finally {
|
|
1884
|
+
span.end();
|
|
1696
1885
|
record.timings[phase] += Math.round(performance.now() - start);
|
|
1697
1886
|
}
|
|
1698
1887
|
},
|
|
@@ -1707,7 +1896,8 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
|
1707
1896
|
capturePages: /* @__PURE__ */ new Map(),
|
|
1708
1897
|
stepBlobs: /* @__PURE__ */ new Map(),
|
|
1709
1898
|
snapshotDir,
|
|
1710
|
-
turnSteps: /* @__PURE__ */ new Map()
|
|
1899
|
+
turnSteps: /* @__PURE__ */ new Map(),
|
|
1900
|
+
turnStatusRead: false
|
|
1711
1901
|
};
|
|
1712
1902
|
return a;
|
|
1713
1903
|
}
|
|
@@ -2122,12 +2312,13 @@ function startHarness(a, harness, box, options) {
|
|
|
2122
2312
|
* the diff, switches to the agent's egress, and starts the harness session with the
|
|
2123
2313
|
* harness-placed skills and MCP servers.
|
|
2124
2314
|
*/
|
|
2125
|
-
async function startSession(a, runId) {
|
|
2315
|
+
async function startSession(a, runId, attempt) {
|
|
2126
2316
|
const startedAt = performance.now();
|
|
2127
2317
|
const { spec } = a;
|
|
2128
2318
|
const env = a.env;
|
|
2129
2319
|
const box = await createSandbox(spec.sandbox, {
|
|
2130
2320
|
runId,
|
|
2321
|
+
attempt,
|
|
2131
2322
|
timeoutMs: spec.timeoutMs + setupAllowance(spec) + 6e4,
|
|
2132
2323
|
signal: a.signal,
|
|
2133
2324
|
env: env.vars,
|
|
@@ -2155,6 +2346,7 @@ async function startSession(a, runId) {
|
|
|
2155
2346
|
a.phases = brokerPhases(box, env.substitutions, spec.egress);
|
|
2156
2347
|
if (a.signal.aborted) box.destroy();
|
|
2157
2348
|
a.signal.throwIfAborted();
|
|
2349
|
+
a.report({ kind: "sandbox-up" });
|
|
2158
2350
|
await a.phases.start();
|
|
2159
2351
|
if (env.substitutions.length) {
|
|
2160
2352
|
const agent = env.substitutions.filter((s) => s.phase === "agent").map((s) => s.name);
|
|
@@ -2660,14 +2852,21 @@ function turnSender(a, takeContext, checks) {
|
|
|
2660
2852
|
const { diff } = changes;
|
|
2661
2853
|
a.record.diff = diff;
|
|
2662
2854
|
if (a.spec.snapshot.when === "turns") await takeSnapshot(a, box, `turn-${turnNumber}`, "harness");
|
|
2855
|
+
const { status: turnStatus, ...rest } = result;
|
|
2663
2856
|
const data = {
|
|
2664
2857
|
prompt,
|
|
2665
|
-
...
|
|
2858
|
+
...rest,
|
|
2859
|
+
status: turnStatus,
|
|
2666
2860
|
diff
|
|
2667
2861
|
};
|
|
2668
2862
|
const turn = Object.freeze(Object.assign({
|
|
2669
2863
|
...data,
|
|
2670
|
-
...checks
|
|
2864
|
+
...checks,
|
|
2865
|
+
[TURN_STATUS]: turnStatus,
|
|
2866
|
+
get status() {
|
|
2867
|
+
a.turnStatusRead = true;
|
|
2868
|
+
return turnStatus;
|
|
2869
|
+
}
|
|
2671
2870
|
}, { [TURN_EDITED]: edited }));
|
|
2672
2871
|
a.turns.push(turn);
|
|
2673
2872
|
return turn;
|
|
@@ -2686,23 +2885,38 @@ async function runAttempt(spec, fn, handle) {
|
|
|
2686
2885
|
traceLimit: config.debug?.traceBufferBytes ?? TRACE_BUFFER_DEFAULT,
|
|
2687
2886
|
...verbosity > 0 ? { emit: handle.debug } : {}
|
|
2688
2887
|
});
|
|
2689
|
-
|
|
2888
|
+
const span = openSpan("attempt", {
|
|
2889
|
+
"reeval.eval.id": id,
|
|
2890
|
+
"reeval.attempt.key": attemptKey(id, spec.cell.key, spec.n),
|
|
2891
|
+
"reeval.cell": spec.cell.key,
|
|
2892
|
+
"reeval.attempt.n": spec.n
|
|
2893
|
+
});
|
|
2894
|
+
let error;
|
|
2895
|
+
try {
|
|
2896
|
+
return await withDebug(debug, () => withAttemptSpan(span, () => attemptBody(spec, fn, handle, id, debug)));
|
|
2897
|
+
} catch (thrown) {
|
|
2898
|
+
error = thrown;
|
|
2899
|
+
throw thrown;
|
|
2900
|
+
} finally {
|
|
2901
|
+
span.end(error);
|
|
2902
|
+
}
|
|
2690
2903
|
}
|
|
2691
2904
|
async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, unawaited, expectError, assertions }, id, debug) {
|
|
2692
2905
|
const ctx = getRunContext();
|
|
2693
2906
|
const { runId, stagingDir, config } = ctx;
|
|
2694
2907
|
tracePlan(debug, spec, ctx);
|
|
2908
|
+
const key = attemptKey(id, spec.cell.key, spec.n);
|
|
2695
2909
|
let timer;
|
|
2696
2910
|
let clock;
|
|
2697
2911
|
const a = createAttempt(spec, testSignal, progress, async (a, who) => {
|
|
2698
2912
|
debug.log("container", `starting the sandbox and session for ${who}`);
|
|
2699
2913
|
clock?.pause();
|
|
2700
2914
|
try {
|
|
2701
|
-
return await a.timed("setupMs", () => startSession(a, runId));
|
|
2915
|
+
return await a.timed("setupMs", () => startSession(a, runId, key), "sandbox");
|
|
2702
2916
|
} finally {
|
|
2703
2917
|
clock?.resume();
|
|
2704
2918
|
}
|
|
2705
|
-
}, snapshotSidecar(stagingDir,
|
|
2919
|
+
}, snapshotSidecar(stagingDir, key));
|
|
2706
2920
|
const destroy = () => void Promise.resolve(a.session?.salvage?.()).catch(() => {}).finally(() => a.sandbox?.destroy());
|
|
2707
2921
|
testSignal.addEventListener("abort", destroy, { once: true });
|
|
2708
2922
|
const takeContext = contextTaker(a);
|
|
@@ -2729,6 +2943,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2729
2943
|
}),
|
|
2730
2944
|
judge: judgeCaller(a, readFile)
|
|
2731
2945
|
};
|
|
2946
|
+
bindAttemptContext(c, a);
|
|
2732
2947
|
const timedOut = () => !!timer?.aborted && !testSignal.aborted;
|
|
2733
2948
|
let failure;
|
|
2734
2949
|
let oom;
|
|
@@ -2778,15 +2993,20 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2778
2993
|
const { record } = a;
|
|
2779
2994
|
const checked = assertionRecords(assertions());
|
|
2780
2995
|
if (checked.length) record.assertions = checked;
|
|
2996
|
+
linkJudgmentAssertions(record.judgements ?? [], checked);
|
|
2781
2997
|
for (const x of checked) {
|
|
2782
2998
|
const why = x.message ? ` · ${clip(redactSecrets(x.message.split("\n")[0] ?? ""), 160)}` : "";
|
|
2783
|
-
|
|
2999
|
+
const mark = x.status === "pass" || x.status === "not_applicable" ? "✓" : "✗";
|
|
3000
|
+
debug.log("check", `assert ${mark} ${x.label} (${x.matcher}, ${x.status})${why}`);
|
|
2784
3001
|
}
|
|
3002
|
+
reclassifyTurnErrors(a);
|
|
2785
3003
|
const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
|
|
2786
3004
|
debug.log("teardown", `outcome ${record.outcome}${error}`);
|
|
2787
3005
|
const rel = (path) => relativeFile(config.root, path) || ".";
|
|
2788
3006
|
const scratch = `.reeval/runs/${runId}`;
|
|
2789
3007
|
debug.log("teardown", `record ${scratch}.json · artifacts ${scratch}/${evalSlug(id)}/${evalSlug(spec.cell.key)}/${spec.n} · debug ${rel(debug.dir)}`);
|
|
3008
|
+
attemptSpan()?.set({ "reeval.outcome": record.outcome });
|
|
3009
|
+
if (record.outcome === "fail" || record.outcome === "infra_error") attemptSpan()?.fail(record.error?.message.split("\n")[0] ?? record.outcome);
|
|
2790
3010
|
setMeta(await writeRecord(a, stagingDir, id));
|
|
2791
3011
|
await debug.finish(debug.verbosity === 2 || record.outcome !== "pass");
|
|
2792
3012
|
}
|
|
@@ -3795,6 +4015,6 @@ function optionsFrom(written, where, base) {
|
|
|
3795
4015
|
}
|
|
3796
4016
|
};
|
|
3797
4017
|
}
|
|
3798
|
-
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
4018
|
+
export { assertionRecords as a, attemptForContext as i, evaluate as n, group as r, declareTree as t };
|
|
3799
4019
|
|
|
3800
|
-
//# sourceMappingURL=evaluate-
|
|
4020
|
+
//# sourceMappingURL=evaluate-wgW1lGQj.js.map
|