reeval 0.16.1 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/THIRD_PARTY_NOTICES.md +14 -14
- package/dist/{adapter-CRKYPkRt.js → adapter-BIlJWXff.js} +9 -13
- package/dist/adapter-BIlJWXff.js.map +1 -0
- package/dist/{agent-BgH4-kFc.js → agent-CE-qdvmh.js} +49 -25
- package/dist/agent-CE-qdvmh.js.map +1 -0
- package/dist/{analysis-C2-aEbr7.js → analysis-CTBb_C0Z.js} +2 -3
- package/dist/{analysis-C2-aEbr7.js.map → analysis-CTBb_C0Z.js.map} +1 -1
- package/dist/app-files-YMmDAHA8.js +10 -0
- package/dist/app-files-YMmDAHA8.js.map +1 -0
- package/dist/{assemble-BVbI6NW-.js → assemble-7RI7Nuhz.js} +38 -23
- package/dist/assemble-7RI7Nuhz.js.map +1 -0
- package/dist/assemble-CANqwtwD.js +1 -0
- package/dist/{attempt-CpbZ9arV.js → attempt-CtKGvaQR.js} +4 -5
- package/dist/{attempt-CpbZ9arV.js.map → attempt-CtKGvaQR.js.map} +1 -1
- package/dist/{attempt-host-CIxjs2Be.js → attempt-host-CJM_wEA4.js} +13 -13
- package/dist/attempt-host-CJM_wEA4.js.map +1 -0
- package/dist/{budget-DBUrH4jX.js → budget-BcxLYY5L.js} +2 -3
- package/dist/budget-BcxLYY5L.js.map +1 -0
- package/dist/{capture-iQ6kYJV0.js → capture-DF8cHsJh.js} +583 -125
- package/dist/capture-DF8cHsJh.js.map +1 -0
- package/dist/capture-DMBZE9cs.js +14 -0
- package/dist/capture-DMBZE9cs.js.map +1 -0
- package/dist/capture-DagM1npp.js +1 -0
- package/dist/{cli-1YQU-66F.js → cli-Bm9ntSxz.js} +342 -101
- package/dist/cli-Bm9ntSxz.js.map +1 -0
- package/dist/{cli-spec-CPWT1mAG.js → cli-spec-ChkleEVm.js} +2 -3
- package/dist/{cli-spec-CPWT1mAG.js.map → cli-spec-ChkleEVm.js.map} +1 -1
- package/dist/cli.js +3 -4
- package/dist/{client-DMmOkbRA.js → client-CF-DOjOU.js} +6 -8
- package/dist/{client-DMmOkbRA.js.map → client-CF-DOjOU.js.map} +1 -1
- package/dist/{colors-QV9dFVhN.js → colors-DcXBvAlU.js} +21 -4
- package/dist/colors-DcXBvAlU.js.map +1 -0
- package/dist/{command-Cr1zjRvl.js → command-B7MslY_w.js} +22 -24
- package/dist/command-B7MslY_w.js.map +1 -0
- package/dist/{command-C-HeGlap.js → command-DDkz6jsD.js} +25 -29
- package/dist/{command-C-HeGlap.js.map → command-DDkz6jsD.js.map} +1 -1
- package/dist/{command-BlYCAqu-.js → command-DPF8hnWS.js} +17 -21
- package/dist/command-DPF8hnWS.js.map +1 -0
- package/dist/{command-8_Cg5v6b.js → command-Dk6wggV-.js} +22 -27
- package/dist/command-Dk6wggV-.js.map +1 -0
- package/dist/{commands-BvTNZ2-b.js → commands-DHT-nhEJ.js} +60 -23
- package/dist/commands-DHT-nhEJ.js.map +1 -0
- package/dist/{commands-Bj9NbmsO.js → commands-DQBayPCV.js} +1 -1
- package/dist/{config-CneBZB84.js → config-BLL5JqeX.js} +118 -81
- package/dist/config-BLL5JqeX.js.map +1 -0
- package/dist/{context-Dgn48BqT.js → context-I6nq3uS7.js} +2 -3
- package/dist/{context-Dgn48BqT.js.map → context-I6nq3uS7.js.map} +1 -1
- package/dist/{contrasts-lhFsekco.js → contrasts-CjOU_pfD.js} +4 -5
- package/dist/{contrasts-lhFsekco.js.map → contrasts-CjOU_pfD.js.map} +1 -1
- package/dist/{core-tLN-z1Be.js → core-CxY40JAs.js} +2 -3
- package/dist/{core-tLN-z1Be.js.map → core-CxY40JAs.js.map} +1 -1
- package/dist/cost-CMzY9VXv.js +1 -0
- package/dist/{cost-ChPa8-Fp.js → cost-e4IppZpJ.js} +5 -6
- package/dist/{cost-ChPa8-Fp.js.map → cost-e4IppZpJ.js.map} +1 -1
- package/dist/cursor-catalog-BeBJTk6A.js +142 -0
- package/dist/cursor-catalog-BeBJTk6A.js.map +1 -0
- package/dist/{db-Cnh3jIv_.js → db-DPiWC7Br.js} +22 -24
- package/dist/db-DPiWC7Br.js.map +1 -0
- package/dist/db-DysoTexX.js +2 -0
- package/dist/{debug-CgG3T071.js → debug-B5jDImcW.js} +5 -6
- package/dist/{debug-CgG3T071.js.map → debug-B5jDImcW.js.map} +1 -1
- package/dist/debug-BnXV7JbX.js +1 -0
- package/dist/default-effort-B7c4V9Qd.js +1 -0
- package/dist/default-effort-ciIXMA7h.js +94 -0
- package/dist/default-effort-ciIXMA7h.js.map +1 -0
- package/dist/{dist-CYrBwyHA.js → dist-CGwueB0T.js} +3 -4
- package/dist/{dist-CYrBwyHA.js.map → dist-CGwueB0T.js.map} +1 -1
- package/dist/{docker-Cqq03H72.js → docker-BUZOxh62.js} +20 -11
- package/dist/docker-BUZOxh62.js.map +1 -0
- package/dist/{docker-BihzJwbV.js → docker-Bcrc39Z8.js} +5 -6
- package/dist/{docker-BihzJwbV.js.map → docker-Bcrc39Z8.js.map} +1 -1
- package/dist/{docker-DCAtFMo9.js → docker-D83s_ADn.js} +1 -1
- package/dist/{doctor-Dl4vMeqt.js → doctor-CIhEAzbw.js} +26 -27
- package/dist/{doctor-Dl4vMeqt.js.map → doctor-CIhEAzbw.js.map} +1 -1
- package/dist/{env-CARj5Axq.js → env-T86lAqUD.js} +3 -4
- package/dist/{env-CARj5Axq.js.map → env-T86lAqUD.js.map} +1 -1
- package/dist/{environment-Cf5HMjib.js → environment-Ccytlpgf.js} +10 -13
- package/dist/environment-Ccytlpgf.js.map +1 -0
- package/dist/{errors-CHebohjc.js → errors-BQV__TzH.js} +9 -5
- package/dist/errors-BQV__TzH.js.map +1 -0
- package/dist/{errors-B6BSMs03.js → errors-ttMTnIYX.js} +2 -3
- package/dist/{errors-B6BSMs03.js.map → errors-ttMTnIYX.js.map} +1 -1
- package/dist/{eval-source-CAE4y2Sr.js → eval-source-D4xqc0iO.js} +8 -9
- package/dist/{eval-source-CAE4y2Sr.js.map → eval-source-D4xqc0iO.js.map} +1 -1
- package/dist/{evaluate-D35T51AC.d.ts → evaluate-BEGAQQuo.d.ts} +2 -4
- package/dist/{evaluate-CUtgyr43.js → evaluate-DNzyRQFe.js} +118 -89
- package/dist/evaluate-DNzyRQFe.js.map +1 -0
- package/dist/{files-CNiJ_fZf.js → files-3ZqUBbwg.js} +6 -7
- package/dist/{files-CNiJ_fZf.js.map → files-3ZqUBbwg.js.map} +1 -1
- package/dist/{files-BSqA2sZE.js → files-B2U5RezT.js} +7 -8
- package/dist/{files-BSqA2sZE.js.map → files-B2U5RezT.js.map} +1 -1
- package/dist/{files-Bk1mC3AN.js → files-BGlHnC46.js} +3 -4
- package/dist/{files-Bk1mC3AN.js.map → files-BGlHnC46.js.map} +1 -1
- package/dist/files-Bw2NQecs.js +1 -0
- package/dist/files-COW5JBJM.js +1 -0
- package/dist/files-Cp1jp0Nq.js +1 -0
- package/dist/{files-BRViE7m-.js → files-D_CBK9Zo.js} +17 -18
- package/dist/files-D_CBK9Zo.js.map +1 -0
- package/dist/{files-D1y8Pg4Z.js → files-p-uTdOjA.js} +4 -5
- package/dist/{files-D1y8Pg4Z.js.map → files-p-uTdOjA.js.map} +1 -1
- package/dist/{gate-BC6SdQZs.js → gate-BoDKeuJu.js} +4 -6
- package/dist/{gate-BC6SdQZs.js.map → gate-BoDKeuJu.js.map} +1 -1
- package/dist/gateway.d.ts +1 -3
- package/dist/gateway.js +1 -2
- package/dist/{git-UNxT_3z0.js → git-s-qf-vww.js} +4 -5
- package/dist/{git-UNxT_3z0.js.map → git-s-qf-vww.js.map} +1 -1
- package/dist/{harness-3NVyTSGK.js → harness-Ch2P81sx.js} +49 -16
- package/dist/harness-Ch2P81sx.js.map +1 -0
- package/dist/harness.d.ts +1 -3
- package/dist/harness.js +1 -1
- package/dist/{ids-Bp6ut7ZD.js → ids-DuK6xl4w.js} +2 -3
- package/dist/{ids-Bp6ut7ZD.js.map → ids-DuK6xl4w.js.map} +1 -1
- package/dist/{images-8A_LzVfw.js → images-h1VBM7TP.js} +9 -10
- package/dist/{images-8A_LzVfw.js.map → images-h1VBM7TP.js.map} +1 -1
- package/dist/index.d.ts +11 -13
- package/dist/index.js +5 -5
- package/dist/init-DJ3WFUt8.js +1 -0
- package/dist/{init-sFBgODhu.js → init-Z1cRnE3E.js} +8 -9
- package/dist/{init-sFBgODhu.js.map → init-Z1cRnE3E.js.map} +1 -1
- package/dist/{invocations-C9ejlOYl.js → invocations-z6hUD15n.js} +6 -8
- package/dist/{invocations-C9ejlOYl.js.map → invocations-z6hUD15n.js.map} +1 -1
- package/dist/{judge-Bg4v4e_K.js → judge-ByEUlzOM.js} +10 -15
- package/dist/judge-ByEUlzOM.js.map +1 -0
- package/dist/judge.d.ts +1 -5
- package/dist/judge.js +2 -3
- package/dist/{live-DeMvnDD5.js → live-D22qY6S_.js} +4 -5
- package/dist/{live-DeMvnDD5.js.map → live-D22qY6S_.js.map} +1 -1
- package/dist/live-DUpitP_9.js +1 -0
- package/dist/live-DpfJfc33.js +120 -0
- package/dist/live-DpfJfc33.js.map +1 -0
- package/dist/{matchers-BKPuaXlj.js → matchers-B4ZtlJyP.js} +33 -5
- package/dist/matchers-B4ZtlJyP.js.map +1 -0
- package/dist/{model-2rBZrdZA.js → model-BL_10Neu.js} +8 -10
- package/dist/model-BL_10Neu.js.map +1 -0
- package/dist/{model-Bz7-j71_.js → model-Bn7ndUQZ.js} +11 -15
- package/dist/model-Bn7ndUQZ.js.map +1 -0
- package/dist/{model-3MJtpho8.js → model-CFSjV-HJ.js} +6 -7
- package/dist/{model-3MJtpho8.js.map → model-CFSjV-HJ.js.map} +1 -1
- package/dist/model-CR3-qY-z.js +1 -0
- package/dist/{model-DkPwypN4.js → model-D1TBKeZj.js} +1 -1
- package/dist/{node-fP-H2uxn.js → node-CwP1G178.js} +11 -13
- package/dist/node-CwP1G178.js.map +1 -0
- package/dist/{object-BkXGPKWY.js → object-CeQ8dgUC.js} +2 -3
- package/dist/{object-BkXGPKWY.js.map → object-CeQ8dgUC.js.map} +1 -1
- package/dist/{opencode-providers-Bz0l7OZI.js → opencode-providers-yNKyXD49.js} +80 -20
- package/dist/opencode-providers-yNKyXD49.js.map +1 -0
- package/dist/{openrouter-models-Bo8PZ4Ew.js → openrouter-models-C8nzPKG2.js} +3 -4
- package/dist/{openrouter-models-Bo8PZ4Ew.js.map → openrouter-models-C8nzPKG2.js.map} +1 -1
- package/dist/{patch-CquGAGX5.js → patch-C7yBztZj.js} +2 -3
- package/dist/{patch-CquGAGX5.js.map → patch-C7yBztZj.js.map} +1 -1
- package/dist/{patch-6T9XAipm.js → patch-LFunkZ-L.js} +5 -6
- package/dist/{patch-6T9XAipm.js.map → patch-LFunkZ-L.js.map} +1 -1
- package/dist/{paths-BmR-7tx3.js → paths-Dwmi7L4X.js} +2 -3
- package/dist/{paths-BmR-7tx3.js.map → paths-Dwmi7L4X.js.map} +1 -1
- package/dist/{plan-Dw2BjafJ.js → plan-BrfiMcHh.js} +66 -12
- package/dist/plan-BrfiMcHh.js.map +1 -0
- package/dist/{plugins-ngQEJBjJ.js → plugins-CW-sQMg3.js} +8 -10
- package/dist/plugins-CW-sQMg3.js.map +1 -0
- package/dist/{preflight-CDRpog9o.js → preflight-BnFpd3Kv.js} +4 -5
- package/dist/{preflight-CDRpog9o.js.map → preflight-BnFpd3Kv.js.map} +1 -1
- package/dist/{preflight-BYXkweNN.js → preflight-CH-7KyhS.js} +1 -1
- package/dist/{prepare-PkARUGoF.js → prepare-0cA77eFD.js} +10 -11
- package/dist/{prepare-PkARUGoF.js.map → prepare-0cA77eFD.js.map} +1 -1
- package/dist/{prepare-B3btUlT6.js → prepare-CRyEwb5q.js} +2 -2
- package/dist/{prices-QaS-kjSQ.js → prices-C4BRuSe8.js} +5 -6
- package/dist/{prices-QaS-kjSQ.js.map → prices-C4BRuSe8.js.map} +1 -1
- package/dist/{provenance-7Hzg88f4.js → provenance-CKIG_1LB.js} +6 -7
- package/dist/{provenance-7Hzg88f4.js.map → provenance-CKIG_1LB.js.map} +1 -1
- package/dist/provider-DCdTHQpk.js +1 -0
- package/dist/{provider-CmAYTQQs.js → provider-DZd8t_XW.js} +3 -4
- package/dist/{provider-CmAYTQQs.js.map → provider-DZd8t_XW.js.map} +1 -1
- package/dist/{providers-DDNMbv1T.js → providers-CCl8ElNW.js} +1 -1
- package/dist/{providers-PKoJtGK0.js → providers-Dt6BEN_1.js} +8 -9
- package/dist/{providers-PKoJtGK0.js.map → providers-Dt6BEN_1.js.map} +1 -1
- package/dist/{query-nBXrgEwl.js → query-BV-zcw2t.js} +5 -6
- package/dist/{query-nBXrgEwl.js.map → query-BV-zcw2t.js.map} +1 -1
- package/dist/query-DSSMxkmw.js +1 -0
- package/dist/{questions-CkAwePPR.js → questions-AyLUMHYX.js} +14 -13
- package/dist/{questions-CkAwePPR.js.map → questions-AyLUMHYX.js.map} +1 -1
- package/dist/{record-DZBCjuwm.js → record-J45gJise.js} +13 -6
- package/dist/record-J45gJise.js.map +1 -0
- package/dist/{redact-DVIaahLv.js → redact-BEEGc37e.js} +8 -7
- package/dist/{redact-DVIaahLv.js.map → redact-BEEGc37e.js.map} +1 -1
- package/dist/{remarks-eBcrIZRu.js → remarks-0EXMh6AD.js} +2 -3
- package/dist/{remarks-eBcrIZRu.js.map → remarks-0EXMh6AD.js.map} +1 -1
- package/dist/remarks-BL18Id09.js +1 -0
- package/dist/{remarks-IFFe_oty.js → remarks-CgO1nIoG.js} +4 -6
- package/dist/{remarks-IFFe_oty.js.map → remarks-CgO1nIoG.js.map} +1 -1
- package/dist/{remarks-Bt0zwiQB.js → remarks-UIZTSIhd.js} +16 -17
- package/dist/remarks-UIZTSIhd.js.map +1 -0
- package/dist/{remote-DfJYiP-R.js → remote-BA9-hhgB.js} +10 -13
- package/dist/remote-BA9-hhgB.js.map +1 -0
- package/dist/resolve-C56s8_EF.js +1 -0
- package/dist/{resolve-D_cLa4tW.js → resolve-FHjE-dN3.js} +11 -12
- package/dist/{resolve-D_cLa4tW.js.map → resolve-FHjE-dN3.js.map} +1 -1
- package/dist/{results-DrKJuIQF.js → results-CFE6Nwb0.js} +8 -9
- package/dist/{results-DrKJuIQF.js.map → results-CFE6Nwb0.js.map} +1 -1
- package/dist/{resume-BmIM52MJ.js → resume-iaTxWRew.js} +8 -10
- package/dist/resume-iaTxWRew.js.map +1 -0
- package/dist/{root-Dlff-zp8.js → root-Co0jhOT5.js} +2 -3
- package/dist/{root-Dlff-zp8.js.map → root-Co0jhOT5.js.map} +1 -1
- package/dist/{runtime-CZVpkr04.js → runtime-SEkGS1Rx.js} +8 -10
- package/dist/{runtime-CZVpkr04.js.map → runtime-SEkGS1Rx.js.map} +1 -1
- package/dist/{sandbox-Cmi6IQML.js → sandbox-CAsXlGro.js} +4 -5
- package/dist/{sandbox-Cmi6IQML.js.map → sandbox-CAsXlGro.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -5
- package/dist/sandbox.js +1 -1
- package/dist/server-BYfqpMD0.js +1 -0
- package/dist/{server-DS8BGhGU.js → server-C9_kV1IP.js} +17 -126
- package/dist/server-C9_kV1IP.js.map +1 -0
- package/dist/{shared-B_lvGqWz.js → shared-ogqPJ0uo.js} +4 -6
- package/dist/shared-ogqPJ0uo.js.map +1 -0
- package/dist/{source-DcXv4de-.js → source-BszY6yAC.js} +3 -4
- package/dist/{source-DcXv4de-.js.map → source-BszY6yAC.js.map} +1 -1
- package/dist/source-C4Q5psjD.js +1 -0
- package/dist/{src-BO8AYqJ5.js → src-8fy4nSWU.js} +67 -13
- package/dist/src-8fy4nSWU.js.map +1 -0
- package/dist/{src-DSZyg0Uq.js → src-B-L8O1Ld.js} +38 -9
- package/dist/src-B-L8O1Ld.js.map +1 -0
- package/dist/src-BBmCEE-M.js +3 -0
- package/dist/{src-1lpTocbd.js → src-BWcgyARO.js} +32 -45
- package/dist/src-BWcgyARO.js.map +1 -0
- package/dist/src-BYpY_vXK.js +2 -0
- package/dist/{src-A8HLtO0a.js → src-B_igzM1O.js} +3 -5
- package/dist/src-B_igzM1O.js.map +1 -0
- package/dist/{src-gn7YlnRg.js → src-CWrKJO92.js} +4 -7
- package/dist/src-CWrKJO92.js.map +1 -0
- package/dist/src-CZANB4pb.js +1 -0
- package/dist/src-CnrLy_r_.js +2 -0
- package/dist/{src-8yTF251P.js → src-DMiwtprP.js} +65 -15
- package/dist/src-DMiwtprP.js.map +1 -0
- package/dist/src-DVGVWOAB.js +2 -0
- package/dist/{src-C0ySiej5.js → src-DaYQ1O9Y.js} +94 -68
- package/dist/src-DaYQ1O9Y.js.map +1 -0
- package/dist/src-DoxUxkU3.js +2 -0
- package/dist/src-DqIQQTu1.js +2 -0
- package/dist/{src-BkiComGh.js → src-Dt6BWDcJ.js} +2 -2
- package/dist/{src-BYLM3hEz.js → src-HTG2SoNz.js} +89 -38
- package/dist/src-HTG2SoNz.js.map +1 -0
- package/dist/{src-BYDOXw_Y.js → src-uluei5nU.js} +40 -17
- package/dist/src-uluei5nU.js.map +1 -0
- package/dist/{src-Cp-2Hdib.js → src-umdk2alV.js} +2 -2
- package/dist/{state-C_kdhp5v.js → state-DfcikRK3.js} +26 -35
- package/dist/state-DfcikRK3.js.map +1 -0
- package/dist/state-haxTOljN.js +1 -0
- package/dist/{stats-hfpuYsAM.js → stats-D-Gqwqt0.js} +2 -3
- package/dist/{stats-hfpuYsAM.js.map → stats-D-Gqwqt0.js.map} +1 -1
- package/dist/{suite-DieLXvCT.js → suite-BVcBo_ua.js} +12 -8
- package/dist/suite-BVcBo_ua.js.map +1 -0
- package/dist/suite-CmeXyn66.js +2 -0
- package/dist/{summary-CSzyBVsF.js → summary-BmooCwQv.js} +29 -15
- package/dist/{summary-CSzyBVsF.js.map → summary-BmooCwQv.js.map} +1 -1
- package/dist/{tarball-CHcahyNK.js → tarball-CESxtK4X.js} +2 -3
- package/dist/{tarball-CHcahyNK.js.map → tarball-CESxtK4X.js.map} +1 -1
- package/dist/testing.d.ts +49 -11
- package/dist/testing.js +5 -6
- package/dist/{text-DTigcA64.js → text-D--Q5KsA.js} +2 -3
- package/dist/{text-DTigcA64.js.map → text-D--Q5KsA.js.map} +1 -1
- package/dist/{tool-target-CKWkd6QI.js → tool-target-DS5d72eH.js} +3 -4
- package/dist/{tool-target-CKWkd6QI.js.map → tool-target-DS5d72eH.js.map} +1 -1
- package/dist/{transcript-BJrKfgRW.js → transcript-C_IpYMRk.js} +8 -8
- package/dist/transcript-C_IpYMRk.js.map +1 -0
- package/dist/{types-DVvW66ci.d.ts → types-BlM-aN9_.d.ts} +168 -33
- package/dist/{types-C5vI3g1C.js → types-C4OkYhZg.js} +2 -3
- package/dist/types-C4OkYhZg.js.map +1 -0
- package/dist/{usage-BNY1ABrU.js → usage-DJiDnh2D.js} +2 -3
- package/dist/{usage-BNY1ABrU.js.map → usage-DJiDnh2D.js.map} +1 -1
- package/dist/{vertex-DypYjx71.js → vertex-6uSpZ5L8.js} +8 -8
- package/dist/vertex-6uSpZ5L8.js.map +1 -0
- package/dist/vertex-BaAz44YN.js +1 -0
- package/dist/view/index.d.ts +21 -32
- package/dist/view/index.js +12 -11
- package/dist/views-CBAsDkCB.js +2 -0
- package/dist/{views-B9STsLB4.js → views-DPng-2-z.js} +7 -8
- package/dist/{views-B9STsLB4.js.map → views-DPng-2-z.js.map} +1 -1
- package/dist/{worker-entry-C5bLCv6s.js → worker-entry-lt5otEb5.js} +12 -17
- package/dist/worker-entry-lt5otEb5.js.map +1 -0
- package/dist/{workspace-CBuI9Uds.js → workspace-1O8x4oDc.js} +8 -9
- package/dist/{workspace-CBuI9Uds.js.map → workspace-1O8x4oDc.js.map} +1 -1
- package/dist/workspace-It7dvpbg.js +1 -0
- package/package.json +1 -1
- package/skills/eval-rubric/SKILL.md +1 -1
- package/dist/adapter-CRKYPkRt.js.map +0 -1
- package/dist/agent-BgH4-kFc.js.map +0 -1
- package/dist/app-files-BMVBr079.js +0 -11
- package/dist/app-files-BMVBr079.js.map +0 -1
- package/dist/assemble-BVbI6NW-.js.map +0 -1
- package/dist/assemble-SiHa4__F.js +0 -1
- package/dist/attempt-host-CIxjs2Be.js.map +0 -1
- package/dist/budget-DBUrH4jX.js.map +0 -1
- package/dist/capture-iQ6kYJV0.js.map +0 -1
- package/dist/capture-px7Ey95_.js +0 -1
- package/dist/cli-1YQU-66F.js.map +0 -1
- package/dist/colors-QV9dFVhN.js.map +0 -1
- package/dist/command-8_Cg5v6b.js.map +0 -1
- package/dist/command-BlYCAqu-.js.map +0 -1
- package/dist/command-Cr1zjRvl.js.map +0 -1
- package/dist/commands-BvTNZ2-b.js.map +0 -1
- package/dist/config-CneBZB84.js.map +0 -1
- package/dist/cost-C5RBTTJr.js +0 -1
- package/dist/db-BnXyamuj.js +0 -2
- package/dist/db-Cnh3jIv_.js.map +0 -1
- package/dist/debug-CAx3qcaP.js +0 -1
- package/dist/docker-Cqq03H72.js.map +0 -1
- package/dist/environment-Cf5HMjib.js.map +0 -1
- package/dist/errors-CHebohjc.js.map +0 -1
- package/dist/evaluate-CUtgyr43.js.map +0 -1
- package/dist/files-BEnUJW1e.js +0 -1
- package/dist/files-BRViE7m-.js.map +0 -1
- package/dist/files-C3tQtD26.js +0 -1
- package/dist/files-Dts7EY3H.js +0 -1
- package/dist/harness-3NVyTSGK.js.map +0 -1
- package/dist/init-okoALIah.js +0 -1
- package/dist/judge-Bg4v4e_K.js.map +0 -1
- package/dist/matchers-BKPuaXlj.js.map +0 -1
- package/dist/model-2rBZrdZA.js.map +0 -1
- package/dist/model-Bz7-j71_.js.map +0 -1
- package/dist/model-bREXVXwT.js +0 -1
- package/dist/node-fP-H2uxn.js.map +0 -1
- package/dist/opencode-providers-Bz0l7OZI.js.map +0 -1
- package/dist/plan-Dw2BjafJ.js.map +0 -1
- package/dist/plugins-ngQEJBjJ.js.map +0 -1
- package/dist/provider-DehR4CMN.js +0 -1
- package/dist/query-Bt6vLJ3m.js +0 -1
- package/dist/record-DZBCjuwm.js.map +0 -1
- package/dist/remarks-Bp6AwPgb.js +0 -1
- package/dist/remarks-Bt0zwiQB.js.map +0 -1
- package/dist/remote-DfJYiP-R.js.map +0 -1
- package/dist/resolve-CT4XZIfV.js +0 -1
- package/dist/resume-BmIM52MJ.js.map +0 -1
- package/dist/server-CKJOnIB0.js +0 -1
- package/dist/server-DS8BGhGU.js.map +0 -1
- package/dist/shared-B_lvGqWz.js.map +0 -1
- package/dist/source-CgUqpumI.js +0 -1
- package/dist/src-1lpTocbd.js.map +0 -1
- package/dist/src-8yTF251P.js.map +0 -1
- package/dist/src-A8HLtO0a.js.map +0 -1
- package/dist/src-BO8AYqJ5.js.map +0 -1
- package/dist/src-BTqJbvj6.js +0 -2
- package/dist/src-BYDOXw_Y.js.map +0 -1
- package/dist/src-BYLM3hEz.js.map +0 -1
- package/dist/src-BvgeZ4tS.js +0 -2
- package/dist/src-C0ySiej5.js.map +0 -1
- package/dist/src-CgJrz5p0.js +0 -3
- package/dist/src-DMo2eCOJ.js +0 -2
- package/dist/src-DSZyg0Uq.js.map +0 -1
- package/dist/src-Dk2Na0b-.js +0 -2
- package/dist/src-OESRtvOi.js +0 -2
- package/dist/src-gn7YlnRg.js.map +0 -1
- package/dist/src-h5uDlk3m.js +0 -1
- package/dist/state-C97udoh3.js +0 -1
- package/dist/state-C_kdhp5v.js.map +0 -1
- package/dist/suite-ByT1lsdD.js +0 -2
- package/dist/suite-DieLXvCT.js.map +0 -1
- package/dist/transcript-BJrKfgRW.js.map +0 -1
- package/dist/types-C5vI3g1C.js.map +0 -1
- package/dist/vertex-Bv7vhMgs.js +0 -1
- package/dist/vertex-DypYjx71.js.map +0 -1
- package/dist/views-Ca-rTixp.js +0 -2
- package/dist/worker-entry-C5bLCv6s.js.map +0 -1
- package/dist/workspace-8uHtBSve.js +0 -1
|
@@ -1,40 +1,43 @@
|
|
|
1
|
-
import { t as isObjectLike } from "./object-
|
|
2
|
-
import { r as withLease } from "./client-
|
|
3
|
-
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-
|
|
4
|
-
import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-
|
|
5
|
-
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-
|
|
6
|
-
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-
|
|
7
|
-
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-
|
|
8
|
-
import { $ as
|
|
9
|
-
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-
|
|
10
|
-
import { v as vertexSettings } from "./vertex-
|
|
11
|
-
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-
|
|
12
|
-
import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-
|
|
13
|
-
import { t as inCI } from "./env-
|
|
14
|
-
import {
|
|
15
|
-
import { t as InfraError } from "./types-
|
|
16
|
-
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-
|
|
17
|
-
import { c as formatBytes } from "./debug-
|
|
18
|
-
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-
|
|
19
|
-
import { t as attemptDebugDir } from "./files-
|
|
20
|
-
import { r as openRouterFallbacks } from "./openrouter-models-
|
|
21
|
-
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-
|
|
22
|
-
import { t as contextTaker } from "./context-
|
|
23
|
-
import { a as skillFiles, i as resolveEnvironment } from "./resolve-
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
import { i as
|
|
1
|
+
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
|
+
import { r as withLease } from "./client-CF-DOjOU.js";
|
|
3
|
+
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
|
|
4
|
+
import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-ogqPJ0uo.js";
|
|
5
|
+
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-Ccytlpgf.js";
|
|
6
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-CW-sQMg3.js";
|
|
7
|
+
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-DuK6xl4w.js";
|
|
8
|
+
import { $ as allowedHosts, A as evalPlan, At as refusedMcp, Bt as judgeProblem, Ct as brokeringFor, D as cascade, E as brokeredIn, Et as expectedBlockedFor, F as matchesTags, Ft as NO_JUDGE_INPUT, G as sandboxSetBy, Gt as decideState, H as runsOf, I as plainHeaderError, J as unbrokeredError, K as secretVarsIn, Kt as filePartsOf, L as plainHeaders, Lt as inputsOn, M as harnessId, Mt as writePluginFiles, Ot as keyRejected, P as majorityOf, Pt as INPUT_KEYS, U as sandboxId, Ut as artifactRefsOf, V as refuseIllegalCompare, W as sandboxProvider, Wt as badJudgeArtifacts, X as resolveStateOptions, Y as mergeJudge, _ as fellBackToDirect, at as unenforceable, b as resolveModelGateway, it as isDirect, j as expand, jt as refusedPlugins, k as checkOptions, kt as layoutOf, nt as egressLabel, q as unbrokered, qt as resolveJudgeArtifacts, st as createFakeSandbox, v as gatewayScope, vt as AGENTS, w as resolveGateway, wt as claudeMcpServers, xt as agentMcp } from "./config-BLL5JqeX.js";
|
|
9
|
+
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-ByEUlzOM.js";
|
|
10
|
+
import { v as vertexSettings } from "./vertex-6uSpZ5L8.js";
|
|
11
|
+
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-BEEGc37e.js";
|
|
12
|
+
import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-D--Q5KsA.js";
|
|
13
|
+
import { t as inCI } from "./env-T86lAqUD.js";
|
|
14
|
+
import { dt as OPAQUE_OUTPUT, mt as shellQuote, t as MCP_TIMER } from "./src-DaYQ1O9Y.js";
|
|
15
|
+
import { t as InfraError } from "./types-C4OkYhZg.js";
|
|
16
|
+
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-SEkGS1Rx.js";
|
|
17
|
+
import { c as formatBytes } from "./debug-B5jDImcW.js";
|
|
18
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CtKGvaQR.js";
|
|
19
|
+
import { t as attemptDebugDir } from "./files-p-uTdOjA.js";
|
|
20
|
+
import { r as openRouterFallbacks } from "./openrouter-models-C8nzPKG2.js";
|
|
21
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-DF8cHsJh.js";
|
|
22
|
+
import { t as contextTaker } from "./context-I6nq3uS7.js";
|
|
23
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-FHjE-dN3.js";
|
|
24
|
+
import { n as isDockerSandbox } from "./docker-BUZOxh62.js";
|
|
25
|
+
import { t as PrepareError } from "./provider-DZd8t_XW.js";
|
|
26
|
+
import { i as preparedWorkspace } from "./workspace-1O8x4oDc.js";
|
|
27
|
+
import { t as patchPath } from "./patch-C7yBztZj.js";
|
|
28
|
+
import { n as judgeDiff } from "./patch-LFunkZ-L.js";
|
|
29
|
+
import { t as transcriptForJudge } from "./transcript-C_IpYMRk.js";
|
|
30
|
+
import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CESxtK4X.js";
|
|
31
|
+
import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-D4xqc0iO.js";
|
|
32
|
+
import { i as withDefaultEffort } from "./default-effort-ciIXMA7h.js";
|
|
31
33
|
import { dirname, join } from "node:path";
|
|
32
34
|
import { parse, stringify } from "yaml";
|
|
33
35
|
import { z } from "zod";
|
|
36
|
+
import { randomBytes } from "node:crypto";
|
|
34
37
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
35
38
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
36
39
|
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
37
|
-
|
|
40
|
+
// src/attempt/assertions.ts
|
|
38
41
|
const MESSAGE_LIMIT = 1024;
|
|
39
42
|
const LABEL_LIMIT = 60;
|
|
40
43
|
/** Masked first, so a secret the limit would cut through is masked whole. */
|
|
@@ -102,8 +105,7 @@ function checkFailure(raw) {
|
|
|
102
105
|
caught.stack = error.stack?.replace(error.message, caught.message);
|
|
103
106
|
return caught;
|
|
104
107
|
}
|
|
105
|
-
|
|
106
|
-
//#region src/attempt/debug.ts
|
|
108
|
+
// src/attempt/debug.ts
|
|
107
109
|
/** Which config and profile the attempt runs under, and where each resolved option came from. */
|
|
108
110
|
function tracePlan(debug, spec, ctx) {
|
|
109
111
|
const config = ctx.configFile ?? "none (no config file found)";
|
|
@@ -188,8 +190,7 @@ function traceEnvironment(debug, spec, env) {
|
|
|
188
190
|
}))
|
|
189
191
|
}, "resolved environment");
|
|
190
192
|
}
|
|
191
|
-
|
|
192
|
-
//#region src/harness/fake.ts
|
|
193
|
+
// src/harness/fake.ts
|
|
193
194
|
const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n") ? 1 : 0) : 0;
|
|
194
195
|
/**
|
|
195
196
|
* Writes canned files and returns a canned message on every turn, after running its `script`'s
|
|
@@ -206,6 +207,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
206
207
|
const hangs = descriptor.startHangs;
|
|
207
208
|
if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
|
|
208
209
|
const layout = layoutOf(descriptor);
|
|
210
|
+
const brokered = descriptor.brokers ? await brokerKey(descriptor, sandbox) : void 0;
|
|
209
211
|
const mcp = agentMcp(layout, all, plugins);
|
|
210
212
|
for (const path of await writePluginFiles(sandbox, layout, plugins)) debugLog("env", `plugin settings → ${path} by the fake agent`);
|
|
211
213
|
const dir = `~/${layout.harnessSkillDir}`;
|
|
@@ -287,7 +289,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
287
289
|
tool: tool.tool,
|
|
288
290
|
input: tool.input,
|
|
289
291
|
output: tool.output,
|
|
290
|
-
ok: tool.ok
|
|
292
|
+
ok: tool.ok === void 0 ? true : tool.ok,
|
|
291
293
|
durationMs: 0
|
|
292
294
|
});
|
|
293
295
|
if (flaked) {
|
|
@@ -304,7 +306,10 @@ const createFakeHarness = (descriptor) => {
|
|
|
304
306
|
for (const step of steps) if ("exec" in step) {
|
|
305
307
|
const time = at();
|
|
306
308
|
debugLog("harness", `tool shell called: $ ${step.exec}`);
|
|
307
|
-
const r = await sandbox.exec(step.exec, {
|
|
309
|
+
const r = await sandbox.exec(step.exec, {
|
|
310
|
+
signal,
|
|
311
|
+
...brokered && { env: brokered }
|
|
312
|
+
});
|
|
308
313
|
debugLog("harness", `tool shell returned exit ${r.exitCode} in ${Date.now() - startedAt - time.atMs}ms`);
|
|
309
314
|
transcript.push({
|
|
310
315
|
...time,
|
|
@@ -341,6 +346,18 @@ const createFakeHarness = (descriptor) => {
|
|
|
341
346
|
message: step.error
|
|
342
347
|
});
|
|
343
348
|
return done("", "error");
|
|
349
|
+
} else if ("refuseKey" in step) {
|
|
350
|
+
const message = `${keyRejected(step.refuseKey)}. 401 Unauthorized (the fake agent's)`;
|
|
351
|
+
debugLog("harness", `turn failed: ${message}`);
|
|
352
|
+
transcript.push({
|
|
353
|
+
...at(),
|
|
354
|
+
type: "error",
|
|
355
|
+
message
|
|
356
|
+
});
|
|
357
|
+
return {
|
|
358
|
+
...done("", "error"),
|
|
359
|
+
rejectedKey: message
|
|
360
|
+
};
|
|
344
361
|
} else scripted = step.reply;
|
|
345
362
|
if (descriptor.delayMs) try {
|
|
346
363
|
await setTimeout$1(descriptor.delayMs, void 0, descriptor.ignoreAbort ? {} : { signal });
|
|
@@ -362,6 +379,30 @@ const createFakeHarness = (descriptor) => {
|
|
|
362
379
|
} };
|
|
363
380
|
};
|
|
364
381
|
/**
|
|
382
|
+
* Has the egress gateway swap the host's key for a placeholder on the hosts of the agent's
|
|
383
|
+
* endpoints, as a real adapter does, and returns the variable its `exec` steps get it in.
|
|
384
|
+
*/
|
|
385
|
+
async function brokerKey(descriptor, sandbox) {
|
|
386
|
+
const { credential, endpoints } = AGENTS[descriptor.layout ?? "cursor"];
|
|
387
|
+
const key = process.env[credential];
|
|
388
|
+
if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
|
|
389
|
+
const network = isDockerSandbox(sandbox) ? sandbox.network : void 0;
|
|
390
|
+
if (!network?.setRequestTransformations) throw new InfraError("fakeHarness({ brokers: true }) needs docker(): this sandbox can't broker");
|
|
391
|
+
const placeholder = `reeval_fake_${randomBytes(24).toString("base64url")}`;
|
|
392
|
+
await network.setRequestTransformations([...new Set(endpoints.map((e) => e.host))].map((host) => ({
|
|
393
|
+
match: {
|
|
394
|
+
host,
|
|
395
|
+
headers: [{
|
|
396
|
+
key: { exact: "authorization" },
|
|
397
|
+
value: { exact: `Bearer ${placeholder}` }
|
|
398
|
+
}]
|
|
399
|
+
},
|
|
400
|
+
transform: { headers: { authorization: `Bearer ${key}` } }
|
|
401
|
+
})));
|
|
402
|
+
debugLog("harness", `${credential} is a placeholder for the fake agent, brokered on its endpoints`);
|
|
403
|
+
return { [credential]: placeholder };
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
365
406
|
* Starts a stdio MCP server in the sandbox, as an agent would, and calls one tool with no
|
|
366
407
|
* arguments: initialize, then the call.
|
|
367
408
|
*/
|
|
@@ -436,17 +477,15 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
436
477
|
ok: !reply.result?.isError
|
|
437
478
|
};
|
|
438
479
|
}
|
|
439
|
-
|
|
440
|
-
//#region src/harness/index.ts
|
|
480
|
+
// src/harness/index.ts
|
|
441
481
|
async function createHarness(descriptor, vertex) {
|
|
442
482
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
443
|
-
const { createAgentHarness } = await import("./agent-
|
|
483
|
+
const { createAgentHarness } = await import("./agent-CE-qdvmh.js");
|
|
444
484
|
return createAgentHarness(descriptor, vertex);
|
|
445
485
|
}
|
|
446
486
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
447
487
|
const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
|
|
448
|
-
|
|
449
|
-
//#region src/attempt/environment.ts
|
|
488
|
+
// src/attempt/environment.ts
|
|
450
489
|
/**
|
|
451
490
|
* Resolves the attempt's environment (files, skills, secrets and their placeholders)
|
|
452
491
|
* and finds the image a Docker sandbox starts from, which preflight prepared. Runs on the test's
|
|
@@ -470,8 +509,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
470
509
|
if (debug) traceEnvironment(debug, spec, env);
|
|
471
510
|
record.image = null;
|
|
472
511
|
if (spec.sandbox.kind === "docker") {
|
|
473
|
-
const { preparedFor } = await import("./preflight-
|
|
474
|
-
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-
|
|
512
|
+
const { preparedFor } = await import("./preflight-CH-7KyhS.js");
|
|
513
|
+
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-CCl8ElNW.js");
|
|
475
514
|
signal.throwIfAborted();
|
|
476
515
|
const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
|
|
477
516
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -497,8 +536,7 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
497
536
|
debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
|
|
498
537
|
} else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
|
|
499
538
|
}
|
|
500
|
-
|
|
501
|
-
//#region src/judge/size.ts
|
|
539
|
+
// src/judge/size.ts
|
|
502
540
|
/** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
|
|
503
541
|
const CHARS_PER_TOKEN = 4;
|
|
504
542
|
const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
|
|
@@ -553,8 +591,7 @@ function stateHint(state, question) {
|
|
|
553
591
|
* no model's limit, so this follows every provider error.
|
|
554
592
|
*/
|
|
555
593
|
const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
|
|
556
|
-
|
|
557
|
-
//#region src/judge/state.ts
|
|
594
|
+
// src/judge/state.ts
|
|
558
595
|
function turnState(t) {
|
|
559
596
|
return {
|
|
560
597
|
...t.task !== void 0 ? { task: t.task } : {},
|
|
@@ -575,8 +612,7 @@ function stateOf(d) {
|
|
|
575
612
|
...transcript !== void 0 ? { transcript } : {}
|
|
576
613
|
};
|
|
577
614
|
}
|
|
578
|
-
|
|
579
|
-
//#region src/attempt/judge.ts
|
|
615
|
+
// src/attempt/judge.ts
|
|
580
616
|
const noJudgeModel = (configFile) => `c.judge() has no model. Set judge: { model: 'typesafe-ai/jev-1.14.0' } in ${configFile}, on the group or eval, or pass { model } to this call.`;
|
|
581
617
|
/** A turn this attempt sent becomes its index; any other value is graded as the output. */
|
|
582
618
|
function judgeOn(turns, on) {
|
|
@@ -806,14 +842,13 @@ function judgeCaller(a, readFile) {
|
|
|
806
842
|
return Object.defineProperty(await ask(order, question, call), SUBJECT, { value: question });
|
|
807
843
|
};
|
|
808
844
|
}
|
|
809
|
-
|
|
810
|
-
//#region src/sandbox/index.ts
|
|
845
|
+
// src/sandbox/index.ts
|
|
811
846
|
async function createSandbox(descriptor, options) {
|
|
812
847
|
try {
|
|
813
848
|
switch (descriptor.kind) {
|
|
814
849
|
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
815
850
|
case "docker": {
|
|
816
|
-
const { createDockerSandbox } = await import("./docker-
|
|
851
|
+
const { createDockerSandbox } = await import("./docker-D83s_ADn.js");
|
|
817
852
|
return await createDockerSandbox(descriptor, options);
|
|
818
853
|
}
|
|
819
854
|
}
|
|
@@ -823,8 +858,7 @@ async function createSandbox(descriptor, options) {
|
|
|
823
858
|
throw new InfraError(`Failed to create ${kind} sandbox: ${error.message}`);
|
|
824
859
|
}
|
|
825
860
|
}
|
|
826
|
-
|
|
827
|
-
//#region src/attempt/record.ts
|
|
861
|
+
// src/attempt/record.ts
|
|
828
862
|
function classify(error, signal) {
|
|
829
863
|
const message = error instanceof Error ? error.message : String(error);
|
|
830
864
|
if (signal.aborted) return isTimeout(signal.reason) ? {
|
|
@@ -914,7 +948,7 @@ async function recordOom(a, sandbox) {
|
|
|
914
948
|
/**
|
|
915
949
|
* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
|
|
916
950
|
* Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
|
|
917
|
-
* summarized in one `-v` line.
|
|
951
|
+
* summarized in one `-v` line. Names that didn't resolve are `unresolved`, never `blocked`.
|
|
918
952
|
*/
|
|
919
953
|
function recordEgress(a, sandbox) {
|
|
920
954
|
const traffic = sandbox?.traffic?.();
|
|
@@ -929,7 +963,8 @@ function recordEgress(a, sandbox) {
|
|
|
929
963
|
...traffic && isDirect(a.spec.egress.policy) ? { partial: true } : {},
|
|
930
964
|
contacted: traffic?.contacted ?? [],
|
|
931
965
|
blocked: refused.filter((h) => !isExpected(h)),
|
|
932
|
-
expectedBlocked
|
|
966
|
+
expectedBlocked,
|
|
967
|
+
unresolved: traffic?.unresolved ?? []
|
|
933
968
|
};
|
|
934
969
|
if (expected && expectedBlocked.length) {
|
|
935
970
|
const hosts = expectedBlocked.map((h) => `${h.host}:${h.port} ×${h.connections}`).join(", ");
|
|
@@ -962,7 +997,7 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
962
997
|
const { record, spec } = a;
|
|
963
998
|
const recordPath = join(stagingDir, "evals", recordFileName(attemptKey(id, spec.cell.key, spec.n)));
|
|
964
999
|
await mkdir(join(stagingDir, "evals"), { recursive: true });
|
|
965
|
-
await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
|
|
1000
|
+
await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), new Map([...a.artifactBlobs, ...a.stepBlobs]), a.capturePages);
|
|
966
1001
|
const tmp = `${recordPath}.${process.pid}.tmp`;
|
|
967
1002
|
await writeFile(tmp, stagedRecordJson(record));
|
|
968
1003
|
await rename(tmp, recordPath);
|
|
@@ -1000,8 +1035,7 @@ function turnErrors(transcripts) {
|
|
|
1000
1035
|
return found?.type === "error" ? [`turn ${i + 1} error: ${clip(redactSecrets(found.message.trim()), 500)}`] : [];
|
|
1001
1036
|
});
|
|
1002
1037
|
}
|
|
1003
|
-
|
|
1004
|
-
//#region src/harness/unpack.ts
|
|
1038
|
+
// src/harness/unpack.ts
|
|
1005
1039
|
const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
1006
1040
|
const FRONTMATTER = /^\uFEFF?---\r?\n/;
|
|
1007
1041
|
const FRONTMATTER_BLOCK = /^\uFEFF?---\r?\n(?:---|([\s\S]*?)\r?\n---)[ \t]*(?:\r?\n|$)/;
|
|
@@ -1112,8 +1146,7 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
|
|
|
1112
1146
|
});
|
|
1113
1147
|
return out;
|
|
1114
1148
|
}
|
|
1115
|
-
|
|
1116
|
-
//#region src/sandbox/phases.ts
|
|
1149
|
+
// src/sandbox/phases.ts
|
|
1117
1150
|
const strip = ({ phase: _, ...sub }) => sub;
|
|
1118
1151
|
/**
|
|
1119
1152
|
* Which brokered secrets the gateway swaps in when. `to: 'checks'` secrets are active only while
|
|
@@ -1141,8 +1174,7 @@ function brokerPhases(box, subs) {
|
|
|
1141
1174
|
}
|
|
1142
1175
|
};
|
|
1143
1176
|
}
|
|
1144
|
-
|
|
1145
|
-
//#region src/attempt/mcp-startup.ts
|
|
1177
|
+
// src/attempt/mcp-startup.ts
|
|
1146
1178
|
/**
|
|
1147
1179
|
* The timer and its log beside the workspace: `/reeval/bin/mcp-timer.cjs` in Docker. `.cjs`, since
|
|
1148
1180
|
* a sandbox's directories may be in a `"type": "module"` package.
|
|
@@ -1205,8 +1237,7 @@ const mcpStartupOf = (entries) => entries.map((e) => ({
|
|
|
1205
1237
|
}));
|
|
1206
1238
|
/** The first server in `entries` that never answered `initialize`. */
|
|
1207
1239
|
const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
|
|
1208
|
-
|
|
1209
|
-
//#region src/attempt/state.ts
|
|
1240
|
+
// src/attempt/state.ts
|
|
1210
1241
|
/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
|
|
1211
1242
|
const SETUP_BUDGET_MS = 12e5;
|
|
1212
1243
|
function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
@@ -1265,6 +1296,8 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
|
1265
1296
|
captureNames: /* @__PURE__ */ new Set(),
|
|
1266
1297
|
backgrounds: [],
|
|
1267
1298
|
artifactBlobs: /* @__PURE__ */ new Map(),
|
|
1299
|
+
capturePages: /* @__PURE__ */ new Map(),
|
|
1300
|
+
stepBlobs: /* @__PURE__ */ new Map(),
|
|
1268
1301
|
snapshotDir
|
|
1269
1302
|
};
|
|
1270
1303
|
return a;
|
|
@@ -1301,8 +1334,7 @@ function evalTimer(ms) {
|
|
|
1301
1334
|
clear: () => clearTimeout(timer)
|
|
1302
1335
|
};
|
|
1303
1336
|
}
|
|
1304
|
-
|
|
1305
|
-
//#region src/attempt/session.ts
|
|
1337
|
+
// src/attempt/session.ts
|
|
1306
1338
|
const tailOf = (text) => text.trim().split("\n").slice(-10).join("\n");
|
|
1307
1339
|
/** Setup steps in order, each with its own timeout; a failure is an infra error. */
|
|
1308
1340
|
async function runSetup(a, box, env) {
|
|
@@ -1589,7 +1621,11 @@ async function startSession(a, runId) {
|
|
|
1589
1621
|
brokering: brokeringFor(spec.harness, spec.vertex),
|
|
1590
1622
|
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
|
|
1591
1623
|
substitutions: env.substitutions,
|
|
1592
|
-
egress: spec.egress
|
|
1624
|
+
egress: spec.egress,
|
|
1625
|
+
onWarning: (text) => a.report({
|
|
1626
|
+
kind: "warning",
|
|
1627
|
+
text
|
|
1628
|
+
})
|
|
1593
1629
|
});
|
|
1594
1630
|
a.sandbox = box;
|
|
1595
1631
|
if (spec.sandbox.kind === "fake") debugLog("container", `fake sandbox at ${box.workdir}`);
|
|
@@ -1657,8 +1693,7 @@ async function startSession(a, runId) {
|
|
|
1657
1693
|
});
|
|
1658
1694
|
return [box, session];
|
|
1659
1695
|
}
|
|
1660
|
-
|
|
1661
|
-
//#region src/attempt/snapshot.ts
|
|
1696
|
+
// src/attempt/snapshot.ts
|
|
1662
1697
|
/** The most a snapshot directory's files may add up to; a larger one isn't taken. */
|
|
1663
1698
|
const SNAPSHOT_DIR_LIMIT = 50 * 2 ** 20;
|
|
1664
1699
|
/** The most a compressed snapshot may be. */
|
|
@@ -1888,8 +1923,7 @@ async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SN
|
|
|
1888
1923
|
failed(error instanceof Error ? error.message : String(error));
|
|
1889
1924
|
}
|
|
1890
1925
|
}
|
|
1891
|
-
|
|
1892
|
-
//#region src/attempt/turns.ts
|
|
1926
|
+
// src/attempt/turns.ts
|
|
1893
1927
|
/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
|
|
1894
1928
|
const TURN_STOP_GRACE_MS = 3e3;
|
|
1895
1929
|
/**
|
|
@@ -2020,7 +2054,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2020
2054
|
durationMs: Date.now() - sendStarted
|
|
2021
2055
|
};
|
|
2022
2056
|
});
|
|
2023
|
-
const { raw, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2057
|
+
const { raw, rejectedKey, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2024
2058
|
if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
|
|
2025
2059
|
throw error;
|
|
2026
2060
|
}).finally(() => {
|
|
@@ -2063,8 +2097,9 @@ function turnSender(a, takeContext, checks) {
|
|
|
2063
2097
|
});
|
|
2064
2098
|
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
2065
2099
|
for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
|
|
2066
|
-
|
|
2067
|
-
|
|
2100
|
+
const infra = timedOut || result.status !== "aborted" && rejectedKey;
|
|
2101
|
+
if (infra) {
|
|
2102
|
+
const error = new InfraError(infra);
|
|
2068
2103
|
a.startFailure ??= { error };
|
|
2069
2104
|
throw error;
|
|
2070
2105
|
}
|
|
@@ -2085,8 +2120,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2085
2120
|
return turn;
|
|
2086
2121
|
};
|
|
2087
2122
|
}
|
|
2088
|
-
|
|
2089
|
-
//#region src/attempt/index.ts
|
|
2123
|
+
// src/attempt/index.ts
|
|
2090
2124
|
const TRACE_BUFFER_DEFAULT = 8 * 2 ** 20;
|
|
2091
2125
|
async function runAttempt(spec, fn, handle) {
|
|
2092
2126
|
const ctx = getRunContext();
|
|
@@ -2205,8 +2239,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2205
2239
|
if (oom) throw oom;
|
|
2206
2240
|
if (failure) throw failure.error;
|
|
2207
2241
|
}
|
|
2208
|
-
|
|
2209
|
-
//#region src/best-of.ts
|
|
2242
|
+
// src/best-of.ts
|
|
2210
2243
|
/** The side an attempt that settled counts on: its record's outcome, else the engine's state. */
|
|
2211
2244
|
function sideOf({ state, meta }) {
|
|
2212
2245
|
const outcome = meta?.outcome ?? (state === "passed" ? "pass" : state === "failed" ? "fail" : null);
|
|
@@ -2229,8 +2262,7 @@ function bestOfCell(bestOf) {
|
|
|
2229
2262
|
settled: (settled) => settle[n - 1]?.(sideOf(settled))
|
|
2230
2263
|
});
|
|
2231
2264
|
}
|
|
2232
|
-
|
|
2233
|
-
//#region src/contract/declaration.ts
|
|
2265
|
+
// src/contract/declaration.ts
|
|
2234
2266
|
/** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
|
|
2235
2267
|
const evalOptionsDataSchema = z.record(z.string(), z.unknown());
|
|
2236
2268
|
const declaredModeSchema = z.enum([
|
|
@@ -2286,8 +2318,7 @@ const NODE_SCHEMAS = {
|
|
|
2286
2318
|
* tree's `children`. Options are left to the cascade.
|
|
2287
2319
|
*/
|
|
2288
2320
|
const declarationProblem = (children) => dataProblem(declarationTreeSchema, { children }, "The declaration tree", (node) => Object.keys(NODE_SCHEMAS[String(node.type)]?.shape ?? {}));
|
|
2289
|
-
|
|
2290
|
-
//#region src/plan-eval.ts
|
|
2321
|
+
// src/plan-eval.ts
|
|
2291
2322
|
/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
|
|
2292
2323
|
const GRACE_MS = 5e3;
|
|
2293
2324
|
function configLevel(config) {
|
|
@@ -2381,7 +2412,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2381
2412
|
...harness,
|
|
2382
2413
|
gateway: route.name
|
|
2383
2414
|
};
|
|
2384
|
-
});
|
|
2415
|
+
}).map((harness) => withDefaultEffort(harness, ctx.cursorEfforts));
|
|
2385
2416
|
const plannedAttempts = resolvedEval.skip ? [] : expand({
|
|
2386
2417
|
...resolved,
|
|
2387
2418
|
harness: routed
|
|
@@ -2603,8 +2634,7 @@ function attemptLimits(planned, concurrency) {
|
|
|
2603
2634
|
if (harness.maxConcurrent) limits[`harness:${harness.kind}`] = harness.maxConcurrent;
|
|
2604
2635
|
return limits;
|
|
2605
2636
|
}
|
|
2606
|
-
|
|
2607
|
-
//#region src/evaluate.ts
|
|
2637
|
+
// src/evaluate.ts
|
|
2608
2638
|
let groupStack = [];
|
|
2609
2639
|
/** Each eval file's declarations as `evaluate` and `group` were called, and each eval's body by id. */
|
|
2610
2640
|
const trees = /* @__PURE__ */ new Map();
|
|
@@ -2893,7 +2923,6 @@ function optionsFrom(written, where, base) {
|
|
|
2893
2923
|
}
|
|
2894
2924
|
};
|
|
2895
2925
|
}
|
|
2896
|
-
//#endregion
|
|
2897
2926
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
2898
2927
|
|
|
2899
|
-
//# sourceMappingURL=evaluate-
|
|
2928
|
+
//# sourceMappingURL=evaluate-DNzyRQFe.js.map
|