reeval 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-BIlJWXff.js → adapter-CptK032x.js} +2 -2
- package/dist/{adapter-BIlJWXff.js.map → adapter-CptK032x.js.map} +1 -1
- package/dist/{agent-CE-qdvmh.js → agent-B63OzUX9.js} +124 -11
- package/dist/agent-B63OzUX9.js.map +1 -0
- package/dist/{analysis-CTBb_C0Z.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-CTBb_C0Z.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-YMmDAHA8.js → app-files-sgqwfjtn.js} +4 -4
- package/dist/app-files-sgqwfjtn.js.map +1 -0
- package/dist/assemble-CxBHkavM.js +1 -0
- package/dist/{assemble-7RI7Nuhz.js → assemble-Me7Whp-y.js} +62 -196
- package/dist/assemble-Me7Whp-y.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
- package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
- package/dist/{attempt-host-CJM_wEA4.js → attempt-host-Cik0SoIn.js} +17 -9
- package/dist/{attempt-host-CJM_wEA4.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
- package/dist/bridge/index.mjs +9 -5
- package/dist/{capture-DF8cHsJh.js → capture-C3RmjPph.js} +392 -609
- package/dist/capture-C3RmjPph.js.map +1 -0
- package/dist/capture-YPwxp_iK.js +1 -0
- package/dist/{cli-Bm9ntSxz.js → cli-BBOlXKPG.js} +1078 -459
- package/dist/cli-BBOlXKPG.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
- package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
- package/dist/client-tZ2WBDxf.js.map +1 -0
- package/dist/{command-Dk6wggV-.js → command-CFQ9c1hk.js} +13 -13
- package/dist/{command-Dk6wggV-.js.map → command-CFQ9c1hk.js.map} +1 -1
- package/dist/{command-DPF8hnWS.js → command-DhiwsZl1.js} +9 -9
- package/dist/{command-DPF8hnWS.js.map → command-DhiwsZl1.js.map} +1 -1
- package/dist/{command-DDkz6jsD.js → command-fO0BFsTq.js} +105 -37
- package/dist/command-fO0BFsTq.js.map +1 -0
- package/dist/{command-B7MslY_w.js → command-qGS802uX.js} +17 -15
- package/dist/command-qGS802uX.js.map +1 -0
- package/dist/{commands-DHT-nhEJ.js → commands-CC60TOT5.js} +64 -18
- package/dist/commands-CC60TOT5.js.map +1 -0
- package/dist/{commands-DQBayPCV.js → commands-xYTMNTPv.js} +1 -1
- package/dist/{config-BLL5JqeX.js → config-C2UtS9vy.js} +745 -80
- package/dist/config-C2UtS9vy.js.map +1 -0
- package/dist/{context-I6nq3uS7.js → context-BajfiH0S.js} +3 -3
- package/dist/{context-I6nq3uS7.js.map → context-BajfiH0S.js.map} +1 -1
- package/dist/{contrasts-CjOU_pfD.js → contrasts-BjOg7wyr.js} +3 -3
- package/dist/{contrasts-CjOU_pfD.js.map → contrasts-BjOg7wyr.js.map} +1 -1
- package/dist/cost-BJU0MAji.js +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
- package/dist/{cursor-catalog-BeBJTk6A.js → cursor-catalog-BTK_apL-.js} +2 -2
- package/dist/{cursor-catalog-BeBJTk6A.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
- package/dist/{db-DPiWC7Br.js → db-C5SZ0510.js} +79 -18
- package/dist/db-C5SZ0510.js.map +1 -0
- package/dist/db-puzLzX4k.js +2 -0
- package/dist/{default-effort-ciIXMA7h.js → default-effort-BVmx7r7L.js} +8 -5
- package/dist/default-effort-BVmx7r7L.js.map +1 -0
- package/dist/{default-effort-B7c4V9Qd.js → default-effort-FEidNAIi.js} +1 -1
- package/dist/{dist-CGwueB0T.js → dist-ByYk_wAs.js} +58 -290
- package/dist/dist-ByYk_wAs.js.map +1 -0
- package/dist/{docker-Bcrc39Z8.js → docker-B2F8-PGC.js} +4 -4
- package/dist/{docker-Bcrc39Z8.js.map → docker-B2F8-PGC.js.map} +1 -1
- package/dist/{docker-D83s_ADn.js → docker-BlSopHpZ.js} +1 -1
- package/dist/{docker-BUZOxh62.js → docker-Dto8ROCq.js} +5 -5
- package/dist/{docker-BUZOxh62.js.map → docker-Dto8ROCq.js.map} +1 -1
- package/dist/{doctor-CIhEAzbw.js → doctor-BcvfKpBX.js} +20 -20
- package/dist/{doctor-CIhEAzbw.js.map → doctor-BcvfKpBX.js.map} +1 -1
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Ccytlpgf.js → environment-CC1RV21n.js} +3 -3
- package/dist/{environment-Ccytlpgf.js.map → environment-CC1RV21n.js.map} +1 -1
- package/dist/{eval-source-D4xqc0iO.js → eval-source-CzxdgFBW.js} +18 -10
- package/dist/eval-source-CzxdgFBW.js.map +1 -0
- package/dist/evaluate-DToSB2bt.d.ts +101 -0
- package/dist/{evaluate-DNzyRQFe.js → evaluate-NTNJ8T9H.js} +135 -54
- package/dist/evaluate-NTNJ8T9H.js.map +1 -0
- package/dist/{files-D_CBK9Zo.js → files-CZXrkPja.js} +6 -6
- package/dist/{files-D_CBK9Zo.js.map → files-CZXrkPja.js.map} +1 -1
- package/dist/files-DFtuRQJx.js +1 -0
- package/dist/{files-3ZqUBbwg.js → files-DsXULUi5.js} +4 -4
- package/dist/{files-3ZqUBbwg.js.map → files-DsXULUi5.js.map} +1 -1
- package/dist/{files-B2U5RezT.js → files-aAzoGNCB.js} +5 -5
- package/dist/{files-B2U5RezT.js.map → files-aAzoGNCB.js.map} +1 -1
- package/dist/files-oOYNR7ZH.js +1 -0
- package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
- package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-s-qf-vww.js → git-C1MB5bjr.js} +3 -3
- package/dist/{git-s-qf-vww.js.map → git-C1MB5bjr.js.map} +1 -1
- package/dist/harness-DGsRZDSo.js +105 -0
- package/dist/harness-DGsRZDSo.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
- package/dist/harness-DYSC4zMh.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{images-h1VBM7TP.js → images-CNayXCp2.js} +8 -8
- package/dist/{images-h1VBM7TP.js.map → images-CNayXCp2.js.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +5 -5
- package/dist/init-8o6W3_fn.js +1 -0
- package/dist/{init-Z1cRnE3E.js → init-D5HtmyMI.js} +5 -5
- package/dist/{init-Z1cRnE3E.js.map → init-D5HtmyMI.js.map} +1 -1
- package/dist/ink-list-Ca-5dunO.js +184 -0
- package/dist/ink-list-Ca-5dunO.js.map +1 -0
- package/dist/{invocations-z6hUD15n.js → invocations-DrGXF9dT.js} +4 -4
- package/dist/{invocations-z6hUD15n.js.map → invocations-DrGXF9dT.js.map} +1 -1
- package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
- package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-m-mb0zv8.js +246 -0
- package/dist/judgments-m-mb0zv8.js.map +1 -0
- package/dist/list-CM_YSJXq.js +374 -0
- package/dist/list-CM_YSJXq.js.map +1 -0
- package/dist/{live-D22qY6S_.js → live-B71o2TkX.js} +22 -4
- package/dist/live-B71o2TkX.js.map +1 -0
- package/dist/live-DQc8jr6o.js +1 -0
- package/dist/{live-DpfJfc33.js → live-uRURFDMZ.js} +7 -7
- package/dist/{live-DpfJfc33.js.map → live-uRURFDMZ.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
- package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
- package/dist/{model-BL_10Neu.js → model-BxBWFOZn.js} +44 -20
- package/dist/{model-BL_10Neu.js.map → model-BxBWFOZn.js.map} +1 -1
- package/dist/{model-D1TBKeZj.js → model-CXVVB4RK.js} +1 -1
- package/dist/{model-Bn7ndUQZ.js → model-Ce9l9_rF.js} +27 -254
- package/dist/model-Ce9l9_rF.js.map +1 -0
- package/dist/model-Cxzo4cjw.js +1 -0
- package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
- package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
- package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
- package/dist/{patch-LFunkZ-L.js → patch-ug5HpMOj.js} +3 -3
- package/dist/{patch-LFunkZ-L.js.map → patch-ug5HpMOj.js.map} +1 -1
- package/dist/{plan-BrfiMcHh.js → plan-DqnzfDH1.js} +15 -14
- package/dist/plan-DqnzfDH1.js.map +1 -0
- package/dist/{plugins-CW-sQMg3.js → plugins-B2KQ0y3P.js} +4 -4
- package/dist/{plugins-CW-sQMg3.js.map → plugins-B2KQ0y3P.js.map} +1 -1
- package/dist/{preflight-CH-7KyhS.js → preflight-DVXEvzFU.js} +1 -1
- package/dist/{preflight-BnFpd3Kv.js → preflight-DyzmvK7i.js} +3 -3
- package/dist/{preflight-BnFpd3Kv.js.map → preflight-DyzmvK7i.js.map} +1 -1
- package/dist/{prepare-0cA77eFD.js → prepare-21WMQZyt.js} +7 -7
- package/dist/{prepare-0cA77eFD.js.map → prepare-21WMQZyt.js.map} +1 -1
- package/dist/{prepare-CRyEwb5q.js → prepare-BcSRa7-3.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
- package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
- package/dist/{provenance-CKIG_1LB.js → provenance-iRxjp4Pu.js} +4 -4
- package/dist/{provenance-CKIG_1LB.js.map → provenance-iRxjp4Pu.js.map} +1 -1
- package/dist/{providers-CCl8ElNW.js → providers-D0ob_tra.js} +1 -1
- package/dist/{providers-Dt6BEN_1.js → providers-vXMXuwn-.js} +5 -5
- package/dist/{providers-Dt6BEN_1.js.map → providers-vXMXuwn-.js.map} +1 -1
- package/dist/{query-BV-zcw2t.js → query-BhaKQ07P.js} +3 -3
- package/dist/{query-BV-zcw2t.js.map → query-BhaKQ07P.js.map} +1 -1
- package/dist/query-CvgUsvep.js +1 -0
- package/dist/{questions-AyLUMHYX.js → questions-BzdLdV3C.js} +39 -9
- package/dist/questions-BzdLdV3C.js.map +1 -0
- package/dist/{record-J45gJise.js → record-DhzwkSXc.js} +22 -2
- package/dist/{record-J45gJise.js.map → record-DhzwkSXc.js.map} +1 -1
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-UIZTSIhd.js → remarks-CauU408t.js} +231 -25
- package/dist/remarks-CauU408t.js.map +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/{remarks-CgO1nIoG.js → remarks-NMgPd30U.js} +4 -3
- package/dist/{remarks-CgO1nIoG.js.map → remarks-NMgPd30U.js.map} +1 -1
- package/dist/{remote-BA9-hhgB.js → remote-Dwy3fyo3.js} +5 -5
- package/dist/{remote-BA9-hhgB.js.map → remote-Dwy3fyo3.js.map} +1 -1
- package/dist/{resolve-FHjE-dN3.js → resolve-B1zMnPwK.js} +7 -7
- package/dist/{resolve-FHjE-dN3.js.map → resolve-B1zMnPwK.js.map} +1 -1
- package/dist/resolve-BIIeVT12.js +1 -0
- package/dist/{results-CFE6Nwb0.js → results-CfC0id_u.js} +7 -7
- package/dist/{results-CFE6Nwb0.js.map → results-CfC0id_u.js.map} +1 -1
- package/dist/{resume-iaTxWRew.js → resume-Dkhrpjow.js} +123 -8
- package/dist/resume-Dkhrpjow.js.map +1 -0
- package/dist/{runtime-SEkGS1Rx.js → runtime-DcvQcr36.js} +2 -2
- package/dist/{runtime-SEkGS1Rx.js.map → runtime-DcvQcr36.js.map} +1 -1
- package/dist/{sandbox-CAsXlGro.js → sandbox-8zCrtuCf.js} +2 -2
- package/dist/{sandbox-CAsXlGro.js.map → sandbox-8zCrtuCf.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-0tuj8VSa.js +1 -0
- package/dist/{server-C9_kV1IP.js → server-Dplq8g0r.js} +8 -8
- package/dist/{server-C9_kV1IP.js.map → server-Dplq8g0r.js.map} +1 -1
- package/dist/{source-BszY6yAC.js → source-NWHqn5-o.js} +2 -2
- package/dist/{source-BszY6yAC.js.map → source-NWHqn5-o.js.map} +1 -1
- package/dist/source-llwLewOU.js +1 -0
- package/dist/{src-Dt6BWDcJ.js → src-B9oE_mNS.js} +1 -1
- package/dist/{src-DaYQ1O9Y.js → src-BQ6BiXoc.js} +86 -31
- package/dist/{src-DaYQ1O9Y.js.map → src-BQ6BiXoc.js.map} +1 -1
- package/dist/src-BQXdf-Gw.js +1448 -0
- package/dist/src-BQXdf-Gw.js.map +1 -0
- package/dist/{src-DVGVWOAB.js → src-BXnv0ViU.js} +1 -1
- package/dist/src-BiS8YNN_.js +3 -0
- package/dist/{src-CZANB4pb.js → src-BkH4GOxu.js} +1 -1
- package/dist/{src-8fy4nSWU.js → src-BvM0jWuZ.js} +28 -2
- package/dist/{src-8fy4nSWU.js.map → src-BvM0jWuZ.js.map} +1 -1
- package/dist/src-C-tH5ebd.js +2 -0
- package/dist/{src-umdk2alV.js → src-CUpoVVb8.js} +2 -2
- package/dist/src-Cg1bFs9g.js +2 -0
- package/dist/{src-CWrKJO92.js → src-D5t4OiRt.js} +2 -2
- package/dist/{src-CWrKJO92.js.map → src-D5t4OiRt.js.map} +1 -1
- package/dist/{src-DoxUxkU3.js → src-DV_UorZv.js} +1 -1
- package/dist/{src-DMiwtprP.js → src-KlDOGD0B.js} +4 -4
- package/dist/{src-DMiwtprP.js.map → src-KlDOGD0B.js.map} +1 -1
- package/dist/{src-BYpY_vXK.js → src-XxUpr2sa.js} +1 -1
- package/dist/{state-DfcikRK3.js → state-DDCGUhva.js} +16 -16
- package/dist/{state-DfcikRK3.js.map → state-DDCGUhva.js.map} +1 -1
- package/dist/state-DZbAO95a.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-M8_7xio7.js +1294 -0
- package/dist/suite-M8_7xio7.js.map +1 -0
- package/dist/suite-u2KJnGpY.js +2 -0
- package/dist/{summary-BmooCwQv.js → summary-Uzc9osfV.js} +47 -13
- package/dist/summary-Uzc9osfV.js.map +1 -0
- package/dist/testing.d.ts +51 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
- package/dist/{types-BlM-aN9_.d.ts → types-DGvexkcp.d.ts} +116 -66
- package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
- package/dist/vertex-BLQMayc1.js.map +1 -0
- package/dist/vertex-h_peaZTN.js +1 -0
- package/dist/view/index.d.ts +143 -34
- package/dist/view/index.js +9 -8
- package/dist/{views-DPng-2-z.js → views-BISrsIjw.js} +6 -6
- package/dist/{views-DPng-2-z.js.map → views-BISrsIjw.js.map} +1 -1
- package/dist/views-Bfab8fM9.js +2 -0
- package/dist/{worker-entry-lt5otEb5.js → worker-entry-DJD_nHkS.js} +5 -5
- package/dist/{worker-entry-lt5otEb5.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
- package/dist/workspace-B8EOCuxl.js +1 -0
- package/dist/{workspace-1O8x4oDc.js → workspace-BO1VhUR9.js} +5 -5
- package/dist/{workspace-1O8x4oDc.js.map → workspace-BO1VhUR9.js.map} +1 -1
- package/docker/Dockerfile +4 -4
- package/package.json +14 -12
- package/skills/eval-rubric/SKILL.md +22 -3
- package/dist/agent-CE-qdvmh.js.map +0 -1
- package/dist/app-files-YMmDAHA8.js.map +0 -1
- package/dist/assemble-7RI7Nuhz.js.map +0 -1
- package/dist/assemble-CANqwtwD.js +0 -1
- package/dist/capture-DF8cHsJh.js.map +0 -1
- package/dist/capture-DagM1npp.js +0 -1
- package/dist/cli-Bm9ntSxz.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-B7MslY_w.js.map +0 -1
- package/dist/command-DDkz6jsD.js.map +0 -1
- package/dist/commands-DHT-nhEJ.js.map +0 -1
- package/dist/config-BLL5JqeX.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-DPiWC7Br.js.map +0 -1
- package/dist/db-DysoTexX.js +0 -2
- package/dist/default-effort-ciIXMA7h.js.map +0 -1
- package/dist/dist-CGwueB0T.js.map +0 -1
- package/dist/eval-source-D4xqc0iO.js.map +0 -1
- package/dist/evaluate-BEGAQQuo.d.ts +0 -43
- package/dist/evaluate-DNzyRQFe.js.map +0 -1
- package/dist/files-Bw2NQecs.js +0 -1
- package/dist/files-Cp1jp0Nq.js +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/init-DJ3WFUt8.js +0 -1
- package/dist/live-D22qY6S_.js.map +0 -1
- package/dist/live-DUpitP_9.js +0 -1
- package/dist/model-Bn7ndUQZ.js.map +0 -1
- package/dist/model-CR3-qY-z.js +0 -1
- package/dist/plan-BrfiMcHh.js.map +0 -1
- package/dist/query-DSSMxkmw.js +0 -1
- package/dist/questions-AyLUMHYX.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-UIZTSIhd.js.map +0 -1
- package/dist/resolve-C56s8_EF.js +0 -1
- package/dist/resume-iaTxWRew.js.map +0 -1
- package/dist/server-BYfqpMD0.js +0 -1
- package/dist/source-C4Q5psjD.js +0 -1
- package/dist/src-BBmCEE-M.js +0 -3
- package/dist/src-BWcgyARO.js +0 -640
- package/dist/src-BWcgyARO.js.map +0 -1
- package/dist/src-CnrLy_r_.js +0 -2
- package/dist/src-DqIQQTu1.js +0 -2
- package/dist/state-haxTOljN.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BVcBo_ua.js +0 -544
- package/dist/suite-BVcBo_ua.js.map +0 -1
- package/dist/suite-CmeXyn66.js +0 -2
- package/dist/summary-BmooCwQv.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CBAsDkCB.js +0 -2
- package/dist/workspace-It7dvpbg.js +0 -1
- /package/dist/bridge/{pnpm-lock.yaml → pnpm-lockfile.yaml} +0 -0
|
@@ -1,31 +1,32 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
2
2
|
import { h as parseDuration } from "./shared-ogqPJ0uo.js";
|
|
3
|
-
import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-
|
|
4
|
-
import { C as pluginBuildsLabel } from "./plugins-
|
|
3
|
+
import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-CC1RV21n.js";
|
|
4
|
+
import { C as pluginBuildsLabel } from "./plugins-B2KQ0y3P.js";
|
|
5
5
|
import { a as fromCwd, o as newRunId } from "./ids-DuK6xl4w.js";
|
|
6
|
-
import { M as harnessId, O as checkLimitHarness, R as plannedSandboxIds, U as sandboxId, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials,
|
|
7
|
-
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-
|
|
8
|
-
import { y as vertexSettingsIn } from "./vertex-
|
|
9
|
-
import {
|
|
10
|
-
import { a as plural, i as formatDuration, n as clip
|
|
11
|
-
import { n as loadEnvFiles, t as inCI } from "./env-
|
|
6
|
+
import { $ as directOnCloud, M as harnessId, O as checkLimitHarness, Q as ciRunner, R as plannedSandboxIds, U as sandboxId, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, it as resolveOutput, l as hostCapacity, m as resolveConcurrency, nt as isDirect, p as requireConfig, u as loadConfig, v as gatewayScope, z as plannedSandboxes } from "./config-C2UtS9vy.js";
|
|
7
|
+
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-DYOF69Oe.js";
|
|
8
|
+
import { y as vertexSettingsIn } from "./vertex-BLQMayc1.js";
|
|
9
|
+
import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
|
|
10
|
+
import { a as plural, i as formatDuration, n as clip } from "./text-B8-W3Cvn.js";
|
|
11
|
+
import { n as loadEnvFiles, t as inCI } from "./env-8x1O8I1M.js";
|
|
12
12
|
import { o as PACKAGE_VERSION } from "./root-Co0jhOT5.js";
|
|
13
|
-
import { W as LABEL$1, st as imageInfo } from "./src-
|
|
14
|
-
import {
|
|
15
|
-
import { r as openRouterFallbacks } from "./openrouter-models-C8nzPKG2.js";
|
|
13
|
+
import { W as LABEL$1, st as imageInfo } from "./src-BQ6BiXoc.js";
|
|
14
|
+
import { r as openRouterFallbacks } from "./openrouter-models-DNvLRYM7.js";
|
|
16
15
|
import { t as PrepareError } from "./provider-DZd8t_XW.js";
|
|
17
|
-
import {
|
|
18
|
-
import { u as suiteResult } from "./suite-
|
|
19
|
-
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-
|
|
16
|
+
import { _ as formatUsage, k as runOutcome } from "./model-Ce9l9_rF.js";
|
|
17
|
+
import { h as measuresLine, i as effectOf, r as countedOf, s as isNotGatedResult, u as suiteResult, v as observedLine } from "./suite-M8_7xio7.js";
|
|
18
|
+
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-DZ0bnP0_.js";
|
|
20
19
|
import { n as stripAnsi, t as colors } from "./colors-DcXBvAlU.js";
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
20
|
+
import { i as shownRemarks } from "./remarks-D9Lcd1fF.js";
|
|
21
|
+
import { _ as statsLine, c as checkLabel } from "./summary-Uzc9osfV.js";
|
|
22
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-iRxjp4Pu.js";
|
|
23
23
|
import { t as budgetMarker } from "./budget-BcxLYY5L.js";
|
|
24
|
-
import { a as
|
|
25
|
-
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-
|
|
26
|
-
import { n as imageHarness } from "./prepare-
|
|
27
|
-
import { i as createProviders, n as attemptImage } from "./providers-
|
|
28
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-
|
|
24
|
+
import { a as liveAttempts, d as savedFilters, f as savedInvocation, i as leftOutAttempts, l as resolveResume, m as writeLiveProgress, o as loadPrevious, p as writeCurrent, r as keptAttempts, s as prepareResume, t as completedKeysFor, u as resumeSettings } from "./resume-Dkhrpjow.js";
|
|
25
|
+
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-DyzmvK7i.js";
|
|
26
|
+
import { n as imageHarness } from "./prepare-21WMQZyt.js";
|
|
27
|
+
import { i as createProviders, n as attemptImage } from "./providers-vXMXuwn-.js";
|
|
28
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DrGXF9dT.js";
|
|
29
|
+
import { c as attemptAxes, d as failureLines, f as offset, g as tailLines, h as progressLine, l as debugLines, m as phaseOf, p as outcomeOf, r as listLines, s as progressText, t as LiveList, u as errorCheck } from "./list-CM_YSJXq.js";
|
|
29
30
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
30
31
|
import { dirname, join } from "node:path";
|
|
31
32
|
import { inspect } from "node:util";
|
|
@@ -218,204 +219,654 @@ async function preflight(uses, options) {
|
|
|
218
219
|
}
|
|
219
220
|
return { images };
|
|
220
221
|
}
|
|
221
|
-
// src/ui/
|
|
222
|
-
/**
|
|
223
|
-
const
|
|
222
|
+
// src/ui/header.ts
|
|
223
|
+
/** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
|
|
224
|
+
const GLYPH = 5;
|
|
225
|
+
/** `Est. Time: `, the longest key. */
|
|
226
|
+
const KEY = 11;
|
|
227
|
+
/** `Warning: `, the longer note label. */
|
|
228
|
+
const LABEL = 9;
|
|
224
229
|
/**
|
|
225
|
-
*
|
|
226
|
-
*
|
|
230
|
+
* `text` broken at spaces into lines of at most `width` columns, not counting color codes; a
|
|
231
|
+
* longer word keeps a line. A `·` stays with the word before it, so no line starts with one.
|
|
227
232
|
*/
|
|
228
|
-
function
|
|
229
|
-
const
|
|
230
|
-
const
|
|
231
|
-
|
|
232
|
-
const
|
|
233
|
-
|
|
233
|
+
function wrap(text, width) {
|
|
234
|
+
const words = [];
|
|
235
|
+
for (const word of text.split(/\s+/).filter(Boolean)) if (word === "·" && words.length) words[words.length - 1] += " ·";
|
|
236
|
+
else words.push(word);
|
|
237
|
+
const lines = [];
|
|
238
|
+
let line = "";
|
|
239
|
+
for (const word of words) if (line && stripAnsi(line).length + 1 + stripAnsi(word).length > width) {
|
|
240
|
+
lines.push(line);
|
|
241
|
+
line = word;
|
|
242
|
+
} else line = line ? `${line} ${word}` : word;
|
|
243
|
+
return line ? [...lines, line] : lines;
|
|
244
|
+
}
|
|
245
|
+
const indent = (n) => " ".repeat(n);
|
|
246
|
+
/** `REEVAL v0.16.1`, then the run's facts, one `➜ Key: value` line each. */
|
|
247
|
+
function headerLines(c, h, width) {
|
|
248
|
+
const name = `REEVAL v${h.version}`;
|
|
249
|
+
const title = c.bold(c.depth === 4 ? name : c.teal(name));
|
|
250
|
+
const valueWidth = width - GLYPH - KEY;
|
|
251
|
+
const row = (key, value) => {
|
|
252
|
+
const values = typeof value === "string" ? [value] : value;
|
|
253
|
+
const joined = values.join(", ");
|
|
254
|
+
return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
|
|
255
|
+
};
|
|
256
|
+
const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
|
|
257
|
+
const resumed = h.resumed ? c.dim(` (resumed, ${plural(h.resumed, "attempt")} done)`) : "";
|
|
258
|
+
const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
|
|
234
259
|
return [
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
...
|
|
260
|
+
h.dryRun ? `${title} ${c.dim("dry run")}` : title,
|
|
261
|
+
"",
|
|
262
|
+
...row("Run", `${h.runId}${resumed}`),
|
|
263
|
+
...h.config ? row("Config", h.config) : [],
|
|
264
|
+
...h.profile ? row("Profile", `${h.profile.name}${reason}`) : [],
|
|
265
|
+
...row(many("Harness", h.harnesses.length, "Harnesses"), h.harnesses.map((id) => harness(c, h, id))),
|
|
266
|
+
...row(many("Sandbox", h.sandboxes.length, "Sandboxes"), h.sandboxes),
|
|
267
|
+
...h.judges.length ? row(many("Judge", h.judges.length), h.judges) : [],
|
|
268
|
+
...row("Scope", scope(c, h)),
|
|
269
|
+
...row("Est. Time", estimate(c, h)),
|
|
270
|
+
...h.record && !h.dryRun ? row("Record", h.record) : []
|
|
238
271
|
];
|
|
239
272
|
}
|
|
240
|
-
/**
|
|
241
|
-
function
|
|
242
|
-
const
|
|
243
|
-
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
273
|
+
/** A harness id, with the effort it runs at when the id doesn't name it. */
|
|
274
|
+
function harness(c, h, id) {
|
|
275
|
+
const effort = h.efforts?.[id];
|
|
276
|
+
if (effort === void 0 || id.includes("[effort=")) return id;
|
|
277
|
+
return `${id} ${`(effort: ${effort})`.split(" ").map((w) => c.dim(w)).join(" ")}`;
|
|
278
|
+
}
|
|
279
|
+
function scope(c, h) {
|
|
280
|
+
const attempts = h.fewest < h.attempts ? `${h.fewest}–${h.attempts} attempts` : plural(h.attempts, "attempt");
|
|
281
|
+
const parts = [
|
|
282
|
+
plural(h.files, "eval file"),
|
|
283
|
+
plural(h.evals, "eval") + (h.skipped ? ` (${h.skipped} skipped)` : ""),
|
|
284
|
+
plural(h.cells, "cell"),
|
|
285
|
+
attempts
|
|
286
|
+
];
|
|
287
|
+
const caps = [h.budget.attempts !== void 0 ? `${h.budget.attempts} ${h.attempts <= h.budget.attempts ? c.green("✓") : c.red("✗")}` : "", h.budget.duration ?? ""].filter(Boolean);
|
|
288
|
+
return `${parts.join(" · ")}${caps.length ? ` (budget: ${caps.join(", ")})` : ""}`;
|
|
289
|
+
}
|
|
290
|
+
function estimate(c, h) {
|
|
291
|
+
const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
|
|
292
|
+
const parallel = Math.min(h.concurrency, h.attempts);
|
|
293
|
+
if (parallel <= 1) return serial;
|
|
294
|
+
return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency})`)}`} · ${serial}`;
|
|
295
|
+
}
|
|
296
|
+
/** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
|
|
297
|
+
function noteLines(c, kind, text, width) {
|
|
298
|
+
const [glyph, label, paint] = kind === "hint" ? [
|
|
299
|
+
"ℹ",
|
|
300
|
+
"Hint:",
|
|
301
|
+
c.blue
|
|
302
|
+
] : [
|
|
303
|
+
"⚠",
|
|
304
|
+
"Warning:",
|
|
305
|
+
kind === "root" ? c.red : c.orange
|
|
306
|
+
];
|
|
307
|
+
const basic = {
|
|
308
|
+
warning: c.yellow,
|
|
309
|
+
hint: c.teal,
|
|
310
|
+
root: c.red
|
|
311
|
+
}[kind];
|
|
312
|
+
const lead = c.depth === 4 && kind !== "root" ? `${basic(glyph)} ${c.bold(label)}` : c.bold(paint(`${glyph} ${label}`));
|
|
313
|
+
const at = 14;
|
|
314
|
+
return wrap(text, Math.max(20, width - at)).map((line, i) => i === 0 ? ` ${lead}${indent(LABEL - label.length)}${line}` : `${indent(at)}${line}`);
|
|
315
|
+
}
|
|
316
|
+
/** A step line, `◆ Preparing 1 sandbox (docker)...`, its text wrapped under itself. */
|
|
317
|
+
function stepLines(glyph, text, width) {
|
|
318
|
+
return wrap(text, Math.max(20, width - GLYPH)).map((line, i) => i === 0 ? ` ${glyph} ${line}` : `${indent(GLYPH)}${line}`);
|
|
319
|
+
}
|
|
320
|
+
/** A run-level notice (a prepared workspace, the Ctrl-C one) as a step line. */
|
|
321
|
+
function noticeLines(c, e, width) {
|
|
322
|
+
const glyph = {
|
|
323
|
+
info: c.dim("ℹ"),
|
|
324
|
+
warn: c.orange("⚠"),
|
|
325
|
+
error: c.red("✗")
|
|
326
|
+
}[e.level];
|
|
327
|
+
const lines = stepLines(glyph, e.text, width);
|
|
328
|
+
return e.level === "info" ? lines.map((l) => c.dim(l)) : lines;
|
|
329
|
+
}
|
|
330
|
+
/** Preflight: which sandboxes it prepares, each one ready, and each failure. */
|
|
331
|
+
function prepareStepLines(c, e, width, max) {
|
|
332
|
+
if (e.state === "start") {
|
|
333
|
+
const n = e.targets.length;
|
|
334
|
+
const names = e.targets.map((t) => t.label).join(", ");
|
|
335
|
+
return stepLines(c.teal("◆"), `Preparing ${plural(n, "sandbox", "sandboxes")} (${names})...`, width);
|
|
286
336
|
}
|
|
337
|
+
const { outcome } = e;
|
|
338
|
+
if (outcome.ok) {
|
|
339
|
+
const took = outcome.ms >= 1e3 ? ` ${c.dim(formatDuration(outcome.ms))}` : "";
|
|
340
|
+
return [` ${c.green("✓")} ${outcome.target.label} ready${took}`];
|
|
341
|
+
}
|
|
342
|
+
const who = outcome.target.setBy.join(", ");
|
|
343
|
+
return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
|
|
287
344
|
}
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
return
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
const
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
case "sandbox-ready": return "sandbox ready";
|
|
305
|
-
case "turn-start": return `turn ${p.turn}`;
|
|
306
|
-
case "turn-end": return `turn ${p.turn} done`;
|
|
307
|
-
case "check-start":
|
|
308
|
-
case "check-end": return "checks";
|
|
309
|
-
case "judge-start":
|
|
310
|
-
case "judge-end": return "judging";
|
|
311
|
-
case "cleanup": return "cleanup";
|
|
312
|
-
case "browser-download": return "browser download";
|
|
313
|
-
case "warning": return "running";
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
function capLines(text, max) {
|
|
317
|
-
const lines = text.replace(/\s+$/, "").split("\n");
|
|
318
|
-
if (lines.length <= max) return lines;
|
|
319
|
-
return [...lines.slice(0, max), `(+${lines.length - max} more lines)`];
|
|
320
|
-
}
|
|
321
|
-
/** An infra error's headline, then its last lines: the cause is at the end. */
|
|
322
|
-
function tailLines(text, max) {
|
|
323
|
-
const lines = text.replace(/\s+$/, "").split("\n");
|
|
324
|
-
if (lines.length <= max) return lines;
|
|
325
|
-
const rest = Math.max(1, max - 1);
|
|
326
|
-
return [
|
|
327
|
-
lines[0] ?? "",
|
|
328
|
-
`(${lines.length - 1 - rest} more lines)`,
|
|
329
|
-
...lines.slice(-rest)
|
|
345
|
+
/** The last line before the attempts' own: how their logs will print. */
|
|
346
|
+
function startLine(c, output) {
|
|
347
|
+
const text = output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
|
|
348
|
+
return ` ${c.teal("◆")} ${text}`;
|
|
349
|
+
}
|
|
350
|
+
/** What the start line becomes once the run ends, on a terminal that redraws it. */
|
|
351
|
+
function endLine(c, status) {
|
|
352
|
+
const text = status === "interrupted" ? "Canceled." : status === "budget" ? "Stopped at the budget." : "Finished.";
|
|
353
|
+
return ` ${c.teal("◆")} ${text}`;
|
|
354
|
+
}
|
|
355
|
+
/** The plan as the header block: its facts, `--dry-run`'s table, then each kind of note. */
|
|
356
|
+
function planLines(c, e, width) {
|
|
357
|
+
const sections = [
|
|
358
|
+
e.details,
|
|
359
|
+
[...e.warnings.flatMap((w) => noteLines(c, "warning", w, width)), ...(e.sandboxWarnings ?? []).flatMap((w) => noteLines(c, w.kind === "root" ? "root" : "warning", w.message, width))],
|
|
360
|
+
e.hints.flatMap((h) => noteLines(c, "hint", h, width))
|
|
330
361
|
];
|
|
362
|
+
const out = headerLines(c, e.header, width);
|
|
363
|
+
for (const lines of sections) if (lines.length) out.push("", ...lines);
|
|
364
|
+
out.push("");
|
|
365
|
+
return out;
|
|
366
|
+
}
|
|
367
|
+
// src/ui/status.ts
|
|
368
|
+
const EFFECT = {
|
|
369
|
+
better: "helped",
|
|
370
|
+
worse: "made it worse",
|
|
371
|
+
same: "same"
|
|
372
|
+
};
|
|
373
|
+
const ERROR_VERB = {
|
|
374
|
+
assertion: "failed",
|
|
375
|
+
timeout: "timed out",
|
|
376
|
+
aborted: "canceled",
|
|
377
|
+
infra: "hit an infrastructure error",
|
|
378
|
+
judge: "could not be judged",
|
|
379
|
+
turn: "ended a turn with an error"
|
|
380
|
+
};
|
|
381
|
+
const MESSAGE_LINES = 4;
|
|
382
|
+
const LINE_MAX = 160;
|
|
383
|
+
const times = (n) => n === 1 ? "once" : `${n} times`;
|
|
384
|
+
/** `cursor (composer-2.5)`, from the record's cell. */
|
|
385
|
+
function harnessLabel(record, cell) {
|
|
386
|
+
const model = record.evals.flatMap((e) => e.cells).find((c) => c.key === cell.key && c.harness === cell.harness)?.model;
|
|
387
|
+
return model && model !== cell.harness ? `${cell.harness} (${model})` : cell.harness;
|
|
388
|
+
}
|
|
389
|
+
/** The cells an eval's outcome reads: the gated ones, or every cell of a study. */
|
|
390
|
+
function deciding(cells) {
|
|
391
|
+
const gated = cells.filter((c) => c.gated);
|
|
392
|
+
return gated.length ? gated : cells;
|
|
393
|
+
}
|
|
394
|
+
const cellsOf = (e) => e.cells ?? (e.cases ?? []).flatMap((k) => k.cells);
|
|
395
|
+
/** Each part's label, adding the environment and sandbox only where the cells differ in them. */
|
|
396
|
+
function labeler(record, cells) {
|
|
397
|
+
const envs = new Set(cells.map((c) => c.environment)).size > 1;
|
|
398
|
+
const sandboxes = new Set(cells.map((c) => c.sandbox)).size > 1;
|
|
399
|
+
return (c) => [
|
|
400
|
+
harnessLabel(record, c),
|
|
401
|
+
envs ? `in ${c.environment}` : "",
|
|
402
|
+
sandboxes ? `on ${c.sandbox}` : ""
|
|
403
|
+
].filter(Boolean).join(" ");
|
|
404
|
+
}
|
|
405
|
+
function tallyBy(cells, label) {
|
|
406
|
+
const by = /* @__PURE__ */ new Map();
|
|
407
|
+
for (const c of cells) {
|
|
408
|
+
const key = label(c);
|
|
409
|
+
const t = by.get(key) ?? {
|
|
410
|
+
label: key,
|
|
411
|
+
passed: 0,
|
|
412
|
+
counted: 0
|
|
413
|
+
};
|
|
414
|
+
t.passed += c.passed;
|
|
415
|
+
t.counted += countedOf(c);
|
|
416
|
+
by.set(key, t);
|
|
417
|
+
}
|
|
418
|
+
return [...by.values()];
|
|
419
|
+
}
|
|
420
|
+
const of = (t) => `${t.passed} of ${t.counted}`;
|
|
421
|
+
/** The outcome sentence, per the rules at the top. */
|
|
422
|
+
function outcomeSentence(tallies) {
|
|
423
|
+
const counted = tallies.filter((t) => t.counted > 0);
|
|
424
|
+
if (!counted.length) return "No run finished.";
|
|
425
|
+
const [one] = counted;
|
|
426
|
+
if (counted.length === 1 && one) {
|
|
427
|
+
const once = one.counted === 1;
|
|
428
|
+
if (one.passed === one.counted) return `Passed on ${one.label}${once ? "" : " every time"}.`;
|
|
429
|
+
if (one.passed === 0) return `Failed on ${one.label}${once ? "" : " every time"}.`;
|
|
430
|
+
return `Passed ${of(one)} on ${one.label}.`;
|
|
431
|
+
}
|
|
432
|
+
const names = counted.map((t) => t.label);
|
|
433
|
+
const all = `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`;
|
|
434
|
+
if (counted.every((t) => t.passed === t.counted)) return `Passed on ${all} every time.`;
|
|
435
|
+
if (counted.every((t) => t.passed === 0)) return `Failed on ${all} every time.`;
|
|
436
|
+
const [first, ...rest] = counted;
|
|
437
|
+
return `${first?.label} passed ${first ? of(first) : ""}${rest.map((t) => `, ${t.label} ${of(t)}`).join("")}.`;
|
|
438
|
+
}
|
|
439
|
+
/** `(cursor 4 of 4, codex 1 of 4)` when the harnesses' pass rates differ; empty when they agree. */
|
|
440
|
+
function disagreement(record, cells) {
|
|
441
|
+
const tallies = tallyBy(cells, (c) => harnessLabel(record, c)).filter((t) => t.counted > 0);
|
|
442
|
+
if (tallies.length < 2) return "";
|
|
443
|
+
return new Set(tallies.map((t) => t.passed / t.counted)).size > 1 ? ` (${tallies.map((t) => `${t.label} ${of(t)}`).join(", ")})` : "";
|
|
444
|
+
}
|
|
445
|
+
/** `, not by minLift 60` when the variant's lift fell short of its `minLift`, which fails it. */
|
|
446
|
+
function minLiftShort(e, own) {
|
|
447
|
+
const keys = new Set(own.map((c) => c.key));
|
|
448
|
+
const pooled = (e.pooled ?? []).filter((p) => keys.has(p.key));
|
|
449
|
+
const missed = pooled.length ? pooled.find((p) => p.minLift !== null && p.outcome === "fail")?.minLift : own.find((c) => c.minLift !== null && c.interval && c.interval.low < c.minLift)?.minLift;
|
|
450
|
+
return missed == null ? "" : `, not by minLift ${Math.round(missed * 100)}`;
|
|
331
451
|
}
|
|
332
452
|
/**
|
|
333
|
-
*
|
|
334
|
-
*
|
|
453
|
+
* Under a variant, its comparison with the baseline (`measuresLine`): `Same pass rate · 5× faster ·
|
|
454
|
+
* judge "tidy" +0.12.`, one line, or one per harness (`cursor: …`) when it ran on several. Only
|
|
455
|
+
* when some measure besides pass rate holds; `verbose` adds the ones that only lean.
|
|
335
456
|
*/
|
|
336
|
-
function
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
const
|
|
340
|
-
|
|
457
|
+
function measureLines(e, own, verbose) {
|
|
458
|
+
const keys = new Set(own.map((c) => c.key));
|
|
459
|
+
const pooled = (e.pooled ?? []).filter((p) => keys.has(p.key));
|
|
460
|
+
const sets = pooled.length ? pooled.map((p) => ({
|
|
461
|
+
harness: own.find((c) => c.key === p.key)?.harness ?? p.key,
|
|
462
|
+
measures: p.measures
|
|
463
|
+
})) : own.flatMap((c) => c.measures ? [{
|
|
464
|
+
harness: c.harness,
|
|
465
|
+
measures: c.measures
|
|
466
|
+
}] : []);
|
|
467
|
+
const several = new Set(sets.map((x) => x.harness)).size > 1;
|
|
468
|
+
return sets.flatMap(({ harness, measures }) => {
|
|
469
|
+
if (!measures.filter((m) => m.measure !== "pass" && (m.verdict === "better" || m.verdict === "worse" || verbose && m.verdict === "leans")).length) return [];
|
|
470
|
+
const line = measuresLine(measures, verbose);
|
|
471
|
+
const text = `${line.slice(0, 1).toUpperCase()}${line.slice(1)}.`;
|
|
472
|
+
return [{
|
|
473
|
+
depth: 2,
|
|
474
|
+
text: several ? `${harness}: ${text}` : text,
|
|
475
|
+
tone: "dim"
|
|
476
|
+
}];
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
function compareLines(record, e, cells, verbose) {
|
|
480
|
+
const base = cells.filter((c) => c.baseline);
|
|
481
|
+
const name = (c) => c.environment === "-" ? "no environment" : c.environment;
|
|
482
|
+
const baseTally = tallyBy(base, () => "baseline")[0] ?? {
|
|
483
|
+
passed: 0,
|
|
484
|
+
counted: 0
|
|
485
|
+
};
|
|
486
|
+
const lines = [{
|
|
487
|
+
depth: 0,
|
|
488
|
+
text: `Baseline ${[...new Set(base.map(name))].join(", ")} passed ${of(baseTally)}${disagreement(record, base)}.`
|
|
489
|
+
}];
|
|
490
|
+
const variants = [...new Set(cells.filter((c) => c.variant !== null).map((c) => c.variant))];
|
|
491
|
+
if (!variants.length) return lines;
|
|
492
|
+
const rows = variants.map((v, order) => {
|
|
493
|
+
const own = cells.filter((c) => c.variant === v);
|
|
494
|
+
const t = tallyBy(own, () => v ?? "")[0] ?? {
|
|
495
|
+
passed: 0,
|
|
496
|
+
counted: 0
|
|
497
|
+
};
|
|
498
|
+
const effect = effectOf({
|
|
499
|
+
pass: t.passed,
|
|
500
|
+
counted: t.counted
|
|
501
|
+
}, {
|
|
502
|
+
pass: baseTally.passed,
|
|
503
|
+
counted: baseTally.counted
|
|
504
|
+
});
|
|
505
|
+
return {
|
|
506
|
+
v: v ?? "",
|
|
507
|
+
t,
|
|
508
|
+
effect,
|
|
509
|
+
own,
|
|
510
|
+
order
|
|
511
|
+
};
|
|
512
|
+
});
|
|
513
|
+
const rate = (t) => t.counted ? t.passed / t.counted : -1;
|
|
514
|
+
rows.sort((a, b) => rate(b.t) - rate(a.t) || a.order - b.order);
|
|
515
|
+
const width = Math.max(...rows.map((r) => r.v.length));
|
|
516
|
+
lines.push({
|
|
517
|
+
depth: 0,
|
|
518
|
+
text: "Variants, by pass rate:",
|
|
519
|
+
tone: "heading"
|
|
520
|
+
});
|
|
521
|
+
for (const r of rows) {
|
|
522
|
+
const short = minLiftShort(e, r.own);
|
|
523
|
+
const word = r.effect ? ` ${EFFECT[r.effect]}${short}` : short ? ` ${short.slice(2)}` : "";
|
|
524
|
+
lines.push({
|
|
525
|
+
depth: 1,
|
|
526
|
+
text: `${r.v.padEnd(width)} ${of(r.t)}${word}${disagreement(record, r.own)}`
|
|
527
|
+
});
|
|
528
|
+
lines.push(...measureLines(e, r.own, verbose));
|
|
529
|
+
if (!verbose) continue;
|
|
530
|
+
for (const cell of r.own) {
|
|
531
|
+
const stats = statsLine(cell, cells);
|
|
532
|
+
const who = r.own.length > 1 ? `${harnessLabel(record, cell)}: ` : "";
|
|
533
|
+
if (stats) lines.push({
|
|
534
|
+
depth: 2,
|
|
535
|
+
text: `${who}${stats}`,
|
|
536
|
+
tone: "dim"
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
return lines;
|
|
541
|
+
}
|
|
542
|
+
function failedChecks(cells, groupOf) {
|
|
543
|
+
const by = /* @__PURE__ */ new Map();
|
|
544
|
+
for (const cell of cells) for (const k of cell.checks) {
|
|
545
|
+
if (k.outcome.status !== "ran" || k.outcome.failed + k.outcome.errored === 0) continue;
|
|
546
|
+
const n = k.outcome.failed + k.outcome.errored;
|
|
547
|
+
const f = by.get(k.label) ?? {
|
|
548
|
+
label: k.label,
|
|
549
|
+
failed: 0,
|
|
550
|
+
byHarness: /* @__PURE__ */ new Map(),
|
|
551
|
+
first: k
|
|
552
|
+
};
|
|
553
|
+
f.failed += n;
|
|
554
|
+
const h = groupOf(cell);
|
|
555
|
+
f.byHarness.set(h, (f.byHarness.get(h) ?? 0) + n);
|
|
556
|
+
if (f.first.outcome.status === "ran" && !f.first.outcome.assertion) f.first = k;
|
|
557
|
+
by.set(k.label, f);
|
|
558
|
+
}
|
|
559
|
+
return [...by.values()].sort((a, b) => b.failed - a.failed || a.first.order - b.first.order);
|
|
560
|
+
}
|
|
561
|
+
/** `, all on H` or `, M of them on H`, per the rules at the top. */
|
|
562
|
+
function where(f, harnesses) {
|
|
563
|
+
if (harnesses < 2) return "";
|
|
564
|
+
const [top] = [...f.byHarness].sort((a, b) => b[1] - a[1]);
|
|
565
|
+
if (!top) return "";
|
|
566
|
+
const [h, n] = top;
|
|
567
|
+
if (n === f.failed) return f.failed === 1 ? `, on ${h}` : `, all on ${h}`;
|
|
568
|
+
return n * 2 > f.failed ? `, ${n} of them on ${h}` : "";
|
|
569
|
+
}
|
|
570
|
+
/** A message's first lines, without the label it repeats or a command's empty output headings. */
|
|
571
|
+
function messageLines(text, label = "", max = LINE_MAX) {
|
|
572
|
+
return (label && text.startsWith(`${label}: `) ? text.slice(label.length + 2) : text).split("\n").map((l) => l.trimEnd()).filter((l) => l.trim() && !/^--- (stderr|stdout) ---$/.test(l.trim())).slice(0, MESSAGE_LINES).map((l) => clip(l, max));
|
|
573
|
+
}
|
|
574
|
+
/** A failure's message is red only on a failed eval: a passing compare's baseline misses are expected. */
|
|
575
|
+
function checkLines(cells, groupOf, tone) {
|
|
576
|
+
const failed = failedChecks(cells, groupOf);
|
|
577
|
+
if (!failed.length) return [];
|
|
578
|
+
const harnesses = new Set(cells.map(groupOf)).size;
|
|
579
|
+
const lines = [{
|
|
580
|
+
depth: 0,
|
|
581
|
+
text: failed.length === 1 ? "Every failure was the same check:" : `${failed.length} checks failed:`,
|
|
582
|
+
tone: "heading"
|
|
583
|
+
}];
|
|
584
|
+
for (const f of failed) {
|
|
585
|
+
lines.push({
|
|
586
|
+
depth: 1,
|
|
587
|
+
text: `"${checkLabel(f.label)}" failed ${times(f.failed)}${where(f, harnesses)}`
|
|
588
|
+
});
|
|
589
|
+
const o = f.first.outcome;
|
|
590
|
+
const judge = f.first.judge;
|
|
591
|
+
const message = o.status === "ran" && o.assertion ? o.assertion : judge ? `judge: ${judge.question} (scores ${judge.scores.join(", ")}, threshold ${judge.threshold})` : "";
|
|
592
|
+
for (const text of messageLines(message, f.label)) lines.push({
|
|
593
|
+
depth: 2,
|
|
594
|
+
text,
|
|
595
|
+
tone
|
|
596
|
+
});
|
|
597
|
+
if (o.status === "ran" && o.at) lines.push({
|
|
598
|
+
depth: 2,
|
|
599
|
+
text: `at ${o.at}`,
|
|
600
|
+
tone: "dim"
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
return lines;
|
|
604
|
+
}
|
|
605
|
+
const CASES_NAMED = 8;
|
|
606
|
+
function failedCaseLines(e) {
|
|
607
|
+
const failed = (e.cases ?? []).filter((k) => k.outcome !== "pass" && k.outcome !== "skipped");
|
|
608
|
+
if (!failed.length) return [];
|
|
609
|
+
const named = failed.slice(0, CASES_NAMED).map((k) => k.label);
|
|
610
|
+
const more = failed.length > CASES_NAMED ? ` and ${failed.length - CASES_NAMED} more` : "";
|
|
611
|
+
return [{
|
|
612
|
+
depth: 0,
|
|
613
|
+
text: `Failed cases: ${named.join(", ")}${more}.`
|
|
614
|
+
}];
|
|
615
|
+
}
|
|
616
|
+
/** Errors no failed check explains: timeouts, infrastructure errors, throws. Never clipped: their remedy is often last. */
|
|
617
|
+
function errorLines(cells, tone) {
|
|
618
|
+
const by = /* @__PURE__ */ new Map();
|
|
619
|
+
for (const c of cells) for (const err of c.errors) {
|
|
620
|
+
const key = `${err.kind}\n${err.message}`;
|
|
621
|
+
const e = by.get(key) ?? {
|
|
622
|
+
kind: err.kind,
|
|
623
|
+
message: err.message,
|
|
624
|
+
n: 0
|
|
625
|
+
};
|
|
626
|
+
e.n += err.attempts;
|
|
627
|
+
by.set(key, e);
|
|
628
|
+
}
|
|
629
|
+
return [...by.values()].flatMap((e) => [{
|
|
630
|
+
depth: 0,
|
|
631
|
+
text: `${plural(e.n, "run")} ${ERROR_VERB[e.kind]}:`,
|
|
632
|
+
tone: "heading"
|
|
633
|
+
}, ...messageLines(e.message, "", Number.POSITIVE_INFINITY).map((text) => ({
|
|
634
|
+
depth: 1,
|
|
635
|
+
text,
|
|
636
|
+
tone
|
|
637
|
+
}))]);
|
|
638
|
+
}
|
|
639
|
+
/** `evals/a.eval.ts:12` from `source:evals/a.eval.ts#L12`. */
|
|
640
|
+
const location = (at) => at.replace(/^source:/, "").replace(/#L(\d+)$/, ":$1");
|
|
641
|
+
/** `"npm test" passed on try 2 of 3.`: a check that needed its `c.exec` retries is flaky. */
|
|
642
|
+
function retriedLines(cells) {
|
|
643
|
+
const seen = /* @__PURE__ */ new Set();
|
|
644
|
+
return cells.flatMap((c) => c.checks.flatMap((k) => {
|
|
645
|
+
if (!k.retried || seen.has(k.label)) return [];
|
|
646
|
+
seen.add(k.label);
|
|
647
|
+
return [`"${checkLabel(k.label)}" ${k.retried}.`];
|
|
648
|
+
}));
|
|
649
|
+
}
|
|
650
|
+
function pageErrorNotes(cells) {
|
|
651
|
+
const [first, ...rest] = cells.flatMap((c) => c.pageErrors ?? []);
|
|
652
|
+
if (!first) return [];
|
|
653
|
+
const more = rest.length ? ` (+${plural(rest.length, "more capture")})` : "";
|
|
654
|
+
return [`${plural(first.errors, "page error")} in capture "${first.capture}" → ${first.file}${more}`];
|
|
655
|
+
}
|
|
656
|
+
function remarkLines(remarks, notes) {
|
|
657
|
+
if (!remarks.length && !notes.length) return [];
|
|
341
658
|
return [
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
659
|
+
{
|
|
660
|
+
depth: 0,
|
|
661
|
+
text: "Worth knowing:",
|
|
662
|
+
tone: "heading"
|
|
663
|
+
},
|
|
664
|
+
...remarks.map((r) => ({
|
|
665
|
+
depth: 1,
|
|
666
|
+
text: r.fix.edit ? `${r.text} Fix at ${location(r.fix.at)}: ${r.fix.edit}.` : `${r.text} At ${location(r.fix.at)}.`
|
|
667
|
+
})),
|
|
668
|
+
...notes.map((text) => ({
|
|
669
|
+
depth: 1,
|
|
670
|
+
text
|
|
671
|
+
}))
|
|
345
672
|
];
|
|
346
673
|
}
|
|
347
|
-
/**
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
674
|
+
/** A compare's failures are told apart by arm: `the baseline`, or the variant's name. */
|
|
675
|
+
const armOf = (c) => c.baseline ? "the baseline" : c.variant ?? c.environment;
|
|
676
|
+
const nameOf = (e) => [...e.group, e.template ?? e.name].join(" › ");
|
|
677
|
+
function iconOf(e) {
|
|
678
|
+
if (e.todo || e.notRun) return "skipped";
|
|
679
|
+
if (e.outcome === "skipped") return "skipped";
|
|
680
|
+
if (e.outcome === "canceled") return "canceled";
|
|
681
|
+
if (e.outcome === "pass") return isNotGatedResult(e) ? "observe" : "pass";
|
|
682
|
+
return "fail";
|
|
351
683
|
}
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
const
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
684
|
+
function evalRow(record, e, remarks, verbose = false) {
|
|
685
|
+
const cells = cellsOf(e);
|
|
686
|
+
const decide = deciding(cells);
|
|
687
|
+
const passed = decide.reduce((n, c) => n + c.passed, 0);
|
|
688
|
+
const ran = decide.reduce((n, c) => n + c.passed + c.failed + c.infraError, 0);
|
|
689
|
+
const compare = cells.some((c) => c.baseline);
|
|
690
|
+
const counted = compare && decide.length < cells.length ? "variant run" : "run";
|
|
691
|
+
const summary = e.todo ? "todo" : e.notRun ? "not run" : ran === 0 ? e.outcome === "canceled" ? "canceled" : "skipped" : `${passed} of ${plural(ran, counted)} passed`;
|
|
692
|
+
const tone = iconOf(e) === "fail" ? "red" : "dim";
|
|
693
|
+
const notes = ran ? [...retriedLines(decide), ...pageErrorNotes(decide)] : [];
|
|
694
|
+
const detail = ran ? [
|
|
695
|
+
...compare ? compareLines(record, e, cells, verbose) : [{
|
|
696
|
+
depth: 0,
|
|
697
|
+
text: outcomeSentence(tallyBy(decide, labeler(record, decide)))
|
|
698
|
+
}],
|
|
699
|
+
...failedCaseLines(e),
|
|
700
|
+
...compare ? checkLines(cells, armOf, tone) : checkLines(decide, (c) => harnessLabel(record, c), tone),
|
|
701
|
+
...errorLines(compare ? cells : decide, tone),
|
|
702
|
+
...iconOf(e) === "fail" ? snapshotLines(cells) : [],
|
|
703
|
+
...questionLines(e.questions ?? []),
|
|
704
|
+
...observedLines(e.observations ?? [], cells, (c) => compare ? armOf(c) : harnessLabel(record, c)),
|
|
705
|
+
...remarkLines(remarks, notes)
|
|
706
|
+
] : remarkLines(remarks, []);
|
|
707
|
+
return {
|
|
708
|
+
id: e.id,
|
|
709
|
+
name: nameOf(e),
|
|
710
|
+
icon: iconOf(e),
|
|
711
|
+
summary,
|
|
712
|
+
detail,
|
|
713
|
+
remarks: remarks.length + notes.length,
|
|
714
|
+
compare
|
|
715
|
+
};
|
|
363
716
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
717
|
+
/** Free-text answers shown per question; the rest are in the record. */
|
|
718
|
+
const ANSWERS_SHOWN = 3;
|
|
719
|
+
/** Rows of `[label, text]` at `depth`, the labels padded to one column. */
|
|
720
|
+
const aligned = (depth, rows) => {
|
|
721
|
+
const w = Math.max(0, ...rows.map(([label]) => label.length));
|
|
722
|
+
return rows.map(([label, text]) => ({
|
|
723
|
+
depth,
|
|
724
|
+
text: `${label.padEnd(w)} ${text}`
|
|
725
|
+
}));
|
|
726
|
+
};
|
|
727
|
+
/** Each question's answer: who won the pairs, the tally of choices, or the first answers. */
|
|
728
|
+
function questionLines(questions) {
|
|
729
|
+
return questions.flatMap((q) => {
|
|
730
|
+
const head = {
|
|
731
|
+
depth: 0,
|
|
732
|
+
text: `Question: ${clip(q.question, 100)}`,
|
|
733
|
+
tone: "heading"
|
|
734
|
+
};
|
|
735
|
+
if (q.kind === "attempts") {
|
|
736
|
+
const failed = q.error === null ? [] : [{
|
|
737
|
+
depth: 1,
|
|
738
|
+
text: `${q.failed} of ${q.asked} not answered: ${clip(q.error, 200)}`,
|
|
739
|
+
tone: "red"
|
|
740
|
+
}];
|
|
741
|
+
if (q.choices) return [
|
|
742
|
+
head,
|
|
743
|
+
{
|
|
744
|
+
depth: 1,
|
|
745
|
+
text: q.choices.map((t) => `${t.choice} ${t.n}`).join(" · ")
|
|
746
|
+
},
|
|
747
|
+
...failed
|
|
748
|
+
];
|
|
749
|
+
const shown = q.answers.slice(0, ANSWERS_SHOWN);
|
|
750
|
+
const more = q.answers.length - shown.length;
|
|
368
751
|
return [
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
752
|
+
head,
|
|
753
|
+
...aligned(1, shown.map((a) => [a.attempt, clip(a.text.replace(/\s+/g, " "), 100)])),
|
|
754
|
+
...more ? [{
|
|
755
|
+
depth: 1,
|
|
756
|
+
text: `${more} more: reeval query attempt_answers`,
|
|
757
|
+
tone: "dim"
|
|
758
|
+
}] : [],
|
|
759
|
+
...failed
|
|
760
|
+
];
|
|
373
761
|
}
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
return [
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
762
|
+
if (q.votes === 0 && q.error === null) return [head, {
|
|
763
|
+
depth: 1,
|
|
764
|
+
text: "No pairs to judge: tries that failed their checks are left out.",
|
|
765
|
+
tone: "dim"
|
|
766
|
+
}];
|
|
767
|
+
if (q.rates.length === 0 && q.error === null) return [head, {
|
|
768
|
+
depth: 1,
|
|
769
|
+
text: `${plural(q.votes, "vote")}, every one a tie.`,
|
|
770
|
+
tone: "dim"
|
|
771
|
+
}];
|
|
772
|
+
const cuts = new Set(q.rates.map((r) => r.cut)).size > 1;
|
|
773
|
+
return [
|
|
774
|
+
head,
|
|
775
|
+
...aligned(1, q.rates.map((r) => [cuts ? `${r.name} (${r.cut})` : r.name, `won ${r.wins} of ${r.n}`])),
|
|
776
|
+
...q.error === null ? [] : [{
|
|
777
|
+
depth: 1,
|
|
778
|
+
text: `The model call failed: ${clip(q.error, 200)}`,
|
|
779
|
+
tone: "red"
|
|
780
|
+
}]
|
|
781
|
+
];
|
|
383
782
|
});
|
|
384
|
-
const formatted = e.formattedErrors[0];
|
|
385
|
-
const body = formatted && e.meta?.errorKind !== "timeout" ? formatted : message;
|
|
386
|
-
return capLines((c.enabled ? body : stripAnsi(body)).replace(/^\n+/, ""), options.max).map((l) => c.enabled && body === message ? c.red(l) : l);
|
|
387
783
|
}
|
|
388
|
-
/**
|
|
389
|
-
function
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
784
|
+
/** Where a failed eval's first failed attempt's final workspace snapshot is, to open it. */
|
|
785
|
+
function snapshotLines(cells) {
|
|
786
|
+
const [first, ...others] = cells.flatMap((c) => c.snapshots ?? []);
|
|
787
|
+
if (!first) return [];
|
|
788
|
+
return [{
|
|
789
|
+
depth: 0,
|
|
790
|
+
text: `Snapshot: ${first}${others.length ? ` (+${others.length} more in summary.md)` : ""}`,
|
|
791
|
+
tone: "dim"
|
|
792
|
+
}];
|
|
793
|
+
}
|
|
794
|
+
/** What `c.observe` recorded, each name's tally on one line, or a line per cell under it. */
|
|
795
|
+
function observedLines(tallies, cells, label) {
|
|
796
|
+
if (!tallies.length) return [];
|
|
797
|
+
const byKey = new Map(cells.map((c) => [c.key, c]));
|
|
798
|
+
const perCell = new Set(tallies.map((t) => t.cell)).size > 1;
|
|
799
|
+
const width = Math.max(...tallies.map((t) => clip(t.name, 40).length));
|
|
800
|
+
const lines = [{
|
|
801
|
+
depth: 0,
|
|
802
|
+
text: "Observed:",
|
|
803
|
+
tone: "heading"
|
|
804
|
+
}];
|
|
805
|
+
if (!perCell) {
|
|
806
|
+
for (const t of tallies) lines.push({
|
|
807
|
+
depth: 1,
|
|
808
|
+
text: `${clip(t.name, 40).padEnd(width)} ${observedLine(t)}`
|
|
809
|
+
});
|
|
810
|
+
return lines;
|
|
394
811
|
}
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
812
|
+
for (const name of new Set(tallies.map((t) => t.name))) {
|
|
813
|
+
lines.push({
|
|
814
|
+
depth: 1,
|
|
815
|
+
text: clip(name, 40)
|
|
816
|
+
});
|
|
817
|
+
const rows = tallies.filter((t) => t.name === name).map((t) => {
|
|
818
|
+
const cell = byKey.get(t.cell);
|
|
819
|
+
return [cell ? label(cell) : t.cell, observedLine(t)];
|
|
820
|
+
});
|
|
821
|
+
const w = Math.max(...rows.map(([who]) => who.length));
|
|
822
|
+
for (const [who, line] of rows) lines.push({
|
|
823
|
+
depth: 2,
|
|
824
|
+
text: `${who.padEnd(w)} ${line}`
|
|
825
|
+
});
|
|
826
|
+
}
|
|
827
|
+
return lines;
|
|
399
828
|
}
|
|
400
|
-
|
|
401
|
-
function
|
|
402
|
-
const
|
|
829
|
+
/** `3 passed · 1 failed · 1 not gated`, from the suite's tally. */
|
|
830
|
+
function tallyText(suite) {
|
|
831
|
+
const t = suite.tally;
|
|
832
|
+
return [
|
|
833
|
+
t.errors ? plural(t.errors, "run error") : "",
|
|
834
|
+
t.fail ? `${t.fail} failed` : "",
|
|
835
|
+
t.infraError ? plural(t.infraError, "infra error") : "",
|
|
836
|
+
t.canceled ? `${t.canceled} canceled` : "",
|
|
837
|
+
t.pass ? `${t.pass} passed` : "",
|
|
838
|
+
t.notGated ? `${t.notGated} not gated` : "",
|
|
839
|
+
t.skipped ? `${t.skipped} skipped` : ""
|
|
840
|
+
].filter(Boolean).join(" · ");
|
|
841
|
+
}
|
|
842
|
+
function statusList(input) {
|
|
843
|
+
const { record, suite } = input;
|
|
844
|
+
const { shown, hidden } = shownRemarks(input.remarks?.remarks ?? [], suite.runId, input.verbose);
|
|
845
|
+
const files = suite.files.map((f) => ({
|
|
846
|
+
path: f.path,
|
|
847
|
+
rows: f.evals.map((e) => evalRow(record, e, shown.filter((r) => r.evalId === e.path), input.verbose))
|
|
848
|
+
}));
|
|
403
849
|
const tokens = formatUsage(record.cost.usage);
|
|
850
|
+
const done = [
|
|
851
|
+
`Done in ${formatDuration(record.durationMs)}`,
|
|
852
|
+
tallyText(suite) || "nothing ran",
|
|
853
|
+
tokens
|
|
854
|
+
].filter(Boolean).join(" · ");
|
|
404
855
|
const notes = [
|
|
405
|
-
...
|
|
406
|
-
...record.status === "
|
|
407
|
-
|
|
408
|
-
formatDuration(record.durationMs),
|
|
409
|
-
tokens,
|
|
410
|
-
`results ${e.summaryPath}`
|
|
411
|
-
].filter(Boolean).join(" · "))
|
|
856
|
+
...input.notRun ? [`Incomplete: ${plural(input.notRun, "attempt")} this resume’s filters left out ${input.notRun === 1 ? "is" : "are"} not run; reeval run --resume ${record.runId} runs ${input.notRun === 1 ? "it" : "them"}`] : [],
|
|
857
|
+
...record.status === "budget" && !input.notRun ? ["Budget reached: attempts that hadn’t started were skipped"] : [],
|
|
858
|
+
...record.status === "interrupted" && !input.notRun ? ["Interrupted: attempts that hadn’t finished were canceled"] : []
|
|
412
859
|
];
|
|
413
|
-
return
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
860
|
+
return {
|
|
861
|
+
files,
|
|
862
|
+
errors: suite.errors,
|
|
863
|
+
done,
|
|
864
|
+
notes,
|
|
865
|
+
hiddenRemarks: hidden
|
|
866
|
+
};
|
|
418
867
|
}
|
|
868
|
+
/** The pointers under the closing line: the studio and the run's summary.md. */
|
|
869
|
+
const pointerLine = (summaryPath) => `Browse it: reeval studio · summary ${summaryPath}`;
|
|
419
870
|
// src/ui/tags.ts
|
|
420
871
|
const STOPWORDS = new Set("a an the and or of to in on for with without from by at as is are be it its that this when than then into via not no".split(" "));
|
|
421
872
|
const MAX_WORDS = 3;
|
|
@@ -475,6 +926,8 @@ var Tagger = class {
|
|
|
475
926
|
};
|
|
476
927
|
// src/ui/agent.ts
|
|
477
928
|
const MAX_LINES = 12;
|
|
929
|
+
/** Agents read whole lines, so the header's notes don't wrap. */
|
|
930
|
+
const WIDTH = Number.POSITIVE_INFINITY;
|
|
478
931
|
var AgentRenderer = class {
|
|
479
932
|
#o;
|
|
480
933
|
#c = colors(false);
|
|
@@ -490,14 +943,13 @@ var AgentRenderer = class {
|
|
|
490
943
|
onEvent(e) {
|
|
491
944
|
switch (e.type) {
|
|
492
945
|
case "plan":
|
|
493
|
-
this.#write(
|
|
494
|
-
for (const line of e.details) this.#write(line);
|
|
495
|
-
for (const w of e.warnings) this.#write(`warning: ${w}`);
|
|
496
|
-
for (const h of e.hints) this.#write(`hint: ${h}`);
|
|
497
|
-
for (const w of e.sandboxWarnings ?? []) this.#write(sandboxWarningLine(w));
|
|
946
|
+
for (const line of planLines(this.#c, e, WIDTH)) this.#write(line);
|
|
498
947
|
return;
|
|
499
948
|
case "run-start":
|
|
500
|
-
this.#write(
|
|
949
|
+
this.#write(startLine(this.#c, {
|
|
950
|
+
...this.#o.output,
|
|
951
|
+
outputLogs: "none"
|
|
952
|
+
}));
|
|
501
953
|
return;
|
|
502
954
|
case "collected":
|
|
503
955
|
this.#tags.addFile(e.evals);
|
|
@@ -519,148 +971,29 @@ var AgentRenderer = class {
|
|
|
519
971
|
return;
|
|
520
972
|
}
|
|
521
973
|
case "prepare":
|
|
522
|
-
for (const line of
|
|
974
|
+
for (const line of prepareStepLines(this.#c, e, WIDTH, MAX_LINES)) this.#write(line);
|
|
523
975
|
return;
|
|
524
976
|
case "notice":
|
|
525
|
-
|
|
977
|
+
for (const line of noticeLines(this.#c, e, WIDTH)) this.#write(line);
|
|
526
978
|
return;
|
|
527
|
-
case "run-end":
|
|
979
|
+
case "run-end": {
|
|
528
980
|
this.#write("");
|
|
529
|
-
|
|
981
|
+
const verbose = (this.#o.verbosity ?? 0) > 0;
|
|
982
|
+
const list = statusList({
|
|
983
|
+
record: e.record,
|
|
984
|
+
suite: suiteResult(e.record),
|
|
985
|
+
remarks: e.remarks ?? null,
|
|
986
|
+
verbose,
|
|
987
|
+
...e.notRun ? { notRun: e.notRun } : {}
|
|
988
|
+
});
|
|
989
|
+
for (const line of listLines(list, this.#c, false, pointerLine(e.summaryPath), verbose)) this.#write(line);
|
|
530
990
|
return;
|
|
991
|
+
}
|
|
531
992
|
default: return;
|
|
532
993
|
}
|
|
533
994
|
}
|
|
534
995
|
close() {}
|
|
535
996
|
};
|
|
536
|
-
// src/ui/header.ts
|
|
537
|
-
/** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
|
|
538
|
-
const GLYPH = 5;
|
|
539
|
-
/** `Est. Time: `, the longest key. */
|
|
540
|
-
const KEY = 11;
|
|
541
|
-
/** `Warning: `, the longer note label. */
|
|
542
|
-
const LABEL = 9;
|
|
543
|
-
/**
|
|
544
|
-
* `text` broken at spaces into lines of at most `width` columns, not counting color codes; a
|
|
545
|
-
* longer word keeps a line. A `·` stays with the word before it, so no line starts with one.
|
|
546
|
-
*/
|
|
547
|
-
function wrap(text, width) {
|
|
548
|
-
const words = [];
|
|
549
|
-
for (const word of text.split(/\s+/).filter(Boolean)) if (word === "·" && words.length) words[words.length - 1] += " ·";
|
|
550
|
-
else words.push(word);
|
|
551
|
-
const lines = [];
|
|
552
|
-
let line = "";
|
|
553
|
-
for (const word of words) if (line && stripAnsi(line).length + 1 + stripAnsi(word).length > width) {
|
|
554
|
-
lines.push(line);
|
|
555
|
-
line = word;
|
|
556
|
-
} else line = line ? `${line} ${word}` : word;
|
|
557
|
-
return line ? [...lines, line] : lines;
|
|
558
|
-
}
|
|
559
|
-
const indent = (n) => " ".repeat(n);
|
|
560
|
-
/** `REEVAL v0.16.1`, then the run's facts, one `➜ Key: value` line each. */
|
|
561
|
-
function headerLines(c, h, width) {
|
|
562
|
-
const name = `REEVAL v${h.version}`;
|
|
563
|
-
const title = c.bold(c.depth === 4 ? name : c.teal(name));
|
|
564
|
-
const valueWidth = width - GLYPH - KEY;
|
|
565
|
-
const row = (key, value) => {
|
|
566
|
-
const values = typeof value === "string" ? [value] : value;
|
|
567
|
-
const joined = values.join(", ");
|
|
568
|
-
return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
|
|
569
|
-
};
|
|
570
|
-
const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
|
|
571
|
-
const resumed = h.resumed ? c.dim(` (resumed, ${plural(h.resumed, "attempt")} done)`) : "";
|
|
572
|
-
const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
|
|
573
|
-
return [
|
|
574
|
-
h.dryRun ? `${title} ${c.dim("dry run")}` : title,
|
|
575
|
-
"",
|
|
576
|
-
...row("Run", `${h.runId}${resumed}`),
|
|
577
|
-
...h.config ? row("Config", h.config) : [],
|
|
578
|
-
...h.profile ? row("Profile", `${h.profile.name}${reason}`) : [],
|
|
579
|
-
...row(many("Harness", h.harnesses.length, "Harnesses"), h.harnesses.map((id) => harness(c, h, id))),
|
|
580
|
-
...row(many("Sandbox", h.sandboxes.length, "Sandboxes"), h.sandboxes),
|
|
581
|
-
...h.judges.length ? row(many("Judge", h.judges.length), h.judges) : [],
|
|
582
|
-
...row("Scope", scope(c, h)),
|
|
583
|
-
...row("Est. Time", estimate(c, h)),
|
|
584
|
-
...h.record && !h.dryRun ? row("Record", h.record) : []
|
|
585
|
-
];
|
|
586
|
-
}
|
|
587
|
-
/** A harness id, with the effort it runs at when the id doesn't name it. */
|
|
588
|
-
function harness(c, h, id) {
|
|
589
|
-
const effort = h.efforts?.[id];
|
|
590
|
-
if (effort === void 0 || id.includes("[effort=")) return id;
|
|
591
|
-
return `${id} ${`(effort: ${effort})`.split(" ").map((w) => c.dim(w)).join(" ")}`;
|
|
592
|
-
}
|
|
593
|
-
function scope(c, h) {
|
|
594
|
-
const attempts = h.fewest < h.attempts ? `${h.fewest}–${h.attempts} attempts` : plural(h.attempts, "attempt");
|
|
595
|
-
const parts = [
|
|
596
|
-
plural(h.files, "eval file"),
|
|
597
|
-
plural(h.evals, "eval") + (h.skipped ? ` (${h.skipped} skipped)` : ""),
|
|
598
|
-
plural(h.cells, "cell"),
|
|
599
|
-
attempts
|
|
600
|
-
];
|
|
601
|
-
const caps = [h.budget.attempts !== void 0 ? `${h.budget.attempts} ${h.attempts <= h.budget.attempts ? c.green("✓") : c.red("✗")}` : "", h.budget.duration ?? ""].filter(Boolean);
|
|
602
|
-
return `${parts.join(" · ")}${caps.length ? ` (budget: ${caps.join(", ")})` : ""}`;
|
|
603
|
-
}
|
|
604
|
-
function estimate(c, h) {
|
|
605
|
-
const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
|
|
606
|
-
const parallel = Math.min(h.concurrency, h.attempts);
|
|
607
|
-
if (parallel <= 1) return serial;
|
|
608
|
-
return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency})`)}`} · ${serial}`;
|
|
609
|
-
}
|
|
610
|
-
/** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
|
|
611
|
-
function noteLines(c, kind, text, width) {
|
|
612
|
-
const [glyph, label, paint] = kind === "hint" ? [
|
|
613
|
-
"ℹ",
|
|
614
|
-
"Hint:",
|
|
615
|
-
c.blue
|
|
616
|
-
] : [
|
|
617
|
-
"⚠",
|
|
618
|
-
"Warning:",
|
|
619
|
-
kind === "root" ? c.red : c.orange
|
|
620
|
-
];
|
|
621
|
-
const basic = {
|
|
622
|
-
warning: c.yellow,
|
|
623
|
-
hint: c.teal,
|
|
624
|
-
root: c.red
|
|
625
|
-
}[kind];
|
|
626
|
-
const lead = c.depth === 4 && kind !== "root" ? `${basic(glyph)} ${c.bold(label)}` : c.bold(paint(`${glyph} ${label}`));
|
|
627
|
-
const at = 14;
|
|
628
|
-
return wrap(text, Math.max(20, width - at)).map((line, i) => i === 0 ? ` ${lead}${indent(LABEL - label.length)}${line}` : `${indent(at)}${line}`);
|
|
629
|
-
}
|
|
630
|
-
/** A step line, `◆ Preparing 1 sandbox (docker)...`, its text wrapped under itself. */
|
|
631
|
-
function stepLines(glyph, text, width) {
|
|
632
|
-
return wrap(text, Math.max(20, width - GLYPH)).map((line, i) => i === 0 ? ` ${glyph} ${line}` : `${indent(GLYPH)}${line}`);
|
|
633
|
-
}
|
|
634
|
-
/** A run-level notice (a prepared workspace, the Ctrl-C one) as a step line. */
|
|
635
|
-
function noticeLines(c, e, width) {
|
|
636
|
-
const glyph = {
|
|
637
|
-
info: c.dim("ℹ"),
|
|
638
|
-
warn: c.orange("⚠"),
|
|
639
|
-
error: c.red("✗")
|
|
640
|
-
}[e.level];
|
|
641
|
-
const lines = stepLines(glyph, e.text, width);
|
|
642
|
-
return e.level === "info" ? lines.map((l) => c.dim(l)) : lines;
|
|
643
|
-
}
|
|
644
|
-
/** Preflight: which sandboxes it prepares, each one ready, and each failure. */
|
|
645
|
-
function prepareStepLines(c, e, width, max) {
|
|
646
|
-
if (e.state === "start") {
|
|
647
|
-
const n = e.targets.length;
|
|
648
|
-
const names = e.targets.map((t) => t.label).join(", ");
|
|
649
|
-
return stepLines(c.teal("◆"), `Preparing ${plural(n, "sandbox", "sandboxes")} (${names})...`, width);
|
|
650
|
-
}
|
|
651
|
-
const { outcome } = e;
|
|
652
|
-
if (outcome.ok) {
|
|
653
|
-
const took = outcome.ms >= 1e3 ? ` ${c.dim(formatDuration(outcome.ms))}` : "";
|
|
654
|
-
return [` ${c.green("✓")} ${outcome.target.label} ready${took}`];
|
|
655
|
-
}
|
|
656
|
-
const who = outcome.target.setBy.join(", ");
|
|
657
|
-
return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
|
|
658
|
-
}
|
|
659
|
-
/** The last line before the attempts' own: how their logs will print. */
|
|
660
|
-
function startLine(c, output) {
|
|
661
|
-
const text = output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
|
|
662
|
-
return ` ${c.teal("◆")} ${text}`;
|
|
663
|
-
}
|
|
664
997
|
// src/ui/plain.ts
|
|
665
998
|
/** In a grouped block, an end line replaces its start line and keeps its start time. */
|
|
666
999
|
const STARTS = {
|
|
@@ -702,6 +1035,8 @@ var PlainRenderer = class {
|
|
|
702
1035
|
#queued = /* @__PURE__ */ new Set();
|
|
703
1036
|
#runStartedAt = 0;
|
|
704
1037
|
#lastWrite = 0;
|
|
1038
|
+
/** The last line written was blank, so the list needs no gap above it. */
|
|
1039
|
+
#blank = false;
|
|
705
1040
|
#timer;
|
|
706
1041
|
/** On a TTY: the status line is drawn under everything else, and redrawn after each write. */
|
|
707
1042
|
#live;
|
|
@@ -710,8 +1045,11 @@ var PlainRenderer = class {
|
|
|
710
1045
|
#evals = /* @__PURE__ */ new Set();
|
|
711
1046
|
#tries = /* @__PURE__ */ new Set();
|
|
712
1047
|
#ended = /* @__PURE__ */ new Set();
|
|
1048
|
+
#list;
|
|
1049
|
+
#lastProgress = 0;
|
|
713
1050
|
constructor(options) {
|
|
714
1051
|
this.#o = options;
|
|
1052
|
+
this.#list = new LiveList(options.harness);
|
|
715
1053
|
this.#c = options.colors;
|
|
716
1054
|
this.#now = options.now ?? Date.now;
|
|
717
1055
|
this.#live = options.output.statusLine;
|
|
@@ -725,6 +1063,7 @@ var PlainRenderer = class {
|
|
|
725
1063
|
const line = `${text.trimEnd()}\n`;
|
|
726
1064
|
this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
|
|
727
1065
|
this.#lastWrite = this.#now();
|
|
1066
|
+
this.#blank = line === "\n";
|
|
728
1067
|
}
|
|
729
1068
|
/** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
|
|
730
1069
|
#drawStatus() {
|
|
@@ -775,31 +1114,17 @@ var PlainRenderer = class {
|
|
|
775
1114
|
onEvent(e) {
|
|
776
1115
|
const c = this.#c;
|
|
777
1116
|
switch (e.type) {
|
|
778
|
-
case "run-start":
|
|
1117
|
+
case "run-start":
|
|
779
1118
|
this.#runStartedAt = e.at;
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
this.#write("");
|
|
783
|
-
return;
|
|
784
|
-
}
|
|
785
|
-
const files = `${e.files} eval file${e.files === 1 ? "" : "s"}`;
|
|
786
|
-
const order = `${this.#o.output.logOrder} logs`;
|
|
787
|
-
const setup = `harness ${e.harness} · sandbox ${e.sandbox} · concurrency ${e.concurrency}`;
|
|
788
|
-
this.#write(c.dim(`reeval ${e.version} · run ${e.runId} · ${files} · ${setup} · ${order}`));
|
|
1119
|
+
this.#write(startLine(c, this.#o.output));
|
|
1120
|
+
this.#write("");
|
|
789
1121
|
return;
|
|
790
|
-
}
|
|
791
1122
|
case "plan":
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
return;
|
|
795
|
-
}
|
|
796
|
-
this.#write(e.line);
|
|
797
|
-
for (const line of e.details) this.#write(c.dim(line));
|
|
798
|
-
for (const w of e.warnings) this.#write(c.yellow(`warning: ${w}`));
|
|
799
|
-
for (const h of e.hints) this.#write(c.dim(`hint: ${h}`));
|
|
800
|
-
for (const w of e.sandboxWarnings ?? []) this.#sandboxWarning(w);
|
|
1123
|
+
for (const line of planLines(c, e, this.#columns)) this.#write(line);
|
|
1124
|
+
if (this.#o.output.githubActions) for (const w of e.sandboxWarnings ?? []) this.#annotate(w);
|
|
801
1125
|
return;
|
|
802
1126
|
case "collected":
|
|
1127
|
+
this.#list.collected(e.file, e.evals);
|
|
803
1128
|
this.#tags.addFile(e.evals);
|
|
804
1129
|
for (const x of e.evals) {
|
|
805
1130
|
if (x.skipped) continue;
|
|
@@ -811,6 +1136,7 @@ var PlainRenderer = class {
|
|
|
811
1136
|
return;
|
|
812
1137
|
case "eval-start": {
|
|
813
1138
|
const a = this.#attempt(e, e.at);
|
|
1139
|
+
this.#list.start(e);
|
|
814
1140
|
a.last = e;
|
|
815
1141
|
this.#queued.delete(e.key);
|
|
816
1142
|
this.#running.set(e.key, a);
|
|
@@ -820,6 +1146,7 @@ var PlainRenderer = class {
|
|
|
820
1146
|
case "eval-progress": {
|
|
821
1147
|
const a = this.#running.get(e.key);
|
|
822
1148
|
if (!a) return;
|
|
1149
|
+
this.#list.progress(e.key, e.progress);
|
|
823
1150
|
if (e.progress.kind !== "warning") a.last = e;
|
|
824
1151
|
const { progress } = e;
|
|
825
1152
|
const paint = a.ungated ? {
|
|
@@ -861,13 +1188,13 @@ var PlainRenderer = class {
|
|
|
861
1188
|
return;
|
|
862
1189
|
}
|
|
863
1190
|
case "eval-end":
|
|
1191
|
+
this.#list.end(e.result);
|
|
864
1192
|
this.#end(e.result, e.at);
|
|
865
1193
|
if (this.#tries.has(e.result.key)) this.#ended.add(e.result.key);
|
|
866
1194
|
this.#drawStatus();
|
|
867
1195
|
return;
|
|
868
|
-
case "prepare":
|
|
869
|
-
const
|
|
870
|
-
for (const line of lines) this.#write(line);
|
|
1196
|
+
case "prepare":
|
|
1197
|
+
for (const line of prepareStepLines(c, e, this.#columns, ERROR_LINES)) this.#write(line);
|
|
871
1198
|
if (e.state === "failed" && this.#o.output.githubActions) {
|
|
872
1199
|
const { target, failure } = e.outcome;
|
|
873
1200
|
const what = target.kind === "workspace" ? "The" : "Sandbox";
|
|
@@ -875,52 +1202,30 @@ var PlainRenderer = class {
|
|
|
875
1202
|
this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
|
|
876
1203
|
}
|
|
877
1204
|
return;
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
if (this.#o.output.rich) {
|
|
881
|
-
for (const line of noticeLines(c, e, this.#columns)) this.#write(line);
|
|
882
|
-
return;
|
|
883
|
-
}
|
|
884
|
-
const paint = {
|
|
885
|
-
info: c.dim,
|
|
886
|
-
warn: c.yellow,
|
|
887
|
-
error: c.red
|
|
888
|
-
}[e.level];
|
|
889
|
-
this.#write(paint(e.text));
|
|
1205
|
+
case "notice":
|
|
1206
|
+
for (const line of noticeLines(c, e, this.#columns)) this.#write(line);
|
|
890
1207
|
return;
|
|
891
|
-
|
|
892
|
-
case "run-end":
|
|
1208
|
+
case "run-end": {
|
|
893
1209
|
this.#clearStatus();
|
|
894
|
-
this.#write("");
|
|
895
|
-
|
|
1210
|
+
if (!this.#blank) this.#write("");
|
|
1211
|
+
const verbose = (this.#o.verbosity ?? 0) > 0;
|
|
1212
|
+
const list = statusList({
|
|
1213
|
+
record: e.record,
|
|
1214
|
+
suite: suiteResult(e.record),
|
|
1215
|
+
remarks: e.remarks ?? null,
|
|
1216
|
+
verbose,
|
|
1217
|
+
...e.notRun ? { notRun: e.notRun } : {}
|
|
1218
|
+
});
|
|
1219
|
+
for (const line of listLines(list, c, true, pointerLine(e.summaryPath), verbose)) this.#write(line);
|
|
896
1220
|
return;
|
|
1221
|
+
}
|
|
897
1222
|
}
|
|
898
1223
|
}
|
|
899
1224
|
get #columns() {
|
|
900
|
-
return this.#o.columns ||
|
|
901
|
-
}
|
|
902
|
-
/** The plan as the rich header: its facts, `--dry-run`'s table, then each kind of note. */
|
|
903
|
-
#richPlan(e) {
|
|
904
|
-
const c = this.#c;
|
|
905
|
-
const width = this.#columns;
|
|
906
|
-
for (const line of headerLines(c, e.header, width)) this.#write(line);
|
|
907
|
-
const sections = [
|
|
908
|
-
e.details,
|
|
909
|
-
[...e.warnings.flatMap((w) => noteLines(c, "warning", w, width)), ...(e.sandboxWarnings ?? []).flatMap((w) => noteLines(c, w.kind === "root" ? "root" : "warning", w.message, width))],
|
|
910
|
-
e.hints.flatMap((h) => noteLines(c, "hint", h, width))
|
|
911
|
-
];
|
|
912
|
-
for (const lines of sections) {
|
|
913
|
-
if (!lines.length) continue;
|
|
914
|
-
this.#write("");
|
|
915
|
-
for (const line of lines) this.#write(line);
|
|
916
|
-
}
|
|
917
|
-
this.#write("");
|
|
1225
|
+
return this.#o.columns || Number.POSITIVE_INFINITY;
|
|
918
1226
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
const line = sandboxWarningLine(w);
|
|
922
|
-
this.#write(w.kind === "root" ? c.bold(c.red(line)) : c.yellow(line));
|
|
923
|
-
if (!this.#o.output.githubActions) return;
|
|
1227
|
+
/** The header block names a sandbox warning; GitHub Actions also annotates it. */
|
|
1228
|
+
#annotate(w) {
|
|
924
1229
|
const file = w.file ? `file=${ghEscape(w.file, true)},` : "";
|
|
925
1230
|
const title = w.kind === "root" ? "Sandbox runs as root" : "Sandbox has extra capabilities";
|
|
926
1231
|
this.#write(`::warning ${file}title=${ghEscape(title, true)}::${ghEscape(w.message)}`);
|
|
@@ -968,13 +1273,23 @@ var PlainRenderer = class {
|
|
|
968
1273
|
}
|
|
969
1274
|
}
|
|
970
1275
|
}
|
|
971
|
-
/**
|
|
1276
|
+
/**
|
|
1277
|
+
* Every `heartbeatMs` while attempts run. Without attempt logs, how many evals are done; with
|
|
1278
|
+
* them, each running attempt's phase, when nothing printed for that long.
|
|
1279
|
+
*/
|
|
972
1280
|
tick(now = this.#now()) {
|
|
973
|
-
const { heartbeatMs } = this.#o.output;
|
|
1281
|
+
const { heartbeatMs, outputLogs } = this.#o.output;
|
|
974
1282
|
if (this.#live || heartbeatMs <= 0) return;
|
|
975
|
-
if (now - this.#lastWrite < heartbeatMs) return;
|
|
976
|
-
const c = this.#c;
|
|
977
1283
|
if (this.#running.size === 0) return;
|
|
1284
|
+
const c = this.#c;
|
|
1285
|
+
if (outputLogs === "none") {
|
|
1286
|
+
if (now - Math.max(this.#lastProgress, this.#runStartedAt) < heartbeatMs) return;
|
|
1287
|
+
const { done, total } = this.#list.counts();
|
|
1288
|
+
this.#write(c.dim(progressText(done, total, now - this.#runStartedAt)));
|
|
1289
|
+
this.#lastProgress = now;
|
|
1290
|
+
return;
|
|
1291
|
+
}
|
|
1292
|
+
if (now - this.#lastWrite < heartbeatMs) return;
|
|
978
1293
|
const running = [...this.#running.values()].map((a) => {
|
|
979
1294
|
const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
|
|
980
1295
|
return `${this.#prefix(a)} ${phaseOf(progress)} ${c.dim(formatDuration(now - a.startedAt))}`;
|
|
@@ -988,6 +1303,188 @@ var PlainRenderer = class {
|
|
|
988
1303
|
this.#clearStatus();
|
|
989
1304
|
}
|
|
990
1305
|
};
|
|
1306
|
+
// src/ui/tty.ts
|
|
1307
|
+
/** The list's state, which the Ink app reads with `useSyncExternalStore`. */
|
|
1308
|
+
var ListStore = class {
|
|
1309
|
+
#state = {
|
|
1310
|
+
files: [],
|
|
1311
|
+
logs: [],
|
|
1312
|
+
counts: {
|
|
1313
|
+
done: 0,
|
|
1314
|
+
total: 0
|
|
1315
|
+
},
|
|
1316
|
+
phase: null,
|
|
1317
|
+
finished: null
|
|
1318
|
+
};
|
|
1319
|
+
#listeners = /* @__PURE__ */ new Set();
|
|
1320
|
+
get = () => this.#state;
|
|
1321
|
+
subscribe = (listener) => {
|
|
1322
|
+
this.#listeners.add(listener);
|
|
1323
|
+
return () => this.#listeners.delete(listener);
|
|
1324
|
+
};
|
|
1325
|
+
set(next) {
|
|
1326
|
+
this.#state = {
|
|
1327
|
+
...this.#state,
|
|
1328
|
+
...next
|
|
1329
|
+
};
|
|
1330
|
+
for (const l of this.#listeners) l();
|
|
1331
|
+
}
|
|
1332
|
+
};
|
|
1333
|
+
var InteractiveRenderer = class {
|
|
1334
|
+
#o;
|
|
1335
|
+
#c;
|
|
1336
|
+
#store = new ListStore();
|
|
1337
|
+
#list;
|
|
1338
|
+
/** The header and run-level lines, as every terminal prints them. */
|
|
1339
|
+
#plain;
|
|
1340
|
+
#app;
|
|
1341
|
+
/** Set once the list couldn't load: from then on, the run prints as a pipe's does. */
|
|
1342
|
+
#fallback;
|
|
1343
|
+
#ended;
|
|
1344
|
+
#closed = false;
|
|
1345
|
+
constructor(options) {
|
|
1346
|
+
this.#o = options;
|
|
1347
|
+
this.#c = options.colors;
|
|
1348
|
+
this.#list = new LiveList(options.harness);
|
|
1349
|
+
const out = { write: (text) => {
|
|
1350
|
+
if (!this.#app || this.#fallback) return options.out.write(text);
|
|
1351
|
+
const lines = String(text).replace(/\n$/, "").split("\n");
|
|
1352
|
+
this.#store.set({ logs: [...this.#store.get().logs, ...lines] });
|
|
1353
|
+
return true;
|
|
1354
|
+
} };
|
|
1355
|
+
this.#plain = new PlainRenderer({
|
|
1356
|
+
...options,
|
|
1357
|
+
out,
|
|
1358
|
+
timers: false,
|
|
1359
|
+
output: {
|
|
1360
|
+
...options.output,
|
|
1361
|
+
statusLine: false,
|
|
1362
|
+
interactive: false,
|
|
1363
|
+
outputLogs: "none"
|
|
1364
|
+
}
|
|
1365
|
+
});
|
|
1366
|
+
}
|
|
1367
|
+
#mount() {
|
|
1368
|
+
if (this.#app || this.#closed) return;
|
|
1369
|
+
const cancel = this.#o.cancel ?? (() => process.kill(process.pid, "SIGINT"));
|
|
1370
|
+
const load = this.#o.load ?? (() => import("./ink-list-Ca-5dunO.js"));
|
|
1371
|
+
this.#app = load().then((m) => m.mountList({
|
|
1372
|
+
store: this.#store,
|
|
1373
|
+
colors: this.#c,
|
|
1374
|
+
cancel,
|
|
1375
|
+
stdout: this.#o.stdout,
|
|
1376
|
+
stdin: this.#o.stdin
|
|
1377
|
+
})).catch((error) => {
|
|
1378
|
+
this.#fallBack(error);
|
|
1379
|
+
});
|
|
1380
|
+
}
|
|
1381
|
+
#fallBack(error) {
|
|
1382
|
+
const out = this.#o.out;
|
|
1383
|
+
const { logs, phase } = this.#store.get();
|
|
1384
|
+
for (const line of logs) out.write(`${line}\n`);
|
|
1385
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
1386
|
+
out.write(`${this.#c.yellow(`Couldn't draw the interactive list (${reason}).`)}\n`);
|
|
1387
|
+
if (phase) out.write(`${phase}\n`);
|
|
1388
|
+
this.#fallback = new PlainRenderer({
|
|
1389
|
+
...this.#o,
|
|
1390
|
+
timers: false,
|
|
1391
|
+
output: {
|
|
1392
|
+
...this.#o.output,
|
|
1393
|
+
statusLine: false,
|
|
1394
|
+
interactive: false
|
|
1395
|
+
}
|
|
1396
|
+
});
|
|
1397
|
+
if (this.#ended) this.#fallback.onEvent(this.#ended.event);
|
|
1398
|
+
}
|
|
1399
|
+
#update() {
|
|
1400
|
+
const { done, total } = this.#list.counts();
|
|
1401
|
+
this.#store.set({
|
|
1402
|
+
files: this.#list.files(),
|
|
1403
|
+
counts: {
|
|
1404
|
+
done,
|
|
1405
|
+
total
|
|
1406
|
+
}
|
|
1407
|
+
});
|
|
1408
|
+
}
|
|
1409
|
+
onEvent(e) {
|
|
1410
|
+
if (this.#fallback) {
|
|
1411
|
+
if (e.type !== "run-start") this.#fallback.onEvent(e);
|
|
1412
|
+
return;
|
|
1413
|
+
}
|
|
1414
|
+
switch (e.type) {
|
|
1415
|
+
case "run-start":
|
|
1416
|
+
this.#store.set({ phase: startLine(this.#c, this.#o.output) });
|
|
1417
|
+
return;
|
|
1418
|
+
case "collected":
|
|
1419
|
+
this.#list.collected(e.file, e.evals);
|
|
1420
|
+
this.#mount();
|
|
1421
|
+
this.#update();
|
|
1422
|
+
return;
|
|
1423
|
+
case "eval-start":
|
|
1424
|
+
this.#list.start(e);
|
|
1425
|
+
this.#update();
|
|
1426
|
+
return;
|
|
1427
|
+
case "eval-progress":
|
|
1428
|
+
if (e.progress.kind === "warning" || e.progress.kind === "browser-download") this.#plain.onEvent(e);
|
|
1429
|
+
this.#list.progress(e.key, e.progress);
|
|
1430
|
+
this.#update();
|
|
1431
|
+
return;
|
|
1432
|
+
case "eval-end":
|
|
1433
|
+
this.#list.end(e.result);
|
|
1434
|
+
this.#update();
|
|
1435
|
+
return;
|
|
1436
|
+
case "debug": return;
|
|
1437
|
+
case "run-end": {
|
|
1438
|
+
const list = statusList({
|
|
1439
|
+
record: e.record,
|
|
1440
|
+
suite: suiteResult(e.record),
|
|
1441
|
+
remarks: e.remarks ?? null,
|
|
1442
|
+
verbose: false,
|
|
1443
|
+
...e.notRun ? { notRun: e.notRun } : {}
|
|
1444
|
+
});
|
|
1445
|
+
if (!this.#app) {
|
|
1446
|
+
this.#plain.onEvent(e);
|
|
1447
|
+
return;
|
|
1448
|
+
}
|
|
1449
|
+
this.#ended = {
|
|
1450
|
+
list,
|
|
1451
|
+
pointer: pointerLine(e.summaryPath),
|
|
1452
|
+
event: e
|
|
1453
|
+
};
|
|
1454
|
+
this.#store.set({
|
|
1455
|
+
files: list.files,
|
|
1456
|
+
...this.#store.get().phase ? { phase: endLine(this.#c, e.record.status) } : {},
|
|
1457
|
+
finished: {
|
|
1458
|
+
done: list.done,
|
|
1459
|
+
notes: list.notes,
|
|
1460
|
+
pointer: pointerLine(e.summaryPath),
|
|
1461
|
+
errors: list.errors
|
|
1462
|
+
}
|
|
1463
|
+
});
|
|
1464
|
+
return;
|
|
1465
|
+
}
|
|
1466
|
+
default: this.#plain.onEvent(e);
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
close() {
|
|
1470
|
+
this.#plain.close();
|
|
1471
|
+
this.#fallback?.close();
|
|
1472
|
+
if (!this.#store.get().finished) {
|
|
1473
|
+
this.#closed = true;
|
|
1474
|
+
this.#app?.then((a) => a?.unmount());
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
/** Until the user quits the list, then the whole list into the scrollback; at once when there's none. */
|
|
1478
|
+
async wait() {
|
|
1479
|
+
const app = await this.#app;
|
|
1480
|
+
if (!app || !this.#ended) return;
|
|
1481
|
+
await app.waitUntilExit();
|
|
1482
|
+
const { phase } = this.#store.get();
|
|
1483
|
+
const { list, pointer } = this.#ended;
|
|
1484
|
+
const text = [...phase ? [phase, ""] : [], ...listLines(list, this.#c, true, pointer)];
|
|
1485
|
+
this.#o.out.write(`${text.join("\n")}\n`);
|
|
1486
|
+
}
|
|
1487
|
+
};
|
|
991
1488
|
// src/ui/index.ts
|
|
992
1489
|
const depthOf = (bits = 4) => bits >= 24 ? 24 : bits >= 8 ? 8 : 4;
|
|
993
1490
|
function createRenderer(options) {
|
|
@@ -996,10 +1493,17 @@ function createRenderer(options) {
|
|
|
996
1493
|
...options,
|
|
997
1494
|
out
|
|
998
1495
|
});
|
|
1496
|
+
const paint = colors(options.output.color, depthOf(options.colorDepth));
|
|
1497
|
+
if (options.output.interactive && options.tty) return new InteractiveRenderer({
|
|
1498
|
+
...options,
|
|
1499
|
+
...options.tty,
|
|
1500
|
+
out,
|
|
1501
|
+
colors: paint
|
|
1502
|
+
});
|
|
999
1503
|
return new PlainRenderer({
|
|
1000
1504
|
...options,
|
|
1001
1505
|
out,
|
|
1002
|
-
colors:
|
|
1506
|
+
colors: paint
|
|
1003
1507
|
});
|
|
1004
1508
|
}
|
|
1005
1509
|
// src/cli.ts
|
|
@@ -1072,7 +1576,7 @@ async function run(argv) {
|
|
|
1072
1576
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
1073
1577
|
}
|
|
1074
1578
|
if (command === "init") {
|
|
1075
|
-
const { initCommand } = await import("./init-
|
|
1579
|
+
const { initCommand } = await import("./init-8o6W3_fn.js");
|
|
1076
1580
|
return initCommand(values, filters, {
|
|
1077
1581
|
out: (s) => console.log(s),
|
|
1078
1582
|
input: process.stdin,
|
|
@@ -1081,7 +1585,7 @@ async function run(argv) {
|
|
|
1081
1585
|
});
|
|
1082
1586
|
}
|
|
1083
1587
|
if (command === "doctor") {
|
|
1084
|
-
const { doctorCommand } = await import("./doctor-
|
|
1588
|
+
const { doctorCommand } = await import("./doctor-BcvfKpBX.js");
|
|
1085
1589
|
return doctorCommand(values, {
|
|
1086
1590
|
out: (s) => console.log(s),
|
|
1087
1591
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -1089,7 +1593,7 @@ async function run(argv) {
|
|
|
1089
1593
|
});
|
|
1090
1594
|
}
|
|
1091
1595
|
if (command === "report") {
|
|
1092
|
-
const { reportCommand } = await import("./command-
|
|
1596
|
+
const { reportCommand } = await import("./command-CFQ9c1hk.js");
|
|
1093
1597
|
return reportCommand(values, filters, {
|
|
1094
1598
|
out: (s) => console.log(s),
|
|
1095
1599
|
err: (s) => console.error(dim(s)),
|
|
@@ -1104,7 +1608,7 @@ async function run(argv) {
|
|
|
1104
1608
|
}
|
|
1105
1609
|
if (command === "studio") {
|
|
1106
1610
|
const config = await loadConfig(configPath, "studio");
|
|
1107
|
-
const { studioCommand } = await import("./command-
|
|
1611
|
+
const { studioCommand } = await import("./command-DhiwsZl1.js");
|
|
1108
1612
|
return studioCommand({
|
|
1109
1613
|
config,
|
|
1110
1614
|
flags: values,
|
|
@@ -1115,7 +1619,7 @@ async function run(argv) {
|
|
|
1115
1619
|
if (command === "mcp") {
|
|
1116
1620
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
1117
1621
|
const config = await loadConfig(configPath, "mcp");
|
|
1118
|
-
const { mcpCommand } = await import("./command-
|
|
1622
|
+
const { mcpCommand } = await import("./command-qGS802uX.js");
|
|
1119
1623
|
return mcpCommand({
|
|
1120
1624
|
config,
|
|
1121
1625
|
flags: values,
|
|
@@ -1124,7 +1628,7 @@ async function run(argv) {
|
|
|
1124
1628
|
}
|
|
1125
1629
|
if (command in HISTORY_COMMANDS) {
|
|
1126
1630
|
const config = await loadConfig(configPath, command);
|
|
1127
|
-
const commands = await import("./commands-
|
|
1631
|
+
const commands = await import("./commands-xYTMNTPv.js");
|
|
1128
1632
|
const ctx = {
|
|
1129
1633
|
config,
|
|
1130
1634
|
flags: values,
|
|
@@ -1137,7 +1641,7 @@ async function run(argv) {
|
|
|
1137
1641
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
1138
1642
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
1139
1643
|
case "query": {
|
|
1140
|
-
const { queryCommand } = await import("./query-
|
|
1644
|
+
const { queryCommand } = await import("./query-CvgUsvep.js");
|
|
1141
1645
|
return queryCommand({
|
|
1142
1646
|
...ctx,
|
|
1143
1647
|
err: (s) => console.error(dim(s))
|
|
@@ -1145,7 +1649,7 @@ async function run(argv) {
|
|
|
1145
1649
|
}
|
|
1146
1650
|
case "analyze": {
|
|
1147
1651
|
loadEnvFiles(configPath);
|
|
1148
|
-
const { analyzeCommand } = await import("./command-
|
|
1652
|
+
const { analyzeCommand } = await import("./command-fO0BFsTq.js");
|
|
1149
1653
|
return analyzeCommand({
|
|
1150
1654
|
...ctx,
|
|
1151
1655
|
err: (s) => console.error(dim(s))
|
|
@@ -1159,12 +1663,12 @@ async function run(argv) {
|
|
|
1159
1663
|
}
|
|
1160
1664
|
/** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
|
|
1161
1665
|
async function dryRunPrices(planned, judgeModel) {
|
|
1162
|
-
const { needsPrices } = await import("./cost-
|
|
1666
|
+
const { needsPrices } = await import("./cost-BJU0MAji.js");
|
|
1163
1667
|
if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
|
|
1164
1668
|
table: null,
|
|
1165
1669
|
why: "nothing to price"
|
|
1166
1670
|
};
|
|
1167
|
-
const { loadPrices } = await import("./prices-
|
|
1671
|
+
const { loadPrices } = await import("./prices-DYJA0tJN.js");
|
|
1168
1672
|
return loadPrices();
|
|
1169
1673
|
}
|
|
1170
1674
|
/** Asks a yes/no question on the terminal; anything but y or yes is no. */
|
|
@@ -1197,8 +1701,8 @@ async function cleanCommand(flags, configPath) {
|
|
|
1197
1701
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1198
1702
|
console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
|
|
1199
1703
|
if (flags.images) {
|
|
1200
|
-
const { dockerAvailable, pruneImages } = await import("./src-
|
|
1201
|
-
const { requireDocker } = await import("./providers-
|
|
1704
|
+
const { dockerAvailable, pruneImages } = await import("./src-BkH4GOxu.js");
|
|
1705
|
+
const { requireDocker } = await import("./providers-D0ob_tra.js");
|
|
1202
1706
|
const engines = [];
|
|
1203
1707
|
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1204
1708
|
if (!engines.length) await requireDocker();
|
|
@@ -1219,7 +1723,7 @@ function verbosityOf(flags) {
|
|
|
1219
1723
|
* does. Needs a config, as `reeval run` does: there is no default image.
|
|
1220
1724
|
*/
|
|
1221
1725
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1222
|
-
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-
|
|
1726
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-CNayXCp2.js");
|
|
1223
1727
|
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
|
|
1224
1728
|
...flags,
|
|
1225
1729
|
"dry-run": true
|
|
@@ -1271,15 +1775,32 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1271
1775
|
return failed.every((b) => !b.ok && b.failure.usage) ? EXIT.usage : EXIT.infra;
|
|
1272
1776
|
}
|
|
1273
1777
|
/** `reeval run`: check, plan, run the engine, then write the run and pick the exit code. */
|
|
1274
|
-
async function runCommand(flags, configPath,
|
|
1275
|
-
const
|
|
1778
|
+
async function runCommand(flags, configPath, given) {
|
|
1779
|
+
const resumed = flags.resume === void 0 ? null : await resumedRun(flags, given, configPath);
|
|
1780
|
+
const runFlags = resumed?.settings ? {
|
|
1781
|
+
...flags,
|
|
1782
|
+
...resumed.settings
|
|
1783
|
+
} : flags;
|
|
1784
|
+
const filters = resumed?.settings?.filters ?? given;
|
|
1785
|
+
const setup = await prepareRun(runFlags, configPath, {
|
|
1786
|
+
noProfile: !!resumed?.saved && resumed.saved.profile === void 0,
|
|
1787
|
+
...resumed?.profileOf ? { profileOf: resumed.profileOf } : {}
|
|
1788
|
+
});
|
|
1789
|
+
const explicit = runFlags.concurrency ?? (process.env.REEVAL_CONCURRENCY || void 0);
|
|
1790
|
+
const startedWith = resumed ? resumed.saved ?? void 0 : {
|
|
1791
|
+
...setup.runs === void 0 ? {} : { runs: setup.runs },
|
|
1792
|
+
...setup.profile ? { profile: setup.profile.name } : {},
|
|
1793
|
+
...explicit === void 0 ? {} : { concurrency: setup.concurrency },
|
|
1794
|
+
filters: savedFilters(setup.config.root, filters),
|
|
1795
|
+
tags: runFlags.tag ?? []
|
|
1796
|
+
};
|
|
1276
1797
|
const ctx = await startRun(setup);
|
|
1277
1798
|
invocation.runId = ctx.runId;
|
|
1278
1799
|
const githubOut = process.env.GITHUB_OUTPUT;
|
|
1279
1800
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1280
1801
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1281
1802
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1282
|
-
const { engineVersions } = await import("./node-
|
|
1803
|
+
const { engineVersions } = await import("./node-DhIRSrrK.js");
|
|
1283
1804
|
const terminal = setup.json && !setup.dryRun ? process.stderr : process.stdout;
|
|
1284
1805
|
const renderer = createRenderer({
|
|
1285
1806
|
out: setup.json && setup.dryRun ? { write: () => true } : terminal,
|
|
@@ -1287,11 +1808,19 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1287
1808
|
colorDepth: terminal.isTTY ? terminal.getColorDepth() : void 0,
|
|
1288
1809
|
output: setup.output,
|
|
1289
1810
|
verbosity: ctx.verbosity ?? 0,
|
|
1290
|
-
harness: setup.config.harness[0]?.kind ?? "agent"
|
|
1811
|
+
harness: setup.config.harness[0]?.kind ?? "agent",
|
|
1812
|
+
tty: {
|
|
1813
|
+
stdout: process.stdout,
|
|
1814
|
+
stdin: process.stdin
|
|
1815
|
+
}
|
|
1291
1816
|
});
|
|
1292
1817
|
let executed;
|
|
1293
1818
|
try {
|
|
1294
|
-
executed = await executeRun(ctx, setup, filters, renderer
|
|
1819
|
+
executed = await executeRun(ctx, setup, filters, renderer, {
|
|
1820
|
+
startedWith,
|
|
1821
|
+
notes: resumed?.notes ?? [],
|
|
1822
|
+
narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length)
|
|
1823
|
+
});
|
|
1295
1824
|
} catch (error) {
|
|
1296
1825
|
renderer.close();
|
|
1297
1826
|
if (setup.resume === void 0) await rm(ctx.stagingDir, {
|
|
@@ -1300,11 +1829,11 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1300
1829
|
});
|
|
1301
1830
|
throw error;
|
|
1302
1831
|
}
|
|
1303
|
-
const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
|
|
1832
|
+
const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions, leftOut } = executed;
|
|
1304
1833
|
invocation.unhandledErrors = engineRun.unhandledErrors;
|
|
1305
1834
|
const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
|
|
1306
1835
|
if ((interrupted || engineRun.canceled) && engines.length) {
|
|
1307
|
-
const { removeRunContainers } = await import("./docker-
|
|
1836
|
+
const { removeRunContainers } = await import("./docker-BlSopHpZ.js");
|
|
1308
1837
|
await removeRunContainers(ctx.runId, engines);
|
|
1309
1838
|
}
|
|
1310
1839
|
if (engineRun.stopped || !plan) {
|
|
@@ -1321,8 +1850,8 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1321
1850
|
}
|
|
1322
1851
|
finishing = true;
|
|
1323
1852
|
live?.phase("finishing");
|
|
1324
|
-
const { lockfiles } = await import("./state-
|
|
1325
|
-
const { writeScratchRun } = await import("./results-
|
|
1853
|
+
const { lockfiles } = await import("./state-DZbAO95a.js");
|
|
1854
|
+
const { writeScratchRun } = await import("./results-CfC0id_u.js");
|
|
1326
1855
|
const run = await writeScratchRun({
|
|
1327
1856
|
ctx,
|
|
1328
1857
|
author: ctx.author,
|
|
@@ -1342,12 +1871,19 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1342
1871
|
git: ctx.git,
|
|
1343
1872
|
previous: ctx.previous,
|
|
1344
1873
|
previousArtifacts: ctx.previousArtifacts,
|
|
1874
|
+
leftOut,
|
|
1345
1875
|
runner: runnerInfo(),
|
|
1876
|
+
...startedWith ? { invocation: startedWith } : {},
|
|
1346
1877
|
...answers && (answers.evals.size || answers.attempts.size) ? { answers } : {}
|
|
1347
1878
|
}, () => live?.stop());
|
|
1348
1879
|
const { record } = run;
|
|
1349
1880
|
const outcome = runOutcome(record);
|
|
1350
|
-
const
|
|
1881
|
+
const notRun = leftOut.filter((l) => !l.done).length;
|
|
1882
|
+
const ran = notRun > 0 && !stopped && record.status === "interrupted" ? runOutcome({
|
|
1883
|
+
...record,
|
|
1884
|
+
status: "complete"
|
|
1885
|
+
}) : outcome;
|
|
1886
|
+
const code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1351
1887
|
const { pruneDebug } = await import("./files-COW5JBJM.js");
|
|
1352
1888
|
await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
|
|
1353
1889
|
const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
|
|
@@ -1357,15 +1893,17 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1357
1893
|
record,
|
|
1358
1894
|
summaryPath: fromCwd(summary),
|
|
1359
1895
|
results: engineRun.evals,
|
|
1360
|
-
remarks: run.remarks
|
|
1896
|
+
remarks: run.remarks,
|
|
1897
|
+
...notRun > 0 && !stopped ? { notRun } : {}
|
|
1361
1898
|
});
|
|
1362
1899
|
renderer.close();
|
|
1363
1900
|
const stepSummary = process.env.GITHUB_STEP_SUMMARY;
|
|
1364
1901
|
if (stepSummary && run.artifactDir) await appendFile(stepSummary, await readFile(summary)).catch(() => {});
|
|
1365
1902
|
const { config } = ctx;
|
|
1366
1903
|
const mode = setup.record;
|
|
1367
|
-
|
|
1368
|
-
|
|
1904
|
+
const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI();
|
|
1905
|
+
if (autoRecord && record.status !== "interrupted") {
|
|
1906
|
+
const { record: recordCmd } = await import("./commands-xYTMNTPv.js");
|
|
1369
1907
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
1370
1908
|
const line = await recordCmd(config, record.runId, {
|
|
1371
1909
|
...flags,
|
|
@@ -1373,15 +1911,58 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1373
1911
|
});
|
|
1374
1912
|
if (setup.json) console.error(line);
|
|
1375
1913
|
else console.log(line);
|
|
1914
|
+
} else if (autoRecord && notRun > 0 && !stopped) {
|
|
1915
|
+
const line = `Not recorded: run ${record.runId} is incomplete. reeval run --resume ${record.runId}${flags.record ? " --record" : ""} runs the rest and records it.`;
|
|
1916
|
+
if (setup.json) console.error(line);
|
|
1917
|
+
else console.log(line);
|
|
1376
1918
|
}
|
|
1377
1919
|
if (setup.json) {
|
|
1378
|
-
const { suiteResult } = await import("./suite-
|
|
1379
|
-
process.stdout.write(`${JSON.stringify(
|
|
1920
|
+
const { suiteResult } = await import("./suite-u2KJnGpY.js");
|
|
1921
|
+
process.stdout.write(`${JSON.stringify({
|
|
1922
|
+
...suiteResult(record),
|
|
1923
|
+
outcome: ran
|
|
1924
|
+
}, null, 2)}\n`);
|
|
1380
1925
|
}
|
|
1926
|
+
await renderer.wait?.();
|
|
1381
1927
|
return code;
|
|
1382
1928
|
}
|
|
1383
|
-
/**
|
|
1384
|
-
|
|
1929
|
+
/**
|
|
1930
|
+
* `--resume`: the run's saved settings with the flags given now. An older reeval's run saved none,
|
|
1931
|
+
* so it plans with the flags given now, with a note.
|
|
1932
|
+
*/
|
|
1933
|
+
async function resumedRun(flags, filters, configPath) {
|
|
1934
|
+
const path = requireConfig(configPath);
|
|
1935
|
+
loadEnvFiles(path);
|
|
1936
|
+
const root = dirname(path);
|
|
1937
|
+
const target = await resolveResume(root, flags.resume || void 0);
|
|
1938
|
+
const saved = await savedInvocation(root, target);
|
|
1939
|
+
if (!saved) return {
|
|
1940
|
+
saved: null,
|
|
1941
|
+
settings: null,
|
|
1942
|
+
notes: [`run ${target.runId} saved no --runs, profile or filters, so this resume plans with the ones given now`],
|
|
1943
|
+
profileOf: void 0
|
|
1944
|
+
};
|
|
1945
|
+
const given = {
|
|
1946
|
+
runs: flags.runs,
|
|
1947
|
+
profile: flags.profile,
|
|
1948
|
+
concurrency: flags.concurrency,
|
|
1949
|
+
tag: flags.tag,
|
|
1950
|
+
filters
|
|
1951
|
+
};
|
|
1952
|
+
const resumed = resumeSettings(target.runId, saved, given, { root });
|
|
1953
|
+
const fromRun = flags.profile === void 0 && resumed.settings.profile !== void 0;
|
|
1954
|
+
return {
|
|
1955
|
+
saved,
|
|
1956
|
+
...resumed,
|
|
1957
|
+
profileOf: fromRun ? target.runId : void 0
|
|
1958
|
+
};
|
|
1959
|
+
}
|
|
1960
|
+
/**
|
|
1961
|
+
* Everything that can fail before the evals are collected: output mode, config, profile, flags.
|
|
1962
|
+
* `noProfile` is a resume of a run that had none, which `defaultProfile` mustn't give one;
|
|
1963
|
+
* `profileOf` names the resumed run whose profile `flags.profile` is.
|
|
1964
|
+
*/
|
|
1965
|
+
async function prepareRun(flags, configPath, { noProfile = false, profileOf } = {}) {
|
|
1385
1966
|
const output = resolveOutput({
|
|
1386
1967
|
flags: {
|
|
1387
1968
|
ui: flags.ui,
|
|
@@ -1392,13 +1973,15 @@ async function prepareRun(flags, configPath) {
|
|
|
1392
1973
|
},
|
|
1393
1974
|
env: process.env,
|
|
1394
1975
|
isTTY: !!(flags.json && !flags["dry-run"] ? process.stderr : process.stdout).isTTY,
|
|
1395
|
-
agent: detectAgent().name
|
|
1976
|
+
agent: detectAgent().name,
|
|
1977
|
+
stdinTTY: !!process.stdin.isTTY && !flags.json && !flags["dry-run"]
|
|
1396
1978
|
});
|
|
1397
1979
|
const path = requireConfig(configPath);
|
|
1398
1980
|
loadEnvFiles(path);
|
|
1399
|
-
const named = flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
|
|
1981
|
+
const named = noProfile ? void 0 : flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
|
|
1400
1982
|
const config = await loadConfig(path, "run", named);
|
|
1401
|
-
|
|
1983
|
+
if (profileOf && flags.profile && !config.profiles[flags.profile]) throw new UsageError(`run ${profileOf} was started with profile ${flags.profile}, which ${config.file} no longer defines. Put it back to resume the run, or start a new run without --resume.`);
|
|
1984
|
+
const profile = noProfile ? null : selectProfile(config, flags.profile);
|
|
1402
1985
|
let runs;
|
|
1403
1986
|
if (flags.runs !== void 0) {
|
|
1404
1987
|
runs = Number(flags.runs);
|
|
@@ -1408,7 +1991,7 @@ async function prepareRun(flags, configPath) {
|
|
|
1408
1991
|
const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
|
|
1409
1992
|
const container = sandboxes.find((s) => s.kind === "docker");
|
|
1410
1993
|
if (!container) return hostCapacity();
|
|
1411
|
-
const { dockerCapacity } = await import("./src-
|
|
1994
|
+
const { dockerCapacity } = await import("./src-BkH4GOxu.js");
|
|
1412
1995
|
return await dockerCapacity(container.engine) ?? hostCapacity();
|
|
1413
1996
|
});
|
|
1414
1997
|
return {
|
|
@@ -1435,11 +2018,11 @@ async function prepareRun(flags, configPath) {
|
|
|
1435
2018
|
async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
|
|
1436
2019
|
const startedAt = /* @__PURE__ */ new Date();
|
|
1437
2020
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
1438
|
-
const { worktreePatch } = await import("./state-
|
|
2021
|
+
const { worktreePatch } = await import("./state-DZbAO95a.js");
|
|
1439
2022
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
1440
2023
|
const who = await author(config.root);
|
|
1441
|
-
const { stagingDir, sweepStaging } = await import("./results-
|
|
1442
|
-
const { scratchDir } = await import("./files-
|
|
2024
|
+
const { stagingDir, sweepStaging } = await import("./results-CfC0id_u.js");
|
|
2025
|
+
const { scratchDir } = await import("./files-DFtuRQJx.js");
|
|
1443
2026
|
const scratch = scratchDir(config.root);
|
|
1444
2027
|
if (resume !== void 0) {
|
|
1445
2028
|
const target = await resolveResume(config.root, resume || void 0);
|
|
@@ -1450,7 +2033,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1450
2033
|
artifacts: target.previousArtifacts
|
|
1451
2034
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
1452
2035
|
if (!dryRun) try {
|
|
1453
|
-
const { removeRunContainers } = await import("./docker-
|
|
2036
|
+
const { removeRunContainers } = await import("./docker-BlSopHpZ.js");
|
|
1454
2037
|
await removeRunContainers(target.runId, ["docker", "podman"]);
|
|
1455
2038
|
} catch {}
|
|
1456
2039
|
return {
|
|
@@ -1497,11 +2080,33 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1497
2080
|
}
|
|
1498
2081
|
function noMatch(config, filters, tags) {
|
|
1499
2082
|
const narrowed = [filters.length ? `filter "${filters.join(" ")}"` : "", tags.length ? `tags ${tags.join(", ")}` : ""].filter(Boolean);
|
|
1500
|
-
return new UsageError(`No evals matched ${config.include.join(", ")}${narrowed.length ? ` with ${narrowed.join(" and ")}` : ""}`);
|
|
2083
|
+
return new UsageError(`No evals matched ${config.include.join(", ")}${narrowed.length ? ` with ${narrowed.join(" and ")}` : ""}${maskedFilterHint(filters)}`);
|
|
2084
|
+
}
|
|
2085
|
+
/**
|
|
2086
|
+
* Why a filter that holds a secret matches nothing: eval names are masked before they're
|
|
2087
|
+
* matched. Printed masked, it's the filter that would match.
|
|
2088
|
+
*/
|
|
2089
|
+
function maskedFilterHint(parts) {
|
|
2090
|
+
const masked = parts.filter((p) => redactSecrets(p) !== p);
|
|
2091
|
+
if (!masked.length) return "";
|
|
2092
|
+
return `. Eval names are masked, so a filter that holds a secret can't match one: filter by the masked name (${masked.map((p) => `"${redactSecrets(p)}"`).join(", ")}) or a part of it without the secret`;
|
|
1501
2093
|
}
|
|
1502
2094
|
function unmatchedFilter(config, parts) {
|
|
1503
2095
|
const which = parts.map((p) => `"${p}"`).join(", ");
|
|
1504
|
-
return new UsageError(`filter ${which} ${parts.length === 1 ? "matches" : "match"} no eval file, eval name or cell in ${config.include.join(", ")}`);
|
|
2096
|
+
return new UsageError(`filter ${which} ${parts.length === 1 ? "matches" : "match"} no eval file, eval name or cell in ${config.include.join(", ")}${maskedFilterHint(parts)}`);
|
|
2097
|
+
}
|
|
2098
|
+
/**
|
|
2099
|
+
* An eval whose id holds a value declared with `secret()`. `evaluate` masks names as the
|
|
2100
|
+
* worker's environment shows secrets, but a declared one is only known once the plan has the
|
|
2101
|
+
* environments, after the names became ids, so it's refused rather than written.
|
|
2102
|
+
*/
|
|
2103
|
+
function namesDeclaredSecret(evals) {
|
|
2104
|
+
for (const { id } of evals) {
|
|
2105
|
+
const [file, ...names] = id.split(" › ");
|
|
2106
|
+
const masked = names.map((n) => redactSecrets(n));
|
|
2107
|
+
if (masked.every((n, i) => n === names[i])) continue;
|
|
2108
|
+
return new UsageError(`${[file, ...masked].join(" › ")} names a secret declared with secret(): name it without the secret.`);
|
|
2109
|
+
}
|
|
1505
2110
|
}
|
|
1506
2111
|
/**
|
|
1507
2112
|
* Reads Cursor's model catalog for the planned Cursor cells, checks their `effort` and `params`
|
|
@@ -1511,12 +2116,12 @@ function unmatchedFilter(config, parts) {
|
|
|
1511
2116
|
*/
|
|
1512
2117
|
async function planEfforts(planned) {
|
|
1513
2118
|
const live = planned.filter((p) => !p.attempt.plan.skip);
|
|
1514
|
-
const { effortNotes, withDefaultEffort } = await import("./default-effort-
|
|
2119
|
+
const { effortNotes, withDefaultEffort } = await import("./default-effort-FEidNAIi.js");
|
|
1515
2120
|
let cursor;
|
|
1516
2121
|
let unchecked = "CURSOR_API_KEY isn't set";
|
|
1517
2122
|
const key = process.env.CURSOR_API_KEY;
|
|
1518
2123
|
if (key && live.some((p) => p.attempt.harness.kind === "cursor")) {
|
|
1519
|
-
const { checkCursorPlan } = await import("./cursor-catalog-
|
|
2124
|
+
const { checkCursorPlan } = await import("./cursor-catalog-BTK_apL-.js");
|
|
1520
2125
|
const checked = await checkCursorPlan(key, live);
|
|
1521
2126
|
if (checked instanceof Error) unchecked = `its request failed: ${checked.message}`;
|
|
1522
2127
|
else {
|
|
@@ -1536,7 +2141,7 @@ async function planEfforts(planned) {
|
|
|
1536
2141
|
* Collects the evals and prints the plan; stops there for `--dry-run`, or on a budget, credential
|
|
1537
2142
|
* or Docker problem (exit 2). Then runs the engine with live output and Ctrl-C handling.
|
|
1538
2143
|
*/
|
|
1539
|
-
async function executeRun(ctx, setup, filters, renderer) {
|
|
2144
|
+
async function executeRun(ctx, setup, filters, renderer, started) {
|
|
1540
2145
|
const { runId, stagingDir, config, runs, tags, concurrency, profile, verbosity, configFile } = ctx;
|
|
1541
2146
|
const run = JSON.parse(JSON.stringify({
|
|
1542
2147
|
runId,
|
|
@@ -1550,8 +2155,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1550
2155
|
configFile,
|
|
1551
2156
|
completed: []
|
|
1552
2157
|
}));
|
|
1553
|
-
const { runEngine } = await import("./node-
|
|
1554
|
-
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-
|
|
2158
|
+
const { runEngine } = await import("./node-DhIRSrrK.js");
|
|
2159
|
+
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-DqnzfDH1.js");
|
|
1555
2160
|
let plan;
|
|
1556
2161
|
let sandboxVersions = { images: [] };
|
|
1557
2162
|
let preflightStop;
|
|
@@ -1560,6 +2165,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1560
2165
|
let toRun = [];
|
|
1561
2166
|
let live;
|
|
1562
2167
|
let kept = /* @__PURE__ */ new Map();
|
|
2168
|
+
let leftOut = [];
|
|
1563
2169
|
let questions;
|
|
1564
2170
|
const onPlan = async (collected) => {
|
|
1565
2171
|
const { planned, excluded, matched, unmatched } = collected;
|
|
@@ -1580,9 +2186,11 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1580
2186
|
} : p);
|
|
1581
2187
|
toRun = skipped;
|
|
1582
2188
|
for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
|
|
2189
|
+
const named = namesDeclaredSecret([...planned, ...excluded]);
|
|
2190
|
+
if (named) throw named;
|
|
1583
2191
|
const efforts = await planEfforts(skipped);
|
|
1584
2192
|
if (efforts) run.cursorEfforts = efforts;
|
|
1585
|
-
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-
|
|
2193
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-CXVVB4RK.js") : void 0;
|
|
1586
2194
|
const judge = runJudgeModel(config, profile);
|
|
1587
2195
|
const judgeModel = judge === void 0 ? null : judgeModelName(judge);
|
|
1588
2196
|
plan = await buildPlan({
|
|
@@ -1601,13 +2209,20 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1601
2209
|
const cloud = ciRunner(process.env);
|
|
1602
2210
|
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
|
|
1603
2211
|
if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
|
|
2212
|
+
plan.warnings.push(...started.notes);
|
|
2213
|
+
if (started.narrowed) {
|
|
2214
|
+
leftOut = leftOutAttempts(planned, stagingDir, ctx.previous, await liveAttempts(stagingDir), new Set(excluded.map((e) => e.id)));
|
|
2215
|
+
const notRun = leftOut.filter((l) => !l.done).length;
|
|
2216
|
+
if (notRun) plan.warnings.push(`${notRun === 1 ? "1 attempt" : `${notRun} attempts`} run ${runId} planned ${notRun === 1 ? "isn't" : "aren't"} in this resume's filters: ${notRun === 1 ? "it stays" : "they stay"} not run and the run stays incomplete. reeval run --resume ${runId} runs ${notRun === 1 ? "it" : "them"}`);
|
|
2217
|
+
}
|
|
1604
2218
|
if (completed.size) plan.resume = {
|
|
1605
2219
|
runId,
|
|
1606
2220
|
completed: completed.size
|
|
1607
2221
|
};
|
|
1608
2222
|
run.completed = [...completed];
|
|
1609
|
-
|
|
1610
|
-
|
|
2223
|
+
const done = leftOut.filter((l) => l.done).map((l) => l.key);
|
|
2224
|
+
kept = keptAttempts([...completed, ...done], stagingDir, ctx.previous);
|
|
2225
|
+
const { imageStages, imageTargets } = await import("./images-CNayXCp2.js");
|
|
1611
2226
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1612
2227
|
id: p.id,
|
|
1613
2228
|
harness: p.attempt.harness,
|
|
@@ -1626,13 +2241,13 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1626
2241
|
return false;
|
|
1627
2242
|
}
|
|
1628
2243
|
const elsewhere = setup.configElsewhere ? configFile : void 0;
|
|
1629
|
-
const { scratchDir } = await import("./files-
|
|
2244
|
+
const { scratchDir } = await import("./files-DFtuRQJx.js");
|
|
1630
2245
|
const lines = [];
|
|
1631
2246
|
if (setup.dryRun) {
|
|
1632
2247
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
1633
2248
|
lines.push(...planTable(plan, todos));
|
|
1634
|
-
const { imageStatusLines } = await import("./images-
|
|
1635
|
-
const { dockerAvailable, engineSpec } = await import("./src-
|
|
2249
|
+
const { imageStatusLines } = await import("./images-CNayXCp2.js");
|
|
2250
|
+
const { dockerAvailable, engineSpec } = await import("./src-BkH4GOxu.js");
|
|
1636
2251
|
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
1637
2252
|
const down = [];
|
|
1638
2253
|
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
@@ -1641,7 +2256,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1641
2256
|
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
1642
2257
|
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
1643
2258
|
}
|
|
1644
|
-
const { costLine } = await import("./cost-
|
|
2259
|
+
const { costLine } = await import("./cost-BJU0MAji.js");
|
|
1645
2260
|
const cost = plan.cost && costLine(plan.cost);
|
|
1646
2261
|
if (cost) lines.push(cost);
|
|
1647
2262
|
}
|
|
@@ -1715,7 +2330,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1715
2330
|
});
|
|
1716
2331
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
1717
2332
|
if (asked.length) {
|
|
1718
|
-
const { questionRunner } = await import("./questions-
|
|
2333
|
+
const { questionRunner } = await import("./questions-BzdLdV3C.js");
|
|
1719
2334
|
const warned = /* @__PURE__ */ new Set();
|
|
1720
2335
|
questions = questionRunner({
|
|
1721
2336
|
evals: asked,
|
|
@@ -1757,7 +2372,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1757
2372
|
}
|
|
1758
2373
|
});
|
|
1759
2374
|
}
|
|
1760
|
-
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-
|
|
2375
|
+
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-B8EOCuxl.js");
|
|
1761
2376
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
1762
2377
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
1763
2378
|
workspace: a.workspace,
|
|
@@ -1861,7 +2476,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1861
2476
|
runId,
|
|
1862
2477
|
startedAt: ctx.startedAt,
|
|
1863
2478
|
planned: toRun,
|
|
1864
|
-
kept
|
|
2479
|
+
kept,
|
|
2480
|
+
leftOut,
|
|
2481
|
+
...started.startedWith ? { invocation: started.startedWith } : {}
|
|
1865
2482
|
});
|
|
1866
2483
|
renderer.onEvent({
|
|
1867
2484
|
type: "run-start",
|
|
@@ -1889,8 +2506,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1889
2506
|
onCancelable: (cancel) => {
|
|
1890
2507
|
let first;
|
|
1891
2508
|
const onSignal = (signal) => {
|
|
1892
|
-
if (first && signal !== first.signal && Date.now() - first.at < SAME_PRESS_MS) {
|
|
1893
|
-
if (
|
|
2509
|
+
if (first && !first.passedOn && signal !== first.signal && Date.now() - first.at < SAME_PRESS_MS) {
|
|
2510
|
+
if (signal === "SIGINT") stopCode = EXIT.interrupted;
|
|
2511
|
+
if (!finishing) renderer.onEvent({
|
|
1894
2512
|
type: "notice",
|
|
1895
2513
|
at: Date.now(),
|
|
1896
2514
|
level: "warn",
|
|
@@ -1932,9 +2550,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1932
2550
|
throw error;
|
|
1933
2551
|
} finally {
|
|
1934
2552
|
clearTimeout(budgetTimer);
|
|
1935
|
-
const { removePrepared } = await import("./preflight-
|
|
2553
|
+
const { removePrepared } = await import("./preflight-DVXEvzFU.js");
|
|
1936
2554
|
removePrepared(config.root, runId);
|
|
1937
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
2555
|
+
const { removePreparedWorkspaces } = await import("./workspace-B8EOCuxl.js");
|
|
1938
2556
|
removePreparedWorkspaces(config.root, runId);
|
|
1939
2557
|
}
|
|
1940
2558
|
if (preflightStop) throw preflightStop;
|
|
@@ -1948,7 +2566,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1948
2566
|
budgetReached: () => reached,
|
|
1949
2567
|
cutShort,
|
|
1950
2568
|
live,
|
|
1951
|
-
questions
|
|
2569
|
+
questions,
|
|
2570
|
+
leftOut
|
|
1952
2571
|
};
|
|
1953
2572
|
}
|
|
1954
2573
|
process.stdout.on("error", (error) => {
|
|
@@ -2016,4 +2635,4 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
2016
2635
|
});
|
|
2017
2636
|
export {};
|
|
2018
2637
|
|
|
2019
|
-
//# sourceMappingURL=cli-
|
|
2638
|
+
//# sourceMappingURL=cli-BBOlXKPG.js.map
|