reeval 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-CptK032x.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-B63OzUX9.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-B63OzUX9.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Me7Whp-y.js → assemble-t6h018An.js} +152 -129
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
- package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-Cik0SoIn.js → attempt-host-CIm1-11m.js} +6 -6
- package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-C3RmjPph.js → capture-iBzFiC0j.js} +27 -25
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-BBOlXKPG.js → cli-20Fpvs5H.js} +442 -707
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-qGS802uX.js → command-B5OyHpOu.js} +21 -80
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-DhiwsZl1.js → command-BLNVDiIg.js} +9 -9
- package/dist/{command-DhiwsZl1.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-fO0BFsTq.js → command-CRh-cb6Z.js} +37 -37
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-CFQ9c1hk.js → command-aUJvENwy.js} +16 -16
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-CC60TOT5.js → commands-CWy-WndG.js} +186 -85
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-xYTMNTPv.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-C2UtS9vy.js → config-B9mB1kF5.js} +261 -1708
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
- package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-C5SZ0510.js → db-zgC1tvKB.js} +130 -84
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-FEidNAIi.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BVmx7r7L.js → default-effort-DGhQmu95.js} +4 -4
- package/dist/{default-effort-BVmx7r7L.js.map → default-effort-DGhQmu95.js.map} +1 -1
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-B2F8-PGC.js → docker-Cqw-CyWz.js} +3 -3
- package/dist/{docker-B2F8-PGC.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker-Dto8ROCq.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-BlSopHpZ.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-BcvfKpBX.js → doctor-B2PohneB.js} +84 -27
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{environment-CC1RV21n.js → environment-DB8o3hGi.js} +79 -39
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-CzxdgFBW.js → eval-source-D3sbAzt-.js} +10 -10
- package/dist/{eval-source-CzxdgFBW.js.map → eval-source-D3sbAzt-.js.map} +1 -1
- package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
- package/dist/{evaluate-NTNJ8T9H.js → evaluate-CwKeN_fo.js} +377 -142
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-DsXULUi5.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-DsXULUi5.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-aAzoGNCB.js → files-BiFU16BW.js} +5 -5
- package/dist/{files-aAzoGNCB.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-CZXrkPja.js → files-Cl7BQ5gE.js} +54 -13
- package/dist/{files-CZXrkPja.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-C1MB5bjr.js → git-fDiJ8UIV.js} +2 -2
- package/dist/{git-C1MB5bjr.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
- package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
- package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-CNayXCp2.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-D5HtmyMI.js → init-DbuZVRrW.js} +7 -6
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
- package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
- package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
- package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
- package/dist/{live-uRURFDMZ.js → live-CnGrRM6j.js} +7 -7
- package/dist/{live-uRURFDMZ.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
- package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-CXVVB4RK.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-BxBWFOZn.js → model-DJjLw-0r.js} +5 -5
- package/dist/{model-BxBWFOZn.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-ug5HpMOj.js → patch-DH4e_Gkd.js} +2 -2
- package/dist/{patch-ug5HpMOj.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-DqnzfDH1.js → plan-Cgq9NCEX.js} +110 -39
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-B2KQ0y3P.js → plugins-BkmPxDiY.js} +10 -267
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-DyzmvK7i.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-DVXEvzFU.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-21WMQZyt.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-21WMQZyt.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-BcSRa7-3.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
- package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-iRxjp4Pu.js → provenance-zsc4--dP.js} +4 -3
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-vXMXuwn-.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-BhaKQ07P.js → query-Bb3SYICk.js} +2 -2
- package/dist/{query-BhaKQ07P.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-BzdLdV3C.js → questions-CWlR9ffu.js} +9 -9
- package/dist/{questions-BzdLdV3C.js.map → questions-CWlR9ffu.js.map} +1 -1
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-CauU408t.js → remarks-vtkPmQb9.js} +115 -35
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-Dwy3fyo3.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-Dwy3fyo3.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B1zMnPwK.js → resolve-Do0ZaYH6.js} +10 -9
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-CfC0id_u.js → results-JyuM64gy.js} +6 -6
- package/dist/{results-CfC0id_u.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/{resume-Dkhrpjow.js → resume-AW5oXzUc.js} +169 -27
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-DcvQcr36.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-8zCrtuCf.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-Dplq8g0r.js → server-Cn8VqO9S.js} +9 -9
- package/dist/{server-Dplq8g0r.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-NWHqn5-o.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-NWHqn5-o.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-B9oE_mNS.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-BQ6BiXoc.js → src-Ck2r_EMT.js} +607 -72
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-CUpoVVb8.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-DDCGUhva.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-DDCGUhva.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +4 -4
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
- package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
- package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
- package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +101 -58
- package/dist/view/index.js +9 -8
- package/dist/{views-BISrsIjw.js → views-DAIiLiV3.js} +7 -6
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-DJD_nHkS.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-BO1VhUR9.js → workspace-uuE51x35.js} +6 -6
- package/dist/{workspace-BO1VhUR9.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +24 -11
- package/skills/eval-rubric/SKILL.md +15 -7
- package/dist/adapter-CptK032x.js.map +0 -1
- package/dist/app-files-sgqwfjtn.js.map +0 -1
- package/dist/assemble-CxBHkavM.js +0 -1
- package/dist/assemble-Me7Whp-y.js.map +0 -1
- package/dist/capture-C3RmjPph.js.map +0 -1
- package/dist/capture-YPwxp_iK.js +0 -1
- package/dist/cli-BBOlXKPG.js.map +0 -1
- package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
- package/dist/client-tZ2WBDxf.js.map +0 -1
- package/dist/command-CFQ9c1hk.js.map +0 -1
- package/dist/command-fO0BFsTq.js.map +0 -1
- package/dist/command-qGS802uX.js.map +0 -1
- package/dist/commands-CC60TOT5.js.map +0 -1
- package/dist/config-C2UtS9vy.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-BjOg7wyr.js +0 -101
- package/dist/contrasts-BjOg7wyr.js.map +0 -1
- package/dist/cost-BJU0MAji.js +0 -1
- package/dist/db-C5SZ0510.js.map +0 -1
- package/dist/db-puzLzX4k.js +0 -2
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker-Dto8ROCq.js.map +0 -1
- package/dist/doctor-BcvfKpBX.js.map +0 -1
- package/dist/environment-CC1RV21n.js.map +0 -1
- package/dist/evaluate-NTNJ8T9H.js.map +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-DFtuRQJx.js +0 -1
- package/dist/files-oOYNR7ZH.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-Cp_pQoEe.js.map +0 -1
- package/dist/harness-DYSC4zMh.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-CNayXCp2.js.map +0 -1
- package/dist/init-8o6W3_fn.js +0 -1
- package/dist/init-D5HtmyMI.js.map +0 -1
- package/dist/ink-list-Ca-5dunO.js.map +0 -1
- package/dist/judge-DYOF69Oe.js.map +0 -1
- package/dist/list-CM_YSJXq.js +0 -374
- package/dist/list-CM_YSJXq.js.map +0 -1
- package/dist/live-DQc8jr6o.js +0 -1
- package/dist/model-Ce9l9_rF.js.map +0 -1
- package/dist/model-Cxzo4cjw.js +0 -1
- package/dist/node-DhIRSrrK.js.map +0 -1
- package/dist/opencode-providers-Co7plCea.js.map +0 -1
- package/dist/plan-DqnzfDH1.js.map +0 -1
- package/dist/plugins-B2KQ0y3P.js.map +0 -1
- package/dist/preflight-DyzmvK7i.js.map +0 -1
- package/dist/provenance-iRxjp4Pu.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-D0ob_tra.js +0 -1
- package/dist/providers-vXMXuwn-.js.map +0 -1
- package/dist/query-CvgUsvep.js +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/remarks-CauU408t.js.map +0 -1
- package/dist/remarks-NMgPd30U.js.map +0 -1
- package/dist/resolve-B1zMnPwK.js.map +0 -1
- package/dist/resolve-BIIeVT12.js +0 -1
- package/dist/resume-Dkhrpjow.js.map +0 -1
- package/dist/runtime-DcvQcr36.js.map +0 -1
- package/dist/sandbox-8zCrtuCf.js.map +0 -1
- package/dist/server-0tuj8VSa.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-llwLewOU.js +0 -1
- package/dist/src-BQ6BiXoc.js.map +0 -1
- package/dist/src-BXnv0ViU.js +0 -2
- package/dist/src-BiS8YNN_.js +0 -3
- package/dist/src-BkH4GOxu.js +0 -1
- package/dist/src-C-tH5ebd.js +0 -2
- package/dist/src-Cg1bFs9g.js +0 -2
- package/dist/src-DV_UorZv.js +0 -2
- package/dist/src-XxUpr2sa.js +0 -2
- package/dist/state-DZbAO95a.js +0 -1
- package/dist/suite-M8_7xio7.js.map +0 -1
- package/dist/suite-u2KJnGpY.js +0 -2
- package/dist/summary-Uzc9osfV.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-h_peaZTN.js +0 -1
- package/dist/views-BISrsIjw.js.map +0 -1
- package/dist/views-Bfab8fM9.js +0 -2
- package/dist/workspace-B8EOCuxl.js +0 -1
|
@@ -1,38 +1,34 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import {
|
|
3
|
-
import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-CC1RV21n.js";
|
|
4
|
-
import { C as pluginBuildsLabel } from "./plugins-B2KQ0y3P.js";
|
|
5
|
-
import { a as fromCwd, o as newRunId } from "./ids-DuK6xl4w.js";
|
|
6
|
-
import { $ as directOnCloud, M as harnessId, O as checkLimitHarness, Q as ciRunner, R as plannedSandboxIds, U as sandboxId, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, it as resolveOutput, l as hostCapacity, m as resolveConcurrency, nt as isDirect, p as requireConfig, u as loadConfig, v as gatewayScope, z as plannedSandboxes } from "./config-C2UtS9vy.js";
|
|
7
|
-
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-DYOF69Oe.js";
|
|
8
|
-
import { y as vertexSettingsIn } from "./vertex-BLQMayc1.js";
|
|
9
|
-
import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
|
|
2
|
+
import { g as parseDuration } from "./shared-Cfi45Bvy.js";
|
|
10
3
|
import { a as plural, i as formatDuration, n as clip } from "./text-B8-W3Cvn.js";
|
|
4
|
+
import { l as sandboxCapacity, n as capabilitiesOf, r as cleanupRun } from "./kinds-C1oY3qyC.js";
|
|
5
|
+
import { N as workspaceLabel, T as profileSandbox, d as environmentSandbox } from "./environment-DB8o3hGi.js";
|
|
6
|
+
import { a as newRunId, i as fromCwd } from "./ids-C_advdJA.js";
|
|
7
|
+
import { I as checkLimitHarness, V as harnessId, X as plannedSandboxes, Y as plannedSandboxIds, _t as resolveOutput, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, dt as ciRunner, f as missingCredentials, ft as directOnCloud, g as selectProfile, h as runJudgeModel, ht as isDirect, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig, v as gatewayScope } from "./config-B9mB1kF5.js";
|
|
8
|
+
import { y as vertexSettingsIn } from "./vertex-Eb2wqahk.js";
|
|
9
|
+
import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
|
|
10
|
+
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-COMXT1Hx.js";
|
|
11
11
|
import { n as loadEnvFiles, t as inCI } from "./env-8x1O8I1M.js";
|
|
12
12
|
import { o as PACKAGE_VERSION } from "./root-Co0jhOT5.js";
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-DZ0bnP0_.js";
|
|
13
|
+
import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
|
|
14
|
+
import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-DKnYkqeK.js";
|
|
15
|
+
import { k as runOutcome } from "./model-V3mMXp6x.js";
|
|
16
|
+
import { u as suiteResult } from "./suite-COglUhPd.js";
|
|
17
|
+
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-Dsr79Qy1.js";
|
|
19
18
|
import { n as stripAnsi, t as colors } from "./colors-DcXBvAlU.js";
|
|
20
|
-
import { i as
|
|
21
|
-
import { _ as statsLine, c as checkLabel } from "./summary-Uzc9osfV.js";
|
|
22
|
-
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-iRxjp4Pu.js";
|
|
19
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-zsc4--dP.js";
|
|
23
20
|
import { t as budgetMarker } from "./budget-BcxLYY5L.js";
|
|
24
|
-
import {
|
|
25
|
-
import { n as
|
|
26
|
-
import {
|
|
27
|
-
import { i as
|
|
28
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DrGXF9dT.js";
|
|
29
|
-
import { c as attemptAxes, d as failureLines, f as offset, g as tailLines, h as progressLine, l as debugLines, m as phaseOf, p as outcomeOf, r as listLines, s as progressText, t as LiveList, u as errorCheck } from "./list-CM_YSJXq.js";
|
|
21
|
+
import { S as writeLiveProgress, _ as resumeSettings, b as stoppedRetry, c as liveAttempts, d as prepareResume, f as previousAttempt, g as restoreRetry, h as resolveRetry, l as loadPrevious, m as resolveResume, n as erroredAttempts, o as keptAttempts, r as holdRetry, s as leftOutAttempts, t as completedKeysFor, u as noteRetries, v as savedFilters, x as writeCurrent, y as savedInvocation } from "./resume-AW5oXzUc.js";
|
|
22
|
+
import { n as preflight } from "./preflight-CtlloeOQ.js";
|
|
23
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CxGNOlh0.js";
|
|
24
|
+
import { _ as phaseOf, c as pointerLine, d as attemptAxes, f as debugLines, g as outcomeOf, h as offset, i as listLines, m as failureLines, n as SPINNER, p as errorCheck, s as progressText, t as LiveList, u as statusList, v as progressLine, y as tailLines } from "./list-DuThsE28.js";
|
|
30
25
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
31
26
|
import { dirname, join } from "node:path";
|
|
32
27
|
import { inspect } from "node:util";
|
|
33
28
|
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
34
29
|
import { detectAgent } from "std-env";
|
|
35
30
|
import { Command, Option } from "commander";
|
|
31
|
+
import stringWidth from "string-width";
|
|
36
32
|
// src/cli-parse.ts
|
|
37
33
|
const OPTIONS = Object.fromEntries(Object.entries(FLAGS).map(([name, spec]) => {
|
|
38
34
|
const short = "short" in spec ? `-${spec.short}, ` : "";
|
|
@@ -163,62 +159,6 @@ function usageError(error, command) {
|
|
|
163
159
|
}
|
|
164
160
|
return new UsageError(first.charAt(0).toUpperCase() + first.slice(1));
|
|
165
161
|
}
|
|
166
|
-
// src/preflight.ts
|
|
167
|
-
/**
|
|
168
|
-
* How the terminal names a sandbox to prepare: its cell id, `+ install`, `+ plugin builds`, and a
|
|
169
|
-
* harness other than Cursor.
|
|
170
|
-
*/
|
|
171
|
-
function prepareLabel({ sandbox, harness, plugins }) {
|
|
172
|
-
if (sandbox.kind !== "docker") return sandboxId(sandbox);
|
|
173
|
-
const baked = imageHarness(harness);
|
|
174
|
-
const install = sandbox.install?.length ? " + install" : "";
|
|
175
|
-
return `${sandboxId(sandbox)}${install}${pluginBuildsLabel(plugins)}${baked === "cursor" ? "" : ` · ${baked}`}`;
|
|
176
|
-
}
|
|
177
|
-
/** A run that stopped in preflight on an infra failure: exit 3. */
|
|
178
|
-
var PreflightInfraError = class extends Error {
|
|
179
|
-
name = "PreflightInfraError";
|
|
180
|
-
};
|
|
181
|
-
/**
|
|
182
|
-
* Before any eval starts: checks each sandbox provider, then prepares every distinct sandbox the
|
|
183
|
-
* planned attempts need (building Docker images), in parallel, and writes the results for the
|
|
184
|
-
* run's workers. A config-level sandbox that fails stops the run (exit 2 for a mistake in the
|
|
185
|
-
* config, exit 3 otherwise); any other failure fails only the attempts that need it.
|
|
186
|
-
*/
|
|
187
|
-
async function preflight(uses, options) {
|
|
188
|
-
const providers = options.providers ?? createProviders();
|
|
189
|
-
const targets = prepareTargets(uses, providers, prepareLabel);
|
|
190
|
-
if (!targets.some((t) => t.kind !== "fake")) return { images: [] };
|
|
191
|
-
let outcomes;
|
|
192
|
-
try {
|
|
193
|
-
outcomes = await prepareAll(targets, {
|
|
194
|
-
providers,
|
|
195
|
-
root: options.root,
|
|
196
|
-
...options.signal ? { signal: options.signal } : {},
|
|
197
|
-
...options.delays ? { delays: options.delays } : {},
|
|
198
|
-
...options.onEvent ? { onEvent: options.onEvent } : {},
|
|
199
|
-
log: (line) => process.env.REEVAL_DEBUG && console.error(line)
|
|
200
|
-
});
|
|
201
|
-
} catch (error) {
|
|
202
|
-
if (error instanceof PrepareError) throw error.failure.usage ? new UsageError(error.message) : new PreflightInfraError(error.message);
|
|
203
|
-
if (error instanceof RunStopError) throw error.outcome.failure.usage ? new UsageError(error.message) : new PreflightInfraError(error.message);
|
|
204
|
-
throw error;
|
|
205
|
-
}
|
|
206
|
-
writePrepared(options.root, options.runId, outcomes);
|
|
207
|
-
const images = [];
|
|
208
|
-
for (const o of outcomes) {
|
|
209
|
-
if (!o.ok || o.target.sandbox.kind !== "docker") continue;
|
|
210
|
-
const { ref, id, agentCli } = attemptImage(o.handle);
|
|
211
|
-
if (images.some((i) => i.id === id)) continue;
|
|
212
|
-
const info = await imageInfo(ref, o.target.sandbox.engine);
|
|
213
|
-
images.push({
|
|
214
|
-
ref,
|
|
215
|
-
id,
|
|
216
|
-
repoDigests: info?.repoDigests ?? [],
|
|
217
|
-
agentCli: agentCli ?? info?.labels[LABEL$1.agentVersion] ?? null
|
|
218
|
-
});
|
|
219
|
-
}
|
|
220
|
-
return { images };
|
|
221
|
-
}
|
|
222
162
|
// src/ui/header.ts
|
|
223
163
|
/** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
|
|
224
164
|
const GLYPH = 5;
|
|
@@ -254,7 +194,8 @@ function headerLines(c, h, width) {
|
|
|
254
194
|
return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
|
|
255
195
|
};
|
|
256
196
|
const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
|
|
257
|
-
const
|
|
197
|
+
const continued = [h.resumed ? `${plural(h.resumed, "attempt")} done` : "", h.retried ? `${plural(h.retried, "error")} to retry` : ""].filter(Boolean);
|
|
198
|
+
const resumed = continued.length ? c.dim(` (resumed, ${continued.join(", ")})`) : "";
|
|
258
199
|
const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
|
|
259
200
|
return [
|
|
260
201
|
h.dryRun ? `${title} ${c.dim("dry run")}` : title,
|
|
@@ -289,9 +230,11 @@ function scope(c, h) {
|
|
|
289
230
|
}
|
|
290
231
|
function estimate(c, h) {
|
|
291
232
|
const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
|
|
292
|
-
const
|
|
233
|
+
const most = Math.min(h.concurrency, h.attempts);
|
|
234
|
+
const parallel = h.parallel ?? most;
|
|
293
235
|
if (parallel <= 1) return serial;
|
|
294
|
-
|
|
236
|
+
const capped = parallel < most ? `, ${parallel} at once` : "";
|
|
237
|
+
return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency}${capped})`)}`} · ${serial}`;
|
|
295
238
|
}
|
|
296
239
|
/** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
|
|
297
240
|
function noteLines(c, kind, text, width) {
|
|
@@ -342,15 +285,17 @@ function prepareStepLines(c, e, width, max) {
|
|
|
342
285
|
const who = outcome.target.setBy.join(", ");
|
|
343
286
|
return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
|
|
344
287
|
}
|
|
288
|
+
/** The start line's text: how the attempts' logs will print. */
|
|
289
|
+
function startText(output) {
|
|
290
|
+
return output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
|
|
291
|
+
}
|
|
345
292
|
/** The last line before the attempts' own: how their logs will print. */
|
|
346
293
|
function startLine(c, output) {
|
|
347
|
-
|
|
348
|
-
return ` ${c.teal("◆")} ${text}`;
|
|
294
|
+
return ` ${c.teal("◆")} ${startText(output)}`;
|
|
349
295
|
}
|
|
350
|
-
/** What the start line becomes once the run ends, on a terminal that redraws it. */
|
|
351
|
-
function
|
|
352
|
-
|
|
353
|
-
return ` ${c.teal("◆")} ${text}`;
|
|
296
|
+
/** What the start line's text becomes once the run ends, on a terminal that redraws it. */
|
|
297
|
+
function endText(status) {
|
|
298
|
+
return status === "interrupted" ? "Canceled." : status === "budget" ? "Stopped at the budget." : "Finished.";
|
|
354
299
|
}
|
|
355
300
|
/** The plan as the header block: its facts, `--dry-run`'s table, then each kind of note. */
|
|
356
301
|
function planLines(c, e, width) {
|
|
@@ -364,509 +309,6 @@ function planLines(c, e, width) {
|
|
|
364
309
|
out.push("");
|
|
365
310
|
return out;
|
|
366
311
|
}
|
|
367
|
-
// src/ui/status.ts
|
|
368
|
-
const EFFECT = {
|
|
369
|
-
better: "helped",
|
|
370
|
-
worse: "made it worse",
|
|
371
|
-
same: "same"
|
|
372
|
-
};
|
|
373
|
-
const ERROR_VERB = {
|
|
374
|
-
assertion: "failed",
|
|
375
|
-
timeout: "timed out",
|
|
376
|
-
aborted: "canceled",
|
|
377
|
-
infra: "hit an infrastructure error",
|
|
378
|
-
judge: "could not be judged",
|
|
379
|
-
turn: "ended a turn with an error"
|
|
380
|
-
};
|
|
381
|
-
const MESSAGE_LINES = 4;
|
|
382
|
-
const LINE_MAX = 160;
|
|
383
|
-
const times = (n) => n === 1 ? "once" : `${n} times`;
|
|
384
|
-
/** `cursor (composer-2.5)`, from the record's cell. */
|
|
385
|
-
function harnessLabel(record, cell) {
|
|
386
|
-
const model = record.evals.flatMap((e) => e.cells).find((c) => c.key === cell.key && c.harness === cell.harness)?.model;
|
|
387
|
-
return model && model !== cell.harness ? `${cell.harness} (${model})` : cell.harness;
|
|
388
|
-
}
|
|
389
|
-
/** The cells an eval's outcome reads: the gated ones, or every cell of a study. */
|
|
390
|
-
function deciding(cells) {
|
|
391
|
-
const gated = cells.filter((c) => c.gated);
|
|
392
|
-
return gated.length ? gated : cells;
|
|
393
|
-
}
|
|
394
|
-
const cellsOf = (e) => e.cells ?? (e.cases ?? []).flatMap((k) => k.cells);
|
|
395
|
-
/** Each part's label, adding the environment and sandbox only where the cells differ in them. */
|
|
396
|
-
function labeler(record, cells) {
|
|
397
|
-
const envs = new Set(cells.map((c) => c.environment)).size > 1;
|
|
398
|
-
const sandboxes = new Set(cells.map((c) => c.sandbox)).size > 1;
|
|
399
|
-
return (c) => [
|
|
400
|
-
harnessLabel(record, c),
|
|
401
|
-
envs ? `in ${c.environment}` : "",
|
|
402
|
-
sandboxes ? `on ${c.sandbox}` : ""
|
|
403
|
-
].filter(Boolean).join(" ");
|
|
404
|
-
}
|
|
405
|
-
function tallyBy(cells, label) {
|
|
406
|
-
const by = /* @__PURE__ */ new Map();
|
|
407
|
-
for (const c of cells) {
|
|
408
|
-
const key = label(c);
|
|
409
|
-
const t = by.get(key) ?? {
|
|
410
|
-
label: key,
|
|
411
|
-
passed: 0,
|
|
412
|
-
counted: 0
|
|
413
|
-
};
|
|
414
|
-
t.passed += c.passed;
|
|
415
|
-
t.counted += countedOf(c);
|
|
416
|
-
by.set(key, t);
|
|
417
|
-
}
|
|
418
|
-
return [...by.values()];
|
|
419
|
-
}
|
|
420
|
-
const of = (t) => `${t.passed} of ${t.counted}`;
|
|
421
|
-
/** The outcome sentence, per the rules at the top. */
|
|
422
|
-
function outcomeSentence(tallies) {
|
|
423
|
-
const counted = tallies.filter((t) => t.counted > 0);
|
|
424
|
-
if (!counted.length) return "No run finished.";
|
|
425
|
-
const [one] = counted;
|
|
426
|
-
if (counted.length === 1 && one) {
|
|
427
|
-
const once = one.counted === 1;
|
|
428
|
-
if (one.passed === one.counted) return `Passed on ${one.label}${once ? "" : " every time"}.`;
|
|
429
|
-
if (one.passed === 0) return `Failed on ${one.label}${once ? "" : " every time"}.`;
|
|
430
|
-
return `Passed ${of(one)} on ${one.label}.`;
|
|
431
|
-
}
|
|
432
|
-
const names = counted.map((t) => t.label);
|
|
433
|
-
const all = `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`;
|
|
434
|
-
if (counted.every((t) => t.passed === t.counted)) return `Passed on ${all} every time.`;
|
|
435
|
-
if (counted.every((t) => t.passed === 0)) return `Failed on ${all} every time.`;
|
|
436
|
-
const [first, ...rest] = counted;
|
|
437
|
-
return `${first?.label} passed ${first ? of(first) : ""}${rest.map((t) => `, ${t.label} ${of(t)}`).join("")}.`;
|
|
438
|
-
}
|
|
439
|
-
/** `(cursor 4 of 4, codex 1 of 4)` when the harnesses' pass rates differ; empty when they agree. */
|
|
440
|
-
function disagreement(record, cells) {
|
|
441
|
-
const tallies = tallyBy(cells, (c) => harnessLabel(record, c)).filter((t) => t.counted > 0);
|
|
442
|
-
if (tallies.length < 2) return "";
|
|
443
|
-
return new Set(tallies.map((t) => t.passed / t.counted)).size > 1 ? ` (${tallies.map((t) => `${t.label} ${of(t)}`).join(", ")})` : "";
|
|
444
|
-
}
|
|
445
|
-
/** `, not by minLift 60` when the variant's lift fell short of its `minLift`, which fails it. */
|
|
446
|
-
function minLiftShort(e, own) {
|
|
447
|
-
const keys = new Set(own.map((c) => c.key));
|
|
448
|
-
const pooled = (e.pooled ?? []).filter((p) => keys.has(p.key));
|
|
449
|
-
const missed = pooled.length ? pooled.find((p) => p.minLift !== null && p.outcome === "fail")?.minLift : own.find((c) => c.minLift !== null && c.interval && c.interval.low < c.minLift)?.minLift;
|
|
450
|
-
return missed == null ? "" : `, not by minLift ${Math.round(missed * 100)}`;
|
|
451
|
-
}
|
|
452
|
-
/**
|
|
453
|
-
* Under a variant, its comparison with the baseline (`measuresLine`): `Same pass rate · 5× faster ·
|
|
454
|
-
* judge "tidy" +0.12.`, one line, or one per harness (`cursor: …`) when it ran on several. Only
|
|
455
|
-
* when some measure besides pass rate holds; `verbose` adds the ones that only lean.
|
|
456
|
-
*/
|
|
457
|
-
function measureLines(e, own, verbose) {
|
|
458
|
-
const keys = new Set(own.map((c) => c.key));
|
|
459
|
-
const pooled = (e.pooled ?? []).filter((p) => keys.has(p.key));
|
|
460
|
-
const sets = pooled.length ? pooled.map((p) => ({
|
|
461
|
-
harness: own.find((c) => c.key === p.key)?.harness ?? p.key,
|
|
462
|
-
measures: p.measures
|
|
463
|
-
})) : own.flatMap((c) => c.measures ? [{
|
|
464
|
-
harness: c.harness,
|
|
465
|
-
measures: c.measures
|
|
466
|
-
}] : []);
|
|
467
|
-
const several = new Set(sets.map((x) => x.harness)).size > 1;
|
|
468
|
-
return sets.flatMap(({ harness, measures }) => {
|
|
469
|
-
if (!measures.filter((m) => m.measure !== "pass" && (m.verdict === "better" || m.verdict === "worse" || verbose && m.verdict === "leans")).length) return [];
|
|
470
|
-
const line = measuresLine(measures, verbose);
|
|
471
|
-
const text = `${line.slice(0, 1).toUpperCase()}${line.slice(1)}.`;
|
|
472
|
-
return [{
|
|
473
|
-
depth: 2,
|
|
474
|
-
text: several ? `${harness}: ${text}` : text,
|
|
475
|
-
tone: "dim"
|
|
476
|
-
}];
|
|
477
|
-
});
|
|
478
|
-
}
|
|
479
|
-
function compareLines(record, e, cells, verbose) {
|
|
480
|
-
const base = cells.filter((c) => c.baseline);
|
|
481
|
-
const name = (c) => c.environment === "-" ? "no environment" : c.environment;
|
|
482
|
-
const baseTally = tallyBy(base, () => "baseline")[0] ?? {
|
|
483
|
-
passed: 0,
|
|
484
|
-
counted: 0
|
|
485
|
-
};
|
|
486
|
-
const lines = [{
|
|
487
|
-
depth: 0,
|
|
488
|
-
text: `Baseline ${[...new Set(base.map(name))].join(", ")} passed ${of(baseTally)}${disagreement(record, base)}.`
|
|
489
|
-
}];
|
|
490
|
-
const variants = [...new Set(cells.filter((c) => c.variant !== null).map((c) => c.variant))];
|
|
491
|
-
if (!variants.length) return lines;
|
|
492
|
-
const rows = variants.map((v, order) => {
|
|
493
|
-
const own = cells.filter((c) => c.variant === v);
|
|
494
|
-
const t = tallyBy(own, () => v ?? "")[0] ?? {
|
|
495
|
-
passed: 0,
|
|
496
|
-
counted: 0
|
|
497
|
-
};
|
|
498
|
-
const effect = effectOf({
|
|
499
|
-
pass: t.passed,
|
|
500
|
-
counted: t.counted
|
|
501
|
-
}, {
|
|
502
|
-
pass: baseTally.passed,
|
|
503
|
-
counted: baseTally.counted
|
|
504
|
-
});
|
|
505
|
-
return {
|
|
506
|
-
v: v ?? "",
|
|
507
|
-
t,
|
|
508
|
-
effect,
|
|
509
|
-
own,
|
|
510
|
-
order
|
|
511
|
-
};
|
|
512
|
-
});
|
|
513
|
-
const rate = (t) => t.counted ? t.passed / t.counted : -1;
|
|
514
|
-
rows.sort((a, b) => rate(b.t) - rate(a.t) || a.order - b.order);
|
|
515
|
-
const width = Math.max(...rows.map((r) => r.v.length));
|
|
516
|
-
lines.push({
|
|
517
|
-
depth: 0,
|
|
518
|
-
text: "Variants, by pass rate:",
|
|
519
|
-
tone: "heading"
|
|
520
|
-
});
|
|
521
|
-
for (const r of rows) {
|
|
522
|
-
const short = minLiftShort(e, r.own);
|
|
523
|
-
const word = r.effect ? ` ${EFFECT[r.effect]}${short}` : short ? ` ${short.slice(2)}` : "";
|
|
524
|
-
lines.push({
|
|
525
|
-
depth: 1,
|
|
526
|
-
text: `${r.v.padEnd(width)} ${of(r.t)}${word}${disagreement(record, r.own)}`
|
|
527
|
-
});
|
|
528
|
-
lines.push(...measureLines(e, r.own, verbose));
|
|
529
|
-
if (!verbose) continue;
|
|
530
|
-
for (const cell of r.own) {
|
|
531
|
-
const stats = statsLine(cell, cells);
|
|
532
|
-
const who = r.own.length > 1 ? `${harnessLabel(record, cell)}: ` : "";
|
|
533
|
-
if (stats) lines.push({
|
|
534
|
-
depth: 2,
|
|
535
|
-
text: `${who}${stats}`,
|
|
536
|
-
tone: "dim"
|
|
537
|
-
});
|
|
538
|
-
}
|
|
539
|
-
}
|
|
540
|
-
return lines;
|
|
541
|
-
}
|
|
542
|
-
function failedChecks(cells, groupOf) {
|
|
543
|
-
const by = /* @__PURE__ */ new Map();
|
|
544
|
-
for (const cell of cells) for (const k of cell.checks) {
|
|
545
|
-
if (k.outcome.status !== "ran" || k.outcome.failed + k.outcome.errored === 0) continue;
|
|
546
|
-
const n = k.outcome.failed + k.outcome.errored;
|
|
547
|
-
const f = by.get(k.label) ?? {
|
|
548
|
-
label: k.label,
|
|
549
|
-
failed: 0,
|
|
550
|
-
byHarness: /* @__PURE__ */ new Map(),
|
|
551
|
-
first: k
|
|
552
|
-
};
|
|
553
|
-
f.failed += n;
|
|
554
|
-
const h = groupOf(cell);
|
|
555
|
-
f.byHarness.set(h, (f.byHarness.get(h) ?? 0) + n);
|
|
556
|
-
if (f.first.outcome.status === "ran" && !f.first.outcome.assertion) f.first = k;
|
|
557
|
-
by.set(k.label, f);
|
|
558
|
-
}
|
|
559
|
-
return [...by.values()].sort((a, b) => b.failed - a.failed || a.first.order - b.first.order);
|
|
560
|
-
}
|
|
561
|
-
/** `, all on H` or `, M of them on H`, per the rules at the top. */
|
|
562
|
-
function where(f, harnesses) {
|
|
563
|
-
if (harnesses < 2) return "";
|
|
564
|
-
const [top] = [...f.byHarness].sort((a, b) => b[1] - a[1]);
|
|
565
|
-
if (!top) return "";
|
|
566
|
-
const [h, n] = top;
|
|
567
|
-
if (n === f.failed) return f.failed === 1 ? `, on ${h}` : `, all on ${h}`;
|
|
568
|
-
return n * 2 > f.failed ? `, ${n} of them on ${h}` : "";
|
|
569
|
-
}
|
|
570
|
-
/** A message's first lines, without the label it repeats or a command's empty output headings. */
|
|
571
|
-
function messageLines(text, label = "", max = LINE_MAX) {
|
|
572
|
-
return (label && text.startsWith(`${label}: `) ? text.slice(label.length + 2) : text).split("\n").map((l) => l.trimEnd()).filter((l) => l.trim() && !/^--- (stderr|stdout) ---$/.test(l.trim())).slice(0, MESSAGE_LINES).map((l) => clip(l, max));
|
|
573
|
-
}
|
|
574
|
-
/** A failure's message is red only on a failed eval: a passing compare's baseline misses are expected. */
|
|
575
|
-
function checkLines(cells, groupOf, tone) {
|
|
576
|
-
const failed = failedChecks(cells, groupOf);
|
|
577
|
-
if (!failed.length) return [];
|
|
578
|
-
const harnesses = new Set(cells.map(groupOf)).size;
|
|
579
|
-
const lines = [{
|
|
580
|
-
depth: 0,
|
|
581
|
-
text: failed.length === 1 ? "Every failure was the same check:" : `${failed.length} checks failed:`,
|
|
582
|
-
tone: "heading"
|
|
583
|
-
}];
|
|
584
|
-
for (const f of failed) {
|
|
585
|
-
lines.push({
|
|
586
|
-
depth: 1,
|
|
587
|
-
text: `"${checkLabel(f.label)}" failed ${times(f.failed)}${where(f, harnesses)}`
|
|
588
|
-
});
|
|
589
|
-
const o = f.first.outcome;
|
|
590
|
-
const judge = f.first.judge;
|
|
591
|
-
const message = o.status === "ran" && o.assertion ? o.assertion : judge ? `judge: ${judge.question} (scores ${judge.scores.join(", ")}, threshold ${judge.threshold})` : "";
|
|
592
|
-
for (const text of messageLines(message, f.label)) lines.push({
|
|
593
|
-
depth: 2,
|
|
594
|
-
text,
|
|
595
|
-
tone
|
|
596
|
-
});
|
|
597
|
-
if (o.status === "ran" && o.at) lines.push({
|
|
598
|
-
depth: 2,
|
|
599
|
-
text: `at ${o.at}`,
|
|
600
|
-
tone: "dim"
|
|
601
|
-
});
|
|
602
|
-
}
|
|
603
|
-
return lines;
|
|
604
|
-
}
|
|
605
|
-
const CASES_NAMED = 8;
|
|
606
|
-
function failedCaseLines(e) {
|
|
607
|
-
const failed = (e.cases ?? []).filter((k) => k.outcome !== "pass" && k.outcome !== "skipped");
|
|
608
|
-
if (!failed.length) return [];
|
|
609
|
-
const named = failed.slice(0, CASES_NAMED).map((k) => k.label);
|
|
610
|
-
const more = failed.length > CASES_NAMED ? ` and ${failed.length - CASES_NAMED} more` : "";
|
|
611
|
-
return [{
|
|
612
|
-
depth: 0,
|
|
613
|
-
text: `Failed cases: ${named.join(", ")}${more}.`
|
|
614
|
-
}];
|
|
615
|
-
}
|
|
616
|
-
/** Errors no failed check explains: timeouts, infrastructure errors, throws. Never clipped: their remedy is often last. */
|
|
617
|
-
function errorLines(cells, tone) {
|
|
618
|
-
const by = /* @__PURE__ */ new Map();
|
|
619
|
-
for (const c of cells) for (const err of c.errors) {
|
|
620
|
-
const key = `${err.kind}\n${err.message}`;
|
|
621
|
-
const e = by.get(key) ?? {
|
|
622
|
-
kind: err.kind,
|
|
623
|
-
message: err.message,
|
|
624
|
-
n: 0
|
|
625
|
-
};
|
|
626
|
-
e.n += err.attempts;
|
|
627
|
-
by.set(key, e);
|
|
628
|
-
}
|
|
629
|
-
return [...by.values()].flatMap((e) => [{
|
|
630
|
-
depth: 0,
|
|
631
|
-
text: `${plural(e.n, "run")} ${ERROR_VERB[e.kind]}:`,
|
|
632
|
-
tone: "heading"
|
|
633
|
-
}, ...messageLines(e.message, "", Number.POSITIVE_INFINITY).map((text) => ({
|
|
634
|
-
depth: 1,
|
|
635
|
-
text,
|
|
636
|
-
tone
|
|
637
|
-
}))]);
|
|
638
|
-
}
|
|
639
|
-
/** `evals/a.eval.ts:12` from `source:evals/a.eval.ts#L12`. */
|
|
640
|
-
const location = (at) => at.replace(/^source:/, "").replace(/#L(\d+)$/, ":$1");
|
|
641
|
-
/** `"npm test" passed on try 2 of 3.`: a check that needed its `c.exec` retries is flaky. */
|
|
642
|
-
function retriedLines(cells) {
|
|
643
|
-
const seen = /* @__PURE__ */ new Set();
|
|
644
|
-
return cells.flatMap((c) => c.checks.flatMap((k) => {
|
|
645
|
-
if (!k.retried || seen.has(k.label)) return [];
|
|
646
|
-
seen.add(k.label);
|
|
647
|
-
return [`"${checkLabel(k.label)}" ${k.retried}.`];
|
|
648
|
-
}));
|
|
649
|
-
}
|
|
650
|
-
function pageErrorNotes(cells) {
|
|
651
|
-
const [first, ...rest] = cells.flatMap((c) => c.pageErrors ?? []);
|
|
652
|
-
if (!first) return [];
|
|
653
|
-
const more = rest.length ? ` (+${plural(rest.length, "more capture")})` : "";
|
|
654
|
-
return [`${plural(first.errors, "page error")} in capture "${first.capture}" → ${first.file}${more}`];
|
|
655
|
-
}
|
|
656
|
-
function remarkLines(remarks, notes) {
|
|
657
|
-
if (!remarks.length && !notes.length) return [];
|
|
658
|
-
return [
|
|
659
|
-
{
|
|
660
|
-
depth: 0,
|
|
661
|
-
text: "Worth knowing:",
|
|
662
|
-
tone: "heading"
|
|
663
|
-
},
|
|
664
|
-
...remarks.map((r) => ({
|
|
665
|
-
depth: 1,
|
|
666
|
-
text: r.fix.edit ? `${r.text} Fix at ${location(r.fix.at)}: ${r.fix.edit}.` : `${r.text} At ${location(r.fix.at)}.`
|
|
667
|
-
})),
|
|
668
|
-
...notes.map((text) => ({
|
|
669
|
-
depth: 1,
|
|
670
|
-
text
|
|
671
|
-
}))
|
|
672
|
-
];
|
|
673
|
-
}
|
|
674
|
-
/** A compare's failures are told apart by arm: `the baseline`, or the variant's name. */
|
|
675
|
-
const armOf = (c) => c.baseline ? "the baseline" : c.variant ?? c.environment;
|
|
676
|
-
const nameOf = (e) => [...e.group, e.template ?? e.name].join(" › ");
|
|
677
|
-
function iconOf(e) {
|
|
678
|
-
if (e.todo || e.notRun) return "skipped";
|
|
679
|
-
if (e.outcome === "skipped") return "skipped";
|
|
680
|
-
if (e.outcome === "canceled") return "canceled";
|
|
681
|
-
if (e.outcome === "pass") return isNotGatedResult(e) ? "observe" : "pass";
|
|
682
|
-
return "fail";
|
|
683
|
-
}
|
|
684
|
-
function evalRow(record, e, remarks, verbose = false) {
|
|
685
|
-
const cells = cellsOf(e);
|
|
686
|
-
const decide = deciding(cells);
|
|
687
|
-
const passed = decide.reduce((n, c) => n + c.passed, 0);
|
|
688
|
-
const ran = decide.reduce((n, c) => n + c.passed + c.failed + c.infraError, 0);
|
|
689
|
-
const compare = cells.some((c) => c.baseline);
|
|
690
|
-
const counted = compare && decide.length < cells.length ? "variant run" : "run";
|
|
691
|
-
const summary = e.todo ? "todo" : e.notRun ? "not run" : ran === 0 ? e.outcome === "canceled" ? "canceled" : "skipped" : `${passed} of ${plural(ran, counted)} passed`;
|
|
692
|
-
const tone = iconOf(e) === "fail" ? "red" : "dim";
|
|
693
|
-
const notes = ran ? [...retriedLines(decide), ...pageErrorNotes(decide)] : [];
|
|
694
|
-
const detail = ran ? [
|
|
695
|
-
...compare ? compareLines(record, e, cells, verbose) : [{
|
|
696
|
-
depth: 0,
|
|
697
|
-
text: outcomeSentence(tallyBy(decide, labeler(record, decide)))
|
|
698
|
-
}],
|
|
699
|
-
...failedCaseLines(e),
|
|
700
|
-
...compare ? checkLines(cells, armOf, tone) : checkLines(decide, (c) => harnessLabel(record, c), tone),
|
|
701
|
-
...errorLines(compare ? cells : decide, tone),
|
|
702
|
-
...iconOf(e) === "fail" ? snapshotLines(cells) : [],
|
|
703
|
-
...questionLines(e.questions ?? []),
|
|
704
|
-
...observedLines(e.observations ?? [], cells, (c) => compare ? armOf(c) : harnessLabel(record, c)),
|
|
705
|
-
...remarkLines(remarks, notes)
|
|
706
|
-
] : remarkLines(remarks, []);
|
|
707
|
-
return {
|
|
708
|
-
id: e.id,
|
|
709
|
-
name: nameOf(e),
|
|
710
|
-
icon: iconOf(e),
|
|
711
|
-
summary,
|
|
712
|
-
detail,
|
|
713
|
-
remarks: remarks.length + notes.length,
|
|
714
|
-
compare
|
|
715
|
-
};
|
|
716
|
-
}
|
|
717
|
-
/** Free-text answers shown per question; the rest are in the record. */
|
|
718
|
-
const ANSWERS_SHOWN = 3;
|
|
719
|
-
/** Rows of `[label, text]` at `depth`, the labels padded to one column. */
|
|
720
|
-
const aligned = (depth, rows) => {
|
|
721
|
-
const w = Math.max(0, ...rows.map(([label]) => label.length));
|
|
722
|
-
return rows.map(([label, text]) => ({
|
|
723
|
-
depth,
|
|
724
|
-
text: `${label.padEnd(w)} ${text}`
|
|
725
|
-
}));
|
|
726
|
-
};
|
|
727
|
-
/** Each question's answer: who won the pairs, the tally of choices, or the first answers. */
|
|
728
|
-
function questionLines(questions) {
|
|
729
|
-
return questions.flatMap((q) => {
|
|
730
|
-
const head = {
|
|
731
|
-
depth: 0,
|
|
732
|
-
text: `Question: ${clip(q.question, 100)}`,
|
|
733
|
-
tone: "heading"
|
|
734
|
-
};
|
|
735
|
-
if (q.kind === "attempts") {
|
|
736
|
-
const failed = q.error === null ? [] : [{
|
|
737
|
-
depth: 1,
|
|
738
|
-
text: `${q.failed} of ${q.asked} not answered: ${clip(q.error, 200)}`,
|
|
739
|
-
tone: "red"
|
|
740
|
-
}];
|
|
741
|
-
if (q.choices) return [
|
|
742
|
-
head,
|
|
743
|
-
{
|
|
744
|
-
depth: 1,
|
|
745
|
-
text: q.choices.map((t) => `${t.choice} ${t.n}`).join(" · ")
|
|
746
|
-
},
|
|
747
|
-
...failed
|
|
748
|
-
];
|
|
749
|
-
const shown = q.answers.slice(0, ANSWERS_SHOWN);
|
|
750
|
-
const more = q.answers.length - shown.length;
|
|
751
|
-
return [
|
|
752
|
-
head,
|
|
753
|
-
...aligned(1, shown.map((a) => [a.attempt, clip(a.text.replace(/\s+/g, " "), 100)])),
|
|
754
|
-
...more ? [{
|
|
755
|
-
depth: 1,
|
|
756
|
-
text: `${more} more: reeval query attempt_answers`,
|
|
757
|
-
tone: "dim"
|
|
758
|
-
}] : [],
|
|
759
|
-
...failed
|
|
760
|
-
];
|
|
761
|
-
}
|
|
762
|
-
if (q.votes === 0 && q.error === null) return [head, {
|
|
763
|
-
depth: 1,
|
|
764
|
-
text: "No pairs to judge: tries that failed their checks are left out.",
|
|
765
|
-
tone: "dim"
|
|
766
|
-
}];
|
|
767
|
-
if (q.rates.length === 0 && q.error === null) return [head, {
|
|
768
|
-
depth: 1,
|
|
769
|
-
text: `${plural(q.votes, "vote")}, every one a tie.`,
|
|
770
|
-
tone: "dim"
|
|
771
|
-
}];
|
|
772
|
-
const cuts = new Set(q.rates.map((r) => r.cut)).size > 1;
|
|
773
|
-
return [
|
|
774
|
-
head,
|
|
775
|
-
...aligned(1, q.rates.map((r) => [cuts ? `${r.name} (${r.cut})` : r.name, `won ${r.wins} of ${r.n}`])),
|
|
776
|
-
...q.error === null ? [] : [{
|
|
777
|
-
depth: 1,
|
|
778
|
-
text: `The model call failed: ${clip(q.error, 200)}`,
|
|
779
|
-
tone: "red"
|
|
780
|
-
}]
|
|
781
|
-
];
|
|
782
|
-
});
|
|
783
|
-
}
|
|
784
|
-
/** Where a failed eval's first failed attempt's final workspace snapshot is, to open it. */
|
|
785
|
-
function snapshotLines(cells) {
|
|
786
|
-
const [first, ...others] = cells.flatMap((c) => c.snapshots ?? []);
|
|
787
|
-
if (!first) return [];
|
|
788
|
-
return [{
|
|
789
|
-
depth: 0,
|
|
790
|
-
text: `Snapshot: ${first}${others.length ? ` (+${others.length} more in summary.md)` : ""}`,
|
|
791
|
-
tone: "dim"
|
|
792
|
-
}];
|
|
793
|
-
}
|
|
794
|
-
/** What `c.observe` recorded, each name's tally on one line, or a line per cell under it. */
|
|
795
|
-
function observedLines(tallies, cells, label) {
|
|
796
|
-
if (!tallies.length) return [];
|
|
797
|
-
const byKey = new Map(cells.map((c) => [c.key, c]));
|
|
798
|
-
const perCell = new Set(tallies.map((t) => t.cell)).size > 1;
|
|
799
|
-
const width = Math.max(...tallies.map((t) => clip(t.name, 40).length));
|
|
800
|
-
const lines = [{
|
|
801
|
-
depth: 0,
|
|
802
|
-
text: "Observed:",
|
|
803
|
-
tone: "heading"
|
|
804
|
-
}];
|
|
805
|
-
if (!perCell) {
|
|
806
|
-
for (const t of tallies) lines.push({
|
|
807
|
-
depth: 1,
|
|
808
|
-
text: `${clip(t.name, 40).padEnd(width)} ${observedLine(t)}`
|
|
809
|
-
});
|
|
810
|
-
return lines;
|
|
811
|
-
}
|
|
812
|
-
for (const name of new Set(tallies.map((t) => t.name))) {
|
|
813
|
-
lines.push({
|
|
814
|
-
depth: 1,
|
|
815
|
-
text: clip(name, 40)
|
|
816
|
-
});
|
|
817
|
-
const rows = tallies.filter((t) => t.name === name).map((t) => {
|
|
818
|
-
const cell = byKey.get(t.cell);
|
|
819
|
-
return [cell ? label(cell) : t.cell, observedLine(t)];
|
|
820
|
-
});
|
|
821
|
-
const w = Math.max(...rows.map(([who]) => who.length));
|
|
822
|
-
for (const [who, line] of rows) lines.push({
|
|
823
|
-
depth: 2,
|
|
824
|
-
text: `${who.padEnd(w)} ${line}`
|
|
825
|
-
});
|
|
826
|
-
}
|
|
827
|
-
return lines;
|
|
828
|
-
}
|
|
829
|
-
/** `3 passed · 1 failed · 1 not gated`, from the suite's tally. */
|
|
830
|
-
function tallyText(suite) {
|
|
831
|
-
const t = suite.tally;
|
|
832
|
-
return [
|
|
833
|
-
t.errors ? plural(t.errors, "run error") : "",
|
|
834
|
-
t.fail ? `${t.fail} failed` : "",
|
|
835
|
-
t.infraError ? plural(t.infraError, "infra error") : "",
|
|
836
|
-
t.canceled ? `${t.canceled} canceled` : "",
|
|
837
|
-
t.pass ? `${t.pass} passed` : "",
|
|
838
|
-
t.notGated ? `${t.notGated} not gated` : "",
|
|
839
|
-
t.skipped ? `${t.skipped} skipped` : ""
|
|
840
|
-
].filter(Boolean).join(" · ");
|
|
841
|
-
}
|
|
842
|
-
function statusList(input) {
|
|
843
|
-
const { record, suite } = input;
|
|
844
|
-
const { shown, hidden } = shownRemarks(input.remarks?.remarks ?? [], suite.runId, input.verbose);
|
|
845
|
-
const files = suite.files.map((f) => ({
|
|
846
|
-
path: f.path,
|
|
847
|
-
rows: f.evals.map((e) => evalRow(record, e, shown.filter((r) => r.evalId === e.path), input.verbose))
|
|
848
|
-
}));
|
|
849
|
-
const tokens = formatUsage(record.cost.usage);
|
|
850
|
-
const done = [
|
|
851
|
-
`Done in ${formatDuration(record.durationMs)}`,
|
|
852
|
-
tallyText(suite) || "nothing ran",
|
|
853
|
-
tokens
|
|
854
|
-
].filter(Boolean).join(" · ");
|
|
855
|
-
const notes = [
|
|
856
|
-
...input.notRun ? [`Incomplete: ${plural(input.notRun, "attempt")} this resume’s filters left out ${input.notRun === 1 ? "is" : "are"} not run; reeval run --resume ${record.runId} runs ${input.notRun === 1 ? "it" : "them"}`] : [],
|
|
857
|
-
...record.status === "budget" && !input.notRun ? ["Budget reached: attempts that hadn’t started were skipped"] : [],
|
|
858
|
-
...record.status === "interrupted" && !input.notRun ? ["Interrupted: attempts that hadn’t finished were canceled"] : []
|
|
859
|
-
];
|
|
860
|
-
return {
|
|
861
|
-
files,
|
|
862
|
-
errors: suite.errors,
|
|
863
|
-
done,
|
|
864
|
-
notes,
|
|
865
|
-
hiddenRemarks: hidden
|
|
866
|
-
};
|
|
867
|
-
}
|
|
868
|
-
/** The pointers under the closing line: the studio and the run's summary.md. */
|
|
869
|
-
const pointerLine = (summaryPath) => `Browse it: reeval studio · summary ${summaryPath}`;
|
|
870
312
|
// src/ui/tags.ts
|
|
871
313
|
const STOPWORDS = new Set("a an the and or of to in on for with without from by at as is are be it its that this when than then into via not no".split(" "));
|
|
872
314
|
const MAX_WORDS = 3;
|
|
@@ -986,7 +428,8 @@ var AgentRenderer = class {
|
|
|
986
428
|
verbose,
|
|
987
429
|
...e.notRun ? { notRun: e.notRun } : {}
|
|
988
430
|
});
|
|
989
|
-
|
|
431
|
+
const pointer = pointerLine(e.summaryPath);
|
|
432
|
+
for (const line of listLines(list, this.#c, false, pointer, verbose, true)) this.#write(line);
|
|
990
433
|
return;
|
|
991
434
|
}
|
|
992
435
|
default: return;
|
|
@@ -1266,7 +709,7 @@ var PlainRenderer = class {
|
|
|
1266
709
|
this.#write(`::error file=${ghEscape(file, true)},line=${line},title=${label}::${ghEscape(x.message)}`);
|
|
1267
710
|
}
|
|
1268
711
|
if (!errorCheck(result)?.at) {
|
|
1269
|
-
const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n"));
|
|
712
|
+
const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n").replace(/^✗ failed {2}/, ""));
|
|
1270
713
|
const at = locate(result);
|
|
1271
714
|
const where = at ? `,line=${at.line},col=${at.col}` : "";
|
|
1272
715
|
this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
|
|
@@ -1330,6 +773,107 @@ var ListStore = class {
|
|
|
1330
773
|
for (const l of this.#listeners) l();
|
|
1331
774
|
}
|
|
1332
775
|
};
|
|
776
|
+
const FRAME_MS = 80;
|
|
777
|
+
const ESC = String.fromCharCode(27);
|
|
778
|
+
const BEL = String.fromCharCode(7);
|
|
779
|
+
/** A CSI sequence (colors, cursor moves) or an OSC one (a hyperlink), which take no columns. */
|
|
780
|
+
const ESCAPE = new RegExp(`${ESC}(?:\\[[0-9;?]*[ -/]*[@-~]|\\][^${BEL}${ESC}]*(?:${BEL}|${ESC}\\\\))`, "g");
|
|
781
|
+
const graphemes = new Intl.Segmenter();
|
|
782
|
+
/**
|
|
783
|
+
* Terminal rows one line takes at `columns`, placed as a terminal places it: a wide character that
|
|
784
|
+
* doesn't fit in the row's last column moves to the next row, `\r` returns to the row's start and
|
|
785
|
+
* `\t` goes to the next stop of eight without wrapping.
|
|
786
|
+
*/
|
|
787
|
+
function rowsOf(line, columns) {
|
|
788
|
+
let rows = 1;
|
|
789
|
+
let column = 0;
|
|
790
|
+
for (const { segment } of graphemes.segment(line.replace(ESCAPE, ""))) if (segment === "\r") column = 0;
|
|
791
|
+
else if (segment === " ") column = Math.min(columns - 1, (Math.floor(column / 8) + 1) * 8);
|
|
792
|
+
else if (segment === "\b") column = Math.max(0, column - 1);
|
|
793
|
+
else {
|
|
794
|
+
const width = stringWidth(segment);
|
|
795
|
+
if (!width) continue;
|
|
796
|
+
if (column + width > columns) {
|
|
797
|
+
rows += 1;
|
|
798
|
+
column = 0;
|
|
799
|
+
}
|
|
800
|
+
column += width;
|
|
801
|
+
}
|
|
802
|
+
return rows;
|
|
803
|
+
}
|
|
804
|
+
/**
|
|
805
|
+
* Before the list is up, the sandbox prepare step: the glyph of its `◆ Preparing…` line spins in
|
|
806
|
+
* place (the cursor goes up to it and back) until each sandbox is ready or failed, then settles to
|
|
807
|
+
* `◆`. What prints under it is kept and measured in rows as the terminal placed it, so a wide
|
|
808
|
+
* character still finds the step line. Once the step line would be out of reach, or the width
|
|
809
|
+
* changes (some terminals reflow what they show, some don't, so its row is unknown), it stops
|
|
810
|
+
* without drawing.
|
|
811
|
+
*/
|
|
812
|
+
var StepSpinner = class {
|
|
813
|
+
#write;
|
|
814
|
+
#c;
|
|
815
|
+
#size;
|
|
816
|
+
#timer;
|
|
817
|
+
#frame = 0;
|
|
818
|
+
/** The step line and everything printed under it since. */
|
|
819
|
+
#text = "";
|
|
820
|
+
/** The width `#text` was printed at. */
|
|
821
|
+
#columns = 0;
|
|
822
|
+
#pending = 0;
|
|
823
|
+
constructor(write, c, size) {
|
|
824
|
+
this.#write = write;
|
|
825
|
+
this.#c = c;
|
|
826
|
+
this.#size = size;
|
|
827
|
+
}
|
|
828
|
+
/** Terminal lines from the step line down to the cursor after `text`. */
|
|
829
|
+
#up(text = this.#text) {
|
|
830
|
+
const columns = Math.max(1, this.#columns);
|
|
831
|
+
const lines = text.split("\n");
|
|
832
|
+
if (lines.at(-1) === "") lines.pop();
|
|
833
|
+
return lines.reduce((n, l) => n + rowsOf(l, columns), 0);
|
|
834
|
+
}
|
|
835
|
+
/** The step line was just written as `text`; it spins until `targets` have reported. */
|
|
836
|
+
start(text, targets) {
|
|
837
|
+
this.settle();
|
|
838
|
+
this.#text = text;
|
|
839
|
+
this.#columns = this.#size().columns;
|
|
840
|
+
this.#pending = targets;
|
|
841
|
+
this.#timer = setInterval(() => this.#draw(), FRAME_MS);
|
|
842
|
+
this.#timer.unref();
|
|
843
|
+
this.#draw();
|
|
844
|
+
}
|
|
845
|
+
/** Before `text` prints under it; settles first if the step line would scroll out of reach. */
|
|
846
|
+
before(text) {
|
|
847
|
+
if (!this.#timer) return;
|
|
848
|
+
const next = this.#text + text;
|
|
849
|
+
if (!this.#reachable(next)) this.settle();
|
|
850
|
+
this.#text = next;
|
|
851
|
+
}
|
|
852
|
+
/** One sandbox reported; the last settles it. */
|
|
853
|
+
reported() {
|
|
854
|
+
this.#pending -= 1;
|
|
855
|
+
if (this.#pending <= 0) this.settle();
|
|
856
|
+
}
|
|
857
|
+
#reachable(text = this.#text) {
|
|
858
|
+
const { rows, columns } = this.#size();
|
|
859
|
+
return columns === this.#columns && this.#up(text) < rows - 1;
|
|
860
|
+
}
|
|
861
|
+
#draw(glyph = SPINNER[this.#frame++ % SPINNER.length] ?? "◆") {
|
|
862
|
+
if (!this.#reachable()) {
|
|
863
|
+
clearInterval(this.#timer);
|
|
864
|
+
this.#timer = void 0;
|
|
865
|
+
return;
|
|
866
|
+
}
|
|
867
|
+
const up = this.#up();
|
|
868
|
+
this.#write(`\x1b[${up}A\r\x1b[2C${this.#c.teal(glyph)}\r\x1b[${up}B`);
|
|
869
|
+
}
|
|
870
|
+
settle() {
|
|
871
|
+
if (!this.#timer) return;
|
|
872
|
+
this.#draw("◆");
|
|
873
|
+
clearInterval(this.#timer);
|
|
874
|
+
this.#timer = void 0;
|
|
875
|
+
}
|
|
876
|
+
};
|
|
1333
877
|
var InteractiveRenderer = class {
|
|
1334
878
|
#o;
|
|
1335
879
|
#c;
|
|
@@ -1342,12 +886,20 @@ var InteractiveRenderer = class {
|
|
|
1342
886
|
#fallback;
|
|
1343
887
|
#ended;
|
|
1344
888
|
#closed = false;
|
|
889
|
+
#spinner;
|
|
890
|
+
/** What the plain renderer wrote since `#written` was reset, while the list isn't up. */
|
|
891
|
+
#written = "";
|
|
1345
892
|
constructor(options) {
|
|
1346
893
|
this.#o = options;
|
|
1347
894
|
this.#c = options.colors;
|
|
1348
895
|
this.#list = new LiveList(options.harness);
|
|
896
|
+
this.#spinner = new StepSpinner((text) => options.out.write(text), options.colors, () => sizeOf(options.stdout));
|
|
1349
897
|
const out = { write: (text) => {
|
|
1350
|
-
if (!this.#app || this.#fallback)
|
|
898
|
+
if (!this.#app || this.#fallback) {
|
|
899
|
+
this.#spinner.before(redactSecrets(String(text)));
|
|
900
|
+
this.#written += String(text);
|
|
901
|
+
return options.out.write(text);
|
|
902
|
+
}
|
|
1351
903
|
const lines = String(text).replace(/\n$/, "").split("\n");
|
|
1352
904
|
this.#store.set({ logs: [...this.#store.get().logs, ...lines] });
|
|
1353
905
|
return true;
|
|
@@ -1367,7 +919,7 @@ var InteractiveRenderer = class {
|
|
|
1367
919
|
#mount() {
|
|
1368
920
|
if (this.#app || this.#closed) return;
|
|
1369
921
|
const cancel = this.#o.cancel ?? (() => process.kill(process.pid, "SIGINT"));
|
|
1370
|
-
const load = this.#o.load ?? (() => import("./ink-list-
|
|
922
|
+
const load = this.#o.load ?? (() => import("./ink-list-BpfidQ1h.js"));
|
|
1371
923
|
this.#app = load().then((m) => m.mountList({
|
|
1372
924
|
store: this.#store,
|
|
1373
925
|
colors: this.#c,
|
|
@@ -1384,7 +936,7 @@ var InteractiveRenderer = class {
|
|
|
1384
936
|
for (const line of logs) out.write(`${line}\n`);
|
|
1385
937
|
const reason = error instanceof Error ? error.message : String(error);
|
|
1386
938
|
out.write(`${this.#c.yellow(`Couldn't draw the interactive list (${reason}).`)}\n`);
|
|
1387
|
-
if (phase) out.write(
|
|
939
|
+
if (phase) out.write(` ${this.#c.teal("◆")} ${phase}\n`);
|
|
1388
940
|
this.#fallback = new PlainRenderer({
|
|
1389
941
|
...this.#o,
|
|
1390
942
|
timers: false,
|
|
@@ -1397,12 +949,14 @@ var InteractiveRenderer = class {
|
|
|
1397
949
|
if (this.#ended) this.#fallback.onEvent(this.#ended.event);
|
|
1398
950
|
}
|
|
1399
951
|
#update() {
|
|
1400
|
-
const { done, total } = this.#list.counts();
|
|
952
|
+
const { done, total, failed, failing } = this.#list.counts();
|
|
1401
953
|
this.#store.set({
|
|
1402
954
|
files: this.#list.files(),
|
|
1403
955
|
counts: {
|
|
1404
956
|
done,
|
|
1405
|
-
total
|
|
957
|
+
total,
|
|
958
|
+
failed,
|
|
959
|
+
failing
|
|
1406
960
|
}
|
|
1407
961
|
});
|
|
1408
962
|
}
|
|
@@ -1413,9 +967,18 @@ var InteractiveRenderer = class {
|
|
|
1413
967
|
}
|
|
1414
968
|
switch (e.type) {
|
|
1415
969
|
case "run-start":
|
|
1416
|
-
this.#
|
|
970
|
+
this.#spinner.settle();
|
|
971
|
+
this.#store.set({ phase: startText(this.#o.output) });
|
|
972
|
+
return;
|
|
973
|
+
case "prepare":
|
|
974
|
+
this.#written = "";
|
|
975
|
+
this.#plain.onEvent(e);
|
|
976
|
+
if (this.#app || this.#o.timers === false || process.env.REEVAL_DEBUG) return;
|
|
977
|
+
if (e.state === "start") this.#spinner.start(this.#written, e.targets.length);
|
|
978
|
+
else this.#spinner.reported();
|
|
1417
979
|
return;
|
|
1418
980
|
case "collected":
|
|
981
|
+
this.#spinner.settle();
|
|
1419
982
|
this.#list.collected(e.file, e.evals);
|
|
1420
983
|
this.#mount();
|
|
1421
984
|
this.#update();
|
|
@@ -1435,25 +998,28 @@ var InteractiveRenderer = class {
|
|
|
1435
998
|
return;
|
|
1436
999
|
case "debug": return;
|
|
1437
1000
|
case "run-end": {
|
|
1438
|
-
const
|
|
1001
|
+
const listOf = (verbose) => statusList({
|
|
1439
1002
|
record: e.record,
|
|
1440
1003
|
suite: suiteResult(e.record),
|
|
1441
1004
|
remarks: e.remarks ?? null,
|
|
1442
|
-
verbose
|
|
1005
|
+
verbose,
|
|
1443
1006
|
...e.notRun ? { notRun: e.notRun } : {}
|
|
1444
1007
|
});
|
|
1008
|
+
const list = listOf(false);
|
|
1009
|
+
const verbose = (this.#o.verbosity ?? 0) > 0;
|
|
1445
1010
|
if (!this.#app) {
|
|
1446
1011
|
this.#plain.onEvent(e);
|
|
1447
1012
|
return;
|
|
1448
1013
|
}
|
|
1449
1014
|
this.#ended = {
|
|
1450
|
-
list,
|
|
1015
|
+
scrollback: verbose ? listOf(true) : list,
|
|
1016
|
+
verbose,
|
|
1451
1017
|
pointer: pointerLine(e.summaryPath),
|
|
1452
1018
|
event: e
|
|
1453
1019
|
};
|
|
1454
1020
|
this.#store.set({
|
|
1455
1021
|
files: list.files,
|
|
1456
|
-
...this.#store.get().phase ? { phase:
|
|
1022
|
+
...this.#store.get().phase ? { phase: endText(e.record.status) } : {},
|
|
1457
1023
|
finished: {
|
|
1458
1024
|
done: list.done,
|
|
1459
1025
|
notes: list.notes,
|
|
@@ -1467,6 +1033,7 @@ var InteractiveRenderer = class {
|
|
|
1467
1033
|
}
|
|
1468
1034
|
}
|
|
1469
1035
|
close() {
|
|
1036
|
+
this.#spinner.settle();
|
|
1470
1037
|
this.#plain.close();
|
|
1471
1038
|
this.#fallback?.close();
|
|
1472
1039
|
if (!this.#store.get().finished) {
|
|
@@ -1480,11 +1047,18 @@ var InteractiveRenderer = class {
|
|
|
1480
1047
|
if (!app || !this.#ended) return;
|
|
1481
1048
|
await app.waitUntilExit();
|
|
1482
1049
|
const { phase } = this.#store.get();
|
|
1483
|
-
const {
|
|
1484
|
-
const text = [...phase ? [phase
|
|
1050
|
+
const { scrollback, verbose, pointer } = this.#ended;
|
|
1051
|
+
const text = [...phase ? [` ${this.#c.teal("◆")} ${phase}`, ""] : [], ...listLines(scrollback, this.#c, true, pointer, verbose)];
|
|
1485
1052
|
this.#o.out.write(`${text.join("\n")}\n`);
|
|
1486
1053
|
}
|
|
1487
1054
|
};
|
|
1055
|
+
const sizeOf = (stdout) => {
|
|
1056
|
+
const { rows, columns } = stdout ?? {};
|
|
1057
|
+
return {
|
|
1058
|
+
rows: rows || 24,
|
|
1059
|
+
columns: columns || 80
|
|
1060
|
+
};
|
|
1061
|
+
};
|
|
1488
1062
|
// src/ui/index.ts
|
|
1489
1063
|
const depthOf = (bits = 4) => bits >= 24 ? 24 : bits >= 8 ? 8 : 4;
|
|
1490
1064
|
function createRenderer(options) {
|
|
@@ -1532,7 +1106,7 @@ const EXIT = {
|
|
|
1532
1106
|
};
|
|
1533
1107
|
const version = PACKAGE_VERSION;
|
|
1534
1108
|
const HELP = helpText(version);
|
|
1535
|
-
const { green, red, dim } = colors(!!process.stdout.isTTY && !process.env.NO_COLOR);
|
|
1109
|
+
const { green, red, dim, yellow } = colors(!!process.stdout.isTTY && !process.env.NO_COLOR);
|
|
1536
1110
|
const startedAt = Date.now();
|
|
1537
1111
|
const argv = process.argv.slice(2);
|
|
1538
1112
|
/** What the invocation log needs from the command as it goes. */
|
|
@@ -1576,7 +1150,7 @@ async function run(argv) {
|
|
|
1576
1150
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
1577
1151
|
}
|
|
1578
1152
|
if (command === "init") {
|
|
1579
|
-
const { initCommand } = await import("./init-
|
|
1153
|
+
const { initCommand } = await import("./init-4RPeD-K-.js");
|
|
1580
1154
|
return initCommand(values, filters, {
|
|
1581
1155
|
out: (s) => console.log(s),
|
|
1582
1156
|
input: process.stdin,
|
|
@@ -1585,7 +1159,7 @@ async function run(argv) {
|
|
|
1585
1159
|
});
|
|
1586
1160
|
}
|
|
1587
1161
|
if (command === "doctor") {
|
|
1588
|
-
const { doctorCommand } = await import("./doctor-
|
|
1162
|
+
const { doctorCommand } = await import("./doctor-B2PohneB.js");
|
|
1589
1163
|
return doctorCommand(values, {
|
|
1590
1164
|
out: (s) => console.log(s),
|
|
1591
1165
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -1593,7 +1167,7 @@ async function run(argv) {
|
|
|
1593
1167
|
});
|
|
1594
1168
|
}
|
|
1595
1169
|
if (command === "report") {
|
|
1596
|
-
const { reportCommand } = await import("./command-
|
|
1170
|
+
const { reportCommand } = await import("./command-aUJvENwy.js");
|
|
1597
1171
|
return reportCommand(values, filters, {
|
|
1598
1172
|
out: (s) => console.log(s),
|
|
1599
1173
|
err: (s) => console.error(dim(s)),
|
|
@@ -1608,7 +1182,7 @@ async function run(argv) {
|
|
|
1608
1182
|
}
|
|
1609
1183
|
if (command === "studio") {
|
|
1610
1184
|
const config = await loadConfig(configPath, "studio");
|
|
1611
|
-
const { studioCommand } = await import("./command-
|
|
1185
|
+
const { studioCommand } = await import("./command-BLNVDiIg.js");
|
|
1612
1186
|
return studioCommand({
|
|
1613
1187
|
config,
|
|
1614
1188
|
flags: values,
|
|
@@ -1619,7 +1193,7 @@ async function run(argv) {
|
|
|
1619
1193
|
if (command === "mcp") {
|
|
1620
1194
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
1621
1195
|
const config = await loadConfig(configPath, "mcp");
|
|
1622
|
-
const { mcpCommand } = await import("./command-
|
|
1196
|
+
const { mcpCommand } = await import("./command-B5OyHpOu.js");
|
|
1623
1197
|
return mcpCommand({
|
|
1624
1198
|
config,
|
|
1625
1199
|
flags: values,
|
|
@@ -1628,7 +1202,7 @@ async function run(argv) {
|
|
|
1628
1202
|
}
|
|
1629
1203
|
if (command in HISTORY_COMMANDS) {
|
|
1630
1204
|
const config = await loadConfig(configPath, command);
|
|
1631
|
-
const commands = await import("./commands-
|
|
1205
|
+
const commands = await import("./commands-CfdBX-mc.js");
|
|
1632
1206
|
const ctx = {
|
|
1633
1207
|
config,
|
|
1634
1208
|
flags: values,
|
|
@@ -1641,7 +1215,7 @@ async function run(argv) {
|
|
|
1641
1215
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
1642
1216
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
1643
1217
|
case "query": {
|
|
1644
|
-
const { queryCommand } = await import("./query-
|
|
1218
|
+
const { queryCommand } = await import("./query-DIfkUcPh.js");
|
|
1645
1219
|
return queryCommand({
|
|
1646
1220
|
...ctx,
|
|
1647
1221
|
err: (s) => console.error(dim(s))
|
|
@@ -1649,7 +1223,7 @@ async function run(argv) {
|
|
|
1649
1223
|
}
|
|
1650
1224
|
case "analyze": {
|
|
1651
1225
|
loadEnvFiles(configPath);
|
|
1652
|
-
const { analyzeCommand } = await import("./command-
|
|
1226
|
+
const { analyzeCommand } = await import("./command-CRh-cb6Z.js");
|
|
1653
1227
|
return analyzeCommand({
|
|
1654
1228
|
...ctx,
|
|
1655
1229
|
err: (s) => console.error(dim(s))
|
|
@@ -1663,12 +1237,12 @@ async function run(argv) {
|
|
|
1663
1237
|
}
|
|
1664
1238
|
/** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
|
|
1665
1239
|
async function dryRunPrices(planned, judgeModel) {
|
|
1666
|
-
const { needsPrices } = await import("./cost-
|
|
1240
|
+
const { needsPrices } = await import("./cost-Cr26Alhq.js");
|
|
1667
1241
|
if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
|
|
1668
1242
|
table: null,
|
|
1669
1243
|
why: "nothing to price"
|
|
1670
1244
|
};
|
|
1671
|
-
const { loadPrices } = await import("./prices-
|
|
1245
|
+
const { loadPrices } = await import("./prices-5m3SzMZ3.js");
|
|
1672
1246
|
return loadPrices();
|
|
1673
1247
|
}
|
|
1674
1248
|
/** Asks a yes/no question on the terminal; anything but y or yes is no. */
|
|
@@ -1695,14 +1269,14 @@ async function cleanCommand(flags, configPath) {
|
|
|
1695
1269
|
keep = Number(flags.keep);
|
|
1696
1270
|
if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
|
|
1697
1271
|
}
|
|
1698
|
-
const { debugRoot, pruneDebug } = await import("./files-
|
|
1699
|
-
const { formatBytes } = await import("./debug-
|
|
1272
|
+
const { debugRoot, pruneDebug } = await import("./files-C1Rk9FYA.js");
|
|
1273
|
+
const { formatBytes } = await import("./debug-CF4p4Lp_.js");
|
|
1700
1274
|
const { runs, bytes } = await pruneDebug(config.root, keep);
|
|
1701
1275
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1702
1276
|
console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
|
|
1703
1277
|
if (flags.images) {
|
|
1704
|
-
const { dockerAvailable, pruneImages } = await import("./src-
|
|
1705
|
-
const { requireDocker } = await import("./providers-
|
|
1278
|
+
const { dockerAvailable, pruneImages } = await import("./src-DtodMT59.js");
|
|
1279
|
+
const { requireDocker } = await import("./providers-CVu8e0xS.js");
|
|
1706
1280
|
const engines = [];
|
|
1707
1281
|
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1708
1282
|
if (!engines.length) await requireDocker();
|
|
@@ -1712,24 +1286,25 @@ async function cleanCommand(flags, configPath) {
|
|
|
1712
1286
|
}
|
|
1713
1287
|
return EXIT.ok;
|
|
1714
1288
|
}
|
|
1715
|
-
/** `-v` is 1 and `-vv`
|
|
1289
|
+
/** `-v` is 1 and `-vv` is 2. */
|
|
1716
1290
|
function verbosityOf(flags) {
|
|
1717
|
-
if (flags.trace) return 2;
|
|
1718
1291
|
return Math.min(2, flags.verbose?.length ?? 0);
|
|
1719
1292
|
}
|
|
1720
1293
|
/**
|
|
1721
|
-
* `reeval image build`: collects the evals (as `--dry-run` does)
|
|
1722
|
-
* sandboxes
|
|
1723
|
-
*
|
|
1294
|
+
* `reeval image build`: collects the evals (as `--dry-run` does), pulls every image their `tart()`
|
|
1295
|
+
* sandboxes clone that Tart doesn't have, and builds every image their Docker sandboxes need; with
|
|
1296
|
+
* `--dry-run`, lists them instead. A plan with no Docker sandbox never runs Docker. `--profile`
|
|
1297
|
+
* plans as `reeval run --profile` does. Needs a config, as `reeval run` does: there is no default
|
|
1298
|
+
* image.
|
|
1724
1299
|
*/
|
|
1725
1300
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1726
|
-
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-
|
|
1301
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-BNgcR77K.js");
|
|
1727
1302
|
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
|
|
1728
1303
|
...flags,
|
|
1729
1304
|
"dry-run": true
|
|
1730
1305
|
}, configPath));
|
|
1731
1306
|
const root = config.root;
|
|
1732
|
-
const
|
|
1307
|
+
const uses = await plannedImageUses({
|
|
1733
1308
|
runId,
|
|
1734
1309
|
stagingDir,
|
|
1735
1310
|
config,
|
|
@@ -1737,46 +1312,116 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1737
1312
|
tags,
|
|
1738
1313
|
concurrency,
|
|
1739
1314
|
profile
|
|
1740
|
-
}, filters, (parts) => unmatchedFilter(config, parts))
|
|
1741
|
-
|
|
1742
|
-
|
|
1315
|
+
}, filters, (parts) => unmatchedFilter(config, parts));
|
|
1316
|
+
const targets = imageTargets(uses);
|
|
1317
|
+
const vms = tartTargets(uses);
|
|
1318
|
+
if (targets.length === 0 && vms.length === 0) {
|
|
1319
|
+
console.log("No eval runs in a Docker or Tart sandbox, so there are no images to build or pull.");
|
|
1743
1320
|
return EXIT.ok;
|
|
1744
1321
|
}
|
|
1322
|
+
const { tartTagWarning } = await import("./tart-ref-7Qv3cm3B.js");
|
|
1323
|
+
for (const t of vms) {
|
|
1324
|
+
const warning = tartTagWarning(t.sandbox.image);
|
|
1325
|
+
if (warning) console.log(yellow(`warning: ${warning}`));
|
|
1326
|
+
}
|
|
1745
1327
|
if (flags["dry-run"]) {
|
|
1746
|
-
for (const line of await
|
|
1328
|
+
if (vms.length) for (const line of await tartPlanLines(vms)) console.log(line);
|
|
1329
|
+
if (targets.length) for (const line of await imagePlanLines(targets, root)) console.log(line);
|
|
1747
1330
|
return EXIT.ok;
|
|
1748
1331
|
}
|
|
1749
|
-
const { PrepareError } = await import("./provider-
|
|
1750
|
-
|
|
1332
|
+
const { PrepareError } = await import("./provider-BnUoz4KB.js");
|
|
1333
|
+
const outcomes = [];
|
|
1334
|
+
let refused = false;
|
|
1335
|
+
const refusal = (id, of, others) => (error) => {
|
|
1336
|
+
if (!others || !(error instanceof PrepareError)) throw error;
|
|
1337
|
+
if (error.failure.usage) refused = true;
|
|
1338
|
+
const evals = of.reduce((n, t) => n + t.evals, 0);
|
|
1339
|
+
outcomes.push({
|
|
1340
|
+
id,
|
|
1341
|
+
evals,
|
|
1342
|
+
ok: false,
|
|
1343
|
+
failure: error.failure
|
|
1344
|
+
});
|
|
1345
|
+
return [];
|
|
1346
|
+
};
|
|
1751
1347
|
try {
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1348
|
+
const pulled = vms.length ? await pullTartImages(vms, root, { log: (line) => console.log(line.startsWith("warning: ") ? yellow(line) : dim(line)) }).catch(refusal("tart", vms, targets.length)) : [];
|
|
1349
|
+
for (const p of pulled) outcomes.push(p.ok ? {
|
|
1350
|
+
id: p.id,
|
|
1351
|
+
evals: p.evals,
|
|
1352
|
+
ok: true,
|
|
1353
|
+
ref: p.image.ref,
|
|
1354
|
+
id12: p.image.digest ? short12(p.image.digest) : "local VM"
|
|
1355
|
+
} : {
|
|
1356
|
+
id: p.id,
|
|
1357
|
+
evals: p.evals,
|
|
1358
|
+
ok: false,
|
|
1359
|
+
failure: p.failure
|
|
1755
1360
|
});
|
|
1361
|
+
if (targets.length) {
|
|
1362
|
+
const built = await buildImages(targets, root, {
|
|
1363
|
+
log: (line) => console.log(dim(line)),
|
|
1364
|
+
force: !!flags.force
|
|
1365
|
+
}).catch(refusal("docker", targets, vms.length));
|
|
1366
|
+
for (const b of built) outcomes.push(b.ok ? {
|
|
1367
|
+
id: b.id,
|
|
1368
|
+
evals: b.evals,
|
|
1369
|
+
ok: true,
|
|
1370
|
+
ref: b.image.ref,
|
|
1371
|
+
id12: short12(b.image.id)
|
|
1372
|
+
} : {
|
|
1373
|
+
id: b.id,
|
|
1374
|
+
evals: b.evals,
|
|
1375
|
+
ok: false,
|
|
1376
|
+
failure: b.failure
|
|
1377
|
+
});
|
|
1378
|
+
}
|
|
1756
1379
|
} catch (error) {
|
|
1757
|
-
if (error instanceof PrepareError
|
|
1758
|
-
throw error;
|
|
1380
|
+
if (!(error instanceof PrepareError)) throw error;
|
|
1381
|
+
if (error.failure.usage) throw new UsageError(error.message);
|
|
1382
|
+
if (error.failure.cause === "enospc") invocation.machineState = true;
|
|
1383
|
+
const { PreflightInfraError } = await import("./preflight-4gWSy0Pe.js");
|
|
1384
|
+
throw new PreflightInfraError(error.message);
|
|
1759
1385
|
}
|
|
1760
|
-
for (const
|
|
1761
|
-
if (
|
|
1762
|
-
|
|
1763
|
-
console.log(`${green("✓")} ${green(`${b.id}: ${b.image.ref}`)}${dim(` ${id}`)}`);
|
|
1386
|
+
for (const o of outcomes) {
|
|
1387
|
+
if (o.ok) {
|
|
1388
|
+
console.log(`${green("✓")} ${green(`${o.id}: ${o.ref}`)}${dim(` ${o.id12}`)}`);
|
|
1764
1389
|
continue;
|
|
1765
1390
|
}
|
|
1766
|
-
const uses =
|
|
1767
|
-
const [headline = "", ...rest] =
|
|
1768
|
-
console.log(`${red("✗")} ${red(`${
|
|
1391
|
+
const uses = o.evals ? ` (${o.evals} eval${o.evals === 1 ? "" : "s"})` : "";
|
|
1392
|
+
const [headline = "", ...rest] = o.failure.message.split("\n");
|
|
1393
|
+
console.log(`${red("✗")} ${red(`${o.id}${uses}: ${headline}`)}`);
|
|
1769
1394
|
for (const line of rest) console.log(` ${line}`);
|
|
1770
1395
|
}
|
|
1771
|
-
const failed =
|
|
1772
|
-
const n =
|
|
1773
|
-
console.log(failed.length ? red(`${failed.length} of ${n} image${n === 1 ? "" : "s"} failed to build`) : dim(`${n} image${n === 1 ? "" : "s"} ready`));
|
|
1396
|
+
const failed = outcomes.filter((o) => !o.ok);
|
|
1397
|
+
const n = outcomes.length;
|
|
1398
|
+
console.log(failed.length ? red(`${failed.length} of ${n} image${n === 1 ? "" : "s"} failed to ${vms.length ? targets.length ? "build or pull" : "pull" : "build"}`) : dim(`${n} image${n === 1 ? "" : "s"} ready`));
|
|
1774
1399
|
if (!failed.length) return EXIT.ok;
|
|
1775
|
-
|
|
1400
|
+
if (refused || failed.every((o) => !o.ok && o.failure.usage)) return EXIT.usage;
|
|
1401
|
+
invocation.machineState = failed.every((o) => !o.ok && o.failure.cause === "enospc");
|
|
1402
|
+
return EXIT.infra;
|
|
1776
1403
|
}
|
|
1404
|
+
/** The first 12 hex digits of an image id or digest. */
|
|
1405
|
+
const short12 = (id) => id.replace(/^sha256:/, "").slice(0, 12);
|
|
1777
1406
|
/** `reeval run`: check, plan, run the engine, then write the run and pick the exit code. */
|
|
1778
1407
|
async function runCommand(flags, configPath, given) {
|
|
1779
|
-
const
|
|
1408
|
+
const retry = flags["retry-errors"];
|
|
1409
|
+
if (retry !== void 0 && flags.resume !== void 0) throw new UsageError("--resume and --retry-errors both continue a run: pass one of them.");
|
|
1410
|
+
if (retry !== void 0 && (given.length || flags.tag?.length || flags.trait?.length)) throw new UsageError("--retry-errors runs every infra-errored attempt of the run again, with its plan: leave out filters, --tag and --trait.");
|
|
1411
|
+
const resumed = flags.resume !== void 0 || retry !== void 0 ? await resumedRun(flags, given, configPath) : null;
|
|
1412
|
+
if (resumed?.nothing) {
|
|
1413
|
+
const { nothing, record } = resumed.nothing;
|
|
1414
|
+
const outcome = runOutcome(record);
|
|
1415
|
+
if (flags.json) {
|
|
1416
|
+
console.error(nothing);
|
|
1417
|
+
const { suiteResult } = await import("./suite-CKBa6MKt.js");
|
|
1418
|
+
process.stdout.write(`${JSON.stringify({
|
|
1419
|
+
...suiteResult(record),
|
|
1420
|
+
outcome
|
|
1421
|
+
}, null, 2)}\n`);
|
|
1422
|
+
} else console.log(nothing);
|
|
1423
|
+
return outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1424
|
+
}
|
|
1780
1425
|
const runFlags = resumed?.settings ? {
|
|
1781
1426
|
...flags,
|
|
1782
1427
|
...resumed.settings
|
|
@@ -1792,7 +1437,8 @@ async function runCommand(flags, configPath, given) {
|
|
|
1792
1437
|
...setup.profile ? { profile: setup.profile.name } : {},
|
|
1793
1438
|
...explicit === void 0 ? {} : { concurrency: setup.concurrency },
|
|
1794
1439
|
filters: savedFilters(setup.config.root, filters),
|
|
1795
|
-
tags: runFlags.tag ?? []
|
|
1440
|
+
tags: runFlags.tag ?? [],
|
|
1441
|
+
...runFlags.trait?.length ? { traits: runFlags.trait } : {}
|
|
1796
1442
|
};
|
|
1797
1443
|
const ctx = await startRun(setup);
|
|
1798
1444
|
invocation.runId = ctx.runId;
|
|
@@ -1800,7 +1446,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1800
1446
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1801
1447
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1802
1448
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1803
|
-
const { engineVersions } = await import("./node-
|
|
1449
|
+
const { engineVersions } = await import("./node-D7pAU--0.js");
|
|
1804
1450
|
const terminal = setup.json && !setup.dryRun ? process.stderr : process.stdout;
|
|
1805
1451
|
const renderer = createRenderer({
|
|
1806
1452
|
out: setup.json && setup.dryRun ? { write: () => true } : terminal,
|
|
@@ -1819,28 +1465,37 @@ async function runCommand(flags, configPath, given) {
|
|
|
1819
1465
|
executed = await executeRun(ctx, setup, filters, renderer, {
|
|
1820
1466
|
startedWith,
|
|
1821
1467
|
notes: resumed?.notes ?? [],
|
|
1822
|
-
narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length)
|
|
1468
|
+
narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length || !!flags.trait?.length)
|
|
1823
1469
|
});
|
|
1824
1470
|
} catch (error) {
|
|
1825
1471
|
renderer.close();
|
|
1826
|
-
if (
|
|
1472
|
+
if (ctx.retry) await restoreRetry(setup.config.root, ctx.runId);
|
|
1473
|
+
else if (setup.resume === void 0) await rm(ctx.stagingDir, {
|
|
1827
1474
|
recursive: true,
|
|
1828
1475
|
force: true
|
|
1829
1476
|
});
|
|
1830
1477
|
throw error;
|
|
1831
1478
|
}
|
|
1832
1479
|
const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions, leftOut } = executed;
|
|
1480
|
+
const { retried } = executed;
|
|
1833
1481
|
invocation.unhandledErrors = engineRun.unhandledErrors;
|
|
1834
|
-
|
|
1835
|
-
if ((interrupted || engineRun.canceled) && engines.length) {
|
|
1836
|
-
const { removeRunContainers } = await import("./docker-BlSopHpZ.js");
|
|
1837
|
-
await removeRunContainers(ctx.runId, engines);
|
|
1838
|
-
}
|
|
1482
|
+
if ((interrupted || engineRun.canceled) && plan) await cleanupRun(ctx.runId, plan.sandboxes);
|
|
1839
1483
|
if (engineRun.stopped || !plan) {
|
|
1840
1484
|
live?.remove();
|
|
1841
1485
|
renderer.close();
|
|
1842
1486
|
return EXIT.ok;
|
|
1843
1487
|
}
|
|
1488
|
+
if (ctx.retry && ctx.previous && retried.size) {
|
|
1489
|
+
const { retryEnvironmentChanges } = await import("./retry-ChzISxmy.js");
|
|
1490
|
+
const changes = retryEnvironmentChanges(ctx.previous, ctx.stagingDir, retried);
|
|
1491
|
+
if (changes.length) {
|
|
1492
|
+
questions?.cancel();
|
|
1493
|
+
live?.remove();
|
|
1494
|
+
renderer.close();
|
|
1495
|
+
await restoreRetry(setup.config.root, ctx.runId);
|
|
1496
|
+
throw new UsageError(`run ${ctx.runId}'s plan changed since it ran: ${changes.join("; ")}. ${retryVerb(ctx.retry, ctx.runId)} only with the plan they ran with, so the run is left as it was: put it back, or start a new run with reeval run.`);
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1844
1499
|
let stopped = cutShort || engineRun.canceled;
|
|
1845
1500
|
if (stopped) questions?.cancel();
|
|
1846
1501
|
let answers = stopped ? await questions?.answered() : await questions?.settle();
|
|
@@ -1850,8 +1505,8 @@ async function runCommand(flags, configPath, given) {
|
|
|
1850
1505
|
}
|
|
1851
1506
|
finishing = true;
|
|
1852
1507
|
live?.phase("finishing");
|
|
1853
|
-
const { lockfiles } = await import("./state-
|
|
1854
|
-
const { writeScratchRun } = await import("./results-
|
|
1508
|
+
const { lockfiles } = await import("./state-kimtIhBP.js");
|
|
1509
|
+
const { writeScratchRun } = await import("./results-JyuM64gy.js");
|
|
1855
1510
|
const run = await writeScratchRun({
|
|
1856
1511
|
ctx,
|
|
1857
1512
|
author: ctx.author,
|
|
@@ -1866,6 +1521,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1866
1521
|
language: { name: "typescript" },
|
|
1867
1522
|
enginePackages: engineVersions,
|
|
1868
1523
|
sandbox: sandboxVersions,
|
|
1524
|
+
...ctx.retry ? { retry: ctx.retry } : {},
|
|
1869
1525
|
worktreePatch: ctx.worktreePatch,
|
|
1870
1526
|
lockfiles: await lockfiles(ctx.config.root),
|
|
1871
1527
|
git: ctx.git,
|
|
@@ -1884,7 +1540,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1884
1540
|
status: "complete"
|
|
1885
1541
|
}) : outcome;
|
|
1886
1542
|
const code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1887
|
-
const { pruneDebug } = await import("./files-
|
|
1543
|
+
const { pruneDebug } = await import("./files-C1Rk9FYA.js");
|
|
1888
1544
|
await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
|
|
1889
1545
|
const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
|
|
1890
1546
|
renderer.onEvent({
|
|
@@ -1897,13 +1553,21 @@ async function runCommand(flags, configPath, given) {
|
|
|
1897
1553
|
...notRun > 0 && !stopped ? { notRun } : {}
|
|
1898
1554
|
});
|
|
1899
1555
|
renderer.close();
|
|
1556
|
+
if (retried.size) {
|
|
1557
|
+
const { retryCounts, retryLine } = await import("./retry-ChzISxmy.js");
|
|
1558
|
+
const line = retryLine(retryCounts(record, retried, ctx.previous));
|
|
1559
|
+
if (setup.json) console.error(line);
|
|
1560
|
+
else console.log(line);
|
|
1561
|
+
}
|
|
1900
1562
|
const stepSummary = process.env.GITHUB_STEP_SUMMARY;
|
|
1901
1563
|
if (stepSummary && run.artifactDir) await appendFile(stepSummary, await readFile(summary)).catch(() => {});
|
|
1902
1564
|
const { config } = ctx;
|
|
1903
1565
|
const mode = setup.record;
|
|
1904
|
-
const
|
|
1566
|
+
const { recordFile } = await import("./files-DnlA1A7N.js");
|
|
1567
|
+
const rerecord = (setup.retry !== void 0 || setup.resume !== void 0) && record.status === "complete" && existsSync(recordFile(config.history.dir, record.runId));
|
|
1568
|
+
const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI() || rerecord;
|
|
1905
1569
|
if (autoRecord && record.status !== "interrupted") {
|
|
1906
|
-
const { record: recordCmd } = await import("./commands-
|
|
1570
|
+
const { record: recordCmd } = await import("./commands-CfdBX-mc.js");
|
|
1907
1571
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
1908
1572
|
const line = await recordCmd(config, record.runId, {
|
|
1909
1573
|
...flags,
|
|
@@ -1917,7 +1581,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1917
1581
|
else console.log(line);
|
|
1918
1582
|
}
|
|
1919
1583
|
if (setup.json) {
|
|
1920
|
-
const { suiteResult } = await import("./suite-
|
|
1584
|
+
const { suiteResult } = await import("./suite-CKBa6MKt.js");
|
|
1921
1585
|
process.stdout.write(`${JSON.stringify({
|
|
1922
1586
|
...suiteResult(record),
|
|
1923
1587
|
outcome: ran
|
|
@@ -1934,12 +1598,24 @@ async function resumedRun(flags, filters, configPath) {
|
|
|
1934
1598
|
const path = requireConfig(configPath);
|
|
1935
1599
|
loadEnvFiles(path);
|
|
1936
1600
|
const root = dirname(path);
|
|
1937
|
-
const target = await
|
|
1601
|
+
const target = await continuedRun(root, flags);
|
|
1602
|
+
if (flags["retry-errors"] !== void 0 && target.previous && !erroredAttempts(target.previous).length) return {
|
|
1603
|
+
nothing: {
|
|
1604
|
+
nothing: `run ${target.runId} has no infra errors, so --retry-errors has nothing to run again.`,
|
|
1605
|
+
record: target.previous
|
|
1606
|
+
},
|
|
1607
|
+
saved: null,
|
|
1608
|
+
settings: null,
|
|
1609
|
+
notes: [],
|
|
1610
|
+
profileOf: void 0
|
|
1611
|
+
};
|
|
1612
|
+
if (flags.resume !== void 0 && await stoppedRetry(root, target) && (filters.length || flags.tag?.length || flags.trait?.length)) throw new UsageError(`run ${target.runId} stopped during a --retry-errors, which runs every infra-errored attempt of the run again, with its plan: leave out filters, --tag and --trait.`);
|
|
1938
1613
|
const saved = await savedInvocation(root, target);
|
|
1939
1614
|
if (!saved) return {
|
|
1615
|
+
nothing: null,
|
|
1940
1616
|
saved: null,
|
|
1941
1617
|
settings: null,
|
|
1942
|
-
notes: [`run ${target.runId} saved no --runs, profile or filters, so this resume plans with the ones given now`],
|
|
1618
|
+
notes: [`run ${target.runId} saved no --runs, profile or filters, so this ${flags.resume === void 0 ? "retry" : "resume"} plans with the ones given now`],
|
|
1943
1619
|
profileOf: void 0
|
|
1944
1620
|
};
|
|
1945
1621
|
const given = {
|
|
@@ -1947,16 +1623,24 @@ async function resumedRun(flags, filters, configPath) {
|
|
|
1947
1623
|
profile: flags.profile,
|
|
1948
1624
|
concurrency: flags.concurrency,
|
|
1949
1625
|
tag: flags.tag,
|
|
1626
|
+
trait: flags.trait,
|
|
1950
1627
|
filters
|
|
1951
1628
|
};
|
|
1952
|
-
const
|
|
1629
|
+
const flag = flags.resume === void 0 ? "--retry-errors" : "--resume";
|
|
1630
|
+
const resumed = resumeSettings(target.runId, saved, given, {
|
|
1631
|
+
root,
|
|
1632
|
+
flag
|
|
1633
|
+
});
|
|
1953
1634
|
const fromRun = flags.profile === void 0 && resumed.settings.profile !== void 0;
|
|
1954
1635
|
return {
|
|
1636
|
+
nothing: null,
|
|
1955
1637
|
saved,
|
|
1956
1638
|
...resumed,
|
|
1957
1639
|
profileOf: fromRun ? target.runId : void 0
|
|
1958
1640
|
};
|
|
1959
1641
|
}
|
|
1642
|
+
/** The run `--resume` continues, or the finished one `--retry-errors` runs the errors of again. */
|
|
1643
|
+
const continuedRun = (root, flags) => flags["retry-errors"] !== void 0 ? resolveRetry(root, flags["retry-errors"] || void 0) : resolveResume(root, flags.resume || void 0);
|
|
1960
1644
|
/**
|
|
1961
1645
|
* Everything that can fail before the evals are collected: output mode, config, profile, flags.
|
|
1962
1646
|
* `noProfile` is a resume of a run that had none, which `defaultProfile` mustn't give one;
|
|
@@ -1987,12 +1671,9 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
|
|
|
1987
1671
|
runs = Number(flags.runs);
|
|
1988
1672
|
if (!(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`--runs must be a whole number of at least 1 (got ${flags.runs})`);
|
|
1989
1673
|
}
|
|
1990
|
-
const sandboxes = [
|
|
1674
|
+
const sandboxes = [...(config.environments.length ? config.environments : [void 0]).map((e) => environmentSandbox(e, config.sandbox)), ...profile ? [profileSandbox(config, profile.defaults)] : []];
|
|
1991
1675
|
const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
|
|
1992
|
-
|
|
1993
|
-
if (!container) return hostCapacity();
|
|
1994
|
-
const { dockerCapacity } = await import("./src-BkH4GOxu.js");
|
|
1995
|
-
return await dockerCapacity(container.engine) ?? hostCapacity();
|
|
1676
|
+
return await sandboxCapacity(sandboxes) ?? hostCapacity();
|
|
1996
1677
|
});
|
|
1997
1678
|
return {
|
|
1998
1679
|
output,
|
|
@@ -2000,6 +1681,7 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
|
|
|
2000
1681
|
profile,
|
|
2001
1682
|
runs,
|
|
2002
1683
|
tags: [...flags.tag ?? [], ...profile?.tags ?? []],
|
|
1684
|
+
traits: traitFlags(flags.trait),
|
|
2003
1685
|
concurrency,
|
|
2004
1686
|
budget: profile?.budget ?? config.budget,
|
|
2005
1687
|
record: profile?.record ?? config.record,
|
|
@@ -2011,37 +1693,48 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
|
|
|
2011
1693
|
/** Named in the plan line when the project root isn't the current directory. */
|
|
2012
1694
|
configElsewhere: dirname(path) !== process.cwd(),
|
|
2013
1695
|
/** `undefined` is a new run; `''` is newest resumable; a string is a run id or prefix. */
|
|
2014
|
-
resume: flags.resume
|
|
1696
|
+
resume: flags.resume,
|
|
1697
|
+
/** Like `resume`, for `--retry-errors`. */
|
|
1698
|
+
retry: flags["retry-errors"]
|
|
2015
1699
|
};
|
|
2016
1700
|
}
|
|
1701
|
+
/** What a retry, or a resume that finishes one, does to the run's errors, for its refusals. */
|
|
1702
|
+
const retryVerb = (retry, runId) => retry.resume ? `--resume finishes run ${runId}'s --retry-errors, which runs its errors again` : "--retry-errors runs its errors again";
|
|
2017
1703
|
/** The run's id and the state it starts from: commit, uncommitted changes, author. */
|
|
2018
|
-
async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
|
|
1704
|
+
async function startRun({ config, profile, runs, tags, traits, concurrency, verbosity, configFile, resume, retry, dryRun }) {
|
|
2019
1705
|
const startedAt = /* @__PURE__ */ new Date();
|
|
2020
1706
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
2021
|
-
const { worktreePatch } = await import("./state-
|
|
1707
|
+
const { worktreePatch } = await import("./state-kimtIhBP.js");
|
|
2022
1708
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
2023
1709
|
const who = await author(config.root);
|
|
2024
|
-
const { stagingDir, sweepStaging } = await import("./results-
|
|
2025
|
-
const { scratchDir } = await import("./files-
|
|
1710
|
+
const { stagingDir, sweepStaging } = await import("./results-JyuM64gy.js");
|
|
1711
|
+
const { scratchDir } = await import("./files-DnlA1A7N.js");
|
|
2026
1712
|
const scratch = scratchDir(config.root);
|
|
2027
|
-
if (resume !== void 0) {
|
|
2028
|
-
const target = await
|
|
2029
|
-
|
|
1713
|
+
if (resume !== void 0 || retry !== void 0) {
|
|
1714
|
+
const target = await continuedRun(config.root, {
|
|
1715
|
+
resume,
|
|
1716
|
+
"retry-errors": retry
|
|
1717
|
+
});
|
|
1718
|
+
if (!dryRun && retry === void 0) await prepareResume(target, config.root);
|
|
1719
|
+
else if (!dryRun) holdRetry(target, config.root);
|
|
1720
|
+
const stopped = retry === void 0 ? await stoppedRetry(config.root, target) : null;
|
|
2030
1721
|
await sweepStaging(config.root, target.runId);
|
|
2031
1722
|
const aside = dryRun ? {
|
|
2032
1723
|
record: target.previous,
|
|
2033
1724
|
artifacts: target.previousArtifacts
|
|
2034
1725
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
2035
1726
|
if (!dryRun) try {
|
|
2036
|
-
|
|
2037
|
-
await removeRunContainers(target.runId, ["docker", "podman"]);
|
|
1727
|
+
await cleanupRun(target.runId);
|
|
2038
1728
|
} catch {}
|
|
1729
|
+
const { unfoldCases } = await import("./cases-Dy0mifal.js");
|
|
1730
|
+
const previous = aside.record ?? target.previous;
|
|
2039
1731
|
return {
|
|
2040
1732
|
runId: target.runId,
|
|
2041
1733
|
stagingDir: stagingDir(config.root, target.runId),
|
|
2042
1734
|
config,
|
|
2043
1735
|
runs,
|
|
2044
1736
|
tags,
|
|
1737
|
+
traits,
|
|
2045
1738
|
concurrency,
|
|
2046
1739
|
profile,
|
|
2047
1740
|
verbosity,
|
|
@@ -2051,8 +1744,17 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
2051
1744
|
worktreePatch: patch,
|
|
2052
1745
|
author: who,
|
|
2053
1746
|
scratch,
|
|
2054
|
-
previous:
|
|
2055
|
-
previousArtifacts: aside.artifacts ?? target.previousArtifacts
|
|
1747
|
+
previous: previous && unfoldCases(previous),
|
|
1748
|
+
previousArtifacts: aside.artifacts ?? target.previousArtifacts,
|
|
1749
|
+
...retry === void 0 ? {} : { retryTarget: target },
|
|
1750
|
+
...retry !== void 0 ? { retry: {
|
|
1751
|
+
startedAt,
|
|
1752
|
+
resume: false
|
|
1753
|
+
} } : stopped ? { retry: {
|
|
1754
|
+
startedAt,
|
|
1755
|
+
resume: true,
|
|
1756
|
+
...stopped.began ? { began: stopped.began } : {}
|
|
1757
|
+
} } : {}
|
|
2056
1758
|
};
|
|
2057
1759
|
}
|
|
2058
1760
|
const stamps = new Set([scratch, config.history.dir].flatMap((dir) => existsSync(dir) ? readdirSync(dir).map((name) => name.slice(0, 16)) : []));
|
|
@@ -2065,6 +1767,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
2065
1767
|
config,
|
|
2066
1768
|
runs,
|
|
2067
1769
|
tags,
|
|
1770
|
+
traits,
|
|
2068
1771
|
concurrency,
|
|
2069
1772
|
profile,
|
|
2070
1773
|
verbosity,
|
|
@@ -2116,12 +1819,12 @@ function namesDeclaredSecret(evals) {
|
|
|
2116
1819
|
*/
|
|
2117
1820
|
async function planEfforts(planned) {
|
|
2118
1821
|
const live = planned.filter((p) => !p.attempt.plan.skip);
|
|
2119
|
-
const { effortNotes, withDefaultEffort } = await import("./default-effort-
|
|
1822
|
+
const { effortNotes, withDefaultEffort } = await import("./default-effort-BAy8ixd1.js");
|
|
2120
1823
|
let cursor;
|
|
2121
1824
|
let unchecked = "CURSOR_API_KEY isn't set";
|
|
2122
1825
|
const key = process.env.CURSOR_API_KEY;
|
|
2123
1826
|
if (key && live.some((p) => p.attempt.harness.kind === "cursor")) {
|
|
2124
|
-
const { checkCursorPlan } = await import("./cursor-catalog-
|
|
1827
|
+
const { checkCursorPlan } = await import("./cursor-catalog-BSn9xjmo.js");
|
|
2125
1828
|
const checked = await checkCursorPlan(key, live);
|
|
2126
1829
|
if (checked instanceof Error) unchecked = `its request failed: ${checked.message}`;
|
|
2127
1830
|
else {
|
|
@@ -2142,21 +1845,23 @@ async function planEfforts(planned) {
|
|
|
2142
1845
|
* or Docker problem (exit 2). Then runs the engine with live output and Ctrl-C handling.
|
|
2143
1846
|
*/
|
|
2144
1847
|
async function executeRun(ctx, setup, filters, renderer, started) {
|
|
2145
|
-
const { runId, stagingDir, config, runs, tags, concurrency, profile, verbosity
|
|
1848
|
+
const { runId, stagingDir, config, runs, tags, traits, concurrency, profile, verbosity } = ctx;
|
|
1849
|
+
const { configFile } = ctx;
|
|
2146
1850
|
const run = JSON.parse(JSON.stringify({
|
|
2147
1851
|
runId,
|
|
2148
1852
|
stagingDir,
|
|
2149
1853
|
config,
|
|
2150
1854
|
runs,
|
|
2151
1855
|
tags,
|
|
1856
|
+
traits,
|
|
2152
1857
|
concurrency,
|
|
2153
1858
|
profile,
|
|
2154
1859
|
verbosity,
|
|
2155
1860
|
configFile,
|
|
2156
1861
|
completed: []
|
|
2157
1862
|
}));
|
|
2158
|
-
const { runEngine } = await import("./node-
|
|
2159
|
-
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-
|
|
1863
|
+
const { runEngine } = await import("./node-D7pAU--0.js");
|
|
1864
|
+
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-Cgq9NCEX.js");
|
|
2160
1865
|
let plan;
|
|
2161
1866
|
let sandboxVersions = { images: [] };
|
|
2162
1867
|
let preflightStop;
|
|
@@ -2166,13 +1871,21 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2166
1871
|
let live;
|
|
2167
1872
|
let kept = /* @__PURE__ */ new Map();
|
|
2168
1873
|
let leftOut = [];
|
|
1874
|
+
let retried = /* @__PURE__ */ new Set();
|
|
2169
1875
|
let questions;
|
|
2170
1876
|
const onPlan = async (collected) => {
|
|
2171
1877
|
const { planned, excluded, matched, unmatched } = collected;
|
|
2172
1878
|
if (unmatched.length) throw unmatchedFilter(config, unmatched);
|
|
2173
1879
|
if (matched === 0) throw noMatch(config, filters, tags);
|
|
2174
1880
|
checkLimitHarness(profile, config.file, collected);
|
|
1881
|
+
checkTraitKeys(traitFilters(traits, profile), collected.traits);
|
|
1882
|
+
const continuing = setup.resume !== void 0 || ctx.retryTarget !== void 0;
|
|
1883
|
+
const { errored } = continuing ? noteRetries(planned, stagingDir, ctx.previous, false) : { errored: /* @__PURE__ */ new Set() };
|
|
2175
1884
|
const completed = completedKeysFor(planned, stagingDir, ctx.previous);
|
|
1885
|
+
if (ctx.retry) for (const { key } of planned) {
|
|
1886
|
+
const was = previousAttempt(ctx.previous, key);
|
|
1887
|
+
if (was && was.outcome !== "infra_error") completed.add(key);
|
|
1888
|
+
}
|
|
2176
1889
|
const skipped = planned.map((p) => completed.has(p.key) ? {
|
|
2177
1890
|
...p,
|
|
2178
1891
|
attempt: {
|
|
@@ -2190,7 +1903,25 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2190
1903
|
if (named) throw named;
|
|
2191
1904
|
const efforts = await planEfforts(skipped);
|
|
2192
1905
|
if (efforts) run.cursorEfforts = efforts;
|
|
2193
|
-
|
|
1906
|
+
if (ctx.retry && ctx.previous) {
|
|
1907
|
+
const ranOn = ctx.previous.stack?.reeval;
|
|
1908
|
+
if (ranOn && ranOn !== version) throw new UsageError(`run ${runId} ran on reeval ${ranOn}, and this is ${version}. ${retryVerb(ctx.retry, runId)} only on the reeval they ran on: install reeval ${ranOn}, or start a new run with reeval run.`);
|
|
1909
|
+
const { retryChanges } = await import("./retry-ChzISxmy.js");
|
|
1910
|
+
const { judgePart } = await import("./assemble-UPCRrtOz.js");
|
|
1911
|
+
const { engineVersions } = await import("./node-D7pAU--0.js");
|
|
1912
|
+
const judged = judgePart(config, profile, version, engineVersions);
|
|
1913
|
+
const runs = new Set(planned.map((p) => p.key).filter((key) => !completed.has(key)));
|
|
1914
|
+
const changes = await retryChanges(ctx.previous, planned, runs, config.root, judged);
|
|
1915
|
+
if (changes.length) throw new UsageError(`run ${runId}'s plan changed since it ran: ${changes.join("; ")}. ${retryVerb(ctx.retry, runId)} only with the plan they ran with: put it back, or start a new run with reeval run.`);
|
|
1916
|
+
if (ctx.retryTarget && !setup.dryRun) {
|
|
1917
|
+
await prepareResume(ctx.retryTarget, config.root, { held: true });
|
|
1918
|
+
const aside = await loadPrevious(stagingDir);
|
|
1919
|
+
ctx.previous = aside.record ?? ctx.previous;
|
|
1920
|
+
ctx.previousArtifacts = aside.artifacts;
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
retried = continuing && !setup.dryRun ? noteRetries(planned, stagingDir, ctx.previous, true).errored : errored;
|
|
1924
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-047Dcd-a.js") : void 0;
|
|
2194
1925
|
const judge = runJudgeModel(config, profile);
|
|
2195
1926
|
const judgeModel = judge === void 0 ? null : judgeModelName(judge);
|
|
2196
1927
|
plan = await buildPlan({
|
|
@@ -2207,7 +1938,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2207
1938
|
});
|
|
2208
1939
|
plan.runId = runId;
|
|
2209
1940
|
const cloud = ciRunner(process.env);
|
|
2210
|
-
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.
|
|
1941
|
+
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
|
|
2211
1942
|
if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
|
|
2212
1943
|
plan.warnings.push(...started.notes);
|
|
2213
1944
|
if (started.narrowed) {
|
|
@@ -2215,16 +1946,17 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2215
1946
|
const notRun = leftOut.filter((l) => !l.done).length;
|
|
2216
1947
|
if (notRun) plan.warnings.push(`${notRun === 1 ? "1 attempt" : `${notRun} attempts`} run ${runId} planned ${notRun === 1 ? "isn't" : "aren't"} in this resume's filters: ${notRun === 1 ? "it stays" : "they stay"} not run and the run stays incomplete. reeval run --resume ${runId} runs ${notRun === 1 ? "it" : "them"}`);
|
|
2217
1948
|
}
|
|
2218
|
-
if (completed.size) plan.resume = {
|
|
1949
|
+
if (completed.size || retried.size) plan.resume = {
|
|
2219
1950
|
runId,
|
|
2220
|
-
completed: completed.size
|
|
1951
|
+
completed: completed.size,
|
|
1952
|
+
...retried.size ? { retried: retried.size } : {}
|
|
2221
1953
|
};
|
|
2222
1954
|
run.completed = [...completed];
|
|
2223
1955
|
const done = leftOut.filter((l) => l.done).map((l) => l.key);
|
|
2224
1956
|
kept = keptAttempts([...completed, ...done], stagingDir, ctx.previous);
|
|
2225
|
-
const { imageStages, imageTargets } = await import("./images-
|
|
1957
|
+
const { imageStages, imageTargets } = await import("./images-BNgcR77K.js");
|
|
2226
1958
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
2227
|
-
id: p.id,
|
|
1959
|
+
id: p.attempt.family?.id ?? p.id,
|
|
2228
1960
|
harness: p.attempt.harness,
|
|
2229
1961
|
sandbox: p.attempt.sandbox,
|
|
2230
1962
|
plugins: p.attempt.pluginBuilds ?? []
|
|
@@ -2241,13 +1973,13 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2241
1973
|
return false;
|
|
2242
1974
|
}
|
|
2243
1975
|
const elsewhere = setup.configElsewhere ? configFile : void 0;
|
|
2244
|
-
const { scratchDir } = await import("./files-
|
|
1976
|
+
const { scratchDir } = await import("./files-DnlA1A7N.js");
|
|
2245
1977
|
const lines = [];
|
|
2246
1978
|
if (setup.dryRun) {
|
|
2247
1979
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
2248
1980
|
lines.push(...planTable(plan, todos));
|
|
2249
|
-
const { imageStatusLines } = await import("./images-
|
|
2250
|
-
const { dockerAvailable, engineSpec } = await import("./src-
|
|
1981
|
+
const { imageStatusLines } = await import("./images-BNgcR77K.js");
|
|
1982
|
+
const { dockerAvailable, engineSpec } = await import("./src-DtodMT59.js");
|
|
2251
1983
|
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
2252
1984
|
const down = [];
|
|
2253
1985
|
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
@@ -2256,7 +1988,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2256
1988
|
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
2257
1989
|
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
2258
1990
|
}
|
|
2259
|
-
const { costLine } = await import("./cost-
|
|
1991
|
+
const { costLine } = await import("./cost-Cr26Alhq.js");
|
|
2260
1992
|
const cost = plan.cost && costLine(plan.cost);
|
|
2261
1993
|
if (cost) lines.push(cost);
|
|
2262
1994
|
}
|
|
@@ -2314,6 +2046,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2314
2046
|
}
|
|
2315
2047
|
const missing = missingCredentials(config, process.env, {
|
|
2316
2048
|
harnesses: plan.harnessDescriptors,
|
|
2049
|
+
sandboxes: plan.sandboxes,
|
|
2317
2050
|
profile,
|
|
2318
2051
|
questionModels: [...questionModels],
|
|
2319
2052
|
judges: [...judges.values()]
|
|
@@ -2330,7 +2063,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2330
2063
|
});
|
|
2331
2064
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
2332
2065
|
if (asked.length) {
|
|
2333
|
-
const { questionRunner } = await import("./questions-
|
|
2066
|
+
const { questionRunner } = await import("./questions-CWlR9ffu.js");
|
|
2334
2067
|
const warned = /* @__PURE__ */ new Set();
|
|
2335
2068
|
questions = questionRunner({
|
|
2336
2069
|
evals: asked,
|
|
@@ -2372,7 +2105,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2372
2105
|
}
|
|
2373
2106
|
});
|
|
2374
2107
|
}
|
|
2375
|
-
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-
|
|
2108
|
+
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-CcQRT31C.js");
|
|
2376
2109
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
2377
2110
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
2378
2111
|
workspace: a.workspace,
|
|
@@ -2550,9 +2283,9 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2550
2283
|
throw error;
|
|
2551
2284
|
} finally {
|
|
2552
2285
|
clearTimeout(budgetTimer);
|
|
2553
|
-
const { removePrepared } = await import("./preflight-
|
|
2286
|
+
const { removePrepared } = await import("./preflight-R-InZCQb.js");
|
|
2554
2287
|
removePrepared(config.root, runId);
|
|
2555
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
2288
|
+
const { removePreparedWorkspaces } = await import("./workspace-CcQRT31C.js");
|
|
2556
2289
|
removePreparedWorkspaces(config.root, runId);
|
|
2557
2290
|
}
|
|
2558
2291
|
if (preflightStop) throw preflightStop;
|
|
@@ -2567,7 +2300,9 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2567
2300
|
cutShort,
|
|
2568
2301
|
live,
|
|
2569
2302
|
questions,
|
|
2570
|
-
leftOut
|
|
2303
|
+
leftOut,
|
|
2304
|
+
/** The attempts that ran again after an infra error, by key; none on a dry run. */
|
|
2305
|
+
retried: setup.dryRun ? /* @__PURE__ */ new Set() : retried
|
|
2571
2306
|
};
|
|
2572
2307
|
}
|
|
2573
2308
|
process.stdout.on("error", (error) => {
|
|
@@ -2601,7 +2336,7 @@ function finish(code, error, kind) {
|
|
|
2601
2336
|
} : null,
|
|
2602
2337
|
unhandledErrors: invocation.unhandledErrors.map((e) => redactSecrets(e))
|
|
2603
2338
|
});
|
|
2604
|
-
const broke = kind === "crash" || code === EXIT.infra || invocation.unhandledErrors.length > 0;
|
|
2339
|
+
const broke = kind === "crash" || code === EXIT.infra && !invocation.machineState || invocation.unhandledErrors.length > 0;
|
|
2605
2340
|
const quiet = !!process.env.REEVAL_NO_REPORT_HINT || command === "report" || !!flags?.json || flags?.ui === "agent" || process.env.REEVAL_UI === "agent" || !flags?.ui && !process.env.REEVAL_UI && !!detectAgent().name;
|
|
2606
2341
|
if (broke && !quiet) console.error(dim("hint: if this looks like a bug in reeval, reeval report collects a redacted report to file (REEVAL_NO_REPORT_HINT=1 hides this)"));
|
|
2607
2342
|
}
|
|
@@ -2635,4 +2370,4 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
2635
2370
|
});
|
|
2636
2371
|
export {};
|
|
2637
2372
|
|
|
2638
|
-
//# sourceMappingURL=cli-
|
|
2373
|
+
//# sourceMappingURL=cli-20Fpvs5H.js.map
|