reeval 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
- package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
- package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
- package/dist/default-effort-DGhQmu95.js.map +1 -0
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
- package/dist/eval-source-D3sbAzt-.js.map +1 -0
- package/dist/evaluate-COdzf7Hl.d.ts +92 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
- package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
- package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/harness-C2berMiM.js +105 -0
- package/dist/harness-C2berMiM.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/ink-list-BpfidQ1h.js +274 -0
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
- package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-CzgpsTqv.js +246 -0
- package/dist/judgments-CzgpsTqv.js.map +1 -0
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
- package/dist/live-CgYbO4Te.js.map +1 -0
- package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
- package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
- package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
- package/dist/questions-CWlR9ffu.js.map +1 -0
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/remarks-DPCONFxZ.js +138 -0
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/resume-AW5oXzUc.js +691 -0
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
- package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/suite-COglUhPd.js +1272 -0
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
- package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
- package/dist/vertex-Eb2wqahk.js.map +1 -0
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +238 -90
- package/dist/view/index.js +11 -9
- package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
- package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +26 -11
- package/skills/eval-rubric/SKILL.md +31 -4
- package/dist/adapter-DyvDj4ZN.js.map +0 -1
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/command-TLmY-rYp.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-DtSX8E_V.js +0 -101
- package/dist/contrasts-DtSX8E_V.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker--2ytOHcu.js.map +0 -1
- package/dist/doctor-eE65YpEv.js.map +0 -1
- package/dist/environment-Cy3FVLR5.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-BoDKeuJu.js.map +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-BVZHDf_z.js.map +0 -1
- package/dist/init-BWnMbUnc.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/judge-ByEUlzOM.js.map +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/node-CwP1G178.js.map +0 -1
- package/dist/opencode-providers-yNKyXD49.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/plugins-BEs2Am3L.js.map +0 -1
- package/dist/preflight-50Qw5e5K.js.map +0 -1
- package/dist/provenance-Cc8fFE7Z.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-CP_bBtYm.js +0 -1
- package/dist/providers-DZSTlC9R.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-DeZZT8PK.js +0 -116
- package/dist/remarks-DeZZT8PK.js.map +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resolve-B_JlQ0La.js.map +0 -1
- package/dist/resume-CexsWuGK.js +0 -434
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/runtime-CSZ1hjlN.js.map +0 -1
- package/dist/sandbox-B5W8lLL1.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BKDZO3ra.js +0 -2
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/src-C9elednc.js +0 -2
- package/dist/src-CM7xdJFA.js +0 -2
- package/dist/src-CWEGknZo.js +0 -2
- package/dist/src-Cpy9MNS3.js.map +0 -1
- package/dist/src-DVaAKCsH.js +0 -1
- package/dist/src-DurP--_D.js +0 -2
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CKoN5NAx.js.map +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -1,37 +1,34 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { l as
|
|
8
|
-
import { y as vertexSettingsIn } from "./vertex-
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { n as loadEnvFiles, t as inCI } from "./env-
|
|
2
|
+
import { g as parseDuration } from "./shared-Cfi45Bvy.js";
|
|
3
|
+
import { a as plural, i as formatDuration, n as clip } from "./text-B8-W3Cvn.js";
|
|
4
|
+
import { l as sandboxCapacity, n as capabilitiesOf, r as cleanupRun } from "./kinds-C1oY3qyC.js";
|
|
5
|
+
import { N as workspaceLabel, T as profileSandbox, d as environmentSandbox } from "./environment-DB8o3hGi.js";
|
|
6
|
+
import { a as newRunId, i as fromCwd } from "./ids-C_advdJA.js";
|
|
7
|
+
import { I as checkLimitHarness, V as harnessId, X as plannedSandboxes, Y as plannedSandboxIds, _t as resolveOutput, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, dt as ciRunner, f as missingCredentials, ft as directOnCloud, g as selectProfile, h as runJudgeModel, ht as isDirect, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig, v as gatewayScope } from "./config-B9mB1kF5.js";
|
|
8
|
+
import { y as vertexSettingsIn } from "./vertex-Eb2wqahk.js";
|
|
9
|
+
import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
|
|
10
|
+
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-COMXT1Hx.js";
|
|
11
|
+
import { n as loadEnvFiles, t as inCI } from "./env-8x1O8I1M.js";
|
|
12
12
|
import { o as PACKAGE_VERSION } from "./root-Co0jhOT5.js";
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import { u as suiteResult } from "./suite-BtHPBFYL.js";
|
|
19
|
-
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-ChkleEVm.js";
|
|
13
|
+
import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
|
|
14
|
+
import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-DKnYkqeK.js";
|
|
15
|
+
import { k as runOutcome } from "./model-V3mMXp6x.js";
|
|
16
|
+
import { u as suiteResult } from "./suite-COglUhPd.js";
|
|
17
|
+
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-Dsr79Qy1.js";
|
|
20
18
|
import { n as stripAnsi, t as colors } from "./colors-DcXBvAlU.js";
|
|
21
|
-
import {
|
|
22
|
-
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-Cc8fFE7Z.js";
|
|
19
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-zsc4--dP.js";
|
|
23
20
|
import { t as budgetMarker } from "./budget-BcxLYY5L.js";
|
|
24
|
-
import {
|
|
25
|
-
import { n as
|
|
26
|
-
import {
|
|
27
|
-
import { i as
|
|
28
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-BGRs1r_C.js";
|
|
21
|
+
import { S as writeLiveProgress, _ as resumeSettings, b as stoppedRetry, c as liveAttempts, d as prepareResume, f as previousAttempt, g as restoreRetry, h as resolveRetry, l as loadPrevious, m as resolveResume, n as erroredAttempts, o as keptAttempts, r as holdRetry, s as leftOutAttempts, t as completedKeysFor, u as noteRetries, v as savedFilters, x as writeCurrent, y as savedInvocation } from "./resume-AW5oXzUc.js";
|
|
22
|
+
import { n as preflight } from "./preflight-CtlloeOQ.js";
|
|
23
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CxGNOlh0.js";
|
|
24
|
+
import { _ as phaseOf, c as pointerLine, d as attemptAxes, f as debugLines, g as outcomeOf, h as offset, i as listLines, m as failureLines, n as SPINNER, p as errorCheck, s as progressText, t as LiveList, u as statusList, v as progressLine, y as tailLines } from "./list-DuThsE28.js";
|
|
29
25
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
30
26
|
import { dirname, join } from "node:path";
|
|
31
27
|
import { inspect } from "node:util";
|
|
32
28
|
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
33
29
|
import { detectAgent } from "std-env";
|
|
34
30
|
import { Command, Option } from "commander";
|
|
31
|
+
import stringWidth from "string-width";
|
|
35
32
|
// src/cli-parse.ts
|
|
36
33
|
const OPTIONS = Object.fromEntries(Object.entries(FLAGS).map(([name, spec]) => {
|
|
37
34
|
const short = "short" in spec ? `-${spec.short}, ` : "";
|
|
@@ -162,259 +159,155 @@ function usageError(error, command) {
|
|
|
162
159
|
}
|
|
163
160
|
return new UsageError(first.charAt(0).toUpperCase() + first.slice(1));
|
|
164
161
|
}
|
|
165
|
-
// src/
|
|
166
|
-
/**
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
const baked = imageHarness(harness);
|
|
173
|
-
const install = sandbox.install?.length ? " + install" : "";
|
|
174
|
-
return `${sandboxId(sandbox)}${install}${pluginBuildsLabel(plugins)}${baked === "cursor" ? "" : ` · ${baked}`}`;
|
|
175
|
-
}
|
|
176
|
-
/** A run that stopped in preflight on an infra failure: exit 3. */
|
|
177
|
-
var PreflightInfraError = class extends Error {
|
|
178
|
-
name = "PreflightInfraError";
|
|
179
|
-
};
|
|
162
|
+
// src/ui/header.ts
|
|
163
|
+
/** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
|
|
164
|
+
const GLYPH = 5;
|
|
165
|
+
/** `Est. Time: `, the longest key. */
|
|
166
|
+
const KEY = 11;
|
|
167
|
+
/** `Warning: `, the longer note label. */
|
|
168
|
+
const LABEL = 9;
|
|
180
169
|
/**
|
|
181
|
-
*
|
|
182
|
-
*
|
|
183
|
-
* run's workers. A config-level sandbox that fails stops the run (exit 2 for a mistake in the
|
|
184
|
-
* config, exit 3 otherwise); any other failure fails only the attempts that need it.
|
|
170
|
+
* `text` broken at spaces into lines of at most `width` columns, not counting color codes; a
|
|
171
|
+
* longer word keeps a line. A `·` stays with the word before it, so no line starts with one.
|
|
185
172
|
*/
|
|
186
|
-
|
|
187
|
-
const
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
...options.onEvent ? { onEvent: options.onEvent } : {},
|
|
198
|
-
log: (line) => process.env.REEVAL_DEBUG && console.error(line)
|
|
199
|
-
});
|
|
200
|
-
} catch (error) {
|
|
201
|
-
if (error instanceof PrepareError) throw error.failure.usage ? new UsageError(error.message) : new PreflightInfraError(error.message);
|
|
202
|
-
if (error instanceof RunStopError) throw error.outcome.failure.usage ? new UsageError(error.message) : new PreflightInfraError(error.message);
|
|
203
|
-
throw error;
|
|
204
|
-
}
|
|
205
|
-
writePrepared(options.root, options.runId, outcomes);
|
|
206
|
-
const images = [];
|
|
207
|
-
for (const o of outcomes) {
|
|
208
|
-
if (!o.ok || o.target.sandbox.kind !== "docker") continue;
|
|
209
|
-
const { ref, id, agentCli } = attemptImage(o.handle);
|
|
210
|
-
if (images.some((i) => i.id === id)) continue;
|
|
211
|
-
const info = await imageInfo(ref, o.target.sandbox.engine);
|
|
212
|
-
images.push({
|
|
213
|
-
ref,
|
|
214
|
-
id,
|
|
215
|
-
repoDigests: info?.repoDigests ?? [],
|
|
216
|
-
agentCli: agentCli ?? info?.labels[LABEL$1.agentVersion] ?? null
|
|
217
|
-
});
|
|
218
|
-
}
|
|
219
|
-
return { images };
|
|
173
|
+
function wrap(text, width) {
|
|
174
|
+
const words = [];
|
|
175
|
+
for (const word of text.split(/\s+/).filter(Boolean)) if (word === "·" && words.length) words[words.length - 1] += " ·";
|
|
176
|
+
else words.push(word);
|
|
177
|
+
const lines = [];
|
|
178
|
+
let line = "";
|
|
179
|
+
for (const word of words) if (line && stripAnsi(line).length + 1 + stripAnsi(word).length > width) {
|
|
180
|
+
lines.push(line);
|
|
181
|
+
line = word;
|
|
182
|
+
} else line = line ? `${line} ${word}` : word;
|
|
183
|
+
return line ? [...lines, line] : lines;
|
|
220
184
|
}
|
|
221
|
-
|
|
222
|
-
/**
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
const
|
|
233
|
-
const
|
|
185
|
+
const indent = (n) => " ".repeat(n);
|
|
186
|
+
/** `REEVAL v0.16.1`, then the run's facts, one `➜ Key: value` line each. */
|
|
187
|
+
function headerLines(c, h, width) {
|
|
188
|
+
const name = `REEVAL v${h.version}`;
|
|
189
|
+
const title = c.bold(c.depth === 4 ? name : c.teal(name));
|
|
190
|
+
const valueWidth = width - GLYPH - KEY;
|
|
191
|
+
const row = (key, value) => {
|
|
192
|
+
const values = typeof value === "string" ? [value] : value;
|
|
193
|
+
const joined = values.join(", ");
|
|
194
|
+
return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
|
|
195
|
+
};
|
|
196
|
+
const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
|
|
197
|
+
const continued = [h.resumed ? `${plural(h.resumed, "attempt")} done` : "", h.retried ? `${plural(h.retried, "error")} to retry` : ""].filter(Boolean);
|
|
198
|
+
const resumed = continued.length ? c.dim(` (resumed, ${continued.join(", ")})`) : "";
|
|
199
|
+
const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
|
|
234
200
|
return [
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
...
|
|
201
|
+
h.dryRun ? `${title} ${c.dim("dry run")}` : title,
|
|
202
|
+
"",
|
|
203
|
+
...row("Run", `${h.runId}${resumed}`),
|
|
204
|
+
...h.config ? row("Config", h.config) : [],
|
|
205
|
+
...h.profile ? row("Profile", `${h.profile.name}${reason}`) : [],
|
|
206
|
+
...row(many("Harness", h.harnesses.length, "Harnesses"), h.harnesses.map((id) => harness(c, h, id))),
|
|
207
|
+
...row(many("Sandbox", h.sandboxes.length, "Sandboxes"), h.sandboxes),
|
|
208
|
+
...h.judges.length ? row(many("Judge", h.judges.length), h.judges) : [],
|
|
209
|
+
...row("Scope", scope(c, h)),
|
|
210
|
+
...row("Est. Time", estimate(c, h)),
|
|
211
|
+
...h.record && !h.dryRun ? row("Record", h.record) : []
|
|
238
212
|
];
|
|
239
213
|
}
|
|
240
|
-
/**
|
|
241
|
-
function
|
|
242
|
-
const
|
|
243
|
-
|
|
244
|
-
}
|
|
245
|
-
function outcomeOf(e) {
|
|
246
|
-
if (e.state === "skipped") return "skipped";
|
|
247
|
-
if (e.meta?.errorKind === "aborted") return "canceled";
|
|
248
|
-
return e.meta?.outcome ?? (e.state === "passed" ? "pass" : "fail");
|
|
249
|
-
}
|
|
250
|
-
/** What tells an eval's attempts apart: the axes that vary (`cursor:composer-2.5 · with-skill`), else the harness. */
|
|
251
|
-
function attemptAxes(a, fallback) {
|
|
252
|
-
return a?.label.replace(/\s*#\d+$/, "") || a?.harness.kind || fallback;
|
|
253
|
-
}
|
|
254
|
-
/** A command on one line: the first line, cut to 80 characters at a word boundary. */
|
|
255
|
-
function oneLine(text, max = 80) {
|
|
256
|
-
const [first = ""] = text.trim().split("\n");
|
|
257
|
-
const cut = clip(first, max);
|
|
258
|
-
return cut === first && text.trim().includes("\n") ? `${first}…` : cut;
|
|
259
|
-
}
|
|
260
|
-
/** One line per worker progress event, without prefix or offset. */
|
|
261
|
-
function progressLine(c, p) {
|
|
262
|
-
switch (p.kind) {
|
|
263
|
-
case "sandbox": return c.dim("sandbox starting");
|
|
264
|
-
case "sandbox-ready": return `sandbox ready ${c.dim(formatDuration(p.ms))}`;
|
|
265
|
-
case "turn-start": return `turn ${p.turn} started`;
|
|
266
|
-
case "turn-end": {
|
|
267
|
-
const status = p.status === "completed" ? "done" : c.red(p.status);
|
|
268
|
-
const activity = [plural(p.tools, "tool call"), plural(p.fileEdits, "file edit")].join(", ");
|
|
269
|
-
const error = p.error ? `: ${c.red(oneLine(p.error, 160))}` : "";
|
|
270
|
-
return `turn ${p.turn} ${status}${error} ${c.dim(`${formatDuration(p.ms)} · ${activity}`)}`;
|
|
271
|
-
}
|
|
272
|
-
case "check-start": return c.dim(`$ ${oneLine(p.command)}`);
|
|
273
|
-
case "check-end": {
|
|
274
|
-
const exit = p.exitCode === 0 ? c.dim("exit 0") : c.red(`exit ${p.exitCode}`);
|
|
275
|
-
const tries = p.try && p.tries ? triesNote(p.exitCode, p.try, p.tries) : null;
|
|
276
|
-
return `$ ${oneLine(p.command)} ${exit} ${c.dim([formatDuration(p.ms), tries].filter(Boolean).join(" · "))}`;
|
|
277
|
-
}
|
|
278
|
-
case "judge-start": return c.dim(`judging: ${oneLine(p.question, 120)}`);
|
|
279
|
-
case "judge-end": return judgeLine(c, p);
|
|
280
|
-
case "cleanup": return c.dim("cleaning up");
|
|
281
|
-
case "warning": return c.yellow(`warning: ${p.text}`);
|
|
282
|
-
case "browser-download": {
|
|
283
|
-
const total = p.bytes == null ? "?" : formatBytes(p.bytes);
|
|
284
|
-
return `downloading ${p.name} ${p.version} ${formatBytes(p.received)} / ${total} → ${p.cache}`;
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
function formatBytes(n) {
|
|
289
|
-
if (n < 1024) return `${n} B`;
|
|
290
|
-
if (n < 1048576) return `${(n / 1024).toFixed(1)} KB`;
|
|
291
|
-
return `${(n / 1048576).toFixed(1)} MB`;
|
|
292
|
-
}
|
|
293
|
-
function judgeLine(c, j) {
|
|
294
|
-
const verdict = j.verdict === "pass" ? c.green("pass") : c.red(j.verdict);
|
|
295
|
-
const score = j.score === null ? "" : ` ${j.score.toFixed(2)}`;
|
|
296
|
-
const threshold = j.threshold === null ? "" : ` ${c.dim(`(≥ ${j.threshold})`)}`;
|
|
297
|
-
return `${c.dim("judge")} ${verdict}${score}${threshold} ${j.question}`;
|
|
298
|
-
}
|
|
299
|
-
/** The phase a running attempt is in, for the heartbeat. */
|
|
300
|
-
function phaseOf(p) {
|
|
301
|
-
switch (p?.kind) {
|
|
302
|
-
case void 0: return "starting";
|
|
303
|
-
case "sandbox": return "sandbox";
|
|
304
|
-
case "sandbox-ready": return "sandbox ready";
|
|
305
|
-
case "turn-start": return `turn ${p.turn}`;
|
|
306
|
-
case "turn-end": return `turn ${p.turn} done`;
|
|
307
|
-
case "check-start":
|
|
308
|
-
case "check-end": return "checks";
|
|
309
|
-
case "judge-start":
|
|
310
|
-
case "judge-end": return "judging";
|
|
311
|
-
case "cleanup": return "cleanup";
|
|
312
|
-
case "browser-download": return "browser download";
|
|
313
|
-
case "warning": return "running";
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
function capLines(text, max) {
|
|
317
|
-
const lines = text.replace(/\s+$/, "").split("\n");
|
|
318
|
-
if (lines.length <= max) return lines;
|
|
319
|
-
return [...lines.slice(0, max), `(+${lines.length - max} more lines)`];
|
|
214
|
+
/** A harness id, with the effort it runs at when the id doesn't name it. */
|
|
215
|
+
function harness(c, h, id) {
|
|
216
|
+
const effort = h.efforts?.[id];
|
|
217
|
+
if (effort === void 0 || id.includes("[effort=")) return id;
|
|
218
|
+
return `${id} ${`(effort: ${effort})`.split(" ").map((w) => c.dim(w)).join(" ")}`;
|
|
320
219
|
}
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
const
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
`(${lines.length - 1 - rest} more lines)`,
|
|
329
|
-
...lines.slice(-rest)
|
|
220
|
+
function scope(c, h) {
|
|
221
|
+
const attempts = h.fewest < h.attempts ? `${h.fewest}–${h.attempts} attempts` : plural(h.attempts, "attempt");
|
|
222
|
+
const parts = [
|
|
223
|
+
plural(h.files, "eval file"),
|
|
224
|
+
plural(h.evals, "eval") + (h.skipped ? ` (${h.skipped} skipped)` : ""),
|
|
225
|
+
plural(h.cells, "cell"),
|
|
226
|
+
attempts
|
|
330
227
|
];
|
|
228
|
+
const caps = [h.budget.attempts !== void 0 ? `${h.budget.attempts} ${h.attempts <= h.budget.attempts ? c.green("✓") : c.red("✗")}` : "", h.budget.duration ?? ""].filter(Boolean);
|
|
229
|
+
return `${parts.join(" · ")}${caps.length ? ` (budget: ${caps.join(", ")})` : ""}`;
|
|
331
230
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
const message = e.errors[0] ?? "";
|
|
340
|
-
const turns = (e.meta?.turnErrors ?? []).filter((t) => !message.includes(t.replace(/^turn \d+ error: /, ""))).map((t) => c.yellow(t));
|
|
341
|
-
return [
|
|
342
|
-
...failureBody(c, e, outcome, message, options),
|
|
343
|
-
...otherChecks(c, e),
|
|
344
|
-
...turns
|
|
345
|
-
];
|
|
231
|
+
function estimate(c, h) {
|
|
232
|
+
const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
|
|
233
|
+
const most = Math.min(h.concurrency, h.attempts);
|
|
234
|
+
const parallel = h.parallel ?? most;
|
|
235
|
+
if (parallel <= 1) return serial;
|
|
236
|
+
const capped = parallel < most ? `, ${parallel} at once` : "";
|
|
237
|
+
return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency}${capped})`)}`} · ${serial}`;
|
|
346
238
|
}
|
|
347
|
-
/**
|
|
348
|
-
function
|
|
349
|
-
const
|
|
350
|
-
|
|
239
|
+
/** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
|
|
240
|
+
function noteLines(c, kind, text, width) {
|
|
241
|
+
const [glyph, label, paint] = kind === "hint" ? [
|
|
242
|
+
"ℹ",
|
|
243
|
+
"Hint:",
|
|
244
|
+
c.blue
|
|
245
|
+
] : [
|
|
246
|
+
"⚠",
|
|
247
|
+
"Warning:",
|
|
248
|
+
kind === "root" ? c.red : c.orange
|
|
249
|
+
];
|
|
250
|
+
const basic = {
|
|
251
|
+
warning: c.yellow,
|
|
252
|
+
hint: c.teal,
|
|
253
|
+
root: c.red
|
|
254
|
+
}[kind];
|
|
255
|
+
const lead = c.depth === 4 && kind !== "root" ? `${basic(glyph)} ${c.bold(label)}` : c.bold(paint(`${glyph} ${label}`));
|
|
256
|
+
const at = 14;
|
|
257
|
+
return wrap(text, Math.max(20, width - at)).map((line, i) => i === 0 ? ` ${lead}${indent(LABEL - label.length)}${line}` : `${indent(at)}${line}`);
|
|
351
258
|
}
|
|
352
|
-
/**
|
|
353
|
-
function
|
|
354
|
-
|
|
355
|
-
return (e.meta?.failedChecks ?? []).filter((x) => x !== shown).map((x) => {
|
|
356
|
-
const why = x.message.startsWith(`${x.label}: `) ? x.message.slice(x.label.length + 2) : x.message;
|
|
357
|
-
return [
|
|
358
|
-
c.red(`✗ ${x.label}`),
|
|
359
|
-
x.at && c.dim(x.at),
|
|
360
|
-
why && c.dim(why)
|
|
361
|
-
].filter(Boolean).join(" ");
|
|
362
|
-
});
|
|
259
|
+
/** A step line, `◆ Preparing 1 sandbox (docker)...`, its text wrapped under itself. */
|
|
260
|
+
function stepLines(glyph, text, width) {
|
|
261
|
+
return wrap(text, Math.max(20, width - GLYPH)).map((line, i) => i === 0 ? ` ${glyph} ${line}` : `${indent(GLYPH)}${line}`);
|
|
363
262
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
}
|
|
374
|
-
return [`${c.yellow("infra error")}`, ...tailLines(message, options.max)];
|
|
375
|
-
}
|
|
376
|
-
const failed = (e.meta?.judgements ?? []).filter((j) => j.verdict !== "pass");
|
|
377
|
-
if (e.meta?.errorKind === "judge" && failed.length) return failed.flatMap((j, i) => {
|
|
378
|
-
const reason = options.reasons?.[i];
|
|
379
|
-
return [judgeLine(c, {
|
|
380
|
-
...j,
|
|
381
|
-
threshold: j.threshold
|
|
382
|
-
}), ...reason ? [` ${reason}`] : []];
|
|
383
|
-
});
|
|
384
|
-
const formatted = e.formattedErrors[0];
|
|
385
|
-
const body = formatted && e.meta?.errorKind !== "timeout" ? formatted : message;
|
|
386
|
-
return capLines((c.enabled ? body : stripAnsi(body)).replace(/^\n+/, ""), options.max).map((l) => c.enabled && body === message ? c.red(l) : l);
|
|
263
|
+
/** A run-level notice (a prepared workspace, the Ctrl-C one) as a step line. */
|
|
264
|
+
function noticeLines(c, e, width) {
|
|
265
|
+
const glyph = {
|
|
266
|
+
info: c.dim("ℹ"),
|
|
267
|
+
warn: c.orange("⚠"),
|
|
268
|
+
error: c.red("✗")
|
|
269
|
+
}[e.level];
|
|
270
|
+
const lines = stepLines(glyph, e.text, width);
|
|
271
|
+
return e.level === "info" ? lines.map((l) => c.dim(l)) : lines;
|
|
387
272
|
}
|
|
388
|
-
/** Preflight
|
|
389
|
-
function
|
|
273
|
+
/** Preflight: which sandboxes it prepares, each one ready, and each failure. */
|
|
274
|
+
function prepareStepLines(c, e, width, max) {
|
|
390
275
|
if (e.state === "start") {
|
|
391
276
|
const n = e.targets.length;
|
|
392
277
|
const names = e.targets.map((t) => t.label).join(", ");
|
|
393
|
-
return
|
|
278
|
+
return stepLines(c.teal("◆"), `Preparing ${plural(n, "sandbox", "sandboxes")} (${names})...`, width);
|
|
394
279
|
}
|
|
395
280
|
const { outcome } = e;
|
|
396
|
-
if (outcome.ok)
|
|
281
|
+
if (outcome.ok) {
|
|
282
|
+
const took = outcome.ms >= 1e3 ? ` ${c.dim(formatDuration(outcome.ms))}` : "";
|
|
283
|
+
return [` ${c.green("✓")} ${outcome.target.label} ready${took}`];
|
|
284
|
+
}
|
|
397
285
|
const who = outcome.target.setBy.join(", ");
|
|
398
|
-
return [
|
|
286
|
+
return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
|
|
287
|
+
}
|
|
288
|
+
/** The start line's text: how the attempts' logs will print. */
|
|
289
|
+
function startText(output) {
|
|
290
|
+
return output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
|
|
291
|
+
}
|
|
292
|
+
/** The last line before the attempts' own: how their logs will print. */
|
|
293
|
+
function startLine(c, output) {
|
|
294
|
+
return ` ${c.teal("◆")} ${startText(output)}`;
|
|
295
|
+
}
|
|
296
|
+
/** What the start line's text becomes once the run ends, on a terminal that redraws it. */
|
|
297
|
+
function endText(status) {
|
|
298
|
+
return status === "interrupted" ? "Canceled." : status === "budget" ? "Stopped at the budget." : "Finished.";
|
|
399
299
|
}
|
|
400
|
-
|
|
401
|
-
function
|
|
402
|
-
const
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were canceled")] : [],
|
|
407
|
-
c.dim([
|
|
408
|
-
formatDuration(record.durationMs),
|
|
409
|
-
tokens,
|
|
410
|
-
`results ${e.summaryPath}`
|
|
411
|
-
].filter(Boolean).join(" · "))
|
|
300
|
+
/** The plan as the header block: its facts, `--dry-run`'s table, then each kind of note. */
|
|
301
|
+
function planLines(c, e, width) {
|
|
302
|
+
const sections = [
|
|
303
|
+
e.details,
|
|
304
|
+
[...e.warnings.flatMap((w) => noteLines(c, "warning", w, width)), ...(e.sandboxWarnings ?? []).flatMap((w) => noteLines(c, w.kind === "root" ? "root" : "warning", w.message, width))],
|
|
305
|
+
e.hints.flatMap((h) => noteLines(c, "hint", h, width))
|
|
412
306
|
];
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
});
|
|
307
|
+
const out = headerLines(c, e.header, width);
|
|
308
|
+
for (const lines of sections) if (lines.length) out.push("", ...lines);
|
|
309
|
+
out.push("");
|
|
310
|
+
return out;
|
|
418
311
|
}
|
|
419
312
|
// src/ui/tags.ts
|
|
420
313
|
const STOPWORDS = new Set("a an the and or of to in on for with without from by at as is are be it its that this when than then into via not no".split(" "));
|
|
@@ -475,6 +368,8 @@ var Tagger = class {
|
|
|
475
368
|
};
|
|
476
369
|
// src/ui/agent.ts
|
|
477
370
|
const MAX_LINES = 12;
|
|
371
|
+
/** Agents read whole lines, so the header's notes don't wrap. */
|
|
372
|
+
const WIDTH = Number.POSITIVE_INFINITY;
|
|
478
373
|
var AgentRenderer = class {
|
|
479
374
|
#o;
|
|
480
375
|
#c = colors(false);
|
|
@@ -490,14 +385,13 @@ var AgentRenderer = class {
|
|
|
490
385
|
onEvent(e) {
|
|
491
386
|
switch (e.type) {
|
|
492
387
|
case "plan":
|
|
493
|
-
this.#write(
|
|
494
|
-
for (const line of e.details) this.#write(line);
|
|
495
|
-
for (const w of e.warnings) this.#write(`warning: ${w}`);
|
|
496
|
-
for (const h of e.hints) this.#write(`hint: ${h}`);
|
|
497
|
-
for (const w of e.sandboxWarnings ?? []) this.#write(sandboxWarningLine(w));
|
|
388
|
+
for (const line of planLines(this.#c, e, WIDTH)) this.#write(line);
|
|
498
389
|
return;
|
|
499
390
|
case "run-start":
|
|
500
|
-
this.#write(
|
|
391
|
+
this.#write(startLine(this.#c, {
|
|
392
|
+
...this.#o.output,
|
|
393
|
+
outputLogs: "none"
|
|
394
|
+
}));
|
|
501
395
|
return;
|
|
502
396
|
case "collected":
|
|
503
397
|
this.#tags.addFile(e.evals);
|
|
@@ -519,148 +413,30 @@ var AgentRenderer = class {
|
|
|
519
413
|
return;
|
|
520
414
|
}
|
|
521
415
|
case "prepare":
|
|
522
|
-
for (const line of
|
|
416
|
+
for (const line of prepareStepLines(this.#c, e, WIDTH, MAX_LINES)) this.#write(line);
|
|
523
417
|
return;
|
|
524
418
|
case "notice":
|
|
525
|
-
|
|
419
|
+
for (const line of noticeLines(this.#c, e, WIDTH)) this.#write(line);
|
|
526
420
|
return;
|
|
527
|
-
case "run-end":
|
|
421
|
+
case "run-end": {
|
|
528
422
|
this.#write("");
|
|
529
|
-
|
|
423
|
+
const verbose = (this.#o.verbosity ?? 0) > 0;
|
|
424
|
+
const list = statusList({
|
|
425
|
+
record: e.record,
|
|
426
|
+
suite: suiteResult(e.record),
|
|
427
|
+
remarks: e.remarks ?? null,
|
|
428
|
+
verbose,
|
|
429
|
+
...e.notRun ? { notRun: e.notRun } : {}
|
|
430
|
+
});
|
|
431
|
+
const pointer = pointerLine(e.summaryPath);
|
|
432
|
+
for (const line of listLines(list, this.#c, false, pointer, verbose, true)) this.#write(line);
|
|
530
433
|
return;
|
|
434
|
+
}
|
|
531
435
|
default: return;
|
|
532
436
|
}
|
|
533
437
|
}
|
|
534
438
|
close() {}
|
|
535
439
|
};
|
|
536
|
-
// src/ui/header.ts
|
|
537
|
-
/** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
|
|
538
|
-
const GLYPH = 5;
|
|
539
|
-
/** `Est. Time: `, the longest key. */
|
|
540
|
-
const KEY = 11;
|
|
541
|
-
/** `Warning: `, the longer note label. */
|
|
542
|
-
const LABEL = 9;
|
|
543
|
-
/**
|
|
544
|
-
* `text` broken at spaces into lines of at most `width` columns, not counting color codes; a
|
|
545
|
-
* longer word keeps a line. A `·` stays with the word before it, so no line starts with one.
|
|
546
|
-
*/
|
|
547
|
-
function wrap(text, width) {
|
|
548
|
-
const words = [];
|
|
549
|
-
for (const word of text.split(/\s+/).filter(Boolean)) if (word === "·" && words.length) words[words.length - 1] += " ·";
|
|
550
|
-
else words.push(word);
|
|
551
|
-
const lines = [];
|
|
552
|
-
let line = "";
|
|
553
|
-
for (const word of words) if (line && stripAnsi(line).length + 1 + stripAnsi(word).length > width) {
|
|
554
|
-
lines.push(line);
|
|
555
|
-
line = word;
|
|
556
|
-
} else line = line ? `${line} ${word}` : word;
|
|
557
|
-
return line ? [...lines, line] : lines;
|
|
558
|
-
}
|
|
559
|
-
const indent = (n) => " ".repeat(n);
|
|
560
|
-
/** `REEVAL v0.16.1`, then the run's facts, one `➜ Key: value` line each. */
|
|
561
|
-
function headerLines(c, h, width) {
|
|
562
|
-
const name = `REEVAL v${h.version}`;
|
|
563
|
-
const title = c.bold(c.depth === 4 ? name : c.teal(name));
|
|
564
|
-
const valueWidth = width - GLYPH - KEY;
|
|
565
|
-
const row = (key, value) => {
|
|
566
|
-
const values = typeof value === "string" ? [value] : value;
|
|
567
|
-
const joined = values.join(", ");
|
|
568
|
-
return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
|
|
569
|
-
};
|
|
570
|
-
const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
|
|
571
|
-
const resumed = h.resumed ? c.dim(` (resumed, ${plural(h.resumed, "attempt")} done)`) : "";
|
|
572
|
-
const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
|
|
573
|
-
return [
|
|
574
|
-
h.dryRun ? `${title} ${c.dim("dry run")}` : title,
|
|
575
|
-
"",
|
|
576
|
-
...row("Run", `${h.runId}${resumed}`),
|
|
577
|
-
...h.config ? row("Config", h.config) : [],
|
|
578
|
-
...h.profile ? row("Profile", `${h.profile.name}${reason}`) : [],
|
|
579
|
-
...row(many("Harness", h.harnesses.length, "Harnesses"), h.harnesses.map((id) => harness(c, h, id))),
|
|
580
|
-
...row(many("Sandbox", h.sandboxes.length, "Sandboxes"), h.sandboxes),
|
|
581
|
-
...h.judges.length ? row(many("Judge", h.judges.length), h.judges) : [],
|
|
582
|
-
...row("Scope", scope(c, h)),
|
|
583
|
-
...row("Est. Time", estimate(c, h)),
|
|
584
|
-
...h.record && !h.dryRun ? row("Record", h.record) : []
|
|
585
|
-
];
|
|
586
|
-
}
|
|
587
|
-
/** A harness id, with the effort it runs at when the id doesn't name it. */
|
|
588
|
-
function harness(c, h, id) {
|
|
589
|
-
const effort = h.efforts?.[id];
|
|
590
|
-
if (effort === void 0 || id.includes("[effort=")) return id;
|
|
591
|
-
return `${id} ${`(effort: ${effort})`.split(" ").map((w) => c.dim(w)).join(" ")}`;
|
|
592
|
-
}
|
|
593
|
-
function scope(c, h) {
|
|
594
|
-
const attempts = h.fewest < h.attempts ? `${h.fewest}–${h.attempts} attempts` : plural(h.attempts, "attempt");
|
|
595
|
-
const parts = [
|
|
596
|
-
plural(h.files, "eval file"),
|
|
597
|
-
plural(h.evals, "eval") + (h.skipped ? ` (${h.skipped} skipped)` : ""),
|
|
598
|
-
plural(h.cells, "cell"),
|
|
599
|
-
attempts
|
|
600
|
-
];
|
|
601
|
-
const caps = [h.budget.attempts !== void 0 ? `${h.budget.attempts} ${h.attempts <= h.budget.attempts ? c.green("✓") : c.red("✗")}` : "", h.budget.duration ?? ""].filter(Boolean);
|
|
602
|
-
return `${parts.join(" · ")}${caps.length ? ` (budget: ${caps.join(", ")})` : ""}`;
|
|
603
|
-
}
|
|
604
|
-
function estimate(c, h) {
|
|
605
|
-
const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
|
|
606
|
-
const parallel = Math.min(h.concurrency, h.attempts);
|
|
607
|
-
if (parallel <= 1) return serial;
|
|
608
|
-
return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency})`)}`} · ${serial}`;
|
|
609
|
-
}
|
|
610
|
-
/** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
|
|
611
|
-
function noteLines(c, kind, text, width) {
|
|
612
|
-
const [glyph, label, paint] = kind === "hint" ? [
|
|
613
|
-
"ℹ",
|
|
614
|
-
"Hint:",
|
|
615
|
-
c.blue
|
|
616
|
-
] : [
|
|
617
|
-
"⚠",
|
|
618
|
-
"Warning:",
|
|
619
|
-
kind === "root" ? c.red : c.orange
|
|
620
|
-
];
|
|
621
|
-
const basic = {
|
|
622
|
-
warning: c.yellow,
|
|
623
|
-
hint: c.teal,
|
|
624
|
-
root: c.red
|
|
625
|
-
}[kind];
|
|
626
|
-
const lead = c.depth === 4 && kind !== "root" ? `${basic(glyph)} ${c.bold(label)}` : c.bold(paint(`${glyph} ${label}`));
|
|
627
|
-
const at = 14;
|
|
628
|
-
return wrap(text, Math.max(20, width - at)).map((line, i) => i === 0 ? ` ${lead}${indent(LABEL - label.length)}${line}` : `${indent(at)}${line}`);
|
|
629
|
-
}
|
|
630
|
-
/** A step line, `◆ Preparing 1 sandbox (docker)...`, its text wrapped under itself. */
|
|
631
|
-
function stepLines(glyph, text, width) {
|
|
632
|
-
return wrap(text, Math.max(20, width - GLYPH)).map((line, i) => i === 0 ? ` ${glyph} ${line}` : `${indent(GLYPH)}${line}`);
|
|
633
|
-
}
|
|
634
|
-
/** A run-level notice (a prepared workspace, the Ctrl-C one) as a step line. */
|
|
635
|
-
function noticeLines(c, e, width) {
|
|
636
|
-
const glyph = {
|
|
637
|
-
info: c.dim("ℹ"),
|
|
638
|
-
warn: c.orange("⚠"),
|
|
639
|
-
error: c.red("✗")
|
|
640
|
-
}[e.level];
|
|
641
|
-
const lines = stepLines(glyph, e.text, width);
|
|
642
|
-
return e.level === "info" ? lines.map((l) => c.dim(l)) : lines;
|
|
643
|
-
}
|
|
644
|
-
/** Preflight: which sandboxes it prepares, each one ready, and each failure. */
|
|
645
|
-
function prepareStepLines(c, e, width, max) {
|
|
646
|
-
if (e.state === "start") {
|
|
647
|
-
const n = e.targets.length;
|
|
648
|
-
const names = e.targets.map((t) => t.label).join(", ");
|
|
649
|
-
return stepLines(c.teal("◆"), `Preparing ${plural(n, "sandbox", "sandboxes")} (${names})...`, width);
|
|
650
|
-
}
|
|
651
|
-
const { outcome } = e;
|
|
652
|
-
if (outcome.ok) {
|
|
653
|
-
const took = outcome.ms >= 1e3 ? ` ${c.dim(formatDuration(outcome.ms))}` : "";
|
|
654
|
-
return [` ${c.green("✓")} ${outcome.target.label} ready${took}`];
|
|
655
|
-
}
|
|
656
|
-
const who = outcome.target.setBy.join(", ");
|
|
657
|
-
return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
|
|
658
|
-
}
|
|
659
|
-
/** The last line before the attempts' own: how their logs will print. */
|
|
660
|
-
function startLine(c, output) {
|
|
661
|
-
const text = output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
|
|
662
|
-
return ` ${c.teal("◆")} ${text}`;
|
|
663
|
-
}
|
|
664
440
|
// src/ui/plain.ts
|
|
665
441
|
/** In a grouped block, an end line replaces its start line and keeps its start time. */
|
|
666
442
|
const STARTS = {
|
|
@@ -702,6 +478,8 @@ var PlainRenderer = class {
|
|
|
702
478
|
#queued = /* @__PURE__ */ new Set();
|
|
703
479
|
#runStartedAt = 0;
|
|
704
480
|
#lastWrite = 0;
|
|
481
|
+
/** The last line written was blank, so the list needs no gap above it. */
|
|
482
|
+
#blank = false;
|
|
705
483
|
#timer;
|
|
706
484
|
/** On a TTY: the status line is drawn under everything else, and redrawn after each write. */
|
|
707
485
|
#live;
|
|
@@ -710,8 +488,11 @@ var PlainRenderer = class {
|
|
|
710
488
|
#evals = /* @__PURE__ */ new Set();
|
|
711
489
|
#tries = /* @__PURE__ */ new Set();
|
|
712
490
|
#ended = /* @__PURE__ */ new Set();
|
|
491
|
+
#list;
|
|
492
|
+
#lastProgress = 0;
|
|
713
493
|
constructor(options) {
|
|
714
494
|
this.#o = options;
|
|
495
|
+
this.#list = new LiveList(options.harness);
|
|
715
496
|
this.#c = options.colors;
|
|
716
497
|
this.#now = options.now ?? Date.now;
|
|
717
498
|
this.#live = options.output.statusLine;
|
|
@@ -725,6 +506,7 @@ var PlainRenderer = class {
|
|
|
725
506
|
const line = `${text.trimEnd()}\n`;
|
|
726
507
|
this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
|
|
727
508
|
this.#lastWrite = this.#now();
|
|
509
|
+
this.#blank = line === "\n";
|
|
728
510
|
}
|
|
729
511
|
/** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
|
|
730
512
|
#drawStatus() {
|
|
@@ -775,31 +557,17 @@ var PlainRenderer = class {
|
|
|
775
557
|
onEvent(e) {
|
|
776
558
|
const c = this.#c;
|
|
777
559
|
switch (e.type) {
|
|
778
|
-
case "run-start":
|
|
560
|
+
case "run-start":
|
|
779
561
|
this.#runStartedAt = e.at;
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
this.#write("");
|
|
783
|
-
return;
|
|
784
|
-
}
|
|
785
|
-
const files = `${e.files} eval file${e.files === 1 ? "" : "s"}`;
|
|
786
|
-
const order = `${this.#o.output.logOrder} logs`;
|
|
787
|
-
const setup = `harness ${e.harness} · sandbox ${e.sandbox} · concurrency ${e.concurrency}`;
|
|
788
|
-
this.#write(c.dim(`reeval ${e.version} · run ${e.runId} · ${files} · ${setup} · ${order}`));
|
|
562
|
+
this.#write(startLine(c, this.#o.output));
|
|
563
|
+
this.#write("");
|
|
789
564
|
return;
|
|
790
|
-
}
|
|
791
565
|
case "plan":
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
return;
|
|
795
|
-
}
|
|
796
|
-
this.#write(e.line);
|
|
797
|
-
for (const line of e.details) this.#write(c.dim(line));
|
|
798
|
-
for (const w of e.warnings) this.#write(c.yellow(`warning: ${w}`));
|
|
799
|
-
for (const h of e.hints) this.#write(c.dim(`hint: ${h}`));
|
|
800
|
-
for (const w of e.sandboxWarnings ?? []) this.#sandboxWarning(w);
|
|
566
|
+
for (const line of planLines(c, e, this.#columns)) this.#write(line);
|
|
567
|
+
if (this.#o.output.githubActions) for (const w of e.sandboxWarnings ?? []) this.#annotate(w);
|
|
801
568
|
return;
|
|
802
569
|
case "collected":
|
|
570
|
+
this.#list.collected(e.file, e.evals);
|
|
803
571
|
this.#tags.addFile(e.evals);
|
|
804
572
|
for (const x of e.evals) {
|
|
805
573
|
if (x.skipped) continue;
|
|
@@ -811,6 +579,7 @@ var PlainRenderer = class {
|
|
|
811
579
|
return;
|
|
812
580
|
case "eval-start": {
|
|
813
581
|
const a = this.#attempt(e, e.at);
|
|
582
|
+
this.#list.start(e);
|
|
814
583
|
a.last = e;
|
|
815
584
|
this.#queued.delete(e.key);
|
|
816
585
|
this.#running.set(e.key, a);
|
|
@@ -820,6 +589,7 @@ var PlainRenderer = class {
|
|
|
820
589
|
case "eval-progress": {
|
|
821
590
|
const a = this.#running.get(e.key);
|
|
822
591
|
if (!a) return;
|
|
592
|
+
this.#list.progress(e.key, e.progress);
|
|
823
593
|
if (e.progress.kind !== "warning") a.last = e;
|
|
824
594
|
const { progress } = e;
|
|
825
595
|
const paint = a.ungated ? {
|
|
@@ -861,13 +631,13 @@ var PlainRenderer = class {
|
|
|
861
631
|
return;
|
|
862
632
|
}
|
|
863
633
|
case "eval-end":
|
|
634
|
+
this.#list.end(e.result);
|
|
864
635
|
this.#end(e.result, e.at);
|
|
865
636
|
if (this.#tries.has(e.result.key)) this.#ended.add(e.result.key);
|
|
866
637
|
this.#drawStatus();
|
|
867
638
|
return;
|
|
868
|
-
case "prepare":
|
|
869
|
-
const
|
|
870
|
-
for (const line of lines) this.#write(line);
|
|
639
|
+
case "prepare":
|
|
640
|
+
for (const line of prepareStepLines(c, e, this.#columns, ERROR_LINES)) this.#write(line);
|
|
871
641
|
if (e.state === "failed" && this.#o.output.githubActions) {
|
|
872
642
|
const { target, failure } = e.outcome;
|
|
873
643
|
const what = target.kind === "workspace" ? "The" : "Sandbox";
|
|
@@ -875,52 +645,30 @@ var PlainRenderer = class {
|
|
|
875
645
|
this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
|
|
876
646
|
}
|
|
877
647
|
return;
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
if (this.#o.output.rich) {
|
|
881
|
-
for (const line of noticeLines(c, e, this.#columns)) this.#write(line);
|
|
882
|
-
return;
|
|
883
|
-
}
|
|
884
|
-
const paint = {
|
|
885
|
-
info: c.dim,
|
|
886
|
-
warn: c.yellow,
|
|
887
|
-
error: c.red
|
|
888
|
-
}[e.level];
|
|
889
|
-
this.#write(paint(e.text));
|
|
648
|
+
case "notice":
|
|
649
|
+
for (const line of noticeLines(c, e, this.#columns)) this.#write(line);
|
|
890
650
|
return;
|
|
891
|
-
|
|
892
|
-
case "run-end":
|
|
651
|
+
case "run-end": {
|
|
893
652
|
this.#clearStatus();
|
|
894
|
-
this.#write("");
|
|
895
|
-
|
|
653
|
+
if (!this.#blank) this.#write("");
|
|
654
|
+
const verbose = (this.#o.verbosity ?? 0) > 0;
|
|
655
|
+
const list = statusList({
|
|
656
|
+
record: e.record,
|
|
657
|
+
suite: suiteResult(e.record),
|
|
658
|
+
remarks: e.remarks ?? null,
|
|
659
|
+
verbose,
|
|
660
|
+
...e.notRun ? { notRun: e.notRun } : {}
|
|
661
|
+
});
|
|
662
|
+
for (const line of listLines(list, c, true, pointerLine(e.summaryPath), verbose)) this.#write(line);
|
|
896
663
|
return;
|
|
664
|
+
}
|
|
897
665
|
}
|
|
898
666
|
}
|
|
899
667
|
get #columns() {
|
|
900
|
-
return this.#o.columns ||
|
|
668
|
+
return this.#o.columns || Number.POSITIVE_INFINITY;
|
|
901
669
|
}
|
|
902
|
-
/** The
|
|
903
|
-
#
|
|
904
|
-
const c = this.#c;
|
|
905
|
-
const width = this.#columns;
|
|
906
|
-
for (const line of headerLines(c, e.header, width)) this.#write(line);
|
|
907
|
-
const sections = [
|
|
908
|
-
e.details,
|
|
909
|
-
[...e.warnings.flatMap((w) => noteLines(c, "warning", w, width)), ...(e.sandboxWarnings ?? []).flatMap((w) => noteLines(c, w.kind === "root" ? "root" : "warning", w.message, width))],
|
|
910
|
-
e.hints.flatMap((h) => noteLines(c, "hint", h, width))
|
|
911
|
-
];
|
|
912
|
-
for (const lines of sections) {
|
|
913
|
-
if (!lines.length) continue;
|
|
914
|
-
this.#write("");
|
|
915
|
-
for (const line of lines) this.#write(line);
|
|
916
|
-
}
|
|
917
|
-
this.#write("");
|
|
918
|
-
}
|
|
919
|
-
#sandboxWarning(w) {
|
|
920
|
-
const c = this.#c;
|
|
921
|
-
const line = sandboxWarningLine(w);
|
|
922
|
-
this.#write(w.kind === "root" ? c.bold(c.red(line)) : c.yellow(line));
|
|
923
|
-
if (!this.#o.output.githubActions) return;
|
|
670
|
+
/** The header block names a sandbox warning; GitHub Actions also annotates it. */
|
|
671
|
+
#annotate(w) {
|
|
924
672
|
const file = w.file ? `file=${ghEscape(w.file, true)},` : "";
|
|
925
673
|
const title = w.kind === "root" ? "Sandbox runs as root" : "Sandbox has extra capabilities";
|
|
926
674
|
this.#write(`::warning ${file}title=${ghEscape(title, true)}::${ghEscape(w.message)}`);
|
|
@@ -961,20 +709,30 @@ var PlainRenderer = class {
|
|
|
961
709
|
this.#write(`::error file=${ghEscape(file, true)},line=${line},title=${label}::${ghEscape(x.message)}`);
|
|
962
710
|
}
|
|
963
711
|
if (!errorCheck(result)?.at) {
|
|
964
|
-
const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n"));
|
|
712
|
+
const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n").replace(/^✗ failed {2}/, ""));
|
|
965
713
|
const at = locate(result);
|
|
966
714
|
const where = at ? `,line=${at.line},col=${at.col}` : "";
|
|
967
715
|
this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
|
|
968
716
|
}
|
|
969
717
|
}
|
|
970
718
|
}
|
|
971
|
-
/**
|
|
719
|
+
/**
|
|
720
|
+
* Every `heartbeatMs` while attempts run. Without attempt logs, how many evals are done; with
|
|
721
|
+
* them, each running attempt's phase, when nothing printed for that long.
|
|
722
|
+
*/
|
|
972
723
|
tick(now = this.#now()) {
|
|
973
|
-
const { heartbeatMs } = this.#o.output;
|
|
724
|
+
const { heartbeatMs, outputLogs } = this.#o.output;
|
|
974
725
|
if (this.#live || heartbeatMs <= 0) return;
|
|
975
|
-
if (now - this.#lastWrite < heartbeatMs) return;
|
|
976
|
-
const c = this.#c;
|
|
977
726
|
if (this.#running.size === 0) return;
|
|
727
|
+
const c = this.#c;
|
|
728
|
+
if (outputLogs === "none") {
|
|
729
|
+
if (now - Math.max(this.#lastProgress, this.#runStartedAt) < heartbeatMs) return;
|
|
730
|
+
const { done, total } = this.#list.counts();
|
|
731
|
+
this.#write(c.dim(progressText(done, total, now - this.#runStartedAt)));
|
|
732
|
+
this.#lastProgress = now;
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
if (now - this.#lastWrite < heartbeatMs) return;
|
|
978
736
|
const running = [...this.#running.values()].map((a) => {
|
|
979
737
|
const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
|
|
980
738
|
return `${this.#prefix(a)} ${phaseOf(progress)} ${c.dim(formatDuration(now - a.startedAt))}`;
|
|
@@ -988,6 +746,319 @@ var PlainRenderer = class {
|
|
|
988
746
|
this.#clearStatus();
|
|
989
747
|
}
|
|
990
748
|
};
|
|
749
|
+
// src/ui/tty.ts
|
|
750
|
+
/** The list's state, which the Ink app reads with `useSyncExternalStore`. */
|
|
751
|
+
var ListStore = class {
|
|
752
|
+
#state = {
|
|
753
|
+
files: [],
|
|
754
|
+
logs: [],
|
|
755
|
+
counts: {
|
|
756
|
+
done: 0,
|
|
757
|
+
total: 0
|
|
758
|
+
},
|
|
759
|
+
phase: null,
|
|
760
|
+
finished: null
|
|
761
|
+
};
|
|
762
|
+
#listeners = /* @__PURE__ */ new Set();
|
|
763
|
+
get = () => this.#state;
|
|
764
|
+
subscribe = (listener) => {
|
|
765
|
+
this.#listeners.add(listener);
|
|
766
|
+
return () => this.#listeners.delete(listener);
|
|
767
|
+
};
|
|
768
|
+
set(next) {
|
|
769
|
+
this.#state = {
|
|
770
|
+
...this.#state,
|
|
771
|
+
...next
|
|
772
|
+
};
|
|
773
|
+
for (const l of this.#listeners) l();
|
|
774
|
+
}
|
|
775
|
+
};
|
|
776
|
+
const FRAME_MS = 80;
|
|
777
|
+
const ESC = String.fromCharCode(27);
|
|
778
|
+
const BEL = String.fromCharCode(7);
|
|
779
|
+
/** A CSI sequence (colors, cursor moves) or an OSC one (a hyperlink), which take no columns. */
|
|
780
|
+
const ESCAPE = new RegExp(`${ESC}(?:\\[[0-9;?]*[ -/]*[@-~]|\\][^${BEL}${ESC}]*(?:${BEL}|${ESC}\\\\))`, "g");
|
|
781
|
+
const graphemes = new Intl.Segmenter();
|
|
782
|
+
/**
|
|
783
|
+
* Terminal rows one line takes at `columns`, placed as a terminal places it: a wide character that
|
|
784
|
+
* doesn't fit in the row's last column moves to the next row, `\r` returns to the row's start and
|
|
785
|
+
* `\t` goes to the next stop of eight without wrapping.
|
|
786
|
+
*/
|
|
787
|
+
function rowsOf(line, columns) {
|
|
788
|
+
let rows = 1;
|
|
789
|
+
let column = 0;
|
|
790
|
+
for (const { segment } of graphemes.segment(line.replace(ESCAPE, ""))) if (segment === "\r") column = 0;
|
|
791
|
+
else if (segment === " ") column = Math.min(columns - 1, (Math.floor(column / 8) + 1) * 8);
|
|
792
|
+
else if (segment === "\b") column = Math.max(0, column - 1);
|
|
793
|
+
else {
|
|
794
|
+
const width = stringWidth(segment);
|
|
795
|
+
if (!width) continue;
|
|
796
|
+
if (column + width > columns) {
|
|
797
|
+
rows += 1;
|
|
798
|
+
column = 0;
|
|
799
|
+
}
|
|
800
|
+
column += width;
|
|
801
|
+
}
|
|
802
|
+
return rows;
|
|
803
|
+
}
|
|
804
|
+
/**
|
|
805
|
+
* Before the list is up, the sandbox prepare step: the glyph of its `◆ Preparing…` line spins in
|
|
806
|
+
* place (the cursor goes up to it and back) until each sandbox is ready or failed, then settles to
|
|
807
|
+
* `◆`. What prints under it is kept and measured in rows as the terminal placed it, so a wide
|
|
808
|
+
* character still finds the step line. Once the step line would be out of reach, or the width
|
|
809
|
+
* changes (some terminals reflow what they show, some don't, so its row is unknown), it stops
|
|
810
|
+
* without drawing.
|
|
811
|
+
*/
|
|
812
|
+
var StepSpinner = class {
|
|
813
|
+
#write;
|
|
814
|
+
#c;
|
|
815
|
+
#size;
|
|
816
|
+
#timer;
|
|
817
|
+
#frame = 0;
|
|
818
|
+
/** The step line and everything printed under it since. */
|
|
819
|
+
#text = "";
|
|
820
|
+
/** The width `#text` was printed at. */
|
|
821
|
+
#columns = 0;
|
|
822
|
+
#pending = 0;
|
|
823
|
+
constructor(write, c, size) {
|
|
824
|
+
this.#write = write;
|
|
825
|
+
this.#c = c;
|
|
826
|
+
this.#size = size;
|
|
827
|
+
}
|
|
828
|
+
/** Terminal lines from the step line down to the cursor after `text`. */
|
|
829
|
+
#up(text = this.#text) {
|
|
830
|
+
const columns = Math.max(1, this.#columns);
|
|
831
|
+
const lines = text.split("\n");
|
|
832
|
+
if (lines.at(-1) === "") lines.pop();
|
|
833
|
+
return lines.reduce((n, l) => n + rowsOf(l, columns), 0);
|
|
834
|
+
}
|
|
835
|
+
/** The step line was just written as `text`; it spins until `targets` have reported. */
|
|
836
|
+
start(text, targets) {
|
|
837
|
+
this.settle();
|
|
838
|
+
this.#text = text;
|
|
839
|
+
this.#columns = this.#size().columns;
|
|
840
|
+
this.#pending = targets;
|
|
841
|
+
this.#timer = setInterval(() => this.#draw(), FRAME_MS);
|
|
842
|
+
this.#timer.unref();
|
|
843
|
+
this.#draw();
|
|
844
|
+
}
|
|
845
|
+
/** Before `text` prints under it; settles first if the step line would scroll out of reach. */
|
|
846
|
+
before(text) {
|
|
847
|
+
if (!this.#timer) return;
|
|
848
|
+
const next = this.#text + text;
|
|
849
|
+
if (!this.#reachable(next)) this.settle();
|
|
850
|
+
this.#text = next;
|
|
851
|
+
}
|
|
852
|
+
/** One sandbox reported; the last settles it. */
|
|
853
|
+
reported() {
|
|
854
|
+
this.#pending -= 1;
|
|
855
|
+
if (this.#pending <= 0) this.settle();
|
|
856
|
+
}
|
|
857
|
+
#reachable(text = this.#text) {
|
|
858
|
+
const { rows, columns } = this.#size();
|
|
859
|
+
return columns === this.#columns && this.#up(text) < rows - 1;
|
|
860
|
+
}
|
|
861
|
+
#draw(glyph = SPINNER[this.#frame++ % SPINNER.length] ?? "◆") {
|
|
862
|
+
if (!this.#reachable()) {
|
|
863
|
+
clearInterval(this.#timer);
|
|
864
|
+
this.#timer = void 0;
|
|
865
|
+
return;
|
|
866
|
+
}
|
|
867
|
+
const up = this.#up();
|
|
868
|
+
this.#write(`\x1b[${up}A\r\x1b[2C${this.#c.teal(glyph)}\r\x1b[${up}B`);
|
|
869
|
+
}
|
|
870
|
+
settle() {
|
|
871
|
+
if (!this.#timer) return;
|
|
872
|
+
this.#draw("◆");
|
|
873
|
+
clearInterval(this.#timer);
|
|
874
|
+
this.#timer = void 0;
|
|
875
|
+
}
|
|
876
|
+
};
|
|
877
|
+
var InteractiveRenderer = class {
|
|
878
|
+
#o;
|
|
879
|
+
#c;
|
|
880
|
+
#store = new ListStore();
|
|
881
|
+
#list;
|
|
882
|
+
/** The header and run-level lines, as every terminal prints them. */
|
|
883
|
+
#plain;
|
|
884
|
+
#app;
|
|
885
|
+
/** Set once the list couldn't load: from then on, the run prints as a pipe's does. */
|
|
886
|
+
#fallback;
|
|
887
|
+
#ended;
|
|
888
|
+
#closed = false;
|
|
889
|
+
#spinner;
|
|
890
|
+
/** What the plain renderer wrote since `#written` was reset, while the list isn't up. */
|
|
891
|
+
#written = "";
|
|
892
|
+
constructor(options) {
|
|
893
|
+
this.#o = options;
|
|
894
|
+
this.#c = options.colors;
|
|
895
|
+
this.#list = new LiveList(options.harness);
|
|
896
|
+
this.#spinner = new StepSpinner((text) => options.out.write(text), options.colors, () => sizeOf(options.stdout));
|
|
897
|
+
const out = { write: (text) => {
|
|
898
|
+
if (!this.#app || this.#fallback) {
|
|
899
|
+
this.#spinner.before(redactSecrets(String(text)));
|
|
900
|
+
this.#written += String(text);
|
|
901
|
+
return options.out.write(text);
|
|
902
|
+
}
|
|
903
|
+
const lines = String(text).replace(/\n$/, "").split("\n");
|
|
904
|
+
this.#store.set({ logs: [...this.#store.get().logs, ...lines] });
|
|
905
|
+
return true;
|
|
906
|
+
} };
|
|
907
|
+
this.#plain = new PlainRenderer({
|
|
908
|
+
...options,
|
|
909
|
+
out,
|
|
910
|
+
timers: false,
|
|
911
|
+
output: {
|
|
912
|
+
...options.output,
|
|
913
|
+
statusLine: false,
|
|
914
|
+
interactive: false,
|
|
915
|
+
outputLogs: "none"
|
|
916
|
+
}
|
|
917
|
+
});
|
|
918
|
+
}
|
|
919
|
+
#mount() {
|
|
920
|
+
if (this.#app || this.#closed) return;
|
|
921
|
+
const cancel = this.#o.cancel ?? (() => process.kill(process.pid, "SIGINT"));
|
|
922
|
+
const load = this.#o.load ?? (() => import("./ink-list-BpfidQ1h.js"));
|
|
923
|
+
this.#app = load().then((m) => m.mountList({
|
|
924
|
+
store: this.#store,
|
|
925
|
+
colors: this.#c,
|
|
926
|
+
cancel,
|
|
927
|
+
stdout: this.#o.stdout,
|
|
928
|
+
stdin: this.#o.stdin
|
|
929
|
+
})).catch((error) => {
|
|
930
|
+
this.#fallBack(error);
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
#fallBack(error) {
|
|
934
|
+
const out = this.#o.out;
|
|
935
|
+
const { logs, phase } = this.#store.get();
|
|
936
|
+
for (const line of logs) out.write(`${line}\n`);
|
|
937
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
938
|
+
out.write(`${this.#c.yellow(`Couldn't draw the interactive list (${reason}).`)}\n`);
|
|
939
|
+
if (phase) out.write(` ${this.#c.teal("◆")} ${phase}\n`);
|
|
940
|
+
this.#fallback = new PlainRenderer({
|
|
941
|
+
...this.#o,
|
|
942
|
+
timers: false,
|
|
943
|
+
output: {
|
|
944
|
+
...this.#o.output,
|
|
945
|
+
statusLine: false,
|
|
946
|
+
interactive: false
|
|
947
|
+
}
|
|
948
|
+
});
|
|
949
|
+
if (this.#ended) this.#fallback.onEvent(this.#ended.event);
|
|
950
|
+
}
|
|
951
|
+
#update() {
|
|
952
|
+
const { done, total, failed, failing } = this.#list.counts();
|
|
953
|
+
this.#store.set({
|
|
954
|
+
files: this.#list.files(),
|
|
955
|
+
counts: {
|
|
956
|
+
done,
|
|
957
|
+
total,
|
|
958
|
+
failed,
|
|
959
|
+
failing
|
|
960
|
+
}
|
|
961
|
+
});
|
|
962
|
+
}
|
|
963
|
+
onEvent(e) {
|
|
964
|
+
if (this.#fallback) {
|
|
965
|
+
if (e.type !== "run-start") this.#fallback.onEvent(e);
|
|
966
|
+
return;
|
|
967
|
+
}
|
|
968
|
+
switch (e.type) {
|
|
969
|
+
case "run-start":
|
|
970
|
+
this.#spinner.settle();
|
|
971
|
+
this.#store.set({ phase: startText(this.#o.output) });
|
|
972
|
+
return;
|
|
973
|
+
case "prepare":
|
|
974
|
+
this.#written = "";
|
|
975
|
+
this.#plain.onEvent(e);
|
|
976
|
+
if (this.#app || this.#o.timers === false || process.env.REEVAL_DEBUG) return;
|
|
977
|
+
if (e.state === "start") this.#spinner.start(this.#written, e.targets.length);
|
|
978
|
+
else this.#spinner.reported();
|
|
979
|
+
return;
|
|
980
|
+
case "collected":
|
|
981
|
+
this.#spinner.settle();
|
|
982
|
+
this.#list.collected(e.file, e.evals);
|
|
983
|
+
this.#mount();
|
|
984
|
+
this.#update();
|
|
985
|
+
return;
|
|
986
|
+
case "eval-start":
|
|
987
|
+
this.#list.start(e);
|
|
988
|
+
this.#update();
|
|
989
|
+
return;
|
|
990
|
+
case "eval-progress":
|
|
991
|
+
if (e.progress.kind === "warning" || e.progress.kind === "browser-download") this.#plain.onEvent(e);
|
|
992
|
+
this.#list.progress(e.key, e.progress);
|
|
993
|
+
this.#update();
|
|
994
|
+
return;
|
|
995
|
+
case "eval-end":
|
|
996
|
+
this.#list.end(e.result);
|
|
997
|
+
this.#update();
|
|
998
|
+
return;
|
|
999
|
+
case "debug": return;
|
|
1000
|
+
case "run-end": {
|
|
1001
|
+
const listOf = (verbose) => statusList({
|
|
1002
|
+
record: e.record,
|
|
1003
|
+
suite: suiteResult(e.record),
|
|
1004
|
+
remarks: e.remarks ?? null,
|
|
1005
|
+
verbose,
|
|
1006
|
+
...e.notRun ? { notRun: e.notRun } : {}
|
|
1007
|
+
});
|
|
1008
|
+
const list = listOf(false);
|
|
1009
|
+
const verbose = (this.#o.verbosity ?? 0) > 0;
|
|
1010
|
+
if (!this.#app) {
|
|
1011
|
+
this.#plain.onEvent(e);
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1014
|
+
this.#ended = {
|
|
1015
|
+
scrollback: verbose ? listOf(true) : list,
|
|
1016
|
+
verbose,
|
|
1017
|
+
pointer: pointerLine(e.summaryPath),
|
|
1018
|
+
event: e
|
|
1019
|
+
};
|
|
1020
|
+
this.#store.set({
|
|
1021
|
+
files: list.files,
|
|
1022
|
+
...this.#store.get().phase ? { phase: endText(e.record.status) } : {},
|
|
1023
|
+
finished: {
|
|
1024
|
+
done: list.done,
|
|
1025
|
+
notes: list.notes,
|
|
1026
|
+
pointer: pointerLine(e.summaryPath),
|
|
1027
|
+
errors: list.errors
|
|
1028
|
+
}
|
|
1029
|
+
});
|
|
1030
|
+
return;
|
|
1031
|
+
}
|
|
1032
|
+
default: this.#plain.onEvent(e);
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
close() {
|
|
1036
|
+
this.#spinner.settle();
|
|
1037
|
+
this.#plain.close();
|
|
1038
|
+
this.#fallback?.close();
|
|
1039
|
+
if (!this.#store.get().finished) {
|
|
1040
|
+
this.#closed = true;
|
|
1041
|
+
this.#app?.then((a) => a?.unmount());
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
/** Until the user quits the list, then the whole list into the scrollback; at once when there's none. */
|
|
1045
|
+
async wait() {
|
|
1046
|
+
const app = await this.#app;
|
|
1047
|
+
if (!app || !this.#ended) return;
|
|
1048
|
+
await app.waitUntilExit();
|
|
1049
|
+
const { phase } = this.#store.get();
|
|
1050
|
+
const { scrollback, verbose, pointer } = this.#ended;
|
|
1051
|
+
const text = [...phase ? [` ${this.#c.teal("◆")} ${phase}`, ""] : [], ...listLines(scrollback, this.#c, true, pointer, verbose)];
|
|
1052
|
+
this.#o.out.write(`${text.join("\n")}\n`);
|
|
1053
|
+
}
|
|
1054
|
+
};
|
|
1055
|
+
const sizeOf = (stdout) => {
|
|
1056
|
+
const { rows, columns } = stdout ?? {};
|
|
1057
|
+
return {
|
|
1058
|
+
rows: rows || 24,
|
|
1059
|
+
columns: columns || 80
|
|
1060
|
+
};
|
|
1061
|
+
};
|
|
991
1062
|
// src/ui/index.ts
|
|
992
1063
|
const depthOf = (bits = 4) => bits >= 24 ? 24 : bits >= 8 ? 8 : 4;
|
|
993
1064
|
function createRenderer(options) {
|
|
@@ -996,10 +1067,17 @@ function createRenderer(options) {
|
|
|
996
1067
|
...options,
|
|
997
1068
|
out
|
|
998
1069
|
});
|
|
1070
|
+
const paint = colors(options.output.color, depthOf(options.colorDepth));
|
|
1071
|
+
if (options.output.interactive && options.tty) return new InteractiveRenderer({
|
|
1072
|
+
...options,
|
|
1073
|
+
...options.tty,
|
|
1074
|
+
out,
|
|
1075
|
+
colors: paint
|
|
1076
|
+
});
|
|
999
1077
|
return new PlainRenderer({
|
|
1000
1078
|
...options,
|
|
1001
1079
|
out,
|
|
1002
|
-
colors:
|
|
1080
|
+
colors: paint
|
|
1003
1081
|
});
|
|
1004
1082
|
}
|
|
1005
1083
|
// src/cli.ts
|
|
@@ -1028,7 +1106,7 @@ const EXIT = {
|
|
|
1028
1106
|
};
|
|
1029
1107
|
const version = PACKAGE_VERSION;
|
|
1030
1108
|
const HELP = helpText(version);
|
|
1031
|
-
const { green, red, dim } = colors(!!process.stdout.isTTY && !process.env.NO_COLOR);
|
|
1109
|
+
const { green, red, dim, yellow } = colors(!!process.stdout.isTTY && !process.env.NO_COLOR);
|
|
1032
1110
|
const startedAt = Date.now();
|
|
1033
1111
|
const argv = process.argv.slice(2);
|
|
1034
1112
|
/** What the invocation log needs from the command as it goes. */
|
|
@@ -1072,7 +1150,7 @@ async function run(argv) {
|
|
|
1072
1150
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
1073
1151
|
}
|
|
1074
1152
|
if (command === "init") {
|
|
1075
|
-
const { initCommand } = await import("./init-
|
|
1153
|
+
const { initCommand } = await import("./init-4RPeD-K-.js");
|
|
1076
1154
|
return initCommand(values, filters, {
|
|
1077
1155
|
out: (s) => console.log(s),
|
|
1078
1156
|
input: process.stdin,
|
|
@@ -1081,7 +1159,7 @@ async function run(argv) {
|
|
|
1081
1159
|
});
|
|
1082
1160
|
}
|
|
1083
1161
|
if (command === "doctor") {
|
|
1084
|
-
const { doctorCommand } = await import("./doctor-
|
|
1162
|
+
const { doctorCommand } = await import("./doctor-B2PohneB.js");
|
|
1085
1163
|
return doctorCommand(values, {
|
|
1086
1164
|
out: (s) => console.log(s),
|
|
1087
1165
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -1089,7 +1167,7 @@ async function run(argv) {
|
|
|
1089
1167
|
});
|
|
1090
1168
|
}
|
|
1091
1169
|
if (command === "report") {
|
|
1092
|
-
const { reportCommand } = await import("./command-
|
|
1170
|
+
const { reportCommand } = await import("./command-aUJvENwy.js");
|
|
1093
1171
|
return reportCommand(values, filters, {
|
|
1094
1172
|
out: (s) => console.log(s),
|
|
1095
1173
|
err: (s) => console.error(dim(s)),
|
|
@@ -1104,7 +1182,7 @@ async function run(argv) {
|
|
|
1104
1182
|
}
|
|
1105
1183
|
if (command === "studio") {
|
|
1106
1184
|
const config = await loadConfig(configPath, "studio");
|
|
1107
|
-
const { studioCommand } = await import("./command-
|
|
1185
|
+
const { studioCommand } = await import("./command-BLNVDiIg.js");
|
|
1108
1186
|
return studioCommand({
|
|
1109
1187
|
config,
|
|
1110
1188
|
flags: values,
|
|
@@ -1115,7 +1193,7 @@ async function run(argv) {
|
|
|
1115
1193
|
if (command === "mcp") {
|
|
1116
1194
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
1117
1195
|
const config = await loadConfig(configPath, "mcp");
|
|
1118
|
-
const { mcpCommand } = await import("./command-
|
|
1196
|
+
const { mcpCommand } = await import("./command-B5OyHpOu.js");
|
|
1119
1197
|
return mcpCommand({
|
|
1120
1198
|
config,
|
|
1121
1199
|
flags: values,
|
|
@@ -1124,7 +1202,7 @@ async function run(argv) {
|
|
|
1124
1202
|
}
|
|
1125
1203
|
if (command in HISTORY_COMMANDS) {
|
|
1126
1204
|
const config = await loadConfig(configPath, command);
|
|
1127
|
-
const commands = await import("./commands-
|
|
1205
|
+
const commands = await import("./commands-CfdBX-mc.js");
|
|
1128
1206
|
const ctx = {
|
|
1129
1207
|
config,
|
|
1130
1208
|
flags: values,
|
|
@@ -1137,7 +1215,7 @@ async function run(argv) {
|
|
|
1137
1215
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
1138
1216
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
1139
1217
|
case "query": {
|
|
1140
|
-
const { queryCommand } = await import("./query-
|
|
1218
|
+
const { queryCommand } = await import("./query-DIfkUcPh.js");
|
|
1141
1219
|
return queryCommand({
|
|
1142
1220
|
...ctx,
|
|
1143
1221
|
err: (s) => console.error(dim(s))
|
|
@@ -1145,7 +1223,7 @@ async function run(argv) {
|
|
|
1145
1223
|
}
|
|
1146
1224
|
case "analyze": {
|
|
1147
1225
|
loadEnvFiles(configPath);
|
|
1148
|
-
const { analyzeCommand } = await import("./command-
|
|
1226
|
+
const { analyzeCommand } = await import("./command-CRh-cb6Z.js");
|
|
1149
1227
|
return analyzeCommand({
|
|
1150
1228
|
...ctx,
|
|
1151
1229
|
err: (s) => console.error(dim(s))
|
|
@@ -1159,12 +1237,12 @@ async function run(argv) {
|
|
|
1159
1237
|
}
|
|
1160
1238
|
/** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
|
|
1161
1239
|
async function dryRunPrices(planned, judgeModel) {
|
|
1162
|
-
const { needsPrices } = await import("./cost-
|
|
1240
|
+
const { needsPrices } = await import("./cost-Cr26Alhq.js");
|
|
1163
1241
|
if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
|
|
1164
1242
|
table: null,
|
|
1165
1243
|
why: "nothing to price"
|
|
1166
1244
|
};
|
|
1167
|
-
const { loadPrices } = await import("./prices-
|
|
1245
|
+
const { loadPrices } = await import("./prices-5m3SzMZ3.js");
|
|
1168
1246
|
return loadPrices();
|
|
1169
1247
|
}
|
|
1170
1248
|
/** Asks a yes/no question on the terminal; anything but y or yes is no. */
|
|
@@ -1191,14 +1269,14 @@ async function cleanCommand(flags, configPath) {
|
|
|
1191
1269
|
keep = Number(flags.keep);
|
|
1192
1270
|
if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
|
|
1193
1271
|
}
|
|
1194
|
-
const { debugRoot, pruneDebug } = await import("./files-
|
|
1195
|
-
const { formatBytes } = await import("./debug-
|
|
1272
|
+
const { debugRoot, pruneDebug } = await import("./files-C1Rk9FYA.js");
|
|
1273
|
+
const { formatBytes } = await import("./debug-CF4p4Lp_.js");
|
|
1196
1274
|
const { runs, bytes } = await pruneDebug(config.root, keep);
|
|
1197
1275
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1198
1276
|
console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
|
|
1199
1277
|
if (flags.images) {
|
|
1200
|
-
const { dockerAvailable, pruneImages } = await import("./src-
|
|
1201
|
-
const { requireDocker } = await import("./providers-
|
|
1278
|
+
const { dockerAvailable, pruneImages } = await import("./src-DtodMT59.js");
|
|
1279
|
+
const { requireDocker } = await import("./providers-CVu8e0xS.js");
|
|
1202
1280
|
const engines = [];
|
|
1203
1281
|
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1204
1282
|
if (!engines.length) await requireDocker();
|
|
@@ -1208,24 +1286,25 @@ async function cleanCommand(flags, configPath) {
|
|
|
1208
1286
|
}
|
|
1209
1287
|
return EXIT.ok;
|
|
1210
1288
|
}
|
|
1211
|
-
/** `-v` is 1 and `-vv`
|
|
1289
|
+
/** `-v` is 1 and `-vv` is 2. */
|
|
1212
1290
|
function verbosityOf(flags) {
|
|
1213
|
-
if (flags.trace) return 2;
|
|
1214
1291
|
return Math.min(2, flags.verbose?.length ?? 0);
|
|
1215
1292
|
}
|
|
1216
1293
|
/**
|
|
1217
|
-
* `reeval image build`: collects the evals (as `--dry-run` does)
|
|
1218
|
-
* sandboxes
|
|
1219
|
-
*
|
|
1294
|
+
* `reeval image build`: collects the evals (as `--dry-run` does), pulls every image their `tart()`
|
|
1295
|
+
* sandboxes clone that Tart doesn't have, and builds every image their Docker sandboxes need; with
|
|
1296
|
+
* `--dry-run`, lists them instead. A plan with no Docker sandbox never runs Docker. `--profile`
|
|
1297
|
+
* plans as `reeval run --profile` does. Needs a config, as `reeval run` does: there is no default
|
|
1298
|
+
* image.
|
|
1220
1299
|
*/
|
|
1221
1300
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1222
|
-
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-
|
|
1301
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-BNgcR77K.js");
|
|
1223
1302
|
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
|
|
1224
1303
|
...flags,
|
|
1225
1304
|
"dry-run": true
|
|
1226
1305
|
}, configPath));
|
|
1227
1306
|
const root = config.root;
|
|
1228
|
-
const
|
|
1307
|
+
const uses = await plannedImageUses({
|
|
1229
1308
|
runId,
|
|
1230
1309
|
stagingDir,
|
|
1231
1310
|
config,
|
|
@@ -1233,53 +1312,141 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1233
1312
|
tags,
|
|
1234
1313
|
concurrency,
|
|
1235
1314
|
profile
|
|
1236
|
-
}, filters, (parts) => unmatchedFilter(config, parts))
|
|
1237
|
-
|
|
1238
|
-
|
|
1315
|
+
}, filters, (parts) => unmatchedFilter(config, parts));
|
|
1316
|
+
const targets = imageTargets(uses);
|
|
1317
|
+
const vms = tartTargets(uses);
|
|
1318
|
+
if (targets.length === 0 && vms.length === 0) {
|
|
1319
|
+
console.log("No eval runs in a Docker or Tart sandbox, so there are no images to build or pull.");
|
|
1239
1320
|
return EXIT.ok;
|
|
1240
1321
|
}
|
|
1322
|
+
const { tartTagWarning } = await import("./tart-ref-7Qv3cm3B.js");
|
|
1323
|
+
for (const t of vms) {
|
|
1324
|
+
const warning = tartTagWarning(t.sandbox.image);
|
|
1325
|
+
if (warning) console.log(yellow(`warning: ${warning}`));
|
|
1326
|
+
}
|
|
1241
1327
|
if (flags["dry-run"]) {
|
|
1242
|
-
for (const line of await
|
|
1328
|
+
if (vms.length) for (const line of await tartPlanLines(vms)) console.log(line);
|
|
1329
|
+
if (targets.length) for (const line of await imagePlanLines(targets, root)) console.log(line);
|
|
1243
1330
|
return EXIT.ok;
|
|
1244
1331
|
}
|
|
1245
|
-
const { PrepareError } = await import("./provider-
|
|
1246
|
-
|
|
1332
|
+
const { PrepareError } = await import("./provider-BnUoz4KB.js");
|
|
1333
|
+
const outcomes = [];
|
|
1334
|
+
let refused = false;
|
|
1335
|
+
const refusal = (id, of, others) => (error) => {
|
|
1336
|
+
if (!others || !(error instanceof PrepareError)) throw error;
|
|
1337
|
+
if (error.failure.usage) refused = true;
|
|
1338
|
+
const evals = of.reduce((n, t) => n + t.evals, 0);
|
|
1339
|
+
outcomes.push({
|
|
1340
|
+
id,
|
|
1341
|
+
evals,
|
|
1342
|
+
ok: false,
|
|
1343
|
+
failure: error.failure
|
|
1344
|
+
});
|
|
1345
|
+
return [];
|
|
1346
|
+
};
|
|
1247
1347
|
try {
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1348
|
+
const pulled = vms.length ? await pullTartImages(vms, root, { log: (line) => console.log(line.startsWith("warning: ") ? yellow(line) : dim(line)) }).catch(refusal("tart", vms, targets.length)) : [];
|
|
1349
|
+
for (const p of pulled) outcomes.push(p.ok ? {
|
|
1350
|
+
id: p.id,
|
|
1351
|
+
evals: p.evals,
|
|
1352
|
+
ok: true,
|
|
1353
|
+
ref: p.image.ref,
|
|
1354
|
+
id12: p.image.digest ? short12(p.image.digest) : "local VM"
|
|
1355
|
+
} : {
|
|
1356
|
+
id: p.id,
|
|
1357
|
+
evals: p.evals,
|
|
1358
|
+
ok: false,
|
|
1359
|
+
failure: p.failure
|
|
1251
1360
|
});
|
|
1361
|
+
if (targets.length) {
|
|
1362
|
+
const built = await buildImages(targets, root, {
|
|
1363
|
+
log: (line) => console.log(dim(line)),
|
|
1364
|
+
force: !!flags.force
|
|
1365
|
+
}).catch(refusal("docker", targets, vms.length));
|
|
1366
|
+
for (const b of built) outcomes.push(b.ok ? {
|
|
1367
|
+
id: b.id,
|
|
1368
|
+
evals: b.evals,
|
|
1369
|
+
ok: true,
|
|
1370
|
+
ref: b.image.ref,
|
|
1371
|
+
id12: short12(b.image.id)
|
|
1372
|
+
} : {
|
|
1373
|
+
id: b.id,
|
|
1374
|
+
evals: b.evals,
|
|
1375
|
+
ok: false,
|
|
1376
|
+
failure: b.failure
|
|
1377
|
+
});
|
|
1378
|
+
}
|
|
1252
1379
|
} catch (error) {
|
|
1253
|
-
if (error instanceof PrepareError
|
|
1254
|
-
throw error;
|
|
1380
|
+
if (!(error instanceof PrepareError)) throw error;
|
|
1381
|
+
if (error.failure.usage) throw new UsageError(error.message);
|
|
1382
|
+
if (error.failure.cause === "enospc") invocation.machineState = true;
|
|
1383
|
+
const { PreflightInfraError } = await import("./preflight-4gWSy0Pe.js");
|
|
1384
|
+
throw new PreflightInfraError(error.message);
|
|
1255
1385
|
}
|
|
1256
|
-
for (const
|
|
1257
|
-
if (
|
|
1258
|
-
|
|
1259
|
-
console.log(`${green("✓")} ${green(`${b.id}: ${b.image.ref}`)}${dim(` ${id}`)}`);
|
|
1386
|
+
for (const o of outcomes) {
|
|
1387
|
+
if (o.ok) {
|
|
1388
|
+
console.log(`${green("✓")} ${green(`${o.id}: ${o.ref}`)}${dim(` ${o.id12}`)}`);
|
|
1260
1389
|
continue;
|
|
1261
1390
|
}
|
|
1262
|
-
const uses =
|
|
1263
|
-
const [headline = "", ...rest] =
|
|
1264
|
-
console.log(`${red("✗")} ${red(`${
|
|
1391
|
+
const uses = o.evals ? ` (${o.evals} eval${o.evals === 1 ? "" : "s"})` : "";
|
|
1392
|
+
const [headline = "", ...rest] = o.failure.message.split("\n");
|
|
1393
|
+
console.log(`${red("✗")} ${red(`${o.id}${uses}: ${headline}`)}`);
|
|
1265
1394
|
for (const line of rest) console.log(` ${line}`);
|
|
1266
1395
|
}
|
|
1267
|
-
const failed =
|
|
1268
|
-
const n =
|
|
1269
|
-
console.log(failed.length ? red(`${failed.length} of ${n} image${n === 1 ? "" : "s"} failed to build`) : dim(`${n} image${n === 1 ? "" : "s"} ready`));
|
|
1396
|
+
const failed = outcomes.filter((o) => !o.ok);
|
|
1397
|
+
const n = outcomes.length;
|
|
1398
|
+
console.log(failed.length ? red(`${failed.length} of ${n} image${n === 1 ? "" : "s"} failed to ${vms.length ? targets.length ? "build or pull" : "pull" : "build"}`) : dim(`${n} image${n === 1 ? "" : "s"} ready`));
|
|
1270
1399
|
if (!failed.length) return EXIT.ok;
|
|
1271
|
-
|
|
1400
|
+
if (refused || failed.every((o) => !o.ok && o.failure.usage)) return EXIT.usage;
|
|
1401
|
+
invocation.machineState = failed.every((o) => !o.ok && o.failure.cause === "enospc");
|
|
1402
|
+
return EXIT.infra;
|
|
1272
1403
|
}
|
|
1404
|
+
/** The first 12 hex digits of an image id or digest. */
|
|
1405
|
+
const short12 = (id) => id.replace(/^sha256:/, "").slice(0, 12);
|
|
1273
1406
|
/** `reeval run`: check, plan, run the engine, then write the run and pick the exit code. */
|
|
1274
|
-
async function runCommand(flags, configPath,
|
|
1275
|
-
const
|
|
1407
|
+
async function runCommand(flags, configPath, given) {
|
|
1408
|
+
const retry = flags["retry-errors"];
|
|
1409
|
+
if (retry !== void 0 && flags.resume !== void 0) throw new UsageError("--resume and --retry-errors both continue a run: pass one of them.");
|
|
1410
|
+
if (retry !== void 0 && (given.length || flags.tag?.length || flags.trait?.length)) throw new UsageError("--retry-errors runs every infra-errored attempt of the run again, with its plan: leave out filters, --tag and --trait.");
|
|
1411
|
+
const resumed = flags.resume !== void 0 || retry !== void 0 ? await resumedRun(flags, given, configPath) : null;
|
|
1412
|
+
if (resumed?.nothing) {
|
|
1413
|
+
const { nothing, record } = resumed.nothing;
|
|
1414
|
+
const outcome = runOutcome(record);
|
|
1415
|
+
if (flags.json) {
|
|
1416
|
+
console.error(nothing);
|
|
1417
|
+
const { suiteResult } = await import("./suite-CKBa6MKt.js");
|
|
1418
|
+
process.stdout.write(`${JSON.stringify({
|
|
1419
|
+
...suiteResult(record),
|
|
1420
|
+
outcome
|
|
1421
|
+
}, null, 2)}\n`);
|
|
1422
|
+
} else console.log(nothing);
|
|
1423
|
+
return outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1424
|
+
}
|
|
1425
|
+
const runFlags = resumed?.settings ? {
|
|
1426
|
+
...flags,
|
|
1427
|
+
...resumed.settings
|
|
1428
|
+
} : flags;
|
|
1429
|
+
const filters = resumed?.settings?.filters ?? given;
|
|
1430
|
+
const setup = await prepareRun(runFlags, configPath, {
|
|
1431
|
+
noProfile: !!resumed?.saved && resumed.saved.profile === void 0,
|
|
1432
|
+
...resumed?.profileOf ? { profileOf: resumed.profileOf } : {}
|
|
1433
|
+
});
|
|
1434
|
+
const explicit = runFlags.concurrency ?? (process.env.REEVAL_CONCURRENCY || void 0);
|
|
1435
|
+
const startedWith = resumed ? resumed.saved ?? void 0 : {
|
|
1436
|
+
...setup.runs === void 0 ? {} : { runs: setup.runs },
|
|
1437
|
+
...setup.profile ? { profile: setup.profile.name } : {},
|
|
1438
|
+
...explicit === void 0 ? {} : { concurrency: setup.concurrency },
|
|
1439
|
+
filters: savedFilters(setup.config.root, filters),
|
|
1440
|
+
tags: runFlags.tag ?? [],
|
|
1441
|
+
...runFlags.trait?.length ? { traits: runFlags.trait } : {}
|
|
1442
|
+
};
|
|
1276
1443
|
const ctx = await startRun(setup);
|
|
1277
1444
|
invocation.runId = ctx.runId;
|
|
1278
1445
|
const githubOut = process.env.GITHUB_OUTPUT;
|
|
1279
1446
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1280
1447
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1281
1448
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1282
|
-
const { engineVersions } = await import("./node-
|
|
1449
|
+
const { engineVersions } = await import("./node-D7pAU--0.js");
|
|
1283
1450
|
const terminal = setup.json && !setup.dryRun ? process.stderr : process.stdout;
|
|
1284
1451
|
const renderer = createRenderer({
|
|
1285
1452
|
out: setup.json && setup.dryRun ? { write: () => true } : terminal,
|
|
@@ -1287,31 +1454,48 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1287
1454
|
colorDepth: terminal.isTTY ? terminal.getColorDepth() : void 0,
|
|
1288
1455
|
output: setup.output,
|
|
1289
1456
|
verbosity: ctx.verbosity ?? 0,
|
|
1290
|
-
harness: setup.config.harness[0]?.kind ?? "agent"
|
|
1457
|
+
harness: setup.config.harness[0]?.kind ?? "agent",
|
|
1458
|
+
tty: {
|
|
1459
|
+
stdout: process.stdout,
|
|
1460
|
+
stdin: process.stdin
|
|
1461
|
+
}
|
|
1291
1462
|
});
|
|
1292
1463
|
let executed;
|
|
1293
1464
|
try {
|
|
1294
|
-
executed = await executeRun(ctx, setup, filters, renderer
|
|
1465
|
+
executed = await executeRun(ctx, setup, filters, renderer, {
|
|
1466
|
+
startedWith,
|
|
1467
|
+
notes: resumed?.notes ?? [],
|
|
1468
|
+
narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length || !!flags.trait?.length)
|
|
1469
|
+
});
|
|
1295
1470
|
} catch (error) {
|
|
1296
1471
|
renderer.close();
|
|
1297
|
-
if (
|
|
1472
|
+
if (ctx.retry) await restoreRetry(setup.config.root, ctx.runId);
|
|
1473
|
+
else if (setup.resume === void 0) await rm(ctx.stagingDir, {
|
|
1298
1474
|
recursive: true,
|
|
1299
1475
|
force: true
|
|
1300
1476
|
});
|
|
1301
1477
|
throw error;
|
|
1302
1478
|
}
|
|
1303
|
-
const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
|
|
1479
|
+
const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions, leftOut } = executed;
|
|
1480
|
+
const { retried } = executed;
|
|
1304
1481
|
invocation.unhandledErrors = engineRun.unhandledErrors;
|
|
1305
|
-
|
|
1306
|
-
if ((interrupted || engineRun.canceled) && engines.length) {
|
|
1307
|
-
const { removeRunContainers } = await import("./docker-CIzHzwlC.js");
|
|
1308
|
-
await removeRunContainers(ctx.runId, engines);
|
|
1309
|
-
}
|
|
1482
|
+
if ((interrupted || engineRun.canceled) && plan) await cleanupRun(ctx.runId, plan.sandboxes);
|
|
1310
1483
|
if (engineRun.stopped || !plan) {
|
|
1311
1484
|
live?.remove();
|
|
1312
1485
|
renderer.close();
|
|
1313
1486
|
return EXIT.ok;
|
|
1314
1487
|
}
|
|
1488
|
+
if (ctx.retry && ctx.previous && retried.size) {
|
|
1489
|
+
const { retryEnvironmentChanges } = await import("./retry-ChzISxmy.js");
|
|
1490
|
+
const changes = retryEnvironmentChanges(ctx.previous, ctx.stagingDir, retried);
|
|
1491
|
+
if (changes.length) {
|
|
1492
|
+
questions?.cancel();
|
|
1493
|
+
live?.remove();
|
|
1494
|
+
renderer.close();
|
|
1495
|
+
await restoreRetry(setup.config.root, ctx.runId);
|
|
1496
|
+
throw new UsageError(`run ${ctx.runId}'s plan changed since it ran: ${changes.join("; ")}. ${retryVerb(ctx.retry, ctx.runId)} only with the plan they ran with, so the run is left as it was: put it back, or start a new run with reeval run.`);
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1315
1499
|
let stopped = cutShort || engineRun.canceled;
|
|
1316
1500
|
if (stopped) questions?.cancel();
|
|
1317
1501
|
let answers = stopped ? await questions?.answered() : await questions?.settle();
|
|
@@ -1321,8 +1505,8 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1321
1505
|
}
|
|
1322
1506
|
finishing = true;
|
|
1323
1507
|
live?.phase("finishing");
|
|
1324
|
-
const { lockfiles } = await import("./state-
|
|
1325
|
-
const { writeScratchRun } = await import("./results-
|
|
1508
|
+
const { lockfiles } = await import("./state-kimtIhBP.js");
|
|
1509
|
+
const { writeScratchRun } = await import("./results-JyuM64gy.js");
|
|
1326
1510
|
const run = await writeScratchRun({
|
|
1327
1511
|
ctx,
|
|
1328
1512
|
author: ctx.author,
|
|
@@ -1337,18 +1521,26 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1337
1521
|
language: { name: "typescript" },
|
|
1338
1522
|
enginePackages: engineVersions,
|
|
1339
1523
|
sandbox: sandboxVersions,
|
|
1524
|
+
...ctx.retry ? { retry: ctx.retry } : {},
|
|
1340
1525
|
worktreePatch: ctx.worktreePatch,
|
|
1341
1526
|
lockfiles: await lockfiles(ctx.config.root),
|
|
1342
1527
|
git: ctx.git,
|
|
1343
1528
|
previous: ctx.previous,
|
|
1344
1529
|
previousArtifacts: ctx.previousArtifacts,
|
|
1530
|
+
leftOut,
|
|
1345
1531
|
runner: runnerInfo(),
|
|
1532
|
+
...startedWith ? { invocation: startedWith } : {},
|
|
1346
1533
|
...answers && (answers.evals.size || answers.attempts.size) ? { answers } : {}
|
|
1347
1534
|
}, () => live?.stop());
|
|
1348
1535
|
const { record } = run;
|
|
1349
1536
|
const outcome = runOutcome(record);
|
|
1350
|
-
const
|
|
1351
|
-
const
|
|
1537
|
+
const notRun = leftOut.filter((l) => !l.done).length;
|
|
1538
|
+
const ran = notRun > 0 && !stopped && record.status === "interrupted" ? runOutcome({
|
|
1539
|
+
...record,
|
|
1540
|
+
status: "complete"
|
|
1541
|
+
}) : outcome;
|
|
1542
|
+
const code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1543
|
+
const { pruneDebug } = await import("./files-C1Rk9FYA.js");
|
|
1352
1544
|
await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
|
|
1353
1545
|
const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
|
|
1354
1546
|
renderer.onEvent({
|
|
@@ -1357,15 +1549,25 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1357
1549
|
record,
|
|
1358
1550
|
summaryPath: fromCwd(summary),
|
|
1359
1551
|
results: engineRun.evals,
|
|
1360
|
-
remarks: run.remarks
|
|
1552
|
+
remarks: run.remarks,
|
|
1553
|
+
...notRun > 0 && !stopped ? { notRun } : {}
|
|
1361
1554
|
});
|
|
1362
1555
|
renderer.close();
|
|
1556
|
+
if (retried.size) {
|
|
1557
|
+
const { retryCounts, retryLine } = await import("./retry-ChzISxmy.js");
|
|
1558
|
+
const line = retryLine(retryCounts(record, retried, ctx.previous));
|
|
1559
|
+
if (setup.json) console.error(line);
|
|
1560
|
+
else console.log(line);
|
|
1561
|
+
}
|
|
1363
1562
|
const stepSummary = process.env.GITHUB_STEP_SUMMARY;
|
|
1364
1563
|
if (stepSummary && run.artifactDir) await appendFile(stepSummary, await readFile(summary)).catch(() => {});
|
|
1365
1564
|
const { config } = ctx;
|
|
1366
1565
|
const mode = setup.record;
|
|
1367
|
-
|
|
1368
|
-
|
|
1566
|
+
const { recordFile } = await import("./files-DnlA1A7N.js");
|
|
1567
|
+
const rerecord = (setup.retry !== void 0 || setup.resume !== void 0) && record.status === "complete" && existsSync(recordFile(config.history.dir, record.runId));
|
|
1568
|
+
const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI() || rerecord;
|
|
1569
|
+
if (autoRecord && record.status !== "interrupted") {
|
|
1570
|
+
const { record: recordCmd } = await import("./commands-CfdBX-mc.js");
|
|
1369
1571
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
1370
1572
|
const line = await recordCmd(config, record.runId, {
|
|
1371
1573
|
...flags,
|
|
@@ -1373,15 +1575,78 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1373
1575
|
});
|
|
1374
1576
|
if (setup.json) console.error(line);
|
|
1375
1577
|
else console.log(line);
|
|
1578
|
+
} else if (autoRecord && notRun > 0 && !stopped) {
|
|
1579
|
+
const line = `Not recorded: run ${record.runId} is incomplete. reeval run --resume ${record.runId}${flags.record ? " --record" : ""} runs the rest and records it.`;
|
|
1580
|
+
if (setup.json) console.error(line);
|
|
1581
|
+
else console.log(line);
|
|
1376
1582
|
}
|
|
1377
1583
|
if (setup.json) {
|
|
1378
|
-
const { suiteResult } = await import("./suite-
|
|
1379
|
-
process.stdout.write(`${JSON.stringify(
|
|
1584
|
+
const { suiteResult } = await import("./suite-CKBa6MKt.js");
|
|
1585
|
+
process.stdout.write(`${JSON.stringify({
|
|
1586
|
+
...suiteResult(record),
|
|
1587
|
+
outcome: ran
|
|
1588
|
+
}, null, 2)}\n`);
|
|
1380
1589
|
}
|
|
1590
|
+
await renderer.wait?.();
|
|
1381
1591
|
return code;
|
|
1382
1592
|
}
|
|
1383
|
-
/**
|
|
1384
|
-
|
|
1593
|
+
/**
|
|
1594
|
+
* `--resume`: the run's saved settings with the flags given now. An older reeval's run saved none,
|
|
1595
|
+
* so it plans with the flags given now, with a note.
|
|
1596
|
+
*/
|
|
1597
|
+
async function resumedRun(flags, filters, configPath) {
|
|
1598
|
+
const path = requireConfig(configPath);
|
|
1599
|
+
loadEnvFiles(path);
|
|
1600
|
+
const root = dirname(path);
|
|
1601
|
+
const target = await continuedRun(root, flags);
|
|
1602
|
+
if (flags["retry-errors"] !== void 0 && target.previous && !erroredAttempts(target.previous).length) return {
|
|
1603
|
+
nothing: {
|
|
1604
|
+
nothing: `run ${target.runId} has no infra errors, so --retry-errors has nothing to run again.`,
|
|
1605
|
+
record: target.previous
|
|
1606
|
+
},
|
|
1607
|
+
saved: null,
|
|
1608
|
+
settings: null,
|
|
1609
|
+
notes: [],
|
|
1610
|
+
profileOf: void 0
|
|
1611
|
+
};
|
|
1612
|
+
if (flags.resume !== void 0 && await stoppedRetry(root, target) && (filters.length || flags.tag?.length || flags.trait?.length)) throw new UsageError(`run ${target.runId} stopped during a --retry-errors, which runs every infra-errored attempt of the run again, with its plan: leave out filters, --tag and --trait.`);
|
|
1613
|
+
const saved = await savedInvocation(root, target);
|
|
1614
|
+
if (!saved) return {
|
|
1615
|
+
nothing: null,
|
|
1616
|
+
saved: null,
|
|
1617
|
+
settings: null,
|
|
1618
|
+
notes: [`run ${target.runId} saved no --runs, profile or filters, so this ${flags.resume === void 0 ? "retry" : "resume"} plans with the ones given now`],
|
|
1619
|
+
profileOf: void 0
|
|
1620
|
+
};
|
|
1621
|
+
const given = {
|
|
1622
|
+
runs: flags.runs,
|
|
1623
|
+
profile: flags.profile,
|
|
1624
|
+
concurrency: flags.concurrency,
|
|
1625
|
+
tag: flags.tag,
|
|
1626
|
+
trait: flags.trait,
|
|
1627
|
+
filters
|
|
1628
|
+
};
|
|
1629
|
+
const flag = flags.resume === void 0 ? "--retry-errors" : "--resume";
|
|
1630
|
+
const resumed = resumeSettings(target.runId, saved, given, {
|
|
1631
|
+
root,
|
|
1632
|
+
flag
|
|
1633
|
+
});
|
|
1634
|
+
const fromRun = flags.profile === void 0 && resumed.settings.profile !== void 0;
|
|
1635
|
+
return {
|
|
1636
|
+
nothing: null,
|
|
1637
|
+
saved,
|
|
1638
|
+
...resumed,
|
|
1639
|
+
profileOf: fromRun ? target.runId : void 0
|
|
1640
|
+
};
|
|
1641
|
+
}
|
|
1642
|
+
/** The run `--resume` continues, or the finished one `--retry-errors` runs the errors of again. */
|
|
1643
|
+
const continuedRun = (root, flags) => flags["retry-errors"] !== void 0 ? resolveRetry(root, flags["retry-errors"] || void 0) : resolveResume(root, flags.resume || void 0);
|
|
1644
|
+
/**
|
|
1645
|
+
* Everything that can fail before the evals are collected: output mode, config, profile, flags.
|
|
1646
|
+
* `noProfile` is a resume of a run that had none, which `defaultProfile` mustn't give one;
|
|
1647
|
+
* `profileOf` names the resumed run whose profile `flags.profile` is.
|
|
1648
|
+
*/
|
|
1649
|
+
async function prepareRun(flags, configPath, { noProfile = false, profileOf } = {}) {
|
|
1385
1650
|
const output = resolveOutput({
|
|
1386
1651
|
flags: {
|
|
1387
1652
|
ui: flags.ui,
|
|
@@ -1392,24 +1657,23 @@ async function prepareRun(flags, configPath) {
|
|
|
1392
1657
|
},
|
|
1393
1658
|
env: process.env,
|
|
1394
1659
|
isTTY: !!(flags.json && !flags["dry-run"] ? process.stderr : process.stdout).isTTY,
|
|
1395
|
-
agent: detectAgent().name
|
|
1660
|
+
agent: detectAgent().name,
|
|
1661
|
+
stdinTTY: !!process.stdin.isTTY && !flags.json && !flags["dry-run"]
|
|
1396
1662
|
});
|
|
1397
1663
|
const path = requireConfig(configPath);
|
|
1398
1664
|
loadEnvFiles(path);
|
|
1399
|
-
const named = flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
|
|
1665
|
+
const named = noProfile ? void 0 : flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
|
|
1400
1666
|
const config = await loadConfig(path, "run", named);
|
|
1401
|
-
|
|
1667
|
+
if (profileOf && flags.profile && !config.profiles[flags.profile]) throw new UsageError(`run ${profileOf} was started with profile ${flags.profile}, which ${config.file} no longer defines. Put it back to resume the run, or start a new run without --resume.`);
|
|
1668
|
+
const profile = noProfile ? null : selectProfile(config, flags.profile);
|
|
1402
1669
|
let runs;
|
|
1403
1670
|
if (flags.runs !== void 0) {
|
|
1404
1671
|
runs = Number(flags.runs);
|
|
1405
1672
|
if (!(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`--runs must be a whole number of at least 1 (got ${flags.runs})`);
|
|
1406
1673
|
}
|
|
1407
|
-
const sandboxes = [
|
|
1674
|
+
const sandboxes = [...(config.environments.length ? config.environments : [void 0]).map((e) => environmentSandbox(e, config.sandbox)), ...profile ? [profileSandbox(config, profile.defaults)] : []];
|
|
1408
1675
|
const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
|
|
1409
|
-
|
|
1410
|
-
if (!container) return hostCapacity();
|
|
1411
|
-
const { dockerCapacity } = await import("./src-DVaAKCsH.js");
|
|
1412
|
-
return await dockerCapacity(container.engine) ?? hostCapacity();
|
|
1676
|
+
return await sandboxCapacity(sandboxes) ?? hostCapacity();
|
|
1413
1677
|
});
|
|
1414
1678
|
return {
|
|
1415
1679
|
output,
|
|
@@ -1417,6 +1681,7 @@ async function prepareRun(flags, configPath) {
|
|
|
1417
1681
|
profile,
|
|
1418
1682
|
runs,
|
|
1419
1683
|
tags: [...flags.tag ?? [], ...profile?.tags ?? []],
|
|
1684
|
+
traits: traitFlags(flags.trait),
|
|
1420
1685
|
concurrency,
|
|
1421
1686
|
budget: profile?.budget ?? config.budget,
|
|
1422
1687
|
record: profile?.record ?? config.record,
|
|
@@ -1428,37 +1693,48 @@ async function prepareRun(flags, configPath) {
|
|
|
1428
1693
|
/** Named in the plan line when the project root isn't the current directory. */
|
|
1429
1694
|
configElsewhere: dirname(path) !== process.cwd(),
|
|
1430
1695
|
/** `undefined` is a new run; `''` is newest resumable; a string is a run id or prefix. */
|
|
1431
|
-
resume: flags.resume
|
|
1696
|
+
resume: flags.resume,
|
|
1697
|
+
/** Like `resume`, for `--retry-errors`. */
|
|
1698
|
+
retry: flags["retry-errors"]
|
|
1432
1699
|
};
|
|
1433
1700
|
}
|
|
1701
|
+
/** What a retry, or a resume that finishes one, does to the run's errors, for its refusals. */
|
|
1702
|
+
const retryVerb = (retry, runId) => retry.resume ? `--resume finishes run ${runId}'s --retry-errors, which runs its errors again` : "--retry-errors runs its errors again";
|
|
1434
1703
|
/** The run's id and the state it starts from: commit, uncommitted changes, author. */
|
|
1435
|
-
async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
|
|
1704
|
+
async function startRun({ config, profile, runs, tags, traits, concurrency, verbosity, configFile, resume, retry, dryRun }) {
|
|
1436
1705
|
const startedAt = /* @__PURE__ */ new Date();
|
|
1437
1706
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
1438
|
-
const { worktreePatch } = await import("./state-
|
|
1707
|
+
const { worktreePatch } = await import("./state-kimtIhBP.js");
|
|
1439
1708
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
1440
1709
|
const who = await author(config.root);
|
|
1441
|
-
const { stagingDir, sweepStaging } = await import("./results-
|
|
1442
|
-
const { scratchDir } = await import("./files-
|
|
1710
|
+
const { stagingDir, sweepStaging } = await import("./results-JyuM64gy.js");
|
|
1711
|
+
const { scratchDir } = await import("./files-DnlA1A7N.js");
|
|
1443
1712
|
const scratch = scratchDir(config.root);
|
|
1444
|
-
if (resume !== void 0) {
|
|
1445
|
-
const target = await
|
|
1446
|
-
|
|
1713
|
+
if (resume !== void 0 || retry !== void 0) {
|
|
1714
|
+
const target = await continuedRun(config.root, {
|
|
1715
|
+
resume,
|
|
1716
|
+
"retry-errors": retry
|
|
1717
|
+
});
|
|
1718
|
+
if (!dryRun && retry === void 0) await prepareResume(target, config.root);
|
|
1719
|
+
else if (!dryRun) holdRetry(target, config.root);
|
|
1720
|
+
const stopped = retry === void 0 ? await stoppedRetry(config.root, target) : null;
|
|
1447
1721
|
await sweepStaging(config.root, target.runId);
|
|
1448
1722
|
const aside = dryRun ? {
|
|
1449
1723
|
record: target.previous,
|
|
1450
1724
|
artifacts: target.previousArtifacts
|
|
1451
1725
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
1452
1726
|
if (!dryRun) try {
|
|
1453
|
-
|
|
1454
|
-
await removeRunContainers(target.runId, ["docker", "podman"]);
|
|
1727
|
+
await cleanupRun(target.runId);
|
|
1455
1728
|
} catch {}
|
|
1729
|
+
const { unfoldCases } = await import("./cases-Dy0mifal.js");
|
|
1730
|
+
const previous = aside.record ?? target.previous;
|
|
1456
1731
|
return {
|
|
1457
1732
|
runId: target.runId,
|
|
1458
1733
|
stagingDir: stagingDir(config.root, target.runId),
|
|
1459
1734
|
config,
|
|
1460
1735
|
runs,
|
|
1461
1736
|
tags,
|
|
1737
|
+
traits,
|
|
1462
1738
|
concurrency,
|
|
1463
1739
|
profile,
|
|
1464
1740
|
verbosity,
|
|
@@ -1468,8 +1744,17 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1468
1744
|
worktreePatch: patch,
|
|
1469
1745
|
author: who,
|
|
1470
1746
|
scratch,
|
|
1471
|
-
previous:
|
|
1472
|
-
previousArtifacts: aside.artifacts ?? target.previousArtifacts
|
|
1747
|
+
previous: previous && unfoldCases(previous),
|
|
1748
|
+
previousArtifacts: aside.artifacts ?? target.previousArtifacts,
|
|
1749
|
+
...retry === void 0 ? {} : { retryTarget: target },
|
|
1750
|
+
...retry !== void 0 ? { retry: {
|
|
1751
|
+
startedAt,
|
|
1752
|
+
resume: false
|
|
1753
|
+
} } : stopped ? { retry: {
|
|
1754
|
+
startedAt,
|
|
1755
|
+
resume: true,
|
|
1756
|
+
...stopped.began ? { began: stopped.began } : {}
|
|
1757
|
+
} } : {}
|
|
1473
1758
|
};
|
|
1474
1759
|
}
|
|
1475
1760
|
const stamps = new Set([scratch, config.history.dir].flatMap((dir) => existsSync(dir) ? readdirSync(dir).map((name) => name.slice(0, 16)) : []));
|
|
@@ -1482,6 +1767,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1482
1767
|
config,
|
|
1483
1768
|
runs,
|
|
1484
1769
|
tags,
|
|
1770
|
+
traits,
|
|
1485
1771
|
concurrency,
|
|
1486
1772
|
profile,
|
|
1487
1773
|
verbosity,
|
|
@@ -1497,11 +1783,33 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1497
1783
|
}
|
|
1498
1784
|
function noMatch(config, filters, tags) {
|
|
1499
1785
|
const narrowed = [filters.length ? `filter "${filters.join(" ")}"` : "", tags.length ? `tags ${tags.join(", ")}` : ""].filter(Boolean);
|
|
1500
|
-
return new UsageError(`No evals matched ${config.include.join(", ")}${narrowed.length ? ` with ${narrowed.join(" and ")}` : ""}`);
|
|
1786
|
+
return new UsageError(`No evals matched ${config.include.join(", ")}${narrowed.length ? ` with ${narrowed.join(" and ")}` : ""}${maskedFilterHint(filters)}`);
|
|
1787
|
+
}
|
|
1788
|
+
/**
|
|
1789
|
+
* Why a filter that holds a secret matches nothing: eval names are masked before they're
|
|
1790
|
+
* matched. Printed masked, it's the filter that would match.
|
|
1791
|
+
*/
|
|
1792
|
+
function maskedFilterHint(parts) {
|
|
1793
|
+
const masked = parts.filter((p) => redactSecrets(p) !== p);
|
|
1794
|
+
if (!masked.length) return "";
|
|
1795
|
+
return `. Eval names are masked, so a filter that holds a secret can't match one: filter by the masked name (${masked.map((p) => `"${redactSecrets(p)}"`).join(", ")}) or a part of it without the secret`;
|
|
1501
1796
|
}
|
|
1502
1797
|
function unmatchedFilter(config, parts) {
|
|
1503
1798
|
const which = parts.map((p) => `"${p}"`).join(", ");
|
|
1504
|
-
return new UsageError(`filter ${which} ${parts.length === 1 ? "matches" : "match"} no eval file, eval name or cell in ${config.include.join(", ")}`);
|
|
1799
|
+
return new UsageError(`filter ${which} ${parts.length === 1 ? "matches" : "match"} no eval file, eval name or cell in ${config.include.join(", ")}${maskedFilterHint(parts)}`);
|
|
1800
|
+
}
|
|
1801
|
+
/**
|
|
1802
|
+
* An eval whose id holds a value declared with `secret()`. `evaluate` masks names as the
|
|
1803
|
+
* worker's environment shows secrets, but a declared one is only known once the plan has the
|
|
1804
|
+
* environments, after the names became ids, so it's refused rather than written.
|
|
1805
|
+
*/
|
|
1806
|
+
function namesDeclaredSecret(evals) {
|
|
1807
|
+
for (const { id } of evals) {
|
|
1808
|
+
const [file, ...names] = id.split(" › ");
|
|
1809
|
+
const masked = names.map((n) => redactSecrets(n));
|
|
1810
|
+
if (masked.every((n, i) => n === names[i])) continue;
|
|
1811
|
+
return new UsageError(`${[file, ...masked].join(" › ")} names a secret declared with secret(): name it without the secret.`);
|
|
1812
|
+
}
|
|
1505
1813
|
}
|
|
1506
1814
|
/**
|
|
1507
1815
|
* Reads Cursor's model catalog for the planned Cursor cells, checks their `effort` and `params`
|
|
@@ -1511,12 +1819,12 @@ function unmatchedFilter(config, parts) {
|
|
|
1511
1819
|
*/
|
|
1512
1820
|
async function planEfforts(planned) {
|
|
1513
1821
|
const live = planned.filter((p) => !p.attempt.plan.skip);
|
|
1514
|
-
const { effortNotes, withDefaultEffort } = await import("./default-effort-
|
|
1822
|
+
const { effortNotes, withDefaultEffort } = await import("./default-effort-BAy8ixd1.js");
|
|
1515
1823
|
let cursor;
|
|
1516
1824
|
let unchecked = "CURSOR_API_KEY isn't set";
|
|
1517
1825
|
const key = process.env.CURSOR_API_KEY;
|
|
1518
1826
|
if (key && live.some((p) => p.attempt.harness.kind === "cursor")) {
|
|
1519
|
-
const { checkCursorPlan } = await import("./cursor-catalog-
|
|
1827
|
+
const { checkCursorPlan } = await import("./cursor-catalog-BSn9xjmo.js");
|
|
1520
1828
|
const checked = await checkCursorPlan(key, live);
|
|
1521
1829
|
if (checked instanceof Error) unchecked = `its request failed: ${checked.message}`;
|
|
1522
1830
|
else {
|
|
@@ -1536,22 +1844,24 @@ async function planEfforts(planned) {
|
|
|
1536
1844
|
* Collects the evals and prints the plan; stops there for `--dry-run`, or on a budget, credential
|
|
1537
1845
|
* or Docker problem (exit 2). Then runs the engine with live output and Ctrl-C handling.
|
|
1538
1846
|
*/
|
|
1539
|
-
async function executeRun(ctx, setup, filters, renderer) {
|
|
1540
|
-
const { runId, stagingDir, config, runs, tags, concurrency, profile, verbosity
|
|
1847
|
+
async function executeRun(ctx, setup, filters, renderer, started) {
|
|
1848
|
+
const { runId, stagingDir, config, runs, tags, traits, concurrency, profile, verbosity } = ctx;
|
|
1849
|
+
const { configFile } = ctx;
|
|
1541
1850
|
const run = JSON.parse(JSON.stringify({
|
|
1542
1851
|
runId,
|
|
1543
1852
|
stagingDir,
|
|
1544
1853
|
config,
|
|
1545
1854
|
runs,
|
|
1546
1855
|
tags,
|
|
1856
|
+
traits,
|
|
1547
1857
|
concurrency,
|
|
1548
1858
|
profile,
|
|
1549
1859
|
verbosity,
|
|
1550
1860
|
configFile,
|
|
1551
1861
|
completed: []
|
|
1552
1862
|
}));
|
|
1553
|
-
const { runEngine } = await import("./node-
|
|
1554
|
-
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-
|
|
1863
|
+
const { runEngine } = await import("./node-D7pAU--0.js");
|
|
1864
|
+
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-Cgq9NCEX.js");
|
|
1555
1865
|
let plan;
|
|
1556
1866
|
let sandboxVersions = { images: [] };
|
|
1557
1867
|
let preflightStop;
|
|
@@ -1560,13 +1870,22 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1560
1870
|
let toRun = [];
|
|
1561
1871
|
let live;
|
|
1562
1872
|
let kept = /* @__PURE__ */ new Map();
|
|
1873
|
+
let leftOut = [];
|
|
1874
|
+
let retried = /* @__PURE__ */ new Set();
|
|
1563
1875
|
let questions;
|
|
1564
1876
|
const onPlan = async (collected) => {
|
|
1565
1877
|
const { planned, excluded, matched, unmatched } = collected;
|
|
1566
1878
|
if (unmatched.length) throw unmatchedFilter(config, unmatched);
|
|
1567
1879
|
if (matched === 0) throw noMatch(config, filters, tags);
|
|
1568
1880
|
checkLimitHarness(profile, config.file, collected);
|
|
1881
|
+
checkTraitKeys(traitFilters(traits, profile), collected.traits);
|
|
1882
|
+
const continuing = setup.resume !== void 0 || ctx.retryTarget !== void 0;
|
|
1883
|
+
const { errored } = continuing ? noteRetries(planned, stagingDir, ctx.previous, false) : { errored: /* @__PURE__ */ new Set() };
|
|
1569
1884
|
const completed = completedKeysFor(planned, stagingDir, ctx.previous);
|
|
1885
|
+
if (ctx.retry) for (const { key } of planned) {
|
|
1886
|
+
const was = previousAttempt(ctx.previous, key);
|
|
1887
|
+
if (was && was.outcome !== "infra_error") completed.add(key);
|
|
1888
|
+
}
|
|
1570
1889
|
const skipped = planned.map((p) => completed.has(p.key) ? {
|
|
1571
1890
|
...p,
|
|
1572
1891
|
attempt: {
|
|
@@ -1580,9 +1899,29 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1580
1899
|
} : p);
|
|
1581
1900
|
toRun = skipped;
|
|
1582
1901
|
for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
|
|
1902
|
+
const named = namesDeclaredSecret([...planned, ...excluded]);
|
|
1903
|
+
if (named) throw named;
|
|
1583
1904
|
const efforts = await planEfforts(skipped);
|
|
1584
1905
|
if (efforts) run.cursorEfforts = efforts;
|
|
1585
|
-
|
|
1906
|
+
if (ctx.retry && ctx.previous) {
|
|
1907
|
+
const ranOn = ctx.previous.stack?.reeval;
|
|
1908
|
+
if (ranOn && ranOn !== version) throw new UsageError(`run ${runId} ran on reeval ${ranOn}, and this is ${version}. ${retryVerb(ctx.retry, runId)} only on the reeval they ran on: install reeval ${ranOn}, or start a new run with reeval run.`);
|
|
1909
|
+
const { retryChanges } = await import("./retry-ChzISxmy.js");
|
|
1910
|
+
const { judgePart } = await import("./assemble-UPCRrtOz.js");
|
|
1911
|
+
const { engineVersions } = await import("./node-D7pAU--0.js");
|
|
1912
|
+
const judged = judgePart(config, profile, version, engineVersions);
|
|
1913
|
+
const runs = new Set(planned.map((p) => p.key).filter((key) => !completed.has(key)));
|
|
1914
|
+
const changes = await retryChanges(ctx.previous, planned, runs, config.root, judged);
|
|
1915
|
+
if (changes.length) throw new UsageError(`run ${runId}'s plan changed since it ran: ${changes.join("; ")}. ${retryVerb(ctx.retry, runId)} only with the plan they ran with: put it back, or start a new run with reeval run.`);
|
|
1916
|
+
if (ctx.retryTarget && !setup.dryRun) {
|
|
1917
|
+
await prepareResume(ctx.retryTarget, config.root, { held: true });
|
|
1918
|
+
const aside = await loadPrevious(stagingDir);
|
|
1919
|
+
ctx.previous = aside.record ?? ctx.previous;
|
|
1920
|
+
ctx.previousArtifacts = aside.artifacts;
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
retried = continuing && !setup.dryRun ? noteRetries(planned, stagingDir, ctx.previous, true).errored : errored;
|
|
1924
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-047Dcd-a.js") : void 0;
|
|
1586
1925
|
const judge = runJudgeModel(config, profile);
|
|
1587
1926
|
const judgeModel = judge === void 0 ? null : judgeModelName(judge);
|
|
1588
1927
|
plan = await buildPlan({
|
|
@@ -1599,17 +1938,25 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1599
1938
|
});
|
|
1600
1939
|
plan.runId = runId;
|
|
1601
1940
|
const cloud = ciRunner(process.env);
|
|
1602
|
-
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.
|
|
1941
|
+
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
|
|
1603
1942
|
if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
|
|
1604
|
-
|
|
1943
|
+
plan.warnings.push(...started.notes);
|
|
1944
|
+
if (started.narrowed) {
|
|
1945
|
+
leftOut = leftOutAttempts(planned, stagingDir, ctx.previous, await liveAttempts(stagingDir), new Set(excluded.map((e) => e.id)));
|
|
1946
|
+
const notRun = leftOut.filter((l) => !l.done).length;
|
|
1947
|
+
if (notRun) plan.warnings.push(`${notRun === 1 ? "1 attempt" : `${notRun} attempts`} run ${runId} planned ${notRun === 1 ? "isn't" : "aren't"} in this resume's filters: ${notRun === 1 ? "it stays" : "they stay"} not run and the run stays incomplete. reeval run --resume ${runId} runs ${notRun === 1 ? "it" : "them"}`);
|
|
1948
|
+
}
|
|
1949
|
+
if (completed.size || retried.size) plan.resume = {
|
|
1605
1950
|
runId,
|
|
1606
|
-
completed: completed.size
|
|
1951
|
+
completed: completed.size,
|
|
1952
|
+
...retried.size ? { retried: retried.size } : {}
|
|
1607
1953
|
};
|
|
1608
1954
|
run.completed = [...completed];
|
|
1609
|
-
|
|
1610
|
-
|
|
1955
|
+
const done = leftOut.filter((l) => l.done).map((l) => l.key);
|
|
1956
|
+
kept = keptAttempts([...completed, ...done], stagingDir, ctx.previous);
|
|
1957
|
+
const { imageStages, imageTargets } = await import("./images-BNgcR77K.js");
|
|
1611
1958
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1612
|
-
id: p.id,
|
|
1959
|
+
id: p.attempt.family?.id ?? p.id,
|
|
1613
1960
|
harness: p.attempt.harness,
|
|
1614
1961
|
sandbox: p.attempt.sandbox,
|
|
1615
1962
|
plugins: p.attempt.pluginBuilds ?? []
|
|
@@ -1626,13 +1973,13 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1626
1973
|
return false;
|
|
1627
1974
|
}
|
|
1628
1975
|
const elsewhere = setup.configElsewhere ? configFile : void 0;
|
|
1629
|
-
const { scratchDir } = await import("./files-
|
|
1976
|
+
const { scratchDir } = await import("./files-DnlA1A7N.js");
|
|
1630
1977
|
const lines = [];
|
|
1631
1978
|
if (setup.dryRun) {
|
|
1632
1979
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
1633
1980
|
lines.push(...planTable(plan, todos));
|
|
1634
|
-
const { imageStatusLines } = await import("./images-
|
|
1635
|
-
const { dockerAvailable, engineSpec } = await import("./src-
|
|
1981
|
+
const { imageStatusLines } = await import("./images-BNgcR77K.js");
|
|
1982
|
+
const { dockerAvailable, engineSpec } = await import("./src-DtodMT59.js");
|
|
1636
1983
|
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
1637
1984
|
const down = [];
|
|
1638
1985
|
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
@@ -1641,7 +1988,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1641
1988
|
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
1642
1989
|
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
1643
1990
|
}
|
|
1644
|
-
const { costLine } = await import("./cost-
|
|
1991
|
+
const { costLine } = await import("./cost-Cr26Alhq.js");
|
|
1645
1992
|
const cost = plan.cost && costLine(plan.cost);
|
|
1646
1993
|
if (cost) lines.push(cost);
|
|
1647
1994
|
}
|
|
@@ -1699,6 +2046,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1699
2046
|
}
|
|
1700
2047
|
const missing = missingCredentials(config, process.env, {
|
|
1701
2048
|
harnesses: plan.harnessDescriptors,
|
|
2049
|
+
sandboxes: plan.sandboxes,
|
|
1702
2050
|
profile,
|
|
1703
2051
|
questionModels: [...questionModels],
|
|
1704
2052
|
judges: [...judges.values()]
|
|
@@ -1715,7 +2063,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1715
2063
|
});
|
|
1716
2064
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
1717
2065
|
if (asked.length) {
|
|
1718
|
-
const { questionRunner } = await import("./questions-
|
|
2066
|
+
const { questionRunner } = await import("./questions-CWlR9ffu.js");
|
|
1719
2067
|
const warned = /* @__PURE__ */ new Set();
|
|
1720
2068
|
questions = questionRunner({
|
|
1721
2069
|
evals: asked,
|
|
@@ -1757,7 +2105,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1757
2105
|
}
|
|
1758
2106
|
});
|
|
1759
2107
|
}
|
|
1760
|
-
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-
|
|
2108
|
+
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-CcQRT31C.js");
|
|
1761
2109
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
1762
2110
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
1763
2111
|
workspace: a.workspace,
|
|
@@ -1861,7 +2209,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1861
2209
|
runId,
|
|
1862
2210
|
startedAt: ctx.startedAt,
|
|
1863
2211
|
planned: toRun,
|
|
1864
|
-
kept
|
|
2212
|
+
kept,
|
|
2213
|
+
leftOut,
|
|
2214
|
+
...started.startedWith ? { invocation: started.startedWith } : {}
|
|
1865
2215
|
});
|
|
1866
2216
|
renderer.onEvent({
|
|
1867
2217
|
type: "run-start",
|
|
@@ -1933,9 +2283,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1933
2283
|
throw error;
|
|
1934
2284
|
} finally {
|
|
1935
2285
|
clearTimeout(budgetTimer);
|
|
1936
|
-
const { removePrepared } = await import("./preflight-
|
|
2286
|
+
const { removePrepared } = await import("./preflight-R-InZCQb.js");
|
|
1937
2287
|
removePrepared(config.root, runId);
|
|
1938
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
2288
|
+
const { removePreparedWorkspaces } = await import("./workspace-CcQRT31C.js");
|
|
1939
2289
|
removePreparedWorkspaces(config.root, runId);
|
|
1940
2290
|
}
|
|
1941
2291
|
if (preflightStop) throw preflightStop;
|
|
@@ -1949,7 +2299,10 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1949
2299
|
budgetReached: () => reached,
|
|
1950
2300
|
cutShort,
|
|
1951
2301
|
live,
|
|
1952
|
-
questions
|
|
2302
|
+
questions,
|
|
2303
|
+
leftOut,
|
|
2304
|
+
/** The attempts that ran again after an infra error, by key; none on a dry run. */
|
|
2305
|
+
retried: setup.dryRun ? /* @__PURE__ */ new Set() : retried
|
|
1953
2306
|
};
|
|
1954
2307
|
}
|
|
1955
2308
|
process.stdout.on("error", (error) => {
|
|
@@ -1983,7 +2336,7 @@ function finish(code, error, kind) {
|
|
|
1983
2336
|
} : null,
|
|
1984
2337
|
unhandledErrors: invocation.unhandledErrors.map((e) => redactSecrets(e))
|
|
1985
2338
|
});
|
|
1986
|
-
const broke = kind === "crash" || code === EXIT.infra || invocation.unhandledErrors.length > 0;
|
|
2339
|
+
const broke = kind === "crash" || code === EXIT.infra && !invocation.machineState || invocation.unhandledErrors.length > 0;
|
|
1987
2340
|
const quiet = !!process.env.REEVAL_NO_REPORT_HINT || command === "report" || !!flags?.json || flags?.ui === "agent" || process.env.REEVAL_UI === "agent" || !flags?.ui && !process.env.REEVAL_UI && !!detectAgent().name;
|
|
1988
2341
|
if (broke && !quiet) console.error(dim("hint: if this looks like a bug in reeval, reeval report collects a redacted report to file (REEVAL_NO_REPORT_HINT=1 hides this)"));
|
|
1989
2342
|
}
|
|
@@ -2017,4 +2370,4 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
2017
2370
|
});
|
|
2018
2371
|
export {};
|
|
2019
2372
|
|
|
2020
|
-
//# sourceMappingURL=cli-
|
|
2373
|
+
//# sourceMappingURL=cli-20Fpvs5H.js.map
|