reeval 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +841 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
- package/dist/adapter-Di9H2etg.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
- package/dist/agent-B-6GFDqA.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
- package/dist/agents-KBTYhSSh.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
- package/dist/app-files-Cmh9FGr5.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
- package/dist/assemble-DNVBTBp_.js.map +1 -0
- package/dist/attach-EbDGKGTO.js +152 -0
- package/dist/attach-EbDGKGTO.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
- package/dist/attempt-host-DQvynDHk.js.map +1 -0
- package/dist/boat-Bzz8L-KQ.js +701 -0
- package/dist/boat-Bzz8L-KQ.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
- package/dist/boat-Dek8ftIt.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/bundle-DZstHDeW.js +718 -0
- package/dist/bundle-DZstHDeW.js.map +1 -0
- package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
- package/dist/capture-BkbPsfej.js.map +1 -0
- package/dist/capture-CUAvk9KM.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
- package/dist/cli-BgBBuxxJ.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
- package/dist/cli-spec-B5L9n-DV.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-DMEYR3MA.js +807 -0
- package/dist/client-DMEYR3MA.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
- package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
- package/dist/command-D_w_kDpI.js.map +1 -0
- package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
- package/dist/command-Dwlh0Vhu.js.map +1 -0
- package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
- package/dist/commands-Cuf57OYb.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
- package/dist/config-C0RKt00M.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
- package/dist/db-3o8s2EyD.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
- package/dist/db-ErnJ-gMD.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
- package/dist/docker-DrMnpL84.js +1 -0
- package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
- package/dist/doctor-B7utCSdh.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
- package/dist/environment-C0Gg8kyN.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
- package/dist/eval-source-CsW5VIXS.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
- package/dist/evaluate-BHQyAszQ.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
- package/dist/fake-bootstrap-VpisF1WE.js +15 -0
- package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
- package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
- package/dist/files-CAzVG3Rr.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
- package/dist/files-D-Lir4aE.js +1 -0
- package/dist/files-DRlJ0Xr2.js +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
- package/dist/files-qjeqXnfE.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/harness-install-Dxho1yaU.js +53 -0
- package/dist/harness-install-Dxho1yaU.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
- package/dist/images-Ch1y1t6_.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
- package/dist/init-C2HqCwnt.js.map +1 -0
- package/dist/init-CM9xwyre.js +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
- package/dist/ink-list-BbKXElcQ.js.map +1 -0
- package/dist/inputs-5UIDpjz-.js +190 -0
- package/dist/inputs-5UIDpjz-.js.map +1 -0
- package/dist/inputs-load-C82HOKnb.js +1 -0
- package/dist/inputs-load-CGPyigMC.js +95 -0
- package/dist/inputs-load-CGPyigMC.js.map +1 -0
- package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
- package/dist/jiti-radm1Soi.js +52 -0
- package/dist/jiti-radm1Soi.js.map +1 -0
- package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
- package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
- package/dist/layout-Czh5wOpz.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
- package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
- package/dist/live-C9Et_BVT.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
- package/dist/live-DGgIrVw0.js.map +1 -0
- package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
- package/dist/machine-service-2lIv5UWF.js +1 -0
- package/dist/machine-service-Ct_1XMMi.js +520 -0
- package/dist/machine-service-Ct_1XMMi.js.map +1 -0
- package/dist/machines-BnquHcBZ.js +120 -0
- package/dist/machines-BnquHcBZ.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
- package/dist/model-Bcy4Qjmm.js +1 -0
- package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
- package/dist/model-DRa7yv6K.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
- package/dist/model-E5X8YBJM.js.map +1 -0
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
- package/dist/node-BjuhqQwa.js.map +1 -0
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
- package/dist/options-C989kuVE.js +1 -0
- package/dist/own-reads-CYmBlrHK.js +15 -0
- package/dist/own-reads-CYmBlrHK.js.map +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
- package/dist/plan-BcT4G5jj.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
- package/dist/plugins-cC0sqoDA.js.map +1 -0
- package/dist/preflight-BMVDBcux.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
- package/dist/preflight-DySHXeFJ.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
- package/dist/prepare-nbX1CXXg.js +409 -0
- package/dist/prepare-nbX1CXXg.js.map +1 -0
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/providers-BkLGzaSY.js +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
- package/dist/providers-bNgfKa6m.js.map +1 -0
- package/dist/query-B2SQddoV.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
- package/dist/record--vHreTLk.js.map +1 -0
- package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
- package/dist/redact-t-L3VGy8.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
- package/dist/remarks-CYFOsnz5.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
- package/dist/resolve-Bm9ZyFR-.js +1 -0
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
- package/dist/resolve-DorSHDBs.js.map +1 -0
- package/dist/resources-DqdwihhT.js +62 -0
- package/dist/resources-DqdwihhT.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
- package/dist/results-DhMgRkR0.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
- package/dist/resume-C8Ihfy-C.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/routes-Cp4gAjBZ.js +427 -0
- package/dist/routes-Cp4gAjBZ.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
- package/dist/runtime-CPUH0Vk5.js.map +1 -0
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
- package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
- package/dist/server-ZaMzUNmV.js +1 -0
- package/dist/services-BG8xuU82.js +39 -0
- package/dist/services-BG8xuU82.js.map +1 -0
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
- package/dist/source-BcBDj-R0.js +1 -0
- package/dist/src-B2eHTCnp.js +3 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
- package/dist/src-Bb3k9Kn_.js.map +1 -0
- package/dist/src-BfOlH_Bu.js +2 -0
- package/dist/src-BuGc40IF.js +2 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C2K5_afU.js +2 -0
- package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
- package/dist/src-C3OhA5pw.js.map +1 -0
- package/dist/src-CSVJjghq.js +4 -0
- package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
- package/dist/src-CWfy8ct8.js.map +1 -0
- package/dist/src-CcrQQB0_.js +2 -0
- package/dist/src-CotUxxQV.js +2 -0
- package/dist/src-D-PwBaBB.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
- package/dist/src-DNWiU__5.js.map +1 -0
- package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
- package/dist/src-DwcTn4JY.js.map +1 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
- package/dist/src-c3ZQu7Uu.js.map +1 -0
- package/dist/src-jYqNkT-F.js +1 -0
- package/dist/start-Db5VRcRZ.js +660 -0
- package/dist/start-Db5VRcRZ.js.map +1 -0
- package/dist/state-CKIjZZRK.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/suite-CH1aaAbs.js +2 -0
- package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
- package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
- package/dist/summary-BK5Sa_N0.js.map +1 -0
- package/dist/sync-DUi_P2UJ.js +559 -0
- package/dist/sync-DUi_P2UJ.js.map +1 -0
- package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
- package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
- package/dist/tart-BQ_Ap0l4.js +508 -0
- package/dist/tart-BQ_Ap0l4.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-D1RttYfM.js +148 -0
- package/dist/tart-bake-D1RttYfM.js.map +1 -0
- package/dist/tart-bake-DjFcsG3N.js +1 -0
- package/dist/tart-cli-BOcluha1.js +1 -0
- package/dist/tart-cli-gA9xEaQL.js +89 -0
- package/dist/tart-cli-gA9xEaQL.js.map +1 -0
- package/dist/tart-hosts-CHNrfV6u.js +313 -0
- package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
- package/dist/tart-hosts-CWse-ASo.js +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
- package/dist/tart-images-CLBjfF9u.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-krsmEo44.js +433 -0
- package/dist/tool-class-krsmEo44.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
- package/dist/ui-DKYjTVpn.js +1455 -0
- package/dist/ui-DKYjTVpn.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/vercel-models-Ci33llXz.js +21 -0
- package/dist/vercel-models-Ci33llXz.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +32 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
- package/dist/views-DCjtw1jF.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
- package/dist/worker-entry-DrAYYt0M.js.map +1 -0
- package/dist/workspace-CnRbV8cM.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
- package/docker/Dockerfile +1 -1
- package/package.json +15 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-BjLqFJIZ.js.map +0 -1
- package/dist/boat-DSNgZt-t.js +0 -322
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/capture-Ccf0Hmyx.js.map +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/command-Ff0T-1Hi.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker--mukta3L.js +0 -1
- package/dist/docker-DG_uFgn2.js +0 -246
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/ink-list-BpfidQ1h.js.map +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CgYbO4Te.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/model-cgt5uvu8.js.map +0 -1
- package/dist/node-D7pAU--0.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prepare-DQEDXoDp.js +0 -268
- package/dist/prepare-DQEDXoDp.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/record-6IUzwP9K.js.map +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/runtime-Xm9oeC7o.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/summary-DDy9DfyJ.js.map +0 -1
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js +0 -586
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js +0 -344
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -1,34 +1,34 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import { g as parseDuration } from "./shared-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import { y as vertexSettingsIn } from "./vertex-
|
|
9
|
-
import {
|
|
10
|
-
import { l as
|
|
2
|
+
import { g as parseDuration } from "./shared-DtZwIAfy.js";
|
|
3
|
+
import { o as PACKAGE_VERSION } from "./root-BzeoyfGd.js";
|
|
4
|
+
import { a as newRunId, c as relativeFile, i as fromCwd } from "./ids-B-JKTCP8.js";
|
|
5
|
+
import { S as stripCredentials, g as redactSecrets, x as registerSecret } from "./redact-t-L3VGy8.js";
|
|
6
|
+
import { H as sandboxCapacity, I as capabilitiesOf, L as cleanupRun, W as sandboxKind } from "./plugins-cC0sqoDA.js";
|
|
7
|
+
import { n as clip } from "./text-B8-W3Cvn.js";
|
|
8
|
+
import { y as vertexSettingsIn } from "./vertex-BEWxHoxH.js";
|
|
9
|
+
import { D as profileSandbox, F as workspaceLabel, d as environmentSandbox } from "./environment-C0Gg8kyN.js";
|
|
10
|
+
import { At as resolveOutput, Dt as rootlessPodmanSwitch, Ot as setupEgress, R as checkLimitHarness, S as resolveModelGateway, U as harnessId, _ as selectProfile, _t as ciRunner, a as configPathFor, b as gatewayScope, c as findConfig, d as loadConfig, et as plannedSandboxIds, f as malformedCredentials, g as runJudgeModel, h as requireConfig, i as configLabel, l as givenConcurrency, m as neededCredentials, o as defaultConcurrency, p as missingCredentials, tt as plannedSandboxes, u as hostCapacity, v as withProfileHistory, vt as directOnCloud, xt as isDirect } from "./config-C0RKt00M.js";
|
|
11
|
+
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-DwzqucLS.js";
|
|
11
12
|
import { n as loadEnvFiles, t as inCI } from "./env-8x1O8I1M.js";
|
|
12
|
-
import {
|
|
13
|
-
import { r as openRouterFallbacks } from "./openrouter-models-
|
|
14
|
-
import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-
|
|
15
|
-
import { k as runOutcome } from "./model-
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-
|
|
13
|
+
import { r as localStudio } from "./resolve-Bul8lHaa.js";
|
|
14
|
+
import { r as openRouterFallbacks } from "./openrouter-models-B1sIOTD2.js";
|
|
15
|
+
import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-CCD8XsmI.js";
|
|
16
|
+
import { k as runOutcome } from "./model-DF77SgZv.js";
|
|
17
|
+
import { c as helpText, i as INSTEAD, n as FLAGS, o as commandHelpText, r as GLOBAL, t as COMMANDS } from "./cli-spec-B5L9n-DV.js";
|
|
18
|
+
import { t as colors } from "./colors-DcXBvAlU.js";
|
|
19
|
+
import { d as storeCredentials, i as isRemote, u as pullStore } from "./sync-DUi_P2UJ.js";
|
|
20
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-CkykifD9.js";
|
|
21
|
+
import { A as writeLiveProgress, C as restoreRetry, D as stoppedRetry, E as savedInvocation, O as sweepStaging, S as resolveRetry, T as savedFilters, _ as pruneEvents, a as eventsFile, b as resolveAttach, d as liveAttempts, f as loadPrevious, h as previousAttempt, i as erroredAttempts, k as writeCurrent, l as keptAttempts, m as prepareResume, o as holdRetry, p as noteRetries, r as completedKeysFor, t as CANCEL_SIGNAL, u as leftOutAttempts, w as resumeSettings, x as resolveResume, y as removeStoppedStaging } from "./resume-C8Ihfy-C.js";
|
|
22
|
+
import { c as isEventsUrl, d as teeRenderer, l as notEventStream, s as isAttachUrl, t as createRenderer, u as openEventStream } from "./ui-DKYjTVpn.js";
|
|
23
|
+
import { n as preflight } from "./preflight-DySHXeFJ.js";
|
|
20
24
|
import { t as budgetMarker } from "./budget-BcxLYY5L.js";
|
|
21
|
-
import {
|
|
22
|
-
import { n as preflight } from "./preflight-GkpgrBxP.js";
|
|
23
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CxGNOlh0.js";
|
|
24
|
-
import { _ as phaseOf, c as pointerLine, d as attemptAxes, f as debugLines, g as outcomeOf, h as offset, i as listLines, m as failureLines, n as SPINNER, p as errorCheck, s as progressText, t as LiveList, u as statusList, v as progressLine, y as tailLines } from "./list-DuThsE28.js";
|
|
25
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-BrZAMuTb.js";
|
|
25
26
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
26
|
-
import { dirname, join } from "node:path";
|
|
27
|
-
import { inspect } from "node:util";
|
|
27
|
+
import { dirname, join, resolve } from "node:path";
|
|
28
28
|
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
29
|
+
import { inspect } from "node:util";
|
|
29
30
|
import { detectAgent } from "std-env";
|
|
30
31
|
import { Command, Option } from "commander";
|
|
31
|
-
import stringWidth from "string-width";
|
|
32
32
|
// src/cli-parse.ts
|
|
33
33
|
const OPTIONS = Object.fromEntries(Object.entries(FLAGS).map(([name, spec]) => {
|
|
34
34
|
const short = "short" in spec ? `-${spec.short}, ` : "";
|
|
@@ -64,9 +64,17 @@ function parseCli(argv) {
|
|
|
64
64
|
for (const [subName, sub] of Object.entries(spec.sub ?? {})) {
|
|
65
65
|
const subcommand = configure(command.command(subName), `reeval ${name} ${subName}`, sub.flags);
|
|
66
66
|
subcommand.argument("[args...]").action(() => {
|
|
67
|
-
const
|
|
67
|
+
const own = valuesOf(command, spec.flags);
|
|
68
|
+
const stray = Object.keys(own).find((flag) => !sub.flags.includes(flag));
|
|
68
69
|
if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}. Run reeval --help.`);
|
|
69
|
-
|
|
70
|
+
const result = done(program, subcommand, [name, subName], sub.flags, sub.positionals);
|
|
71
|
+
parsed = {
|
|
72
|
+
...result,
|
|
73
|
+
values: {
|
|
74
|
+
...own,
|
|
75
|
+
...result.values
|
|
76
|
+
}
|
|
77
|
+
};
|
|
70
78
|
});
|
|
71
79
|
}
|
|
72
80
|
}
|
|
@@ -148,7 +156,10 @@ function usageError(error, command) {
|
|
|
148
156
|
const known = arg.startsWith("--") ? FLAGS[flag] : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[1];
|
|
149
157
|
const long = arg.startsWith("--") ? flag : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[0];
|
|
150
158
|
if (known && value !== void 0 && known.type === "boolean") return new UsageError(`--${flag} doesn't take a value: pass --${flag} on its own.`);
|
|
151
|
-
if (known)
|
|
159
|
+
if (known) {
|
|
160
|
+
const instead = INSTEAD[`${command} --${long}`];
|
|
161
|
+
return new UsageError(`--${long} isn't an option of ${command}. ${instead ? `${instead}.` : "Run reeval --help."}`);
|
|
162
|
+
}
|
|
152
163
|
const shown = arg.replace(/\r/g, "\\r").replace(/\n/g, "\\n");
|
|
153
164
|
return new UsageError(`Unknown option '${shown}'. ${suggestion ? `${suggestion} ` : ""}Run reeval --help.`);
|
|
154
165
|
}
|
|
@@ -159,927 +170,6 @@ function usageError(error, command) {
|
|
|
159
170
|
}
|
|
160
171
|
return new UsageError(first.charAt(0).toUpperCase() + first.slice(1));
|
|
161
172
|
}
|
|
162
|
-
// src/ui/header.ts
|
|
163
|
-
/** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
|
|
164
|
-
const GLYPH = 5;
|
|
165
|
-
/** `Est. Time: `, the longest key. */
|
|
166
|
-
const KEY = 11;
|
|
167
|
-
/** `Warning: `, the longer note label. */
|
|
168
|
-
const LABEL = 9;
|
|
169
|
-
/**
|
|
170
|
-
* `text` broken at spaces into lines of at most `width` columns, not counting color codes; a
|
|
171
|
-
* longer word keeps a line. A `·` stays with the word before it, so no line starts with one.
|
|
172
|
-
*/
|
|
173
|
-
function wrap(text, width) {
|
|
174
|
-
const words = [];
|
|
175
|
-
for (const word of text.split(/\s+/).filter(Boolean)) if (word === "·" && words.length) words[words.length - 1] += " ·";
|
|
176
|
-
else words.push(word);
|
|
177
|
-
const lines = [];
|
|
178
|
-
let line = "";
|
|
179
|
-
for (const word of words) if (line && stripAnsi(line).length + 1 + stripAnsi(word).length > width) {
|
|
180
|
-
lines.push(line);
|
|
181
|
-
line = word;
|
|
182
|
-
} else line = line ? `${line} ${word}` : word;
|
|
183
|
-
return line ? [...lines, line] : lines;
|
|
184
|
-
}
|
|
185
|
-
const indent = (n) => " ".repeat(n);
|
|
186
|
-
/** `REEVAL v0.16.1`, then the run's facts, one `➜ Key: value` line each. */
|
|
187
|
-
function headerLines(c, h, width) {
|
|
188
|
-
const name = `REEVAL v${h.version}`;
|
|
189
|
-
const title = c.bold(c.depth === 4 ? name : c.teal(name));
|
|
190
|
-
const valueWidth = width - GLYPH - KEY;
|
|
191
|
-
const row = (key, value) => {
|
|
192
|
-
const values = typeof value === "string" ? [value] : value;
|
|
193
|
-
const joined = values.join(", ");
|
|
194
|
-
return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
|
|
195
|
-
};
|
|
196
|
-
const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
|
|
197
|
-
const continued = [h.resumed ? `${plural(h.resumed, "attempt")} done` : "", h.retried ? `${plural(h.retried, "error")} to retry` : ""].filter(Boolean);
|
|
198
|
-
const resumed = continued.length ? c.dim(` (resumed, ${continued.join(", ")})`) : "";
|
|
199
|
-
const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
|
|
200
|
-
return [
|
|
201
|
-
h.dryRun ? `${title} ${c.dim("dry run")}` : title,
|
|
202
|
-
"",
|
|
203
|
-
...row("Run", `${h.runId}${resumed}`),
|
|
204
|
-
...h.config ? row("Config", h.config) : [],
|
|
205
|
-
...h.profile ? row("Profile", `${h.profile.name}${reason}`) : [],
|
|
206
|
-
...row(many("Harness", h.harnesses.length, "Harnesses"), h.harnesses.map((id) => harness(c, h, id))),
|
|
207
|
-
...row(many("Sandbox", h.sandboxes.length, "Sandboxes"), h.sandboxes),
|
|
208
|
-
...h.judges.length ? row(many("Judge", h.judges.length), h.judges) : [],
|
|
209
|
-
...row("Scope", scope(c, h)),
|
|
210
|
-
...row("Est. Time", estimate(c, h)),
|
|
211
|
-
...h.record && !h.dryRun ? row("Record", h.record) : []
|
|
212
|
-
];
|
|
213
|
-
}
|
|
214
|
-
/** A harness id, with the effort it runs at when the id doesn't name it. */
|
|
215
|
-
function harness(c, h, id) {
|
|
216
|
-
const effort = h.efforts?.[id];
|
|
217
|
-
if (effort === void 0 || id.includes("[effort=")) return id;
|
|
218
|
-
return `${id} ${`(effort: ${effort})`.split(" ").map((w) => c.dim(w)).join(" ")}`;
|
|
219
|
-
}
|
|
220
|
-
function scope(c, h) {
|
|
221
|
-
const attempts = h.fewest < h.attempts ? `${h.fewest}–${h.attempts} attempts` : plural(h.attempts, "attempt");
|
|
222
|
-
const parts = [
|
|
223
|
-
plural(h.files, "eval file"),
|
|
224
|
-
plural(h.evals, "eval") + (h.skipped ? ` (${h.skipped} skipped)` : ""),
|
|
225
|
-
plural(h.cells, "cell"),
|
|
226
|
-
attempts
|
|
227
|
-
];
|
|
228
|
-
const caps = [h.budget.attempts !== void 0 ? `${h.budget.attempts} ${h.attempts <= h.budget.attempts ? c.green("✓") : c.red("✗")}` : "", h.budget.duration ?? ""].filter(Boolean);
|
|
229
|
-
return `${parts.join(" · ")}${caps.length ? ` (budget: ${caps.join(", ")})` : ""}`;
|
|
230
|
-
}
|
|
231
|
-
function estimate(c, h) {
|
|
232
|
-
const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
|
|
233
|
-
const most = Math.min(h.concurrency, h.attempts);
|
|
234
|
-
const parallel = h.parallel ?? most;
|
|
235
|
-
if (parallel <= 1) return serial;
|
|
236
|
-
const capped = parallel < most ? `, ${parallel} at once` : "";
|
|
237
|
-
return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency}${capped})`)}`} · ${serial}`;
|
|
238
|
-
}
|
|
239
|
-
/** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
|
|
240
|
-
function noteLines(c, kind, text, width) {
|
|
241
|
-
const [glyph, label, paint] = kind === "hint" ? [
|
|
242
|
-
"ℹ",
|
|
243
|
-
"Hint:",
|
|
244
|
-
c.blue
|
|
245
|
-
] : [
|
|
246
|
-
"⚠",
|
|
247
|
-
"Warning:",
|
|
248
|
-
kind === "root" ? c.red : c.orange
|
|
249
|
-
];
|
|
250
|
-
const basic = {
|
|
251
|
-
warning: c.yellow,
|
|
252
|
-
hint: c.teal,
|
|
253
|
-
root: c.red
|
|
254
|
-
}[kind];
|
|
255
|
-
const lead = c.depth === 4 && kind !== "root" ? `${basic(glyph)} ${c.bold(label)}` : c.bold(paint(`${glyph} ${label}`));
|
|
256
|
-
const at = 14;
|
|
257
|
-
return wrap(text, Math.max(20, width - at)).map((line, i) => i === 0 ? ` ${lead}${indent(LABEL - label.length)}${line}` : `${indent(at)}${line}`);
|
|
258
|
-
}
|
|
259
|
-
/** A step line, `◆ Preparing 1 sandbox (docker)...`, its text wrapped under itself. */
|
|
260
|
-
function stepLines(glyph, text, width) {
|
|
261
|
-
return wrap(text, Math.max(20, width - GLYPH)).map((line, i) => i === 0 ? ` ${glyph} ${line}` : `${indent(GLYPH)}${line}`);
|
|
262
|
-
}
|
|
263
|
-
/** A run-level notice (a prepared workspace, the Ctrl-C one) as a step line. */
|
|
264
|
-
function noticeLines(c, e, width) {
|
|
265
|
-
const glyph = {
|
|
266
|
-
info: c.dim("ℹ"),
|
|
267
|
-
warn: c.orange("⚠"),
|
|
268
|
-
error: c.red("✗")
|
|
269
|
-
}[e.level];
|
|
270
|
-
const lines = stepLines(glyph, e.text, width);
|
|
271
|
-
return e.level === "info" ? lines.map((l) => c.dim(l)) : lines;
|
|
272
|
-
}
|
|
273
|
-
/** Preflight: which sandboxes it prepares, each one ready, and each failure. */
|
|
274
|
-
function prepareStepLines(c, e, width, max) {
|
|
275
|
-
if (e.state === "start") {
|
|
276
|
-
const n = e.targets.length;
|
|
277
|
-
const names = e.targets.map((t) => t.label).join(", ");
|
|
278
|
-
return stepLines(c.teal("◆"), `Preparing ${plural(n, "sandbox", "sandboxes")} (${names})...`, width);
|
|
279
|
-
}
|
|
280
|
-
const { outcome } = e;
|
|
281
|
-
if (outcome.ok) {
|
|
282
|
-
const took = outcome.ms >= 1e3 ? ` ${c.dim(formatDuration(outcome.ms))}` : "";
|
|
283
|
-
return [` ${c.green("✓")} ${outcome.target.label} ready${took}`];
|
|
284
|
-
}
|
|
285
|
-
const who = outcome.target.setBy.join(", ");
|
|
286
|
-
return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
|
|
287
|
-
}
|
|
288
|
-
/** The start line's text: how the attempts' logs will print. */
|
|
289
|
-
function startText(output) {
|
|
290
|
-
return output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
|
|
291
|
-
}
|
|
292
|
-
/** The last line before the attempts' own: how their logs will print. */
|
|
293
|
-
function startLine(c, output) {
|
|
294
|
-
return ` ${c.teal("◆")} ${startText(output)}`;
|
|
295
|
-
}
|
|
296
|
-
/** What the start line's text becomes once the run ends, on a terminal that redraws it. */
|
|
297
|
-
function endText(status) {
|
|
298
|
-
return status === "interrupted" ? "Canceled." : status === "budget" ? "Stopped at the budget." : "Finished.";
|
|
299
|
-
}
|
|
300
|
-
/** The plan as the header block: its facts, `--dry-run`'s table, then each kind of note. */
|
|
301
|
-
function planLines(c, e, width) {
|
|
302
|
-
const sections = [
|
|
303
|
-
e.details,
|
|
304
|
-
[...e.warnings.flatMap((w) => noteLines(c, "warning", w, width)), ...(e.sandboxWarnings ?? []).flatMap((w) => noteLines(c, w.kind === "root" ? "root" : "warning", w.message, width))],
|
|
305
|
-
e.hints.flatMap((h) => noteLines(c, "hint", h, width))
|
|
306
|
-
];
|
|
307
|
-
const out = headerLines(c, e.header, width);
|
|
308
|
-
for (const lines of sections) if (lines.length) out.push("", ...lines);
|
|
309
|
-
out.push("");
|
|
310
|
-
return out;
|
|
311
|
-
}
|
|
312
|
-
// src/ui/tags.ts
|
|
313
|
-
const STOPWORDS = new Set("a an the and or of to in on for with without from by at as is are be it its that this when than then into via not no".split(" "));
|
|
314
|
-
const MAX_WORDS = 3;
|
|
315
|
-
const MAX_LENGTH = 24;
|
|
316
|
-
const words = (name) => name.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((w) => w && !STOPWORDS.has(w));
|
|
317
|
-
/**
|
|
318
|
-
* Short handles for the evals of one file, e.g. `greet`: the first distinctive words of each
|
|
319
|
-
* name, where words shared by most of the file's evals ("recommends Genkit over …") don't
|
|
320
|
-
* count. Depends only on the file's evals, so a tag stays the same across runs and filters.
|
|
321
|
-
*/
|
|
322
|
-
function fileTags(evals, taken) {
|
|
323
|
-
const lists = evals.map((e) => words(e.name));
|
|
324
|
-
const df = /* @__PURE__ */ new Map();
|
|
325
|
-
for (const list of lists) for (const w of new Set(list)) df.set(w, (df.get(w) ?? 0) + 1);
|
|
326
|
-
const common = (w) => evals.length > 1 && (df.get(w) ?? 0) > evals.length / 2;
|
|
327
|
-
const candidates = lists.map((list) => {
|
|
328
|
-
const rare = list.filter((w) => !common(w));
|
|
329
|
-
return rare.length ? rare : list.length ? list : ["eval"];
|
|
330
|
-
});
|
|
331
|
-
const tags = /* @__PURE__ */ new Map();
|
|
332
|
-
const clip = (tag) => tag.slice(0, MAX_LENGTH).replace(/-+$/, "");
|
|
333
|
-
const prefix = (list, n) => clip(list.slice(0, n).join("-"));
|
|
334
|
-
evals.forEach((e, i) => {
|
|
335
|
-
const own = candidates[i] ?? ["eval"];
|
|
336
|
-
let tag = prefix(own, 1);
|
|
337
|
-
for (let n = 1; n <= MAX_WORDS; n++) {
|
|
338
|
-
tag = prefix(own, n);
|
|
339
|
-
if (!candidates.some((other, j) => j !== i && prefix(other, n) === tag) && !taken.has(tag)) break;
|
|
340
|
-
}
|
|
341
|
-
let unique = tag;
|
|
342
|
-
for (let k = 2; taken.has(unique); k++) unique = `${tag}-${k}`;
|
|
343
|
-
taken.add(unique);
|
|
344
|
-
tags.set(e.id, unique);
|
|
345
|
-
});
|
|
346
|
-
return tags;
|
|
347
|
-
}
|
|
348
|
-
/** Tags evals as their files are collected; later files never rename earlier tags. */
|
|
349
|
-
var Tagger = class {
|
|
350
|
-
#tags = /* @__PURE__ */ new Map();
|
|
351
|
-
#taken = /* @__PURE__ */ new Set();
|
|
352
|
-
addFile(attempts) {
|
|
353
|
-
const fresh = [...new Map(attempts.map((a) => [a.id, a])).values()].filter((e) => !this.#tags.has(e.id));
|
|
354
|
-
for (const [id, tag] of fileTags(fresh, this.#taken)) this.#tags.set(id, tag);
|
|
355
|
-
}
|
|
356
|
-
/** `greet#1`: the eval's tag and run number. */
|
|
357
|
-
label(id, attempt) {
|
|
358
|
-
let tag = this.#tags.get(id);
|
|
359
|
-
if (!tag) {
|
|
360
|
-
this.addFile([{
|
|
361
|
-
id,
|
|
362
|
-
name: id.split(" › ").at(-1) ?? id
|
|
363
|
-
}]);
|
|
364
|
-
tag = this.#tags.get(id) ?? "eval";
|
|
365
|
-
}
|
|
366
|
-
return `${tag}#${attempt}`;
|
|
367
|
-
}
|
|
368
|
-
};
|
|
369
|
-
// src/ui/agent.ts
|
|
370
|
-
const MAX_LINES = 12;
|
|
371
|
-
/** Agents read whole lines, so the header's notes don't wrap. */
|
|
372
|
-
const WIDTH = Number.POSITIVE_INFINITY;
|
|
373
|
-
var AgentRenderer = class {
|
|
374
|
-
#o;
|
|
375
|
-
#c = colors(false);
|
|
376
|
-
/** `[key]` prefixes for `-v` lines, from each attempt's start. */
|
|
377
|
-
#labels = /* @__PURE__ */ new Map();
|
|
378
|
-
#tags = new Tagger();
|
|
379
|
-
constructor(options) {
|
|
380
|
-
this.#o = options;
|
|
381
|
-
}
|
|
382
|
-
#write(text) {
|
|
383
|
-
this.#o.out.write(`${text.trimEnd()}\n`);
|
|
384
|
-
}
|
|
385
|
-
onEvent(e) {
|
|
386
|
-
switch (e.type) {
|
|
387
|
-
case "plan":
|
|
388
|
-
for (const line of planLines(this.#c, e, WIDTH)) this.#write(line);
|
|
389
|
-
return;
|
|
390
|
-
case "run-start":
|
|
391
|
-
this.#write(startLine(this.#c, {
|
|
392
|
-
...this.#o.output,
|
|
393
|
-
outputLogs: "none"
|
|
394
|
-
}));
|
|
395
|
-
return;
|
|
396
|
-
case "collected":
|
|
397
|
-
this.#tags.addFile(e.evals);
|
|
398
|
-
return;
|
|
399
|
-
case "eval-start": {
|
|
400
|
-
const tag = this.#tags.label(e.id, e.attempt?.n ?? 1);
|
|
401
|
-
this.#labels.set(e.key, `[${tag} ${attemptAxes(e.attempt, "agent")}]`);
|
|
402
|
-
return;
|
|
403
|
-
}
|
|
404
|
-
case "debug": {
|
|
405
|
-
const label = this.#labels.get(e.key) ?? `[${e.key}]`;
|
|
406
|
-
for (const line of debugLines(this.#c, e.line, this.#o.verbosity === 2)) this.#write(`${label} ${line}`);
|
|
407
|
-
return;
|
|
408
|
-
}
|
|
409
|
-
case "eval-progress": {
|
|
410
|
-
if (e.progress.kind !== "browser-download" && e.progress.kind !== "warning") return;
|
|
411
|
-
const label = this.#labels.get(e.key) ?? `[${e.key}]`;
|
|
412
|
-
this.#write(`${label} ${progressLine(this.#c, e.progress)}`);
|
|
413
|
-
return;
|
|
414
|
-
}
|
|
415
|
-
case "prepare":
|
|
416
|
-
for (const line of prepareStepLines(this.#c, e, WIDTH, MAX_LINES)) this.#write(line);
|
|
417
|
-
return;
|
|
418
|
-
case "notice":
|
|
419
|
-
for (const line of noticeLines(this.#c, e, WIDTH)) this.#write(line);
|
|
420
|
-
return;
|
|
421
|
-
case "run-end": {
|
|
422
|
-
this.#write("");
|
|
423
|
-
const verbose = (this.#o.verbosity ?? 0) > 0;
|
|
424
|
-
const list = statusList({
|
|
425
|
-
record: e.record,
|
|
426
|
-
suite: suiteResult(e.record),
|
|
427
|
-
remarks: e.remarks ?? null,
|
|
428
|
-
verbose,
|
|
429
|
-
...e.notRun ? { notRun: e.notRun } : {}
|
|
430
|
-
});
|
|
431
|
-
const pointer = pointerLine(e.summaryPath);
|
|
432
|
-
for (const line of listLines(list, this.#c, false, pointer, verbose, true)) this.#write(line);
|
|
433
|
-
return;
|
|
434
|
-
}
|
|
435
|
-
default: return;
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
close() {}
|
|
439
|
-
};
|
|
440
|
-
// src/ui/plain.ts
|
|
441
|
-
/** In a grouped block, an end line replaces its start line and keeps its start time. */
|
|
442
|
-
const STARTS = {
|
|
443
|
-
"sandbox-ready": "sandbox",
|
|
444
|
-
"turn-end": "turn-start",
|
|
445
|
-
"check-end": "check-start",
|
|
446
|
-
"judge-end": "judge-start",
|
|
447
|
-
"browser-download": "browser-download"
|
|
448
|
-
};
|
|
449
|
-
const ERROR_LINES = 12;
|
|
450
|
-
/** Back to the start of the line and clear it, for the live status line. */
|
|
451
|
-
const CLEAR = "\r\x1B[2K";
|
|
452
|
-
/** GitHub Actions workflow-command escaping. */
|
|
453
|
-
const ghEscape = (s, property = false) => {
|
|
454
|
-
const out = s.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
|
|
455
|
-
return property ? out.replace(/:/g, "%3A").replace(/,/g, "%2C") : out;
|
|
456
|
-
};
|
|
457
|
-
/** The eval file's line in Vitest's formatted error (`❯ evals/a.eval.ts:7:33`), if any. */
|
|
458
|
-
function locate(result) {
|
|
459
|
-
const text = stripAnsi(result.formattedErrors[0] ?? "");
|
|
460
|
-
const file = result.file.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
461
|
-
const match = new RegExp(`❯ ${file}:(\\d+):(\\d+)`).exec(text);
|
|
462
|
-
return match?.[1] && match[2] ? {
|
|
463
|
-
line: match[1],
|
|
464
|
-
col: match[2]
|
|
465
|
-
} : void 0;
|
|
466
|
-
}
|
|
467
|
-
/** `evals/a.eval.ts:7` as its file and line. */
|
|
468
|
-
const splitAt = (at) => {
|
|
469
|
-
const i = at.lastIndexOf(":");
|
|
470
|
-
return [at.slice(0, i), at.slice(i + 1)];
|
|
471
|
-
};
|
|
472
|
-
var PlainRenderer = class {
|
|
473
|
-
#o;
|
|
474
|
-
#c;
|
|
475
|
-
#tags = new Tagger();
|
|
476
|
-
#running = /* @__PURE__ */ new Map();
|
|
477
|
-
#now;
|
|
478
|
-
#queued = /* @__PURE__ */ new Set();
|
|
479
|
-
#runStartedAt = 0;
|
|
480
|
-
#lastWrite = 0;
|
|
481
|
-
/** The last line written was blank, so the list needs no gap above it. */
|
|
482
|
-
#blank = false;
|
|
483
|
-
#timer;
|
|
484
|
-
/** On a TTY: the status line is drawn under everything else, and redrawn after each write. */
|
|
485
|
-
#live;
|
|
486
|
-
#status = "";
|
|
487
|
-
/** Evals and tries collected (not skipped), and the tries that ended. */
|
|
488
|
-
#evals = /* @__PURE__ */ new Set();
|
|
489
|
-
#tries = /* @__PURE__ */ new Set();
|
|
490
|
-
#ended = /* @__PURE__ */ new Set();
|
|
491
|
-
#list;
|
|
492
|
-
#lastProgress = 0;
|
|
493
|
-
constructor(options) {
|
|
494
|
-
this.#o = options;
|
|
495
|
-
this.#list = new LiveList(options.harness);
|
|
496
|
-
this.#c = options.colors;
|
|
497
|
-
this.#now = options.now ?? Date.now;
|
|
498
|
-
this.#live = options.output.statusLine;
|
|
499
|
-
const { heartbeatMs } = options.output;
|
|
500
|
-
if (options.timers !== false && heartbeatMs > 0) {
|
|
501
|
-
this.#timer = setInterval(() => this.tick(), Math.min(heartbeatMs, 5e3));
|
|
502
|
-
this.#timer.unref();
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
#write(text) {
|
|
506
|
-
const line = `${text.trimEnd()}\n`;
|
|
507
|
-
this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
|
|
508
|
-
this.#lastWrite = this.#now();
|
|
509
|
-
this.#blank = line === "\n";
|
|
510
|
-
}
|
|
511
|
-
/** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
|
|
512
|
-
#drawStatus() {
|
|
513
|
-
if (!this.#live || !this.#tries.size) return;
|
|
514
|
-
const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
|
|
515
|
-
const status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
|
|
516
|
-
if (status === this.#status) return;
|
|
517
|
-
this.#o.out.write(`${CLEAR}${status}`);
|
|
518
|
-
this.#status = status;
|
|
519
|
-
}
|
|
520
|
-
#clearStatus() {
|
|
521
|
-
if (!this.#status) return;
|
|
522
|
-
this.#o.out.write(CLEAR);
|
|
523
|
-
this.#status = "";
|
|
524
|
-
}
|
|
525
|
-
#attempt(ref, startedAt) {
|
|
526
|
-
const n = ref.attempt?.n ?? 1;
|
|
527
|
-
const label = `[${this.#tags.label(ref.id, n)} ${attemptAxes(ref.attempt, this.#o.harness)}]`;
|
|
528
|
-
const ungated = ref.attempt?.gate === "none";
|
|
529
|
-
return {
|
|
530
|
-
key: ref.key,
|
|
531
|
-
id: ref.id,
|
|
532
|
-
label,
|
|
533
|
-
startedAt,
|
|
534
|
-
lines: [],
|
|
535
|
-
last: void 0,
|
|
536
|
-
ungated
|
|
537
|
-
};
|
|
538
|
-
}
|
|
539
|
-
#prefix(a) {
|
|
540
|
-
return this.#c.tag(a.label)(a.label);
|
|
541
|
-
}
|
|
542
|
-
#detail(a, line, prefixed) {
|
|
543
|
-
const body = line.debug ? ` ${line.text}` : ` ${this.#c.dim(offset(line.at - a.startedAt))} ${line.text}`;
|
|
544
|
-
return prefixed ? `${this.#prefix(a)}${body}` : body.slice(1);
|
|
545
|
-
}
|
|
546
|
-
get #streaming() {
|
|
547
|
-
return this.#o.output.logOrder === "stream" && this.#o.output.outputLogs === "full";
|
|
548
|
-
}
|
|
549
|
-
#add(a, line) {
|
|
550
|
-
if (line.kind === "warning" && !this.#streaming) {
|
|
551
|
-
this.#write(this.#detail(a, line, true));
|
|
552
|
-
return;
|
|
553
|
-
}
|
|
554
|
-
a.lines.push(line);
|
|
555
|
-
if (this.#streaming) this.#write(this.#detail(a, line, true));
|
|
556
|
-
}
|
|
557
|
-
onEvent(e) {
|
|
558
|
-
const c = this.#c;
|
|
559
|
-
switch (e.type) {
|
|
560
|
-
case "run-start":
|
|
561
|
-
this.#runStartedAt = e.at;
|
|
562
|
-
this.#write(startLine(c, this.#o.output));
|
|
563
|
-
this.#write("");
|
|
564
|
-
return;
|
|
565
|
-
case "plan":
|
|
566
|
-
for (const line of planLines(c, e, this.#columns)) this.#write(line);
|
|
567
|
-
if (this.#o.output.githubActions) for (const w of e.sandboxWarnings ?? []) this.#annotate(w);
|
|
568
|
-
return;
|
|
569
|
-
case "collected":
|
|
570
|
-
this.#list.collected(e.file, e.evals);
|
|
571
|
-
this.#tags.addFile(e.evals);
|
|
572
|
-
for (const x of e.evals) {
|
|
573
|
-
if (x.skipped) continue;
|
|
574
|
-
this.#queued.add(x.key);
|
|
575
|
-
this.#evals.add(x.id);
|
|
576
|
-
this.#tries.add(x.key);
|
|
577
|
-
}
|
|
578
|
-
this.#drawStatus();
|
|
579
|
-
return;
|
|
580
|
-
case "eval-start": {
|
|
581
|
-
const a = this.#attempt(e, e.at);
|
|
582
|
-
this.#list.start(e);
|
|
583
|
-
a.last = e;
|
|
584
|
-
this.#queued.delete(e.key);
|
|
585
|
-
this.#running.set(e.key, a);
|
|
586
|
-
if (this.#streaming) this.#write(`${this.#prefix(a)} ${c.dim("▶")} ${e.id}`);
|
|
587
|
-
return;
|
|
588
|
-
}
|
|
589
|
-
case "eval-progress": {
|
|
590
|
-
const a = this.#running.get(e.key);
|
|
591
|
-
if (!a) return;
|
|
592
|
-
this.#list.progress(e.key, e.progress);
|
|
593
|
-
if (e.progress.kind !== "warning") a.last = e;
|
|
594
|
-
const { progress } = e;
|
|
595
|
-
const paint = a.ungated ? {
|
|
596
|
-
...c,
|
|
597
|
-
red: c.dim
|
|
598
|
-
} : c;
|
|
599
|
-
const line = {
|
|
600
|
-
at: progress.at,
|
|
601
|
-
text: progressLine(paint, progress),
|
|
602
|
-
kind: progress.kind
|
|
603
|
-
};
|
|
604
|
-
const start = STARTS[progress.kind];
|
|
605
|
-
const open = start && a.lines.findLastIndex((l) => l.kind === start);
|
|
606
|
-
if (!this.#streaming && open !== void 0 && open >= 0) {
|
|
607
|
-
line.at = a.lines[open]?.at ?? line.at;
|
|
608
|
-
a.lines.splice(open, 1);
|
|
609
|
-
}
|
|
610
|
-
this.#add(a, line);
|
|
611
|
-
return;
|
|
612
|
-
}
|
|
613
|
-
case "debug": {
|
|
614
|
-
const a = this.#running.get(e.key);
|
|
615
|
-
if (!a) return;
|
|
616
|
-
for (const text of debugLines(c, e.line, this.#o.verbosity === 2)) this.#add(a, {
|
|
617
|
-
at: e.line.at,
|
|
618
|
-
text,
|
|
619
|
-
debug: true
|
|
620
|
-
});
|
|
621
|
-
return;
|
|
622
|
-
}
|
|
623
|
-
case "log": {
|
|
624
|
-
const a = e.key ? this.#running.get(e.key) : void 0;
|
|
625
|
-
const label = e.stream === "stderr" ? c.yellow("stderr |") : c.dim("stdout |");
|
|
626
|
-
for (const text of e.text.replace(/\n$/, "").split("\n")) if (a) this.#add(a, {
|
|
627
|
-
at: e.at,
|
|
628
|
-
text: `${label} ${text}`
|
|
629
|
-
});
|
|
630
|
-
else this.#write(`${label} ${text}`);
|
|
631
|
-
return;
|
|
632
|
-
}
|
|
633
|
-
case "eval-end":
|
|
634
|
-
this.#list.end(e.result);
|
|
635
|
-
this.#end(e.result, e.at);
|
|
636
|
-
if (this.#tries.has(e.result.key)) this.#ended.add(e.result.key);
|
|
637
|
-
this.#drawStatus();
|
|
638
|
-
return;
|
|
639
|
-
case "prepare":
|
|
640
|
-
for (const line of prepareStepLines(c, e, this.#columns, ERROR_LINES)) this.#write(line);
|
|
641
|
-
if (e.state === "failed" && this.#o.output.githubActions) {
|
|
642
|
-
const { target, failure } = e.outcome;
|
|
643
|
-
const what = target.kind === "workspace" ? "The" : "Sandbox";
|
|
644
|
-
const title = ghEscape(`${what} ${target.label} failed to prepare`, true);
|
|
645
|
-
this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
|
|
646
|
-
}
|
|
647
|
-
return;
|
|
648
|
-
case "notice":
|
|
649
|
-
for (const line of noticeLines(c, e, this.#columns)) this.#write(line);
|
|
650
|
-
return;
|
|
651
|
-
case "run-end": {
|
|
652
|
-
this.#clearStatus();
|
|
653
|
-
if (!this.#blank) this.#write("");
|
|
654
|
-
const verbose = (this.#o.verbosity ?? 0) > 0;
|
|
655
|
-
const list = statusList({
|
|
656
|
-
record: e.record,
|
|
657
|
-
suite: suiteResult(e.record),
|
|
658
|
-
remarks: e.remarks ?? null,
|
|
659
|
-
verbose,
|
|
660
|
-
...e.notRun ? { notRun: e.notRun } : {}
|
|
661
|
-
});
|
|
662
|
-
for (const line of listLines(list, c, true, pointerLine(e.summaryPath), verbose)) this.#write(line);
|
|
663
|
-
return;
|
|
664
|
-
}
|
|
665
|
-
}
|
|
666
|
-
}
|
|
667
|
-
get #columns() {
|
|
668
|
-
return this.#o.columns || Number.POSITIVE_INFINITY;
|
|
669
|
-
}
|
|
670
|
-
/** The header block names a sandbox warning; GitHub Actions also annotates it. */
|
|
671
|
-
#annotate(w) {
|
|
672
|
-
const file = w.file ? `file=${ghEscape(w.file, true)},` : "";
|
|
673
|
-
const title = w.kind === "root" ? "Sandbox runs as root" : "Sandbox has extra capabilities";
|
|
674
|
-
this.#write(`::warning ${file}title=${ghEscape(title, true)}::${ghEscape(w.message)}`);
|
|
675
|
-
}
|
|
676
|
-
/**
|
|
677
|
-
* An ended attempt's logs, as `--output-logs` picks: grouped, its block under a `▶` header;
|
|
678
|
-
* streamed, only its error lines, since the rest already printed. A sandbox that failed to
|
|
679
|
-
* prepare was annotated once, for all its attempts.
|
|
680
|
-
*/
|
|
681
|
-
#end(result, at) {
|
|
682
|
-
const c = this.#c;
|
|
683
|
-
const a = this.#running.get(result.key) ?? this.#attempt(result, at - result.durationMs);
|
|
684
|
-
this.#running.delete(result.key);
|
|
685
|
-
this.#queued.delete(result.key);
|
|
686
|
-
const outcome = outcomeOf(result);
|
|
687
|
-
const failed = outcome !== "pass" && outcome !== "skipped";
|
|
688
|
-
const ungated = failed && result.attempt?.gate === "none";
|
|
689
|
-
const missed = ungated && outcome === "fail";
|
|
690
|
-
const { outputLogs, githubActions, logPrefix } = this.#o.output;
|
|
691
|
-
const showDetail = !this.#streaming && a.lines.length > 0 && (outputLogs === "full" || outputLogs === "errors-only" && failed);
|
|
692
|
-
const judged = result.meta?.errorKind === "judge" && (showDetail || this.#streaming);
|
|
693
|
-
const errors = failed && outputLogs !== "none" && !judged ? missed ? failureLines(colors(false), result, { max: 1 }).filter((l) => l.trim()).slice(0, 1).map((l) => c.dim(l)) : failureLines(c, result, { max: ERROR_LINES }) : [];
|
|
694
|
-
const lines = a.lines.filter((l) => l.kind !== "cleanup").sort((x, y) => x.at - y.at);
|
|
695
|
-
const prefixed = logPrefix === "task";
|
|
696
|
-
const head = `${this.#prefix(a)} ${c.dim("▶")} ${result.id}`;
|
|
697
|
-
const group = githubActions && showDetail && (!failed || missed);
|
|
698
|
-
if (showDetail || !this.#streaming && errors.length) this.#write(group ? `::group::${stripAnsi(head)}` : head);
|
|
699
|
-
if (showDetail) for (const line of lines) this.#write(this.#detail(a, line, prefixed));
|
|
700
|
-
for (const line of errors) this.#write(prefixed ? `${this.#prefix(a)} ${line}` : ` ${line}`);
|
|
701
|
-
if (group) this.#write("::endgroup::");
|
|
702
|
-
if (githubActions && failed && !ungated && !result.meta?.prepare) {
|
|
703
|
-
const which = result.attempt?.label ? ` [${result.attempt.label}]` : "";
|
|
704
|
-
const title = ghEscape(`${result.id}${which}`, true);
|
|
705
|
-
const checks = (result.meta?.failedChecks ?? []).filter((x) => x.at);
|
|
706
|
-
for (const x of checks) {
|
|
707
|
-
const [file, line] = splitAt(x.at);
|
|
708
|
-
const label = ghEscape(`${result.id}${which}: ${x.label}`, true);
|
|
709
|
-
this.#write(`::error file=${ghEscape(file, true)},line=${line},title=${label}::${ghEscape(x.message)}`);
|
|
710
|
-
}
|
|
711
|
-
if (!errorCheck(result)?.at) {
|
|
712
|
-
const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n").replace(/^✗ failed {2}/, ""));
|
|
713
|
-
const at = locate(result);
|
|
714
|
-
const where = at ? `,line=${at.line},col=${at.col}` : "";
|
|
715
|
-
this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
}
|
|
719
|
-
/**
|
|
720
|
-
* Every `heartbeatMs` while attempts run. Without attempt logs, how many evals are done; with
|
|
721
|
-
* them, each running attempt's phase, when nothing printed for that long.
|
|
722
|
-
*/
|
|
723
|
-
tick(now = this.#now()) {
|
|
724
|
-
const { heartbeatMs, outputLogs } = this.#o.output;
|
|
725
|
-
if (this.#live || heartbeatMs <= 0) return;
|
|
726
|
-
if (this.#running.size === 0) return;
|
|
727
|
-
const c = this.#c;
|
|
728
|
-
if (outputLogs === "none") {
|
|
729
|
-
if (now - Math.max(this.#lastProgress, this.#runStartedAt) < heartbeatMs) return;
|
|
730
|
-
const { done, total } = this.#list.counts();
|
|
731
|
-
this.#write(c.dim(progressText(done, total, now - this.#runStartedAt)));
|
|
732
|
-
this.#lastProgress = now;
|
|
733
|
-
return;
|
|
734
|
-
}
|
|
735
|
-
if (now - this.#lastWrite < heartbeatMs) return;
|
|
736
|
-
const running = [...this.#running.values()].map((a) => {
|
|
737
|
-
const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
|
|
738
|
-
return `${this.#prefix(a)} ${phaseOf(progress)} ${c.dim(formatDuration(now - a.startedAt))}`;
|
|
739
|
-
});
|
|
740
|
-
const queued = this.#queued.size ? `, ${this.#queued.size} queued` : "";
|
|
741
|
-
const elapsed = formatDuration(now - this.#runStartedAt);
|
|
742
|
-
this.#write(`${c.dim(`… ${this.#running.size} running${queued} after ${elapsed}:`)} ${running.join(c.dim(", "))}`);
|
|
743
|
-
}
|
|
744
|
-
close() {
|
|
745
|
-
clearInterval(this.#timer);
|
|
746
|
-
this.#clearStatus();
|
|
747
|
-
}
|
|
748
|
-
};
|
|
749
|
-
// src/ui/tty.ts
|
|
750
|
-
/** The list's state, which the Ink app reads with `useSyncExternalStore`. */
|
|
751
|
-
var ListStore = class {
|
|
752
|
-
#state = {
|
|
753
|
-
files: [],
|
|
754
|
-
logs: [],
|
|
755
|
-
counts: {
|
|
756
|
-
done: 0,
|
|
757
|
-
total: 0
|
|
758
|
-
},
|
|
759
|
-
phase: null,
|
|
760
|
-
finished: null
|
|
761
|
-
};
|
|
762
|
-
#listeners = /* @__PURE__ */ new Set();
|
|
763
|
-
get = () => this.#state;
|
|
764
|
-
subscribe = (listener) => {
|
|
765
|
-
this.#listeners.add(listener);
|
|
766
|
-
return () => this.#listeners.delete(listener);
|
|
767
|
-
};
|
|
768
|
-
set(next) {
|
|
769
|
-
this.#state = {
|
|
770
|
-
...this.#state,
|
|
771
|
-
...next
|
|
772
|
-
};
|
|
773
|
-
for (const l of this.#listeners) l();
|
|
774
|
-
}
|
|
775
|
-
};
|
|
776
|
-
const FRAME_MS = 80;
|
|
777
|
-
const ESC = String.fromCharCode(27);
|
|
778
|
-
const BEL = String.fromCharCode(7);
|
|
779
|
-
/** A CSI sequence (colors, cursor moves) or an OSC one (a hyperlink), which take no columns. */
|
|
780
|
-
const ESCAPE = new RegExp(`${ESC}(?:\\[[0-9;?]*[ -/]*[@-~]|\\][^${BEL}${ESC}]*(?:${BEL}|${ESC}\\\\))`, "g");
|
|
781
|
-
const graphemes = new Intl.Segmenter();
|
|
782
|
-
/**
|
|
783
|
-
* Terminal rows one line takes at `columns`, placed as a terminal places it: a wide character that
|
|
784
|
-
* doesn't fit in the row's last column moves to the next row, `\r` returns to the row's start and
|
|
785
|
-
* `\t` goes to the next stop of eight without wrapping.
|
|
786
|
-
*/
|
|
787
|
-
function rowsOf(line, columns) {
|
|
788
|
-
let rows = 1;
|
|
789
|
-
let column = 0;
|
|
790
|
-
for (const { segment } of graphemes.segment(line.replace(ESCAPE, ""))) if (segment === "\r") column = 0;
|
|
791
|
-
else if (segment === " ") column = Math.min(columns - 1, (Math.floor(column / 8) + 1) * 8);
|
|
792
|
-
else if (segment === "\b") column = Math.max(0, column - 1);
|
|
793
|
-
else {
|
|
794
|
-
const width = stringWidth(segment);
|
|
795
|
-
if (!width) continue;
|
|
796
|
-
if (column + width > columns) {
|
|
797
|
-
rows += 1;
|
|
798
|
-
column = 0;
|
|
799
|
-
}
|
|
800
|
-
column += width;
|
|
801
|
-
}
|
|
802
|
-
return rows;
|
|
803
|
-
}
|
|
804
|
-
/**
|
|
805
|
-
* Before the list is up, the sandbox prepare step: the glyph of its `◆ Preparing…` line spins in
|
|
806
|
-
* place (the cursor goes up to it and back) until each sandbox is ready or failed, then settles to
|
|
807
|
-
* `◆`. What prints under it is kept and measured in rows as the terminal placed it, so a wide
|
|
808
|
-
* character still finds the step line. Once the step line would be out of reach, or the width
|
|
809
|
-
* changes (some terminals reflow what they show, some don't, so its row is unknown), it stops
|
|
810
|
-
* without drawing.
|
|
811
|
-
*/
|
|
812
|
-
var StepSpinner = class {
|
|
813
|
-
#write;
|
|
814
|
-
#c;
|
|
815
|
-
#size;
|
|
816
|
-
#timer;
|
|
817
|
-
#frame = 0;
|
|
818
|
-
/** The step line and everything printed under it since. */
|
|
819
|
-
#text = "";
|
|
820
|
-
/** The width `#text` was printed at. */
|
|
821
|
-
#columns = 0;
|
|
822
|
-
#pending = 0;
|
|
823
|
-
constructor(write, c, size) {
|
|
824
|
-
this.#write = write;
|
|
825
|
-
this.#c = c;
|
|
826
|
-
this.#size = size;
|
|
827
|
-
}
|
|
828
|
-
/** Terminal lines from the step line down to the cursor after `text`. */
|
|
829
|
-
#up(text = this.#text) {
|
|
830
|
-
const columns = Math.max(1, this.#columns);
|
|
831
|
-
const lines = text.split("\n");
|
|
832
|
-
if (lines.at(-1) === "") lines.pop();
|
|
833
|
-
return lines.reduce((n, l) => n + rowsOf(l, columns), 0);
|
|
834
|
-
}
|
|
835
|
-
/** The step line was just written as `text`; it spins until `targets` have reported. */
|
|
836
|
-
start(text, targets) {
|
|
837
|
-
this.settle();
|
|
838
|
-
this.#text = text;
|
|
839
|
-
this.#columns = this.#size().columns;
|
|
840
|
-
this.#pending = targets;
|
|
841
|
-
this.#timer = setInterval(() => this.#draw(), FRAME_MS);
|
|
842
|
-
this.#timer.unref();
|
|
843
|
-
this.#draw();
|
|
844
|
-
}
|
|
845
|
-
/** Before `text` prints under it; settles first if the step line would scroll out of reach. */
|
|
846
|
-
before(text) {
|
|
847
|
-
if (!this.#timer) return;
|
|
848
|
-
const next = this.#text + text;
|
|
849
|
-
if (!this.#reachable(next)) this.settle();
|
|
850
|
-
this.#text = next;
|
|
851
|
-
}
|
|
852
|
-
/** One sandbox reported; the last settles it. */
|
|
853
|
-
reported() {
|
|
854
|
-
this.#pending -= 1;
|
|
855
|
-
if (this.#pending <= 0) this.settle();
|
|
856
|
-
}
|
|
857
|
-
#reachable(text = this.#text) {
|
|
858
|
-
const { rows, columns } = this.#size();
|
|
859
|
-
return columns === this.#columns && this.#up(text) < rows - 1;
|
|
860
|
-
}
|
|
861
|
-
#draw(glyph = SPINNER[this.#frame++ % SPINNER.length] ?? "◆") {
|
|
862
|
-
if (!this.#reachable()) {
|
|
863
|
-
clearInterval(this.#timer);
|
|
864
|
-
this.#timer = void 0;
|
|
865
|
-
return;
|
|
866
|
-
}
|
|
867
|
-
const up = this.#up();
|
|
868
|
-
this.#write(`\x1b[${up}A\r\x1b[2C${this.#c.teal(glyph)}\r\x1b[${up}B`);
|
|
869
|
-
}
|
|
870
|
-
settle() {
|
|
871
|
-
if (!this.#timer) return;
|
|
872
|
-
this.#draw("◆");
|
|
873
|
-
clearInterval(this.#timer);
|
|
874
|
-
this.#timer = void 0;
|
|
875
|
-
}
|
|
876
|
-
};
|
|
877
|
-
var InteractiveRenderer = class {
|
|
878
|
-
#o;
|
|
879
|
-
#c;
|
|
880
|
-
#store = new ListStore();
|
|
881
|
-
#list;
|
|
882
|
-
/** The header and run-level lines, as every terminal prints them. */
|
|
883
|
-
#plain;
|
|
884
|
-
#app;
|
|
885
|
-
/** Set once the list couldn't load: from then on, the run prints as a pipe's does. */
|
|
886
|
-
#fallback;
|
|
887
|
-
#ended;
|
|
888
|
-
#closed = false;
|
|
889
|
-
#spinner;
|
|
890
|
-
/** What the plain renderer wrote since `#written` was reset, while the list isn't up. */
|
|
891
|
-
#written = "";
|
|
892
|
-
constructor(options) {
|
|
893
|
-
this.#o = options;
|
|
894
|
-
this.#c = options.colors;
|
|
895
|
-
this.#list = new LiveList(options.harness);
|
|
896
|
-
this.#spinner = new StepSpinner((text) => options.out.write(text), options.colors, () => sizeOf(options.stdout));
|
|
897
|
-
const out = { write: (text) => {
|
|
898
|
-
if (!this.#app || this.#fallback) {
|
|
899
|
-
this.#spinner.before(redactSecrets(String(text)));
|
|
900
|
-
this.#written += String(text);
|
|
901
|
-
return options.out.write(text);
|
|
902
|
-
}
|
|
903
|
-
const lines = String(text).replace(/\n$/, "").split("\n");
|
|
904
|
-
this.#store.set({ logs: [...this.#store.get().logs, ...lines] });
|
|
905
|
-
return true;
|
|
906
|
-
} };
|
|
907
|
-
this.#plain = new PlainRenderer({
|
|
908
|
-
...options,
|
|
909
|
-
out,
|
|
910
|
-
timers: false,
|
|
911
|
-
output: {
|
|
912
|
-
...options.output,
|
|
913
|
-
statusLine: false,
|
|
914
|
-
interactive: false,
|
|
915
|
-
outputLogs: "none"
|
|
916
|
-
}
|
|
917
|
-
});
|
|
918
|
-
}
|
|
919
|
-
#mount() {
|
|
920
|
-
if (this.#app || this.#closed) return;
|
|
921
|
-
const cancel = this.#o.cancel ?? (() => process.kill(process.pid, "SIGINT"));
|
|
922
|
-
const load = this.#o.load ?? (() => import("./ink-list-BpfidQ1h.js"));
|
|
923
|
-
this.#app = load().then((m) => m.mountList({
|
|
924
|
-
store: this.#store,
|
|
925
|
-
colors: this.#c,
|
|
926
|
-
cancel,
|
|
927
|
-
stdout: this.#o.stdout,
|
|
928
|
-
stdin: this.#o.stdin
|
|
929
|
-
})).catch((error) => {
|
|
930
|
-
this.#fallBack(error);
|
|
931
|
-
});
|
|
932
|
-
}
|
|
933
|
-
#fallBack(error) {
|
|
934
|
-
const out = this.#o.out;
|
|
935
|
-
const { logs, phase } = this.#store.get();
|
|
936
|
-
for (const line of logs) out.write(`${line}\n`);
|
|
937
|
-
const reason = error instanceof Error ? error.message : String(error);
|
|
938
|
-
out.write(`${this.#c.yellow(`Couldn't draw the interactive list (${reason}).`)}\n`);
|
|
939
|
-
if (phase) out.write(` ${this.#c.teal("◆")} ${phase}\n`);
|
|
940
|
-
this.#fallback = new PlainRenderer({
|
|
941
|
-
...this.#o,
|
|
942
|
-
timers: false,
|
|
943
|
-
output: {
|
|
944
|
-
...this.#o.output,
|
|
945
|
-
statusLine: false,
|
|
946
|
-
interactive: false
|
|
947
|
-
}
|
|
948
|
-
});
|
|
949
|
-
if (this.#ended) this.#fallback.onEvent(this.#ended.event);
|
|
950
|
-
}
|
|
951
|
-
#update() {
|
|
952
|
-
const { done, total, failed, failing } = this.#list.counts();
|
|
953
|
-
this.#store.set({
|
|
954
|
-
files: this.#list.files(),
|
|
955
|
-
counts: {
|
|
956
|
-
done,
|
|
957
|
-
total,
|
|
958
|
-
failed,
|
|
959
|
-
failing
|
|
960
|
-
}
|
|
961
|
-
});
|
|
962
|
-
}
|
|
963
|
-
onEvent(e) {
|
|
964
|
-
if (this.#fallback) {
|
|
965
|
-
if (e.type !== "run-start") this.#fallback.onEvent(e);
|
|
966
|
-
return;
|
|
967
|
-
}
|
|
968
|
-
switch (e.type) {
|
|
969
|
-
case "run-start":
|
|
970
|
-
this.#spinner.settle();
|
|
971
|
-
this.#store.set({ phase: startText(this.#o.output) });
|
|
972
|
-
return;
|
|
973
|
-
case "prepare":
|
|
974
|
-
this.#written = "";
|
|
975
|
-
this.#plain.onEvent(e);
|
|
976
|
-
if (this.#app || this.#o.timers === false || process.env.REEVAL_DEBUG) return;
|
|
977
|
-
if (e.state === "start") this.#spinner.start(this.#written, e.targets.length);
|
|
978
|
-
else this.#spinner.reported();
|
|
979
|
-
return;
|
|
980
|
-
case "collected":
|
|
981
|
-
this.#spinner.settle();
|
|
982
|
-
this.#list.collected(e.file, e.evals);
|
|
983
|
-
this.#mount();
|
|
984
|
-
this.#update();
|
|
985
|
-
return;
|
|
986
|
-
case "eval-start":
|
|
987
|
-
this.#list.start(e);
|
|
988
|
-
this.#update();
|
|
989
|
-
return;
|
|
990
|
-
case "eval-progress":
|
|
991
|
-
if (e.progress.kind === "warning" || e.progress.kind === "browser-download") this.#plain.onEvent(e);
|
|
992
|
-
this.#list.progress(e.key, e.progress);
|
|
993
|
-
this.#update();
|
|
994
|
-
return;
|
|
995
|
-
case "eval-end":
|
|
996
|
-
this.#list.end(e.result);
|
|
997
|
-
this.#update();
|
|
998
|
-
return;
|
|
999
|
-
case "debug": return;
|
|
1000
|
-
case "run-end": {
|
|
1001
|
-
const listOf = (verbose) => statusList({
|
|
1002
|
-
record: e.record,
|
|
1003
|
-
suite: suiteResult(e.record),
|
|
1004
|
-
remarks: e.remarks ?? null,
|
|
1005
|
-
verbose,
|
|
1006
|
-
...e.notRun ? { notRun: e.notRun } : {}
|
|
1007
|
-
});
|
|
1008
|
-
const list = listOf(false);
|
|
1009
|
-
const verbose = (this.#o.verbosity ?? 0) > 0;
|
|
1010
|
-
if (!this.#app) {
|
|
1011
|
-
this.#plain.onEvent(e);
|
|
1012
|
-
return;
|
|
1013
|
-
}
|
|
1014
|
-
this.#ended = {
|
|
1015
|
-
scrollback: verbose ? listOf(true) : list,
|
|
1016
|
-
verbose,
|
|
1017
|
-
pointer: pointerLine(e.summaryPath),
|
|
1018
|
-
event: e
|
|
1019
|
-
};
|
|
1020
|
-
this.#store.set({
|
|
1021
|
-
files: list.files,
|
|
1022
|
-
...this.#store.get().phase ? { phase: endText(e.record.status) } : {},
|
|
1023
|
-
finished: {
|
|
1024
|
-
done: list.done,
|
|
1025
|
-
notes: list.notes,
|
|
1026
|
-
pointer: pointerLine(e.summaryPath),
|
|
1027
|
-
errors: list.errors
|
|
1028
|
-
}
|
|
1029
|
-
});
|
|
1030
|
-
return;
|
|
1031
|
-
}
|
|
1032
|
-
default: this.#plain.onEvent(e);
|
|
1033
|
-
}
|
|
1034
|
-
}
|
|
1035
|
-
close() {
|
|
1036
|
-
this.#spinner.settle();
|
|
1037
|
-
this.#plain.close();
|
|
1038
|
-
this.#fallback?.close();
|
|
1039
|
-
if (!this.#store.get().finished) {
|
|
1040
|
-
this.#closed = true;
|
|
1041
|
-
this.#app?.then((a) => a?.unmount());
|
|
1042
|
-
}
|
|
1043
|
-
}
|
|
1044
|
-
/** Until the user quits the list, then the whole list into the scrollback; at once when there's none. */
|
|
1045
|
-
async wait() {
|
|
1046
|
-
const app = await this.#app;
|
|
1047
|
-
if (!app || !this.#ended) return;
|
|
1048
|
-
await app.waitUntilExit();
|
|
1049
|
-
const { phase } = this.#store.get();
|
|
1050
|
-
const { scrollback, verbose, pointer } = this.#ended;
|
|
1051
|
-
const text = [...phase ? [` ${this.#c.teal("◆")} ${phase}`, ""] : [], ...listLines(scrollback, this.#c, true, pointer, verbose)];
|
|
1052
|
-
this.#o.out.write(`${text.join("\n")}\n`);
|
|
1053
|
-
}
|
|
1054
|
-
};
|
|
1055
|
-
const sizeOf = (stdout) => {
|
|
1056
|
-
const { rows, columns } = stdout ?? {};
|
|
1057
|
-
return {
|
|
1058
|
-
rows: rows || 24,
|
|
1059
|
-
columns: columns || 80
|
|
1060
|
-
};
|
|
1061
|
-
};
|
|
1062
|
-
// src/ui/index.ts
|
|
1063
|
-
const depthOf = (bits = 4) => bits >= 24 ? 24 : bits >= 8 ? 8 : 4;
|
|
1064
|
-
function createRenderer(options) {
|
|
1065
|
-
const out = { write: (text) => options.out.write(redactSecrets(text)) };
|
|
1066
|
-
if (options.output.mode === "agent") return new AgentRenderer({
|
|
1067
|
-
...options,
|
|
1068
|
-
out
|
|
1069
|
-
});
|
|
1070
|
-
const paint = colors(options.output.color, depthOf(options.colorDepth));
|
|
1071
|
-
if (options.output.interactive && options.tty) return new InteractiveRenderer({
|
|
1072
|
-
...options,
|
|
1073
|
-
...options.tty,
|
|
1074
|
-
out,
|
|
1075
|
-
colors: paint
|
|
1076
|
-
});
|
|
1077
|
-
return new PlainRenderer({
|
|
1078
|
-
...options,
|
|
1079
|
-
out,
|
|
1080
|
-
colors: paint
|
|
1081
|
-
});
|
|
1082
|
-
}
|
|
1083
173
|
// src/cli.ts
|
|
1084
174
|
const HISTORY_COMMANDS = {
|
|
1085
175
|
record: 1,
|
|
@@ -1088,13 +178,31 @@ const HISTORY_COMMANDS = {
|
|
|
1088
178
|
compare: 1,
|
|
1089
179
|
checkout: 1,
|
|
1090
180
|
query: 1,
|
|
1091
|
-
analyze: 1
|
|
181
|
+
analyze: 1,
|
|
182
|
+
sync: 1
|
|
183
|
+
};
|
|
184
|
+
/** The commands that pull a remote store's new runs into its copy before reading it. */
|
|
185
|
+
const PULLS = {
|
|
186
|
+
history: 1,
|
|
187
|
+
show: 1,
|
|
188
|
+
compare: 1,
|
|
189
|
+
checkout: 1,
|
|
190
|
+
query: 1,
|
|
191
|
+
analyze: 1,
|
|
192
|
+
mcp: 1
|
|
1092
193
|
};
|
|
1093
194
|
const EXPECTED_ERRORS = [
|
|
1094
195
|
"CommandError",
|
|
1095
196
|
"RefError",
|
|
1096
197
|
"HistoryError",
|
|
1097
|
-
"StudioBuildError"
|
|
198
|
+
"StudioBuildError",
|
|
199
|
+
"NewerSchemaError"
|
|
200
|
+
];
|
|
201
|
+
/** Errors that exit 3: reeval couldn't reach what the run needs, not a mistake in the command. */
|
|
202
|
+
const INFRA_ERRORS = [
|
|
203
|
+
"PreflightInfraError",
|
|
204
|
+
"StoreError",
|
|
205
|
+
"EventStreamError"
|
|
1098
206
|
];
|
|
1099
207
|
const EXIT = {
|
|
1100
208
|
ok: 0,
|
|
@@ -1114,6 +222,8 @@ const invocation = {
|
|
|
1114
222
|
runId: null,
|
|
1115
223
|
unhandledErrors: []
|
|
1116
224
|
};
|
|
225
|
+
/** `reeval run --events`: the run's event stream, which ends with the command's exit code. */
|
|
226
|
+
let eventStream;
|
|
1117
227
|
/** Set by the first Ctrl-C or SIGTERM of `reeval run`. */
|
|
1118
228
|
let interrupted = false;
|
|
1119
229
|
/** The last signal that canceled the run: 130 after SIGINT, 143 after SIGTERM. */
|
|
@@ -1133,6 +243,10 @@ function quitWithoutRecord(runId, stagingDir) {
|
|
|
1133
243
|
` Deleting ${fromCwd(stagingDir)} discards the run.`
|
|
1134
244
|
].join("\n");
|
|
1135
245
|
}
|
|
246
|
+
/** Sweeps stale staging, naming each stale run it keeps because it can still be resumed. */
|
|
247
|
+
async function sweep(root, keep) {
|
|
248
|
+
for (const runId of await sweepStaging(root, keep)) process.stderr.write(`reeval: run ${runId} stopped over a day ago and can still be resumed: reeval run --resume ${runId} finishes it, reeval clean --staging deletes it.\n`);
|
|
249
|
+
}
|
|
1136
250
|
async function run(argv) {
|
|
1137
251
|
const { values, positionals } = parseCli(argv);
|
|
1138
252
|
invocation.flags = values;
|
|
@@ -1150,7 +264,7 @@ async function run(argv) {
|
|
|
1150
264
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
1151
265
|
}
|
|
1152
266
|
if (command === "init") {
|
|
1153
|
-
const { initCommand } = await import("./init-
|
|
267
|
+
const { initCommand } = await import("./init-CM9xwyre.js");
|
|
1154
268
|
return initCommand(values, filters, {
|
|
1155
269
|
out: (s) => console.log(s),
|
|
1156
270
|
input: process.stdin,
|
|
@@ -1159,7 +273,7 @@ async function run(argv) {
|
|
|
1159
273
|
});
|
|
1160
274
|
}
|
|
1161
275
|
if (command === "doctor") {
|
|
1162
|
-
const { doctorCommand } = await import("./doctor-
|
|
276
|
+
const { doctorCommand } = await import("./doctor-B7utCSdh.js");
|
|
1163
277
|
return doctorCommand(values, {
|
|
1164
278
|
out: (s) => console.log(s),
|
|
1165
279
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -1167,7 +281,7 @@ async function run(argv) {
|
|
|
1167
281
|
});
|
|
1168
282
|
}
|
|
1169
283
|
if (command === "report") {
|
|
1170
|
-
const { reportCommand } = await import("./command-
|
|
284
|
+
const { reportCommand } = await import("./command-loXH2jHr.js");
|
|
1171
285
|
return reportCommand(values, filters, {
|
|
1172
286
|
out: (s) => console.log(s),
|
|
1173
287
|
err: (s) => console.error(dim(s)),
|
|
@@ -1181,8 +295,9 @@ async function run(argv) {
|
|
|
1181
295
|
return imageBuildCommand(values, configPath, filters.slice(1));
|
|
1182
296
|
}
|
|
1183
297
|
if (command === "studio") {
|
|
1184
|
-
const config = await
|
|
1185
|
-
|
|
298
|
+
const config = await historyConfig(configPath, "studio", values.profile);
|
|
299
|
+
localStudio(config);
|
|
300
|
+
const { studioCommand } = await import("./command-BKb8eN_X.js");
|
|
1186
301
|
return studioCommand({
|
|
1187
302
|
config,
|
|
1188
303
|
flags: values,
|
|
@@ -1192,8 +307,8 @@ async function run(argv) {
|
|
|
1192
307
|
}
|
|
1193
308
|
if (command === "mcp") {
|
|
1194
309
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
1195
|
-
const config = await
|
|
1196
|
-
const { mcpCommand } = await import("./command-
|
|
310
|
+
const config = await historyConfig(configPath, "mcp", values.profile, values);
|
|
311
|
+
const { mcpCommand } = await import("./command-Dwlh0Vhu.js");
|
|
1197
312
|
return mcpCommand({
|
|
1198
313
|
config,
|
|
1199
314
|
flags: values,
|
|
@@ -1201,8 +316,8 @@ async function run(argv) {
|
|
|
1201
316
|
});
|
|
1202
317
|
}
|
|
1203
318
|
if (command in HISTORY_COMMANDS) {
|
|
1204
|
-
const config = await
|
|
1205
|
-
const commands = await import("./commands-
|
|
319
|
+
const config = await historyConfig(configPath, command, values.profile, values);
|
|
320
|
+
const commands = await import("./commands-Dqli45sR.js");
|
|
1206
321
|
const ctx = {
|
|
1207
322
|
config,
|
|
1208
323
|
flags: values,
|
|
@@ -1214,16 +329,16 @@ async function run(argv) {
|
|
|
1214
329
|
case "show": return commands.showCommand(ctx, filters);
|
|
1215
330
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
1216
331
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
332
|
+
case "sync": return commands.syncCommand(ctx, filters);
|
|
1217
333
|
case "query": {
|
|
1218
|
-
const { queryCommand } = await import("./query-
|
|
334
|
+
const { queryCommand } = await import("./query-B2SQddoV.js");
|
|
1219
335
|
return queryCommand({
|
|
1220
336
|
...ctx,
|
|
1221
337
|
err: (s) => console.error(dim(s))
|
|
1222
338
|
}, filters);
|
|
1223
339
|
}
|
|
1224
340
|
case "analyze": {
|
|
1225
|
-
|
|
1226
|
-
const { analyzeCommand } = await import("./command-DSL7I-SI.js");
|
|
341
|
+
const { analyzeCommand } = await import("./command-D_w_kDpI.js");
|
|
1227
342
|
return analyzeCommand({
|
|
1228
343
|
...ctx,
|
|
1229
344
|
err: (s) => console.error(dim(s))
|
|
@@ -1232,17 +347,79 @@ async function run(argv) {
|
|
|
1232
347
|
}
|
|
1233
348
|
}
|
|
1234
349
|
if (command === "clean") return cleanCommand(values, configPath);
|
|
350
|
+
if (command === "attach") return attachCommand(values, filters[0], configPath);
|
|
351
|
+
if (command === "bundle") {
|
|
352
|
+
const { bundleCommand } = await import("./bundle-DZstHDeW.js");
|
|
353
|
+
return bundleCommand(values, filters, configPath, argv.slice(1), (s) => console.log(s));
|
|
354
|
+
}
|
|
1235
355
|
if (command !== "run") throw new UsageError(`Unknown command "${command}". Run reeval --help.`);
|
|
356
|
+
if (values.bundle !== void 0) return bundleRun(values, filters);
|
|
1236
357
|
return runCommand(values, configPath, filters);
|
|
1237
358
|
}
|
|
359
|
+
/**
|
|
360
|
+
* `reeval run --bundle`: the bundle's own filters, profile, tags, traits and runs, from the
|
|
361
|
+
* project it extracts to. Other run flags (`--json`, `--record`, `--concurrency`, …) apply as given.
|
|
362
|
+
*/
|
|
363
|
+
async function bundleRun(flags, filters) {
|
|
364
|
+
const planning = [
|
|
365
|
+
"profile",
|
|
366
|
+
"tag",
|
|
367
|
+
"trait",
|
|
368
|
+
"runs",
|
|
369
|
+
"config",
|
|
370
|
+
"resume",
|
|
371
|
+
"retry-errors"
|
|
372
|
+
].filter((f) => flags[f] !== void 0).map((f) => `--${f}`);
|
|
373
|
+
if (filters.length || planning.length) throw new UsageError(`--bundle runs the bundle with the filters, profile, tags, traits and runs it was made with: leave out ${[...filters.length ? ["filters"] : [], ...planning].join(", ")}.`);
|
|
374
|
+
const file = flags.bundle;
|
|
375
|
+
const log = (s) => flags.json ? console.error(s) : console.log(s);
|
|
376
|
+
const { openBundle } = await import("./bundle-DZstHDeW.js");
|
|
377
|
+
const opened = await openBundle(file, log);
|
|
378
|
+
if (opened.exit !== void 0) return opened.exit;
|
|
379
|
+
const { args } = opened.manifest;
|
|
380
|
+
log(`reeval: running ${file} in ${opened.project}, where its .reeval/ stays`);
|
|
381
|
+
process.chdir(opened.project);
|
|
382
|
+
return runCommand({
|
|
383
|
+
...flags,
|
|
384
|
+
bundle: void 0,
|
|
385
|
+
...args.profile ? { profile: args.profile } : {},
|
|
386
|
+
tag: args.tags,
|
|
387
|
+
trait: args.traits,
|
|
388
|
+
...args.runs ? { runs: args.runs } : {}
|
|
389
|
+
}, opened.config, args.filters);
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* The config a command that reads the history runs on: the profile `--profile`, `REEVAL_PROFILE`
|
|
393
|
+
* or `defaultProfile` picks sets which store it reads, as it does for `reeval run`. `.env` and
|
|
394
|
+
* `.env.local` load first, as for `reeval run`: they can hold the store's keys and the profile.
|
|
395
|
+
*/
|
|
396
|
+
async function historyConfig(configPath, command, flag, values = {}) {
|
|
397
|
+
loadEnvFiles(configPath);
|
|
398
|
+
const named = flag ?? (process.env.REEVAL_PROFILE || void 0);
|
|
399
|
+
const loaded = await loadConfig(configPath, command, named);
|
|
400
|
+
const config = withProfileHistory(loaded, selectProfile(loaded, flag));
|
|
401
|
+
if (!config.history.scope) return config;
|
|
402
|
+
const history = values["all-branches"] ? {
|
|
403
|
+
...config.history,
|
|
404
|
+
scope: "all"
|
|
405
|
+
} : config.history;
|
|
406
|
+
if (isRemote(history) && command in PULLS && !values.offline) {
|
|
407
|
+
const { cachedAt, unreachable } = await pullStore(history);
|
|
408
|
+
if (unreachable) console.error(yellow(`${history.store.name} is unreachable (${unreachable}); showing the runs pulled at ${cachedAt}`));
|
|
409
|
+
}
|
|
410
|
+
return {
|
|
411
|
+
...config,
|
|
412
|
+
history
|
|
413
|
+
};
|
|
414
|
+
}
|
|
1238
415
|
/** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
|
|
1239
416
|
async function dryRunPrices(planned, judgeModel) {
|
|
1240
|
-
const { needsPrices } = await import("./cost-
|
|
417
|
+
const { needsPrices } = await import("./cost-IM80LyGD.js");
|
|
1241
418
|
if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
|
|
1242
419
|
table: null,
|
|
1243
420
|
why: "nothing to price"
|
|
1244
421
|
};
|
|
1245
|
-
const { loadPrices } = await import("./prices-
|
|
422
|
+
const { loadPrices } = await import("./prices-B1vm_lPk.js");
|
|
1246
423
|
return loadPrices();
|
|
1247
424
|
}
|
|
1248
425
|
/** Asks a yes/no question on the terminal; anything but y or yes is no. */
|
|
@@ -1260,7 +437,8 @@ async function confirm(question) {
|
|
|
1260
437
|
}
|
|
1261
438
|
/**
|
|
1262
439
|
* `reeval clean`: removes local debug data and snapshots, all of them or all but the newest
|
|
1263
|
-
* `--keep` runs'
|
|
440
|
+
* `--keep` runs', with `--staging` every stopped run's staging, and with `--bundles` every copy
|
|
441
|
+
* `run --bundle` extracted whose process has ended.
|
|
1264
442
|
*/
|
|
1265
443
|
async function cleanCommand(flags, configPath) {
|
|
1266
444
|
const config = await loadConfig(configPath, "clean");
|
|
@@ -1269,20 +447,38 @@ async function cleanCommand(flags, configPath) {
|
|
|
1269
447
|
keep = Number(flags.keep);
|
|
1270
448
|
if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
|
|
1271
449
|
}
|
|
1272
|
-
const { debugRoot, pruneDebug } = await import("./files-
|
|
450
|
+
const { debugRoot, pruneDebug } = await import("./files-BjOJw4sn.js");
|
|
1273
451
|
const { formatBytes } = await import("./debug-CF4p4Lp_.js");
|
|
1274
452
|
const { runs, bytes } = await pruneDebug(config.root, keep);
|
|
1275
453
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1276
454
|
console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
|
|
455
|
+
if (flags.staging) {
|
|
456
|
+
const removed = await removeStoppedStaging(config.root);
|
|
457
|
+
console.log(removed.length ? `Removed the staging of ${removed.length} stopped run${removed.length === 1 ? "" : "s"}: ${removed.join(", ")}` : "No stopped runs in .reeval/staging to remove");
|
|
458
|
+
}
|
|
459
|
+
const streams = await pruneEvents(config.root, keep);
|
|
460
|
+
if (streams.length) console.log(`Removed the event streams of ${streams.length} finished run${streams.length === 1 ? "" : "s"} from .reeval/events`);
|
|
461
|
+
if (flags.bundles) {
|
|
462
|
+
const { removeExtractedBundles } = await import("./bundle-DZstHDeW.js");
|
|
463
|
+
const removed = await removeExtractedBundles();
|
|
464
|
+
console.log(removed.length ? `Removed ${removed.length} extracted bundle${removed.length === 1 ? "" : "s"}: ${removed.join(", ")}` : "No extracted bundles to remove");
|
|
465
|
+
}
|
|
1277
466
|
if (flags.images) {
|
|
1278
|
-
const { dockerAvailable, pruneImages } = await import("./src-
|
|
1279
|
-
const { requireDocker } = await import("./providers-
|
|
467
|
+
const { dockerAvailable, pruneImages } = await import("./src-jYqNkT-F.js");
|
|
468
|
+
const { requireDocker } = await import("./providers-BkLGzaSY.js");
|
|
1280
469
|
const engines = [];
|
|
1281
470
|
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1282
|
-
|
|
471
|
+
const { tartCli } = await import("./tart-cli-BOcluha1.js");
|
|
472
|
+
const tart = await tartCli(["--version"]).then((r) => r.exitCode === 0, () => false);
|
|
473
|
+
if (!engines.length && !tart) await requireDocker();
|
|
1283
474
|
let removed = 0;
|
|
1284
475
|
for (const engine of engines) removed += await pruneImages(engine);
|
|
1285
|
-
console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
|
|
476
|
+
if (engines.length) console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
|
|
477
|
+
if (tart) {
|
|
478
|
+
const { deleteBakedImages } = await import("./tart-bake-DjFcsG3N.js");
|
|
479
|
+
const baked = await deleteBakedImages();
|
|
480
|
+
console.log(baked.length ? `Deleted ${baked.length} Tart image${baked.length === 1 ? "" : "s"} with a harness baked in (reeval image build bakes them again): ${baked.join(", ")}` : "No Tart images with a harness baked in to delete");
|
|
481
|
+
}
|
|
1286
482
|
}
|
|
1287
483
|
return EXIT.ok;
|
|
1288
484
|
}
|
|
@@ -1298,11 +494,12 @@ function verbosityOf(flags) {
|
|
|
1298
494
|
* image.
|
|
1299
495
|
*/
|
|
1300
496
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1301
|
-
const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-
|
|
1302
|
-
const
|
|
497
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-Ch1y1t6_.js");
|
|
498
|
+
const setup = await prepareRun({
|
|
1303
499
|
...flags,
|
|
1304
500
|
"dry-run": true
|
|
1305
|
-
}, configPath)
|
|
501
|
+
}, configPath);
|
|
502
|
+
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(setup);
|
|
1306
503
|
const root = config.root;
|
|
1307
504
|
const uses = await plannedImageUses({
|
|
1308
505
|
runId,
|
|
@@ -1320,8 +517,8 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1320
517
|
return EXIT.ok;
|
|
1321
518
|
}
|
|
1322
519
|
const { tartTagWarning } = await import("./tart-ref-7Qv3cm3B.js");
|
|
1323
|
-
for (const
|
|
1324
|
-
const warning = tartTagWarning(
|
|
520
|
+
for (const image of new Set(vms.map((t) => t.sandbox.image))) {
|
|
521
|
+
const warning = tartTagWarning(image);
|
|
1325
522
|
if (warning) console.log(yellow(`warning: ${warning}`));
|
|
1326
523
|
}
|
|
1327
524
|
if (flags["dry-run"]) {
|
|
@@ -1329,7 +526,7 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1329
526
|
if (targets.length) for (const line of await imagePlanLines(targets, root)) console.log(line);
|
|
1330
527
|
return EXIT.ok;
|
|
1331
528
|
}
|
|
1332
|
-
const { PrepareError } = await import("./provider-
|
|
529
|
+
const { PrepareError } = await import("./provider-DHT20Gl8.js");
|
|
1333
530
|
const outcomes = [];
|
|
1334
531
|
let refused = false;
|
|
1335
532
|
const refusal = (id, of, others) => (error) => {
|
|
@@ -1345,7 +542,10 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1345
542
|
return [];
|
|
1346
543
|
};
|
|
1347
544
|
try {
|
|
1348
|
-
const pulled = vms.length ? await pullTartImages(vms, root, {
|
|
545
|
+
const pulled = vms.length ? await pullTartImages(vms, root, {
|
|
546
|
+
log: (line) => console.log(line.startsWith("warning: ") ? yellow(line) : dim(line)),
|
|
547
|
+
force: !!flags.force
|
|
548
|
+
}).catch(refusal("tart", vms, targets.length)) : [];
|
|
1349
549
|
for (const p of pulled) outcomes.push(p.ok ? {
|
|
1350
550
|
id: p.id,
|
|
1351
551
|
evals: p.evals,
|
|
@@ -1361,7 +561,9 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1361
561
|
if (targets.length) {
|
|
1362
562
|
const built = await buildImages(targets, root, {
|
|
1363
563
|
log: (line) => console.log(dim(line)),
|
|
1364
|
-
force: !!flags.force
|
|
564
|
+
force: !!flags.force,
|
|
565
|
+
registry: setup.registry,
|
|
566
|
+
push: !!flags.push
|
|
1365
567
|
}).catch(refusal("docker", targets, vms.length));
|
|
1366
568
|
for (const b of built) outcomes.push(b.ok ? {
|
|
1367
569
|
id: b.id,
|
|
@@ -1380,7 +582,7 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1380
582
|
if (!(error instanceof PrepareError)) throw error;
|
|
1381
583
|
if (error.failure.usage) throw new UsageError(error.message);
|
|
1382
584
|
if (error.failure.cause === "enospc") invocation.machineState = true;
|
|
1383
|
-
const { PreflightInfraError } = await import("./preflight-
|
|
585
|
+
const { PreflightInfraError } = await import("./preflight-BMVDBcux.js");
|
|
1384
586
|
throw new PreflightInfraError(error.message);
|
|
1385
587
|
}
|
|
1386
588
|
for (const o of outcomes) {
|
|
@@ -1403,18 +605,69 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1403
605
|
}
|
|
1404
606
|
/** The first 12 hex digits of an image id or digest. */
|
|
1405
607
|
const short12 = (id) => id.replace(/^sha256:/, "").slice(0, 12);
|
|
608
|
+
/**
|
|
609
|
+
* `reeval attach`: a run's socket URL or stream file as given, else the run a ref names (by default the one in
|
|
610
|
+
* progress) and the stream file it writes. Exits with the run's exit code.
|
|
611
|
+
*/
|
|
612
|
+
async function attachCommand(flags, target, configPath) {
|
|
613
|
+
let source;
|
|
614
|
+
let root = configPath === void 0 ? void 0 : dirname(configPath);
|
|
615
|
+
if (target && (isAttachUrl(target) || existsSync(target))) {
|
|
616
|
+
source = target;
|
|
617
|
+
const inEvents = isAttachUrl(target) ? null : /^(.*)[\\/]\.reeval[\\/]events[\\/][^\\/]+\.ndjson$/.exec(resolve(target));
|
|
618
|
+
if (inEvents?.[1]) root = inEvents[1];
|
|
619
|
+
} else {
|
|
620
|
+
root = dirname(requireConfig(configPath));
|
|
621
|
+
source = (await resolveAttach(root, target)).file;
|
|
622
|
+
}
|
|
623
|
+
const { attach } = await import("./attach-EbDGKGTO.js");
|
|
624
|
+
return attach({
|
|
625
|
+
source,
|
|
626
|
+
...root === void 0 ? {} : { root },
|
|
627
|
+
out: process.stdout,
|
|
628
|
+
columns: process.stdout.columns,
|
|
629
|
+
colorDepth: process.stdout.isTTY ? process.stdout.getColorDepth() : void 0,
|
|
630
|
+
tty: {
|
|
631
|
+
stdout: process.stdout,
|
|
632
|
+
stdin: process.stdin
|
|
633
|
+
},
|
|
634
|
+
output: (verbosity) => resolveOutput({
|
|
635
|
+
flags: {
|
|
636
|
+
ui: flags.ui,
|
|
637
|
+
logOrder: flags["log-order"],
|
|
638
|
+
logPrefix: flags["log-prefix"],
|
|
639
|
+
outputLogs: flags["output-logs"],
|
|
640
|
+
verbose: verbosity > 0
|
|
641
|
+
},
|
|
642
|
+
env: process.env,
|
|
643
|
+
isTTY: !!process.stdout.isTTY,
|
|
644
|
+
agent: detectAgent().name,
|
|
645
|
+
stdinTTY: !!process.stdin.isTTY
|
|
646
|
+
})
|
|
647
|
+
});
|
|
648
|
+
}
|
|
1406
649
|
/** `reeval run`: check, plan, run the engine, then write the run and pick the exit code. */
|
|
1407
650
|
async function runCommand(flags, configPath, given) {
|
|
1408
651
|
const retry = flags["retry-errors"];
|
|
652
|
+
const url = flags["events-url"];
|
|
653
|
+
const parsed = url === void 0 ? null : URL.parse(url);
|
|
654
|
+
if (url !== void 0 && !(parsed && isEventsUrl(url))) throw new UsageError(`--events-url takes an http or https URL (got ${stripCredentials(url)})`);
|
|
655
|
+
if (url !== void 0 && (parsed?.username || parsed?.password)) throw new UsageError(`--events-url can't carry a user name or password (got ${stripCredentials(url)}): put a token in the path or query instead`);
|
|
656
|
+
const eventsFileFlag = flags["events-file"];
|
|
657
|
+
if (eventsFileFlag !== void 0) {
|
|
658
|
+
const refused = notEventStream(resolve(eventsFileFlag));
|
|
659
|
+
if (refused) throw new UsageError(`--events-file: ${refused}`);
|
|
660
|
+
}
|
|
1409
661
|
if (retry !== void 0 && flags.resume !== void 0) throw new UsageError("--resume and --retry-errors both continue a run: pass one of them.");
|
|
1410
662
|
if (retry !== void 0 && (given.length || flags.tag?.length || flags.trait?.length)) throw new UsageError("--retry-errors runs every infra-errored attempt of the run again, with its plan: leave out filters, --tag and --trait.");
|
|
1411
|
-
const
|
|
663
|
+
const continued = flags.resume !== void 0 || retry !== void 0;
|
|
664
|
+
const resumed = continued ? await resumedRun(flags, given, configPath) : null;
|
|
1412
665
|
if (resumed?.nothing) {
|
|
1413
666
|
const { nothing, record } = resumed.nothing;
|
|
1414
667
|
const outcome = runOutcome(record);
|
|
1415
668
|
if (flags.json) {
|
|
1416
669
|
console.error(nothing);
|
|
1417
|
-
const { suiteResult } = await import("./suite-
|
|
670
|
+
const { suiteResult } = await import("./suite-CH1aaAbs.js");
|
|
1418
671
|
process.stdout.write(`${JSON.stringify({
|
|
1419
672
|
...suiteResult(record),
|
|
1420
673
|
outcome
|
|
@@ -1432,40 +685,63 @@ async function runCommand(flags, configPath, given) {
|
|
|
1432
685
|
...resumed?.profileOf ? { profileOf: resumed.profileOf } : {}
|
|
1433
686
|
});
|
|
1434
687
|
const explicit = runFlags.concurrency ?? (process.env.REEVAL_CONCURRENCY || void 0);
|
|
688
|
+
const ctx = await startRun(setup);
|
|
1435
689
|
const startedWith = resumed ? resumed.saved ?? void 0 : {
|
|
1436
690
|
...setup.runs === void 0 ? {} : { runs: setup.runs },
|
|
1437
691
|
...setup.profile ? { profile: setup.profile.name } : {},
|
|
1438
|
-
...explicit === void 0 ? {} : { concurrency:
|
|
692
|
+
...explicit === void 0 ? {} : { concurrency: ctx.concurrency },
|
|
1439
693
|
filters: savedFilters(setup.config.root, filters),
|
|
1440
694
|
tags: runFlags.tag ?? [],
|
|
1441
695
|
...runFlags.trait?.length ? { traits: runFlags.trait } : {}
|
|
1442
696
|
};
|
|
1443
|
-
const ctx = await startRun(setup);
|
|
1444
697
|
invocation.runId = ctx.runId;
|
|
1445
698
|
const githubOut = process.env.GITHUB_OUTPUT;
|
|
1446
699
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1447
700
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1448
701
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1449
|
-
const { engineVersions } = await import("./node-
|
|
702
|
+
const { engineVersions } = await import("./node-BjuhqQwa.js");
|
|
1450
703
|
const terminal = setup.json && !setup.dryRun ? process.stderr : process.stdout;
|
|
1451
|
-
const
|
|
704
|
+
const harness = setup.config.harness[0]?.kind ?? "agent";
|
|
705
|
+
const drawn = createRenderer({
|
|
1452
706
|
out: setup.json && setup.dryRun ? { write: () => true } : terminal,
|
|
1453
707
|
columns: terminal.columns,
|
|
1454
708
|
colorDepth: terminal.isTTY ? terminal.getColorDepth() : void 0,
|
|
1455
709
|
output: setup.output,
|
|
1456
710
|
verbosity: ctx.verbosity ?? 0,
|
|
1457
|
-
harness
|
|
711
|
+
harness,
|
|
1458
712
|
tty: {
|
|
1459
713
|
stdout: process.stdout,
|
|
1460
714
|
stdin: process.stdin
|
|
1461
715
|
}
|
|
1462
716
|
});
|
|
717
|
+
const root = setup.config.root;
|
|
718
|
+
const eventsTo = eventsFileFlag !== void 0 ? resolve(eventsFileFlag) : flags.events ? eventsFile(root, ctx.runId) : resumed?.saved?.events ? resolve(root, resumed.saved.events) : void 0;
|
|
719
|
+
const savedEvents = eventsTo ? relativeFile(root, eventsTo) : void 0;
|
|
720
|
+
if (startedWith && savedEvents) startedWith.events = savedEvents;
|
|
721
|
+
const eventsUrl = flags["events-url"];
|
|
722
|
+
if (eventsTo || eventsUrl) eventStream = openEventStream({
|
|
723
|
+
...eventsTo ? {
|
|
724
|
+
file: eventsTo,
|
|
725
|
+
append: continued
|
|
726
|
+
} : {},
|
|
727
|
+
...eventsUrl ? { url: eventsUrl } : {},
|
|
728
|
+
header: {
|
|
729
|
+
reeval: version,
|
|
730
|
+
runId: ctx.runId,
|
|
731
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
732
|
+
harness,
|
|
733
|
+
verbosity: ctx.verbosity ?? 0
|
|
734
|
+
},
|
|
735
|
+
warn: (message) => process.stderr.write(yellow(`reeval: ${message}\n`))
|
|
736
|
+
});
|
|
737
|
+
const renderer = eventStream ? teeRenderer(drawn, eventStream) : drawn;
|
|
1463
738
|
let executed;
|
|
1464
739
|
try {
|
|
1465
740
|
executed = await executeRun(ctx, setup, filters, renderer, {
|
|
1466
741
|
startedWith,
|
|
1467
742
|
notes: resumed?.notes ?? [],
|
|
1468
|
-
narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length || !!flags.trait?.length)
|
|
743
|
+
narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length || !!flags.trait?.length),
|
|
744
|
+
...savedEvents ? { events: savedEvents } : {}
|
|
1469
745
|
});
|
|
1470
746
|
} catch (error) {
|
|
1471
747
|
renderer.close();
|
|
@@ -1486,7 +762,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1486
762
|
return EXIT.ok;
|
|
1487
763
|
}
|
|
1488
764
|
if (ctx.retry && ctx.previous && retried.size) {
|
|
1489
|
-
const { retryEnvironmentChanges } = await import("./retry-
|
|
765
|
+
const { retryEnvironmentChanges } = await import("./retry-Bfk7pDti.js");
|
|
1490
766
|
const changes = retryEnvironmentChanges(ctx.previous, ctx.stagingDir, retried);
|
|
1491
767
|
if (changes.length) {
|
|
1492
768
|
questions?.cancel();
|
|
@@ -1505,8 +781,8 @@ async function runCommand(flags, configPath, given) {
|
|
|
1505
781
|
}
|
|
1506
782
|
finishing = true;
|
|
1507
783
|
live?.phase("finishing");
|
|
1508
|
-
const { lockfiles } = await import("./state-
|
|
1509
|
-
const { writeScratchRun } = await import("./results-
|
|
784
|
+
const { lockfiles } = await import("./state-CKIjZZRK.js");
|
|
785
|
+
const { writeScratchRun } = await import("./results-DhMgRkR0.js");
|
|
1510
786
|
const run = await writeScratchRun({
|
|
1511
787
|
ctx,
|
|
1512
788
|
author: ctx.author,
|
|
@@ -1539,8 +815,8 @@ async function runCommand(flags, configPath, given) {
|
|
|
1539
815
|
...record,
|
|
1540
816
|
status: "complete"
|
|
1541
817
|
}) : outcome;
|
|
1542
|
-
|
|
1543
|
-
const { pruneDebug } = await import("./files-
|
|
818
|
+
let code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
819
|
+
const { pruneDebug } = await import("./files-BjOJw4sn.js");
|
|
1544
820
|
await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
|
|
1545
821
|
const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
|
|
1546
822
|
renderer.onEvent({
|
|
@@ -1554,7 +830,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1554
830
|
});
|
|
1555
831
|
renderer.close();
|
|
1556
832
|
if (retried.size) {
|
|
1557
|
-
const { retryCounts, retryLine } = await import("./retry-
|
|
833
|
+
const { retryCounts, retryLine } = await import("./retry-Bfk7pDti.js");
|
|
1558
834
|
const line = retryLine(retryCounts(record, retried, ctx.previous));
|
|
1559
835
|
if (setup.json) console.error(line);
|
|
1560
836
|
else console.log(line);
|
|
@@ -1563,15 +839,22 @@ async function runCommand(flags, configPath, given) {
|
|
|
1563
839
|
if (stepSummary && run.artifactDir) await appendFile(stepSummary, await readFile(summary)).catch(() => {});
|
|
1564
840
|
const { config } = ctx;
|
|
1565
841
|
const mode = setup.record;
|
|
1566
|
-
const { recordFile } = await import("./files-
|
|
842
|
+
const { readRecord, recordFile } = await import("./files-CAzVG3Rr.js");
|
|
1567
843
|
const rerecord = (setup.retry !== void 0 || setup.resume !== void 0) && record.status === "complete" && existsSync(recordFile(config.history.dir, record.runId));
|
|
1568
844
|
const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI() || rerecord;
|
|
1569
845
|
if (autoRecord && record.status !== "interrupted") {
|
|
1570
|
-
const { record: recordCmd } = await import("./commands-
|
|
846
|
+
const { record: recordCmd } = await import("./commands-Dqli45sR.js");
|
|
1571
847
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
848
|
+
const level = rerecord ? { artifacts: (await readRecord(recordFile(config.history.dir, record.runId))).artifacts } : {};
|
|
1572
849
|
const line = await recordCmd(config, record.runId, {
|
|
1573
850
|
...flags,
|
|
1574
|
-
...allow
|
|
851
|
+
...allow,
|
|
852
|
+
...level
|
|
853
|
+
}).catch((error) => {
|
|
854
|
+
if (error.name !== "StoreError") throw error;
|
|
855
|
+
if (code !== EXIT.interrupted && code !== EXIT.terminated) code = EXIT.infra;
|
|
856
|
+
invocation.machineState = true;
|
|
857
|
+
return red(`reeval: ${redactSecrets(error.message)}`);
|
|
1575
858
|
});
|
|
1576
859
|
if (setup.json) console.error(line);
|
|
1577
860
|
else console.log(line);
|
|
@@ -1581,7 +864,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1581
864
|
else console.log(line);
|
|
1582
865
|
}
|
|
1583
866
|
if (setup.json) {
|
|
1584
|
-
const { suiteResult } = await import("./suite-
|
|
867
|
+
const { suiteResult } = await import("./suite-CH1aaAbs.js");
|
|
1585
868
|
process.stdout.write(`${JSON.stringify({
|
|
1586
869
|
...suiteResult(record),
|
|
1587
870
|
outcome: ran
|
|
@@ -1661,20 +944,28 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
|
|
|
1661
944
|
stdinTTY: !!process.stdin.isTTY && !flags.json && !flags["dry-run"]
|
|
1662
945
|
});
|
|
1663
946
|
const path = requireConfig(configPath);
|
|
1664
|
-
|
|
947
|
+
const tracked = !!(flags["dry-run"] && flags.json) ? (await import("./inputs-load-C82HOKnb.js")).trackLoad(dirname(path)) : void 0;
|
|
948
|
+
let configInputs = [];
|
|
949
|
+
let loaded;
|
|
1665
950
|
const named = noProfile ? void 0 : flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
951
|
+
try {
|
|
952
|
+
loadEnvFiles(path);
|
|
953
|
+
loaded = await loadConfig(path, "run", named);
|
|
954
|
+
} finally {
|
|
955
|
+
if (tracked) configInputs = tracked.stop(path, "config");
|
|
956
|
+
}
|
|
957
|
+
if (profileOf && flags.profile && !loaded.profiles[flags.profile]) throw new UsageError(`run ${profileOf} was started with profile ${flags.profile}, which ${loaded.file} no longer defines. Put it back to resume the run, or start a new run without --resume.`);
|
|
958
|
+
const profile = noProfile ? null : selectProfile(loaded, flags.profile);
|
|
959
|
+
const config = withProfileHistory(loaded, profile);
|
|
960
|
+
if ((flags.record || (profile?.record ?? config.record) === "always" || (profile?.record ?? config.record) === "ci" && inCI()) && !flags["dry-run"]) storeCredentials(config.history);
|
|
1669
961
|
let runs;
|
|
1670
962
|
if (flags.runs !== void 0) {
|
|
1671
963
|
runs = Number(flags.runs);
|
|
1672
964
|
if (!(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`--runs must be a whole number of at least 1 (got ${flags.runs})`);
|
|
1673
965
|
}
|
|
1674
966
|
const sandboxes = [...(config.environments.length ? config.environments : [void 0]).map((e) => environmentSandbox(e, config.sandbox)), ...profile ? [profileSandbox(config, profile.defaults)] : []];
|
|
1675
|
-
const concurrency =
|
|
1676
|
-
|
|
1677
|
-
});
|
|
967
|
+
const concurrency = givenConcurrency(config, flags.concurrency);
|
|
968
|
+
const registry = await registryFlag(flags);
|
|
1678
969
|
return {
|
|
1679
970
|
output,
|
|
1680
971
|
config,
|
|
@@ -1682,12 +973,23 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
|
|
|
1682
973
|
runs,
|
|
1683
974
|
tags: [...flags.tag ?? [], ...profile?.tags ?? []],
|
|
1684
975
|
traits: traitFlags(flags.trait),
|
|
1685
|
-
|
|
976
|
+
sandboxes,
|
|
977
|
+
/**
|
|
978
|
+
* The run's concurrency. Without one given, it's sized from where the first sandbox that
|
|
979
|
+
* runs off the host runs (the Docker or Podman VM, the Boat account) when it answers, as a
|
|
980
|
+
* real run is, so --dry-run shows the same; a resume reads it once the stopped run's
|
|
981
|
+
* sandboxes are gone, since they count against it.
|
|
982
|
+
*/
|
|
983
|
+
sizeRun: async () => concurrency ?? defaultConcurrency(await sandboxCapacity(sandboxes) ?? hostCapacity()),
|
|
1686
984
|
budget: profile?.budget ?? config.budget,
|
|
1687
985
|
record: profile?.record ?? config.record,
|
|
1688
986
|
dryRun: !!flags["dry-run"],
|
|
1689
987
|
json: !!flags.json,
|
|
988
|
+
/** What loading the config read, for `--dry-run --json`'s `inputs`. */
|
|
989
|
+
configInputs,
|
|
1690
990
|
force: !!flags.force,
|
|
991
|
+
/** `--registry`: where a missing sandbox image is pulled from before it's built. */
|
|
992
|
+
registry,
|
|
1691
993
|
verbosity: verbosityOf(flags),
|
|
1692
994
|
configFile: configLabel(path),
|
|
1693
995
|
/** Named in the plan line when the project root isn't the current directory. */
|
|
@@ -1698,17 +1000,26 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
|
|
|
1698
1000
|
retry: flags["retry-errors"]
|
|
1699
1001
|
};
|
|
1700
1002
|
}
|
|
1003
|
+
/** `--registry`, checked, or null; `--push` needs it. */
|
|
1004
|
+
async function registryFlag(flags) {
|
|
1005
|
+
if (flags.push && flags.registry === void 0) throw new UsageError("--push needs --registry, the registry to push to, e.g. --registry ghcr.io/acme/reeval");
|
|
1006
|
+
if (flags.registry === void 0) return null;
|
|
1007
|
+
const { registryProblem } = await import("./src-jYqNkT-F.js");
|
|
1008
|
+
const problem = registryProblem(flags.registry);
|
|
1009
|
+
if (problem) throw new UsageError(problem);
|
|
1010
|
+
return flags.registry.replace(/\/+$/, "");
|
|
1011
|
+
}
|
|
1701
1012
|
/** What a retry, or a resume that finishes one, does to the run's errors, for its refusals. */
|
|
1702
1013
|
const retryVerb = (retry, runId) => retry.resume ? `--resume finishes run ${runId}'s --retry-errors, which runs its errors again` : "--retry-errors runs its errors again";
|
|
1703
1014
|
/** The run's id and the state it starts from: commit, uncommitted changes, author. */
|
|
1704
|
-
async function startRun({ config, profile, runs, tags, traits,
|
|
1015
|
+
async function startRun({ config, profile, runs, tags, traits, sandboxes, sizeRun, verbosity, configFile, resume, retry, dryRun }) {
|
|
1705
1016
|
const startedAt = /* @__PURE__ */ new Date();
|
|
1706
1017
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
1707
|
-
const { worktreePatch } = await import("./state-
|
|
1018
|
+
const { worktreePatch } = await import("./state-CKIjZZRK.js");
|
|
1708
1019
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
1709
1020
|
const who = await author(config.root);
|
|
1710
|
-
const { stagingDir
|
|
1711
|
-
const { scratchDir } = await import("./files-
|
|
1021
|
+
const { stagingDir } = await import("./results-DhMgRkR0.js");
|
|
1022
|
+
const { scratchDir } = await import("./files-CAzVG3Rr.js");
|
|
1712
1023
|
const scratch = scratchDir(config.root);
|
|
1713
1024
|
if (resume !== void 0 || retry !== void 0) {
|
|
1714
1025
|
const target = await continuedRun(config.root, {
|
|
@@ -1718,15 +1029,18 @@ async function startRun({ config, profile, runs, tags, traits, concurrency, verb
|
|
|
1718
1029
|
if (!dryRun && retry === void 0) await prepareResume(target, config.root);
|
|
1719
1030
|
else if (!dryRun) holdRetry(target, config.root);
|
|
1720
1031
|
const stopped = retry === void 0 ? await stoppedRetry(config.root, target) : null;
|
|
1721
|
-
await
|
|
1032
|
+
if (!dryRun) await sweep(config.root, target.runId);
|
|
1722
1033
|
const aside = dryRun ? {
|
|
1723
1034
|
record: target.previous,
|
|
1724
1035
|
artifacts: target.previousArtifacts
|
|
1725
1036
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
1726
|
-
if (!dryRun)
|
|
1727
|
-
await cleanupRun(target.runId);
|
|
1728
|
-
|
|
1729
|
-
|
|
1037
|
+
if (!dryRun) {
|
|
1038
|
+
await cleanupRun(target.runId).catch(() => {});
|
|
1039
|
+
const scoped = sandboxes.filter((s) => sandboxKind(s).cleanupScope?.(s)?.atStart);
|
|
1040
|
+
if (scoped.length) await cleanupRun(target.runId, scoped).catch(() => {});
|
|
1041
|
+
}
|
|
1042
|
+
const concurrency = await sizeRun();
|
|
1043
|
+
const { unfoldCases } = await import("./cases-FvhbM8M5.js");
|
|
1730
1044
|
const previous = aside.record ?? target.previous;
|
|
1731
1045
|
return {
|
|
1732
1046
|
runId: target.runId,
|
|
@@ -1760,7 +1074,8 @@ async function startRun({ config, profile, runs, tags, traits, concurrency, verb
|
|
|
1760
1074
|
const stamps = new Set([scratch, config.history.dir].flatMap((dir) => existsSync(dir) ? readdirSync(dir).map((name) => name.slice(0, 16)) : []));
|
|
1761
1075
|
let runId = newRunId(startedAt, git?.sha ?? null, who);
|
|
1762
1076
|
for (let n = 1; stamps.has(runId.slice(0, 16)); n++) runId = newRunId(new Date(startedAt.getTime() + n * 1e3), git?.sha ?? null, who);
|
|
1763
|
-
await
|
|
1077
|
+
if (!dryRun) await sweep(config.root);
|
|
1078
|
+
const concurrency = await sizeRun();
|
|
1764
1079
|
return {
|
|
1765
1080
|
runId,
|
|
1766
1081
|
stagingDir: stagingDir(config.root, runId),
|
|
@@ -1819,12 +1134,12 @@ function namesDeclaredSecret(evals) {
|
|
|
1819
1134
|
*/
|
|
1820
1135
|
async function planEfforts(planned) {
|
|
1821
1136
|
const live = planned.filter((p) => !p.attempt.plan.skip);
|
|
1822
|
-
const { effortNotes, withDefaultEffort } = await import("./default-effort-
|
|
1137
|
+
const { effortNotes, withDefaultEffort } = await import("./default-effort-DHzj84dD.js");
|
|
1823
1138
|
let cursor;
|
|
1824
1139
|
let unchecked = "CURSOR_API_KEY isn't set";
|
|
1825
1140
|
const key = process.env.CURSOR_API_KEY;
|
|
1826
1141
|
if (key && live.some((p) => p.attempt.harness.kind === "cursor")) {
|
|
1827
|
-
const { checkCursorPlan } = await import("./cursor-catalog-
|
|
1142
|
+
const { checkCursorPlan } = await import("./cursor-catalog-CRsFxorF.js");
|
|
1828
1143
|
const checked = await checkCursorPlan(key, live);
|
|
1829
1144
|
if (checked instanceof Error) unchecked = `its request failed: ${checked.message}`;
|
|
1830
1145
|
else {
|
|
@@ -1845,6 +1160,7 @@ async function planEfforts(planned) {
|
|
|
1845
1160
|
* or Docker problem (exit 2). Then runs the engine with live output and Ctrl-C handling.
|
|
1846
1161
|
*/
|
|
1847
1162
|
async function executeRun(ctx, setup, filters, renderer, started) {
|
|
1163
|
+
const { events } = started;
|
|
1848
1164
|
const { runId, stagingDir, config, runs, tags, traits, concurrency, profile, verbosity } = ctx;
|
|
1849
1165
|
const { configFile } = ctx;
|
|
1850
1166
|
const run = JSON.parse(JSON.stringify({
|
|
@@ -1858,10 +1174,11 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1858
1174
|
profile,
|
|
1859
1175
|
verbosity,
|
|
1860
1176
|
configFile,
|
|
1861
|
-
completed: []
|
|
1177
|
+
completed: [],
|
|
1178
|
+
...setup.dryRun && setup.json ? { inputs: true } : {}
|
|
1862
1179
|
}));
|
|
1863
|
-
const { runEngine } = await import("./node-
|
|
1864
|
-
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-
|
|
1180
|
+
const { runEngine } = await import("./node-BjuhqQwa.js");
|
|
1181
|
+
const { buildPlan, checkAttemptBudget, planDocument, planInputs, planLine, planTable, runHeader } = await import("./plan-BcT4G5jj.js");
|
|
1865
1182
|
let plan;
|
|
1866
1183
|
let sandboxVersions = { images: [] };
|
|
1867
1184
|
let preflightStop;
|
|
@@ -1906,9 +1223,9 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1906
1223
|
if (ctx.retry && ctx.previous) {
|
|
1907
1224
|
const ranOn = ctx.previous.stack?.reeval;
|
|
1908
1225
|
if (ranOn && ranOn !== version) throw new UsageError(`run ${runId} ran on reeval ${ranOn}, and this is ${version}. ${retryVerb(ctx.retry, runId)} only on the reeval they ran on: install reeval ${ranOn}, or start a new run with reeval run.`);
|
|
1909
|
-
const { retryChanges } = await import("./retry-
|
|
1910
|
-
const { judgePart } = await import("./assemble-
|
|
1911
|
-
const { engineVersions } = await import("./node-
|
|
1226
|
+
const { retryChanges } = await import("./retry-Bfk7pDti.js");
|
|
1227
|
+
const { judgePart } = await import("./assemble-CBMRIhso.js");
|
|
1228
|
+
const { engineVersions } = await import("./node-BjuhqQwa.js");
|
|
1912
1229
|
const judged = judgePart(config, profile, version, engineVersions);
|
|
1913
1230
|
const runs = new Set(planned.map((p) => p.key).filter((key) => !completed.has(key)));
|
|
1914
1231
|
const changes = await retryChanges(ctx.previous, planned, runs, config.root, judged);
|
|
@@ -1921,7 +1238,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1921
1238
|
}
|
|
1922
1239
|
}
|
|
1923
1240
|
retried = continuing && !setup.dryRun ? noteRetries(planned, stagingDir, ctx.previous, true).errored : errored;
|
|
1924
|
-
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-
|
|
1241
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-B1tdv9-d.js") : void 0;
|
|
1925
1242
|
const judge = runJudgeModel(config, profile);
|
|
1926
1243
|
const judgeModel = judge === void 0 ? null : judgeModelName(judge);
|
|
1927
1244
|
plan = await buildPlan({
|
|
@@ -1938,8 +1255,11 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1938
1255
|
});
|
|
1939
1256
|
plan.runId = runId;
|
|
1940
1257
|
const cloud = ciRunner(process.env);
|
|
1941
|
-
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
|
|
1258
|
+
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy) || isDirect(setupEgress(p.attempt.egress).policy)).map((p) => p.id))] : [];
|
|
1942
1259
|
if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
|
|
1260
|
+
const switching = concurrency > 1 ? [...new Set(skipped.filter(({ attempt }) => !attempt.plan.skip && !!attempt.egress.setup && attempt.sandbox.kind === "docker" && attempt.sandbox.engine === "podman").map((p) => p.id))] : [];
|
|
1261
|
+
const { podmanRootless } = await import("./providers-BkLGzaSY.js");
|
|
1262
|
+
if (switching.length && await podmanRootless()) plan.warnings.push(rootlessPodmanSwitch(switching));
|
|
1943
1263
|
plan.warnings.push(...started.notes);
|
|
1944
1264
|
if (started.narrowed) {
|
|
1945
1265
|
leftOut = leftOutAttempts(planned, stagingDir, ctx.previous, await liveAttempts(stagingDir), new Set(excluded.map((e) => e.id)));
|
|
@@ -1954,7 +1274,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1954
1274
|
run.completed = [...completed];
|
|
1955
1275
|
const done = leftOut.filter((l) => l.done).map((l) => l.key);
|
|
1956
1276
|
kept = keptAttempts([...completed, ...done], stagingDir, ctx.previous);
|
|
1957
|
-
const { imageStages, imageTargets } = await import("./images-
|
|
1277
|
+
const { imageStages, imageTargets } = await import("./images-Ch1y1t6_.js");
|
|
1958
1278
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1959
1279
|
id: p.attempt.family?.id ?? p.id,
|
|
1960
1280
|
harness: p.attempt.harness,
|
|
@@ -1968,18 +1288,33 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1968
1288
|
evals: t.evals,
|
|
1969
1289
|
stages: await imageStages(t, config.root)
|
|
1970
1290
|
})));
|
|
1971
|
-
const
|
|
1291
|
+
const { lockfiles } = await import("./state-CKIjZZRK.js");
|
|
1292
|
+
const locks = (await lockfiles(config.root)).map(({ path }) => ({
|
|
1293
|
+
path: join(config.root, path),
|
|
1294
|
+
kind: "lockfile"
|
|
1295
|
+
}));
|
|
1296
|
+
const inputs = planInputs(config.root, [...setup.configInputs, ...locks], skipped);
|
|
1297
|
+
const running = skipped.filter((p) => !p.attempt.plan.skip);
|
|
1298
|
+
const secrets = new Set(running.flatMap((p) => p.attempt.secretVars));
|
|
1299
|
+
const needed = neededCredentials(config, {
|
|
1300
|
+
harnesses: plan.harnessDescriptors,
|
|
1301
|
+
sandboxes: plan.sandboxes,
|
|
1302
|
+
profile,
|
|
1303
|
+
judges: running.flatMap((p) => p.attempt.judge ? [p.attempt.judge] : [])
|
|
1304
|
+
});
|
|
1305
|
+
for (const { names } of needed) for (const name of names) secrets.add(name);
|
|
1306
|
+
const doc = planDocument(plan, skipped, excluded, images, inputs, [...secrets].sort());
|
|
1972
1307
|
process.stdout.write(`${JSON.stringify(doc, null, 2)}\n`);
|
|
1973
1308
|
return false;
|
|
1974
1309
|
}
|
|
1975
1310
|
const elsewhere = setup.configElsewhere ? configFile : void 0;
|
|
1976
|
-
const { scratchDir } = await import("./files-
|
|
1311
|
+
const { scratchDir } = await import("./files-CAzVG3Rr.js");
|
|
1977
1312
|
const lines = [];
|
|
1978
1313
|
if (setup.dryRun) {
|
|
1979
1314
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
1980
1315
|
lines.push(...planTable(plan, todos));
|
|
1981
|
-
const { imageStatusLines } = await import("./images-
|
|
1982
|
-
const { dockerAvailable, engineSpec } = await import("./src-
|
|
1316
|
+
const { imageStatusLines } = await import("./images-Ch1y1t6_.js");
|
|
1317
|
+
const { dockerAvailable, engineSpec } = await import("./src-jYqNkT-F.js");
|
|
1983
1318
|
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
1984
1319
|
const down = [];
|
|
1985
1320
|
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
@@ -1988,7 +1323,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1988
1323
|
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
1989
1324
|
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
1990
1325
|
}
|
|
1991
|
-
const { costLine } = await import("./cost-
|
|
1326
|
+
const { costLine } = await import("./cost-IM80LyGD.js");
|
|
1992
1327
|
const cost = plan.cost && costLine(plan.cost);
|
|
1993
1328
|
if (cost) lines.push(cost);
|
|
1994
1329
|
}
|
|
@@ -2011,6 +1346,15 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2011
1346
|
sandboxWarnings: plan.sandboxWarnings
|
|
2012
1347
|
});
|
|
2013
1348
|
if (setup.dryRun) return false;
|
|
1349
|
+
if (continuing) {
|
|
1350
|
+
const away = /* @__PURE__ */ new Map();
|
|
1351
|
+
for (const { attempt } of planned) {
|
|
1352
|
+
const { sandbox } = attempt;
|
|
1353
|
+
const scope = sandboxKind(sandbox).cleanupScope?.(sandbox);
|
|
1354
|
+
if (scope) away.set(`${sandbox.kind}\0${scope.key}`, sandbox);
|
|
1355
|
+
}
|
|
1356
|
+
if (away.size) await cleanupRun(runId, [...away.values()]).catch(() => {});
|
|
1357
|
+
}
|
|
2014
1358
|
checkAttemptBudget(plan, setup.force, config.file);
|
|
2015
1359
|
const questionModels = /* @__PURE__ */ new Set();
|
|
2016
1360
|
const asked = [];
|
|
@@ -2063,7 +1407,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2063
1407
|
});
|
|
2064
1408
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
2065
1409
|
if (asked.length) {
|
|
2066
|
-
const { questionRunner } = await import("./questions-
|
|
1410
|
+
const { questionRunner } = await import("./questions-Csv64WCb.js");
|
|
2067
1411
|
const warned = /* @__PURE__ */ new Set();
|
|
2068
1412
|
questions = questionRunner({
|
|
2069
1413
|
evals: asked,
|
|
@@ -2105,7 +1449,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2105
1449
|
}
|
|
2106
1450
|
});
|
|
2107
1451
|
}
|
|
2108
|
-
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-
|
|
1452
|
+
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-CnRbV8cM.js");
|
|
2109
1453
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
2110
1454
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
2111
1455
|
workspace: a.workspace,
|
|
@@ -2170,7 +1514,8 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2170
1514
|
type: "prepare",
|
|
2171
1515
|
at: Date.now(),
|
|
2172
1516
|
...e
|
|
2173
|
-
})
|
|
1517
|
+
}),
|
|
1518
|
+
...setup.registry ? { registry: setup.registry } : {}
|
|
2174
1519
|
});
|
|
2175
1520
|
} catch (error) {
|
|
2176
1521
|
if (error.name !== "PreflightInfraError") throw error;
|
|
@@ -2197,11 +1542,17 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2197
1542
|
};
|
|
2198
1543
|
let engineRun;
|
|
2199
1544
|
let cutShort = false;
|
|
1545
|
+
const { MACHINES, MachineService } = await import("./machine-service-2lIv5UWF.js");
|
|
1546
|
+
const machines = new MachineService();
|
|
2200
1547
|
try {
|
|
2201
1548
|
engineRun = await runEngine({
|
|
2202
1549
|
run,
|
|
2203
1550
|
filters,
|
|
2204
1551
|
onPlan,
|
|
1552
|
+
services: {
|
|
1553
|
+
call: (owner, service, body) => service === MACHINES ? machines.call(owner, body) : Promise.reject(/* @__PURE__ */ new Error(`This run has no ${service} service`)),
|
|
1554
|
+
released: (owner) => machines.released(owner)
|
|
1555
|
+
},
|
|
2205
1556
|
onRunStart: (files) => {
|
|
2206
1557
|
writeCurrent(setup.config.root, runId);
|
|
2207
1558
|
live = writeLiveProgress({
|
|
@@ -2211,7 +1562,8 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2211
1562
|
planned: toRun,
|
|
2212
1563
|
kept,
|
|
2213
1564
|
leftOut,
|
|
2214
|
-
...started.startedWith ? { invocation: started.startedWith } : {}
|
|
1565
|
+
...started.startedWith ? { invocation: started.startedWith } : {},
|
|
1566
|
+
...events ? { events } : {}
|
|
2215
1567
|
});
|
|
2216
1568
|
renderer.onEvent({
|
|
2217
1569
|
type: "run-start",
|
|
@@ -2238,7 +1590,15 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2238
1590
|
},
|
|
2239
1591
|
onCancelable: (cancel) => {
|
|
2240
1592
|
let first;
|
|
2241
|
-
const onSignal = (signal) => {
|
|
1593
|
+
const onSignal = (signal, attach) => {
|
|
1594
|
+
if (first?.attach && !attach && Date.now() - first.at < SAME_PRESS_MS) {
|
|
1595
|
+
stopCode = signal === "SIGTERM" ? EXIT.terminated : EXIT.interrupted;
|
|
1596
|
+
first = {
|
|
1597
|
+
signal,
|
|
1598
|
+
at: Date.now()
|
|
1599
|
+
};
|
|
1600
|
+
return;
|
|
1601
|
+
}
|
|
2242
1602
|
if (first && !first.passedOn && signal !== first.signal && Date.now() - first.at < SAME_PRESS_MS) {
|
|
2243
1603
|
if (signal === "SIGINT") stopCode = EXIT.interrupted;
|
|
2244
1604
|
if (!finishing) renderer.onEvent({
|
|
@@ -2260,7 +1620,8 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2260
1620
|
interrupted = true;
|
|
2261
1621
|
first = {
|
|
2262
1622
|
signal,
|
|
2263
|
-
at: Date.now()
|
|
1623
|
+
at: Date.now(),
|
|
1624
|
+
...attach ? { attach } : {}
|
|
2264
1625
|
};
|
|
2265
1626
|
if (finishing) return;
|
|
2266
1627
|
questions?.cancel();
|
|
@@ -2273,8 +1634,11 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2273
1634
|
});
|
|
2274
1635
|
cancel();
|
|
2275
1636
|
};
|
|
2276
|
-
process.on("SIGINT", onSignal);
|
|
2277
|
-
process.on("SIGTERM", onSignal);
|
|
1637
|
+
process.on("SIGINT", () => onSignal("SIGINT", false));
|
|
1638
|
+
process.on("SIGTERM", () => onSignal("SIGTERM", false));
|
|
1639
|
+
if (process.platform !== "win32") process.on(CANCEL_SIGNAL, () => {
|
|
1640
|
+
if (!interrupted && !finishing) onSignal("SIGINT", true);
|
|
1641
|
+
});
|
|
2278
1642
|
}
|
|
2279
1643
|
});
|
|
2280
1644
|
cutShort = interrupted;
|
|
@@ -2283,9 +1647,10 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2283
1647
|
throw error;
|
|
2284
1648
|
} finally {
|
|
2285
1649
|
clearTimeout(budgetTimer);
|
|
2286
|
-
|
|
1650
|
+
await machines.close();
|
|
1651
|
+
const { removePrepared } = await import("./preflight-zsAn3heN.js");
|
|
2287
1652
|
removePrepared(config.root, runId);
|
|
2288
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
1653
|
+
const { removePreparedWorkspaces } = await import("./workspace-CnRbV8cM.js");
|
|
2289
1654
|
removePreparedWorkspaces(config.root, runId);
|
|
2290
1655
|
}
|
|
2291
1656
|
if (preflightStop) throw preflightStop;
|
|
@@ -2313,6 +1678,7 @@ process.stdout.on("error", (error) => {
|
|
|
2313
1678
|
* error or an infra error), prints one line suggesting `reeval report`. Not for agents or JSON.
|
|
2314
1679
|
*/
|
|
2315
1680
|
function finish(code, error, kind) {
|
|
1681
|
+
eventStream?.end(code);
|
|
2316
1682
|
const flags = invocation.flags;
|
|
2317
1683
|
let root = process.cwd();
|
|
2318
1684
|
try {
|
|
@@ -2361,13 +1727,15 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
2361
1727
|
process.exitCode = code;
|
|
2362
1728
|
}, (error) => {
|
|
2363
1729
|
const name = error.name;
|
|
2364
|
-
const
|
|
1730
|
+
const infra = INFRA_ERRORS.includes(name);
|
|
1731
|
+
if (name === "StoreError") invocation.machineState = true;
|
|
1732
|
+
const expected = error instanceof UsageError || EXPECTED_ERRORS.includes(name) || infra;
|
|
2365
1733
|
console.error(red(`reeval: ${redactSecrets(error.message)}`));
|
|
2366
1734
|
if (!expected && process.env.REEVAL_DEBUG) console.error(redactSecrets(inspect(error)));
|
|
2367
|
-
const code =
|
|
1735
|
+
const code = infra ? EXIT.infra : EXIT.usage;
|
|
2368
1736
|
finish(code, error, error instanceof UsageError ? "usage" : expected ? "expected" : "crash");
|
|
2369
1737
|
process.exitCode = code;
|
|
2370
1738
|
});
|
|
2371
1739
|
export {};
|
|
2372
1740
|
|
|
2373
|
-
//# sourceMappingURL=cli-
|
|
1741
|
+
//# sourceMappingURL=cli-BgBBuxxJ.js.map
|