reeval 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -40
- package/THIRD_PARTY_NOTICES.md +218 -11
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +30 -30
- package/dist/adapter-DWlpOtRn.js +279 -0
- package/dist/adapter-DWlpOtRn.js.map +1 -0
- package/dist/{agent-B72lZ6w5.js → agent-CURAwx_q.js} +6 -6
- package/dist/{agent-B72lZ6w5.js.map → agent-CURAwx_q.js.map} +1 -1
- package/dist/{agent-Cfj1wD4D.js → agent-CaS_2Sce.js} +92 -26
- package/dist/agent-CaS_2Sce.js.map +1 -0
- package/dist/{analysis-BTNnKh1w.js → analysis-Bxub9LMT.js} +4 -4
- package/dist/{analysis-BTNnKh1w.js.map → analysis-Bxub9LMT.js.map} +1 -1
- package/dist/{assemble-BbFVjHeS.js → assemble-B9Alds-z.js} +290 -30
- package/dist/assemble-B9Alds-z.js.map +1 -0
- package/dist/assemble-J5MukZFY.js +1 -0
- package/dist/{client-CwzU9CiW.js → assert-EUIoRdIt.js} +38 -112
- package/dist/assert-EUIoRdIt.js.map +1 -0
- package/dist/bridge/index.mjs +248 -55
- package/dist/{capture-BXhBfmsM.js → capture-CNb9SuU5.js} +204 -39
- package/dist/capture-CNb9SuU5.js.map +1 -0
- package/dist/capture-jquXZBKK.js +1 -0
- package/dist/{cli-BMjTKY-D.js → cli-D303Vj5N.js} +257 -146
- package/dist/cli-D303Vj5N.js.map +1 -0
- package/dist/{cli-spec-DSvKsOK4.js → cli-spec-BPZbQG2p.js} +38 -19
- package/dist/cli-spec-BPZbQG2p.js.map +1 -0
- package/dist/cli.js +4 -2
- package/dist/cli.js.map +1 -1
- package/dist/client-fcwgwOAD.js +106 -0
- package/dist/client-fcwgwOAD.js.map +1 -0
- package/dist/{command-JPGNzFsA.js → command-BLfT7vnQ.js} +19 -18
- package/dist/command-BLfT7vnQ.js.map +1 -0
- package/dist/{command-BK4kOgkH.js → command-BT9a37ZR.js} +381 -78
- package/dist/command-BT9a37ZR.js.map +1 -0
- package/dist/{command-BqJtFiPy.js → command-Bge8F7Ko.js} +384 -101
- package/dist/command-Bge8F7Ko.js.map +1 -0
- package/dist/command-BkFjijN2.js +1 -0
- package/dist/{command-BdstSEHu.js → command-CMJtHKRa.js} +14 -14
- package/dist/{command-BdstSEHu.js.map → command-CMJtHKRa.js.map} +1 -1
- package/dist/commands-1p-bIWhN.js +1 -0
- package/dist/{commands-B618jg3C.js → commands-BFx16WB8.js} +8 -8
- package/dist/commands-BFx16WB8.js.map +1 -0
- package/dist/config-DEgnUiU-.js +756 -0
- package/dist/config-DEgnUiU-.js.map +1 -0
- package/dist/{context-CvUv7DUb.js → context-C1mxXh80.js} +11 -7
- package/dist/context-C1mxXh80.js.map +1 -0
- package/dist/{db-DFbtGIYy.js → db-CdHizc4_.js} +582 -53
- package/dist/db-CdHizc4_.js.map +1 -0
- package/dist/db-Dm1cRJJP.js +2 -0
- package/dist/debug-D7bEv_UV.js +1 -0
- package/dist/{dist-CTzThfu_.js → dist-B8hJImYt.js} +17 -4
- package/dist/dist-B8hJImYt.js.map +1 -0
- package/dist/{docker-2qns7hJ2.js → docker-7b_2SaH7.js} +35 -22
- package/dist/docker-7b_2SaH7.js.map +1 -0
- package/dist/{docker-IFRW8K5f.js → docker-sZrNrr82.js} +3 -3
- package/dist/docker-sZrNrr82.js.map +1 -0
- package/dist/docker-yauK-9X8.js +1 -0
- package/dist/{doctor-B6oYTPoR.js → doctor-BYNok4UO.js} +263 -84
- package/dist/doctor-BYNok4UO.js.map +1 -0
- package/dist/{duration-JsJrIWXq.js → duration-CEOQww97.js} +2 -2
- package/dist/{duration-JsJrIWXq.js.map → duration-CEOQww97.js.map} +1 -1
- package/dist/{egress-IUKdeayc.js → egress-C4PZTSSe.js} +147 -14
- package/dist/egress-C4PZTSSe.js.map +1 -0
- package/dist/environment-DNjRk0nU.js +1133 -0
- package/dist/environment-DNjRk0nU.js.map +1 -0
- package/dist/{errors-yDAEwmZo.js → errors-B6BSMs03.js} +2 -2
- package/dist/errors-B6BSMs03.js.map +1 -0
- package/dist/{errors-C9w_ud7g.js → errors-r025B0Zb.js} +4 -4
- package/dist/errors-r025B0Zb.js.map +1 -0
- package/dist/evaluate-C1UVL21l.d.ts +92 -0
- package/dist/evaluate-D4zeRGBF.js +2351 -0
- package/dist/evaluate-D4zeRGBF.js.map +1 -0
- package/dist/family-D2b7WD10.js +67 -0
- package/dist/family-D2b7WD10.js.map +1 -0
- package/dist/files-BAt0i00X.js +1 -0
- package/dist/{files-CMSUXAEg.js → files-BXUdY8tr.js} +10 -9
- package/dist/{files-CMSUXAEg.js.map → files-BXUdY8tr.js.map} +1 -1
- package/dist/files-BhyltlDB.js +1 -0
- package/dist/files-Bk1mC3AN.js +43 -0
- package/dist/files-Bk1mC3AN.js.map +1 -0
- package/dist/{files-B6aVVGrk.js → files-DQmHE7pb.js} +5 -5
- package/dist/{files-B6aVVGrk.js.map → files-DQmHE7pb.js.map} +1 -1
- package/dist/files-ZZPqfAo6.js +1 -0
- package/dist/{git-EmdFJ4DY.js → git-DfG-YJ1u.js} +2 -2
- package/dist/{git-EmdFJ4DY.js.map → git-DfG-YJ1u.js.map} +1 -1
- package/dist/harness-B8QC9u0O.js +323 -0
- package/dist/harness-B8QC9u0O.js.map +1 -0
- package/dist/harness.d.ts +19 -31
- package/dist/harness.js +1 -2
- package/dist/{header-_F0EWob3.js → header-hK-Z_Po9.js} +10 -3
- package/dist/{header-_F0EWob3.js.map → header-hK-Z_Po9.js.map} +1 -1
- package/dist/{images-CscddrJU.js → images-0On0cztu.js} +114 -45
- package/dist/images-0On0cztu.js.map +1 -0
- package/dist/index.d.ts +10 -26
- package/dist/index.js +6 -2108
- package/dist/{init-NQfIvi-0.js → init-B77Rpcqc.js} +26 -8
- package/dist/init-B77Rpcqc.js.map +1 -0
- package/dist/init-BrOWin3W.js +1 -0
- package/dist/{invocations-nPvrlZPj.js → invocations-DbGfBAA_.js} +4 -4
- package/dist/{invocations-nPvrlZPj.js.map → invocations-DbGfBAA_.js.map} +1 -1
- package/dist/{judge-CeYPN8P_.js → judge-DW_GDOmB.js} +11 -19
- package/dist/{judge-CeYPN8P_.js.map → judge-DW_GDOmB.js.map} +1 -1
- package/dist/judge.d.ts +2 -2
- package/dist/judge.js +1 -1
- package/dist/{live-S0qODb6o.js → live-jlfszAbb.js} +9 -20
- package/dist/live-jlfszAbb.js.map +1 -0
- package/dist/matchers-Ddmc2z7H.js +118 -0
- package/dist/matchers-Ddmc2z7H.js.map +1 -0
- package/dist/{model-DEQo3FkZ.js → model-DaVCY3rb.js} +2 -2
- package/dist/{model-DEQo3FkZ.js.map → model-DaVCY3rb.js.map} +1 -1
- package/dist/model-NM3ID0uR.js +1 -0
- package/dist/{model-Cl507trk.js → model-h_cDtcRr.js} +19 -18
- package/dist/model-h_cDtcRr.js.map +1 -0
- package/dist/model-nwWJvaUR.js +1 -0
- package/dist/{model-BndnmcLm.js → model-oqcDplJG.js} +271 -34
- package/dist/model-oqcDplJG.js.map +1 -0
- package/dist/{node-Co7EXge0.js → node-B3ydHo6t.js} +10 -47
- package/dist/node-B3ydHo6t.js.map +1 -0
- package/dist/object-CLf2-tNq.js +9 -0
- package/dist/object-CLf2-tNq.js.map +1 -0
- package/dist/{opencode-providers-ByD85Q0l.js → opencode-providers-J9colP0-.js} +96 -21
- package/dist/opencode-providers-J9colP0-.js.map +1 -0
- package/dist/options-Bh2W63dK.js +1119 -0
- package/dist/options-Bh2W63dK.js.map +1 -0
- package/dist/{pairs-DDsjgm5F.js → pairs-CnInRu-G.js} +76 -2
- package/dist/{pairs-DDsjgm5F.js.map → pairs-CnInRu-G.js.map} +1 -1
- package/dist/{patch-CGCVipPu.js → patch-Zac5aIRM.js} +6 -6
- package/dist/{patch-CGCVipPu.js.map → patch-Zac5aIRM.js.map} +1 -1
- package/dist/{plan-PPc4-6xd.js → plan-C9Cy27Ay.js} +75 -32
- package/dist/plan-C9Cy27Ay.js.map +1 -0
- package/dist/preflight-CPWpQeBI.js +1 -0
- package/dist/{preflight-CddycK92.js → preflight-D7iFPKeK.js} +8 -7
- package/dist/preflight-D7iFPKeK.js.map +1 -0
- package/dist/prepare-BGjePFQl.js +2 -0
- package/dist/{prepare-CsP-057p.js → prepare-doGxaAGR.js} +38 -30
- package/dist/prepare-doGxaAGR.js.map +1 -0
- package/dist/process-CSVO_jsc.js +119 -0
- package/dist/process-CSVO_jsc.js.map +1 -0
- package/dist/{provenance-CsX6PHDR.js → provenance-DmE9uPVt.js} +4 -4
- package/dist/{provenance-CsX6PHDR.js.map → provenance-DmE9uPVt.js.map} +1 -1
- package/dist/provider-DehR4CMN.js +1 -0
- package/dist/providers-gDMko8fl.js +1 -0
- package/dist/providers-tAOD1NQN.js +165 -0
- package/dist/providers-tAOD1NQN.js.map +1 -0
- package/dist/{query-BUcwCYsI.js → query-Bes9Kh6Y.js} +3 -3
- package/dist/{query-BUcwCYsI.js.map → query-Bes9Kh6Y.js.map} +1 -1
- package/dist/query-C_G_svLU.js +1 -0
- package/dist/questions-DjAwMxZ3.js +524 -0
- package/dist/questions-DjAwMxZ3.js.map +1 -0
- package/dist/{record-BRP9vZrj.js → record-BKmMvZ6O.js} +9 -1
- package/dist/{record-BRP9vZrj.js.map → record-BKmMvZ6O.js.map} +1 -1
- package/dist/remote-HDRt16ew.js +378 -0
- package/dist/remote-HDRt16ew.js.map +1 -0
- package/dist/{results-BcSHWbgB.js → results-DuauQI3Z.js} +4 -4
- package/dist/{results-BcSHWbgB.js.map → results-DuauQI3Z.js.map} +1 -1
- package/dist/{resume-DZ9AmfPv.js → resume-BRfa8E_s.js} +10 -10
- package/dist/{resume-DZ9AmfPv.js.map → resume-BRfa8E_s.js.map} +1 -1
- package/dist/{root-CvdRlRZX.js → root-DICDHCtp.js} +4 -4
- package/dist/{root-CvdRlRZX.js.map → root-DICDHCtp.js.map} +1 -1
- package/dist/{runtime-Dfb2r-cS.js → runtime-PX_oXiCS.js} +105 -11
- package/dist/runtime-PX_oXiCS.js.map +1 -0
- package/dist/sandbox-D3A8oUb7.js +70 -0
- package/dist/sandbox-D3A8oUb7.js.map +1 -0
- package/dist/sandbox.d.ts +22 -8
- package/dist/sandbox.js +1 -2
- package/dist/server-C7sg1ndo.js +1 -0
- package/dist/{server-Da60oe4b.js → server-XDleKaF_.js} +11 -11
- package/dist/server-XDleKaF_.js.map +1 -0
- package/dist/shared-DYpIb1BI.js +166 -0
- package/dist/shared-DYpIb1BI.js.map +1 -0
- package/dist/{source-BVOnL3sn.js → source-C8xumIWj.js} +2 -2
- package/dist/{source-BVOnL3sn.js.map → source-C8xumIWj.js.map} +1 -1
- package/dist/source-fGd0SaKP.js +1 -0
- package/dist/{src-TdJuDYqB.js → src-B7treWa8.js} +1597 -292
- package/dist/src-B7treWa8.js.map +1 -0
- package/dist/{src-C1E9HDWO.js → src-C7_1SqDP.js} +3 -3
- package/dist/{src-C1E9HDWO.js.map → src-C7_1SqDP.js.map} +1 -1
- package/dist/{src-DDNXsD9H.js → src-CF17Ug08.js} +5 -4
- package/dist/src-CF17Ug08.js.map +1 -0
- package/dist/{src-D_cXqmnI.js → src-CFp86GtV.js} +60 -60
- package/dist/src-CFp86GtV.js.map +1 -0
- package/dist/src-C_lPtowT.js +2 -0
- package/dist/src-DIL0jST0.js +368 -0
- package/dist/src-DIL0jST0.js.map +1 -0
- package/dist/src-DNnG4e4R.js +2 -0
- package/dist/src-DTbrY1hi.js +2 -0
- package/dist/src-DdyJEh2v.js +2 -0
- package/dist/src-Dj9PAroc.js +2 -0
- package/dist/src-Du8H85Qq.js +2 -0
- package/dist/src-DvCg0NBs.js +324 -0
- package/dist/src-DvCg0NBs.js.map +1 -0
- package/dist/{src-CxTC5kNI.js → src-DvK0EIZx.js} +8 -10
- package/dist/{src-CxTC5kNI.js.map → src-DvK0EIZx.js.map} +1 -1
- package/dist/{src-DVy58-zZ.js → src-FGXhkEkH.js} +5 -5
- package/dist/{src-DVy58-zZ.js.map → src-FGXhkEkH.js.map} +1 -1
- package/dist/src-jzagbuqH.js +2 -0
- package/dist/src-ofUaYbuY.js +3 -0
- package/dist/src-pJZOrYR9.js +1 -0
- package/dist/{src-D95O-Smv.js → src-u8_ixn_F.js} +15 -11
- package/dist/src-u8_ixn_F.js.map +1 -0
- package/dist/{state-B72h_jEw.js → state-C__9swoG.js} +36 -24
- package/dist/state-C__9swoG.js.map +1 -0
- package/dist/state-CeR6oz0k.js +66 -0
- package/dist/state-CeR6oz0k.js.map +1 -0
- package/dist/state-CgjQo29K.js +1 -0
- package/dist/studio-app/assets/index-CwQN3f_h.css +2 -0
- package/dist/studio-app/assets/{index-Bgr-efZm.js → index-DrQc2Yxd.js} +12 -12
- package/dist/studio-app/index.html +2 -2
- package/dist/{summary-k3mhWWXv.js → summary-BLAVLySU.js} +202 -17
- package/dist/summary-BLAVLySU.js.map +1 -0
- package/dist/{summary-CJdxaRdM.js → summary-DgyjUh_l.js} +7 -7
- package/dist/{summary-CJdxaRdM.js.map → summary-DgyjUh_l.js.map} +1 -1
- package/dist/testing.d.ts +3 -2
- package/dist/testing.js +3 -2
- package/dist/{tool-target-BU1JnSE9.js → tool-target-D2mcYlSQ.js} +14 -7
- package/dist/tool-target-D2mcYlSQ.js.map +1 -0
- package/dist/{tools-D0gF2QoV.js → tools-GCyNYSc1.js} +7 -7
- package/dist/tools-GCyNYSc1.js.map +1 -0
- package/dist/transcript-BNNANU8A.js +64 -0
- package/dist/transcript-BNNANU8A.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-_LOFdPUf.d.ts → types-g17bRAzt.d.ts} +578 -49
- package/dist/{usage-DDF_6tkV.js → usage-CleNKkly.js} +4 -2
- package/dist/{usage-DDF_6tkV.js.map → usage-CleNKkly.js.map} +1 -1
- package/dist/view/index.d.ts +83 -17
- package/dist/view/index.js +7 -6
- package/dist/views-BW6oCDUQ.js +2 -0
- package/dist/{views-DZ1YDqey.js → views-BmQF4UCI.js} +13 -8
- package/dist/views-BmQF4UCI.js.map +1 -0
- package/dist/{worker-entry-DXeyWY-m.js → worker-entry-yOchGYYv.js} +16 -225
- package/dist/worker-entry-yOchGYYv.js.map +1 -0
- package/dist/{workspace-DpXdrslo.js → workspace-CgyHEM0l.js} +7 -7
- package/dist/{workspace-DpXdrslo.js.map → workspace-CgyHEM0l.js.map} +1 -1
- package/dist/workspace-D8sSgpiy.js +1 -0
- package/docker/Dockerfile +7 -5
- package/package.json +12 -12
- package/dist/agent-Cfj1wD4D.js.map +0 -1
- package/dist/assemble-BbFVjHeS.js.map +0 -1
- package/dist/assemble-CXiTVyHD.js +0 -2
- package/dist/capture-BXhBfmsM.js.map +0 -1
- package/dist/capture-N7r3yeak.js +0 -2
- package/dist/cli-BMjTKY-D.js.map +0 -1
- package/dist/cli-spec-DSvKsOK4.js.map +0 -1
- package/dist/client-CwzU9CiW.js.map +0 -1
- package/dist/command-BK4kOgkH.js.map +0 -1
- package/dist/command-BqJtFiPy.js.map +0 -1
- package/dist/command-CTVGuH8y.js +0 -2
- package/dist/command-JPGNzFsA.js.map +0 -1
- package/dist/commands-B618jg3C.js.map +0 -1
- package/dist/commands-Duo2ur3r.js +0 -2
- package/dist/config-B-E0xRkf.js +0 -447
- package/dist/config-B-E0xRkf.js.map +0 -1
- package/dist/context-CvUv7DUb.js.map +0 -1
- package/dist/db-CnoUy-FZ.js +0 -3
- package/dist/db-DFbtGIYy.js.map +0 -1
- package/dist/debug-SEmW4w8f.js +0 -2
- package/dist/dist-CTzThfu_.js.map +0 -1
- package/dist/docker-2qns7hJ2.js.map +0 -1
- package/dist/docker-D6t23fZl.js +0 -2
- package/dist/docker-IFRW8K5f.js.map +0 -1
- package/dist/doctor-B6oYTPoR.js.map +0 -1
- package/dist/egress-IUKdeayc.js.map +0 -1
- package/dist/environment-DiCljXMD.js +0 -657
- package/dist/environment-DiCljXMD.js.map +0 -1
- package/dist/errors-C9w_ud7g.js.map +0 -1
- package/dist/errors-yDAEwmZo.js.map +0 -1
- package/dist/fake-instructions-CUI33I_F.js +0 -32
- package/dist/fake-instructions-CUI33I_F.js.map +0 -1
- package/dist/files-BM78_nnF.js +0 -2
- package/dist/files-BpYgF2z0.js +0 -2
- package/dist/files-YCCVaTQC.js +0 -2
- package/dist/harness-iqWeJQMM.js +0 -154
- package/dist/harness-iqWeJQMM.js.map +0 -1
- package/dist/images-CscddrJU.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/init-DBHyi92L.js +0 -2
- package/dist/init-NQfIvi-0.js.map +0 -1
- package/dist/jiti-BcJHCtrb.js +0 -46
- package/dist/jiti-BcJHCtrb.js.map +0 -1
- package/dist/live-S0qODb6o.js.map +0 -1
- package/dist/model-BbLGRtOs.js +0 -2
- package/dist/model-BndnmcLm.js.map +0 -1
- package/dist/model-Cl507trk.js.map +0 -1
- package/dist/model-DMOraCcm.js +0 -2
- package/dist/node-Co7EXge0.js.map +0 -1
- package/dist/opencode-providers-ByD85Q0l.js.map +0 -1
- package/dist/options-DPUlm4FI.js +0 -735
- package/dist/options-DPUlm4FI.js.map +0 -1
- package/dist/plan-PPc4-6xd.js.map +0 -1
- package/dist/preflight-CddycK92.js.map +0 -1
- package/dist/preflight-Zumy0adT.js +0 -2
- package/dist/prepare-CKdMoC7e.js +0 -3
- package/dist/prepare-CsP-057p.js.map +0 -1
- package/dist/provider-BDbNOQhQ.js +0 -2
- package/dist/providers-DkwiLKDS.js +0 -107
- package/dist/providers-DkwiLKDS.js.map +0 -1
- package/dist/providers-yXZZcPfx.js +0 -2
- package/dist/query-noGbekGF.js +0 -2
- package/dist/questions-BEPj98fZ.js +0 -234
- package/dist/questions-BEPj98fZ.js.map +0 -1
- package/dist/runtime-Dfb2r-cS.js.map +0 -1
- package/dist/sandbox--ZXuscAZ.js +0 -134
- package/dist/sandbox--ZXuscAZ.js.map +0 -1
- package/dist/server-Da60oe4b.js.map +0 -1
- package/dist/server-DiGR4JxK.js +0 -2
- package/dist/source-Dqu0jKwx.js +0 -2
- package/dist/src-BHNiP-5A.js +0 -4
- package/dist/src-BT4dB5CO.js +0 -2
- package/dist/src-BT5pwco0.js +0 -3
- package/dist/src-BrMbt1B2.js +0 -3
- package/dist/src-D13vpMh0.js +0 -3
- package/dist/src-D95O-Smv.js.map +0 -1
- package/dist/src-DDNXsD9H.js.map +0 -1
- package/dist/src-D_cXqmnI.js.map +0 -1
- package/dist/src-DdRlaiRc.js +0 -3
- package/dist/src-TdJuDYqB.js.map +0 -1
- package/dist/src-xczuQbE6.js +0 -3
- package/dist/state-B72h_jEw.js.map +0 -1
- package/dist/state-Bf_Vq_pV.js +0 -2
- package/dist/studio-app/assets/index-CUx6GoL5.css +0 -2
- package/dist/summary-k3mhWWXv.js.map +0 -1
- package/dist/tool-target-BU1JnSE9.js.map +0 -1
- package/dist/tools-D0gF2QoV.js.map +0 -1
- package/dist/views-DZ1YDqey.js.map +0 -1
- package/dist/views-Ds0GXWtX.js +0 -3
- package/dist/worker-entry-DXeyWY-m.js.map +0 -1
- package/dist/workspace-DzNL-3iO.js +0 -2
|
@@ -1,32 +1,34 @@
|
|
|
1
|
-
import { n as
|
|
2
|
-
import {
|
|
1
|
+
import { n as isOneOf } from "./object-CLf2-tNq.js";
|
|
2
|
+
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
+
import { t as parseDuration } from "./duration-CEOQww97.js";
|
|
4
|
+
import { f as sandboxId, s as harnessId } from "./options-Bh2W63dK.js";
|
|
5
|
+
import { A as workspaceLabel, p as environmentSandbox } from "./environment-DNjRk0nU.js";
|
|
6
|
+
import { a as fromCwd, i as evalSlug, n as attemptKey, o as newRunId } from "./ids-Bp6ut7ZD.js";
|
|
7
|
+
import { a as configPathFor, c as findConfig, d as malformedCredentials, f as missingCredentials, h as selectProfile, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig } from "./config-DEgnUiU-.js";
|
|
3
8
|
import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
|
|
4
9
|
import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
|
|
5
|
-
import { o as PACKAGE_VERSION } from "./root-
|
|
6
|
-
import {
|
|
7
|
-
import { a as fromCwd, i as evalSlug, n as attemptKey, o as newRunId } from "./ids-Bp6ut7ZD.js";
|
|
8
|
-
import { I as LABEL, X as imageInfo } from "./src-TdJuDYqB.js";
|
|
9
|
-
import { d as sandboxId, s as harnessId } from "./options-DPUlm4FI.js";
|
|
10
|
-
import { c as hostCapacity, d as missingCredentials, f as resolveConcurrency, i as configPathFor, l as loadConfig, p as selectProfile, r as configLabel, s as findConfig, u as malformedCredentials } from "./config-B-E0xRkf.js";
|
|
10
|
+
import { o as PACKAGE_VERSION } from "./root-DICDHCtp.js";
|
|
11
|
+
import { I as LABEL, X as imageInfo } from "./src-B7treWa8.js";
|
|
11
12
|
import { s as debugTime } from "./debug-D24NOQH1.js";
|
|
12
13
|
import { i as sandboxWarningLine, n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
|
|
13
14
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
14
|
-
import {
|
|
15
|
-
import { a as
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
15
|
+
import { B as judgeRatesLabel, H as summaryJudgeRates, I as collapsedJudgeLabel, P as cellUsage, T as runOutcome, Y as cellContrast, g as formatUsage, l as evalMarkOutcome, tt as effortLabel, v as hasUngatedCells, x as isUngatedMiss } from "./model-oqcDplJG.js";
|
|
16
|
+
import { a as familyLiftLabel, i as familyHeadLabel, n as familyCases, o as familyRateLabel, r as familyEffortLabel, t as caseLabel } from "./family-D2b7WD10.js";
|
|
17
|
+
import { n as headToHeadLines, s as sampleReasons } from "./pairs-CnInRu-G.js";
|
|
18
|
+
import { a as contrastLines, f as undiscriminating, i as checksLabel, n as answersLabel, o as gateLabel, p as undiscriminatingLabel, s as liftLabel } from "./summary-BLAVLySU.js";
|
|
19
|
+
import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-BRfa8E_s.js";
|
|
20
|
+
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-BPZbQG2p.js";
|
|
21
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-DmE9uPVt.js";
|
|
20
22
|
import { t as budgetMarker } from "./budget-DBUrH4jX.js";
|
|
21
|
-
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
24
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-
|
|
23
|
+
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-D7iFPKeK.js";
|
|
24
|
+
import { n as imageHarness } from "./prepare-doGxaAGR.js";
|
|
25
|
+
import { r as createProviders, t as attemptImage } from "./providers-tAOD1NQN.js";
|
|
26
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DbGfBAA_.js";
|
|
25
27
|
import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
|
|
26
28
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
27
29
|
import { dirname, join } from "node:path";
|
|
28
|
-
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
29
30
|
import { inspect } from "node:util";
|
|
31
|
+
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
30
32
|
import { detectAgent } from "std-env";
|
|
31
33
|
import { Command, Option } from "commander";
|
|
32
34
|
//#region src/cli-parse.ts
|
|
@@ -64,13 +66,13 @@ function parseCli(argv) {
|
|
|
64
66
|
const subcommand = configure(command.command(subName), `reeval ${name} ${subName}`, sub.flags);
|
|
65
67
|
subcommand.argument("[args...]").action(() => {
|
|
66
68
|
const [stray] = Object.keys(valuesOf(command, spec.flags));
|
|
67
|
-
if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}
|
|
69
|
+
if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}. Run reeval --help.`);
|
|
68
70
|
parsed = done(program, subcommand, [name, subName], sub.flags, sub.positionals);
|
|
69
71
|
});
|
|
70
72
|
}
|
|
71
73
|
}
|
|
72
74
|
program.parse(argv, { from: "user" });
|
|
73
|
-
if (!parsed) throw new UsageError("Run reeval --help.");
|
|
75
|
+
if (!parsed) throw new UsageError("reeval couldn't read that command line. Run reeval --help.");
|
|
74
76
|
return parsed;
|
|
75
77
|
}
|
|
76
78
|
function configure(command, name, flags) {
|
|
@@ -123,11 +125,16 @@ function usageError(error, command) {
|
|
|
123
125
|
const [flag = "", value] = arg.replace(/^--?/, "").split("=");
|
|
124
126
|
const known = arg.startsWith("--") ? FLAGS[flag] : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[1];
|
|
125
127
|
const long = arg.startsWith("--") ? flag : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[0];
|
|
126
|
-
if (known && value !== void 0 && known.type === "boolean") return new UsageError(
|
|
128
|
+
if (known && value !== void 0 && known.type === "boolean") return new UsageError(`--${flag} doesn't take a value: pass --${flag} on its own.`);
|
|
127
129
|
if (known) return new UsageError(`--${long} isn't an option of ${command}. Run reeval --help.`);
|
|
128
130
|
const hint = rest.join(" ").replace(/^\(|\)$/g, "");
|
|
129
131
|
return new UsageError(`Unknown option '${arg}'. ${hint ? `${hint} ` : ""}Run reeval --help.`);
|
|
130
132
|
}
|
|
133
|
+
if (error.code === "commander.optionMissingArgument") {
|
|
134
|
+
const usage = /'([^']+)'/.exec(first)?.[1] ?? "";
|
|
135
|
+
const flag = usage.split(/[ ,]/).find((part) => part.startsWith("--")) ?? usage;
|
|
136
|
+
return new UsageError(`${flag} needs a value, e.g. ${usage}. Run reeval --help.`);
|
|
137
|
+
}
|
|
131
138
|
return new UsageError(first.charAt(0).toUpperCase() + first.slice(1));
|
|
132
139
|
}
|
|
133
140
|
//#endregion
|
|
@@ -171,10 +178,10 @@ async function preflight(uses, options) {
|
|
|
171
178
|
writePrepared(options.root, options.runId, outcomes);
|
|
172
179
|
const images = [];
|
|
173
180
|
for (const o of outcomes) {
|
|
174
|
-
if (!o.ok || o.target.kind !== "docker") continue;
|
|
175
|
-
const { ref, id, agentCli } = o.handle;
|
|
181
|
+
if (!o.ok || o.target.sandbox.kind !== "docker") continue;
|
|
182
|
+
const { ref, id, agentCli } = attemptImage(o.handle);
|
|
176
183
|
if (images.some((i) => i.id === id)) continue;
|
|
177
|
-
const info = await imageInfo(ref);
|
|
184
|
+
const info = await imageInfo(ref, o.target.sandbox.engine);
|
|
178
185
|
images.push({
|
|
179
186
|
ref,
|
|
180
187
|
id,
|
|
@@ -416,8 +423,10 @@ function resultFor(entry, attempt, results) {
|
|
|
416
423
|
function entryOutcome(e) {
|
|
417
424
|
const outcomes = e.attempts.map(attemptOutcome);
|
|
418
425
|
if (outcomes.length && outcomes.every((o) => o === "cancelled")) return "cancelled";
|
|
419
|
-
return e
|
|
426
|
+
return evalMarkOutcome(e);
|
|
420
427
|
}
|
|
428
|
+
/** Every cell is `gate: 'none'`: the eval reports its attempts but nothing gates it. */
|
|
429
|
+
const allUngated = (e) => e.cells.length > 0 && e.cells.every((c) => c.gate === "none");
|
|
421
430
|
const pct = (rate) => rate === null ? " – " : `${Math.round(rate * 100)}%`.padStart(4);
|
|
422
431
|
function evalColumns(c, e) {
|
|
423
432
|
const [attempt, ...more] = e.attempts;
|
|
@@ -426,7 +435,7 @@ function evalColumns(c, e) {
|
|
|
426
435
|
if (more.length) {
|
|
427
436
|
const pass = e.attempts.filter((a) => a.outcome === "pass").length;
|
|
428
437
|
const counted = e.attempts.filter((a) => a.outcome === "pass" || a.outcome === "fail").length;
|
|
429
|
-
const rate = hasUngatedCells(e.cells) ? ` · ${pct(e.passRate).trim()} gated` : ` ${pct(e.passRate)}`;
|
|
438
|
+
const rate = allUngated(e) ? " · not gated" : hasUngatedCells(e.cells) ? ` · pass@1 ${pct(e.passRate).trim()} gated` : ` pass@1 ${pct(e.passRate)}`;
|
|
430
439
|
return c.dim(`${`${pass}/${counted}`.padStart(7)}${rate} ${e.attempts.length} attempts`);
|
|
431
440
|
}
|
|
432
441
|
if (attemptOutcome(attempt) === "cancelled") return c.yellow("cancelled");
|
|
@@ -439,7 +448,7 @@ function evalColumns(c, e) {
|
|
|
439
448
|
if (tokens) cols.push(tokens);
|
|
440
449
|
return c.dim(cols.join(" "));
|
|
441
450
|
}
|
|
442
|
-
/**
|
|
451
|
+
/** pass@1 per cell for evals with several attempts; with compare, each cell's role and lift. */
|
|
443
452
|
function cellRows(c, e, indent) {
|
|
444
453
|
if (e.attempts.length < 2) return [];
|
|
445
454
|
const compare = e.cells.some((cell) => cell.role === "variant");
|
|
@@ -458,27 +467,92 @@ function cellRows(c, e, indent) {
|
|
|
458
467
|
if (cell.gate !== "all" && cell.gate !== "none") note += ` ${c.dim(`gate ${gateLabel(cell.gate)}`)}`;
|
|
459
468
|
const tokens = formatUsage(cellUsage(e, cell.key));
|
|
460
469
|
const usage = tokens ? ` ${c.dim(tokens)}` : "";
|
|
461
|
-
const rows = [`${indent}${c.dim(cell.key.padEnd(width))}${role} ${counts} ${pct(cell.passRate)}${note}${infra}${usage}`];
|
|
462
|
-
const rates = judgeRatesLabel(
|
|
470
|
+
const rows = [`${indent}${c.dim(cell.key.padEnd(width))}${role} ${counts} pass@1 ${pct(cell.passRate)}${note}${infra}${usage}`];
|
|
471
|
+
const rates = judgeRatesLabel(summaryJudgeRates(e, cell));
|
|
463
472
|
if (rates) rows.push(`${indent} ${c.dim(rates)}`);
|
|
464
473
|
const effort = cellContrast(e, cell.key);
|
|
465
474
|
if (effort) rows.push(`${indent} ${c.dim(`effort: ${effortLabel(effort)}`)}`);
|
|
475
|
+
const checks = checksLabel(cell);
|
|
476
|
+
if (checks) rows.push(`${indent} ${c.dim(`checks: ${checks}`)}`);
|
|
466
477
|
if (cell.overrides.length) rows.push(`${indent} ${c.dim(`overrides: ${cell.overrides.join(", ")}`)}`);
|
|
467
478
|
return rows;
|
|
468
479
|
});
|
|
469
480
|
}
|
|
481
|
+
/** With two variants or more, each pair of arms on one harness: lift, checks and effort. */
|
|
482
|
+
function contrastRows(c, e, indent) {
|
|
483
|
+
const lines = contrastLines(e);
|
|
484
|
+
if (!lines.length) return [];
|
|
485
|
+
return [`${indent}${c.dim("contrasts (each pair of arms, same harness)")}`, ...lines.map((line) => `${indent} ${c.dim(line)}`)];
|
|
486
|
+
}
|
|
470
487
|
/** One row per compare question: its win rates, or why it has none. They never gate. */
|
|
471
488
|
function questionRows(c, e, indent) {
|
|
472
489
|
return (e.answers ?? []).flatMap((a) => {
|
|
473
490
|
const label = answersLabel(a);
|
|
474
|
-
return [
|
|
491
|
+
return [
|
|
492
|
+
`${indent}${c.dim("question")} ${a.error !== void 0 ? c.yellow(label) : label}`,
|
|
493
|
+
...headToHeadLines(a.votes).map((line) => `${indent} ${line}`),
|
|
494
|
+
...sampleReasons(a.votes).map((reason) => `${indent} ${c.dim(reason)}`)
|
|
495
|
+
];
|
|
475
496
|
});
|
|
476
497
|
}
|
|
477
|
-
/**
|
|
478
|
-
function
|
|
498
|
+
/** A family's mark: its pooled gate's and its cases' worst. */
|
|
499
|
+
function familyOutcome(family, cases) {
|
|
500
|
+
const outcomes = [family.outcome, ...cases.map(entryOutcome)];
|
|
501
|
+
for (const o of [
|
|
502
|
+
"fail",
|
|
503
|
+
"infra_error",
|
|
504
|
+
"cancelled"
|
|
505
|
+
]) if (outcomes.includes(o)) return o;
|
|
506
|
+
return outcomes.every((o) => o === "skipped") ? "skipped" : "pass";
|
|
507
|
+
}
|
|
508
|
+
/** The width case marks wrap at. */
|
|
509
|
+
const CASE_WIDTH = 96;
|
|
510
|
+
/** A pooled family's rows under its header: its cases' marks, then each pooled rate or lift. */
|
|
511
|
+
function familyRows(c, family, cases, indent) {
|
|
512
|
+
const marks = cases.map((e) => `${mark(c, entryOutcome(e))} ${caseLabel(family.name, e.name)}`);
|
|
513
|
+
const plain = cases.map((e) => `x ${caseLabel(family.name, e.name)}`);
|
|
479
514
|
const rows = [];
|
|
515
|
+
let line = [];
|
|
516
|
+
let width = 0;
|
|
517
|
+
marks.forEach((m, i) => {
|
|
518
|
+
const w = plain[i].length + 2;
|
|
519
|
+
if (line.length && width + w > CASE_WIDTH) {
|
|
520
|
+
rows.push(`${indent}${line.join(" ")}`);
|
|
521
|
+
line = [];
|
|
522
|
+
width = 0;
|
|
523
|
+
}
|
|
524
|
+
line.push(m);
|
|
525
|
+
width += w;
|
|
526
|
+
});
|
|
527
|
+
if (line.length) rows.push(`${indent}${line.join(" ")}`);
|
|
528
|
+
if (family.lifts.length) {
|
|
529
|
+
const keyWidth = Math.max(...family.lifts.map((l) => l.variant.length));
|
|
530
|
+
for (const lift of family.lifts) {
|
|
531
|
+
const below = lift.outcome === "fail";
|
|
532
|
+
const gate = lift.minLift !== void 0 ? ` (minLift ${Math.round(lift.minLift * 100)}${below ? " ✗" : ""})` : "";
|
|
533
|
+
const text = `${familyLiftLabel(lift)}${gate}`;
|
|
534
|
+
rows.push(`${indent}${c.dim(lift.variant.padEnd(keyWidth))} ${below ? c.red(text) : text}`);
|
|
535
|
+
const effort = familyEffortLabel(lift);
|
|
536
|
+
if (effort) rows.push(`${indent} ${c.dim(`effort: ${effort}`)}`);
|
|
537
|
+
}
|
|
538
|
+
} else if (family.rates.length === 1) rows.push(`${indent}${familyRateLabel(family.rates[0])}`);
|
|
539
|
+
else {
|
|
540
|
+
const keyWidth = Math.max(...family.rates.map((r) => r.cell.length));
|
|
541
|
+
for (const rate of family.rates) rows.push(`${indent}${c.dim(rate.cell.padEnd(keyWidth))} ${familyRateLabel(rate)}`);
|
|
542
|
+
}
|
|
543
|
+
return rows;
|
|
544
|
+
}
|
|
545
|
+
/**
|
|
546
|
+
* The run's evals as a file › group tree, one row per eval and per cell of a matrix. A pooled
|
|
547
|
+
* family is one row, its template, with its cases' marks and the pooled results under it.
|
|
548
|
+
*/
|
|
549
|
+
function tree(c, evals, families = []) {
|
|
550
|
+
const rows = [];
|
|
551
|
+
const shown = /* @__PURE__ */ new Set();
|
|
480
552
|
let path = [];
|
|
481
553
|
for (const e of evals) {
|
|
554
|
+
const family = e.family && families.find((f) => f.id === e.family?.id);
|
|
555
|
+
if (family && shown.has(family.id)) continue;
|
|
482
556
|
const next = [e.file, ...e.group];
|
|
483
557
|
let same = 0;
|
|
484
558
|
while (same < path.length && same < next.length && path[same] === next[same]) same++;
|
|
@@ -487,19 +561,32 @@ function tree(c, evals) {
|
|
|
487
561
|
text: i === 0 ? c.bold(next[i] ?? "") : next[i] ?? ""
|
|
488
562
|
});
|
|
489
563
|
path = next;
|
|
490
|
-
|
|
564
|
+
if (family) {
|
|
565
|
+
shown.add(family.id);
|
|
566
|
+
rows.push({
|
|
567
|
+
indent: next.length,
|
|
568
|
+
text: family.name,
|
|
569
|
+
family
|
|
570
|
+
});
|
|
571
|
+
} else rows.push({
|
|
491
572
|
indent: next.length,
|
|
492
573
|
text: e.name,
|
|
493
574
|
entry: e
|
|
494
575
|
});
|
|
495
576
|
}
|
|
496
|
-
const width = Math.max(...rows.filter((r) => r.entry).map((r) => r.indent * 2 + 2 + r.text.length));
|
|
497
|
-
return rows.flatMap(({ indent, text, entry }) => {
|
|
577
|
+
const width = Math.max(...rows.filter((r) => r.entry || r.family).map((r) => r.indent * 2 + 2 + r.text.length));
|
|
578
|
+
return rows.flatMap(({ indent, text, entry, family }) => {
|
|
498
579
|
const pad = " ".repeat(indent);
|
|
580
|
+
const fill = " ".repeat(Math.max(0, width - indent * 2 - 2 - text.length));
|
|
581
|
+
if (family) {
|
|
582
|
+
const cases = familyCases(family, evals);
|
|
583
|
+
return [` ${pad}${mark(c, familyOutcome(family, cases))} ${text}${fill} ${c.dim(familyHeadLabel(family))}`, ...familyRows(c, family, cases, ` ${pad} `)];
|
|
584
|
+
}
|
|
499
585
|
if (!entry) return [` ${pad}${text}`];
|
|
500
586
|
return [
|
|
501
|
-
` ${pad}${`${
|
|
587
|
+
` ${pad}${`${entryOutcome(entry) === "pass" && allUngated(entry) && entry.attempts.some((a) => isUngatedMiss(entry, a)) ? MISS_MARK(c) : mark(c, entryOutcome(entry))} ${text}`}${fill} ${evalColumns(c, entry)}`,
|
|
502
588
|
...cellRows(c, entry, ` ${pad} `),
|
|
589
|
+
...contrastRows(c, entry, ` ${pad} `),
|
|
503
590
|
...questionRows(c, entry, ` ${pad} `)
|
|
504
591
|
];
|
|
505
592
|
});
|
|
@@ -517,6 +604,12 @@ function totals(c, s, cancelled = 0) {
|
|
|
517
604
|
const scope = s.attempts === s.evals ? `${s.evals}` : `${s.attempts} attempts, ${s.evals} evals`;
|
|
518
605
|
return parts.length ? `${parts.join(c.dim(" · "))} ${c.dim(`(${scope})`)}` : "no evals ran";
|
|
519
606
|
}
|
|
607
|
+
/** `1 passed · 1 failed (2)`: the families' pooled gates. */
|
|
608
|
+
function familyTotals(c, families) {
|
|
609
|
+
const fail = families.filter((f) => f.outcome === "fail").length;
|
|
610
|
+
const pass = families.length - fail;
|
|
611
|
+
return `${[pass && c.green(`${pass} passed`), fail && c.red(`${fail} failed`)].filter(Boolean).join(c.dim(" · "))} ${c.dim(`(${families.length})`)}`;
|
|
612
|
+
}
|
|
520
613
|
/** Failing attempts, in run order, including expected misses. */
|
|
521
614
|
function failingAttempts(evals) {
|
|
522
615
|
return evals.flatMap((entry) => entry.attempts.map((attempt) => ({
|
|
@@ -528,7 +621,7 @@ function failingAttempts(evals) {
|
|
|
528
621
|
function renderSummary(c, e, attemptDir) {
|
|
529
622
|
const { record } = e;
|
|
530
623
|
const lines = [];
|
|
531
|
-
if (record.evals.length) lines.push(c.bold("Summary"), ...tree(c, record.evals));
|
|
624
|
+
if (record.evals.length) lines.push(c.bold("Summary"), ...tree(c, record.evals, record.families));
|
|
532
625
|
const all = failingAttempts(record.evals);
|
|
533
626
|
const missed = all.filter(({ entry, attempt }) => isUngatedMiss(entry, attempt));
|
|
534
627
|
const failing = all.filter((f) => !missed.includes(f));
|
|
@@ -550,7 +643,18 @@ function renderSummary(c, e, attemptDir) {
|
|
|
550
643
|
}
|
|
551
644
|
if (missed.length) {
|
|
552
645
|
lines.push("", c.bold(`Expected misses, not gated (${missed.length})`));
|
|
646
|
+
const pooled = /* @__PURE__ */ new Map();
|
|
647
|
+
for (const { entry, attempt } of missed) {
|
|
648
|
+
if (!entry.family) continue;
|
|
649
|
+
const key = `${entry.family.id}\0${attempt.cell}`;
|
|
650
|
+
pooled.set(key, (pooled.get(key) ?? 0) + 1);
|
|
651
|
+
}
|
|
652
|
+
for (const [key, n] of pooled) {
|
|
653
|
+
const [id, cell] = key.split("\0");
|
|
654
|
+
lines.push(` ${MISS_MARK(c)} ${id} ${c.dim(`${n} ${n === 1 ? "attempt" : "attempts"} on ${cell}`)}`);
|
|
655
|
+
}
|
|
553
656
|
for (const { entry, attempt } of missed) {
|
|
657
|
+
if (entry.family) continue;
|
|
554
658
|
const which = entry.attempts.length > 1 ? ` ${c.dim(attempt.id)}` : "";
|
|
555
659
|
const [reason] = failureLines(colors(false), resultFor(entry, attempt, e.results), { max: 1 }).filter((l) => l.trim());
|
|
556
660
|
lines.push(` ${MISS_MARK(c)} ${entry.id}${which}`);
|
|
@@ -562,12 +666,23 @@ function renderSummary(c, e, attemptDir) {
|
|
|
562
666
|
const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
|
|
563
667
|
const label = record.summary.attempts === record.summary.evals ? " Evals" : "Attempts";
|
|
564
668
|
const lifted = record.evals.flatMap((entry) => entry.cells.flatMap((cell) => cell.lift?.minLift !== void 0 && cell.lift.interval.lo < cell.lift.minLift ? [`${entry.id} ${cell.key} ${liftLabel(cell.lift)}: its lower bound is below minLift ${Math.round(cell.lift.minLift * 100)}`] : []));
|
|
669
|
+
for (const family of record.families ?? []) for (const lift of family.lifts) {
|
|
670
|
+
if (lift.outcome !== "fail" || lift.minLift === void 0) continue;
|
|
671
|
+
lifted.push(`${family.id} ${lift.variant} pooled ${familyLiftLabel(lift)}: its lower bound is below minLift ${Math.round(lift.minLift * 100)}`);
|
|
672
|
+
}
|
|
673
|
+
const idle = undiscriminating(record.evals);
|
|
674
|
+
if (idle.length) {
|
|
675
|
+
lines.push("", c.bold(`Checks that don't discriminate (${idle.length})`));
|
|
676
|
+
for (const x of idle) lines.push(` ${c.dim("·")} ${undiscriminatingLabel(x)}`);
|
|
677
|
+
}
|
|
565
678
|
if (lifted.length) {
|
|
566
679
|
lines.push("", c.bold(c.red(`Lift below minLift (${lifted.length})`)));
|
|
567
680
|
for (const line of lifted) lines.push(` ${c.red("✗")} ${line}`);
|
|
568
681
|
}
|
|
569
682
|
if (record.status === "budget") lines.push("", c.yellow("Budget reached: attempts that hadn’t started were skipped"));
|
|
570
|
-
lines.push(""
|
|
683
|
+
lines.push("");
|
|
684
|
+
if (record.families?.length) lines.push(`${c.dim("Families")} ${familyTotals(c, record.families)}`);
|
|
685
|
+
lines.push(`${c.dim(label)} ${totals(c, record.summary, cancelled)}`, `${c.dim("Duration")} ${formatDuration(record.durationMs)}`);
|
|
571
686
|
const tokens = formatUsage(record.cost.usage);
|
|
572
687
|
if (tokens) lines.push(` ${c.dim("Tokens")} ${tokens}`);
|
|
573
688
|
lines.push(` ${c.dim("Results")} ${e.summaryPath}`);
|
|
@@ -732,8 +847,17 @@ var AgentRenderer = class {
|
|
|
732
847
|
for (const error of e.unhandledErrors) this.#write(`ERROR ${error}`);
|
|
733
848
|
for (const entry of e.record.evals) for (const a of entry.answers ?? []) {
|
|
734
849
|
this.#write(`question ${entry.id}: ${answersLabel(a)}`);
|
|
850
|
+
for (const line of headToHeadLines(a.votes)) this.#write(` ${line}`);
|
|
735
851
|
for (const reason of sampleReasons(a.votes)) this.#write(` ${reason}`);
|
|
736
852
|
}
|
|
853
|
+
for (const x of undiscriminating(e.record.evals)) this.#write(`check doesn't discriminate: ${undiscriminatingLabel(x)}`);
|
|
854
|
+
for (const family of e.record.families ?? []) {
|
|
855
|
+
const pooled = family.lifts.length ? family.lifts.map((l) => {
|
|
856
|
+
const gate = l.minLift === void 0 ? "" : ` (minLift ${Math.round(l.minLift * 100)}${l.outcome === "fail" ? ", FAIL" : ""})`;
|
|
857
|
+
return `${l.variant} ${familyLiftLabel(l)}${gate}`;
|
|
858
|
+
}) : family.rates.map((r) => `${r.cell} ${familyRateLabel(r)}`);
|
|
859
|
+
this.#write(`family ${family.id}: ${familyHeadLabel(family)} · ${pooled.join(" · ")}`);
|
|
860
|
+
}
|
|
737
861
|
const s = e.record.summary;
|
|
738
862
|
const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
|
|
739
863
|
const parts = [
|
|
@@ -1009,13 +1133,13 @@ const CI_VARS = [
|
|
|
1009
1133
|
const isCI = (env) => CI_VARS.some((name) => set(env, name));
|
|
1010
1134
|
function pick(name, value, allowed) {
|
|
1011
1135
|
if (value === void 0 || value === "") return void 0;
|
|
1012
|
-
if (value === "auto" || allowed
|
|
1013
|
-
throw new UsageError(`${name}
|
|
1136
|
+
if (value === "auto" || isOneOf(allowed, value)) return value;
|
|
1137
|
+
throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
|
|
1014
1138
|
}
|
|
1015
1139
|
function parseHeartbeat(value) {
|
|
1016
1140
|
if (!value) return 6e4;
|
|
1017
1141
|
const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
|
|
1018
|
-
if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s or 0`);
|
|
1142
|
+
if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
|
|
1019
1143
|
const unit = {
|
|
1020
1144
|
ms: 1,
|
|
1021
1145
|
s: 1e3,
|
|
@@ -1115,12 +1239,16 @@ async function run(argv) {
|
|
|
1115
1239
|
return EXIT.ok;
|
|
1116
1240
|
}
|
|
1117
1241
|
const [command, ...filters] = positionals;
|
|
1242
|
+
if (values.help && command) {
|
|
1243
|
+
console.log(commandHelpText(version, positionals));
|
|
1244
|
+
return EXIT.ok;
|
|
1245
|
+
}
|
|
1118
1246
|
if (values.help || !command) {
|
|
1119
1247
|
console.log(HELP);
|
|
1120
1248
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
1121
1249
|
}
|
|
1122
1250
|
if (command === "init") {
|
|
1123
|
-
const { initCommand } = await import("./init-
|
|
1251
|
+
const { initCommand } = await import("./init-BrOWin3W.js");
|
|
1124
1252
|
return initCommand(values, filters, {
|
|
1125
1253
|
out: (s) => console.log(s),
|
|
1126
1254
|
input: process.stdin,
|
|
@@ -1129,7 +1257,7 @@ async function run(argv) {
|
|
|
1129
1257
|
});
|
|
1130
1258
|
}
|
|
1131
1259
|
if (command === "doctor") {
|
|
1132
|
-
const { doctorCommand } = await import("./doctor-
|
|
1260
|
+
const { doctorCommand } = await import("./doctor-BYNok4UO.js");
|
|
1133
1261
|
return doctorCommand(values, {
|
|
1134
1262
|
out: (s) => console.log(s),
|
|
1135
1263
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -1137,7 +1265,7 @@ async function run(argv) {
|
|
|
1137
1265
|
});
|
|
1138
1266
|
}
|
|
1139
1267
|
if (command === "report") {
|
|
1140
|
-
const { reportCommand } = await import("./command-
|
|
1268
|
+
const { reportCommand } = await import("./command-BLfT7vnQ.js");
|
|
1141
1269
|
return reportCommand(values, filters, {
|
|
1142
1270
|
out: (s) => console.log(s),
|
|
1143
1271
|
err: (s) => console.error(dim(s)),
|
|
@@ -1152,7 +1280,7 @@ async function run(argv) {
|
|
|
1152
1280
|
}
|
|
1153
1281
|
if (command === "studio") {
|
|
1154
1282
|
const config = await loadConfig(configPath, "studio");
|
|
1155
|
-
const { studioCommand } = await import("./command-
|
|
1283
|
+
const { studioCommand } = await import("./command-CMJtHKRa.js");
|
|
1156
1284
|
return studioCommand({
|
|
1157
1285
|
config,
|
|
1158
1286
|
flags: values,
|
|
@@ -1163,7 +1291,7 @@ async function run(argv) {
|
|
|
1163
1291
|
if (command === "mcp") {
|
|
1164
1292
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
1165
1293
|
const config = await loadConfig(configPath, "mcp");
|
|
1166
|
-
const { mcpCommand } = await import("./command-
|
|
1294
|
+
const { mcpCommand } = await import("./command-BT9a37ZR.js");
|
|
1167
1295
|
return mcpCommand({
|
|
1168
1296
|
config,
|
|
1169
1297
|
flags: values,
|
|
@@ -1172,7 +1300,7 @@ async function run(argv) {
|
|
|
1172
1300
|
}
|
|
1173
1301
|
if (command in HISTORY_COMMANDS) {
|
|
1174
1302
|
const config = await loadConfig(configPath, command);
|
|
1175
|
-
const commands = await import("./commands-
|
|
1303
|
+
const commands = await import("./commands-1p-bIWhN.js");
|
|
1176
1304
|
const ctx = {
|
|
1177
1305
|
config,
|
|
1178
1306
|
flags: values,
|
|
@@ -1185,7 +1313,7 @@ async function run(argv) {
|
|
|
1185
1313
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
1186
1314
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
1187
1315
|
case "query": {
|
|
1188
|
-
const { queryCommand } = await import("./query-
|
|
1316
|
+
const { queryCommand } = await import("./query-C_G_svLU.js");
|
|
1189
1317
|
return queryCommand({
|
|
1190
1318
|
...ctx,
|
|
1191
1319
|
err: (s) => console.error(dim(s))
|
|
@@ -1193,7 +1321,7 @@ async function run(argv) {
|
|
|
1193
1321
|
}
|
|
1194
1322
|
case "analyze": {
|
|
1195
1323
|
loadEnvFiles(configPath);
|
|
1196
|
-
const { analyzeCommand } = await import("./command-
|
|
1324
|
+
const { analyzeCommand } = await import("./command-BkFjijN2.js");
|
|
1197
1325
|
return analyzeCommand({
|
|
1198
1326
|
...ctx,
|
|
1199
1327
|
err: (s) => console.error(dim(s))
|
|
@@ -1226,16 +1354,19 @@ async function cleanCommand(flags, configPath) {
|
|
|
1226
1354
|
keep = Number(flags.keep);
|
|
1227
1355
|
if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
|
|
1228
1356
|
}
|
|
1229
|
-
const { debugRoot, pruneDebug } = await import("./files-
|
|
1230
|
-
const { formatBytes } = await import("./debug-
|
|
1357
|
+
const { debugRoot, pruneDebug } = await import("./files-BAt0i00X.js");
|
|
1358
|
+
const { formatBytes } = await import("./debug-D7bEv_UV.js");
|
|
1231
1359
|
const { runs, bytes } = await pruneDebug(config.root, keep);
|
|
1232
1360
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1233
1361
|
console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
|
|
1234
1362
|
if (flags.images) {
|
|
1235
|
-
const { pruneImages } = await import("./src-
|
|
1236
|
-
const { requireDocker } = await import("./providers-
|
|
1237
|
-
|
|
1238
|
-
const
|
|
1363
|
+
const { dockerAvailable, pruneImages } = await import("./src-pJZOrYR9.js");
|
|
1364
|
+
const { requireDocker } = await import("./providers-gDMko8fl.js");
|
|
1365
|
+
const engines = [];
|
|
1366
|
+
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1367
|
+
if (!engines.length) await requireDocker();
|
|
1368
|
+
let removed = 0;
|
|
1369
|
+
for (const engine of engines) removed += await pruneImages(engine);
|
|
1239
1370
|
console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
|
|
1240
1371
|
}
|
|
1241
1372
|
return EXIT.ok;
|
|
@@ -1248,55 +1379,24 @@ function verbosityOf(flags) {
|
|
|
1248
1379
|
/**
|
|
1249
1380
|
* `reeval image build`: collects the evals (as `--dry-run` does) and builds every image their Docker
|
|
1250
1381
|
* sandboxes need; with `--dry-run`, lists them instead. `--profile` plans as `reeval run --profile`
|
|
1251
|
-
* does.
|
|
1382
|
+
* does. Needs a config, as `reeval run` does: there is no default image.
|
|
1252
1383
|
*/
|
|
1253
1384
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1254
|
-
const { buildImages, imagePlanLines, imageTargets } = await import("./images-
|
|
1255
|
-
const {
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
root = config.root;
|
|
1270
|
-
const { runEngine } = await import("./node-Co7EXge0.js");
|
|
1271
|
-
let planned = [];
|
|
1272
|
-
await runEngine({
|
|
1273
|
-
run: JSON.parse(JSON.stringify({
|
|
1274
|
-
runId,
|
|
1275
|
-
stagingDir,
|
|
1276
|
-
config,
|
|
1277
|
-
runs,
|
|
1278
|
-
tags,
|
|
1279
|
-
concurrency,
|
|
1280
|
-
profile
|
|
1281
|
-
})),
|
|
1282
|
-
filters,
|
|
1283
|
-
onPlan: async (collected) => {
|
|
1284
|
-
if (collected.unmatched.length) throw unmatchedFilter(config, collected.unmatched);
|
|
1285
|
-
planned = collected.planned.filter((p) => !p.attempt.plan.skip);
|
|
1286
|
-
return false;
|
|
1287
|
-
}
|
|
1288
|
-
});
|
|
1289
|
-
uses = planned.length ? planned.map((p) => ({
|
|
1290
|
-
id: p.id,
|
|
1291
|
-
harness: p.attempt.harness,
|
|
1292
|
-
sandbox: p.attempt.sandbox
|
|
1293
|
-
})) : config.harness.map((harness) => ({
|
|
1294
|
-
id: "",
|
|
1295
|
-
harness,
|
|
1296
|
-
sandbox: config.sandbox
|
|
1297
|
-
}));
|
|
1298
|
-
}
|
|
1299
|
-
const targets = imageTargets(uses);
|
|
1385
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-0On0cztu.js");
|
|
1386
|
+
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
|
|
1387
|
+
...flags,
|
|
1388
|
+
"dry-run": true
|
|
1389
|
+
}, configPath));
|
|
1390
|
+
const root = config.root;
|
|
1391
|
+
const targets = imageTargets(await plannedImageUses({
|
|
1392
|
+
runId,
|
|
1393
|
+
stagingDir,
|
|
1394
|
+
config,
|
|
1395
|
+
runs,
|
|
1396
|
+
tags,
|
|
1397
|
+
concurrency,
|
|
1398
|
+
profile
|
|
1399
|
+
}, filters, (parts) => unmatchedFilter(config, parts)));
|
|
1300
1400
|
if (targets.length === 0) {
|
|
1301
1401
|
console.log("No eval runs in a Docker sandbox, so there are no images to build.");
|
|
1302
1402
|
return EXIT.ok;
|
|
@@ -1305,7 +1405,7 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1305
1405
|
for (const line of await imagePlanLines(targets, root)) console.log(line);
|
|
1306
1406
|
return EXIT.ok;
|
|
1307
1407
|
}
|
|
1308
|
-
const { PrepareError } = await import("./provider-
|
|
1408
|
+
const { PrepareError } = await import("./provider-DehR4CMN.js");
|
|
1309
1409
|
let built;
|
|
1310
1410
|
try {
|
|
1311
1411
|
built = await buildImages(targets, root, {
|
|
@@ -1342,7 +1442,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1342
1442
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1343
1443
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1344
1444
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1345
|
-
const { engineVersions } = await import("./node-
|
|
1445
|
+
const { engineVersions } = await import("./node-B3ydHo6t.js");
|
|
1346
1446
|
const renderer = createRenderer({
|
|
1347
1447
|
out: setup.json ? { write: () => true } : process.stdout,
|
|
1348
1448
|
output: setup.output,
|
|
@@ -1363,15 +1463,16 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1363
1463
|
}
|
|
1364
1464
|
const { engineRun, plan, sandboxVersions, budgetReached, live, notes, questions } = executed;
|
|
1365
1465
|
invocation.unhandledErrors = engineRun.unhandledErrors;
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
await
|
|
1466
|
+
const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
|
|
1467
|
+
if ((interrupted || engineRun.cancelled) && engines.length) {
|
|
1468
|
+
const { removeRunContainers } = await import("./docker-yauK-9X8.js");
|
|
1469
|
+
await removeRunContainers(ctx.runId, engines);
|
|
1369
1470
|
}
|
|
1370
1471
|
if (notes) {
|
|
1371
1472
|
const cancelled = interrupted || engineRun.cancelled;
|
|
1372
1473
|
const written = await notes.finish(cancelled);
|
|
1373
1474
|
if (!cancelled && written.length) {
|
|
1374
|
-
const { notesLine } = await import("./agent-
|
|
1475
|
+
const { notesLine } = await import("./agent-CURAwx_q.js");
|
|
1375
1476
|
const text = notesLine(ctx.config.root, ctx.runId, written);
|
|
1376
1477
|
renderer.onEvent({
|
|
1377
1478
|
type: "notice",
|
|
@@ -1388,8 +1489,8 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1388
1489
|
}
|
|
1389
1490
|
const stopped = interrupted || engineRun.cancelled;
|
|
1390
1491
|
const answers = questions && !stopped ? await questions.settle() : void 0;
|
|
1391
|
-
const { lockfiles } = await import("./state-
|
|
1392
|
-
const { writeScratchRun } = await import("./results-
|
|
1492
|
+
const { lockfiles } = await import("./state-CgjQo29K.js");
|
|
1493
|
+
const { writeScratchRun } = await import("./results-DuauQI3Z.js");
|
|
1393
1494
|
const run = await writeScratchRun({
|
|
1394
1495
|
ctx,
|
|
1395
1496
|
author: ctx.author,
|
|
@@ -1399,7 +1500,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1399
1500
|
interrupted: stopped,
|
|
1400
1501
|
budgetReached: budgetReached(),
|
|
1401
1502
|
plan,
|
|
1402
|
-
|
|
1503
|
+
reeval: version,
|
|
1403
1504
|
runtime: runtimeInfo(),
|
|
1404
1505
|
language: { name: "typescript" },
|
|
1405
1506
|
enginePackages: engineVersions,
|
|
@@ -1414,8 +1515,8 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1414
1515
|
}, () => live?.stop());
|
|
1415
1516
|
const { record } = run;
|
|
1416
1517
|
if (notes && !stopped) {
|
|
1417
|
-
const { summarizeRun } = await import("./agent-
|
|
1418
|
-
const { resolveAnalysisModel } = await import("./model-
|
|
1518
|
+
const { summarizeRun } = await import("./agent-CURAwx_q.js");
|
|
1519
|
+
const { resolveAnalysisModel } = await import("./model-NM3ID0uR.js");
|
|
1419
1520
|
const model = resolveAnalysisModel(ctx.config, "agent");
|
|
1420
1521
|
const { level, text } = await summarizeRun(ctx.config, record.runId, model);
|
|
1421
1522
|
renderer.onEvent({
|
|
@@ -1425,7 +1526,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1425
1526
|
text
|
|
1426
1527
|
});
|
|
1427
1528
|
}
|
|
1428
|
-
const { pruneDebug } = await import("./files-
|
|
1529
|
+
const { pruneDebug } = await import("./files-BAt0i00X.js");
|
|
1429
1530
|
await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
|
|
1430
1531
|
const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
|
|
1431
1532
|
renderer.onEvent({
|
|
@@ -1442,7 +1543,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1442
1543
|
const { config } = ctx;
|
|
1443
1544
|
const mode = setup.record;
|
|
1444
1545
|
if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
|
|
1445
|
-
const { record: recordCmd } = await import("./commands-
|
|
1546
|
+
const { record: recordCmd } = await import("./commands-1p-bIWhN.js");
|
|
1446
1547
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
1447
1548
|
console.log(await recordCmd(config, record.runId, {
|
|
1448
1549
|
...flags,
|
|
@@ -1468,9 +1569,10 @@ async function prepareRun(flags, configPath) {
|
|
|
1468
1569
|
agent: detectAgent().name
|
|
1469
1570
|
});
|
|
1470
1571
|
if (flags.json && !flags["dry-run"]) throw new UsageError("reeval run --json only prints the plan: add --dry-run. A run writes its record to .reeval/runs; read it with reeval show --json.");
|
|
1471
|
-
|
|
1572
|
+
const path = requireConfig(configPath);
|
|
1573
|
+
loadEnvFiles(path);
|
|
1472
1574
|
const named = flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
|
|
1473
|
-
const config = await loadConfig(
|
|
1575
|
+
const config = await loadConfig(path, "run", named);
|
|
1474
1576
|
const profile = selectProfile(config, flags.profile);
|
|
1475
1577
|
let runs;
|
|
1476
1578
|
if (flags.runs !== void 0) {
|
|
@@ -1479,9 +1581,10 @@ async function prepareRun(flags, configPath) {
|
|
|
1479
1581
|
}
|
|
1480
1582
|
const sandboxes = [environmentSandbox(config.environment, config.sandbox), ...profile?.defaults.environment ? [environmentSandbox(profile.defaults.environment, config.sandbox)] : []];
|
|
1481
1583
|
const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1584
|
+
const container = sandboxes.find((s) => s.kind === "docker");
|
|
1585
|
+
if (!container) return hostCapacity();
|
|
1586
|
+
const { dockerCapacity } = await import("./src-pJZOrYR9.js");
|
|
1587
|
+
return await dockerCapacity(container.engine) ?? hostCapacity();
|
|
1485
1588
|
});
|
|
1486
1589
|
return {
|
|
1487
1590
|
output,
|
|
@@ -1496,9 +1599,9 @@ async function prepareRun(flags, configPath) {
|
|
|
1496
1599
|
json: !!flags.json,
|
|
1497
1600
|
force: !!flags.force,
|
|
1498
1601
|
verbosity: verbosityOf(flags),
|
|
1499
|
-
configFile:
|
|
1602
|
+
configFile: configLabel(path),
|
|
1500
1603
|
/** Named in the plan line when the project root isn't the current directory. */
|
|
1501
|
-
configElsewhere:
|
|
1604
|
+
configElsewhere: dirname(path) !== process.cwd(),
|
|
1502
1605
|
/** `undefined` is a new run; `''` is newest resumable; a string is a run id or prefix. */
|
|
1503
1606
|
resume: flags.resume
|
|
1504
1607
|
};
|
|
@@ -1507,11 +1610,11 @@ async function prepareRun(flags, configPath) {
|
|
|
1507
1610
|
async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
|
|
1508
1611
|
const startedAt = /* @__PURE__ */ new Date();
|
|
1509
1612
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
1510
|
-
const { worktreePatch } = await import("./state-
|
|
1613
|
+
const { worktreePatch } = await import("./state-CgjQo29K.js");
|
|
1511
1614
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
1512
1615
|
const who = await author(config.root);
|
|
1513
|
-
const { stagingDir, sweepStaging } = await import("./results-
|
|
1514
|
-
const { scratchDir } = await import("./files-
|
|
1616
|
+
const { stagingDir, sweepStaging } = await import("./results-DuauQI3Z.js");
|
|
1617
|
+
const { scratchDir } = await import("./files-ZZPqfAo6.js");
|
|
1515
1618
|
const scratch = scratchDir(config.root);
|
|
1516
1619
|
if (resume !== void 0) {
|
|
1517
1620
|
const target = await resolveResume(config.root, resume || void 0);
|
|
@@ -1522,8 +1625,8 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1522
1625
|
artifacts: target.previousArtifacts
|
|
1523
1626
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
1524
1627
|
if (!dryRun) try {
|
|
1525
|
-
const { removeRunContainers } = await import("./docker-
|
|
1526
|
-
await removeRunContainers(target.runId);
|
|
1628
|
+
const { removeRunContainers } = await import("./docker-yauK-9X8.js");
|
|
1629
|
+
await removeRunContainers(target.runId, ["docker", "podman"]);
|
|
1527
1630
|
} catch {}
|
|
1528
1631
|
return {
|
|
1529
1632
|
runId: target.runId,
|
|
@@ -1593,8 +1696,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1593
1696
|
configFile,
|
|
1594
1697
|
completed: []
|
|
1595
1698
|
}));
|
|
1596
|
-
const { runEngine } = await import("./node-
|
|
1597
|
-
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-
|
|
1699
|
+
const { runEngine } = await import("./node-B3ydHo6t.js");
|
|
1700
|
+
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-C9Cy27Ay.js");
|
|
1598
1701
|
let plan;
|
|
1599
1702
|
let sandboxVersions = { images: [] };
|
|
1600
1703
|
let preflightStop;
|
|
@@ -1621,7 +1724,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1621
1724
|
} : p);
|
|
1622
1725
|
toRun = skipped;
|
|
1623
1726
|
for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
|
|
1624
|
-
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-
|
|
1727
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-NM3ID0uR.js") : void 0;
|
|
1625
1728
|
plan = await buildPlan({
|
|
1626
1729
|
planned: skipped,
|
|
1627
1730
|
profile,
|
|
@@ -1637,7 +1740,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1637
1740
|
completed: completed.size
|
|
1638
1741
|
};
|
|
1639
1742
|
run.completed = [...completed];
|
|
1640
|
-
const { imageStages, imageTargets } = await import("./images-
|
|
1743
|
+
const { imageStages, imageTargets } = await import("./images-0On0cztu.js");
|
|
1641
1744
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1642
1745
|
id: p.id,
|
|
1643
1746
|
harness: p.attempt.harness,
|
|
@@ -1658,10 +1761,16 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1658
1761
|
if (setup.dryRun) {
|
|
1659
1762
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
1660
1763
|
lines.push(...planTable(plan, todos));
|
|
1661
|
-
const { imageStatusLines } = await import("./images-
|
|
1662
|
-
const { dockerAvailable } = await import("./src-
|
|
1663
|
-
|
|
1664
|
-
|
|
1764
|
+
const { imageStatusLines } = await import("./images-0On0cztu.js");
|
|
1765
|
+
const { dockerAvailable, engineSpec } = await import("./src-pJZOrYR9.js");
|
|
1766
|
+
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
1767
|
+
const down = [];
|
|
1768
|
+
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
1769
|
+
if (targets.length && !down.length) lines.push(...await imageStatusLines(targets, config.root));
|
|
1770
|
+
else if (targets.length) {
|
|
1771
|
+
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
1772
|
+
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
1773
|
+
}
|
|
1665
1774
|
}
|
|
1666
1775
|
lines.push(...plan.warnings.map((w) => `warning: ${w}`));
|
|
1667
1776
|
lines.push(...plan.hints.map((h) => `hint: ${h}`));
|
|
@@ -1673,7 +1782,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1673
1782
|
});
|
|
1674
1783
|
if (setup.dryRun) return false;
|
|
1675
1784
|
checkAttemptBudget(plan, setup.force, config.file);
|
|
1676
|
-
const analysis = config.analysis.agent ? await import("./model-
|
|
1785
|
+
const analysis = config.analysis.agent ? await import("./model-NM3ID0uR.js") : null;
|
|
1677
1786
|
analysis?.analysisModelName(config, "agent");
|
|
1678
1787
|
const questionModels = /* @__PURE__ */ new Set();
|
|
1679
1788
|
const asked = [];
|
|
@@ -1682,7 +1791,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1682
1791
|
const questions = (row.plan.questions ?? []).map((q) => ({
|
|
1683
1792
|
question: q.question,
|
|
1684
1793
|
model: models.questionModel(config, q, row.id),
|
|
1685
|
-
...q.across ? { across: q.across } : {}
|
|
1794
|
+
...q.across ? { across: q.across } : {},
|
|
1795
|
+
...q.state ? { state: q.state } : {},
|
|
1796
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
1686
1797
|
}));
|
|
1687
1798
|
for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
|
|
1688
1799
|
if (questions.length) asked.push({
|
|
@@ -1703,7 +1814,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1703
1814
|
});
|
|
1704
1815
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
1705
1816
|
if (analysis) {
|
|
1706
|
-
const { startNotes } = await import("./agent-
|
|
1817
|
+
const { startNotes } = await import("./agent-CURAwx_q.js");
|
|
1707
1818
|
notes = startNotes({
|
|
1708
1819
|
config,
|
|
1709
1820
|
model: analysis.resolveAnalysisModel(config, "agent"),
|
|
@@ -1719,7 +1830,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1719
1830
|
});
|
|
1720
1831
|
}
|
|
1721
1832
|
if (asked.length) {
|
|
1722
|
-
const { questionRunner } = await import("./questions-
|
|
1833
|
+
const { questionRunner } = await import("./questions-DjAwMxZ3.js");
|
|
1723
1834
|
questions = questionRunner({
|
|
1724
1835
|
evals: asked,
|
|
1725
1836
|
stagingDir,
|
|
@@ -1736,7 +1847,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1736
1847
|
}
|
|
1737
1848
|
});
|
|
1738
1849
|
}
|
|
1739
|
-
const { prepareWorkspaces, workspaceKey } = await import("./workspace-
|
|
1850
|
+
const { prepareWorkspaces, workspaceKey } = await import("./workspace-D8sSgpiy.js");
|
|
1740
1851
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
1741
1852
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
1742
1853
|
workspace: a.workspace,
|
|
@@ -1884,14 +1995,14 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1884
1995
|
throw error;
|
|
1885
1996
|
} finally {
|
|
1886
1997
|
clearTimeout(budgetTimer);
|
|
1887
|
-
const { removePrepared } = await import("./preflight-
|
|
1998
|
+
const { removePrepared } = await import("./preflight-CPWpQeBI.js");
|
|
1888
1999
|
removePrepared(config.root, runId);
|
|
1889
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
2000
|
+
const { removePreparedWorkspaces } = await import("./workspace-D8sSgpiy.js");
|
|
1890
2001
|
removePreparedWorkspaces(config.root, runId);
|
|
1891
2002
|
}
|
|
1892
2003
|
if (preflightStop) throw preflightStop;
|
|
1893
2004
|
if (engineRun.noFiles || !engineRun.stopped && engineRun.evals.length === 0 && !interrupted) live?.remove();
|
|
1894
|
-
if (engineRun.noFiles && !filters.length && !tags.length) throw new UsageError(`No eval files match ${config.include.join(", ")}
|
|
2005
|
+
if (engineRun.noFiles && !filters.length && !tags.length) throw new UsageError(`No eval files in ${config.root} match include: ${config.include.join(", ")}. Name eval files to match it, or set include in ${config.file}.`);
|
|
1895
2006
|
if (engineRun.noFiles || !engineRun.stopped && engineRun.evals.length === 0 && !interrupted) throw noMatch(config, filters, tags);
|
|
1896
2007
|
return {
|
|
1897
2008
|
engineRun,
|
|
@@ -1920,7 +2031,7 @@ function finish(code, error, kind) {
|
|
|
1920
2031
|
const command = commandOf(argv);
|
|
1921
2032
|
logInvocation(root, {
|
|
1922
2033
|
at: new Date(startedAt).toISOString(),
|
|
1923
|
-
|
|
2034
|
+
reeval: version,
|
|
1924
2035
|
argv: redactArgv(argv),
|
|
1925
2036
|
command,
|
|
1926
2037
|
exitCode: code,
|
|
@@ -1969,4 +2080,4 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
1969
2080
|
//#endregion
|
|
1970
2081
|
export {};
|
|
1971
2082
|
|
|
1972
|
-
//# sourceMappingURL=cli-
|
|
2083
|
+
//# sourceMappingURL=cli-D303Vj5N.js.map
|