reeval 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +803 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-BdzhdRYo.js} +8 -6
- package/dist/adapter-BdzhdRYo.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-Dx-bDmlV.js} +179 -31
- package/dist/agent-Dx-bDmlV.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-l0-k7vw3.js} +361 -101
- package/dist/agents-l0-k7vw3.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
- package/dist/app-files-B-CQYB0c.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CYMpN8LC.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-CsZIcJo8.js} +171 -18
- package/dist/assemble-CsZIcJo8.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-BwSmNYkX.js} +17 -8
- package/dist/attempt-host-BwSmNYkX.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
- package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
- package/dist/{boat-DSNgZt-t.js → boat-su1Y7LPd.js} +17 -14
- package/dist/boat-su1Y7LPd.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/{capture-Ccf0Hmyx.js → capture-BqIAeFyC.js} +55 -29
- package/dist/{capture-Ccf0Hmyx.js.map → capture-BqIAeFyC.js.map} +1 -1
- package/dist/capture-D0Ln5aCP.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-s_3opn_I.js} +219 -109
- package/dist/cli-s_3opn_I.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
- package/dist/cli-spec-DxzIjmq_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-C5fIzHQa.js +803 -0
- package/dist/client-C5fIzHQa.js.map +1 -0
- package/dist/{command-DSL7I-SI.js → command-B5OAIjp7.js} +25 -21
- package/dist/command-B5OAIjp7.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BJ7KqczP.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BJ7KqczP.js.map} +1 -1
- package/dist/{command-D8oMB2fa.js → command-BXLZsEwQ.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-BXLZsEwQ.js.map} +1 -1
- package/dist/{command-Ff0T-1Hi.js → command-DA_SHQUQ.js} +16 -16
- package/dist/{command-Ff0T-1Hi.js.map → command-DA_SHQUQ.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-BKBamOcX.js} +174 -40
- package/dist/commands-BKBamOcX.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-BPvAOEmb.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0URar0t.js} +255 -89
- package/dist/config-C0URar0t.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-Sj58aJEu.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-Sj58aJEu.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-FWgSo-T1.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
- package/dist/db-Bc4L-Wcr.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-BjAWxVAG.js} +142 -248
- package/dist/db-BjAWxVAG.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-CWlhZ6U1.js} +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-Ck2GYIxX.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DG_uFgn2.js → docker-CDdK6Ity.js} +41 -16
- package/dist/docker-CDdK6Ity.js.map +1 -0
- package/dist/{docker-DFrjoNs-.js → docker-D74l67k8.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-D74l67k8.js.map} +1 -1
- package/dist/{docker--mukta3L.js → docker-DMug3l-a.js} +1 -1
- package/dist/{doctor-DWK4ttex.js → doctor-eFxyn6at.js} +85 -46
- package/dist/doctor-eFxyn6at.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-CJIKz-Ud.js} +49 -11
- package/dist/environment-CJIKz-Ud.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CXIRJ91v.js} +24 -12
- package/dist/eval-source-CXIRJ91v.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-DA4QsrHO.js} +792 -281
- package/dist/evaluate-DA4QsrHO.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
- package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
- package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
- package/dist/{files-BOBdC0uI.js → files-BVVWAS0A.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BVVWAS0A.js.map} +1 -1
- package/dist/files-BaqXvB88.js +1 -0
- package/dist/{files-tiNWe8Ih.js → files-Bey1VAex.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-Bey1VAex.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/files-CNHhSyb3.js +1 -0
- package/dist/files-CW1lChNh.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-D6j-XpaK.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-D6j-XpaK.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-F44NRCN0.js} +143 -20
- package/dist/files-F44NRCN0.js.map +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-Jqkf9SBa.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-Jqkf9SBa.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Dsc4Wi-F.js} +65 -30
- package/dist/images-Dsc4Wi-F.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/init-C0oUG3jh.js +1 -0
- package/dist/{init-Bi1Q3lRi.js → init-QB5ecPDA.js} +33 -14
- package/dist/init-QB5ecPDA.js.map +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
- package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
- package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
- package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
- package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
- package/dist/layout-DfQa4mJs.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
- package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
- package/dist/live-DwjWJTdn.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
- package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
- package/dist/{live-DL0se4Zd.js → live-y1p3oNXH.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-y1p3oNXH.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
- package/dist/model-3pZZzloU.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-DEjwetkJ.js} +1 -1
- package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-jRKSQBg6.js} +5 -5
- package/dist/{model-cgt5uvu8.js.map → model-jRKSQBg6.js.map} +1 -1
- package/dist/model-nfewGTzB.js +1 -0
- package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
- package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
- package/dist/options-DpsJ_5CU.js +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-Bx-qp8Li.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-Bx-qp8Li.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-ChO0A-xn.js} +63 -12
- package/dist/plan-ChO0A-xn.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-BioMWncI.js} +636 -9
- package/dist/plugins-BioMWncI.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-BUjk_6wT.js} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-CJSGzMzc.js} +8 -9
- package/dist/preflight-CJSGzMzc.js.map +1 -0
- package/dist/preflight-DD3fI4Hz.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-qFCXmxha.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-qFCXmxha.js.map} +1 -1
- package/dist/{prepare-DQEDXoDp.js → prepare-D7oSSFNm.js} +9 -9
- package/dist/{prepare-DQEDXoDp.js.map → prepare-D7oSSFNm.js.map} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-xHU2BpN-.js} +2 -2
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-B-flBAzR.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-B-flBAzR.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-B6StVyfG.js} +215 -48
- package/dist/providers-B6StVyfG.js.map +1 -0
- package/dist/providers-Dljf6al6.js +1 -0
- package/dist/query-BAc2tQZj.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-CkLmJv7R.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-CkLmJv7R.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-CJIh2FSX.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-CJIh2FSX.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
- package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
- package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
- package/dist/redact-DL2BTykw.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-B3wjKscW.js} +46 -11
- package/dist/remarks-B3wjKscW.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-C88amFRN.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-C88amFRN.js.map} +1 -1
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/resolve-DW92wpIZ.js +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-G576PiFJ.js} +14 -11
- package/dist/resolve-G576PiFJ.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-zVHIW6xw.js} +9 -22
- package/dist/results-zVHIW6xw.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-D_PcUXfG.js} +94 -16
- package/dist/resume-D_PcUXfG.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-BvTzCxdc.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-BvTzCxdc.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
- package/dist/routes-B0F2R-K7.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-B3r_pAq6.js} +4 -4
- package/dist/{runtime-Xm9oeC7o.js.map → runtime-B3r_pAq6.js.map} +1 -1
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-Duh3z1TV.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-Duh3z1TV.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CGEsvVRG.js +1 -0
- package/dist/{server-C00zd6zl.js → server-DUWMRKik.js} +8 -8
- package/dist/{server-C00zd6zl.js.map → server-DUWMRKik.js.map} +1 -1
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-CHuN4Nfg.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-CHuN4Nfg.js.map} +1 -1
- package/dist/source-Wz1Nrfue.js +1 -0
- package/dist/src--yl1OypK.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
- package/dist/src-B-HuDqpH.js.map +1 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-y0nOeRRE.js → src-BHJ4MWvY.js} +494 -112
- package/dist/src-BHJ4MWvY.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
- package/dist/src-BI0v8qlZ.js.map +1 -0
- package/dist/src-BL-_0QoE.js +1 -0
- package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
- package/dist/src-BqNzHHWn.js.map +1 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C-mOSczH.js +2 -0
- package/dist/src-CZaYu6ER.js +3 -0
- package/dist/src-CeBhwD8k.js +2 -0
- package/dist/src-CkEiiyON.js +2 -0
- package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
- package/dist/src-CmA5Yev6.js.map +1 -0
- package/dist/src-CwH_nEaw.js +2 -0
- package/dist/src-DA95rM6v.js +4 -0
- package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
- package/dist/src-DL-EE4DU.js.map +1 -0
- package/dist/src-De0KY5S_.js +2 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/state-B3jex5KU.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUBRDyMk.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUBRDyMk.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
- package/dist/suite-Bhs9-Z7O.js +2 -0
- package/dist/{summary-DDy9DfyJ.js → summary-CSp4cfnr.js} +25 -7
- package/dist/{summary-DDy9DfyJ.js.map → summary-CSp4cfnr.js.map} +1 -1
- package/dist/sync-CxAY6Dki.js +559 -0
- package/dist/sync-CxAY6Dki.js.map +1 -0
- package/dist/{tart-dWoJvWHc.js → tart-CcrrtSrp.js} +179 -61
- package/dist/tart-CcrrtSrp.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-F-6seRC9.js +192 -0
- package/dist/tart-bake-F-6seRC9.js.map +1 -0
- package/dist/tart-bake-RK7RaT9E.js +1 -0
- package/dist/tart-cli-B86Jqo1t.js +89 -0
- package/dist/tart-cli-B86Jqo1t.js.map +1 -0
- package/dist/tart-cli-ClvRniLQ.js +1 -0
- package/dist/tart-hosts-Brii7zHM.js +1 -0
- package/dist/tart-hosts-DPyZsg-1.js +313 -0
- package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
- package/dist/tart-images-CWSf03w8.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-CVDdJy0H.js +433 -0
- package/dist/tool-class-CVDdJy0H.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
- package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +27 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-DRxXCyIH.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-DRxXCyIH.js.map} +1 -1
- package/dist/views-i67V1ly3.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-C3Q4U63F.js} +12 -9
- package/dist/worker-entry-C3Q4U63F.js.map +1 -0
- package/dist/workspace-DhUOW8if.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-P72SXLXr.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-P72SXLXr.js.map} +1 -1
- package/package.json +12 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -1,31 +1,33 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import { g as parseDuration } from "./shared-
|
|
2
|
+
import { g as parseDuration } from "./shared-DtZwIAfy.js";
|
|
3
|
+
import { o as PACKAGE_VERSION } from "./root-BzeoyfGd.js";
|
|
4
|
+
import { a as newRunId, i as fromCwd } from "./ids-B-JKTCP8.js";
|
|
5
|
+
import { h as redactSecrets, y as registerSecret } from "./redact-DL2BTykw.js";
|
|
6
|
+
import { F as cleanupRun, P as capabilitiesOf, V as sandboxCapacity } from "./plugins-BioMWncI.js";
|
|
3
7
|
import { a as plural, i as formatDuration, n as clip } from "./text-B8-W3Cvn.js";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { a as
|
|
7
|
-
import {
|
|
8
|
-
import { y as vertexSettingsIn } from "./vertex-Eb2wqahk.js";
|
|
9
|
-
import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
|
|
10
|
-
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-COMXT1Hx.js";
|
|
8
|
+
import { y as vertexSettingsIn } from "./vertex-BEWxHoxH.js";
|
|
9
|
+
import { E as profileSandbox, P as workspaceLabel, d as environmentSandbox } from "./environment-CJIKz-Ud.js";
|
|
10
|
+
import { $ as plannedSandboxIds, Et as setupEgress, H as harnessId, L as checkLimitHarness, Ot as resolveOutput, Tt as rootlessPodmanSwitch, _ as withProfileHistory, _t as directOnCloud, a as configPathFor, bt as isDirect, c as findConfig, d as loadConfig, et as plannedSandboxes, f as malformedCredentials, g as selectProfile, gt as ciRunner, h as runJudgeModel, i as configLabel, l as givenConcurrency, m as requireConfig, o as defaultConcurrency, p as missingCredentials, u as hostCapacity, x as resolveModelGateway, y as gatewayScope } from "./config-C0URar0t.js";
|
|
11
|
+
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-eGH39PaP.js";
|
|
11
12
|
import { n as loadEnvFiles, t as inCI } from "./env-8x1O8I1M.js";
|
|
12
|
-
import {
|
|
13
|
-
import { r as openRouterFallbacks } from "./openrouter-models-
|
|
14
|
-
import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-
|
|
15
|
-
import { k as runOutcome } from "./model-
|
|
16
|
-
import { u as suiteResult } from "./suite-
|
|
17
|
-
import {
|
|
13
|
+
import { r as localStudio } from "./resolve-Bul8lHaa.js";
|
|
14
|
+
import { r as openRouterFallbacks } from "./openrouter-models-C0aRo633.js";
|
|
15
|
+
import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-CCD8XsmI.js";
|
|
16
|
+
import { k as runOutcome } from "./model-DNpKiHqO.js";
|
|
17
|
+
import { u as suiteResult } from "./suite-B9J6Afg6.js";
|
|
18
|
+
import { c as helpText, i as INSTEAD, n as FLAGS, o as commandHelpText, r as GLOBAL, t as COMMANDS } from "./cli-spec-DxzIjmq_.js";
|
|
18
19
|
import { n as stripAnsi, t as colors } from "./colors-DcXBvAlU.js";
|
|
19
|
-
import {
|
|
20
|
+
import { d as storeCredentials, i as isRemote, u as pullStore } from "./sync-CxAY6Dki.js";
|
|
21
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-B-flBAzR.js";
|
|
20
22
|
import { t as budgetMarker } from "./budget-BcxLYY5L.js";
|
|
21
|
-
import { S as
|
|
22
|
-
import { n as preflight } from "./preflight-
|
|
23
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-
|
|
24
|
-
import { _ as phaseOf, c as pointerLine, d as attemptAxes, f as debugLines, g as outcomeOf, h as offset, i as listLines, m as failureLines, n as SPINNER, p as errorCheck, s as progressText, t as LiveList, u as statusList, v as progressLine, y as tailLines } from "./list-
|
|
23
|
+
import { C as writeCurrent, S as sweepStaging, _ as restoreRetry, b as savedInvocation, c as liveAttempts, d as prepareResume, f as previousAttempt, g as resolveRetry, h as resolveResume, l as loadPrevious, m as removeStoppedStaging, n as erroredAttempts, o as keptAttempts, r as holdRetry, s as leftOutAttempts, t as completedKeysFor, u as noteRetries, v as resumeSettings, w as writeLiveProgress, x as stoppedRetry, y as savedFilters } from "./resume-D_PcUXfG.js";
|
|
24
|
+
import { n as preflight } from "./preflight-CJSGzMzc.js";
|
|
25
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-BSr4lRzI.js";
|
|
26
|
+
import { _ as phaseOf, c as pointerLine, d as attemptAxes, f as debugLines, g as outcomeOf, h as offset, i as listLines, m as failureLines, n as SPINNER, p as errorCheck, s as progressText, t as LiveList, u as statusList, v as progressLine, y as tailLines } from "./list-Dh6T0kdr.js";
|
|
25
27
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
26
28
|
import { dirname, join } from "node:path";
|
|
27
|
-
import { inspect } from "node:util";
|
|
28
29
|
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
30
|
+
import { inspect } from "node:util";
|
|
29
31
|
import { detectAgent } from "std-env";
|
|
30
32
|
import { Command, Option } from "commander";
|
|
31
33
|
import stringWidth from "string-width";
|
|
@@ -64,9 +66,17 @@ function parseCli(argv) {
|
|
|
64
66
|
for (const [subName, sub] of Object.entries(spec.sub ?? {})) {
|
|
65
67
|
const subcommand = configure(command.command(subName), `reeval ${name} ${subName}`, sub.flags);
|
|
66
68
|
subcommand.argument("[args...]").action(() => {
|
|
67
|
-
const
|
|
69
|
+
const own = valuesOf(command, spec.flags);
|
|
70
|
+
const stray = Object.keys(own).find((flag) => !sub.flags.includes(flag));
|
|
68
71
|
if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}. Run reeval --help.`);
|
|
69
|
-
|
|
72
|
+
const result = done(program, subcommand, [name, subName], sub.flags, sub.positionals);
|
|
73
|
+
parsed = {
|
|
74
|
+
...result,
|
|
75
|
+
values: {
|
|
76
|
+
...own,
|
|
77
|
+
...result.values
|
|
78
|
+
}
|
|
79
|
+
};
|
|
70
80
|
});
|
|
71
81
|
}
|
|
72
82
|
}
|
|
@@ -148,7 +158,10 @@ function usageError(error, command) {
|
|
|
148
158
|
const known = arg.startsWith("--") ? FLAGS[flag] : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[1];
|
|
149
159
|
const long = arg.startsWith("--") ? flag : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[0];
|
|
150
160
|
if (known && value !== void 0 && known.type === "boolean") return new UsageError(`--${flag} doesn't take a value: pass --${flag} on its own.`);
|
|
151
|
-
if (known)
|
|
161
|
+
if (known) {
|
|
162
|
+
const instead = INSTEAD[`${command} --${long}`];
|
|
163
|
+
return new UsageError(`--${long} isn't an option of ${command}. ${instead ? `${instead}.` : "Run reeval --help."}`);
|
|
164
|
+
}
|
|
152
165
|
const shown = arg.replace(/\r/g, "\\r").replace(/\n/g, "\\n");
|
|
153
166
|
return new UsageError(`Unknown option '${shown}'. ${suggestion ? `${suggestion} ` : ""}Run reeval --help.`);
|
|
154
167
|
}
|
|
@@ -484,9 +497,9 @@ var PlainRenderer = class {
|
|
|
484
497
|
/** On a TTY: the status line is drawn under everything else, and redrawn after each write. */
|
|
485
498
|
#live;
|
|
486
499
|
#status = "";
|
|
487
|
-
/**
|
|
488
|
-
#evals = /* @__PURE__ */ new Set();
|
|
500
|
+
/** Tries collected (not skipped), as many as the plan has, and the tries that ended. */
|
|
489
501
|
#tries = /* @__PURE__ */ new Set();
|
|
502
|
+
#plannedTries = 0;
|
|
490
503
|
#ended = /* @__PURE__ */ new Set();
|
|
491
504
|
#list;
|
|
492
505
|
#lastProgress = 0;
|
|
@@ -508,11 +521,15 @@ var PlainRenderer = class {
|
|
|
508
521
|
this.#lastWrite = this.#now();
|
|
509
522
|
this.#blank = line === "\n";
|
|
510
523
|
}
|
|
511
|
-
/**
|
|
524
|
+
/**
|
|
525
|
+
* `7 evals · 12/28 tries`: what the plan runs (or what's collected, when that's more) and how
|
|
526
|
+
* many tries ended.
|
|
527
|
+
*/
|
|
512
528
|
#drawStatus() {
|
|
513
529
|
if (!this.#live || !this.#tries.size) return;
|
|
514
|
-
const
|
|
515
|
-
const
|
|
530
|
+
const { total } = this.#list.counts();
|
|
531
|
+
const tries = Math.max(this.#plannedTries, this.#tries.size);
|
|
532
|
+
const status = this.#c.dim(`${plural(total, "eval")} · ${this.#ended.size}/${tries} tries`);
|
|
516
533
|
if (status === this.#status) return;
|
|
517
534
|
this.#o.out.write(`${CLEAR}${status}`);
|
|
518
535
|
this.#status = status;
|
|
@@ -563,6 +580,8 @@ var PlainRenderer = class {
|
|
|
563
580
|
this.#write("");
|
|
564
581
|
return;
|
|
565
582
|
case "plan":
|
|
583
|
+
this.#list.planned(e.header.evals);
|
|
584
|
+
this.#plannedTries = e.header.attempts;
|
|
566
585
|
for (const line of planLines(c, e, this.#columns)) this.#write(line);
|
|
567
586
|
if (this.#o.output.githubActions) for (const w of e.sandboxWarnings ?? []) this.#annotate(w);
|
|
568
587
|
return;
|
|
@@ -572,7 +591,6 @@ var PlainRenderer = class {
|
|
|
572
591
|
for (const x of e.evals) {
|
|
573
592
|
if (x.skipped) continue;
|
|
574
593
|
this.#queued.add(x.key);
|
|
575
|
-
this.#evals.add(x.id);
|
|
576
594
|
this.#tries.add(x.key);
|
|
577
595
|
}
|
|
578
596
|
this.#drawStatus();
|
|
@@ -919,7 +937,7 @@ var InteractiveRenderer = class {
|
|
|
919
937
|
#mount() {
|
|
920
938
|
if (this.#app || this.#closed) return;
|
|
921
939
|
const cancel = this.#o.cancel ?? (() => process.kill(process.pid, "SIGINT"));
|
|
922
|
-
const load = this.#o.load ?? (() => import("./ink-list-
|
|
940
|
+
const load = this.#o.load ?? (() => import("./ink-list-mCJypk3d.js"));
|
|
923
941
|
this.#app = load().then((m) => m.mountList({
|
|
924
942
|
store: this.#store,
|
|
925
943
|
colors: this.#c,
|
|
@@ -977,6 +995,10 @@ var InteractiveRenderer = class {
|
|
|
977
995
|
if (e.state === "start") this.#spinner.start(this.#written, e.targets.length);
|
|
978
996
|
else this.#spinner.reported();
|
|
979
997
|
return;
|
|
998
|
+
case "plan":
|
|
999
|
+
this.#list.planned(e.header.evals);
|
|
1000
|
+
this.#plain.onEvent(e);
|
|
1001
|
+
return;
|
|
980
1002
|
case "collected":
|
|
981
1003
|
this.#spinner.settle();
|
|
982
1004
|
this.#list.collected(e.file, e.evals);
|
|
@@ -1088,7 +1110,18 @@ const HISTORY_COMMANDS = {
|
|
|
1088
1110
|
compare: 1,
|
|
1089
1111
|
checkout: 1,
|
|
1090
1112
|
query: 1,
|
|
1091
|
-
analyze: 1
|
|
1113
|
+
analyze: 1,
|
|
1114
|
+
sync: 1
|
|
1115
|
+
};
|
|
1116
|
+
/** The commands that pull a remote store's new runs into its copy before reading it. */
|
|
1117
|
+
const PULLS = {
|
|
1118
|
+
history: 1,
|
|
1119
|
+
show: 1,
|
|
1120
|
+
compare: 1,
|
|
1121
|
+
checkout: 1,
|
|
1122
|
+
query: 1,
|
|
1123
|
+
analyze: 1,
|
|
1124
|
+
mcp: 1
|
|
1092
1125
|
};
|
|
1093
1126
|
const EXPECTED_ERRORS = [
|
|
1094
1127
|
"CommandError",
|
|
@@ -1096,6 +1129,8 @@ const EXPECTED_ERRORS = [
|
|
|
1096
1129
|
"HistoryError",
|
|
1097
1130
|
"StudioBuildError"
|
|
1098
1131
|
];
|
|
1132
|
+
/** Errors that exit 3: reeval couldn't reach what the run needs, not a mistake in the command. */
|
|
1133
|
+
const INFRA_ERRORS = ["PreflightInfraError", "StoreError"];
|
|
1099
1134
|
const EXIT = {
|
|
1100
1135
|
ok: 0,
|
|
1101
1136
|
failed: 1,
|
|
@@ -1133,6 +1168,10 @@ function quitWithoutRecord(runId, stagingDir) {
|
|
|
1133
1168
|
` Deleting ${fromCwd(stagingDir)} discards the run.`
|
|
1134
1169
|
].join("\n");
|
|
1135
1170
|
}
|
|
1171
|
+
/** Sweeps stale staging, naming each stale run it keeps because it can still be resumed. */
|
|
1172
|
+
async function sweep(root, keep) {
|
|
1173
|
+
for (const runId of await sweepStaging(root, keep)) process.stderr.write(`reeval: run ${runId} stopped over a day ago and can still be resumed: reeval run --resume ${runId} finishes it, reeval clean --staging deletes it.\n`);
|
|
1174
|
+
}
|
|
1136
1175
|
async function run(argv) {
|
|
1137
1176
|
const { values, positionals } = parseCli(argv);
|
|
1138
1177
|
invocation.flags = values;
|
|
@@ -1150,7 +1189,7 @@ async function run(argv) {
|
|
|
1150
1189
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
1151
1190
|
}
|
|
1152
1191
|
if (command === "init") {
|
|
1153
|
-
const { initCommand } = await import("./init-
|
|
1192
|
+
const { initCommand } = await import("./init-C0oUG3jh.js");
|
|
1154
1193
|
return initCommand(values, filters, {
|
|
1155
1194
|
out: (s) => console.log(s),
|
|
1156
1195
|
input: process.stdin,
|
|
@@ -1159,7 +1198,7 @@ async function run(argv) {
|
|
|
1159
1198
|
});
|
|
1160
1199
|
}
|
|
1161
1200
|
if (command === "doctor") {
|
|
1162
|
-
const { doctorCommand } = await import("./doctor-
|
|
1201
|
+
const { doctorCommand } = await import("./doctor-eFxyn6at.js");
|
|
1163
1202
|
return doctorCommand(values, {
|
|
1164
1203
|
out: (s) => console.log(s),
|
|
1165
1204
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -1167,7 +1206,7 @@ async function run(argv) {
|
|
|
1167
1206
|
});
|
|
1168
1207
|
}
|
|
1169
1208
|
if (command === "report") {
|
|
1170
|
-
const { reportCommand } = await import("./command-
|
|
1209
|
+
const { reportCommand } = await import("./command-BXLZsEwQ.js");
|
|
1171
1210
|
return reportCommand(values, filters, {
|
|
1172
1211
|
out: (s) => console.log(s),
|
|
1173
1212
|
err: (s) => console.error(dim(s)),
|
|
@@ -1181,8 +1220,9 @@ async function run(argv) {
|
|
|
1181
1220
|
return imageBuildCommand(values, configPath, filters.slice(1));
|
|
1182
1221
|
}
|
|
1183
1222
|
if (command === "studio") {
|
|
1184
|
-
const config = await
|
|
1185
|
-
|
|
1223
|
+
const config = await historyConfig(configPath, "studio", values.profile);
|
|
1224
|
+
localStudio(config);
|
|
1225
|
+
const { studioCommand } = await import("./command-BJ7KqczP.js");
|
|
1186
1226
|
return studioCommand({
|
|
1187
1227
|
config,
|
|
1188
1228
|
flags: values,
|
|
@@ -1192,8 +1232,8 @@ async function run(argv) {
|
|
|
1192
1232
|
}
|
|
1193
1233
|
if (command === "mcp") {
|
|
1194
1234
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
1195
|
-
const config = await
|
|
1196
|
-
const { mcpCommand } = await import("./command-
|
|
1235
|
+
const config = await historyConfig(configPath, "mcp", values.profile, values);
|
|
1236
|
+
const { mcpCommand } = await import("./command-DA_SHQUQ.js");
|
|
1197
1237
|
return mcpCommand({
|
|
1198
1238
|
config,
|
|
1199
1239
|
flags: values,
|
|
@@ -1201,8 +1241,8 @@ async function run(argv) {
|
|
|
1201
1241
|
});
|
|
1202
1242
|
}
|
|
1203
1243
|
if (command in HISTORY_COMMANDS) {
|
|
1204
|
-
const config = await
|
|
1205
|
-
const commands = await import("./commands-
|
|
1244
|
+
const config = await historyConfig(configPath, command, values.profile, values);
|
|
1245
|
+
const commands = await import("./commands-BPvAOEmb.js");
|
|
1206
1246
|
const ctx = {
|
|
1207
1247
|
config,
|
|
1208
1248
|
flags: values,
|
|
@@ -1214,16 +1254,16 @@ async function run(argv) {
|
|
|
1214
1254
|
case "show": return commands.showCommand(ctx, filters);
|
|
1215
1255
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
1216
1256
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
1257
|
+
case "sync": return commands.syncCommand(ctx, filters);
|
|
1217
1258
|
case "query": {
|
|
1218
|
-
const { queryCommand } = await import("./query-
|
|
1259
|
+
const { queryCommand } = await import("./query-BAc2tQZj.js");
|
|
1219
1260
|
return queryCommand({
|
|
1220
1261
|
...ctx,
|
|
1221
1262
|
err: (s) => console.error(dim(s))
|
|
1222
1263
|
}, filters);
|
|
1223
1264
|
}
|
|
1224
1265
|
case "analyze": {
|
|
1225
|
-
|
|
1226
|
-
const { analyzeCommand } = await import("./command-DSL7I-SI.js");
|
|
1266
|
+
const { analyzeCommand } = await import("./command-B5OAIjp7.js");
|
|
1227
1267
|
return analyzeCommand({
|
|
1228
1268
|
...ctx,
|
|
1229
1269
|
err: (s) => console.error(dim(s))
|
|
@@ -1235,14 +1275,38 @@ async function run(argv) {
|
|
|
1235
1275
|
if (command !== "run") throw new UsageError(`Unknown command "${command}". Run reeval --help.`);
|
|
1236
1276
|
return runCommand(values, configPath, filters);
|
|
1237
1277
|
}
|
|
1278
|
+
/**
|
|
1279
|
+
* The config a command that reads the history runs on: the profile `--profile`, `REEVAL_PROFILE`
|
|
1280
|
+
* or `defaultProfile` picks sets which store it reads, as it does for `reeval run`. `.env` and
|
|
1281
|
+
* `.env.local` load first, as for `reeval run`: they can hold the store's keys and the profile.
|
|
1282
|
+
*/
|
|
1283
|
+
async function historyConfig(configPath, command, flag, values = {}) {
|
|
1284
|
+
loadEnvFiles(configPath);
|
|
1285
|
+
const named = flag ?? (process.env.REEVAL_PROFILE || void 0);
|
|
1286
|
+
const loaded = await loadConfig(configPath, command, named);
|
|
1287
|
+
const config = withProfileHistory(loaded, selectProfile(loaded, flag));
|
|
1288
|
+
if (!config.history.scope) return config;
|
|
1289
|
+
const history = values["all-branches"] ? {
|
|
1290
|
+
...config.history,
|
|
1291
|
+
scope: "all"
|
|
1292
|
+
} : config.history;
|
|
1293
|
+
if (isRemote(history) && command in PULLS && !values.offline) {
|
|
1294
|
+
const { cachedAt, unreachable } = await pullStore(history);
|
|
1295
|
+
if (unreachable) console.error(yellow(`${history.store.name} is unreachable (${unreachable}); showing the runs pulled at ${cachedAt}`));
|
|
1296
|
+
}
|
|
1297
|
+
return {
|
|
1298
|
+
...config,
|
|
1299
|
+
history
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1238
1302
|
/** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
|
|
1239
1303
|
async function dryRunPrices(planned, judgeModel) {
|
|
1240
|
-
const { needsPrices } = await import("./cost-
|
|
1304
|
+
const { needsPrices } = await import("./cost-IM80LyGD.js");
|
|
1241
1305
|
if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
|
|
1242
1306
|
table: null,
|
|
1243
1307
|
why: "nothing to price"
|
|
1244
1308
|
};
|
|
1245
|
-
const { loadPrices } = await import("./prices-
|
|
1309
|
+
const { loadPrices } = await import("./prices-B1vm_lPk.js");
|
|
1246
1310
|
return loadPrices();
|
|
1247
1311
|
}
|
|
1248
1312
|
/** Asks a yes/no question on the terminal; anything but y or yes is no. */
|
|
@@ -1260,7 +1324,7 @@ async function confirm(question) {
|
|
|
1260
1324
|
}
|
|
1261
1325
|
/**
|
|
1262
1326
|
* `reeval clean`: removes local debug data and snapshots, all of them or all but the newest
|
|
1263
|
-
* `--keep` runs'.
|
|
1327
|
+
* `--keep` runs', and with `--staging` every stopped run's staging.
|
|
1264
1328
|
*/
|
|
1265
1329
|
async function cleanCommand(flags, configPath) {
|
|
1266
1330
|
const config = await loadConfig(configPath, "clean");
|
|
@@ -1269,20 +1333,31 @@ async function cleanCommand(flags, configPath) {
|
|
|
1269
1333
|
keep = Number(flags.keep);
|
|
1270
1334
|
if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
|
|
1271
1335
|
}
|
|
1272
|
-
const { debugRoot, pruneDebug } = await import("./files-
|
|
1336
|
+
const { debugRoot, pruneDebug } = await import("./files-BjOJw4sn.js");
|
|
1273
1337
|
const { formatBytes } = await import("./debug-CF4p4Lp_.js");
|
|
1274
1338
|
const { runs, bytes } = await pruneDebug(config.root, keep);
|
|
1275
1339
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1276
1340
|
console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
|
|
1341
|
+
if (flags.staging) {
|
|
1342
|
+
const removed = await removeStoppedStaging(config.root);
|
|
1343
|
+
console.log(removed.length ? `Removed the staging of ${removed.length} stopped run${removed.length === 1 ? "" : "s"}: ${removed.join(", ")}` : "No stopped runs in .reeval/staging to remove");
|
|
1344
|
+
}
|
|
1277
1345
|
if (flags.images) {
|
|
1278
|
-
const { dockerAvailable, pruneImages } = await import("./src-
|
|
1279
|
-
const { requireDocker } = await import("./providers-
|
|
1346
|
+
const { dockerAvailable, pruneImages } = await import("./src-BL-_0QoE.js");
|
|
1347
|
+
const { requireDocker } = await import("./providers-Dljf6al6.js");
|
|
1280
1348
|
const engines = [];
|
|
1281
1349
|
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1282
|
-
|
|
1350
|
+
const { tartCli } = await import("./tart-cli-ClvRniLQ.js");
|
|
1351
|
+
const tart = await tartCli(["--version"]).then((r) => r.exitCode === 0, () => false);
|
|
1352
|
+
if (!engines.length && !tart) await requireDocker();
|
|
1283
1353
|
let removed = 0;
|
|
1284
1354
|
for (const engine of engines) removed += await pruneImages(engine);
|
|
1285
|
-
console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
|
|
1355
|
+
if (engines.length) console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
|
|
1356
|
+
if (tart) {
|
|
1357
|
+
const { deleteBakedImages } = await import("./tart-bake-RK7RaT9E.js");
|
|
1358
|
+
const baked = await deleteBakedImages();
|
|
1359
|
+
console.log(baked.length ? `Deleted ${baked.length} Tart image${baked.length === 1 ? "" : "s"} with a harness baked in (reeval image build bakes them again): ${baked.join(", ")}` : "No Tart images with a harness baked in to delete");
|
|
1360
|
+
}
|
|
1286
1361
|
}
|
|
1287
1362
|
return EXIT.ok;
|
|
1288
1363
|
}
|
|
@@ -1298,7 +1373,7 @@ function verbosityOf(flags) {
|
|
|
1298
1373
|
* image.
|
|
1299
1374
|
*/
|
|
1300
1375
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1301
|
-
const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-
|
|
1376
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-Dsc4Wi-F.js");
|
|
1302
1377
|
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
|
|
1303
1378
|
...flags,
|
|
1304
1379
|
"dry-run": true
|
|
@@ -1320,8 +1395,8 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1320
1395
|
return EXIT.ok;
|
|
1321
1396
|
}
|
|
1322
1397
|
const { tartTagWarning } = await import("./tart-ref-7Qv3cm3B.js");
|
|
1323
|
-
for (const
|
|
1324
|
-
const warning = tartTagWarning(
|
|
1398
|
+
for (const image of new Set(vms.map((t) => t.sandbox.image))) {
|
|
1399
|
+
const warning = tartTagWarning(image);
|
|
1325
1400
|
if (warning) console.log(yellow(`warning: ${warning}`));
|
|
1326
1401
|
}
|
|
1327
1402
|
if (flags["dry-run"]) {
|
|
@@ -1329,7 +1404,7 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1329
1404
|
if (targets.length) for (const line of await imagePlanLines(targets, root)) console.log(line);
|
|
1330
1405
|
return EXIT.ok;
|
|
1331
1406
|
}
|
|
1332
|
-
const { PrepareError } = await import("./provider-
|
|
1407
|
+
const { PrepareError } = await import("./provider-DHT20Gl8.js");
|
|
1333
1408
|
const outcomes = [];
|
|
1334
1409
|
let refused = false;
|
|
1335
1410
|
const refusal = (id, of, others) => (error) => {
|
|
@@ -1345,7 +1420,10 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1345
1420
|
return [];
|
|
1346
1421
|
};
|
|
1347
1422
|
try {
|
|
1348
|
-
const pulled = vms.length ? await pullTartImages(vms, root, {
|
|
1423
|
+
const pulled = vms.length ? await pullTartImages(vms, root, {
|
|
1424
|
+
log: (line) => console.log(line.startsWith("warning: ") ? yellow(line) : dim(line)),
|
|
1425
|
+
force: !!flags.force
|
|
1426
|
+
}).catch(refusal("tart", vms, targets.length)) : [];
|
|
1349
1427
|
for (const p of pulled) outcomes.push(p.ok ? {
|
|
1350
1428
|
id: p.id,
|
|
1351
1429
|
evals: p.evals,
|
|
@@ -1380,7 +1458,7 @@ async function imageBuildCommand(flags, configPath, filters) {
|
|
|
1380
1458
|
if (!(error instanceof PrepareError)) throw error;
|
|
1381
1459
|
if (error.failure.usage) throw new UsageError(error.message);
|
|
1382
1460
|
if (error.failure.cause === "enospc") invocation.machineState = true;
|
|
1383
|
-
const { PreflightInfraError } = await import("./preflight-
|
|
1461
|
+
const { PreflightInfraError } = await import("./preflight-DD3fI4Hz.js");
|
|
1384
1462
|
throw new PreflightInfraError(error.message);
|
|
1385
1463
|
}
|
|
1386
1464
|
for (const o of outcomes) {
|
|
@@ -1414,7 +1492,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1414
1492
|
const outcome = runOutcome(record);
|
|
1415
1493
|
if (flags.json) {
|
|
1416
1494
|
console.error(nothing);
|
|
1417
|
-
const { suiteResult } = await import("./suite-
|
|
1495
|
+
const { suiteResult } = await import("./suite-Bhs9-Z7O.js");
|
|
1418
1496
|
process.stdout.write(`${JSON.stringify({
|
|
1419
1497
|
...suiteResult(record),
|
|
1420
1498
|
outcome
|
|
@@ -1432,21 +1510,21 @@ async function runCommand(flags, configPath, given) {
|
|
|
1432
1510
|
...resumed?.profileOf ? { profileOf: resumed.profileOf } : {}
|
|
1433
1511
|
});
|
|
1434
1512
|
const explicit = runFlags.concurrency ?? (process.env.REEVAL_CONCURRENCY || void 0);
|
|
1513
|
+
const ctx = await startRun(setup);
|
|
1435
1514
|
const startedWith = resumed ? resumed.saved ?? void 0 : {
|
|
1436
1515
|
...setup.runs === void 0 ? {} : { runs: setup.runs },
|
|
1437
1516
|
...setup.profile ? { profile: setup.profile.name } : {},
|
|
1438
|
-
...explicit === void 0 ? {} : { concurrency:
|
|
1517
|
+
...explicit === void 0 ? {} : { concurrency: ctx.concurrency },
|
|
1439
1518
|
filters: savedFilters(setup.config.root, filters),
|
|
1440
1519
|
tags: runFlags.tag ?? [],
|
|
1441
1520
|
...runFlags.trait?.length ? { traits: runFlags.trait } : {}
|
|
1442
1521
|
};
|
|
1443
|
-
const ctx = await startRun(setup);
|
|
1444
1522
|
invocation.runId = ctx.runId;
|
|
1445
1523
|
const githubOut = process.env.GITHUB_OUTPUT;
|
|
1446
1524
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1447
1525
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1448
1526
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1449
|
-
const { engineVersions } = await import("./node-
|
|
1527
|
+
const { engineVersions } = await import("./node-DM-mLGdX.js");
|
|
1450
1528
|
const terminal = setup.json && !setup.dryRun ? process.stderr : process.stdout;
|
|
1451
1529
|
const renderer = createRenderer({
|
|
1452
1530
|
out: setup.json && setup.dryRun ? { write: () => true } : terminal,
|
|
@@ -1486,7 +1564,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1486
1564
|
return EXIT.ok;
|
|
1487
1565
|
}
|
|
1488
1566
|
if (ctx.retry && ctx.previous && retried.size) {
|
|
1489
|
-
const { retryEnvironmentChanges } = await import("./retry-
|
|
1567
|
+
const { retryEnvironmentChanges } = await import("./retry-BvTzCxdc.js");
|
|
1490
1568
|
const changes = retryEnvironmentChanges(ctx.previous, ctx.stagingDir, retried);
|
|
1491
1569
|
if (changes.length) {
|
|
1492
1570
|
questions?.cancel();
|
|
@@ -1505,8 +1583,8 @@ async function runCommand(flags, configPath, given) {
|
|
|
1505
1583
|
}
|
|
1506
1584
|
finishing = true;
|
|
1507
1585
|
live?.phase("finishing");
|
|
1508
|
-
const { lockfiles } = await import("./state-
|
|
1509
|
-
const { writeScratchRun } = await import("./results-
|
|
1586
|
+
const { lockfiles } = await import("./state-B3jex5KU.js");
|
|
1587
|
+
const { writeScratchRun } = await import("./results-zVHIW6xw.js");
|
|
1510
1588
|
const run = await writeScratchRun({
|
|
1511
1589
|
ctx,
|
|
1512
1590
|
author: ctx.author,
|
|
@@ -1539,8 +1617,8 @@ async function runCommand(flags, configPath, given) {
|
|
|
1539
1617
|
...record,
|
|
1540
1618
|
status: "complete"
|
|
1541
1619
|
}) : outcome;
|
|
1542
|
-
|
|
1543
|
-
const { pruneDebug } = await import("./files-
|
|
1620
|
+
let code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1621
|
+
const { pruneDebug } = await import("./files-BjOJw4sn.js");
|
|
1544
1622
|
await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
|
|
1545
1623
|
const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
|
|
1546
1624
|
renderer.onEvent({
|
|
@@ -1554,7 +1632,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1554
1632
|
});
|
|
1555
1633
|
renderer.close();
|
|
1556
1634
|
if (retried.size) {
|
|
1557
|
-
const { retryCounts, retryLine } = await import("./retry-
|
|
1635
|
+
const { retryCounts, retryLine } = await import("./retry-BvTzCxdc.js");
|
|
1558
1636
|
const line = retryLine(retryCounts(record, retried, ctx.previous));
|
|
1559
1637
|
if (setup.json) console.error(line);
|
|
1560
1638
|
else console.log(line);
|
|
@@ -1563,15 +1641,22 @@ async function runCommand(flags, configPath, given) {
|
|
|
1563
1641
|
if (stepSummary && run.artifactDir) await appendFile(stepSummary, await readFile(summary)).catch(() => {});
|
|
1564
1642
|
const { config } = ctx;
|
|
1565
1643
|
const mode = setup.record;
|
|
1566
|
-
const { recordFile } = await import("./files-
|
|
1644
|
+
const { readRecord, recordFile } = await import("./files-BaqXvB88.js");
|
|
1567
1645
|
const rerecord = (setup.retry !== void 0 || setup.resume !== void 0) && record.status === "complete" && existsSync(recordFile(config.history.dir, record.runId));
|
|
1568
1646
|
const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI() || rerecord;
|
|
1569
1647
|
if (autoRecord && record.status !== "interrupted") {
|
|
1570
|
-
const { record: recordCmd } = await import("./commands-
|
|
1648
|
+
const { record: recordCmd } = await import("./commands-BPvAOEmb.js");
|
|
1571
1649
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
1650
|
+
const level = rerecord ? { artifacts: (await readRecord(recordFile(config.history.dir, record.runId))).artifacts } : {};
|
|
1572
1651
|
const line = await recordCmd(config, record.runId, {
|
|
1573
1652
|
...flags,
|
|
1574
|
-
...allow
|
|
1653
|
+
...allow,
|
|
1654
|
+
...level
|
|
1655
|
+
}).catch((error) => {
|
|
1656
|
+
if (error.name !== "StoreError") throw error;
|
|
1657
|
+
if (code !== EXIT.interrupted && code !== EXIT.terminated) code = EXIT.infra;
|
|
1658
|
+
invocation.machineState = true;
|
|
1659
|
+
return red(`reeval: ${redactSecrets(error.message)}`);
|
|
1575
1660
|
});
|
|
1576
1661
|
if (setup.json) console.error(line);
|
|
1577
1662
|
else console.log(line);
|
|
@@ -1581,7 +1666,7 @@ async function runCommand(flags, configPath, given) {
|
|
|
1581
1666
|
else console.log(line);
|
|
1582
1667
|
}
|
|
1583
1668
|
if (setup.json) {
|
|
1584
|
-
const { suiteResult } = await import("./suite-
|
|
1669
|
+
const { suiteResult } = await import("./suite-Bhs9-Z7O.js");
|
|
1585
1670
|
process.stdout.write(`${JSON.stringify({
|
|
1586
1671
|
...suiteResult(record),
|
|
1587
1672
|
outcome: ran
|
|
@@ -1663,18 +1748,18 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
|
|
|
1663
1748
|
const path = requireConfig(configPath);
|
|
1664
1749
|
loadEnvFiles(path);
|
|
1665
1750
|
const named = noProfile ? void 0 : flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
|
|
1666
|
-
const
|
|
1667
|
-
if (profileOf && flags.profile && !
|
|
1668
|
-
const profile = noProfile ? null : selectProfile(
|
|
1751
|
+
const loaded = await loadConfig(path, "run", named);
|
|
1752
|
+
if (profileOf && flags.profile && !loaded.profiles[flags.profile]) throw new UsageError(`run ${profileOf} was started with profile ${flags.profile}, which ${loaded.file} no longer defines. Put it back to resume the run, or start a new run without --resume.`);
|
|
1753
|
+
const profile = noProfile ? null : selectProfile(loaded, flags.profile);
|
|
1754
|
+
const config = withProfileHistory(loaded, profile);
|
|
1755
|
+
if ((flags.record || (profile?.record ?? config.record) === "always" || (profile?.record ?? config.record) === "ci" && inCI()) && !flags["dry-run"]) storeCredentials(config.history);
|
|
1669
1756
|
let runs;
|
|
1670
1757
|
if (flags.runs !== void 0) {
|
|
1671
1758
|
runs = Number(flags.runs);
|
|
1672
1759
|
if (!(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`--runs must be a whole number of at least 1 (got ${flags.runs})`);
|
|
1673
1760
|
}
|
|
1674
1761
|
const sandboxes = [...(config.environments.length ? config.environments : [void 0]).map((e) => environmentSandbox(e, config.sandbox)), ...profile ? [profileSandbox(config, profile.defaults)] : []];
|
|
1675
|
-
const concurrency =
|
|
1676
|
-
return await sandboxCapacity(sandboxes) ?? hostCapacity();
|
|
1677
|
-
});
|
|
1762
|
+
const concurrency = givenConcurrency(config, flags.concurrency);
|
|
1678
1763
|
return {
|
|
1679
1764
|
output,
|
|
1680
1765
|
config,
|
|
@@ -1682,7 +1767,14 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
|
|
|
1682
1767
|
runs,
|
|
1683
1768
|
tags: [...flags.tag ?? [], ...profile?.tags ?? []],
|
|
1684
1769
|
traits: traitFlags(flags.trait),
|
|
1685
|
-
|
|
1770
|
+
sandboxes,
|
|
1771
|
+
/**
|
|
1772
|
+
* The run's concurrency. Without one given, it's sized from where the first sandbox that
|
|
1773
|
+
* runs off the host runs (the Docker or Podman VM, the Boat account) when it answers, as a
|
|
1774
|
+
* real run is, so --dry-run shows the same; a resume reads it once the stopped run's
|
|
1775
|
+
* sandboxes are gone, since they count against it.
|
|
1776
|
+
*/
|
|
1777
|
+
sizeRun: async () => concurrency ?? defaultConcurrency(await sandboxCapacity(sandboxes) ?? hostCapacity()),
|
|
1686
1778
|
budget: profile?.budget ?? config.budget,
|
|
1687
1779
|
record: profile?.record ?? config.record,
|
|
1688
1780
|
dryRun: !!flags["dry-run"],
|
|
@@ -1701,14 +1793,14 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
|
|
|
1701
1793
|
/** What a retry, or a resume that finishes one, does to the run's errors, for its refusals. */
|
|
1702
1794
|
const retryVerb = (retry, runId) => retry.resume ? `--resume finishes run ${runId}'s --retry-errors, which runs its errors again` : "--retry-errors runs its errors again";
|
|
1703
1795
|
/** The run's id and the state it starts from: commit, uncommitted changes, author. */
|
|
1704
|
-
async function startRun({ config, profile, runs, tags, traits,
|
|
1796
|
+
async function startRun({ config, profile, runs, tags, traits, sandboxes, sizeRun, verbosity, configFile, resume, retry, dryRun }) {
|
|
1705
1797
|
const startedAt = /* @__PURE__ */ new Date();
|
|
1706
1798
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
1707
|
-
const { worktreePatch } = await import("./state-
|
|
1799
|
+
const { worktreePatch } = await import("./state-B3jex5KU.js");
|
|
1708
1800
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
1709
1801
|
const who = await author(config.root);
|
|
1710
|
-
const { stagingDir
|
|
1711
|
-
const { scratchDir } = await import("./files-
|
|
1802
|
+
const { stagingDir } = await import("./results-zVHIW6xw.js");
|
|
1803
|
+
const { scratchDir } = await import("./files-BaqXvB88.js");
|
|
1712
1804
|
const scratch = scratchDir(config.root);
|
|
1713
1805
|
if (resume !== void 0 || retry !== void 0) {
|
|
1714
1806
|
const target = await continuedRun(config.root, {
|
|
@@ -1718,15 +1810,18 @@ async function startRun({ config, profile, runs, tags, traits, concurrency, verb
|
|
|
1718
1810
|
if (!dryRun && retry === void 0) await prepareResume(target, config.root);
|
|
1719
1811
|
else if (!dryRun) holdRetry(target, config.root);
|
|
1720
1812
|
const stopped = retry === void 0 ? await stoppedRetry(config.root, target) : null;
|
|
1721
|
-
await
|
|
1813
|
+
if (!dryRun) await sweep(config.root, target.runId);
|
|
1722
1814
|
const aside = dryRun ? {
|
|
1723
1815
|
record: target.previous,
|
|
1724
1816
|
artifacts: target.previousArtifacts
|
|
1725
1817
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
1726
|
-
if (!dryRun)
|
|
1727
|
-
await cleanupRun(target.runId);
|
|
1728
|
-
|
|
1729
|
-
|
|
1818
|
+
if (!dryRun) {
|
|
1819
|
+
await cleanupRun(target.runId).catch(() => {});
|
|
1820
|
+
const boats = sandboxes.filter((s) => s.kind === "boat");
|
|
1821
|
+
if (boats.length) await cleanupRun(target.runId, boats).catch(() => {});
|
|
1822
|
+
}
|
|
1823
|
+
const concurrency = await sizeRun();
|
|
1824
|
+
const { unfoldCases } = await import("./cases-FvhbM8M5.js");
|
|
1730
1825
|
const previous = aside.record ?? target.previous;
|
|
1731
1826
|
return {
|
|
1732
1827
|
runId: target.runId,
|
|
@@ -1760,7 +1855,8 @@ async function startRun({ config, profile, runs, tags, traits, concurrency, verb
|
|
|
1760
1855
|
const stamps = new Set([scratch, config.history.dir].flatMap((dir) => existsSync(dir) ? readdirSync(dir).map((name) => name.slice(0, 16)) : []));
|
|
1761
1856
|
let runId = newRunId(startedAt, git?.sha ?? null, who);
|
|
1762
1857
|
for (let n = 1; stamps.has(runId.slice(0, 16)); n++) runId = newRunId(new Date(startedAt.getTime() + n * 1e3), git?.sha ?? null, who);
|
|
1763
|
-
await
|
|
1858
|
+
if (!dryRun) await sweep(config.root);
|
|
1859
|
+
const concurrency = await sizeRun();
|
|
1764
1860
|
return {
|
|
1765
1861
|
runId,
|
|
1766
1862
|
stagingDir: stagingDir(config.root, runId),
|
|
@@ -1819,12 +1915,12 @@ function namesDeclaredSecret(evals) {
|
|
|
1819
1915
|
*/
|
|
1820
1916
|
async function planEfforts(planned) {
|
|
1821
1917
|
const live = planned.filter((p) => !p.attempt.plan.skip);
|
|
1822
|
-
const { effortNotes, withDefaultEffort } = await import("./default-effort-
|
|
1918
|
+
const { effortNotes, withDefaultEffort } = await import("./default-effort-CWlhZ6U1.js");
|
|
1823
1919
|
let cursor;
|
|
1824
1920
|
let unchecked = "CURSOR_API_KEY isn't set";
|
|
1825
1921
|
const key = process.env.CURSOR_API_KEY;
|
|
1826
1922
|
if (key && live.some((p) => p.attempt.harness.kind === "cursor")) {
|
|
1827
|
-
const { checkCursorPlan } = await import("./cursor-catalog-
|
|
1923
|
+
const { checkCursorPlan } = await import("./cursor-catalog-FWgSo-T1.js");
|
|
1828
1924
|
const checked = await checkCursorPlan(key, live);
|
|
1829
1925
|
if (checked instanceof Error) unchecked = `its request failed: ${checked.message}`;
|
|
1830
1926
|
else {
|
|
@@ -1860,8 +1956,8 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1860
1956
|
configFile,
|
|
1861
1957
|
completed: []
|
|
1862
1958
|
}));
|
|
1863
|
-
const { runEngine } = await import("./node-
|
|
1864
|
-
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-
|
|
1959
|
+
const { runEngine } = await import("./node-DM-mLGdX.js");
|
|
1960
|
+
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-ChO0A-xn.js");
|
|
1865
1961
|
let plan;
|
|
1866
1962
|
let sandboxVersions = { images: [] };
|
|
1867
1963
|
let preflightStop;
|
|
@@ -1906,9 +2002,9 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1906
2002
|
if (ctx.retry && ctx.previous) {
|
|
1907
2003
|
const ranOn = ctx.previous.stack?.reeval;
|
|
1908
2004
|
if (ranOn && ranOn !== version) throw new UsageError(`run ${runId} ran on reeval ${ranOn}, and this is ${version}. ${retryVerb(ctx.retry, runId)} only on the reeval they ran on: install reeval ${ranOn}, or start a new run with reeval run.`);
|
|
1909
|
-
const { retryChanges } = await import("./retry-
|
|
1910
|
-
const { judgePart } = await import("./assemble-
|
|
1911
|
-
const { engineVersions } = await import("./node-
|
|
2005
|
+
const { retryChanges } = await import("./retry-BvTzCxdc.js");
|
|
2006
|
+
const { judgePart } = await import("./assemble-CYMpN8LC.js");
|
|
2007
|
+
const { engineVersions } = await import("./node-DM-mLGdX.js");
|
|
1912
2008
|
const judged = judgePart(config, profile, version, engineVersions);
|
|
1913
2009
|
const runs = new Set(planned.map((p) => p.key).filter((key) => !completed.has(key)));
|
|
1914
2010
|
const changes = await retryChanges(ctx.previous, planned, runs, config.root, judged);
|
|
@@ -1921,7 +2017,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1921
2017
|
}
|
|
1922
2018
|
}
|
|
1923
2019
|
retried = continuing && !setup.dryRun ? noteRetries(planned, stagingDir, ctx.previous, true).errored : errored;
|
|
1924
|
-
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-
|
|
2020
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-DEjwetkJ.js") : void 0;
|
|
1925
2021
|
const judge = runJudgeModel(config, profile);
|
|
1926
2022
|
const judgeModel = judge === void 0 ? null : judgeModelName(judge);
|
|
1927
2023
|
plan = await buildPlan({
|
|
@@ -1938,8 +2034,11 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1938
2034
|
});
|
|
1939
2035
|
plan.runId = runId;
|
|
1940
2036
|
const cloud = ciRunner(process.env);
|
|
1941
|
-
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
|
|
2037
|
+
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy) || isDirect(setupEgress(p.attempt.egress).policy)).map((p) => p.id))] : [];
|
|
1942
2038
|
if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
|
|
2039
|
+
const switching = concurrency > 1 ? [...new Set(skipped.filter(({ attempt }) => !attempt.plan.skip && !!attempt.egress.setup && attempt.sandbox.kind === "docker" && attempt.sandbox.engine === "podman").map((p) => p.id))] : [];
|
|
2040
|
+
const { podmanRootless } = await import("./providers-Dljf6al6.js");
|
|
2041
|
+
if (switching.length && await podmanRootless()) plan.warnings.push(rootlessPodmanSwitch(switching));
|
|
1943
2042
|
plan.warnings.push(...started.notes);
|
|
1944
2043
|
if (started.narrowed) {
|
|
1945
2044
|
leftOut = leftOutAttempts(planned, stagingDir, ctx.previous, await liveAttempts(stagingDir), new Set(excluded.map((e) => e.id)));
|
|
@@ -1954,7 +2053,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1954
2053
|
run.completed = [...completed];
|
|
1955
2054
|
const done = leftOut.filter((l) => l.done).map((l) => l.key);
|
|
1956
2055
|
kept = keptAttempts([...completed, ...done], stagingDir, ctx.previous);
|
|
1957
|
-
const { imageStages, imageTargets } = await import("./images-
|
|
2056
|
+
const { imageStages, imageTargets } = await import("./images-Dsc4Wi-F.js");
|
|
1958
2057
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1959
2058
|
id: p.attempt.family?.id ?? p.id,
|
|
1960
2059
|
harness: p.attempt.harness,
|
|
@@ -1973,13 +2072,13 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1973
2072
|
return false;
|
|
1974
2073
|
}
|
|
1975
2074
|
const elsewhere = setup.configElsewhere ? configFile : void 0;
|
|
1976
|
-
const { scratchDir } = await import("./files-
|
|
2075
|
+
const { scratchDir } = await import("./files-BaqXvB88.js");
|
|
1977
2076
|
const lines = [];
|
|
1978
2077
|
if (setup.dryRun) {
|
|
1979
2078
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
1980
2079
|
lines.push(...planTable(plan, todos));
|
|
1981
|
-
const { imageStatusLines } = await import("./images-
|
|
1982
|
-
const { dockerAvailable, engineSpec } = await import("./src-
|
|
2080
|
+
const { imageStatusLines } = await import("./images-Dsc4Wi-F.js");
|
|
2081
|
+
const { dockerAvailable, engineSpec } = await import("./src-BL-_0QoE.js");
|
|
1983
2082
|
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
1984
2083
|
const down = [];
|
|
1985
2084
|
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
@@ -1988,7 +2087,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
1988
2087
|
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
1989
2088
|
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
1990
2089
|
}
|
|
1991
|
-
const { costLine } = await import("./cost-
|
|
2090
|
+
const { costLine } = await import("./cost-IM80LyGD.js");
|
|
1992
2091
|
const cost = plan.cost && costLine(plan.cost);
|
|
1993
2092
|
if (cost) lines.push(cost);
|
|
1994
2093
|
}
|
|
@@ -2011,6 +2110,15 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2011
2110
|
sandboxWarnings: plan.sandboxWarnings
|
|
2012
2111
|
});
|
|
2013
2112
|
if (setup.dryRun) return false;
|
|
2113
|
+
if (continuing) {
|
|
2114
|
+
const away = /* @__PURE__ */ new Map();
|
|
2115
|
+
for (const { attempt } of planned) {
|
|
2116
|
+
const { sandbox } = attempt;
|
|
2117
|
+
if (sandbox.kind === "tart" && sandbox.hosts) away.set(JSON.stringify(sandbox.hosts), sandbox);
|
|
2118
|
+
if (sandbox.kind === "boat") away.set(JSON.stringify(sandbox), sandbox);
|
|
2119
|
+
}
|
|
2120
|
+
if (away.size) await cleanupRun(runId, [...away.values()]).catch(() => {});
|
|
2121
|
+
}
|
|
2014
2122
|
checkAttemptBudget(plan, setup.force, config.file);
|
|
2015
2123
|
const questionModels = /* @__PURE__ */ new Set();
|
|
2016
2124
|
const asked = [];
|
|
@@ -2063,7 +2171,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2063
2171
|
});
|
|
2064
2172
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
2065
2173
|
if (asked.length) {
|
|
2066
|
-
const { questionRunner } = await import("./questions-
|
|
2174
|
+
const { questionRunner } = await import("./questions-CJIh2FSX.js");
|
|
2067
2175
|
const warned = /* @__PURE__ */ new Set();
|
|
2068
2176
|
questions = questionRunner({
|
|
2069
2177
|
evals: asked,
|
|
@@ -2105,7 +2213,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2105
2213
|
}
|
|
2106
2214
|
});
|
|
2107
2215
|
}
|
|
2108
|
-
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-
|
|
2216
|
+
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-DhUOW8if.js");
|
|
2109
2217
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
2110
2218
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
2111
2219
|
workspace: a.workspace,
|
|
@@ -2283,9 +2391,9 @@ async function executeRun(ctx, setup, filters, renderer, started) {
|
|
|
2283
2391
|
throw error;
|
|
2284
2392
|
} finally {
|
|
2285
2393
|
clearTimeout(budgetTimer);
|
|
2286
|
-
const { removePrepared } = await import("./preflight-
|
|
2394
|
+
const { removePrepared } = await import("./preflight-BUjk_6wT.js");
|
|
2287
2395
|
removePrepared(config.root, runId);
|
|
2288
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
2396
|
+
const { removePreparedWorkspaces } = await import("./workspace-DhUOW8if.js");
|
|
2289
2397
|
removePreparedWorkspaces(config.root, runId);
|
|
2290
2398
|
}
|
|
2291
2399
|
if (preflightStop) throw preflightStop;
|
|
@@ -2361,13 +2469,15 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
2361
2469
|
process.exitCode = code;
|
|
2362
2470
|
}, (error) => {
|
|
2363
2471
|
const name = error.name;
|
|
2364
|
-
const
|
|
2472
|
+
const infra = INFRA_ERRORS.includes(name);
|
|
2473
|
+
if (name === "StoreError") invocation.machineState = true;
|
|
2474
|
+
const expected = error instanceof UsageError || EXPECTED_ERRORS.includes(name) || infra;
|
|
2365
2475
|
console.error(red(`reeval: ${redactSecrets(error.message)}`));
|
|
2366
2476
|
if (!expected && process.env.REEVAL_DEBUG) console.error(redactSecrets(inspect(error)));
|
|
2367
|
-
const code =
|
|
2477
|
+
const code = infra ? EXIT.infra : EXIT.usage;
|
|
2368
2478
|
finish(code, error, error instanceof UsageError ? "usage" : expected ? "expected" : "crash");
|
|
2369
2479
|
process.exitCode = code;
|
|
2370
2480
|
});
|
|
2371
2481
|
export {};
|
|
2372
2482
|
|
|
2373
|
-
//# sourceMappingURL=cli-
|
|
2483
|
+
//# sourceMappingURL=cli-s_3opn_I.js.map
|