reeval 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
- package/dist/adapter-CRKYPkRt.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
- package/dist/agent-BgH4-kFc.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
- package/dist/command-8_Cg5v6b.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
- package/dist/command-C-HeGlap.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
- package/dist/command-Cr1zjRvl.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/db-Cnh3jIv_.js +1530 -0
- package/dist/db-Cnh3jIv_.js.map +1 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
- package/dist/docker-Cqq03H72.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
- package/dist/doctor-Dl4vMeqt.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
- package/dist/environment-Cf5HMjib.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
- package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/eval-source-CAE4y2Sr.js +146 -0
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/evaluate-D35T51AC.d.ts +45 -0
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
- package/dist/files-BRViE7m-.js.map +1 -0
- package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
- package/dist/files-BSqA2sZE.js.map +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/files-CNiJ_fZf.js +37 -0
- package/dist/files-CNiJ_fZf.js.map +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
- package/dist/init-sFBgODhu.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
- package/dist/invocations-C9ejlOYl.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
- package/dist/judge-Bg4v4e_K.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
- package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
- package/dist/model-2rBZrdZA.js.map +1 -0
- package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
- package/dist/model-3MJtpho8.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/model-DkPwypN4.js +1 -0
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
- package/dist/node-fP-H2uxn.js.map +1 -0
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
- package/dist/remarks-IFFe_oty.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
- package/dist/remote-DfJYiP-R.js.map +1 -0
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
- package/dist/resolve-D_cLa4tW.js.map +1 -0
- package/dist/results-DrKJuIQF.js +63 -0
- package/dist/results-DrKJuIQF.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
- package/dist/resume-BmIM52MJ.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/source-DcXv4de-.js +59 -0
- package/dist/source-DcXv4de-.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
- package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/src-BTqJbvj6.js +2 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BkiComGh.js +2 -0
- package/dist/src-BvgeZ4tS.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/src-CgJrz5p0.js +3 -0
- package/dist/src-Cp-2Hdib.js +2 -0
- package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/src-OESRtvOi.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +426 -143
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
- package/dist/transcript-BJrKfgRW.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +548 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +46 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/client-D5hNjpEx.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D24NOQH1.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/matchers-D4eUqx4j.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js +0 -78
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js +0 -40
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,33 +1,35 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { h as parseDuration } from "./shared-B_lvGqWz.js";
|
|
3
|
+
import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-Cf5HMjib.js";
|
|
4
|
+
import { C as pluginBuildsLabel } from "./plugins-ngQEJBjJ.js";
|
|
4
5
|
import { a as fromCwd, o as newRunId } from "./ids-Bp6ut7ZD.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
6
|
+
import { A as harnessId, B as sandboxId, E as checkLimitHarness, I as plannedSandboxes, Q as directOnCloud, Z as ciRunner, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, rt as resolveOutput, tt as isDirect, u as loadConfig, v as gatewayScope } from "./config-CneBZB84.js";
|
|
7
|
+
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-Bg4v4e_K.js";
|
|
8
|
+
import { y as vertexSettingsIn } from "./vertex-DypYjx71.js";
|
|
9
|
+
import { _ as registerSecret, m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
10
|
+
import { a as plural, i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-DTigcA64.js";
|
|
11
|
+
import { n as loadEnvFiles, t as inCI } from "./env-CARj5Axq.js";
|
|
10
12
|
import { o as PACKAGE_VERSION } from "./root-Dlff-zp8.js";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
13
|
+
import { U as LABEL, ot as imageInfo } from "./src-C0ySiej5.js";
|
|
14
|
+
import { s as debugTime } from "./debug-CgG3T071.js";
|
|
15
|
+
import { r as openRouterFallbacks } from "./openrouter-models-Bo8PZ4Ew.js";
|
|
14
16
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
15
|
-
import { D as runOutcome,
|
|
16
|
-
import { u as suiteResult } from "./suite-
|
|
17
|
-
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-
|
|
18
|
-
import { p as render } from "./summary-DIGEVr08.js";
|
|
17
|
+
import { D as runOutcome, G as triesNote, _ as formatUsage } from "./model-Bz7-j71_.js";
|
|
18
|
+
import { u as suiteResult } from "./suite-DieLXvCT.js";
|
|
19
|
+
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-CPWT1mAG.js";
|
|
19
20
|
import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
|
|
20
|
-
import {
|
|
21
|
+
import { p as render } from "./summary-CSzyBVsF.js";
|
|
22
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-7Hzg88f4.js";
|
|
21
23
|
import { t as budgetMarker } from "./budget-DBUrH4jX.js";
|
|
22
|
-
import { c as
|
|
23
|
-
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-
|
|
24
|
-
import { n as imageHarness } from "./prepare-
|
|
25
|
-
import { i as createProviders, n as attemptImage } from "./providers-
|
|
26
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-
|
|
24
|
+
import { a as prepareResume, c as writeCurrent, i as loadPrevious, l as writeLiveProgress, r as keptAttempts, s as resolveResume, t as completedKeysFor } from "./resume-BmIM52MJ.js";
|
|
25
|
+
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-CDRpog9o.js";
|
|
26
|
+
import { n as imageHarness } from "./prepare-PkARUGoF.js";
|
|
27
|
+
import { i as createProviders, n as attemptImage } from "./providers-PKoJtGK0.js";
|
|
28
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-C9ejlOYl.js";
|
|
27
29
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
28
30
|
import { dirname, join } from "node:path";
|
|
29
|
-
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
30
31
|
import { inspect } from "node:util";
|
|
32
|
+
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
31
33
|
import { detectAgent } from "std-env";
|
|
32
34
|
import { Command, Option } from "commander";
|
|
33
35
|
//#region src/cli-parse.ts
|
|
@@ -244,14 +246,13 @@ function offset(ms) {
|
|
|
244
246
|
}
|
|
245
247
|
function outcomeOf(e) {
|
|
246
248
|
if (e.state === "skipped") return "skipped";
|
|
247
|
-
if (e.meta?.errorKind === "aborted") return "
|
|
249
|
+
if (e.meta?.errorKind === "aborted") return "canceled";
|
|
248
250
|
return e.meta?.outcome ?? (e.state === "passed" ? "pass" : "fail");
|
|
249
251
|
}
|
|
250
252
|
/** What tells an eval's attempts apart: the axes that vary (`cursor:composer-2.5 · with-skill`), else the harness. */
|
|
251
253
|
function attemptAxes(a, fallback) {
|
|
252
254
|
return a?.label.replace(/\s*#\d+$/, "") || a?.harness.kind || fallback;
|
|
253
255
|
}
|
|
254
|
-
const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
|
|
255
256
|
/** A command on one line: the first line, cut to 80 characters at a word boundary. */
|
|
256
257
|
function oneLine(text, max = 80) {
|
|
257
258
|
const [first = ""] = text.trim().split("\n");
|
|
@@ -334,10 +335,31 @@ function tailLines(text, max) {
|
|
|
334
335
|
*/
|
|
335
336
|
function failureLines(c, e, options) {
|
|
336
337
|
const outcome = outcomeOf(e);
|
|
337
|
-
if (outcome === "
|
|
338
|
+
if (outcome === "canceled") return [c.yellow("canceled")];
|
|
338
339
|
const message = e.errors[0] ?? "";
|
|
339
340
|
const turns = (e.meta?.turnErrors ?? []).filter((t) => !message.includes(t.replace(/^turn \d+ error: /, ""))).map((t) => c.yellow(t));
|
|
340
|
-
return [
|
|
341
|
+
return [
|
|
342
|
+
...failureBody(c, e, outcome, message, options),
|
|
343
|
+
...otherChecks(c, e),
|
|
344
|
+
...turns
|
|
345
|
+
];
|
|
346
|
+
}
|
|
347
|
+
/** The failed check the attempt's error is, when it is one. */
|
|
348
|
+
function errorCheck(e) {
|
|
349
|
+
const message = e.errors[0] ?? "";
|
|
350
|
+
return (e.meta?.failedChecks ?? []).find((x) => x.message && message.includes(x.message.replace(/…$/, "")));
|
|
351
|
+
}
|
|
352
|
+
/** The failed checks the error isn't, each with its line: an `expect` failure doesn't end the body. */
|
|
353
|
+
function otherChecks(c, e) {
|
|
354
|
+
const shown = errorCheck(e);
|
|
355
|
+
return (e.meta?.failedChecks ?? []).filter((x) => x !== shown).map((x) => {
|
|
356
|
+
const why = x.message.startsWith(`${x.label}: `) ? x.message.slice(x.label.length + 2) : x.message;
|
|
357
|
+
return [
|
|
358
|
+
c.red(`✗ ${x.label}`),
|
|
359
|
+
x.at && c.dim(x.at),
|
|
360
|
+
why && c.dim(why)
|
|
361
|
+
].filter(Boolean).join(" ");
|
|
362
|
+
});
|
|
341
363
|
}
|
|
342
364
|
function failureBody(c, e, outcome, message, options) {
|
|
343
365
|
if (outcome === "infra_error") {
|
|
@@ -382,18 +404,17 @@ function resultLines(c, e, verbose) {
|
|
|
382
404
|
const tokens = formatUsage(record.cost.usage);
|
|
383
405
|
const notes = [
|
|
384
406
|
...record.status === "budget" ? [c.yellow("Budget reached: attempts that hadn’t started were skipped")] : [],
|
|
385
|
-
...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were
|
|
407
|
+
...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were canceled")] : [],
|
|
386
408
|
c.dim([
|
|
387
409
|
formatDuration(record.durationMs),
|
|
388
410
|
tokens,
|
|
389
411
|
`results ${e.summaryPath}`
|
|
390
412
|
].filter(Boolean).join(" · "))
|
|
391
413
|
];
|
|
392
|
-
return render(suiteResult(record), e.
|
|
414
|
+
return render(suiteResult(record), e.remarks ?? null, {
|
|
393
415
|
color: c.enabled,
|
|
394
416
|
verbose,
|
|
395
|
-
notes
|
|
396
|
-
hint: e.hint
|
|
417
|
+
notes
|
|
397
418
|
});
|
|
398
419
|
}
|
|
399
420
|
//#endregion
|
|
@@ -540,6 +561,11 @@ function locate(result) {
|
|
|
540
561
|
col: match[2]
|
|
541
562
|
} : void 0;
|
|
542
563
|
}
|
|
564
|
+
/** `evals/a.eval.ts:7` as its file and line. */
|
|
565
|
+
const splitAt = (at) => {
|
|
566
|
+
const i = at.lastIndexOf(":");
|
|
567
|
+
return [at.slice(0, i), at.slice(i + 1)];
|
|
568
|
+
};
|
|
543
569
|
var PlainRenderer = class {
|
|
544
570
|
#o;
|
|
545
571
|
#c;
|
|
@@ -557,13 +583,11 @@ var PlainRenderer = class {
|
|
|
557
583
|
#evals = /* @__PURE__ */ new Set();
|
|
558
584
|
#tries = /* @__PURE__ */ new Set();
|
|
559
585
|
#ended = /* @__PURE__ */ new Set();
|
|
560
|
-
/** Insights after the last attempt, once they've started: they take the status line. */
|
|
561
|
-
#insights = null;
|
|
562
586
|
constructor(options) {
|
|
563
587
|
this.#o = options;
|
|
564
588
|
this.#c = options.colors;
|
|
565
589
|
this.#now = options.now ?? Date.now;
|
|
566
|
-
this.#live = options.output.
|
|
590
|
+
this.#live = options.output.statusLine;
|
|
567
591
|
const { heartbeatMs } = options.output;
|
|
568
592
|
if (options.timers !== false && heartbeatMs > 0) {
|
|
569
593
|
this.#timer = setInterval(() => this.tick(), Math.min(heartbeatMs, 5e3));
|
|
@@ -575,16 +599,11 @@ var PlainRenderer = class {
|
|
|
575
599
|
this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
|
|
576
600
|
this.#lastWrite = this.#now();
|
|
577
601
|
}
|
|
578
|
-
/** `7 evals · 12/28 tries`: what's collected so far and how many tries ended
|
|
602
|
+
/** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
|
|
579
603
|
#drawStatus() {
|
|
580
|
-
if (!this.#live) return;
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
else {
|
|
584
|
-
if (!this.#tries.size) return;
|
|
585
|
-
const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
|
|
586
|
-
status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
|
|
587
|
-
}
|
|
604
|
+
if (!this.#live || !this.#tries.size) return;
|
|
605
|
+
const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
|
|
606
|
+
const status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
|
|
588
607
|
if (status === this.#status) return;
|
|
589
608
|
this.#o.out.write(`${CLEAR}${status}`);
|
|
590
609
|
this.#status = status;
|
|
@@ -712,14 +731,6 @@ var PlainRenderer = class {
|
|
|
712
731
|
this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
|
|
713
732
|
}
|
|
714
733
|
return;
|
|
715
|
-
case "insights-progress":
|
|
716
|
-
this.#insights = {
|
|
717
|
-
done: e.done,
|
|
718
|
-
total: e.total,
|
|
719
|
-
startedAt: this.#insights?.startedAt ?? e.at
|
|
720
|
-
};
|
|
721
|
-
this.#drawStatus();
|
|
722
|
-
return;
|
|
723
734
|
case "notice": {
|
|
724
735
|
const paint = {
|
|
725
736
|
info: c.dim,
|
|
@@ -774,24 +785,26 @@ var PlainRenderer = class {
|
|
|
774
785
|
if (githubActions && failed && !ungated && !result.meta?.prepare) {
|
|
775
786
|
const which = result.attempt?.label ? ` [${result.attempt.label}]` : "";
|
|
776
787
|
const title = ghEscape(`${result.id}${which}`, true);
|
|
777
|
-
const
|
|
778
|
-
const
|
|
779
|
-
|
|
780
|
-
|
|
788
|
+
const checks = (result.meta?.failedChecks ?? []).filter((x) => x.at);
|
|
789
|
+
for (const x of checks) {
|
|
790
|
+
const [file, line] = splitAt(x.at);
|
|
791
|
+
const label = ghEscape(`${result.id}${which}: ${x.label}`, true);
|
|
792
|
+
this.#write(`::error file=${ghEscape(file, true)},line=${line},title=${label}::${ghEscape(x.message)}`);
|
|
793
|
+
}
|
|
794
|
+
if (!errorCheck(result)?.at) {
|
|
795
|
+
const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n"));
|
|
796
|
+
const at = locate(result);
|
|
797
|
+
const where = at ? `,line=${at.line},col=${at.col}` : "";
|
|
798
|
+
this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
|
|
799
|
+
}
|
|
781
800
|
}
|
|
782
801
|
}
|
|
783
|
-
/** Prints the heartbeat if nothing was written for `heartbeatMs` and evals
|
|
802
|
+
/** Prints the heartbeat if nothing was written for `heartbeatMs` and evals are running. */
|
|
784
803
|
tick(now = this.#now()) {
|
|
785
804
|
const { heartbeatMs } = this.#o.output;
|
|
786
805
|
if (this.#live || heartbeatMs <= 0) return;
|
|
787
806
|
if (now - this.#lastWrite < heartbeatMs) return;
|
|
788
807
|
const c = this.#c;
|
|
789
|
-
const insights = this.#insights;
|
|
790
|
-
if (insights) {
|
|
791
|
-
const elapsed = formatDuration(now - insights.startedAt);
|
|
792
|
-
if (insights.done < insights.total) this.#write(c.dim(`… insights ${insights.done}/${insights.total} after ${elapsed}`));
|
|
793
|
-
return;
|
|
794
|
-
}
|
|
795
808
|
if (this.#running.size === 0) return;
|
|
796
809
|
const running = [...this.#running.values()].map((a) => {
|
|
797
810
|
const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
|
|
@@ -857,13 +870,13 @@ const invocation = {
|
|
|
857
870
|
};
|
|
858
871
|
/** Set by the first Ctrl-C or SIGTERM of `reeval run`. */
|
|
859
872
|
let interrupted = false;
|
|
860
|
-
/** The last signal that
|
|
873
|
+
/** The last signal that canceled the run: 130 after SIGINT, 143 after SIGTERM. */
|
|
861
874
|
let stopCode = EXIT.interrupted;
|
|
862
875
|
/**
|
|
863
|
-
* Set
|
|
864
|
-
*
|
|
876
|
+
* Set once the attempts and questions are done and whether the run stopped is decided, as
|
|
877
|
+
* live.json turns `finishing`: a signal then has nothing left to cancel.
|
|
865
878
|
*/
|
|
866
|
-
let
|
|
879
|
+
let finishing = false;
|
|
867
880
|
async function run(argv) {
|
|
868
881
|
const { values, positionals } = parseCli(argv);
|
|
869
882
|
invocation.flags = values;
|
|
@@ -881,7 +894,7 @@ async function run(argv) {
|
|
|
881
894
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
882
895
|
}
|
|
883
896
|
if (command === "init") {
|
|
884
|
-
const { initCommand } = await import("./init-
|
|
897
|
+
const { initCommand } = await import("./init-okoALIah.js");
|
|
885
898
|
return initCommand(values, filters, {
|
|
886
899
|
out: (s) => console.log(s),
|
|
887
900
|
input: process.stdin,
|
|
@@ -890,7 +903,7 @@ async function run(argv) {
|
|
|
890
903
|
});
|
|
891
904
|
}
|
|
892
905
|
if (command === "doctor") {
|
|
893
|
-
const { doctorCommand } = await import("./doctor-
|
|
906
|
+
const { doctorCommand } = await import("./doctor-Dl4vMeqt.js");
|
|
894
907
|
return doctorCommand(values, {
|
|
895
908
|
out: (s) => console.log(s),
|
|
896
909
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -898,7 +911,7 @@ async function run(argv) {
|
|
|
898
911
|
});
|
|
899
912
|
}
|
|
900
913
|
if (command === "report") {
|
|
901
|
-
const { reportCommand } = await import("./command-
|
|
914
|
+
const { reportCommand } = await import("./command-8_Cg5v6b.js");
|
|
902
915
|
return reportCommand(values, filters, {
|
|
903
916
|
out: (s) => console.log(s),
|
|
904
917
|
err: (s) => console.error(dim(s)),
|
|
@@ -913,7 +926,7 @@ async function run(argv) {
|
|
|
913
926
|
}
|
|
914
927
|
if (command === "studio") {
|
|
915
928
|
const config = await loadConfig(configPath, "studio");
|
|
916
|
-
const { studioCommand } = await import("./command-
|
|
929
|
+
const { studioCommand } = await import("./command-BlYCAqu-.js");
|
|
917
930
|
return studioCommand({
|
|
918
931
|
config,
|
|
919
932
|
flags: values,
|
|
@@ -924,7 +937,7 @@ async function run(argv) {
|
|
|
924
937
|
if (command === "mcp") {
|
|
925
938
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
926
939
|
const config = await loadConfig(configPath, "mcp");
|
|
927
|
-
const { mcpCommand } = await import("./command-
|
|
940
|
+
const { mcpCommand } = await import("./command-Cr1zjRvl.js");
|
|
928
941
|
return mcpCommand({
|
|
929
942
|
config,
|
|
930
943
|
flags: values,
|
|
@@ -933,7 +946,7 @@ async function run(argv) {
|
|
|
933
946
|
}
|
|
934
947
|
if (command in HISTORY_COMMANDS) {
|
|
935
948
|
const config = await loadConfig(configPath, command);
|
|
936
|
-
const commands = await import("./commands-
|
|
949
|
+
const commands = await import("./commands-Bj9NbmsO.js");
|
|
937
950
|
const ctx = {
|
|
938
951
|
config,
|
|
939
952
|
flags: values,
|
|
@@ -946,7 +959,7 @@ async function run(argv) {
|
|
|
946
959
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
947
960
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
948
961
|
case "query": {
|
|
949
|
-
const { queryCommand } = await import("./query-
|
|
962
|
+
const { queryCommand } = await import("./query-Bt6vLJ3m.js");
|
|
950
963
|
return queryCommand({
|
|
951
964
|
...ctx,
|
|
952
965
|
err: (s) => console.error(dim(s))
|
|
@@ -954,7 +967,7 @@ async function run(argv) {
|
|
|
954
967
|
}
|
|
955
968
|
case "analyze": {
|
|
956
969
|
loadEnvFiles(configPath);
|
|
957
|
-
const { analyzeCommand } = await import("./command-
|
|
970
|
+
const { analyzeCommand } = await import("./command-C-HeGlap.js");
|
|
958
971
|
return analyzeCommand({
|
|
959
972
|
...ctx,
|
|
960
973
|
err: (s) => console.error(dim(s))
|
|
@@ -968,12 +981,12 @@ async function run(argv) {
|
|
|
968
981
|
}
|
|
969
982
|
/** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
|
|
970
983
|
async function dryRunPrices(planned, judgeModel) {
|
|
971
|
-
const { needsPrices } = await import("./cost-
|
|
984
|
+
const { needsPrices } = await import("./cost-C5RBTTJr.js");
|
|
972
985
|
if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
|
|
973
986
|
table: null,
|
|
974
987
|
why: "nothing to price"
|
|
975
988
|
};
|
|
976
|
-
const { loadPrices } = await import("./prices-
|
|
989
|
+
const { loadPrices } = await import("./prices-QaS-kjSQ.js");
|
|
977
990
|
return loadPrices();
|
|
978
991
|
}
|
|
979
992
|
/** Asks a yes/no question on the terminal; anything but y or yes is no. */
|
|
@@ -989,7 +1002,10 @@ async function confirm(question) {
|
|
|
989
1002
|
rl.close();
|
|
990
1003
|
}
|
|
991
1004
|
}
|
|
992
|
-
/**
|
|
1005
|
+
/**
|
|
1006
|
+
* `reeval clean`: removes local debug data and snapshots, all of them or all but the newest
|
|
1007
|
+
* `--keep` runs'.
|
|
1008
|
+
*/
|
|
993
1009
|
async function cleanCommand(flags, configPath) {
|
|
994
1010
|
const config = await loadConfig(configPath, "clean");
|
|
995
1011
|
let keep = 0;
|
|
@@ -997,14 +1013,14 @@ async function cleanCommand(flags, configPath) {
|
|
|
997
1013
|
keep = Number(flags.keep);
|
|
998
1014
|
if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
|
|
999
1015
|
}
|
|
1000
|
-
const { debugRoot, pruneDebug } = await import("./files-
|
|
1001
|
-
const { formatBytes } = await import("./debug-
|
|
1016
|
+
const { debugRoot, pruneDebug } = await import("./files-C3tQtD26.js");
|
|
1017
|
+
const { formatBytes } = await import("./debug-CAx3qcaP.js");
|
|
1002
1018
|
const { runs, bytes } = await pruneDebug(config.root, keep);
|
|
1003
1019
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1004
|
-
console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
|
|
1020
|
+
console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
|
|
1005
1021
|
if (flags.images) {
|
|
1006
|
-
const { dockerAvailable, pruneImages } = await import("./src-
|
|
1007
|
-
const { requireDocker } = await import("./providers-
|
|
1022
|
+
const { dockerAvailable, pruneImages } = await import("./src-h5uDlk3m.js");
|
|
1023
|
+
const { requireDocker } = await import("./providers-DDNMbv1T.js");
|
|
1008
1024
|
const engines = [];
|
|
1009
1025
|
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1010
1026
|
if (!engines.length) await requireDocker();
|
|
@@ -1025,7 +1041,7 @@ function verbosityOf(flags) {
|
|
|
1025
1041
|
* does. Needs a config, as `reeval run` does: there is no default image.
|
|
1026
1042
|
*/
|
|
1027
1043
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1028
|
-
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-
|
|
1044
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-8A_LzVfw.js");
|
|
1029
1045
|
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
|
|
1030
1046
|
...flags,
|
|
1031
1047
|
"dry-run": true
|
|
@@ -1085,7 +1101,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1085
1101
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1086
1102
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1087
1103
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1088
|
-
const { engineVersions } = await import("./node-
|
|
1104
|
+
const { engineVersions } = await import("./node-fP-H2uxn.js");
|
|
1089
1105
|
const renderer = createRenderer({
|
|
1090
1106
|
out: setup.json ? setup.dryRun ? { write: () => true } : process.stderr : process.stdout,
|
|
1091
1107
|
output: setup.output,
|
|
@@ -1106,8 +1122,8 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1106
1122
|
const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
|
|
1107
1123
|
invocation.unhandledErrors = engineRun.unhandledErrors;
|
|
1108
1124
|
const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
|
|
1109
|
-
if ((interrupted || engineRun.
|
|
1110
|
-
const { removeRunContainers } = await import("./docker-
|
|
1125
|
+
if ((interrupted || engineRun.canceled) && engines.length) {
|
|
1126
|
+
const { removeRunContainers } = await import("./docker-DCAtFMo9.js");
|
|
1111
1127
|
await removeRunContainers(ctx.runId, engines);
|
|
1112
1128
|
}
|
|
1113
1129
|
if (engineRun.stopped || !plan) {
|
|
@@ -1115,15 +1131,17 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1115
1131
|
renderer.close();
|
|
1116
1132
|
return EXIT.ok;
|
|
1117
1133
|
}
|
|
1118
|
-
let stopped = cutShort || engineRun.
|
|
1134
|
+
let stopped = cutShort || engineRun.canceled;
|
|
1119
1135
|
if (stopped) questions?.cancel();
|
|
1120
1136
|
let answers = stopped ? await questions?.answered() : await questions?.settle();
|
|
1121
1137
|
if (!stopped && interrupted && questions) {
|
|
1122
1138
|
stopped = true;
|
|
1123
1139
|
answers = await questions?.answered();
|
|
1124
1140
|
}
|
|
1125
|
-
|
|
1126
|
-
|
|
1141
|
+
finishing = true;
|
|
1142
|
+
live?.phase("finishing");
|
|
1143
|
+
const { lockfiles } = await import("./state-C97udoh3.js");
|
|
1144
|
+
const { writeScratchRun } = await import("./results-DrKJuIQF.js");
|
|
1127
1145
|
const run = await writeScratchRun({
|
|
1128
1146
|
ctx,
|
|
1129
1147
|
author: ctx.author,
|
|
@@ -1149,56 +1167,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1149
1167
|
const { record } = run;
|
|
1150
1168
|
const outcome = runOutcome(record);
|
|
1151
1169
|
const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1152
|
-
|
|
1153
|
-
let insightLines = [];
|
|
1154
|
-
const { runInsights, wantsInsights } = await import("./run-CwIEK16q.js");
|
|
1155
|
-
const wanted = !stopped && wantsInsights(ctx.config, !flags["no-insights"]);
|
|
1156
|
-
if (wanted && interrupted) insightLines = [`insights skipped (${stopCode === EXIT.terminated ? "SIGTERM" : "Ctrl-C"}): reeval analyze ${record.runId} --insights to write them`];
|
|
1157
|
-
else if (wanted) {
|
|
1158
|
-
const { scratchDir } = await import("./files-C2sjmXhk.js");
|
|
1159
|
-
const stop = new AbortController();
|
|
1160
|
-
let stopSignal = "SIGINT";
|
|
1161
|
-
onInsightsSignal = (signal) => {
|
|
1162
|
-
if (stop.signal.aborted) {
|
|
1163
|
-
renderer.close();
|
|
1164
|
-
process.exit(code);
|
|
1165
|
-
}
|
|
1166
|
-
stopSignal = signal;
|
|
1167
|
-
stop.abort(/* @__PURE__ */ new Error(signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"));
|
|
1168
|
-
};
|
|
1169
|
-
try {
|
|
1170
|
-
const done = await runInsights({
|
|
1171
|
-
config: ctx.config,
|
|
1172
|
-
record,
|
|
1173
|
-
runDir: run.artifactDir ?? join(scratchDir(ctx.config.root), record.runId),
|
|
1174
|
-
remarks: run.insights,
|
|
1175
|
-
verbose: (ctx.verbosity ?? 0) > 0,
|
|
1176
|
-
signal: stop.signal,
|
|
1177
|
-
stopSignal: () => stopSignal,
|
|
1178
|
-
onStart: (text) => {
|
|
1179
|
-
renderer.onEvent({
|
|
1180
|
-
type: "notice",
|
|
1181
|
-
at: Date.now(),
|
|
1182
|
-
level: "info",
|
|
1183
|
-
text
|
|
1184
|
-
});
|
|
1185
|
-
},
|
|
1186
|
-
onProgress: (n, total) => {
|
|
1187
|
-
renderer.onEvent({
|
|
1188
|
-
type: "insights-progress",
|
|
1189
|
-
at: Date.now(),
|
|
1190
|
-
done: n,
|
|
1191
|
-
total
|
|
1192
|
-
});
|
|
1193
|
-
}
|
|
1194
|
-
});
|
|
1195
|
-
insights = done.insights ?? insights;
|
|
1196
|
-
insightLines = done.lines;
|
|
1197
|
-
} finally {
|
|
1198
|
-
onInsightsSignal = null;
|
|
1199
|
-
}
|
|
1200
|
-
}
|
|
1201
|
-
const { pruneDebug } = await import("./files-BAt0i00X.js");
|
|
1170
|
+
const { pruneDebug } = await import("./files-C3tQtD26.js");
|
|
1202
1171
|
await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
|
|
1203
1172
|
const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
|
|
1204
1173
|
renderer.onEvent({
|
|
@@ -1207,9 +1176,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1207
1176
|
record,
|
|
1208
1177
|
summaryPath: fromCwd(summary),
|
|
1209
1178
|
results: engineRun.evals,
|
|
1210
|
-
|
|
1211
|
-
insights,
|
|
1212
|
-
insightLines
|
|
1179
|
+
remarks: run.remarks
|
|
1213
1180
|
});
|
|
1214
1181
|
renderer.close();
|
|
1215
1182
|
const stepSummary = process.env.GITHUB_STEP_SUMMARY;
|
|
@@ -1217,7 +1184,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1217
1184
|
const { config } = ctx;
|
|
1218
1185
|
const mode = setup.record;
|
|
1219
1186
|
if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
|
|
1220
|
-
const { record: recordCmd } = await import("./commands-
|
|
1187
|
+
const { record: recordCmd } = await import("./commands-Bj9NbmsO.js");
|
|
1221
1188
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
1222
1189
|
const line = await recordCmd(config, record.runId, {
|
|
1223
1190
|
...flags,
|
|
@@ -1227,7 +1194,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1227
1194
|
else console.log(line);
|
|
1228
1195
|
}
|
|
1229
1196
|
if (setup.json) {
|
|
1230
|
-
const { suiteResult } = await import("./suite-
|
|
1197
|
+
const { suiteResult } = await import("./suite-ByT1lsdD.js");
|
|
1231
1198
|
process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
|
|
1232
1199
|
}
|
|
1233
1200
|
return code;
|
|
@@ -1260,7 +1227,7 @@ async function prepareRun(flags, configPath) {
|
|
|
1260
1227
|
const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
|
|
1261
1228
|
const container = sandboxes.find((s) => s.kind === "docker");
|
|
1262
1229
|
if (!container) return hostCapacity();
|
|
1263
|
-
const { dockerCapacity } = await import("./src-
|
|
1230
|
+
const { dockerCapacity } = await import("./src-h5uDlk3m.js");
|
|
1264
1231
|
return await dockerCapacity(container.engine) ?? hostCapacity();
|
|
1265
1232
|
});
|
|
1266
1233
|
return {
|
|
@@ -1275,8 +1242,6 @@ async function prepareRun(flags, configPath) {
|
|
|
1275
1242
|
dryRun: !!flags["dry-run"],
|
|
1276
1243
|
json: !!flags.json,
|
|
1277
1244
|
force: !!flags.force,
|
|
1278
|
-
/** `--no-insights` turns them off for this run. */
|
|
1279
|
-
insights: !flags["no-insights"],
|
|
1280
1245
|
verbosity: verbosityOf(flags),
|
|
1281
1246
|
configFile: configLabel(path),
|
|
1282
1247
|
/** Named in the plan line when the project root isn't the current directory. */
|
|
@@ -1289,11 +1254,11 @@ async function prepareRun(flags, configPath) {
|
|
|
1289
1254
|
async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
|
|
1290
1255
|
const startedAt = /* @__PURE__ */ new Date();
|
|
1291
1256
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
1292
|
-
const { worktreePatch } = await import("./state-
|
|
1257
|
+
const { worktreePatch } = await import("./state-C97udoh3.js");
|
|
1293
1258
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
1294
1259
|
const who = await author(config.root);
|
|
1295
|
-
const { stagingDir, sweepStaging } = await import("./results-
|
|
1296
|
-
const { scratchDir } = await import("./files-
|
|
1260
|
+
const { stagingDir, sweepStaging } = await import("./results-DrKJuIQF.js");
|
|
1261
|
+
const { scratchDir } = await import("./files-BEnUJW1e.js");
|
|
1297
1262
|
const scratch = scratchDir(config.root);
|
|
1298
1263
|
if (resume !== void 0) {
|
|
1299
1264
|
const target = await resolveResume(config.root, resume || void 0);
|
|
@@ -1304,7 +1269,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1304
1269
|
artifacts: target.previousArtifacts
|
|
1305
1270
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
1306
1271
|
if (!dryRun) try {
|
|
1307
|
-
const { removeRunContainers } = await import("./docker-
|
|
1272
|
+
const { removeRunContainers } = await import("./docker-DCAtFMo9.js");
|
|
1308
1273
|
await removeRunContainers(target.runId, ["docker", "podman"]);
|
|
1309
1274
|
} catch {}
|
|
1310
1275
|
return {
|
|
@@ -1375,8 +1340,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1375
1340
|
configFile,
|
|
1376
1341
|
completed: []
|
|
1377
1342
|
}));
|
|
1378
|
-
const { runEngine } = await import("./node-
|
|
1379
|
-
const { buildPlan, checkAttemptBudget, planDocument,
|
|
1343
|
+
const { runEngine } = await import("./node-fP-H2uxn.js");
|
|
1344
|
+
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-Dw2BjafJ.js");
|
|
1380
1345
|
let plan;
|
|
1381
1346
|
let sandboxVersions = { images: [] };
|
|
1382
1347
|
let preflightStop;
|
|
@@ -1384,10 +1349,13 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1384
1349
|
let budgetTimer;
|
|
1385
1350
|
let toRun = [];
|
|
1386
1351
|
let live;
|
|
1352
|
+
let kept = /* @__PURE__ */ new Map();
|
|
1387
1353
|
let questions;
|
|
1388
|
-
const onPlan = async (
|
|
1354
|
+
const onPlan = async (collected) => {
|
|
1355
|
+
const { planned, excluded, matched, unmatched } = collected;
|
|
1389
1356
|
if (unmatched.length) throw unmatchedFilter(config, unmatched);
|
|
1390
1357
|
if (matched === 0) throw noMatch(config, filters, tags);
|
|
1358
|
+
checkLimitHarness(profile, config.file, collected);
|
|
1391
1359
|
const completed = completedKeysFor(planned, stagingDir, ctx.previous);
|
|
1392
1360
|
const skipped = planned.map((p) => completed.has(p.key) ? {
|
|
1393
1361
|
...p,
|
|
@@ -1402,7 +1370,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1402
1370
|
} : p);
|
|
1403
1371
|
toRun = skipped;
|
|
1404
1372
|
for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
|
|
1405
|
-
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-
|
|
1373
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-DkPwypN4.js") : void 0;
|
|
1406
1374
|
const judge = runJudgeModel(config, profile);
|
|
1407
1375
|
const judgeModel = judge === void 0 ? null : judgeModelName(judge);
|
|
1408
1376
|
plan = await buildPlan({
|
|
@@ -1413,16 +1381,10 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1413
1381
|
root: config.root,
|
|
1414
1382
|
historyDir: config.history.dir,
|
|
1415
1383
|
questionModel: models?.questionModelLabel(config, judge) ?? null,
|
|
1416
|
-
insights: await planInsights(config, setup.insights),
|
|
1417
1384
|
judgeModel,
|
|
1418
1385
|
...setup.dryRun ? { prices: await dryRunPrices(skipped, judgeModel) } : {}
|
|
1419
1386
|
});
|
|
1420
1387
|
plan.runId = runId;
|
|
1421
|
-
if (plan.insights) {
|
|
1422
|
-
const { analysisKeyProblem } = await import("./model-Ch8kX87u.js");
|
|
1423
|
-
const problem = analysisKeyProblem(config);
|
|
1424
|
-
if (problem) plan.warnings.push(`insights will be skipped: ${problem}`);
|
|
1425
|
-
}
|
|
1426
1388
|
const cloud = ciRunner(process.env);
|
|
1427
1389
|
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
|
|
1428
1390
|
if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
|
|
@@ -1431,7 +1393,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1431
1393
|
completed: completed.size
|
|
1432
1394
|
};
|
|
1433
1395
|
run.completed = [...completed];
|
|
1434
|
-
|
|
1396
|
+
kept = keptAttempts(completed, stagingDir, ctx.previous);
|
|
1397
|
+
const { imageStages, imageTargets } = await import("./images-8A_LzVfw.js");
|
|
1435
1398
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1436
1399
|
id: p.id,
|
|
1437
1400
|
harness: p.attempt.harness,
|
|
@@ -1453,8 +1416,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1453
1416
|
if (setup.dryRun) {
|
|
1454
1417
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
1455
1418
|
lines.push(...planTable(plan, todos));
|
|
1456
|
-
const { imageStatusLines } = await import("./images-
|
|
1457
|
-
const { dockerAvailable, engineSpec } = await import("./src-
|
|
1419
|
+
const { imageStatusLines } = await import("./images-8A_LzVfw.js");
|
|
1420
|
+
const { dockerAvailable, engineSpec } = await import("./src-h5uDlk3m.js");
|
|
1458
1421
|
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
1459
1422
|
const down = [];
|
|
1460
1423
|
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
@@ -1463,14 +1426,10 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1463
1426
|
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
1464
1427
|
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
1465
1428
|
}
|
|
1466
|
-
const { costLine } = await import("./cost-
|
|
1429
|
+
const { costLine } = await import("./cost-C5RBTTJr.js");
|
|
1467
1430
|
const cost = plan.cost && costLine(plan.cost);
|
|
1468
1431
|
if (cost) lines.push(cost);
|
|
1469
1432
|
}
|
|
1470
|
-
if (setup.dryRun && plan.insights) {
|
|
1471
|
-
const { formatTokens } = await import("./usage-DF3_p5pE.js");
|
|
1472
|
-
lines.push(`insights ${plan.insights.model} · up to ${formatTokens(plan.insights.maxTokens)} tokens`);
|
|
1473
|
-
}
|
|
1474
1433
|
lines.push(...plan.warnings.map((w) => `warning: ${w}`));
|
|
1475
1434
|
lines.push(...plan.hints.map((h) => `hint: ${h}`));
|
|
1476
1435
|
renderer.onEvent({
|
|
@@ -1494,9 +1453,11 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1494
1453
|
question: q.question,
|
|
1495
1454
|
model,
|
|
1496
1455
|
...gateway ? { gateway } : {},
|
|
1456
|
+
...gateway === "openrouter" && openRouterFallbacks(scope.gateways) ? { fallbacks: true } : {},
|
|
1457
|
+
...gateway === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {},
|
|
1497
1458
|
...q.across !== void 0 ? { across: q.across } : {},
|
|
1498
1459
|
...q.choices ? { choices: q.choices } : {},
|
|
1499
|
-
|
|
1460
|
+
state: q.state,
|
|
1500
1461
|
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
1501
1462
|
};
|
|
1502
1463
|
});
|
|
@@ -1506,10 +1467,16 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1506
1467
|
questions
|
|
1507
1468
|
});
|
|
1508
1469
|
}
|
|
1470
|
+
const judges = /* @__PURE__ */ new Map();
|
|
1471
|
+
for (const p of skipped) {
|
|
1472
|
+
const j = p.attempt.judge;
|
|
1473
|
+
if (j && !p.attempt.plan.skip) judges.set(`${j.model}\0${j.gateway ?? ""}`, j);
|
|
1474
|
+
}
|
|
1509
1475
|
const missing = missingCredentials(config, process.env, {
|
|
1510
1476
|
harnesses: plan.harnessDescriptors,
|
|
1511
1477
|
profile,
|
|
1512
|
-
questionModels: [...questionModels]
|
|
1478
|
+
questionModels: [...questionModels],
|
|
1479
|
+
judges: [...judges.values()]
|
|
1513
1480
|
});
|
|
1514
1481
|
if (missing.length) {
|
|
1515
1482
|
const hint = missing.some((m) => m.includes("(for direct())")) ? unusedGatewayKey(process.env) : "";
|
|
@@ -1518,15 +1485,21 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1518
1485
|
const malformed = malformedCredentials(config, process.env, {
|
|
1519
1486
|
harnesses: plan.harnessDescriptors,
|
|
1520
1487
|
profile,
|
|
1521
|
-
questionModels: [...questionModels]
|
|
1488
|
+
questionModels: [...questionModels],
|
|
1489
|
+
judges: [...judges.values()]
|
|
1522
1490
|
});
|
|
1523
1491
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
1524
1492
|
if (asked.length) {
|
|
1525
|
-
const { questionRunner } = await import("./questions-
|
|
1493
|
+
const { questionRunner } = await import("./questions-CkAwePPR.js");
|
|
1526
1494
|
const warned = /* @__PURE__ */ new Set();
|
|
1527
1495
|
questions = questionRunner({
|
|
1528
1496
|
evals: asked,
|
|
1529
1497
|
stagingDir,
|
|
1498
|
+
resumed: {
|
|
1499
|
+
completed: new Set(run.completed),
|
|
1500
|
+
previous: ctx.previous,
|
|
1501
|
+
artifacts: ctx.previousArtifacts
|
|
1502
|
+
},
|
|
1530
1503
|
maxInputTokens: config.analysis.maxInputTokens,
|
|
1531
1504
|
maxOutputTokens: config.analysis.maxOutputTokens,
|
|
1532
1505
|
onAnswer: (id, answer) => {
|
|
@@ -1559,7 +1532,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1559
1532
|
}
|
|
1560
1533
|
});
|
|
1561
1534
|
}
|
|
1562
|
-
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-
|
|
1535
|
+
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-8uHtBSve.js");
|
|
1563
1536
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
1564
1537
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
1565
1538
|
workspace: a.workspace,
|
|
@@ -1662,7 +1635,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1662
1635
|
dir: stagingDir,
|
|
1663
1636
|
runId,
|
|
1664
1637
|
startedAt: ctx.startedAt,
|
|
1665
|
-
planned: toRun
|
|
1638
|
+
planned: toRun,
|
|
1639
|
+
kept
|
|
1666
1640
|
});
|
|
1667
1641
|
renderer.onEvent({
|
|
1668
1642
|
type: "run-start",
|
|
@@ -1687,22 +1661,22 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1687
1661
|
if (interrupted && e.type === "eval-end" && e.result.state === "skipped") return;
|
|
1688
1662
|
renderer.onEvent(e);
|
|
1689
1663
|
},
|
|
1690
|
-
|
|
1664
|
+
onCancelable: (cancel) => {
|
|
1691
1665
|
const onSignal = (signal) => {
|
|
1692
|
-
if (onInsightsSignal) return onInsightsSignal(signal === "SIGTERM" ? "SIGTERM" : "SIGINT");
|
|
1693
1666
|
stopCode = signal === "SIGTERM" ? EXIT.terminated : EXIT.interrupted;
|
|
1694
1667
|
if (interrupted) {
|
|
1695
1668
|
renderer.close();
|
|
1696
1669
|
process.exit(stopCode);
|
|
1697
1670
|
}
|
|
1698
1671
|
interrupted = true;
|
|
1672
|
+
if (finishing) return;
|
|
1699
1673
|
questions?.cancel();
|
|
1700
1674
|
live?.phase("cancelling");
|
|
1701
1675
|
renderer.onEvent({
|
|
1702
1676
|
type: "notice",
|
|
1703
1677
|
at: Date.now(),
|
|
1704
1678
|
level: "warn",
|
|
1705
|
-
text: "
|
|
1679
|
+
text: "Canceling: stopping agents and destroying sandboxes (Ctrl-C again to force)"
|
|
1706
1680
|
});
|
|
1707
1681
|
cancel();
|
|
1708
1682
|
};
|
|
@@ -1711,15 +1685,14 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1711
1685
|
}
|
|
1712
1686
|
});
|
|
1713
1687
|
cutShort = interrupted;
|
|
1714
|
-
live?.phase("finishing");
|
|
1715
1688
|
} catch (error) {
|
|
1716
1689
|
live?.remove();
|
|
1717
1690
|
throw error;
|
|
1718
1691
|
} finally {
|
|
1719
1692
|
clearTimeout(budgetTimer);
|
|
1720
|
-
const { removePrepared } = await import("./preflight-
|
|
1693
|
+
const { removePrepared } = await import("./preflight-BYXkweNN.js");
|
|
1721
1694
|
removePrepared(config.root, runId);
|
|
1722
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
1695
|
+
const { removePreparedWorkspaces } = await import("./workspace-8uHtBSve.js");
|
|
1723
1696
|
removePreparedWorkspaces(config.root, runId);
|
|
1724
1697
|
}
|
|
1725
1698
|
if (preflightStop) throw preflightStop;
|
|
@@ -1802,4 +1775,4 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
1802
1775
|
//#endregion
|
|
1803
1776
|
export {};
|
|
1804
1777
|
|
|
1805
|
-
//# sourceMappingURL=cli-
|
|
1778
|
+
//# sourceMappingURL=cli-1YQU-66F.js.map
|