reeval 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -259
- package/THIRD_PARTY_NOTICES.md +121 -121
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +41 -30
- package/dist/{adapter-DigeVFWC.js → adapter-BloGVZ_a.js} +20 -8
- package/dist/adapter-BloGVZ_a.js.map +1 -0
- package/dist/{agent-CatUEo5h.js → agent-m-_t-IuW.js} +54 -26
- package/dist/{agent-CatUEo5h.js.map → agent-m-_t-IuW.js.map} +1 -1
- package/dist/{analysis-Bxub9LMT.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-Bxub9LMT.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-C-jwhRDt.js +11 -0
- package/dist/app-files-C-jwhRDt.js.map +1 -0
- package/dist/{assemble-CF063ms6.js → assemble-BDEpelpi.js} +95 -38
- package/dist/assemble-BDEpelpi.js.map +1 -0
- package/dist/assemble-C5bfncHU.js +1 -0
- package/dist/{attempt-CGPV3BZY.js → attempt-jI0OWxlz.js} +2 -2
- package/dist/{attempt-CGPV3BZY.js.map → attempt-jI0OWxlz.js.map} +1 -1
- package/dist/{capture-BiD7lVaM.js → capture-CVRwRTnb.js} +574 -38
- package/dist/capture-CVRwRTnb.js.map +1 -0
- package/dist/capture-CherbjzB.js +1 -0
- package/dist/{cli-B5zQCR2r.js → cli-C_4gaJ50.js} +145 -142
- package/dist/cli-C_4gaJ50.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
- package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
- package/dist/client-D5hNjpEx.js.map +1 -0
- package/dist/{command-C94CQNh8.js → command-BWAuwQNk.js} +12 -12
- package/dist/{command-C94CQNh8.js.map → command-BWAuwQNk.js.map} +1 -1
- package/dist/{command-CaZgCTIw.js → command-BXLUfw4h.js} +14 -14
- package/dist/command-BXLUfw4h.js.map +1 -0
- package/dist/{command-CVtgyqlM.js → command-Be6BSE2L.js} +12 -12
- package/dist/{command-CVtgyqlM.js.map → command-Be6BSE2L.js.map} +1 -1
- package/dist/{command-BMHXHqMT.js → command-Bxdy62dW.js} +15 -14
- package/dist/command-Bxdy62dW.js.map +1 -0
- package/dist/{command-DTgBZR0k.js → command-I1euLLio.js} +35 -15
- package/dist/command-I1euLLio.js.map +1 -0
- package/dist/{commands-CjpGBUWr.js → commands-jzigbNFb.js} +23 -14
- package/dist/commands-jzigbNFb.js.map +1 -0
- package/dist/{commands-BXzIDB3X.js → commands-p1S4nw_H.js} +1 -1
- package/dist/config-BrdHA9bJ.js +3549 -0
- package/dist/config-BrdHA9bJ.js.map +1 -0
- package/dist/{contrasts-CcoMLC_y.js → contrasts-DRxMQQTC.js} +3 -2
- package/dist/contrasts-DRxMQQTC.js.map +1 -0
- package/dist/cost-D4elx933.js +359 -0
- package/dist/cost-D4elx933.js.map +1 -0
- package/dist/cost-DmnAod9x.js +1 -0
- package/dist/db-CAfksinX.js +2 -0
- package/dist/{db-BeqRiWLl.js → db-DO0p7xgP.js} +25 -8
- package/dist/db-DO0p7xgP.js.map +1 -0
- package/dist/{dist-CMjp3qqF.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-CMjp3qqF.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-BbTlMmYi.js → docker-BnRlTpZA.js} +3 -3
- package/dist/{docker-BbTlMmYi.js.map → docker-BnRlTpZA.js.map} +1 -1
- package/dist/{docker-CCXdbREk.js → docker-D01slDF3.js} +29 -28
- package/dist/docker-D01slDF3.js.map +1 -0
- package/dist/{docker-DC69dKxB.js → docker-DivfZl2u.js} +1 -1
- package/dist/{doctor-CqttrZcs.js → doctor-DQ2p0euG.js} +136 -38
- package/dist/doctor-DQ2p0euG.js.map +1 -0
- package/dist/{environment-BBz1Yes9.js → environment-CO0P0soF.js} +31 -11
- package/dist/environment-CO0P0soF.js.map +1 -0
- package/dist/{evaluate-CXQ1o-pd.js → evaluate-BUmsN7yG.js} +68 -29
- package/dist/evaluate-BUmsN7yG.js.map +1 -0
- package/dist/{evaluate-yjZKUexW.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
- package/dist/{facts-DtENlf9s.js → facts-DzOUTreB.js} +13 -5
- package/dist/facts-DzOUTreB.js.map +1 -0
- package/dist/{files-B-eUpCZ1.js → files-B-CJeeEh.js} +14 -9
- package/dist/{files-B-eUpCZ1.js.map → files-B-CJeeEh.js.map} +1 -1
- package/dist/files-Bp5j5BqI.js +1 -0
- package/dist/files-C2sjmXhk.js +1 -0
- package/dist/{files-BsQbYz2i.js → files-C_gQNVB-.js} +4 -4
- package/dist/{files-BsQbYz2i.js.map → files-C_gQNVB-.js.map} +1 -1
- package/dist/{files-COHQLEHE.js → files-kUCrswI6.js} +3 -3
- package/dist/{files-COHQLEHE.js.map → files-kUCrswI6.js.map} +1 -1
- package/dist/gateway.d.ts +17 -0
- package/dist/gateway.js +17 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{generate-Cs2TMH_k.js → generate-DZ7vp69F.js} +18 -18
- package/dist/generate-DZ7vp69F.js.map +1 -0
- package/dist/{git-DhykwgKF.js → git-Bk8DSwHy.js} +3 -3
- package/dist/{git-DhykwgKF.js.map → git-Bk8DSwHy.js.map} +1 -1
- package/dist/{harness-BAjw1fcr.js → harness-COVXSwJ6.js} +46 -15
- package/dist/harness-COVXSwJ6.js.map +1 -0
- package/dist/harness.d.ts +26 -18
- package/dist/harness.js +1 -1
- package/dist/{header-kSmWatcO.js → header-BLPP6aCU.js} +2 -2
- package/dist/{header-kSmWatcO.js.map → header-BLPP6aCU.js.map} +1 -1
- package/dist/{images-BmJRuTpm.js → images-DzLeFjLx.js} +8 -8
- package/dist/{images-BmJRuTpm.js.map → images-DzLeFjLx.js.map} +1 -1
- package/dist/index.d.ts +3 -45
- package/dist/index.js +5 -6
- package/dist/init-RgCQXZhb.js +1 -0
- package/dist/{init-Bwc9byQr.js → init-SRalM5xF.js} +6 -8
- package/dist/init-SRalM5xF.js.map +1 -0
- package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
- package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
- package/dist/{invocations-CQTbP092.js → invocations-DzyKSJBa.js} +3 -3
- package/dist/{invocations-CQTbP092.js.map → invocations-DzyKSJBa.js.map} +1 -1
- package/dist/judge-yLvjC5A3.js +385 -0
- package/dist/judge-yLvjC5A3.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +7 -5
- package/dist/judge.js.map +1 -1
- package/dist/match-Bz0nSdzN.js +204 -0
- package/dist/match-Bz0nSdzN.js.map +1 -0
- package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
- package/dist/{model-B-mel-Gv.js → model-1ZLr9Dcv.js} +59 -35
- package/dist/model-1ZLr9Dcv.js.map +1 -0
- package/dist/model-B45lDUV1.js +68 -0
- package/dist/model-B45lDUV1.js.map +1 -0
- package/dist/model-BTZq8HIZ.js +1 -0
- package/dist/{model-CWlAqmbU.js → model-CgnOpss4.js} +58 -316
- package/dist/model-CgnOpss4.js.map +1 -0
- package/dist/model-Ch8kX87u.js +1 -0
- package/dist/{node-D0mtjRAO.js → node-BSq5qHnQ.js} +12 -5
- package/dist/node-BSq5qHnQ.js.map +1 -0
- package/dist/{opencode-providers-J9colP0-.js → opencode-providers-klkqgzvb.js} +72 -62
- package/dist/opencode-providers-klkqgzvb.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
- package/dist/pairs-CHfZNRT6.js.map +1 -0
- package/dist/{patch-D4oluWzH.js → patch-DWf-I_Q_.js} +2 -2
- package/dist/{patch-D4oluWzH.js.map → patch-DWf-I_Q_.js.map} +1 -1
- package/dist/{plan-OkwUcJQ9.js → plan-BIwUzlB8.js} +63 -26
- package/dist/plan-BIwUzlB8.js.map +1 -0
- package/dist/{plugins-CUWMnSNb.js → plugins-Dkp_H3pB.js} +5 -5
- package/dist/{plugins-CUWMnSNb.js.map → plugins-Dkp_H3pB.js.map} +1 -1
- package/dist/{preflight-B6pY89bF.js → preflight-D9y-HvZQ.js} +1 -1
- package/dist/{preflight-Bjm7_ylE.js → preflight-DJCD_BDn.js} +3 -3
- package/dist/{preflight-Bjm7_ylE.js.map → preflight-DJCD_BDn.js.map} +1 -1
- package/dist/{prepare-DbdgtZKe.js → prepare-BJG_pD54.js} +2 -2
- package/dist/{prepare-CQmK3SBn.js → prepare-CIxCGG0M.js} +8 -8
- package/dist/{prepare-CQmK3SBn.js.map → prepare-CIxCGG0M.js.map} +1 -1
- package/dist/prices-BcLEXvdY.js +250 -0
- package/dist/prices-BcLEXvdY.js.map +1 -0
- package/dist/{provenance-B3Y17oCC.js → provenance-CfZpFpAh.js} +3 -3
- package/dist/{provenance-B3Y17oCC.js.map → provenance-CfZpFpAh.js.map} +1 -1
- package/dist/{providers-CXN2RFz-.js → providers-D1Zxp2he.js} +1 -1
- package/dist/{providers-eoniqK6w.js → providers-DFk_6W4K.js} +5 -5
- package/dist/{providers-eoniqK6w.js.map → providers-DFk_6W4K.js.map} +1 -1
- package/dist/{query-CLRZmWBT.js → query-CemclveB.js} +2 -2
- package/dist/{query-CLRZmWBT.js.map → query-CemclveB.js.map} +1 -1
- package/dist/query-Cvbwl804.js +1 -0
- package/dist/{questions-1XOulevV.js → questions-bAudIc04.js} +309 -47
- package/dist/questions-bAudIc04.js.map +1 -0
- package/dist/{redact-DJ9JCShu.js → redact-CRGdfV2s.js} +4 -2
- package/dist/redact-CRGdfV2s.js.map +1 -0
- package/dist/{remarks-DlJCqWRs.js → remarks-CXwSnSjR.js} +9 -9
- package/dist/{remarks-DlJCqWRs.js.map → remarks-CXwSnSjR.js.map} +1 -1
- package/dist/remarks-CyAu2DiB.js +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
- package/dist/remarks-mujJv1Ey.js.map +1 -0
- package/dist/{remote-cE2VWfkJ.js → remote-CKcEKja4.js} +129 -174
- package/dist/remote-CKcEKja4.js.map +1 -0
- package/dist/resolve-B5-uqmJ6.js +1 -0
- package/dist/{resolve-C4zeccCH.js → resolve-CBoKzQMP.js} +9 -10
- package/dist/resolve-CBoKzQMP.js.map +1 -0
- package/dist/{results-Drtt0uFl.js → results-CMgZEANy.js} +8 -8
- package/dist/{results-Drtt0uFl.js.map → results-CMgZEANy.js.map} +1 -1
- package/dist/{resume-BEFazUfy.js → resume-B9SOSnK9.js} +3 -3
- package/dist/{resume-BEFazUfy.js.map → resume-B9SOSnK9.js.map} +1 -1
- package/dist/{root-DICDHCtp.js → root-Dlff-zp8.js} +5 -5
- package/dist/root-Dlff-zp8.js.map +1 -0
- package/dist/{rules-BFzQqPIm.js → rules-VhftPOOT.js} +53 -4
- package/dist/rules-VhftPOOT.js.map +1 -0
- package/dist/{run-BeMy92Us.js → run-CwIEK16q.js} +12 -12
- package/dist/run-CwIEK16q.js.map +1 -0
- package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
- package/dist/runtime-BgQRyCLl.js.map +1 -0
- package/dist/{sandbox-CQaJj7n9.js → sandbox-BOYZ0hlo.js} +2 -2
- package/dist/{sandbox-CQaJj7n9.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CHRSgKRp.js +1 -0
- package/dist/{server-xK0les4f.js → server-TGW4WyAz.js} +26 -27
- package/dist/{server-xK0les4f.js.map → server-TGW4WyAz.js.map} +1 -1
- package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
- package/dist/shared-BDUzStad.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/{source-Q1hEKbCs.js → source-B1WvBIQt.js} +2 -2
- package/dist/{source-Q1hEKbCs.js.map → source-B1WvBIQt.js.map} +1 -1
- package/dist/source-DAzztZwU.js +1 -0
- package/dist/src-1jHbFaGW.js +1 -0
- package/dist/{src-LekQrr_S.js → src-8zoX_qrb.js} +614 -171
- package/dist/src-8zoX_qrb.js.map +1 -0
- package/dist/{src-C0CeKiLU.js → src-BYLM3hEz.js} +4 -3
- package/dist/src-BYLM3hEz.js.map +1 -0
- package/dist/{src-CFp86GtV.js → src-CHIwOy0o.js} +87 -55
- package/dist/{src-CFp86GtV.js.map → src-CHIwOy0o.js.map} +1 -1
- package/dist/src-C_9wgowC.js +2 -0
- package/dist/{src-C7_1SqDP.js → src-ChhtI6BP.js} +2 -2
- package/dist/{src-C7_1SqDP.js.map → src-ChhtI6BP.js.map} +1 -1
- package/dist/src-CyaDNG9r.js +2 -0
- package/dist/src-D005Cbqz.js +2 -0
- package/dist/src-DIfRbp5a.js +3 -0
- package/dist/{src-BhmaorOU.js → src-DKPVX2Us.js} +43 -17
- package/dist/{src-BhmaorOU.js.map → src-DKPVX2Us.js.map} +1 -1
- package/dist/src-Dg8121Xy.js +2 -0
- package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
- package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
- package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
- package/dist/src-DjRI-IeE.js.map +1 -0
- package/dist/{src-DiLMkIr9.js → src-DmUgeTlE.js} +47 -17
- package/dist/src-DmUgeTlE.js.map +1 -0
- package/dist/src-ElBqEN-S.js +2 -0
- package/dist/{src-qGoFkuJF.js → src-NSdQ9zI6.js} +88 -23
- package/dist/src-NSdQ9zI6.js.map +1 -0
- package/dist/src-YHGaUrGE.js +2 -0
- package/dist/src-mJVxw1Jm.js +2 -0
- package/dist/{state-BmZ8QVZ8.js → state-B1t143wG.js} +22 -20
- package/dist/{state-BmZ8QVZ8.js.map → state-B1t143wG.js.map} +1 -1
- package/dist/state-vznMh2wl.js +1 -0
- package/dist/stats-DSy5HfOv.js +315 -0
- package/dist/stats-DSy5HfOv.js.map +1 -0
- package/dist/suite-BsWGHzlG.js +2 -0
- package/dist/{suite-C8sw2pox.js → suite-WgT7m6Vs.js} +76 -13
- package/dist/suite-WgT7m6Vs.js.map +1 -0
- package/dist/{summary-0oQNGskv.js → summary-DIGEVr08.js} +58 -17
- package/dist/summary-DIGEVr08.js.map +1 -0
- package/dist/testing.d.ts +174 -2
- package/dist/testing.js +14 -3
- package/dist/testing.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-GraZtfHN.d.ts → types-DXBbP9Mk.d.ts} +302 -72
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/usage-DF3_p5pE.js +1 -0
- package/dist/{validate-BdoXUuL_.js → validate-BhwqrVsm.js} +48 -16
- package/dist/validate-BhwqrVsm.js.map +1 -0
- package/dist/view/index.d.ts +143 -91
- package/dist/view/index.js +7 -6
- package/dist/views-BW_C5eVH.js +2 -0
- package/dist/{views-BoUoX4_A.js → views-sXpGE6a2.js} +10 -8
- package/dist/views-sXpGE6a2.js.map +1 -0
- package/dist/{worker-entry-D0-GErmU.js → worker-entry-Duo9oAeT.js} +8 -8
- package/dist/{worker-entry-D0-GErmU.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
- package/dist/{workspace-tjZFkjxC.js → workspace-CDI68wtX.js} +45 -11
- package/dist/workspace-CDI68wtX.js.map +1 -0
- package/dist/workspace-QZCacP8A.js +1 -0
- package/package.json +8 -2
- package/skills/eval-rubric/SKILL.md +9 -0
- package/dist/adapter-DigeVFWC.js.map +0 -1
- package/dist/assemble-CF063ms6.js.map +0 -1
- package/dist/assemble-CJoD_Rzr.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-BiD7lVaM.js.map +0 -1
- package/dist/capture-Dp3q7K7Y.js +0 -1
- package/dist/cli-B5zQCR2r.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-BMHXHqMT.js.map +0 -1
- package/dist/command-CaZgCTIw.js.map +0 -1
- package/dist/command-DTgBZR0k.js.map +0 -1
- package/dist/commands-CjpGBUWr.js.map +0 -1
- package/dist/config-Dv5MG7Rp.js +0 -753
- package/dist/config-Dv5MG7Rp.js.map +0 -1
- package/dist/contrasts-CcoMLC_y.js.map +0 -1
- package/dist/db-BeqRiWLl.js.map +0 -1
- package/dist/db-CJDRVQal.js +0 -2
- package/dist/docker-CCXdbREk.js.map +0 -1
- package/dist/doctor-CqttrZcs.js.map +0 -1
- package/dist/egress-BfK_O9ce.js +0 -1241
- package/dist/egress-BfK_O9ce.js.map +0 -1
- package/dist/environment-BBz1Yes9.js.map +0 -1
- package/dist/evaluate-CXQ1o-pd.js.map +0 -1
- package/dist/facts-DtENlf9s.js.map +0 -1
- package/dist/files-DqCBW7i8.js +0 -1
- package/dist/files-Ha-jsQXf.js +0 -1
- package/dist/generate-Cs2TMH_k.js.map +0 -1
- package/dist/harness-BAjw1fcr.js.map +0 -1
- package/dist/init-Bwc9byQr.js.map +0 -1
- package/dist/init-D7Y1ny1z.js +0 -1
- package/dist/judge-Dt6KW6_O.js +0 -153
- package/dist/judge-Dt6KW6_O.js.map +0 -1
- package/dist/model-B-mel-Gv.js.map +0 -1
- package/dist/model-BGKYVpoj.js +0 -1
- package/dist/model-C7tSwT7f.js +0 -44
- package/dist/model-C7tSwT7f.js.map +0 -1
- package/dist/model-CWlAqmbU.js.map +0 -1
- package/dist/model-DHY-dfoO.js +0 -1
- package/dist/node-D0mtjRAO.js.map +0 -1
- package/dist/opencode-providers-J9colP0-.js.map +0 -1
- package/dist/options-CPUW97_7.js +0 -1116
- package/dist/options-CPUW97_7.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/plan-OkwUcJQ9.js.map +0 -1
- package/dist/query-CmBDYt0_.js +0 -1
- package/dist/questions-1XOulevV.js.map +0 -1
- package/dist/redact-DJ9JCShu.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remote-cE2VWfkJ.js.map +0 -1
- package/dist/resolve-C4zeccCH.js.map +0 -1
- package/dist/resolve-D30yZYmB.js +0 -1
- package/dist/root-DICDHCtp.js.map +0 -1
- package/dist/rules-BFzQqPIm.js.map +0 -1
- package/dist/run-BeMy92Us.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-V4-2w0l8.js +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-BDow_oW8.js +0 -1
- package/dist/src-B9L9yGKI.js +0 -1
- package/dist/src-Bcah6ZkT.js +0 -3
- package/dist/src-C0CeKiLU.js.map +0 -1
- package/dist/src-C0xowZ3C.js +0 -2
- package/dist/src-CHIfIKUu.js +0 -2
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-DLtyz4w5.js +0 -2
- package/dist/src-DPIjmReZ.js +0 -2
- package/dist/src-DXNh5mTr.js +0 -2
- package/dist/src-DiLMkIr9.js.map +0 -1
- package/dist/src-LekQrr_S.js.map +0 -1
- package/dist/src-bCnsA6Vp.js +0 -2
- package/dist/src-qGoFkuJF.js.map +0 -1
- package/dist/src-ynBVosfX.js +0 -2
- package/dist/state-BQR_BCm4.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-CnYJijyI.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-BaHPmAGy.js +0 -1
- package/dist/suite-C8sw2pox.js.map +0 -1
- package/dist/summary-0oQNGskv.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-BdoXUuL_.js.map +0 -1
- package/dist/views-BoUoX4_A.js.map +0 -1
- package/dist/views-noVQNCUP.js +0 -2
- package/dist/workspace-B90YDU_a.js +0 -1
- package/dist/workspace-tjZFkjxC.js.map +0 -1
|
@@ -1,34 +1,33 @@
|
|
|
1
|
-
import { n as isOneOf } from "./object-CLf2-tNq.js";
|
|
2
1
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
-
import { p as parseDuration } from "./shared-
|
|
4
|
-
import {
|
|
5
|
-
import { A as workspaceLabel, p as environmentSandbox } from "./environment-BBz1Yes9.js";
|
|
2
|
+
import { p as parseDuration } from "./shared-BDUzStad.js";
|
|
3
|
+
import { M as workspaceLabel, T as profileSandbox, p as environmentSandbox } from "./environment-CO0P0soF.js";
|
|
6
4
|
import { a as fromCwd, o as newRunId } from "./ids-Bp6ut7ZD.js";
|
|
7
|
-
import { _ as selectProfile, a as
|
|
5
|
+
import { G as ciRunner, K as directOnCloud, O as harnessId, P as sandboxId, Y as isDirect, Z as resolveOutput, _ as selectProfile, a as configPathFor, c as findConfig, d as loadConfig, f as malformedCredentials, g as runJudgeModel, h as resolveConcurrency, i as configLabel, j as plannedSandboxes, l as hasAnalysisModel, m as requireConfig, p as missingCredentials, u as hostCapacity, v as gatewayScope, y as resolveModelGateway } from "./config-BrdHA9bJ.js";
|
|
6
|
+
import { c as judgeModelName, p as unusedGatewayKey } from "./judge-yLvjC5A3.js";
|
|
7
|
+
import { c as registerSecret, s as redactSecrets } from "./redact-CRGdfV2s.js";
|
|
8
8
|
import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
|
|
9
|
-
import {
|
|
10
|
-
import { o as PACKAGE_VERSION } from "./root-
|
|
11
|
-
import {
|
|
12
|
-
import { C as pluginBuildsLabel } from "./plugins-CUWMnSNb.js";
|
|
13
|
-
import { s as debugTime } from "./debug-D24NOQH1.js";
|
|
9
|
+
import { G as LABEL, ot as imageInfo } from "./src-8zoX_qrb.js";
|
|
10
|
+
import { o as PACKAGE_VERSION } from "./root-Dlff-zp8.js";
|
|
11
|
+
import { C as pluginBuildsLabel } from "./plugins-Dkp_H3pB.js";
|
|
14
12
|
import { i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-CrVy5ioL.js";
|
|
13
|
+
import { s as debugTime } from "./debug-D24NOQH1.js";
|
|
15
14
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-
|
|
19
|
-
import { p as render } from "./summary-
|
|
15
|
+
import { D as runOutcome, K as triesNote, _ as formatUsage } from "./model-CgnOpss4.js";
|
|
16
|
+
import { u as suiteResult } from "./suite-WgT7m6Vs.js";
|
|
17
|
+
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-CpnbATtI.js";
|
|
18
|
+
import { p as render } from "./summary-DIGEVr08.js";
|
|
20
19
|
import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
|
|
21
|
-
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-
|
|
20
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-CfZpFpAh.js";
|
|
22
21
|
import { t as budgetMarker } from "./budget-DBUrH4jX.js";
|
|
23
|
-
import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-
|
|
24
|
-
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-
|
|
25
|
-
import { n as imageHarness } from "./prepare-
|
|
26
|
-
import { i as createProviders, n as attemptImage } from "./providers-
|
|
27
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-
|
|
22
|
+
import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-B9SOSnK9.js";
|
|
23
|
+
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-DJCD_BDn.js";
|
|
24
|
+
import { n as imageHarness } from "./prepare-CIxCGG0M.js";
|
|
25
|
+
import { i as createProviders, n as attemptImage } from "./providers-DFk_6W4K.js";
|
|
26
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DzyKSJBa.js";
|
|
28
27
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
29
28
|
import { dirname, join } from "node:path";
|
|
30
|
-
import { inspect } from "node:util";
|
|
31
29
|
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
30
|
+
import { inspect } from "node:util";
|
|
32
31
|
import { detectAgent } from "std-env";
|
|
33
32
|
import { Command, Option } from "commander";
|
|
34
33
|
//#region src/cli-parse.ts
|
|
@@ -274,7 +273,8 @@ function progressLine(c, p) {
|
|
|
274
273
|
case "check-start": return c.dim(`$ ${oneLine(p.command)}`);
|
|
275
274
|
case "check-end": {
|
|
276
275
|
const exit = p.exitCode === 0 ? c.dim("exit 0") : c.red(`exit ${p.exitCode}`);
|
|
277
|
-
|
|
276
|
+
const tries = p.try && p.tries ? triesNote(p.exitCode, p.try, p.tries) : null;
|
|
277
|
+
return `$ ${oneLine(p.command)} ${exit} ${c.dim([formatDuration(p.ms), tries].filter(Boolean).join(" · "))}`;
|
|
278
278
|
}
|
|
279
279
|
case "judge-start": return c.dim(`judging: ${oneLine(p.question, 120)}`);
|
|
280
280
|
case "judge-end": return judgeLine(c, p);
|
|
@@ -807,73 +807,6 @@ var PlainRenderer = class {
|
|
|
807
807
|
}
|
|
808
808
|
};
|
|
809
809
|
//#endregion
|
|
810
|
-
//#region src/ui/mode.ts
|
|
811
|
-
const set = (env, name) => {
|
|
812
|
-
const value = env[name];
|
|
813
|
-
return !!value && value !== "false" && value !== "0";
|
|
814
|
-
};
|
|
815
|
-
const CI_VARS = [
|
|
816
|
-
"BUILD_ID",
|
|
817
|
-
"BUILD_NUMBER",
|
|
818
|
-
"CI",
|
|
819
|
-
"CI_APP_ID",
|
|
820
|
-
"CI_BUILD_ID",
|
|
821
|
-
"CI_BUILD_NUMBER",
|
|
822
|
-
"CI_NAME",
|
|
823
|
-
"CONTINUOUS_INTEGRATION",
|
|
824
|
-
"RUN_ID",
|
|
825
|
-
"TEAMCITY_VERSION",
|
|
826
|
-
"GITHUB_ACTIONS"
|
|
827
|
-
];
|
|
828
|
-
const isCI = (env) => CI_VARS.some((name) => set(env, name));
|
|
829
|
-
function pick(name, value, allowed) {
|
|
830
|
-
if (value === void 0 || value === "") return void 0;
|
|
831
|
-
if (value === "auto" || isOneOf(allowed, value)) return value;
|
|
832
|
-
throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
|
|
833
|
-
}
|
|
834
|
-
function parseHeartbeat(value) {
|
|
835
|
-
if (!value) return 6e4;
|
|
836
|
-
const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
|
|
837
|
-
if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
|
|
838
|
-
const unit = {
|
|
839
|
-
ms: 1,
|
|
840
|
-
s: 1e3,
|
|
841
|
-
m: 6e4
|
|
842
|
-
}[match[2] ?? "s"] ?? 1e3;
|
|
843
|
-
return Number(match[1]) * unit;
|
|
844
|
-
}
|
|
845
|
-
function resolveOutput(input) {
|
|
846
|
-
const { flags, env } = input;
|
|
847
|
-
const ci = isCI(env);
|
|
848
|
-
const githubActions = set(env, "GITHUB_ACTIONS");
|
|
849
|
-
const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
|
|
850
|
-
"live",
|
|
851
|
-
"plain",
|
|
852
|
-
"agent"
|
|
853
|
-
]);
|
|
854
|
-
const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
|
|
855
|
-
const mode = requested === "live" ? "plain" : requested;
|
|
856
|
-
const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
|
|
857
|
-
const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
|
|
858
|
-
const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
|
|
859
|
-
const logPrefix = prefix && prefix !== "auto" ? prefix : githubActions && logOrder === "grouped" ? "none" : "task";
|
|
860
|
-
const logs = pick("--output-logs", flags.outputLogs ?? env.REEVAL_OUTPUT_LOGS, [
|
|
861
|
-
"full",
|
|
862
|
-
"errors-only",
|
|
863
|
-
"none"
|
|
864
|
-
]);
|
|
865
|
-
return {
|
|
866
|
-
requested,
|
|
867
|
-
mode,
|
|
868
|
-
logOrder,
|
|
869
|
-
logPrefix,
|
|
870
|
-
outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
|
|
871
|
-
color: env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions,
|
|
872
|
-
githubActions,
|
|
873
|
-
heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
|
|
874
|
-
};
|
|
875
|
-
}
|
|
876
|
-
//#endregion
|
|
877
810
|
//#region src/ui/index.ts
|
|
878
811
|
function createRenderer(options) {
|
|
879
812
|
const out = { write: (text) => options.out.write(redactSecrets(text)) };
|
|
@@ -948,7 +881,7 @@ async function run(argv) {
|
|
|
948
881
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
949
882
|
}
|
|
950
883
|
if (command === "init") {
|
|
951
|
-
const { initCommand } = await import("./init-
|
|
884
|
+
const { initCommand } = await import("./init-RgCQXZhb.js");
|
|
952
885
|
return initCommand(values, filters, {
|
|
953
886
|
out: (s) => console.log(s),
|
|
954
887
|
input: process.stdin,
|
|
@@ -957,7 +890,7 @@ async function run(argv) {
|
|
|
957
890
|
});
|
|
958
891
|
}
|
|
959
892
|
if (command === "doctor") {
|
|
960
|
-
const { doctorCommand } = await import("./doctor-
|
|
893
|
+
const { doctorCommand } = await import("./doctor-DQ2p0euG.js");
|
|
961
894
|
return doctorCommand(values, {
|
|
962
895
|
out: (s) => console.log(s),
|
|
963
896
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -965,7 +898,7 @@ async function run(argv) {
|
|
|
965
898
|
});
|
|
966
899
|
}
|
|
967
900
|
if (command === "report") {
|
|
968
|
-
const { reportCommand } = await import("./command-
|
|
901
|
+
const { reportCommand } = await import("./command-BXLUfw4h.js");
|
|
969
902
|
return reportCommand(values, filters, {
|
|
970
903
|
out: (s) => console.log(s),
|
|
971
904
|
err: (s) => console.error(dim(s)),
|
|
@@ -980,7 +913,7 @@ async function run(argv) {
|
|
|
980
913
|
}
|
|
981
914
|
if (command === "studio") {
|
|
982
915
|
const config = await loadConfig(configPath, "studio");
|
|
983
|
-
const { studioCommand } = await import("./command-
|
|
916
|
+
const { studioCommand } = await import("./command-I1euLLio.js");
|
|
984
917
|
return studioCommand({
|
|
985
918
|
config,
|
|
986
919
|
flags: values,
|
|
@@ -991,7 +924,7 @@ async function run(argv) {
|
|
|
991
924
|
if (command === "mcp") {
|
|
992
925
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
993
926
|
const config = await loadConfig(configPath, "mcp");
|
|
994
|
-
const { mcpCommand } = await import("./command-
|
|
927
|
+
const { mcpCommand } = await import("./command-Bxdy62dW.js");
|
|
995
928
|
return mcpCommand({
|
|
996
929
|
config,
|
|
997
930
|
flags: values,
|
|
@@ -1000,7 +933,7 @@ async function run(argv) {
|
|
|
1000
933
|
}
|
|
1001
934
|
if (command in HISTORY_COMMANDS) {
|
|
1002
935
|
const config = await loadConfig(configPath, command);
|
|
1003
|
-
const commands = await import("./commands-
|
|
936
|
+
const commands = await import("./commands-p1S4nw_H.js");
|
|
1004
937
|
const ctx = {
|
|
1005
938
|
config,
|
|
1006
939
|
flags: values,
|
|
@@ -1013,7 +946,7 @@ async function run(argv) {
|
|
|
1013
946
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
1014
947
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
1015
948
|
case "query": {
|
|
1016
|
-
const { queryCommand } = await import("./query-
|
|
949
|
+
const { queryCommand } = await import("./query-Cvbwl804.js");
|
|
1017
950
|
return queryCommand({
|
|
1018
951
|
...ctx,
|
|
1019
952
|
err: (s) => console.error(dim(s))
|
|
@@ -1021,7 +954,7 @@ async function run(argv) {
|
|
|
1021
954
|
}
|
|
1022
955
|
case "analyze": {
|
|
1023
956
|
loadEnvFiles(configPath);
|
|
1024
|
-
const { analyzeCommand } = await import("./command-
|
|
957
|
+
const { analyzeCommand } = await import("./command-Be6BSE2L.js");
|
|
1025
958
|
return analyzeCommand({
|
|
1026
959
|
...ctx,
|
|
1027
960
|
err: (s) => console.error(dim(s))
|
|
@@ -1033,6 +966,16 @@ async function run(argv) {
|
|
|
1033
966
|
if (command !== "run") throw new UsageError(`Unknown command "${command}". Run reeval --help.`);
|
|
1034
967
|
return runCommand(values, configPath, filters);
|
|
1035
968
|
}
|
|
969
|
+
/** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
|
|
970
|
+
async function dryRunPrices(planned, judgeModel) {
|
|
971
|
+
const { needsPrices } = await import("./cost-DmnAod9x.js");
|
|
972
|
+
if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
|
|
973
|
+
table: null,
|
|
974
|
+
why: "nothing to price"
|
|
975
|
+
};
|
|
976
|
+
const { loadPrices } = await import("./prices-BcLEXvdY.js");
|
|
977
|
+
return loadPrices();
|
|
978
|
+
}
|
|
1036
979
|
/** Asks a yes/no question on the terminal; anything but y or yes is no. */
|
|
1037
980
|
async function confirm(question) {
|
|
1038
981
|
const { createInterface } = await import("node:readline/promises");
|
|
@@ -1060,8 +1003,8 @@ async function cleanCommand(flags, configPath) {
|
|
|
1060
1003
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1061
1004
|
console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
|
|
1062
1005
|
if (flags.images) {
|
|
1063
|
-
const { dockerAvailable, pruneImages } = await import("./src-
|
|
1064
|
-
const { requireDocker } = await import("./providers-
|
|
1006
|
+
const { dockerAvailable, pruneImages } = await import("./src-1jHbFaGW.js");
|
|
1007
|
+
const { requireDocker } = await import("./providers-D1Zxp2he.js");
|
|
1065
1008
|
const engines = [];
|
|
1066
1009
|
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1067
1010
|
if (!engines.length) await requireDocker();
|
|
@@ -1082,7 +1025,7 @@ function verbosityOf(flags) {
|
|
|
1082
1025
|
* does. Needs a config, as `reeval run` does: there is no default image.
|
|
1083
1026
|
*/
|
|
1084
1027
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1085
|
-
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-
|
|
1028
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-DzLeFjLx.js");
|
|
1086
1029
|
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
|
|
1087
1030
|
...flags,
|
|
1088
1031
|
"dry-run": true
|
|
@@ -1142,7 +1085,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1142
1085
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1143
1086
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1144
1087
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1145
|
-
const { engineVersions } = await import("./node-
|
|
1088
|
+
const { engineVersions } = await import("./node-BSq5qHnQ.js");
|
|
1146
1089
|
const renderer = createRenderer({
|
|
1147
1090
|
out: setup.json ? setup.dryRun ? { write: () => true } : process.stderr : process.stdout,
|
|
1148
1091
|
output: setup.output,
|
|
@@ -1160,11 +1103,11 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1160
1103
|
});
|
|
1161
1104
|
throw error;
|
|
1162
1105
|
}
|
|
1163
|
-
const { engineRun, plan, sandboxVersions, budgetReached, live, questions } = executed;
|
|
1106
|
+
const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
|
|
1164
1107
|
invocation.unhandledErrors = engineRun.unhandledErrors;
|
|
1165
1108
|
const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
|
|
1166
1109
|
if ((interrupted || engineRun.cancelled) && engines.length) {
|
|
1167
|
-
const { removeRunContainers } = await import("./docker-
|
|
1110
|
+
const { removeRunContainers } = await import("./docker-DivfZl2u.js");
|
|
1168
1111
|
await removeRunContainers(ctx.runId, engines);
|
|
1169
1112
|
}
|
|
1170
1113
|
if (engineRun.stopped || !plan) {
|
|
@@ -1172,10 +1115,15 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1172
1115
|
renderer.close();
|
|
1173
1116
|
return EXIT.ok;
|
|
1174
1117
|
}
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1118
|
+
let stopped = cutShort || engineRun.cancelled;
|
|
1119
|
+
if (stopped) questions?.cancel();
|
|
1120
|
+
let answers = stopped ? await questions?.answered() : await questions?.settle();
|
|
1121
|
+
if (!stopped && interrupted && questions) {
|
|
1122
|
+
stopped = true;
|
|
1123
|
+
answers = await questions?.answered();
|
|
1124
|
+
}
|
|
1125
|
+
const { lockfiles } = await import("./state-vznMh2wl.js");
|
|
1126
|
+
const { writeScratchRun } = await import("./results-CMgZEANy.js");
|
|
1179
1127
|
const run = await writeScratchRun({
|
|
1180
1128
|
ctx,
|
|
1181
1129
|
author: ctx.author,
|
|
@@ -1196,18 +1144,18 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1196
1144
|
previous: ctx.previous,
|
|
1197
1145
|
previousArtifacts: ctx.previousArtifacts,
|
|
1198
1146
|
runner: runnerInfo(),
|
|
1199
|
-
...answers
|
|
1147
|
+
...answers && (answers.evals.size || answers.attempts.size) ? { answers } : {}
|
|
1200
1148
|
}, () => live?.stop());
|
|
1201
1149
|
const { record } = run;
|
|
1202
1150
|
const outcome = runOutcome(record);
|
|
1203
1151
|
const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1204
1152
|
let insights = run.insights;
|
|
1205
1153
|
let insightLines = [];
|
|
1206
|
-
const { runInsights, wantsInsights } = await import("./run-
|
|
1154
|
+
const { runInsights, wantsInsights } = await import("./run-CwIEK16q.js");
|
|
1207
1155
|
const wanted = !stopped && wantsInsights(ctx.config, !flags["no-insights"]);
|
|
1208
1156
|
if (wanted && interrupted) insightLines = [`insights skipped (${stopCode === EXIT.terminated ? "SIGTERM" : "Ctrl-C"}): reeval analyze ${record.runId} --insights to write them`];
|
|
1209
1157
|
else if (wanted) {
|
|
1210
|
-
const { scratchDir } = await import("./files-
|
|
1158
|
+
const { scratchDir } = await import("./files-C2sjmXhk.js");
|
|
1211
1159
|
const stop = new AbortController();
|
|
1212
1160
|
let stopSignal = "SIGINT";
|
|
1213
1161
|
onInsightsSignal = (signal) => {
|
|
@@ -1269,7 +1217,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1269
1217
|
const { config } = ctx;
|
|
1270
1218
|
const mode = setup.record;
|
|
1271
1219
|
if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
|
|
1272
|
-
const { record: recordCmd } = await import("./commands-
|
|
1220
|
+
const { record: recordCmd } = await import("./commands-p1S4nw_H.js");
|
|
1273
1221
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
1274
1222
|
const line = await recordCmd(config, record.runId, {
|
|
1275
1223
|
...flags,
|
|
@@ -1279,7 +1227,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1279
1227
|
else console.log(line);
|
|
1280
1228
|
}
|
|
1281
1229
|
if (setup.json) {
|
|
1282
|
-
const { suiteResult } = await import("./suite-
|
|
1230
|
+
const { suiteResult } = await import("./suite-BsWGHzlG.js");
|
|
1283
1231
|
process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
|
|
1284
1232
|
}
|
|
1285
1233
|
return code;
|
|
@@ -1308,11 +1256,11 @@ async function prepareRun(flags, configPath) {
|
|
|
1308
1256
|
runs = Number(flags.runs);
|
|
1309
1257
|
if (!(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`--runs must be a whole number of at least 1 (got ${flags.runs})`);
|
|
1310
1258
|
}
|
|
1311
|
-
const sandboxes = [environmentSandbox(config.environment, config.sandbox), ...profile
|
|
1259
|
+
const sandboxes = [environmentSandbox(config.environment, config.sandbox), ...profile ? [profileSandbox(config, profile.defaults)] : []];
|
|
1312
1260
|
const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
|
|
1313
1261
|
const container = sandboxes.find((s) => s.kind === "docker");
|
|
1314
1262
|
if (!container) return hostCapacity();
|
|
1315
|
-
const { dockerCapacity } = await import("./src-
|
|
1263
|
+
const { dockerCapacity } = await import("./src-1jHbFaGW.js");
|
|
1316
1264
|
return await dockerCapacity(container.engine) ?? hostCapacity();
|
|
1317
1265
|
});
|
|
1318
1266
|
return {
|
|
@@ -1341,11 +1289,11 @@ async function prepareRun(flags, configPath) {
|
|
|
1341
1289
|
async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
|
|
1342
1290
|
const startedAt = /* @__PURE__ */ new Date();
|
|
1343
1291
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
1344
|
-
const { worktreePatch } = await import("./state-
|
|
1292
|
+
const { worktreePatch } = await import("./state-vznMh2wl.js");
|
|
1345
1293
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
1346
1294
|
const who = await author(config.root);
|
|
1347
|
-
const { stagingDir, sweepStaging } = await import("./results-
|
|
1348
|
-
const { scratchDir } = await import("./files-
|
|
1295
|
+
const { stagingDir, sweepStaging } = await import("./results-CMgZEANy.js");
|
|
1296
|
+
const { scratchDir } = await import("./files-C2sjmXhk.js");
|
|
1349
1297
|
const scratch = scratchDir(config.root);
|
|
1350
1298
|
if (resume !== void 0) {
|
|
1351
1299
|
const target = await resolveResume(config.root, resume || void 0);
|
|
@@ -1356,7 +1304,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1356
1304
|
artifacts: target.previousArtifacts
|
|
1357
1305
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
1358
1306
|
if (!dryRun) try {
|
|
1359
|
-
const { removeRunContainers } = await import("./docker-
|
|
1307
|
+
const { removeRunContainers } = await import("./docker-DivfZl2u.js");
|
|
1360
1308
|
await removeRunContainers(target.runId, ["docker", "podman"]);
|
|
1361
1309
|
} catch {}
|
|
1362
1310
|
return {
|
|
@@ -1427,8 +1375,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1427
1375
|
configFile,
|
|
1428
1376
|
completed: []
|
|
1429
1377
|
}));
|
|
1430
|
-
const { runEngine } = await import("./node-
|
|
1431
|
-
const { buildPlan, checkAttemptBudget, planDocument, planInsights, planLine, planTable } = await import("./plan-
|
|
1378
|
+
const { runEngine } = await import("./node-BSq5qHnQ.js");
|
|
1379
|
+
const { buildPlan, checkAttemptBudget, planDocument, planInsights, planLine, planTable } = await import("./plan-BIwUzlB8.js");
|
|
1432
1380
|
let plan;
|
|
1433
1381
|
let sandboxVersions = { images: [] };
|
|
1434
1382
|
let preflightStop;
|
|
@@ -1454,7 +1402,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1454
1402
|
} : p);
|
|
1455
1403
|
toRun = skipped;
|
|
1456
1404
|
for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
|
|
1457
|
-
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-
|
|
1405
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-Ch8kX87u.js") : void 0;
|
|
1406
|
+
const judge = runJudgeModel(config, profile);
|
|
1407
|
+
const judgeModel = judge === void 0 ? null : judgeModelName(judge);
|
|
1458
1408
|
plan = await buildPlan({
|
|
1459
1409
|
planned: skipped,
|
|
1460
1410
|
profile,
|
|
@@ -1462,16 +1412,26 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1462
1412
|
concurrency,
|
|
1463
1413
|
root: config.root,
|
|
1464
1414
|
historyDir: config.history.dir,
|
|
1465
|
-
questionModel: models?.questionModelLabel(config) ?? null,
|
|
1466
|
-
insights: await planInsights(config, setup.insights)
|
|
1415
|
+
questionModel: models?.questionModelLabel(config, judge) ?? null,
|
|
1416
|
+
insights: await planInsights(config, setup.insights),
|
|
1417
|
+
judgeModel,
|
|
1418
|
+
...setup.dryRun ? { prices: await dryRunPrices(skipped, judgeModel) } : {}
|
|
1467
1419
|
});
|
|
1468
1420
|
plan.runId = runId;
|
|
1421
|
+
if (plan.insights) {
|
|
1422
|
+
const { analysisKeyProblem } = await import("./model-Ch8kX87u.js");
|
|
1423
|
+
const problem = analysisKeyProblem(config);
|
|
1424
|
+
if (problem) plan.warnings.push(`insights will be skipped: ${problem}`);
|
|
1425
|
+
}
|
|
1426
|
+
const cloud = ciRunner(process.env);
|
|
1427
|
+
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
|
|
1428
|
+
if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
|
|
1469
1429
|
if (completed.size) plan.resume = {
|
|
1470
1430
|
runId,
|
|
1471
1431
|
completed: completed.size
|
|
1472
1432
|
};
|
|
1473
1433
|
run.completed = [...completed];
|
|
1474
|
-
const { imageStages, imageTargets } = await import("./images-
|
|
1434
|
+
const { imageStages, imageTargets } = await import("./images-DzLeFjLx.js");
|
|
1475
1435
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1476
1436
|
id: p.id,
|
|
1477
1437
|
harness: p.attempt.harness,
|
|
@@ -1493,8 +1453,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1493
1453
|
if (setup.dryRun) {
|
|
1494
1454
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
1495
1455
|
lines.push(...planTable(plan, todos));
|
|
1496
|
-
const { imageStatusLines } = await import("./images-
|
|
1497
|
-
const { dockerAvailable, engineSpec } = await import("./src-
|
|
1456
|
+
const { imageStatusLines } = await import("./images-DzLeFjLx.js");
|
|
1457
|
+
const { dockerAvailable, engineSpec } = await import("./src-1jHbFaGW.js");
|
|
1498
1458
|
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
1499
1459
|
const down = [];
|
|
1500
1460
|
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
@@ -1503,9 +1463,12 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1503
1463
|
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
1504
1464
|
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
1505
1465
|
}
|
|
1466
|
+
const { costLine } = await import("./cost-DmnAod9x.js");
|
|
1467
|
+
const cost = plan.cost && costLine(plan.cost);
|
|
1468
|
+
if (cost) lines.push(cost);
|
|
1506
1469
|
}
|
|
1507
1470
|
if (setup.dryRun && plan.insights) {
|
|
1508
|
-
const { formatTokens } = await import("./usage-
|
|
1471
|
+
const { formatTokens } = await import("./usage-DF3_p5pE.js");
|
|
1509
1472
|
lines.push(`insights ${plan.insights.model} · up to ${formatTokens(plan.insights.maxTokens)} tokens`);
|
|
1510
1473
|
}
|
|
1511
1474
|
lines.push(...plan.warnings.map((w) => `warning: ${w}`));
|
|
@@ -1520,15 +1483,23 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1520
1483
|
checkAttemptBudget(plan, setup.force, config.file);
|
|
1521
1484
|
const questionModels = /* @__PURE__ */ new Set();
|
|
1522
1485
|
const asked = [];
|
|
1486
|
+
const scope = gatewayScope(config, profile);
|
|
1523
1487
|
for (const row of plan.rows) {
|
|
1524
1488
|
if (row.plan.skip || !models) continue;
|
|
1525
|
-
const questions = (row.plan.questions ?? []).map((q) =>
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1489
|
+
const questions = (row.plan.questions ?? []).map((q) => {
|
|
1490
|
+
const model = models.questionModel(config, q, row.id, judge);
|
|
1491
|
+
const gateway = resolveModelGateway(model, void 0, scope, "the question model");
|
|
1492
|
+
return {
|
|
1493
|
+
n: q.n,
|
|
1494
|
+
question: q.question,
|
|
1495
|
+
model,
|
|
1496
|
+
...gateway ? { gateway } : {},
|
|
1497
|
+
...q.across !== void 0 ? { across: q.across } : {},
|
|
1498
|
+
...q.choices ? { choices: q.choices } : {},
|
|
1499
|
+
...q.state ? { state: q.state } : {},
|
|
1500
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
1501
|
+
};
|
|
1502
|
+
});
|
|
1532
1503
|
for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
|
|
1533
1504
|
if (questions.length) asked.push({
|
|
1534
1505
|
row,
|
|
@@ -1540,7 +1511,10 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1540
1511
|
profile,
|
|
1541
1512
|
questionModels: [...questionModels]
|
|
1542
1513
|
});
|
|
1543
|
-
if (missing.length)
|
|
1514
|
+
if (missing.length) {
|
|
1515
|
+
const hint = missing.some((m) => m.includes("(for direct())")) ? unusedGatewayKey(process.env) : "";
|
|
1516
|
+
throw new UsageError(`Missing credentials: ${missing.join("; ")}. Set them in the environment or .env.local.${hint}`);
|
|
1517
|
+
}
|
|
1544
1518
|
const malformed = malformedCredentials(config, process.env, {
|
|
1545
1519
|
harnesses: plan.harnessDescriptors,
|
|
1546
1520
|
profile,
|
|
@@ -1548,7 +1522,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1548
1522
|
});
|
|
1549
1523
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
1550
1524
|
if (asked.length) {
|
|
1551
|
-
const { questionRunner } = await import("./questions-
|
|
1525
|
+
const { questionRunner } = await import("./questions-bAudIc04.js");
|
|
1526
|
+
const warned = /* @__PURE__ */ new Set();
|
|
1552
1527
|
questions = questionRunner({
|
|
1553
1528
|
evals: asked,
|
|
1554
1529
|
stagingDir,
|
|
@@ -1560,12 +1535,31 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1560
1535
|
type: "notice",
|
|
1561
1536
|
at: Date.now(),
|
|
1562
1537
|
level: "warn",
|
|
1563
|
-
text: `
|
|
1538
|
+
text: `question "${clip(answer.question, 80)}" on ${id} failed: ${answer.error}. Outcomes are unchanged.`
|
|
1539
|
+
});
|
|
1540
|
+
},
|
|
1541
|
+
onAttemptAnswer: (id, attempt, answer) => {
|
|
1542
|
+
const once = `${id}\0${answer.question}`;
|
|
1543
|
+
if (answer.error === void 0 || warned.has(once)) return;
|
|
1544
|
+
warned.add(once);
|
|
1545
|
+
renderer.onEvent({
|
|
1546
|
+
type: "notice",
|
|
1547
|
+
at: Date.now(),
|
|
1548
|
+
level: "warn",
|
|
1549
|
+
text: `question "${clip(answer.question, 80)}" on ${attempt} failed: ${answer.error}. Outcomes are unchanged; later failures of it on ${id} aren't shown.`
|
|
1550
|
+
});
|
|
1551
|
+
},
|
|
1552
|
+
onUnshown: (id, error) => {
|
|
1553
|
+
renderer.onEvent({
|
|
1554
|
+
type: "notice",
|
|
1555
|
+
at: Date.now(),
|
|
1556
|
+
level: "warn",
|
|
1557
|
+
text: `${error} on ${id}, so each attempt's answer is that error. Outcomes are unchanged.`
|
|
1564
1558
|
});
|
|
1565
1559
|
}
|
|
1566
1560
|
});
|
|
1567
1561
|
}
|
|
1568
|
-
const { prepareWorkspaces, workspaceKey } = await import("./workspace-
|
|
1562
|
+
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-QZCacP8A.js");
|
|
1569
1563
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
1570
1564
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
1571
1565
|
workspace: a.workspace,
|
|
@@ -1611,6 +1605,12 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1611
1605
|
}
|
|
1612
1606
|
});
|
|
1613
1607
|
}
|
|
1608
|
+
for (const text of sameTreeArms(live, outcomes)) renderer.onEvent({
|
|
1609
|
+
type: "notice",
|
|
1610
|
+
at: Date.now(),
|
|
1611
|
+
level: "warn",
|
|
1612
|
+
text
|
|
1613
|
+
});
|
|
1614
1614
|
try {
|
|
1615
1615
|
sandboxVersions = await preflight(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1616
1616
|
harness: p.attempt.harness,
|
|
@@ -1650,6 +1650,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1650
1650
|
budgetTimer.unref();
|
|
1651
1651
|
};
|
|
1652
1652
|
let engineRun;
|
|
1653
|
+
let cutShort = false;
|
|
1653
1654
|
try {
|
|
1654
1655
|
engineRun = await runEngine({
|
|
1655
1656
|
run,
|
|
@@ -1669,7 +1670,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1669
1670
|
version,
|
|
1670
1671
|
files,
|
|
1671
1672
|
harness: (plan?.harnesses ?? config.harness.map(harnessId)).join(", "),
|
|
1672
|
-
sandbox:
|
|
1673
|
+
sandbox: plannedSandboxes(plan?.sandboxes ?? [], profileSandbox(config, profile?.defaults)),
|
|
1673
1674
|
concurrency,
|
|
1674
1675
|
runId
|
|
1675
1676
|
});
|
|
@@ -1709,15 +1710,16 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1709
1710
|
process.on("SIGTERM", onSignal);
|
|
1710
1711
|
}
|
|
1711
1712
|
});
|
|
1713
|
+
cutShort = interrupted;
|
|
1712
1714
|
live?.phase("finishing");
|
|
1713
1715
|
} catch (error) {
|
|
1714
1716
|
live?.remove();
|
|
1715
1717
|
throw error;
|
|
1716
1718
|
} finally {
|
|
1717
1719
|
clearTimeout(budgetTimer);
|
|
1718
|
-
const { removePrepared } = await import("./preflight-
|
|
1720
|
+
const { removePrepared } = await import("./preflight-D9y-HvZQ.js");
|
|
1719
1721
|
removePrepared(config.root, runId);
|
|
1720
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
1722
|
+
const { removePreparedWorkspaces } = await import("./workspace-QZCacP8A.js");
|
|
1721
1723
|
removePreparedWorkspaces(config.root, runId);
|
|
1722
1724
|
}
|
|
1723
1725
|
if (preflightStop) throw preflightStop;
|
|
@@ -1729,6 +1731,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1729
1731
|
plan,
|
|
1730
1732
|
sandboxVersions,
|
|
1731
1733
|
budgetReached: () => reached,
|
|
1734
|
+
cutShort,
|
|
1732
1735
|
live,
|
|
1733
1736
|
questions
|
|
1734
1737
|
};
|
|
@@ -1799,4 +1802,4 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
1799
1802
|
//#endregion
|
|
1800
1803
|
export {};
|
|
1801
1804
|
|
|
1802
|
-
//# sourceMappingURL=cli-
|
|
1805
|
+
//# sourceMappingURL=cli-C_4gaJ50.js.map
|