reeval 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/THIRD_PARTY_NOTICES.md +121 -121
- package/dist/{adapter-IKKo4sZj.js → adapter-BloGVZ_a.js} +20 -8
- package/dist/adapter-BloGVZ_a.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-m-_t-IuW.js} +53 -25
- package/dist/{agent-ClE-g9VL.js.map → agent-m-_t-IuW.js.map} +1 -1
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-C-jwhRDt.js +11 -0
- package/dist/app-files-C-jwhRDt.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-BDEpelpi.js} +92 -35
- package/dist/assemble-BDEpelpi.js.map +1 -0
- package/dist/assemble-C5bfncHU.js +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-CVRwRTnb.js} +572 -36
- package/dist/capture-CVRwRTnb.js.map +1 -0
- package/dist/capture-CherbjzB.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-C_4gaJ50.js} +132 -140
- package/dist/cli-C_4gaJ50.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
- package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
- package/dist/client-D5hNjpEx.js.map +1 -0
- package/dist/{command-C6vwmjS7.js → command-BWAuwQNk.js} +12 -12
- package/dist/{command-C6vwmjS7.js.map → command-BWAuwQNk.js.map} +1 -1
- package/dist/{command-CANun0m0.js → command-BXLUfw4h.js} +12 -12
- package/dist/{command-CANun0m0.js.map → command-BXLUfw4h.js.map} +1 -1
- package/dist/{command-CBaHukyl.js → command-Be6BSE2L.js} +12 -12
- package/dist/{command-CBaHukyl.js.map → command-Be6BSE2L.js.map} +1 -1
- package/dist/{command-D_kEnYcX.js → command-Bxdy62dW.js} +14 -13
- package/dist/command-Bxdy62dW.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-I1euLLio.js} +35 -15
- package/dist/command-I1euLLio.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-jzigbNFb.js} +23 -14
- package/dist/commands-jzigbNFb.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-p1S4nw_H.js} +1 -1
- package/dist/config-BrdHA9bJ.js +3549 -0
- package/dist/config-BrdHA9bJ.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DRxMQQTC.js} +3 -2
- package/dist/contrasts-DRxMQQTC.js.map +1 -0
- package/dist/cost-D4elx933.js +359 -0
- package/dist/cost-D4elx933.js.map +1 -0
- package/dist/cost-DmnAod9x.js +1 -0
- package/dist/db-CAfksinX.js +2 -0
- package/dist/{db-DJUuFoWQ.js → db-DO0p7xgP.js} +25 -8
- package/dist/db-DO0p7xgP.js.map +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-BnRlTpZA.js} +2 -2
- package/dist/{docker-DuHcQidD.js.map → docker-BnRlTpZA.js.map} +1 -1
- package/dist/{docker-CQxucRPe.js → docker-D01slDF3.js} +26 -10
- package/dist/docker-D01slDF3.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-DivfZl2u.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-DQ2p0euG.js} +135 -36
- package/dist/doctor-DQ2p0euG.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-CO0P0soF.js} +8 -7
- package/dist/environment-CO0P0soF.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BUmsN7yG.js} +66 -27
- package/dist/evaluate-BUmsN7yG.js.map +1 -0
- package/dist/{evaluate-C9-ritA5.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
- package/dist/{facts-Cfx1H70y.js → facts-DzOUTreB.js} +12 -4
- package/dist/facts-DzOUTreB.js.map +1 -0
- package/dist/{files-CN1QfenC.js → files-B-CJeeEh.js} +13 -8
- package/dist/{files-CN1QfenC.js.map → files-B-CJeeEh.js.map} +1 -1
- package/dist/files-Bp5j5BqI.js +1 -0
- package/dist/files-C2sjmXhk.js +1 -0
- package/dist/{files-Bs6H3HiG.js → files-C_gQNVB-.js} +3 -3
- package/dist/{files-Bs6H3HiG.js.map → files-C_gQNVB-.js.map} +1 -1
- package/dist/{files-D27nUk0K.js → files-kUCrswI6.js} +2 -2
- package/dist/{files-D27nUk0K.js.map → files-kUCrswI6.js.map} +1 -1
- package/dist/gateway.d.ts +17 -0
- package/dist/gateway.js +17 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{generate-BnrvBAzF.js → generate-DZ7vp69F.js} +18 -17
- package/dist/generate-DZ7vp69F.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-Bk8DSwHy.js} +2 -2
- package/dist/{git-rPfl7AD1.js.map → git-Bk8DSwHy.js.map} +1 -1
- package/dist/{harness-1OkumOAm.js → harness-COVXSwJ6.js} +46 -15
- package/dist/harness-COVXSwJ6.js.map +1 -0
- package/dist/harness.d.ts +26 -18
- package/dist/harness.js +1 -1
- package/dist/{header-BVzbRnNv.js → header-BLPP6aCU.js} +2 -2
- package/dist/{header-BVzbRnNv.js.map → header-BLPP6aCU.js.map} +1 -1
- package/dist/{images-m784ueGK.js → images-DzLeFjLx.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-DzLeFjLx.js.map} +1 -1
- package/dist/index.d.ts +3 -45
- package/dist/index.js +5 -6
- package/dist/init-RgCQXZhb.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-SRalM5xF.js} +6 -8
- package/dist/init-SRalM5xF.js.map +1 -0
- package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
- package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
- package/dist/{invocations-D_ePcwyL.js → invocations-DzyKSJBa.js} +2 -2
- package/dist/{invocations-D_ePcwyL.js.map → invocations-DzyKSJBa.js.map} +1 -1
- package/dist/judge-yLvjC5A3.js +385 -0
- package/dist/judge-yLvjC5A3.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +7 -5
- package/dist/judge.js.map +1 -1
- package/dist/match-Bz0nSdzN.js +204 -0
- package/dist/match-Bz0nSdzN.js.map +1 -0
- package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-1ZLr9Dcv.js} +40 -26
- package/dist/model-1ZLr9Dcv.js.map +1 -0
- package/dist/model-B45lDUV1.js +68 -0
- package/dist/model-B45lDUV1.js.map +1 -0
- package/dist/model-BTZq8HIZ.js +1 -0
- package/dist/{model-UgHDMhI1.js → model-CgnOpss4.js} +57 -318
- package/dist/model-CgnOpss4.js.map +1 -0
- package/dist/{model-B8H3gL69.js → model-Ch8kX87u.js} +1 -1
- package/dist/{node-CGZYh4Fj.js → node-BSq5qHnQ.js} +12 -5
- package/dist/node-BSq5qHnQ.js.map +1 -0
- package/dist/{opencode-providers-snfeWd6Z.js → opencode-providers-klkqgzvb.js} +23 -52
- package/dist/opencode-providers-klkqgzvb.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
- package/dist/pairs-CHfZNRT6.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DWf-I_Q_.js} +2 -2
- package/dist/{patch-uMq9rCl_.js.map → patch-DWf-I_Q_.js.map} +1 -1
- package/dist/{plan-CZYgXnZh.js → plan-BIwUzlB8.js} +63 -26
- package/dist/plan-BIwUzlB8.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-Dkp_H3pB.js} +4 -4
- package/dist/{plugins-P-uh54VF.js.map → plugins-Dkp_H3pB.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-D9y-HvZQ.js} +1 -1
- package/dist/{preflight-D3HvKeAx.js → preflight-DJCD_BDn.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-DJCD_BDn.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-BJG_pD54.js} +2 -2
- package/dist/{prepare-Dx7dOnJB.js → prepare-CIxCGG0M.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-CIxCGG0M.js.map} +1 -1
- package/dist/prices-BcLEXvdY.js +250 -0
- package/dist/prices-BcLEXvdY.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-CfZpFpAh.js} +2 -2
- package/dist/{provenance-C_-8y-x3.js.map → provenance-CfZpFpAh.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-D1Zxp2he.js} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-DFk_6W4K.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-DFk_6W4K.js.map} +1 -1
- package/dist/{query-DXthKmq0.js → query-CemclveB.js} +2 -2
- package/dist/{query-DXthKmq0.js.map → query-CemclveB.js.map} +1 -1
- package/dist/query-Cvbwl804.js +1 -0
- package/dist/{questions-KENpWnfV.js → questions-bAudIc04.js} +308 -45
- package/dist/questions-bAudIc04.js.map +1 -0
- package/dist/{remarks-Ml6bcSdV.js → remarks-CXwSnSjR.js} +9 -9
- package/dist/{remarks-Ml6bcSdV.js.map → remarks-CXwSnSjR.js.map} +1 -1
- package/dist/remarks-CyAu2DiB.js +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
- package/dist/remarks-mujJv1Ey.js.map +1 -0
- package/dist/{remote-Dkeghk8k.js → remote-CKcEKja4.js} +129 -174
- package/dist/remote-CKcEKja4.js.map +1 -0
- package/dist/resolve-B5-uqmJ6.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-CBoKzQMP.js} +8 -9
- package/dist/resolve-CBoKzQMP.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-CMgZEANy.js} +8 -8
- package/dist/{results-D-naai_Q.js.map → results-CMgZEANy.js.map} +1 -1
- package/dist/{resume-8-nnW8iO.js → resume-B9SOSnK9.js} +3 -3
- package/dist/{resume-8-nnW8iO.js.map → resume-B9SOSnK9.js.map} +1 -1
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{rules-CcV3zLDR.js → rules-VhftPOOT.js} +53 -5
- package/dist/rules-VhftPOOT.js.map +1 -0
- package/dist/{run-Bel7Vde4.js → run-CwIEK16q.js} +9 -9
- package/dist/{run-Bel7Vde4.js.map → run-CwIEK16q.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
- package/dist/runtime-BgQRyCLl.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-BOYZ0hlo.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CHRSgKRp.js +1 -0
- package/dist/{server-De8oH3bt.js → server-TGW4WyAz.js} +25 -26
- package/dist/{server-De8oH3bt.js.map → server-TGW4WyAz.js.map} +1 -1
- package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
- package/dist/shared-BDUzStad.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/{source-C686eZpY.js → source-B1WvBIQt.js} +2 -2
- package/dist/{source-C686eZpY.js.map → source-B1WvBIQt.js.map} +1 -1
- package/dist/source-DAzztZwU.js +1 -0
- package/dist/src-1jHbFaGW.js +1 -0
- package/dist/{src-CIpKL45U.js → src-8zoX_qrb.js} +36 -26
- package/dist/{src-CIpKL45U.js.map → src-8zoX_qrb.js.map} +1 -1
- package/dist/{src-CzQrAcF8.js → src-CHIwOy0o.js} +84 -52
- package/dist/src-CHIwOy0o.js.map +1 -0
- package/dist/src-C_9wgowC.js +2 -0
- package/dist/{src-djNNYIdP.js → src-ChhtI6BP.js} +2 -2
- package/dist/{src-djNNYIdP.js.map → src-ChhtI6BP.js.map} +1 -1
- package/dist/{src-DrJSnoMR.js → src-CyaDNG9r.js} +1 -1
- package/dist/src-D005Cbqz.js +2 -0
- package/dist/src-DIfRbp5a.js +3 -0
- package/dist/{src-C7W6kg81.js → src-DKPVX2Us.js} +34 -16
- package/dist/{src-C7W6kg81.js.map → src-DKPVX2Us.js.map} +1 -1
- package/dist/src-Dg8121Xy.js +2 -0
- package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
- package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
- package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
- package/dist/src-DjRI-IeE.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-DmUgeTlE.js} +4 -4
- package/dist/{src-DeY52Gyf.js.map → src-DmUgeTlE.js.map} +1 -1
- package/dist/src-ElBqEN-S.js +2 -0
- package/dist/{src-Xj_BZKbq.js → src-NSdQ9zI6.js} +31 -16
- package/dist/{src-Xj_BZKbq.js.map → src-NSdQ9zI6.js.map} +1 -1
- package/dist/src-YHGaUrGE.js +2 -0
- package/dist/src-mJVxw1Jm.js +2 -0
- package/dist/{state-C0i_ujQZ.js → state-B1t143wG.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-B1t143wG.js.map} +1 -1
- package/dist/state-vznMh2wl.js +1 -0
- package/dist/stats-DSy5HfOv.js +315 -0
- package/dist/stats-DSy5HfOv.js.map +1 -0
- package/dist/suite-BsWGHzlG.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-WgT7m6Vs.js} +76 -11
- package/dist/suite-WgT7m6Vs.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DIGEVr08.js} +58 -17
- package/dist/summary-DIGEVr08.js.map +1 -0
- package/dist/testing.d.ts +174 -2
- package/dist/testing.js +14 -3
- package/dist/testing.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-DXBbP9Mk.d.ts} +298 -70
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/usage-DF3_p5pE.js +1 -0
- package/dist/{validate-DWjkshl1.js → validate-BhwqrVsm.js} +44 -12
- package/dist/validate-BhwqrVsm.js.map +1 -0
- package/dist/view/index.d.ts +133 -86
- package/dist/view/index.js +7 -6
- package/dist/views-BW_C5eVH.js +2 -0
- package/dist/{views-Dk8MCQKr.js → views-sXpGE6a2.js} +10 -8
- package/dist/views-sXpGE6a2.js.map +1 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-Duo9oAeT.js} +8 -8
- package/dist/{worker-entry-CqrzGPF4.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
- package/dist/{workspace-CLeOybkr.js → workspace-CDI68wtX.js} +4 -4
- package/dist/{workspace-CLeOybkr.js.map → workspace-CDI68wtX.js.map} +1 -1
- package/dist/workspace-QZCacP8A.js +1 -0
- package/package.json +6 -2
- package/skills/eval-rubric/SKILL.md +9 -0
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -1,34 +1,33 @@
|
|
|
1
|
-
import { n as isOneOf } from "./object-CLf2-tNq.js";
|
|
2
1
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
-
import { p as parseDuration } from "./shared-
|
|
4
|
-
import {
|
|
5
|
-
import { M as workspaceLabel, T as profileSandbox, p as environmentSandbox } from "./environment-B4IuURK1.js";
|
|
2
|
+
import { p as parseDuration } from "./shared-BDUzStad.js";
|
|
3
|
+
import { M as workspaceLabel, T as profileSandbox, p as environmentSandbox } from "./environment-CO0P0soF.js";
|
|
6
4
|
import { a as fromCwd, o as newRunId } from "./ids-Bp6ut7ZD.js";
|
|
5
|
+
import { G as ciRunner, K as directOnCloud, O as harnessId, P as sandboxId, Y as isDirect, Z as resolveOutput, _ as selectProfile, a as configPathFor, c as findConfig, d as loadConfig, f as malformedCredentials, g as runJudgeModel, h as resolveConcurrency, i as configLabel, j as plannedSandboxes, l as hasAnalysisModel, m as requireConfig, p as missingCredentials, u as hostCapacity, v as gatewayScope, y as resolveModelGateway } from "./config-BrdHA9bJ.js";
|
|
6
|
+
import { c as judgeModelName, p as unusedGatewayKey } from "./judge-yLvjC5A3.js";
|
|
7
7
|
import { c as registerSecret, s as redactSecrets } from "./redact-CRGdfV2s.js";
|
|
8
|
-
import { a as configPathFor, c as findConfig, d as loadConfig, f as malformedCredentials, g as selectProfile, h as resolveConcurrency, i as configLabel, l as hasAnalysisModel, m as requireConfig, p as missingCredentials, u as hostCapacity } from "./config-D7TFvUpL.js";
|
|
9
8
|
import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
|
|
10
|
-
import {
|
|
11
|
-
import { o as PACKAGE_VERSION } from "./root-
|
|
12
|
-
import { C as pluginBuildsLabel } from "./plugins-
|
|
13
|
-
import { s as debugTime } from "./debug-D24NOQH1.js";
|
|
9
|
+
import { G as LABEL, ot as imageInfo } from "./src-8zoX_qrb.js";
|
|
10
|
+
import { o as PACKAGE_VERSION } from "./root-Dlff-zp8.js";
|
|
11
|
+
import { C as pluginBuildsLabel } from "./plugins-Dkp_H3pB.js";
|
|
14
12
|
import { i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-CrVy5ioL.js";
|
|
13
|
+
import { s as debugTime } from "./debug-D24NOQH1.js";
|
|
15
14
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-
|
|
19
|
-
import { p as render } from "./summary-
|
|
15
|
+
import { D as runOutcome, K as triesNote, _ as formatUsage } from "./model-CgnOpss4.js";
|
|
16
|
+
import { u as suiteResult } from "./suite-WgT7m6Vs.js";
|
|
17
|
+
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-CpnbATtI.js";
|
|
18
|
+
import { p as render } from "./summary-DIGEVr08.js";
|
|
20
19
|
import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
|
|
21
|
-
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-
|
|
20
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-CfZpFpAh.js";
|
|
22
21
|
import { t as budgetMarker } from "./budget-DBUrH4jX.js";
|
|
23
|
-
import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-
|
|
24
|
-
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-
|
|
25
|
-
import { n as imageHarness } from "./prepare-
|
|
26
|
-
import { i as createProviders, n as attemptImage } from "./providers-
|
|
27
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-
|
|
22
|
+
import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-B9SOSnK9.js";
|
|
23
|
+
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-DJCD_BDn.js";
|
|
24
|
+
import { n as imageHarness } from "./prepare-CIxCGG0M.js";
|
|
25
|
+
import { i as createProviders, n as attemptImage } from "./providers-DFk_6W4K.js";
|
|
26
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DzyKSJBa.js";
|
|
28
27
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
29
28
|
import { dirname, join } from "node:path";
|
|
30
|
-
import { inspect } from "node:util";
|
|
31
29
|
import { appendFile, readFile, rm } from "node:fs/promises";
|
|
30
|
+
import { inspect } from "node:util";
|
|
32
31
|
import { detectAgent } from "std-env";
|
|
33
32
|
import { Command, Option } from "commander";
|
|
34
33
|
//#region src/cli-parse.ts
|
|
@@ -274,7 +273,8 @@ function progressLine(c, p) {
|
|
|
274
273
|
case "check-start": return c.dim(`$ ${oneLine(p.command)}`);
|
|
275
274
|
case "check-end": {
|
|
276
275
|
const exit = p.exitCode === 0 ? c.dim("exit 0") : c.red(`exit ${p.exitCode}`);
|
|
277
|
-
|
|
276
|
+
const tries = p.try && p.tries ? triesNote(p.exitCode, p.try, p.tries) : null;
|
|
277
|
+
return `$ ${oneLine(p.command)} ${exit} ${c.dim([formatDuration(p.ms), tries].filter(Boolean).join(" · "))}`;
|
|
278
278
|
}
|
|
279
279
|
case "judge-start": return c.dim(`judging: ${oneLine(p.question, 120)}`);
|
|
280
280
|
case "judge-end": return judgeLine(c, p);
|
|
@@ -807,73 +807,6 @@ var PlainRenderer = class {
|
|
|
807
807
|
}
|
|
808
808
|
};
|
|
809
809
|
//#endregion
|
|
810
|
-
//#region src/ui/mode.ts
|
|
811
|
-
const set = (env, name) => {
|
|
812
|
-
const value = env[name];
|
|
813
|
-
return !!value && value !== "false" && value !== "0";
|
|
814
|
-
};
|
|
815
|
-
const CI_VARS = [
|
|
816
|
-
"BUILD_ID",
|
|
817
|
-
"BUILD_NUMBER",
|
|
818
|
-
"CI",
|
|
819
|
-
"CI_APP_ID",
|
|
820
|
-
"CI_BUILD_ID",
|
|
821
|
-
"CI_BUILD_NUMBER",
|
|
822
|
-
"CI_NAME",
|
|
823
|
-
"CONTINUOUS_INTEGRATION",
|
|
824
|
-
"RUN_ID",
|
|
825
|
-
"TEAMCITY_VERSION",
|
|
826
|
-
"GITHUB_ACTIONS"
|
|
827
|
-
];
|
|
828
|
-
const isCI = (env) => CI_VARS.some((name) => set(env, name));
|
|
829
|
-
function pick(name, value, allowed) {
|
|
830
|
-
if (value === void 0 || value === "") return void 0;
|
|
831
|
-
if (value === "auto" || isOneOf(allowed, value)) return value;
|
|
832
|
-
throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
|
|
833
|
-
}
|
|
834
|
-
function parseHeartbeat(value) {
|
|
835
|
-
if (!value) return 6e4;
|
|
836
|
-
const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
|
|
837
|
-
if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
|
|
838
|
-
const unit = {
|
|
839
|
-
ms: 1,
|
|
840
|
-
s: 1e3,
|
|
841
|
-
m: 6e4
|
|
842
|
-
}[match[2] ?? "s"] ?? 1e3;
|
|
843
|
-
return Number(match[1]) * unit;
|
|
844
|
-
}
|
|
845
|
-
function resolveOutput(input) {
|
|
846
|
-
const { flags, env } = input;
|
|
847
|
-
const ci = isCI(env);
|
|
848
|
-
const githubActions = set(env, "GITHUB_ACTIONS");
|
|
849
|
-
const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
|
|
850
|
-
"live",
|
|
851
|
-
"plain",
|
|
852
|
-
"agent"
|
|
853
|
-
]);
|
|
854
|
-
const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
|
|
855
|
-
const mode = requested === "live" ? "plain" : requested;
|
|
856
|
-
const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
|
|
857
|
-
const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
|
|
858
|
-
const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
|
|
859
|
-
const logPrefix = prefix && prefix !== "auto" ? prefix : githubActions && logOrder === "grouped" ? "none" : "task";
|
|
860
|
-
const logs = pick("--output-logs", flags.outputLogs ?? env.REEVAL_OUTPUT_LOGS, [
|
|
861
|
-
"full",
|
|
862
|
-
"errors-only",
|
|
863
|
-
"none"
|
|
864
|
-
]);
|
|
865
|
-
return {
|
|
866
|
-
requested,
|
|
867
|
-
mode,
|
|
868
|
-
logOrder,
|
|
869
|
-
logPrefix,
|
|
870
|
-
outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
|
|
871
|
-
color: env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions,
|
|
872
|
-
githubActions,
|
|
873
|
-
heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
|
|
874
|
-
};
|
|
875
|
-
}
|
|
876
|
-
//#endregion
|
|
877
810
|
//#region src/ui/index.ts
|
|
878
811
|
function createRenderer(options) {
|
|
879
812
|
const out = { write: (text) => options.out.write(redactSecrets(text)) };
|
|
@@ -948,7 +881,7 @@ async function run(argv) {
|
|
|
948
881
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
949
882
|
}
|
|
950
883
|
if (command === "init") {
|
|
951
|
-
const { initCommand } = await import("./init-
|
|
884
|
+
const { initCommand } = await import("./init-RgCQXZhb.js");
|
|
952
885
|
return initCommand(values, filters, {
|
|
953
886
|
out: (s) => console.log(s),
|
|
954
887
|
input: process.stdin,
|
|
@@ -957,7 +890,7 @@ async function run(argv) {
|
|
|
957
890
|
});
|
|
958
891
|
}
|
|
959
892
|
if (command === "doctor") {
|
|
960
|
-
const { doctorCommand } = await import("./doctor-
|
|
893
|
+
const { doctorCommand } = await import("./doctor-DQ2p0euG.js");
|
|
961
894
|
return doctorCommand(values, {
|
|
962
895
|
out: (s) => console.log(s),
|
|
963
896
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -965,7 +898,7 @@ async function run(argv) {
|
|
|
965
898
|
});
|
|
966
899
|
}
|
|
967
900
|
if (command === "report") {
|
|
968
|
-
const { reportCommand } = await import("./command-
|
|
901
|
+
const { reportCommand } = await import("./command-BXLUfw4h.js");
|
|
969
902
|
return reportCommand(values, filters, {
|
|
970
903
|
out: (s) => console.log(s),
|
|
971
904
|
err: (s) => console.error(dim(s)),
|
|
@@ -980,7 +913,7 @@ async function run(argv) {
|
|
|
980
913
|
}
|
|
981
914
|
if (command === "studio") {
|
|
982
915
|
const config = await loadConfig(configPath, "studio");
|
|
983
|
-
const { studioCommand } = await import("./command-
|
|
916
|
+
const { studioCommand } = await import("./command-I1euLLio.js");
|
|
984
917
|
return studioCommand({
|
|
985
918
|
config,
|
|
986
919
|
flags: values,
|
|
@@ -991,7 +924,7 @@ async function run(argv) {
|
|
|
991
924
|
if (command === "mcp") {
|
|
992
925
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
993
926
|
const config = await loadConfig(configPath, "mcp");
|
|
994
|
-
const { mcpCommand } = await import("./command-
|
|
927
|
+
const { mcpCommand } = await import("./command-Bxdy62dW.js");
|
|
995
928
|
return mcpCommand({
|
|
996
929
|
config,
|
|
997
930
|
flags: values,
|
|
@@ -1000,7 +933,7 @@ async function run(argv) {
|
|
|
1000
933
|
}
|
|
1001
934
|
if (command in HISTORY_COMMANDS) {
|
|
1002
935
|
const config = await loadConfig(configPath, command);
|
|
1003
|
-
const commands = await import("./commands-
|
|
936
|
+
const commands = await import("./commands-p1S4nw_H.js");
|
|
1004
937
|
const ctx = {
|
|
1005
938
|
config,
|
|
1006
939
|
flags: values,
|
|
@@ -1013,7 +946,7 @@ async function run(argv) {
|
|
|
1013
946
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
1014
947
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
1015
948
|
case "query": {
|
|
1016
|
-
const { queryCommand } = await import("./query-
|
|
949
|
+
const { queryCommand } = await import("./query-Cvbwl804.js");
|
|
1017
950
|
return queryCommand({
|
|
1018
951
|
...ctx,
|
|
1019
952
|
err: (s) => console.error(dim(s))
|
|
@@ -1021,7 +954,7 @@ async function run(argv) {
|
|
|
1021
954
|
}
|
|
1022
955
|
case "analyze": {
|
|
1023
956
|
loadEnvFiles(configPath);
|
|
1024
|
-
const { analyzeCommand } = await import("./command-
|
|
957
|
+
const { analyzeCommand } = await import("./command-Be6BSE2L.js");
|
|
1025
958
|
return analyzeCommand({
|
|
1026
959
|
...ctx,
|
|
1027
960
|
err: (s) => console.error(dim(s))
|
|
@@ -1033,6 +966,16 @@ async function run(argv) {
|
|
|
1033
966
|
if (command !== "run") throw new UsageError(`Unknown command "${command}". Run reeval --help.`);
|
|
1034
967
|
return runCommand(values, configPath, filters);
|
|
1035
968
|
}
|
|
969
|
+
/** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
|
|
970
|
+
async function dryRunPrices(planned, judgeModel) {
|
|
971
|
+
const { needsPrices } = await import("./cost-DmnAod9x.js");
|
|
972
|
+
if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
|
|
973
|
+
table: null,
|
|
974
|
+
why: "nothing to price"
|
|
975
|
+
};
|
|
976
|
+
const { loadPrices } = await import("./prices-BcLEXvdY.js");
|
|
977
|
+
return loadPrices();
|
|
978
|
+
}
|
|
1036
979
|
/** Asks a yes/no question on the terminal; anything but y or yes is no. */
|
|
1037
980
|
async function confirm(question) {
|
|
1038
981
|
const { createInterface } = await import("node:readline/promises");
|
|
@@ -1060,8 +1003,8 @@ async function cleanCommand(flags, configPath) {
|
|
|
1060
1003
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1061
1004
|
console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
|
|
1062
1005
|
if (flags.images) {
|
|
1063
|
-
const { dockerAvailable, pruneImages } = await import("./src-
|
|
1064
|
-
const { requireDocker } = await import("./providers-
|
|
1006
|
+
const { dockerAvailable, pruneImages } = await import("./src-1jHbFaGW.js");
|
|
1007
|
+
const { requireDocker } = await import("./providers-D1Zxp2he.js");
|
|
1065
1008
|
const engines = [];
|
|
1066
1009
|
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1067
1010
|
if (!engines.length) await requireDocker();
|
|
@@ -1082,7 +1025,7 @@ function verbosityOf(flags) {
|
|
|
1082
1025
|
* does. Needs a config, as `reeval run` does: there is no default image.
|
|
1083
1026
|
*/
|
|
1084
1027
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1085
|
-
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-
|
|
1028
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-DzLeFjLx.js");
|
|
1086
1029
|
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
|
|
1087
1030
|
...flags,
|
|
1088
1031
|
"dry-run": true
|
|
@@ -1142,7 +1085,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1142
1085
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1143
1086
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1144
1087
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1145
|
-
const { engineVersions } = await import("./node-
|
|
1088
|
+
const { engineVersions } = await import("./node-BSq5qHnQ.js");
|
|
1146
1089
|
const renderer = createRenderer({
|
|
1147
1090
|
out: setup.json ? setup.dryRun ? { write: () => true } : process.stderr : process.stdout,
|
|
1148
1091
|
output: setup.output,
|
|
@@ -1160,11 +1103,11 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1160
1103
|
});
|
|
1161
1104
|
throw error;
|
|
1162
1105
|
}
|
|
1163
|
-
const { engineRun, plan, sandboxVersions, budgetReached, live, questions } = executed;
|
|
1106
|
+
const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
|
|
1164
1107
|
invocation.unhandledErrors = engineRun.unhandledErrors;
|
|
1165
1108
|
const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
|
|
1166
1109
|
if ((interrupted || engineRun.cancelled) && engines.length) {
|
|
1167
|
-
const { removeRunContainers } = await import("./docker-
|
|
1110
|
+
const { removeRunContainers } = await import("./docker-DivfZl2u.js");
|
|
1168
1111
|
await removeRunContainers(ctx.runId, engines);
|
|
1169
1112
|
}
|
|
1170
1113
|
if (engineRun.stopped || !plan) {
|
|
@@ -1172,10 +1115,15 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1172
1115
|
renderer.close();
|
|
1173
1116
|
return EXIT.ok;
|
|
1174
1117
|
}
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1118
|
+
let stopped = cutShort || engineRun.cancelled;
|
|
1119
|
+
if (stopped) questions?.cancel();
|
|
1120
|
+
let answers = stopped ? await questions?.answered() : await questions?.settle();
|
|
1121
|
+
if (!stopped && interrupted && questions) {
|
|
1122
|
+
stopped = true;
|
|
1123
|
+
answers = await questions?.answered();
|
|
1124
|
+
}
|
|
1125
|
+
const { lockfiles } = await import("./state-vznMh2wl.js");
|
|
1126
|
+
const { writeScratchRun } = await import("./results-CMgZEANy.js");
|
|
1179
1127
|
const run = await writeScratchRun({
|
|
1180
1128
|
ctx,
|
|
1181
1129
|
author: ctx.author,
|
|
@@ -1196,18 +1144,18 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1196
1144
|
previous: ctx.previous,
|
|
1197
1145
|
previousArtifacts: ctx.previousArtifacts,
|
|
1198
1146
|
runner: runnerInfo(),
|
|
1199
|
-
...answers
|
|
1147
|
+
...answers && (answers.evals.size || answers.attempts.size) ? { answers } : {}
|
|
1200
1148
|
}, () => live?.stop());
|
|
1201
1149
|
const { record } = run;
|
|
1202
1150
|
const outcome = runOutcome(record);
|
|
1203
1151
|
const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1204
1152
|
let insights = run.insights;
|
|
1205
1153
|
let insightLines = [];
|
|
1206
|
-
const { runInsights, wantsInsights } = await import("./run-
|
|
1154
|
+
const { runInsights, wantsInsights } = await import("./run-CwIEK16q.js");
|
|
1207
1155
|
const wanted = !stopped && wantsInsights(ctx.config, !flags["no-insights"]);
|
|
1208
1156
|
if (wanted && interrupted) insightLines = [`insights skipped (${stopCode === EXIT.terminated ? "SIGTERM" : "Ctrl-C"}): reeval analyze ${record.runId} --insights to write them`];
|
|
1209
1157
|
else if (wanted) {
|
|
1210
|
-
const { scratchDir } = await import("./files-
|
|
1158
|
+
const { scratchDir } = await import("./files-C2sjmXhk.js");
|
|
1211
1159
|
const stop = new AbortController();
|
|
1212
1160
|
let stopSignal = "SIGINT";
|
|
1213
1161
|
onInsightsSignal = (signal) => {
|
|
@@ -1269,7 +1217,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1269
1217
|
const { config } = ctx;
|
|
1270
1218
|
const mode = setup.record;
|
|
1271
1219
|
if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
|
|
1272
|
-
const { record: recordCmd } = await import("./commands-
|
|
1220
|
+
const { record: recordCmd } = await import("./commands-p1S4nw_H.js");
|
|
1273
1221
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
1274
1222
|
const line = await recordCmd(config, record.runId, {
|
|
1275
1223
|
...flags,
|
|
@@ -1279,7 +1227,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1279
1227
|
else console.log(line);
|
|
1280
1228
|
}
|
|
1281
1229
|
if (setup.json) {
|
|
1282
|
-
const { suiteResult } = await import("./suite-
|
|
1230
|
+
const { suiteResult } = await import("./suite-BsWGHzlG.js");
|
|
1283
1231
|
process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
|
|
1284
1232
|
}
|
|
1285
1233
|
return code;
|
|
@@ -1312,7 +1260,7 @@ async function prepareRun(flags, configPath) {
|
|
|
1312
1260
|
const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
|
|
1313
1261
|
const container = sandboxes.find((s) => s.kind === "docker");
|
|
1314
1262
|
if (!container) return hostCapacity();
|
|
1315
|
-
const { dockerCapacity } = await import("./src-
|
|
1263
|
+
const { dockerCapacity } = await import("./src-1jHbFaGW.js");
|
|
1316
1264
|
return await dockerCapacity(container.engine) ?? hostCapacity();
|
|
1317
1265
|
});
|
|
1318
1266
|
return {
|
|
@@ -1341,11 +1289,11 @@ async function prepareRun(flags, configPath) {
|
|
|
1341
1289
|
async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
|
|
1342
1290
|
const startedAt = /* @__PURE__ */ new Date();
|
|
1343
1291
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
1344
|
-
const { worktreePatch } = await import("./state-
|
|
1292
|
+
const { worktreePatch } = await import("./state-vznMh2wl.js");
|
|
1345
1293
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
1346
1294
|
const who = await author(config.root);
|
|
1347
|
-
const { stagingDir, sweepStaging } = await import("./results-
|
|
1348
|
-
const { scratchDir } = await import("./files-
|
|
1295
|
+
const { stagingDir, sweepStaging } = await import("./results-CMgZEANy.js");
|
|
1296
|
+
const { scratchDir } = await import("./files-C2sjmXhk.js");
|
|
1349
1297
|
const scratch = scratchDir(config.root);
|
|
1350
1298
|
if (resume !== void 0) {
|
|
1351
1299
|
const target = await resolveResume(config.root, resume || void 0);
|
|
@@ -1356,7 +1304,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1356
1304
|
artifacts: target.previousArtifacts
|
|
1357
1305
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
1358
1306
|
if (!dryRun) try {
|
|
1359
|
-
const { removeRunContainers } = await import("./docker-
|
|
1307
|
+
const { removeRunContainers } = await import("./docker-DivfZl2u.js");
|
|
1360
1308
|
await removeRunContainers(target.runId, ["docker", "podman"]);
|
|
1361
1309
|
} catch {}
|
|
1362
1310
|
return {
|
|
@@ -1427,8 +1375,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1427
1375
|
configFile,
|
|
1428
1376
|
completed: []
|
|
1429
1377
|
}));
|
|
1430
|
-
const { runEngine } = await import("./node-
|
|
1431
|
-
const { buildPlan, checkAttemptBudget, planDocument, planInsights, planLine, planTable } = await import("./plan-
|
|
1378
|
+
const { runEngine } = await import("./node-BSq5qHnQ.js");
|
|
1379
|
+
const { buildPlan, checkAttemptBudget, planDocument, planInsights, planLine, planTable } = await import("./plan-BIwUzlB8.js");
|
|
1432
1380
|
let plan;
|
|
1433
1381
|
let sandboxVersions = { images: [] };
|
|
1434
1382
|
let preflightStop;
|
|
@@ -1454,7 +1402,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1454
1402
|
} : p);
|
|
1455
1403
|
toRun = skipped;
|
|
1456
1404
|
for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
|
|
1457
|
-
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-
|
|
1405
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-Ch8kX87u.js") : void 0;
|
|
1406
|
+
const judge = runJudgeModel(config, profile);
|
|
1407
|
+
const judgeModel = judge === void 0 ? null : judgeModelName(judge);
|
|
1458
1408
|
plan = await buildPlan({
|
|
1459
1409
|
planned: skipped,
|
|
1460
1410
|
profile,
|
|
@@ -1462,21 +1412,26 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1462
1412
|
concurrency,
|
|
1463
1413
|
root: config.root,
|
|
1464
1414
|
historyDir: config.history.dir,
|
|
1465
|
-
questionModel: models?.questionModelLabel(config) ?? null,
|
|
1466
|
-
insights: await planInsights(config, setup.insights)
|
|
1415
|
+
questionModel: models?.questionModelLabel(config, judge) ?? null,
|
|
1416
|
+
insights: await planInsights(config, setup.insights),
|
|
1417
|
+
judgeModel,
|
|
1418
|
+
...setup.dryRun ? { prices: await dryRunPrices(skipped, judgeModel) } : {}
|
|
1467
1419
|
});
|
|
1468
1420
|
plan.runId = runId;
|
|
1469
1421
|
if (plan.insights) {
|
|
1470
|
-
const { analysisKeyProblem } = await import("./model-
|
|
1422
|
+
const { analysisKeyProblem } = await import("./model-Ch8kX87u.js");
|
|
1471
1423
|
const problem = analysisKeyProblem(config);
|
|
1472
1424
|
if (problem) plan.warnings.push(`insights will be skipped: ${problem}`);
|
|
1473
1425
|
}
|
|
1426
|
+
const cloud = ciRunner(process.env);
|
|
1427
|
+
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
|
|
1428
|
+
if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
|
|
1474
1429
|
if (completed.size) plan.resume = {
|
|
1475
1430
|
runId,
|
|
1476
1431
|
completed: completed.size
|
|
1477
1432
|
};
|
|
1478
1433
|
run.completed = [...completed];
|
|
1479
|
-
const { imageStages, imageTargets } = await import("./images-
|
|
1434
|
+
const { imageStages, imageTargets } = await import("./images-DzLeFjLx.js");
|
|
1480
1435
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1481
1436
|
id: p.id,
|
|
1482
1437
|
harness: p.attempt.harness,
|
|
@@ -1498,8 +1453,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1498
1453
|
if (setup.dryRun) {
|
|
1499
1454
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
1500
1455
|
lines.push(...planTable(plan, todos));
|
|
1501
|
-
const { imageStatusLines } = await import("./images-
|
|
1502
|
-
const { dockerAvailable, engineSpec } = await import("./src-
|
|
1456
|
+
const { imageStatusLines } = await import("./images-DzLeFjLx.js");
|
|
1457
|
+
const { dockerAvailable, engineSpec } = await import("./src-1jHbFaGW.js");
|
|
1503
1458
|
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
1504
1459
|
const down = [];
|
|
1505
1460
|
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
@@ -1508,9 +1463,12 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1508
1463
|
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
1509
1464
|
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
1510
1465
|
}
|
|
1466
|
+
const { costLine } = await import("./cost-DmnAod9x.js");
|
|
1467
|
+
const cost = plan.cost && costLine(plan.cost);
|
|
1468
|
+
if (cost) lines.push(cost);
|
|
1511
1469
|
}
|
|
1512
1470
|
if (setup.dryRun && plan.insights) {
|
|
1513
|
-
const { formatTokens } = await import("./usage-
|
|
1471
|
+
const { formatTokens } = await import("./usage-DF3_p5pE.js");
|
|
1514
1472
|
lines.push(`insights ${plan.insights.model} · up to ${formatTokens(plan.insights.maxTokens)} tokens`);
|
|
1515
1473
|
}
|
|
1516
1474
|
lines.push(...plan.warnings.map((w) => `warning: ${w}`));
|
|
@@ -1525,15 +1483,23 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1525
1483
|
checkAttemptBudget(plan, setup.force, config.file);
|
|
1526
1484
|
const questionModels = /* @__PURE__ */ new Set();
|
|
1527
1485
|
const asked = [];
|
|
1486
|
+
const scope = gatewayScope(config, profile);
|
|
1528
1487
|
for (const row of plan.rows) {
|
|
1529
1488
|
if (row.plan.skip || !models) continue;
|
|
1530
|
-
const questions = (row.plan.questions ?? []).map((q) =>
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1489
|
+
const questions = (row.plan.questions ?? []).map((q) => {
|
|
1490
|
+
const model = models.questionModel(config, q, row.id, judge);
|
|
1491
|
+
const gateway = resolveModelGateway(model, void 0, scope, "the question model");
|
|
1492
|
+
return {
|
|
1493
|
+
n: q.n,
|
|
1494
|
+
question: q.question,
|
|
1495
|
+
model,
|
|
1496
|
+
...gateway ? { gateway } : {},
|
|
1497
|
+
...q.across !== void 0 ? { across: q.across } : {},
|
|
1498
|
+
...q.choices ? { choices: q.choices } : {},
|
|
1499
|
+
...q.state ? { state: q.state } : {},
|
|
1500
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
1501
|
+
};
|
|
1502
|
+
});
|
|
1537
1503
|
for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
|
|
1538
1504
|
if (questions.length) asked.push({
|
|
1539
1505
|
row,
|
|
@@ -1545,7 +1511,10 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1545
1511
|
profile,
|
|
1546
1512
|
questionModels: [...questionModels]
|
|
1547
1513
|
});
|
|
1548
|
-
if (missing.length)
|
|
1514
|
+
if (missing.length) {
|
|
1515
|
+
const hint = missing.some((m) => m.includes("(for direct())")) ? unusedGatewayKey(process.env) : "";
|
|
1516
|
+
throw new UsageError(`Missing credentials: ${missing.join("; ")}. Set them in the environment or .env.local.${hint}`);
|
|
1517
|
+
}
|
|
1549
1518
|
const malformed = malformedCredentials(config, process.env, {
|
|
1550
1519
|
harnesses: plan.harnessDescriptors,
|
|
1551
1520
|
profile,
|
|
@@ -1553,7 +1522,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1553
1522
|
});
|
|
1554
1523
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
1555
1524
|
if (asked.length) {
|
|
1556
|
-
const { questionRunner } = await import("./questions-
|
|
1525
|
+
const { questionRunner } = await import("./questions-bAudIc04.js");
|
|
1526
|
+
const warned = /* @__PURE__ */ new Set();
|
|
1557
1527
|
questions = questionRunner({
|
|
1558
1528
|
evals: asked,
|
|
1559
1529
|
stagingDir,
|
|
@@ -1565,12 +1535,31 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1565
1535
|
type: "notice",
|
|
1566
1536
|
at: Date.now(),
|
|
1567
1537
|
level: "warn",
|
|
1568
|
-
text: `
|
|
1538
|
+
text: `question "${clip(answer.question, 80)}" on ${id} failed: ${answer.error}. Outcomes are unchanged.`
|
|
1539
|
+
});
|
|
1540
|
+
},
|
|
1541
|
+
onAttemptAnswer: (id, attempt, answer) => {
|
|
1542
|
+
const once = `${id}\0${answer.question}`;
|
|
1543
|
+
if (answer.error === void 0 || warned.has(once)) return;
|
|
1544
|
+
warned.add(once);
|
|
1545
|
+
renderer.onEvent({
|
|
1546
|
+
type: "notice",
|
|
1547
|
+
at: Date.now(),
|
|
1548
|
+
level: "warn",
|
|
1549
|
+
text: `question "${clip(answer.question, 80)}" on ${attempt} failed: ${answer.error}. Outcomes are unchanged; later failures of it on ${id} aren't shown.`
|
|
1550
|
+
});
|
|
1551
|
+
},
|
|
1552
|
+
onUnshown: (id, error) => {
|
|
1553
|
+
renderer.onEvent({
|
|
1554
|
+
type: "notice",
|
|
1555
|
+
at: Date.now(),
|
|
1556
|
+
level: "warn",
|
|
1557
|
+
text: `${error} on ${id}, so each attempt's answer is that error. Outcomes are unchanged.`
|
|
1569
1558
|
});
|
|
1570
1559
|
}
|
|
1571
1560
|
});
|
|
1572
1561
|
}
|
|
1573
|
-
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-
|
|
1562
|
+
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-QZCacP8A.js");
|
|
1574
1563
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
1575
1564
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
1576
1565
|
workspace: a.workspace,
|
|
@@ -1661,6 +1650,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1661
1650
|
budgetTimer.unref();
|
|
1662
1651
|
};
|
|
1663
1652
|
let engineRun;
|
|
1653
|
+
let cutShort = false;
|
|
1664
1654
|
try {
|
|
1665
1655
|
engineRun = await runEngine({
|
|
1666
1656
|
run,
|
|
@@ -1720,15 +1710,16 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1720
1710
|
process.on("SIGTERM", onSignal);
|
|
1721
1711
|
}
|
|
1722
1712
|
});
|
|
1713
|
+
cutShort = interrupted;
|
|
1723
1714
|
live?.phase("finishing");
|
|
1724
1715
|
} catch (error) {
|
|
1725
1716
|
live?.remove();
|
|
1726
1717
|
throw error;
|
|
1727
1718
|
} finally {
|
|
1728
1719
|
clearTimeout(budgetTimer);
|
|
1729
|
-
const { removePrepared } = await import("./preflight-
|
|
1720
|
+
const { removePrepared } = await import("./preflight-D9y-HvZQ.js");
|
|
1730
1721
|
removePrepared(config.root, runId);
|
|
1731
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
1722
|
+
const { removePreparedWorkspaces } = await import("./workspace-QZCacP8A.js");
|
|
1732
1723
|
removePreparedWorkspaces(config.root, runId);
|
|
1733
1724
|
}
|
|
1734
1725
|
if (preflightStop) throw preflightStop;
|
|
@@ -1740,6 +1731,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1740
1731
|
plan,
|
|
1741
1732
|
sandboxVersions,
|
|
1742
1733
|
budgetReached: () => reached,
|
|
1734
|
+
cutShort,
|
|
1743
1735
|
live,
|
|
1744
1736
|
questions
|
|
1745
1737
|
};
|
|
@@ -1810,4 +1802,4 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
1810
1802
|
//#endregion
|
|
1811
1803
|
export {};
|
|
1812
1804
|
|
|
1813
|
-
//# sourceMappingURL=cli-
|
|
1805
|
+
//# sourceMappingURL=cli-C_4gaJ50.js.map
|