reeval 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +190 -190
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-ChbvRx3P.js +11 -0
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/attempt-host-BOp8JklD.js +295 -0
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
- package/dist/client-Ct6s6OnF.js.map +1 -0
- package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
- package/dist/command-DbSMcxZL.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/config-k6m7XbAz.js +3777 -0
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
- package/dist/contrasts-DLZF4dGF.js.map +1 -0
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/cost-ChPa8-Fp.js +359 -0
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +30 -0
- package/dist/gateway.js +39 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +33 -21
- package/dist/harness.js +1 -1
- package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +34 -35
- package/dist/index.js +5 -6
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/judge-CKdgEXN7.js +575 -0
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +8 -9
- package/dist/judge.js +7 -9
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/model-vRiUycWh.js +77 -0
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/prices-QaS-kjSQ.js +253 -0
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/questions-BRhU8MzB.js +861 -0
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/remarks-C7i2LMRb.js +596 -0
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/remote-DFIzcx_6.js +273 -0
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
- package/dist/src-1lpTocbd.js.map +1 -0
- package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
- package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
- package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
- package/dist/src-DgHuStEi.js.map +1 -0
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
- package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/stats-hfpuYsAM.js +315 -0
- package/dist/stats-hfpuYsAM.js.map +1 -0
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +449 -2
- package/dist/testing.js +16 -3
- package/dist/testing.js.map +1 -0
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +505 -43
- package/dist/view/index.js +10 -9
- package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
- package/dist/views-BFO1BLKu.js.map +1 -0
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +10 -2
- package/skills/eval-rubric/SKILL.md +43 -103
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/agent-ClE-g9VL.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/cli-spec-BXIBUlaK.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-C6vwmjS7.js +0 -91
- package/dist/command-C6vwmjS7.js.map +0 -1
- package/dist/command-CANun0m0.js.map +0 -1
- package/dist/command-CBaHukyl.js +0 -335
- package/dist/command-CBaHukyl.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js +0 -602
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/docker-DuHcQidD.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-C9-ritA5.d.ts +0 -92
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js +0 -898
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-Bs6H3HiG.js +0 -65
- package/dist/files-Bs6H3HiG.js.map +0 -1
- package/dist/files-CN1QfenC.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D27nUk0K.js.map +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js +0 -1646
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js +0 -323
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/header-BVzbRnNv.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/insights-B2Xmhz9o.js +0 -64
- package/dist/insights-B2Xmhz9o.js.map +0 -1
- package/dist/invocations-D_ePcwyL.js.map +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-B8H3gL69.js +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js +0 -3305
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/patch-uMq9rCl_.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/plugins-P-uh54VF.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js +0 -523
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remarks-Ml6bcSdV.js +0 -278
- package/dist/remarks-Ml6bcSdV.js.map +0 -1
- package/dist/remote-Dkeghk8k.js +0 -378
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/results-D-naai_Q.js.map +0 -1
- package/dist/resume-8-nnW8iO.js.map +0 -1
- package/dist/rules-CcV3zLDR.js +0 -318
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/run-Bel7Vde4.js +0 -88
- package/dist/run-Bel7Vde4.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/server-De8oH3bt.js.map +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-C686eZpY.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-CIpKL45U.js.map +0 -1
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-Xj_BZKbq.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/src-djNNYIdP.js.map +0 -1
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/worker-entry-CqrzGPF4.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -1,30 +1,31 @@
|
|
|
1
|
-
import { n as isOneOf } from "./object-CLf2-tNq.js";
|
|
2
1
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
2
|
+
import { h as parseDuration } from "./shared-B_lvGqWz.js";
|
|
3
|
+
import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-BOFgzFDg.js";
|
|
4
|
+
import { C as pluginBuildsLabel } from "./plugins-D-CP6mMd.js";
|
|
6
5
|
import { a as fromCwd, o as newRunId } from "./ids-Bp6ut7ZD.js";
|
|
7
|
-
import { c as
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { o as
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
6
|
+
import { $ as isDirect, A as harnessId, E as checkLimitHarness, F as plannedSandboxes, R as sandboxId, X as directOnCloud, Y as ciRunner, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, tt as resolveOutput, u as loadConfig, v as gatewayScope } from "./config-k6m7XbAz.js";
|
|
7
|
+
import { l as judgeModelName, m as unusedGatewayKey } from "./judge-CKdgEXN7.js";
|
|
8
|
+
import { y as vertexSettingsIn } from "./vertex-DypYjx71.js";
|
|
9
|
+
import { _ as registerSecret, m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
10
|
+
import { a as plural, i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-DTigcA64.js";
|
|
11
|
+
import { n as loadEnvFiles, t as inCI } from "./env-CARj5Axq.js";
|
|
12
|
+
import { o as PACKAGE_VERSION } from "./root-Dlff-zp8.js";
|
|
13
|
+
import { U as LABEL, ot as imageInfo } from "./src-D91Hd6hT.js";
|
|
14
|
+
import { s as debugTime } from "./debug-BhizgAsU.js";
|
|
15
|
+
import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
|
|
15
16
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-
|
|
19
|
-
import { p as render } from "./summary-BFMHMTz2.js";
|
|
17
|
+
import { D as runOutcome, G as triesNote, _ as formatUsage } from "./model-CTnyvh7H.js";
|
|
18
|
+
import { u as suiteResult } from "./suite-Co8cwelb.js";
|
|
19
|
+
import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-D0qyrPZi.js";
|
|
20
20
|
import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
|
|
21
|
-
import {
|
|
21
|
+
import { f as render } from "./summary-DLFHVoux.js";
|
|
22
|
+
import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-BizwUn3h.js";
|
|
22
23
|
import { t as budgetMarker } from "./budget-DBUrH4jX.js";
|
|
23
|
-
import { c as
|
|
24
|
-
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-
|
|
25
|
-
import { n as imageHarness } from "./prepare-
|
|
26
|
-
import { i as createProviders, n as attemptImage } from "./providers-
|
|
27
|
-
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-
|
|
24
|
+
import { a as prepareResume, c as writeCurrent, i as loadPrevious, l as writeLiveProgress, r as keptAttempts, s as resolveResume, t as completedKeysFor } from "./resume-j-rcIryi.js";
|
|
25
|
+
import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-9ixuXzsx.js";
|
|
26
|
+
import { n as imageHarness } from "./prepare-BfyikQit.js";
|
|
27
|
+
import { i as createProviders, n as attemptImage } from "./providers-B6b9Tnyi.js";
|
|
28
|
+
import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DW-lTk2f.js";
|
|
28
29
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
29
30
|
import { dirname, join } from "node:path";
|
|
30
31
|
import { inspect } from "node:util";
|
|
@@ -245,14 +246,13 @@ function offset(ms) {
|
|
|
245
246
|
}
|
|
246
247
|
function outcomeOf(e) {
|
|
247
248
|
if (e.state === "skipped") return "skipped";
|
|
248
|
-
if (e.meta?.errorKind === "aborted") return "
|
|
249
|
+
if (e.meta?.errorKind === "aborted") return "canceled";
|
|
249
250
|
return e.meta?.outcome ?? (e.state === "passed" ? "pass" : "fail");
|
|
250
251
|
}
|
|
251
252
|
/** What tells an eval's attempts apart: the axes that vary (`cursor:composer-2.5 · with-skill`), else the harness. */
|
|
252
253
|
function attemptAxes(a, fallback) {
|
|
253
254
|
return a?.label.replace(/\s*#\d+$/, "") || a?.harness.kind || fallback;
|
|
254
255
|
}
|
|
255
|
-
const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
|
|
256
256
|
/** A command on one line: the first line, cut to 80 characters at a word boundary. */
|
|
257
257
|
function oneLine(text, max = 80) {
|
|
258
258
|
const [first = ""] = text.trim().split("\n");
|
|
@@ -274,7 +274,8 @@ function progressLine(c, p) {
|
|
|
274
274
|
case "check-start": return c.dim(`$ ${oneLine(p.command)}`);
|
|
275
275
|
case "check-end": {
|
|
276
276
|
const exit = p.exitCode === 0 ? c.dim("exit 0") : c.red(`exit ${p.exitCode}`);
|
|
277
|
-
|
|
277
|
+
const tries = p.try && p.tries ? triesNote(p.exitCode, p.try, p.tries) : null;
|
|
278
|
+
return `$ ${oneLine(p.command)} ${exit} ${c.dim([formatDuration(p.ms), tries].filter(Boolean).join(" · "))}`;
|
|
278
279
|
}
|
|
279
280
|
case "judge-start": return c.dim(`judging: ${oneLine(p.question, 120)}`);
|
|
280
281
|
case "judge-end": return judgeLine(c, p);
|
|
@@ -334,7 +335,7 @@ function tailLines(text, max) {
|
|
|
334
335
|
*/
|
|
335
336
|
function failureLines(c, e, options) {
|
|
336
337
|
const outcome = outcomeOf(e);
|
|
337
|
-
if (outcome === "
|
|
338
|
+
if (outcome === "canceled") return [c.yellow("canceled")];
|
|
338
339
|
const message = e.errors[0] ?? "";
|
|
339
340
|
const turns = (e.meta?.turnErrors ?? []).filter((t) => !message.includes(t.replace(/^turn \d+ error: /, ""))).map((t) => c.yellow(t));
|
|
340
341
|
return [...failureBody(c, e, outcome, message, options), ...turns];
|
|
@@ -382,18 +383,17 @@ function resultLines(c, e, verbose) {
|
|
|
382
383
|
const tokens = formatUsage(record.cost.usage);
|
|
383
384
|
const notes = [
|
|
384
385
|
...record.status === "budget" ? [c.yellow("Budget reached: attempts that hadn’t started were skipped")] : [],
|
|
385
|
-
...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were
|
|
386
|
+
...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were canceled")] : [],
|
|
386
387
|
c.dim([
|
|
387
388
|
formatDuration(record.durationMs),
|
|
388
389
|
tokens,
|
|
389
390
|
`results ${e.summaryPath}`
|
|
390
391
|
].filter(Boolean).join(" · "))
|
|
391
392
|
];
|
|
392
|
-
return render(suiteResult(record), e.
|
|
393
|
+
return render(suiteResult(record), e.remarks ?? null, {
|
|
393
394
|
color: c.enabled,
|
|
394
395
|
verbose,
|
|
395
|
-
notes
|
|
396
|
-
hint: e.hint
|
|
396
|
+
notes
|
|
397
397
|
});
|
|
398
398
|
}
|
|
399
399
|
//#endregion
|
|
@@ -557,13 +557,11 @@ var PlainRenderer = class {
|
|
|
557
557
|
#evals = /* @__PURE__ */ new Set();
|
|
558
558
|
#tries = /* @__PURE__ */ new Set();
|
|
559
559
|
#ended = /* @__PURE__ */ new Set();
|
|
560
|
-
/** Insights after the last attempt, once they've started: they take the status line. */
|
|
561
|
-
#insights = null;
|
|
562
560
|
constructor(options) {
|
|
563
561
|
this.#o = options;
|
|
564
562
|
this.#c = options.colors;
|
|
565
563
|
this.#now = options.now ?? Date.now;
|
|
566
|
-
this.#live = options.output.
|
|
564
|
+
this.#live = options.output.statusLine;
|
|
567
565
|
const { heartbeatMs } = options.output;
|
|
568
566
|
if (options.timers !== false && heartbeatMs > 0) {
|
|
569
567
|
this.#timer = setInterval(() => this.tick(), Math.min(heartbeatMs, 5e3));
|
|
@@ -575,16 +573,11 @@ var PlainRenderer = class {
|
|
|
575
573
|
this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
|
|
576
574
|
this.#lastWrite = this.#now();
|
|
577
575
|
}
|
|
578
|
-
/** `7 evals · 12/28 tries`: what's collected so far and how many tries ended
|
|
576
|
+
/** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
|
|
579
577
|
#drawStatus() {
|
|
580
|
-
if (!this.#live) return;
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
else {
|
|
584
|
-
if (!this.#tries.size) return;
|
|
585
|
-
const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
|
|
586
|
-
status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
|
|
587
|
-
}
|
|
578
|
+
if (!this.#live || !this.#tries.size) return;
|
|
579
|
+
const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
|
|
580
|
+
const status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
|
|
588
581
|
if (status === this.#status) return;
|
|
589
582
|
this.#o.out.write(`${CLEAR}${status}`);
|
|
590
583
|
this.#status = status;
|
|
@@ -712,14 +705,6 @@ var PlainRenderer = class {
|
|
|
712
705
|
this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
|
|
713
706
|
}
|
|
714
707
|
return;
|
|
715
|
-
case "insights-progress":
|
|
716
|
-
this.#insights = {
|
|
717
|
-
done: e.done,
|
|
718
|
-
total: e.total,
|
|
719
|
-
startedAt: this.#insights?.startedAt ?? e.at
|
|
720
|
-
};
|
|
721
|
-
this.#drawStatus();
|
|
722
|
-
return;
|
|
723
708
|
case "notice": {
|
|
724
709
|
const paint = {
|
|
725
710
|
info: c.dim,
|
|
@@ -780,18 +765,12 @@ var PlainRenderer = class {
|
|
|
780
765
|
this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
|
|
781
766
|
}
|
|
782
767
|
}
|
|
783
|
-
/** Prints the heartbeat if nothing was written for `heartbeatMs` and evals
|
|
768
|
+
/** Prints the heartbeat if nothing was written for `heartbeatMs` and evals are running. */
|
|
784
769
|
tick(now = this.#now()) {
|
|
785
770
|
const { heartbeatMs } = this.#o.output;
|
|
786
771
|
if (this.#live || heartbeatMs <= 0) return;
|
|
787
772
|
if (now - this.#lastWrite < heartbeatMs) return;
|
|
788
773
|
const c = this.#c;
|
|
789
|
-
const insights = this.#insights;
|
|
790
|
-
if (insights) {
|
|
791
|
-
const elapsed = formatDuration(now - insights.startedAt);
|
|
792
|
-
if (insights.done < insights.total) this.#write(c.dim(`… insights ${insights.done}/${insights.total} after ${elapsed}`));
|
|
793
|
-
return;
|
|
794
|
-
}
|
|
795
774
|
if (this.#running.size === 0) return;
|
|
796
775
|
const running = [...this.#running.values()].map((a) => {
|
|
797
776
|
const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
|
|
@@ -807,73 +786,6 @@ var PlainRenderer = class {
|
|
|
807
786
|
}
|
|
808
787
|
};
|
|
809
788
|
//#endregion
|
|
810
|
-
//#region src/ui/mode.ts
|
|
811
|
-
const set = (env, name) => {
|
|
812
|
-
const value = env[name];
|
|
813
|
-
return !!value && value !== "false" && value !== "0";
|
|
814
|
-
};
|
|
815
|
-
const CI_VARS = [
|
|
816
|
-
"BUILD_ID",
|
|
817
|
-
"BUILD_NUMBER",
|
|
818
|
-
"CI",
|
|
819
|
-
"CI_APP_ID",
|
|
820
|
-
"CI_BUILD_ID",
|
|
821
|
-
"CI_BUILD_NUMBER",
|
|
822
|
-
"CI_NAME",
|
|
823
|
-
"CONTINUOUS_INTEGRATION",
|
|
824
|
-
"RUN_ID",
|
|
825
|
-
"TEAMCITY_VERSION",
|
|
826
|
-
"GITHUB_ACTIONS"
|
|
827
|
-
];
|
|
828
|
-
const isCI = (env) => CI_VARS.some((name) => set(env, name));
|
|
829
|
-
function pick(name, value, allowed) {
|
|
830
|
-
if (value === void 0 || value === "") return void 0;
|
|
831
|
-
if (value === "auto" || isOneOf(allowed, value)) return value;
|
|
832
|
-
throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
|
|
833
|
-
}
|
|
834
|
-
function parseHeartbeat(value) {
|
|
835
|
-
if (!value) return 6e4;
|
|
836
|
-
const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
|
|
837
|
-
if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
|
|
838
|
-
const unit = {
|
|
839
|
-
ms: 1,
|
|
840
|
-
s: 1e3,
|
|
841
|
-
m: 6e4
|
|
842
|
-
}[match[2] ?? "s"] ?? 1e3;
|
|
843
|
-
return Number(match[1]) * unit;
|
|
844
|
-
}
|
|
845
|
-
function resolveOutput(input) {
|
|
846
|
-
const { flags, env } = input;
|
|
847
|
-
const ci = isCI(env);
|
|
848
|
-
const githubActions = set(env, "GITHUB_ACTIONS");
|
|
849
|
-
const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
|
|
850
|
-
"live",
|
|
851
|
-
"plain",
|
|
852
|
-
"agent"
|
|
853
|
-
]);
|
|
854
|
-
const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
|
|
855
|
-
const mode = requested === "live" ? "plain" : requested;
|
|
856
|
-
const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
|
|
857
|
-
const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
|
|
858
|
-
const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
|
|
859
|
-
const logPrefix = prefix && prefix !== "auto" ? prefix : githubActions && logOrder === "grouped" ? "none" : "task";
|
|
860
|
-
const logs = pick("--output-logs", flags.outputLogs ?? env.REEVAL_OUTPUT_LOGS, [
|
|
861
|
-
"full",
|
|
862
|
-
"errors-only",
|
|
863
|
-
"none"
|
|
864
|
-
]);
|
|
865
|
-
return {
|
|
866
|
-
requested,
|
|
867
|
-
mode,
|
|
868
|
-
logOrder,
|
|
869
|
-
logPrefix,
|
|
870
|
-
outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
|
|
871
|
-
color: env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions,
|
|
872
|
-
githubActions,
|
|
873
|
-
heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
|
|
874
|
-
};
|
|
875
|
-
}
|
|
876
|
-
//#endregion
|
|
877
789
|
//#region src/ui/index.ts
|
|
878
790
|
function createRenderer(options) {
|
|
879
791
|
const out = { write: (text) => options.out.write(redactSecrets(text)) };
|
|
@@ -924,13 +836,13 @@ const invocation = {
|
|
|
924
836
|
};
|
|
925
837
|
/** Set by the first Ctrl-C or SIGTERM of `reeval run`. */
|
|
926
838
|
let interrupted = false;
|
|
927
|
-
/** The last signal that
|
|
839
|
+
/** The last signal that canceled the run: 130 after SIGINT, 143 after SIGTERM. */
|
|
928
840
|
let stopCode = EXIT.interrupted;
|
|
929
841
|
/**
|
|
930
|
-
* Set
|
|
931
|
-
*
|
|
842
|
+
* Set once the attempts and questions are done and whether the run stopped is decided, as
|
|
843
|
+
* live.json turns `finishing`: a signal then has nothing left to cancel.
|
|
932
844
|
*/
|
|
933
|
-
let
|
|
845
|
+
let finishing = false;
|
|
934
846
|
async function run(argv) {
|
|
935
847
|
const { values, positionals } = parseCli(argv);
|
|
936
848
|
invocation.flags = values;
|
|
@@ -948,7 +860,7 @@ async function run(argv) {
|
|
|
948
860
|
return values.help ? EXIT.ok : EXIT.usage;
|
|
949
861
|
}
|
|
950
862
|
if (command === "init") {
|
|
951
|
-
const { initCommand } = await import("./init-
|
|
863
|
+
const { initCommand } = await import("./init-BDEOopK7.js");
|
|
952
864
|
return initCommand(values, filters, {
|
|
953
865
|
out: (s) => console.log(s),
|
|
954
866
|
input: process.stdin,
|
|
@@ -957,7 +869,7 @@ async function run(argv) {
|
|
|
957
869
|
});
|
|
958
870
|
}
|
|
959
871
|
if (command === "doctor") {
|
|
960
|
-
const { doctorCommand } = await import("./doctor-
|
|
872
|
+
const { doctorCommand } = await import("./doctor-BuQnfz4e.js");
|
|
961
873
|
return doctorCommand(values, {
|
|
962
874
|
out: (s) => console.log(s),
|
|
963
875
|
color: !!process.stdout.isTTY && !process.env.NO_COLOR,
|
|
@@ -965,7 +877,7 @@ async function run(argv) {
|
|
|
965
877
|
});
|
|
966
878
|
}
|
|
967
879
|
if (command === "report") {
|
|
968
|
-
const { reportCommand } = await import("./command-
|
|
880
|
+
const { reportCommand } = await import("./command-4pJYVCT4.js");
|
|
969
881
|
return reportCommand(values, filters, {
|
|
970
882
|
out: (s) => console.log(s),
|
|
971
883
|
err: (s) => console.error(dim(s)),
|
|
@@ -980,7 +892,7 @@ async function run(argv) {
|
|
|
980
892
|
}
|
|
981
893
|
if (command === "studio") {
|
|
982
894
|
const config = await loadConfig(configPath, "studio");
|
|
983
|
-
const { studioCommand } = await import("./command-
|
|
895
|
+
const { studioCommand } = await import("./command-DbSMcxZL.js");
|
|
984
896
|
return studioCommand({
|
|
985
897
|
config,
|
|
986
898
|
flags: values,
|
|
@@ -991,7 +903,7 @@ async function run(argv) {
|
|
|
991
903
|
if (command === "mcp") {
|
|
992
904
|
if (!values.http) console.log = console.info = console.debug = console.error;
|
|
993
905
|
const config = await loadConfig(configPath, "mcp");
|
|
994
|
-
const { mcpCommand } = await import("./command-
|
|
906
|
+
const { mcpCommand } = await import("./command-BqBxLgTc.js");
|
|
995
907
|
return mcpCommand({
|
|
996
908
|
config,
|
|
997
909
|
flags: values,
|
|
@@ -1000,7 +912,7 @@ async function run(argv) {
|
|
|
1000
912
|
}
|
|
1001
913
|
if (command in HISTORY_COMMANDS) {
|
|
1002
914
|
const config = await loadConfig(configPath, command);
|
|
1003
|
-
const commands = await import("./commands-
|
|
915
|
+
const commands = await import("./commands-Bi2SEQtr.js");
|
|
1004
916
|
const ctx = {
|
|
1005
917
|
config,
|
|
1006
918
|
flags: values,
|
|
@@ -1013,7 +925,7 @@ async function run(argv) {
|
|
|
1013
925
|
case "compare": return commands.compareCommand(ctx, filters);
|
|
1014
926
|
case "checkout": return commands.checkoutCommand(ctx, filters);
|
|
1015
927
|
case "query": {
|
|
1016
|
-
const { queryCommand } = await import("./query-
|
|
928
|
+
const { queryCommand } = await import("./query-Clm2mNSm.js");
|
|
1017
929
|
return queryCommand({
|
|
1018
930
|
...ctx,
|
|
1019
931
|
err: (s) => console.error(dim(s))
|
|
@@ -1021,7 +933,7 @@ async function run(argv) {
|
|
|
1021
933
|
}
|
|
1022
934
|
case "analyze": {
|
|
1023
935
|
loadEnvFiles(configPath);
|
|
1024
|
-
const { analyzeCommand } = await import("./command-
|
|
936
|
+
const { analyzeCommand } = await import("./command-9xm5ImqI.js");
|
|
1025
937
|
return analyzeCommand({
|
|
1026
938
|
...ctx,
|
|
1027
939
|
err: (s) => console.error(dim(s))
|
|
@@ -1033,6 +945,16 @@ async function run(argv) {
|
|
|
1033
945
|
if (command !== "run") throw new UsageError(`Unknown command "${command}". Run reeval --help.`);
|
|
1034
946
|
return runCommand(values, configPath, filters);
|
|
1035
947
|
}
|
|
948
|
+
/** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
|
|
949
|
+
async function dryRunPrices(planned, judgeModel) {
|
|
950
|
+
const { needsPrices } = await import("./cost-C5RBTTJr.js");
|
|
951
|
+
if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
|
|
952
|
+
table: null,
|
|
953
|
+
why: "nothing to price"
|
|
954
|
+
};
|
|
955
|
+
const { loadPrices } = await import("./prices-QaS-kjSQ.js");
|
|
956
|
+
return loadPrices();
|
|
957
|
+
}
|
|
1036
958
|
/** Asks a yes/no question on the terminal; anything but y or yes is no. */
|
|
1037
959
|
async function confirm(question) {
|
|
1038
960
|
const { createInterface } = await import("node:readline/promises");
|
|
@@ -1055,13 +977,13 @@ async function cleanCommand(flags, configPath) {
|
|
|
1055
977
|
if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
|
|
1056
978
|
}
|
|
1057
979
|
const { debugRoot, pruneDebug } = await import("./files-BAt0i00X.js");
|
|
1058
|
-
const { formatBytes } = await import("./debug-
|
|
980
|
+
const { formatBytes } = await import("./debug-DeM5pswK.js");
|
|
1059
981
|
const { runs, bytes } = await pruneDebug(config.root, keep);
|
|
1060
982
|
const where = fromCwd(debugRoot(config.root)) || ".";
|
|
1061
983
|
console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
|
|
1062
984
|
if (flags.images) {
|
|
1063
|
-
const { dockerAvailable, pruneImages } = await import("./src-
|
|
1064
|
-
const { requireDocker } = await import("./providers-
|
|
985
|
+
const { dockerAvailable, pruneImages } = await import("./src-DH-9IO7f.js");
|
|
986
|
+
const { requireDocker } = await import("./providers-DkOupEBl.js");
|
|
1065
987
|
const engines = [];
|
|
1066
988
|
for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
|
|
1067
989
|
if (!engines.length) await requireDocker();
|
|
@@ -1082,7 +1004,7 @@ function verbosityOf(flags) {
|
|
|
1082
1004
|
* does. Needs a config, as `reeval run` does: there is no default image.
|
|
1083
1005
|
*/
|
|
1084
1006
|
async function imageBuildCommand(flags, configPath, filters) {
|
|
1085
|
-
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-
|
|
1007
|
+
const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-4mwhNkvh.js");
|
|
1086
1008
|
const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
|
|
1087
1009
|
...flags,
|
|
1088
1010
|
"dry-run": true
|
|
@@ -1142,7 +1064,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1142
1064
|
if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
|
|
1143
1065
|
const stepStart = process.env.GITHUB_STEP_SUMMARY;
|
|
1144
1066
|
if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
|
|
1145
|
-
const { engineVersions } = await import("./node-
|
|
1067
|
+
const { engineVersions } = await import("./node-BVbbnFtv.js");
|
|
1146
1068
|
const renderer = createRenderer({
|
|
1147
1069
|
out: setup.json ? setup.dryRun ? { write: () => true } : process.stderr : process.stdout,
|
|
1148
1070
|
output: setup.output,
|
|
@@ -1160,11 +1082,11 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1160
1082
|
});
|
|
1161
1083
|
throw error;
|
|
1162
1084
|
}
|
|
1163
|
-
const { engineRun, plan, sandboxVersions, budgetReached, live, questions } = executed;
|
|
1085
|
+
const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
|
|
1164
1086
|
invocation.unhandledErrors = engineRun.unhandledErrors;
|
|
1165
1087
|
const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
|
|
1166
|
-
if ((interrupted || engineRun.
|
|
1167
|
-
const { removeRunContainers } = await import("./docker-
|
|
1088
|
+
if ((interrupted || engineRun.canceled) && engines.length) {
|
|
1089
|
+
const { removeRunContainers } = await import("./docker-YFj4H1Cr.js");
|
|
1168
1090
|
await removeRunContainers(ctx.runId, engines);
|
|
1169
1091
|
}
|
|
1170
1092
|
if (engineRun.stopped || !plan) {
|
|
@@ -1172,10 +1094,17 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1172
1094
|
renderer.close();
|
|
1173
1095
|
return EXIT.ok;
|
|
1174
1096
|
}
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1097
|
+
let stopped = cutShort || engineRun.canceled;
|
|
1098
|
+
if (stopped) questions?.cancel();
|
|
1099
|
+
let answers = stopped ? await questions?.answered() : await questions?.settle();
|
|
1100
|
+
if (!stopped && interrupted && questions) {
|
|
1101
|
+
stopped = true;
|
|
1102
|
+
answers = await questions?.answered();
|
|
1103
|
+
}
|
|
1104
|
+
finishing = true;
|
|
1105
|
+
live?.phase("finishing");
|
|
1106
|
+
const { lockfiles } = await import("./state-B5PLm5qY.js");
|
|
1107
|
+
const { writeScratchRun } = await import("./results-BC3pcG_Z.js");
|
|
1179
1108
|
const run = await writeScratchRun({
|
|
1180
1109
|
ctx,
|
|
1181
1110
|
author: ctx.author,
|
|
@@ -1196,60 +1125,11 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1196
1125
|
previous: ctx.previous,
|
|
1197
1126
|
previousArtifacts: ctx.previousArtifacts,
|
|
1198
1127
|
runner: runnerInfo(),
|
|
1199
|
-
...answers
|
|
1128
|
+
...answers && (answers.evals.size || answers.attempts.size) ? { answers } : {}
|
|
1200
1129
|
}, () => live?.stop());
|
|
1201
1130
|
const { record } = run;
|
|
1202
1131
|
const outcome = runOutcome(record);
|
|
1203
1132
|
const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
|
|
1204
|
-
let insights = run.insights;
|
|
1205
|
-
let insightLines = [];
|
|
1206
|
-
const { runInsights, wantsInsights } = await import("./run-Bel7Vde4.js");
|
|
1207
|
-
const wanted = !stopped && wantsInsights(ctx.config, !flags["no-insights"]);
|
|
1208
|
-
if (wanted && interrupted) insightLines = [`insights skipped (${stopCode === EXIT.terminated ? "SIGTERM" : "Ctrl-C"}): reeval analyze ${record.runId} --insights to write them`];
|
|
1209
|
-
else if (wanted) {
|
|
1210
|
-
const { scratchDir } = await import("./files-CtjEXWXm.js");
|
|
1211
|
-
const stop = new AbortController();
|
|
1212
|
-
let stopSignal = "SIGINT";
|
|
1213
|
-
onInsightsSignal = (signal) => {
|
|
1214
|
-
if (stop.signal.aborted) {
|
|
1215
|
-
renderer.close();
|
|
1216
|
-
process.exit(code);
|
|
1217
|
-
}
|
|
1218
|
-
stopSignal = signal;
|
|
1219
|
-
stop.abort(/* @__PURE__ */ new Error(signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"));
|
|
1220
|
-
};
|
|
1221
|
-
try {
|
|
1222
|
-
const done = await runInsights({
|
|
1223
|
-
config: ctx.config,
|
|
1224
|
-
record,
|
|
1225
|
-
runDir: run.artifactDir ?? join(scratchDir(ctx.config.root), record.runId),
|
|
1226
|
-
remarks: run.insights,
|
|
1227
|
-
verbose: (ctx.verbosity ?? 0) > 0,
|
|
1228
|
-
signal: stop.signal,
|
|
1229
|
-
stopSignal: () => stopSignal,
|
|
1230
|
-
onStart: (text) => {
|
|
1231
|
-
renderer.onEvent({
|
|
1232
|
-
type: "notice",
|
|
1233
|
-
at: Date.now(),
|
|
1234
|
-
level: "info",
|
|
1235
|
-
text
|
|
1236
|
-
});
|
|
1237
|
-
},
|
|
1238
|
-
onProgress: (n, total) => {
|
|
1239
|
-
renderer.onEvent({
|
|
1240
|
-
type: "insights-progress",
|
|
1241
|
-
at: Date.now(),
|
|
1242
|
-
done: n,
|
|
1243
|
-
total
|
|
1244
|
-
});
|
|
1245
|
-
}
|
|
1246
|
-
});
|
|
1247
|
-
insights = done.insights ?? insights;
|
|
1248
|
-
insightLines = done.lines;
|
|
1249
|
-
} finally {
|
|
1250
|
-
onInsightsSignal = null;
|
|
1251
|
-
}
|
|
1252
|
-
}
|
|
1253
1133
|
const { pruneDebug } = await import("./files-BAt0i00X.js");
|
|
1254
1134
|
await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
|
|
1255
1135
|
const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
|
|
@@ -1259,9 +1139,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1259
1139
|
record,
|
|
1260
1140
|
summaryPath: fromCwd(summary),
|
|
1261
1141
|
results: engineRun.evals,
|
|
1262
|
-
|
|
1263
|
-
insights,
|
|
1264
|
-
insightLines
|
|
1142
|
+
remarks: run.remarks
|
|
1265
1143
|
});
|
|
1266
1144
|
renderer.close();
|
|
1267
1145
|
const stepSummary = process.env.GITHUB_STEP_SUMMARY;
|
|
@@ -1269,7 +1147,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1269
1147
|
const { config } = ctx;
|
|
1270
1148
|
const mode = setup.record;
|
|
1271
1149
|
if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
|
|
1272
|
-
const { record: recordCmd } = await import("./commands-
|
|
1150
|
+
const { record: recordCmd } = await import("./commands-Bi2SEQtr.js");
|
|
1273
1151
|
const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
|
|
1274
1152
|
const line = await recordCmd(config, record.runId, {
|
|
1275
1153
|
...flags,
|
|
@@ -1279,7 +1157,7 @@ async function runCommand(flags, configPath, filters) {
|
|
|
1279
1157
|
else console.log(line);
|
|
1280
1158
|
}
|
|
1281
1159
|
if (setup.json) {
|
|
1282
|
-
const { suiteResult } = await import("./suite-
|
|
1160
|
+
const { suiteResult } = await import("./suite-22Kt9Grx.js");
|
|
1283
1161
|
process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
|
|
1284
1162
|
}
|
|
1285
1163
|
return code;
|
|
@@ -1312,7 +1190,7 @@ async function prepareRun(flags, configPath) {
|
|
|
1312
1190
|
const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
|
|
1313
1191
|
const container = sandboxes.find((s) => s.kind === "docker");
|
|
1314
1192
|
if (!container) return hostCapacity();
|
|
1315
|
-
const { dockerCapacity } = await import("./src-
|
|
1193
|
+
const { dockerCapacity } = await import("./src-DH-9IO7f.js");
|
|
1316
1194
|
return await dockerCapacity(container.engine) ?? hostCapacity();
|
|
1317
1195
|
});
|
|
1318
1196
|
return {
|
|
@@ -1327,8 +1205,6 @@ async function prepareRun(flags, configPath) {
|
|
|
1327
1205
|
dryRun: !!flags["dry-run"],
|
|
1328
1206
|
json: !!flags.json,
|
|
1329
1207
|
force: !!flags.force,
|
|
1330
|
-
/** `--no-insights` turns them off for this run. */
|
|
1331
|
-
insights: !flags["no-insights"],
|
|
1332
1208
|
verbosity: verbosityOf(flags),
|
|
1333
1209
|
configFile: configLabel(path),
|
|
1334
1210
|
/** Named in the plan line when the project root isn't the current directory. */
|
|
@@ -1341,11 +1217,11 @@ async function prepareRun(flags, configPath) {
|
|
|
1341
1217
|
async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
|
|
1342
1218
|
const startedAt = /* @__PURE__ */ new Date();
|
|
1343
1219
|
const git = await gitInfo(config.root, [config.history.dir]);
|
|
1344
|
-
const { worktreePatch } = await import("./state-
|
|
1220
|
+
const { worktreePatch } = await import("./state-B5PLm5qY.js");
|
|
1345
1221
|
const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
|
|
1346
1222
|
const who = await author(config.root);
|
|
1347
|
-
const { stagingDir, sweepStaging } = await import("./results-
|
|
1348
|
-
const { scratchDir } = await import("./files-
|
|
1223
|
+
const { stagingDir, sweepStaging } = await import("./results-BC3pcG_Z.js");
|
|
1224
|
+
const { scratchDir } = await import("./files-DmUrRmmL.js");
|
|
1349
1225
|
const scratch = scratchDir(config.root);
|
|
1350
1226
|
if (resume !== void 0) {
|
|
1351
1227
|
const target = await resolveResume(config.root, resume || void 0);
|
|
@@ -1356,7 +1232,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
|
|
|
1356
1232
|
artifacts: target.previousArtifacts
|
|
1357
1233
|
} : await loadPrevious(stagingDir(config.root, target.runId));
|
|
1358
1234
|
if (!dryRun) try {
|
|
1359
|
-
const { removeRunContainers } = await import("./docker-
|
|
1235
|
+
const { removeRunContainers } = await import("./docker-YFj4H1Cr.js");
|
|
1360
1236
|
await removeRunContainers(target.runId, ["docker", "podman"]);
|
|
1361
1237
|
} catch {}
|
|
1362
1238
|
return {
|
|
@@ -1427,8 +1303,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1427
1303
|
configFile,
|
|
1428
1304
|
completed: []
|
|
1429
1305
|
}));
|
|
1430
|
-
const { runEngine } = await import("./node-
|
|
1431
|
-
const { buildPlan, checkAttemptBudget, planDocument,
|
|
1306
|
+
const { runEngine } = await import("./node-BVbbnFtv.js");
|
|
1307
|
+
const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-DdWkM6SW.js");
|
|
1432
1308
|
let plan;
|
|
1433
1309
|
let sandboxVersions = { images: [] };
|
|
1434
1310
|
let preflightStop;
|
|
@@ -1436,10 +1312,13 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1436
1312
|
let budgetTimer;
|
|
1437
1313
|
let toRun = [];
|
|
1438
1314
|
let live;
|
|
1315
|
+
let kept = /* @__PURE__ */ new Map();
|
|
1439
1316
|
let questions;
|
|
1440
|
-
const onPlan = async (
|
|
1317
|
+
const onPlan = async (collected) => {
|
|
1318
|
+
const { planned, excluded, matched, unmatched } = collected;
|
|
1441
1319
|
if (unmatched.length) throw unmatchedFilter(config, unmatched);
|
|
1442
1320
|
if (matched === 0) throw noMatch(config, filters, tags);
|
|
1321
|
+
checkLimitHarness(profile, config.file, collected);
|
|
1443
1322
|
const completed = completedKeysFor(planned, stagingDir, ctx.previous);
|
|
1444
1323
|
const skipped = planned.map((p) => completed.has(p.key) ? {
|
|
1445
1324
|
...p,
|
|
@@ -1454,7 +1333,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1454
1333
|
} : p);
|
|
1455
1334
|
toRun = skipped;
|
|
1456
1335
|
for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
|
|
1457
|
-
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-
|
|
1336
|
+
const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-g3XnzUv0.js") : void 0;
|
|
1337
|
+
const judge = runJudgeModel(config, profile);
|
|
1338
|
+
const judgeModel = judge === void 0 ? null : judgeModelName(judge);
|
|
1458
1339
|
plan = await buildPlan({
|
|
1459
1340
|
planned: skipped,
|
|
1460
1341
|
profile,
|
|
@@ -1462,21 +1343,21 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1462
1343
|
concurrency,
|
|
1463
1344
|
root: config.root,
|
|
1464
1345
|
historyDir: config.history.dir,
|
|
1465
|
-
questionModel: models?.questionModelLabel(config) ?? null,
|
|
1466
|
-
|
|
1346
|
+
questionModel: models?.questionModelLabel(config, judge) ?? null,
|
|
1347
|
+
judgeModel,
|
|
1348
|
+
...setup.dryRun ? { prices: await dryRunPrices(skipped, judgeModel) } : {}
|
|
1467
1349
|
});
|
|
1468
1350
|
plan.runId = runId;
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
if (problem) plan.warnings.push(`insights will be skipped: ${problem}`);
|
|
1473
|
-
}
|
|
1351
|
+
const cloud = ciRunner(process.env);
|
|
1352
|
+
const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
|
|
1353
|
+
if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
|
|
1474
1354
|
if (completed.size) plan.resume = {
|
|
1475
1355
|
runId,
|
|
1476
1356
|
completed: completed.size
|
|
1477
1357
|
};
|
|
1478
1358
|
run.completed = [...completed];
|
|
1479
|
-
|
|
1359
|
+
kept = keptAttempts(completed, stagingDir, ctx.previous);
|
|
1360
|
+
const { imageStages, imageTargets } = await import("./images-4mwhNkvh.js");
|
|
1480
1361
|
const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
|
|
1481
1362
|
id: p.id,
|
|
1482
1363
|
harness: p.attempt.harness,
|
|
@@ -1498,8 +1379,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1498
1379
|
if (setup.dryRun) {
|
|
1499
1380
|
const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
|
|
1500
1381
|
lines.push(...planTable(plan, todos));
|
|
1501
|
-
const { imageStatusLines } = await import("./images-
|
|
1502
|
-
const { dockerAvailable, engineSpec } = await import("./src-
|
|
1382
|
+
const { imageStatusLines } = await import("./images-4mwhNkvh.js");
|
|
1383
|
+
const { dockerAvailable, engineSpec } = await import("./src-DH-9IO7f.js");
|
|
1503
1384
|
const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
|
|
1504
1385
|
const down = [];
|
|
1505
1386
|
for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
|
|
@@ -1508,10 +1389,9 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1508
1389
|
const names = down.map((e) => engineSpec(e).name).join(" and ");
|
|
1509
1390
|
lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
|
|
1510
1391
|
}
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
lines.push(`insights ${plan.insights.model} · up to ${formatTokens(plan.insights.maxTokens)} tokens`);
|
|
1392
|
+
const { costLine } = await import("./cost-C5RBTTJr.js");
|
|
1393
|
+
const cost = plan.cost && costLine(plan.cost);
|
|
1394
|
+
if (cost) lines.push(cost);
|
|
1515
1395
|
}
|
|
1516
1396
|
lines.push(...plan.warnings.map((w) => `warning: ${w}`));
|
|
1517
1397
|
lines.push(...plan.hints.map((h) => `hint: ${h}`));
|
|
@@ -1525,38 +1405,64 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1525
1405
|
checkAttemptBudget(plan, setup.force, config.file);
|
|
1526
1406
|
const questionModels = /* @__PURE__ */ new Set();
|
|
1527
1407
|
const asked = [];
|
|
1408
|
+
const scope = gatewayScope(config, profile);
|
|
1528
1409
|
for (const row of plan.rows) {
|
|
1529
1410
|
if (row.plan.skip || !models) continue;
|
|
1530
|
-
const questions = (row.plan.questions ?? []).map((q) =>
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1411
|
+
const questions = (row.plan.questions ?? []).map((q) => {
|
|
1412
|
+
const model = models.questionModel(config, q, row.id, judge);
|
|
1413
|
+
const gateway = resolveModelGateway(model, void 0, scope, "the question model");
|
|
1414
|
+
return {
|
|
1415
|
+
n: q.n,
|
|
1416
|
+
question: q.question,
|
|
1417
|
+
model,
|
|
1418
|
+
...gateway ? { gateway } : {},
|
|
1419
|
+
...gateway === "openrouter" && openRouterFallbacks(scope.gateways) ? { fallbacks: true } : {},
|
|
1420
|
+
...gateway === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {},
|
|
1421
|
+
...q.across !== void 0 ? { across: q.across } : {},
|
|
1422
|
+
...q.choices ? { choices: q.choices } : {},
|
|
1423
|
+
...q.state ? { state: q.state } : {},
|
|
1424
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
1425
|
+
};
|
|
1426
|
+
});
|
|
1537
1427
|
for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
|
|
1538
1428
|
if (questions.length) asked.push({
|
|
1539
1429
|
row,
|
|
1540
1430
|
questions
|
|
1541
1431
|
});
|
|
1542
1432
|
}
|
|
1433
|
+
const judges = /* @__PURE__ */ new Map();
|
|
1434
|
+
for (const p of skipped) {
|
|
1435
|
+
const j = p.attempt.judge;
|
|
1436
|
+
if (j && !p.attempt.plan.skip) judges.set(`${j.model}\0${j.gateway ?? ""}`, j);
|
|
1437
|
+
}
|
|
1543
1438
|
const missing = missingCredentials(config, process.env, {
|
|
1544
1439
|
harnesses: plan.harnessDescriptors,
|
|
1545
1440
|
profile,
|
|
1546
|
-
questionModels: [...questionModels]
|
|
1441
|
+
questionModels: [...questionModels],
|
|
1442
|
+
judges: [...judges.values()]
|
|
1547
1443
|
});
|
|
1548
|
-
if (missing.length)
|
|
1444
|
+
if (missing.length) {
|
|
1445
|
+
const hint = missing.some((m) => m.includes("(for direct())")) ? unusedGatewayKey(process.env) : "";
|
|
1446
|
+
throw new UsageError(`Missing credentials: ${missing.join("; ")}. Set them in the environment or .env.local.${hint}`);
|
|
1447
|
+
}
|
|
1549
1448
|
const malformed = malformedCredentials(config, process.env, {
|
|
1550
1449
|
harnesses: plan.harnessDescriptors,
|
|
1551
1450
|
profile,
|
|
1552
|
-
questionModels: [...questionModels]
|
|
1451
|
+
questionModels: [...questionModels],
|
|
1452
|
+
judges: [...judges.values()]
|
|
1553
1453
|
});
|
|
1554
1454
|
if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
1555
1455
|
if (asked.length) {
|
|
1556
|
-
const { questionRunner } = await import("./questions-
|
|
1456
|
+
const { questionRunner } = await import("./questions-BRhU8MzB.js");
|
|
1457
|
+
const warned = /* @__PURE__ */ new Set();
|
|
1557
1458
|
questions = questionRunner({
|
|
1558
1459
|
evals: asked,
|
|
1559
1460
|
stagingDir,
|
|
1461
|
+
resumed: {
|
|
1462
|
+
completed: new Set(run.completed),
|
|
1463
|
+
previous: ctx.previous,
|
|
1464
|
+
artifacts: ctx.previousArtifacts
|
|
1465
|
+
},
|
|
1560
1466
|
maxInputTokens: config.analysis.maxInputTokens,
|
|
1561
1467
|
maxOutputTokens: config.analysis.maxOutputTokens,
|
|
1562
1468
|
onAnswer: (id, answer) => {
|
|
@@ -1565,12 +1471,31 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1565
1471
|
type: "notice",
|
|
1566
1472
|
at: Date.now(),
|
|
1567
1473
|
level: "warn",
|
|
1568
|
-
text: `
|
|
1474
|
+
text: `question "${clip(answer.question, 80)}" on ${id} failed: ${answer.error}. Outcomes are unchanged.`
|
|
1475
|
+
});
|
|
1476
|
+
},
|
|
1477
|
+
onAttemptAnswer: (id, attempt, answer) => {
|
|
1478
|
+
const once = `${id}\0${answer.question}`;
|
|
1479
|
+
if (answer.error === void 0 || warned.has(once)) return;
|
|
1480
|
+
warned.add(once);
|
|
1481
|
+
renderer.onEvent({
|
|
1482
|
+
type: "notice",
|
|
1483
|
+
at: Date.now(),
|
|
1484
|
+
level: "warn",
|
|
1485
|
+
text: `question "${clip(answer.question, 80)}" on ${attempt} failed: ${answer.error}. Outcomes are unchanged; later failures of it on ${id} aren't shown.`
|
|
1486
|
+
});
|
|
1487
|
+
},
|
|
1488
|
+
onUnshown: (id, error) => {
|
|
1489
|
+
renderer.onEvent({
|
|
1490
|
+
type: "notice",
|
|
1491
|
+
at: Date.now(),
|
|
1492
|
+
level: "warn",
|
|
1493
|
+
text: `${error} on ${id}, so each attempt's answer is that error. Outcomes are unchanged.`
|
|
1569
1494
|
});
|
|
1570
1495
|
}
|
|
1571
1496
|
});
|
|
1572
1497
|
}
|
|
1573
|
-
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-
|
|
1498
|
+
const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-DDHzkMjm.js");
|
|
1574
1499
|
const live = skipped.filter((p) => !p.attempt.plan.skip);
|
|
1575
1500
|
const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
|
|
1576
1501
|
workspace: a.workspace,
|
|
@@ -1661,6 +1586,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1661
1586
|
budgetTimer.unref();
|
|
1662
1587
|
};
|
|
1663
1588
|
let engineRun;
|
|
1589
|
+
let cutShort = false;
|
|
1664
1590
|
try {
|
|
1665
1591
|
engineRun = await runEngine({
|
|
1666
1592
|
run,
|
|
@@ -1672,7 +1598,8 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1672
1598
|
dir: stagingDir,
|
|
1673
1599
|
runId,
|
|
1674
1600
|
startedAt: ctx.startedAt,
|
|
1675
|
-
planned: toRun
|
|
1601
|
+
planned: toRun,
|
|
1602
|
+
kept
|
|
1676
1603
|
});
|
|
1677
1604
|
renderer.onEvent({
|
|
1678
1605
|
type: "run-start",
|
|
@@ -1697,22 +1624,22 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1697
1624
|
if (interrupted && e.type === "eval-end" && e.result.state === "skipped") return;
|
|
1698
1625
|
renderer.onEvent(e);
|
|
1699
1626
|
},
|
|
1700
|
-
|
|
1627
|
+
onCancelable: (cancel) => {
|
|
1701
1628
|
const onSignal = (signal) => {
|
|
1702
|
-
if (onInsightsSignal) return onInsightsSignal(signal === "SIGTERM" ? "SIGTERM" : "SIGINT");
|
|
1703
1629
|
stopCode = signal === "SIGTERM" ? EXIT.terminated : EXIT.interrupted;
|
|
1704
1630
|
if (interrupted) {
|
|
1705
1631
|
renderer.close();
|
|
1706
1632
|
process.exit(stopCode);
|
|
1707
1633
|
}
|
|
1708
1634
|
interrupted = true;
|
|
1635
|
+
if (finishing) return;
|
|
1709
1636
|
questions?.cancel();
|
|
1710
1637
|
live?.phase("cancelling");
|
|
1711
1638
|
renderer.onEvent({
|
|
1712
1639
|
type: "notice",
|
|
1713
1640
|
at: Date.now(),
|
|
1714
1641
|
level: "warn",
|
|
1715
|
-
text: "
|
|
1642
|
+
text: "Canceling: stopping agents and destroying sandboxes (Ctrl-C again to force)"
|
|
1716
1643
|
});
|
|
1717
1644
|
cancel();
|
|
1718
1645
|
};
|
|
@@ -1720,15 +1647,15 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1720
1647
|
process.on("SIGTERM", onSignal);
|
|
1721
1648
|
}
|
|
1722
1649
|
});
|
|
1723
|
-
|
|
1650
|
+
cutShort = interrupted;
|
|
1724
1651
|
} catch (error) {
|
|
1725
1652
|
live?.remove();
|
|
1726
1653
|
throw error;
|
|
1727
1654
|
} finally {
|
|
1728
1655
|
clearTimeout(budgetTimer);
|
|
1729
|
-
const { removePrepared } = await import("./preflight-
|
|
1656
|
+
const { removePrepared } = await import("./preflight-DTiVv0gY.js");
|
|
1730
1657
|
removePrepared(config.root, runId);
|
|
1731
|
-
const { removePreparedWorkspaces } = await import("./workspace-
|
|
1658
|
+
const { removePreparedWorkspaces } = await import("./workspace-DDHzkMjm.js");
|
|
1732
1659
|
removePreparedWorkspaces(config.root, runId);
|
|
1733
1660
|
}
|
|
1734
1661
|
if (preflightStop) throw preflightStop;
|
|
@@ -1740,6 +1667,7 @@ async function executeRun(ctx, setup, filters, renderer) {
|
|
|
1740
1667
|
plan,
|
|
1741
1668
|
sandboxVersions,
|
|
1742
1669
|
budgetReached: () => reached,
|
|
1670
|
+
cutShort,
|
|
1743
1671
|
live,
|
|
1744
1672
|
questions
|
|
1745
1673
|
};
|
|
@@ -1810,4 +1738,4 @@ run(process.argv.slice(2)).finally(() => {
|
|
|
1810
1738
|
//#endregion
|
|
1811
1739
|
export {};
|
|
1812
1740
|
|
|
1813
|
-
//# sourceMappingURL=cli-
|
|
1741
|
+
//# sourceMappingURL=cli-PdYrcoKM.js.map
|