reeval 0.15.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -14
- package/dist/{adapter-CbqbMR7y.js → adapter-CqUt4A0w.js} +3 -3
- package/dist/{adapter-CbqbMR7y.js.map → adapter-CqUt4A0w.js.map} +1 -1
- package/dist/{agent-Zgn-kiqq.js → agent-BtAbRTFq.js} +6 -6
- package/dist/{agent-Zgn-kiqq.js.map → agent-BtAbRTFq.js.map} +1 -1
- package/dist/{app-files-ChbvRx3P.js → app-files-CDttHPOz.js} +3 -3
- package/dist/app-files-CDttHPOz.js.map +1 -0
- package/dist/assemble-Bb7zqpRw.js +1 -0
- package/dist/{assemble-SKQGW_QF.js → assemble-J9rkE29m.js} +114 -31
- package/dist/assemble-J9rkE29m.js.map +1 -0
- package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
- package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{attempt-host-BOp8JklD.js → attempt-host-DDedP7Qv.js} +17 -8
- package/dist/attempt-host-DDedP7Qv.js.map +1 -0
- package/dist/capture-Bvc4FtWT.js +15 -0
- package/dist/capture-Bvc4FtWT.js.map +1 -0
- package/dist/capture-D68TiQQy.js +1 -0
- package/dist/{capture-BM_3100g.js → capture-DheNqeYS.js} +1495 -203
- package/dist/capture-DheNqeYS.js.map +1 -0
- package/dist/{cli-PdYrcoKM.js → cli-KTHjNFoo.js} +329 -74
- package/dist/cli-KTHjNFoo.js.map +1 -0
- package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{colors-QV9dFVhN.js → colors-Y6O3mtl_.js} +20 -2
- package/dist/colors-Y6O3mtl_.js.map +1 -0
- package/dist/{command-9xm5ImqI.js → command-1FBECMs-.js} +16 -16
- package/dist/{command-9xm5ImqI.js.map → command-1FBECMs-.js.map} +1 -1
- package/dist/{command-BqBxLgTc.js → command-Ce03LLtg.js} +14 -14
- package/dist/{command-BqBxLgTc.js.map → command-Ce03LLtg.js.map} +1 -1
- package/dist/{command-4pJYVCT4.js → command-CztRuwUM.js} +12 -12
- package/dist/{command-4pJYVCT4.js.map → command-CztRuwUM.js.map} +1 -1
- package/dist/{command-DbSMcxZL.js → command-nNN0mx65.js} +7 -7
- package/dist/{command-DbSMcxZL.js.map → command-nNN0mx65.js.map} +1 -1
- package/dist/{commands-BU6YJCz9.js → commands-DP13JX9i.js} +24 -12
- package/dist/commands-DP13JX9i.js.map +1 -0
- package/dist/{commands-Bi2SEQtr.js → commands-ZwQUezI1.js} +1 -1
- package/dist/{config-k6m7XbAz.js → config-BjjDryd5.js} +464 -220
- package/dist/config-BjjDryd5.js.map +1 -0
- package/dist/{contrasts-DLZF4dGF.js → contrasts-DSsRZpR8.js} +2 -2
- package/dist/{contrasts-DLZF4dGF.js.map → contrasts-DSsRZpR8.js.map} +1 -1
- package/dist/cursor-catalog-D7bFLEAD.js +143 -0
- package/dist/cursor-catalog-D7bFLEAD.js.map +1 -0
- package/dist/{db-CpsifVln.js → db-B9Ityuit.js} +8 -8
- package/dist/{db-CpsifVln.js.map → db-B9Ityuit.js.map} +1 -1
- package/dist/db-Bk6LWwza.js +2 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/default-effort-DK8gospa.js +91 -0
- package/dist/default-effort-DK8gospa.js.map +1 -0
- package/dist/default-effort-DmXHpqo9.js +1 -0
- package/dist/{docker-YFj4H1Cr.js → docker-BLCNfUie.js} +1 -1
- package/dist/{docker-B4-rx4dA.js → docker-SKpYOlGi.js} +5 -5
- package/dist/{docker-B4-rx4dA.js.map → docker-SKpYOlGi.js.map} +1 -1
- package/dist/{docker-51WvhN_X.js → docker-hwp3EucJ.js} +9 -5
- package/dist/docker-hwp3EucJ.js.map +1 -0
- package/dist/{doctor-BuQnfz4e.js → doctor-CT0Sgjng.js} +16 -16
- package/dist/{doctor-BuQnfz4e.js.map → doctor-CT0Sgjng.js.map} +1 -1
- package/dist/{environment-BOFgzFDg.js → environment-CZdercoj.js} +3 -3
- package/dist/{environment-BOFgzFDg.js.map → environment-CZdercoj.js.map} +1 -1
- package/dist/{errors-UjcLWelF.js → errors-BZVKVKYd.js} +15 -4
- package/dist/errors-BZVKVKYd.js.map +1 -0
- package/dist/{eval-source-Dje3rERU.js → eval-source-nZD7-FSz.js} +9 -8
- package/dist/eval-source-nZD7-FSz.js.map +1 -0
- package/dist/{evaluate-dqXccn4k.d.ts → evaluate-CNi0lZ7n.d.ts} +7 -6
- package/dist/{evaluate-BPP9s3s6.js → evaluate-fhC1PCd_.js} +345 -47
- package/dist/evaluate-fhC1PCd_.js.map +1 -0
- package/dist/{files-Clipx4OW.js → files-BGy05Dlr.js} +4 -4
- package/dist/{files-Clipx4OW.js.map → files-BGy05Dlr.js.map} +1 -1
- package/dist/{files-Bi461Be2.js → files-BIGxpL08.js} +25 -16
- package/dist/{files-Bi461Be2.js.map → files-BIGxpL08.js.map} +1 -1
- package/dist/files-C2RC7K4m.js +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/{files-H24pz0hL.js → files-I1i6Xyh1.js} +4 -4
- package/dist/{files-H24pz0hL.js.map → files-I1i6Xyh1.js.map} +1 -1
- package/dist/files-dJmrfmOM.js +1 -0
- package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-CsVZkNNL.js → git-DR02tHtU.js} +2 -2
- package/dist/{git-CsVZkNNL.js.map → git-DR02tHtU.js.map} +1 -1
- package/dist/{harness-3NVyTSGK.js → harness-ylJLvrbp.js} +39 -5
- package/dist/{harness-3NVyTSGK.js.map → harness-ylJLvrbp.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{images-4mwhNkvh.js → images---X8Wu9d.js} +8 -8
- package/dist/{images-4mwhNkvh.js.map → images---X8Wu9d.js.map} +1 -1
- package/dist/index.d.ts +71 -16
- package/dist/index.js +5 -5
- package/dist/init-BNTXrtMO.js +1 -0
- package/dist/{init-cOk3D44b.js → init-CA95ZnkV.js} +4 -4
- package/dist/{init-cOk3D44b.js.map → init-CA95ZnkV.js.map} +1 -1
- package/dist/{invocations-DW-lTk2f.js → invocations-BdoROAfb.js} +3 -3
- package/dist/{invocations-DW-lTk2f.js.map → invocations-BdoROAfb.js.map} +1 -1
- package/dist/{judge-CKdgEXN7.js → judge-Dp7FPJ1P.js} +3 -3
- package/dist/{judge-CKdgEXN7.js.map → judge-Dp7FPJ1P.js.map} +1 -1
- package/dist/judge.d.ts +2 -2
- package/dist/judge.js +1 -1
- package/dist/{live-XJQ6nqK4.js → live-CBFkdK1K.js} +3 -3
- package/dist/{live-XJQ6nqK4.js.map → live-CBFkdK1K.js.map} +1 -1
- package/dist/{matchers-DT2TaGix.js → matchers-Cv74mJRJ.js} +43 -13
- package/dist/matchers-Cv74mJRJ.js.map +1 -0
- package/dist/{model-Bq3G0OG1.js → model-6dfKFkQd.js} +4 -4
- package/dist/{model-Bq3G0OG1.js.map → model-6dfKFkQd.js.map} +1 -1
- package/dist/{model-CTnyvh7H.js → model-Be7fPvgi.js} +4 -51
- package/dist/model-Be7fPvgi.js.map +1 -0
- package/dist/{model-g3XnzUv0.js → model-CN4ccjNz.js} +1 -1
- package/dist/model-C_5TspZS.js +1 -0
- package/dist/{model-vRiUycWh.js → model-DkJVmVd2.js} +4 -4
- package/dist/{model-vRiUycWh.js.map → model-DkJVmVd2.js.map} +1 -1
- package/dist/{node-BVbbnFtv.js → node-D_Zkwhq2.js} +2 -2
- package/dist/{node-BVbbnFtv.js.map → node-D_Zkwhq2.js.map} +1 -1
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/{opencode-providers-Bz0l7OZI.js → opencode-providers-DPgdqRfg.js} +26 -2
- package/dist/{opencode-providers-Bz0l7OZI.js.map → opencode-providers-DPgdqRfg.js.map} +1 -1
- package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-CR00FWAa.js} +2 -2
- package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-CR00FWAa.js.map} +1 -1
- package/dist/{patch-DKJiC5Sq.js → patch-BJNdOZsf.js} +4 -4
- package/dist/patch-BJNdOZsf.js.map +1 -0
- package/dist/{plan-DdWkM6SW.js → plan-C1UpGZTk.js} +76 -12
- package/dist/plan-C1UpGZTk.js.map +1 -0
- package/dist/{plugins-D-CP6mMd.js → plugins-Cx6ReJWE.js} +24 -3
- package/dist/plugins-Cx6ReJWE.js.map +1 -0
- package/dist/{preflight-9ixuXzsx.js → preflight-BZ3L7P3a.js} +3 -3
- package/dist/{preflight-9ixuXzsx.js.map → preflight-BZ3L7P3a.js.map} +1 -1
- package/dist/{preflight-DTiVv0gY.js → preflight-CUooKgtT.js} +1 -1
- package/dist/{prepare-BfyikQit.js → prepare-BbBoTeDj.js} +7 -7
- package/dist/{prepare-BfyikQit.js.map → prepare-BbBoTeDj.js.map} +1 -1
- package/dist/{prepare-tmBBf4W8.js → prepare-CEZaEwaq.js} +2 -2
- package/dist/{provenance-BizwUn3h.js → provenance-BWvTIlQM.js} +2 -2
- package/dist/{provenance-BizwUn3h.js.map → provenance-BWvTIlQM.js.map} +1 -1
- package/dist/{providers-B6b9Tnyi.js → providers-B3yqakc7.js} +5 -5
- package/dist/{providers-B6b9Tnyi.js.map → providers-B3yqakc7.js.map} +1 -1
- package/dist/{providers-DkOupEBl.js → providers-DdyZW_ru.js} +1 -1
- package/dist/query-BkwQ0W-c.js +1 -0
- package/dist/{query-DyM-VExS.js → query-CZcqpOI9.js} +2 -2
- package/dist/{query-DyM-VExS.js.map → query-CZcqpOI9.js.map} +1 -1
- package/dist/{questions-BRhU8MzB.js → questions-CFU7-crW.js} +20 -19
- package/dist/questions-CFU7-crW.js.map +1 -0
- package/dist/{record-DZBCjuwm.js → record-Bu1Xi6MQ.js} +8 -4
- package/dist/record-Bu1Xi6MQ.js.map +1 -0
- package/dist/{remarks-C7i2LMRb.js → remarks-A3j-SkJ_.js} +42 -8
- package/dist/remarks-A3j-SkJ_.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{remarks-D_3qhBWU.js → remarks-Dj_x9l5n.js} +4 -3
- package/dist/{remarks-D_3qhBWU.js.map → remarks-Dj_x9l5n.js.map} +1 -1
- package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-DFIzcx_6.js → remote-EIFHdMd8.js} +5 -5
- package/dist/{remote-DFIzcx_6.js.map → remote-EIFHdMd8.js.map} +1 -1
- package/dist/{resolve-DWSJpKHX.js → resolve-8H5rz8mi.js} +5 -5
- package/dist/{resolve-DWSJpKHX.js.map → resolve-8H5rz8mi.js.map} +1 -1
- package/dist/resolve-Tm9pX7Hv.js +1 -0
- package/dist/{results-BC3pcG_Z.js → results-E7JdVjq2.js} +9 -9
- package/dist/{results-BC3pcG_Z.js.map → results-E7JdVjq2.js.map} +1 -1
- package/dist/{resume-j-rcIryi.js → resume-Blu4yIcq.js} +4 -4
- package/dist/{resume-j-rcIryi.js.map → resume-Blu4yIcq.js.map} +1 -1
- package/dist/{runtime-d56Gg3J5.js → runtime-CNbsMJQ7.js} +111 -37
- package/dist/runtime-CNbsMJQ7.js.map +1 -0
- package/dist/{sandbox-Cmln11gU.js → sandbox-RButGfIE.js} +2 -2
- package/dist/{sandbox-Cmln11gU.js.map → sandbox-RButGfIE.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-Dqrq-Sia.js +1 -0
- package/dist/{server-BOoH10J8.js → server-XgCYs_MF.js} +35 -11
- package/dist/server-XgCYs_MF.js.map +1 -0
- package/dist/{source-wD04-rRG.js → source-BOkbJCD2.js} +17 -3
- package/dist/{source-wD04-rRG.js.map → source-BOkbJCD2.js.map} +1 -1
- package/dist/source-DooSTtyz.js +1 -0
- package/dist/src-1M0srjji.js +1 -0
- package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
- package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-D91Hd6hT.js → src-BJCKWacl.js} +38 -18
- package/dist/src-BJCKWacl.js.map +1 -0
- package/dist/src-BZlhaDhE.js +2 -0
- package/dist/{src-BO8AYqJ5.js → src-BkhEPVJZ.js} +12 -3
- package/dist/{src-BO8AYqJ5.js.map → src-BkhEPVJZ.js.map} +1 -1
- package/dist/{src-C7upnZSd.js → src-Bp6X4193.js} +1 -1
- package/dist/{src-BYLM3hEz.js → src-C2Japg_I.js} +21 -10
- package/dist/{src-BYLM3hEz.js.map → src-C2Japg_I.js.map} +1 -1
- package/dist/src-C5q3_fRV.js +3 -0
- package/dist/src-C9EHSYyP.js +2 -0
- package/dist/src-CJdpsrXi.js +2 -0
- package/dist/{src-1lpTocbd.js → src-D1ET6oPL.js} +3 -27
- package/dist/{src-1lpTocbd.js.map → src-D1ET6oPL.js.map} +1 -1
- package/dist/{src-Bp4aN3rA.js → src-D4CZjMQ4.js} +1 -1
- package/dist/{src-BYPMM1x7.js → src-GkJ-1Rdy.js} +1 -1
- package/dist/{src-kYeaJJrl.js → src-OOKE-5GA.js} +1 -1
- package/dist/state-BCyNaz13.js +1 -0
- package/dist/{state-DrJ687ut.js → state-D6-tvqtp.js} +8 -8
- package/dist/{state-DrJ687ut.js.map → state-D6-tvqtp.js.map} +1 -1
- package/dist/suite-BiWajSlA.js +2 -0
- package/dist/{suite-Co8cwelb.js → suite-DYI1k2Ie.js} +22 -5
- package/dist/suite-DYI1k2Ie.js.map +1 -0
- package/dist/{summary-DLFHVoux.js → summary-CN2aqAUz.js} +59 -17
- package/dist/summary-CN2aqAUz.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +96 -44
- package/dist/testing.js +4 -4
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
- package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
- package/dist/{types-coMWHaU2.d.ts → types-DDeCnNmp.d.ts} +437 -181
- package/dist/view/index.d.ts +34 -5
- package/dist/view/index.js +8 -7
- package/dist/views-7bNY17Z6.js +2 -0
- package/dist/{views-BFO1BLKu.js → views-Bd-H71_t.js} +5 -5
- package/dist/{views-BFO1BLKu.js.map → views-Bd-H71_t.js.map} +1 -1
- package/dist/{worker-entry-CJtdsKBP.js → worker-entry-BIHJqE_z.js} +34 -13
- package/dist/worker-entry-BIHJqE_z.js.map +1 -0
- package/dist/workspace-CxwHv8mG.js +1 -0
- package/dist/{workspace-BRxjfa8N.js → workspace-Sy5g8zW9.js} +4 -4
- package/dist/{workspace-BRxjfa8N.js.map → workspace-Sy5g8zW9.js.map} +1 -1
- package/package.json +1 -1
- package/skills/eval-rubric/SKILL.md +11 -0
- package/dist/app-files-ChbvRx3P.js.map +0 -1
- package/dist/assemble-SKQGW_QF.js.map +0 -1
- package/dist/assemble-XBWuK3Ed.js +0 -1
- package/dist/attempt-host-BOp8JklD.js.map +0 -1
- package/dist/capture-BM_3100g.js.map +0 -1
- package/dist/capture-DSK8O5bA.js +0 -1
- package/dist/cli-PdYrcoKM.js.map +0 -1
- package/dist/cli-spec-D0qyrPZi.js.map +0 -1
- package/dist/client-Ct6s6OnF.js.map +0 -1
- package/dist/colors-QV9dFVhN.js.map +0 -1
- package/dist/commands-BU6YJCz9.js.map +0 -1
- package/dist/config-k6m7XbAz.js.map +0 -1
- package/dist/db-DkD-F-X-.js +0 -2
- package/dist/debug-BhizgAsU.js.map +0 -1
- package/dist/debug-DeM5pswK.js +0 -1
- package/dist/docker-51WvhN_X.js.map +0 -1
- package/dist/errors-UjcLWelF.js.map +0 -1
- package/dist/eval-source-Dje3rERU.js.map +0 -1
- package/dist/evaluate-BPP9s3s6.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-C2h2Cw4F.js +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-DmUrRmmL.js +0 -1
- package/dist/init-BDEOopK7.js +0 -1
- package/dist/matchers-DT2TaGix.js.map +0 -1
- package/dist/model-CTnyvh7H.js.map +0 -1
- package/dist/model-D-SWrLZG.js +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/pairs-puYi-zj2.js.map +0 -1
- package/dist/patch-DKJiC5Sq.js.map +0 -1
- package/dist/plan-DdWkM6SW.js.map +0 -1
- package/dist/plugins-D-CP6mMd.js.map +0 -1
- package/dist/query-Clm2mNSm.js +0 -1
- package/dist/questions-BRhU8MzB.js.map +0 -1
- package/dist/record-DZBCjuwm.js.map +0 -1
- package/dist/remarks-BwJ8pcTQ.js +0 -1
- package/dist/remarks-C7i2LMRb.js.map +0 -1
- package/dist/remarks-dA8LyoPc.js.map +0 -1
- package/dist/resolve--rQZ8u6Z.js +0 -1
- package/dist/runtime-d56Gg3J5.js.map +0 -1
- package/dist/server-BOoH10J8.js.map +0 -1
- package/dist/server-DpuVAbrO.js +0 -1
- package/dist/source-BFd6f0-R.js +0 -1
- package/dist/src-6_zz7oJs.js +0 -2
- package/dist/src-D91Hd6hT.js.map +0 -1
- package/dist/src-DH-9IO7f.js +0 -1
- package/dist/src-HSxBazcg.js +0 -3
- package/dist/src-LD1BAJFa.js +0 -2
- package/dist/src-dlX-Lmxg.js +0 -2
- package/dist/state-B5PLm5qY.js +0 -1
- package/dist/suite-22Kt9Grx.js +0 -2
- package/dist/suite-Co8cwelb.js.map +0 -1
- package/dist/summary-DLFHVoux.js.map +0 -1
- package/dist/views-BOf8qEL8.js +0 -2
- package/dist/worker-entry-CJtdsKBP.js.map +0 -1
- package/dist/workspace-DDHzkMjm.js +0 -1
|
@@ -1,29 +1,29 @@
|
|
|
1
|
-
import { n as isOneOf, t as isObjectLike } from "./object-
|
|
1
|
+
import { n as isOneOf, t as isObjectLike } from "./object-BkXGPKWY.js";
|
|
2
2
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
3
|
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-B_lvGqWz.js";
|
|
4
|
-
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-
|
|
5
|
-
import { B as gatePolicySchema, D as pluginServerName, L as egressPolicySchema, V as gateProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-
|
|
6
|
-
import { _ as
|
|
4
|
+
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-CZdercoj.js";
|
|
5
|
+
import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-Cx6ReJWE.js";
|
|
6
|
+
import { _ as GATEWAY_NAMES, b as gatewayEntryDataSchema, f as FACTORY, g as harnessProblem, h as harnessDescriptorSchema, u as opencodeModelError, v as directOffPin, x as gatewayNameSchema, y as gatewayDescriptorSchema } from "./harness-ylJLvrbp.js";
|
|
7
7
|
import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
8
|
-
import { a as
|
|
9
|
-
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-
|
|
8
|
+
import { a as providerHosts, i as providerEndpoints, l as resolveOpenCodeProvider, o as providerModelResponses, s as providerServedModel } from "./opencode-providers-DPgdqRfg.js";
|
|
9
|
+
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-Dp7FPJ1P.js";
|
|
10
10
|
import { a as VERTEX_ROUTES, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-DypYjx71.js";
|
|
11
11
|
import { r as badCharacterAt } from "./redact-DVIaahLv.js";
|
|
12
12
|
import { s as withoutBom } from "./text-DTigcA64.js";
|
|
13
13
|
import { t as inCI } from "./env-CARj5Axq.js";
|
|
14
14
|
import { u as ownFileURL } from "./root-Dlff-zp8.js";
|
|
15
|
-
import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS,
|
|
15
|
+
import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, ft as run, pt as shellQuote } from "./src-BJCKWacl.js";
|
|
16
16
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
17
|
-
import {
|
|
17
|
+
import { n as gateLabel, u as pairFactors } from "./gate-BC6SdQZs.js";
|
|
18
18
|
import "./stats-hfpuYsAM.js";
|
|
19
|
-
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-
|
|
20
|
-
import { t as docker } from "./sandbox-
|
|
19
|
+
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CqUt4A0w.js";
|
|
20
|
+
import { t as docker } from "./sandbox-RButGfIE.js";
|
|
21
21
|
import { existsSync, readFileSync } from "node:fs";
|
|
22
22
|
import { availableParallelism, tmpdir, totalmem } from "node:os";
|
|
23
23
|
import { dirname, join, resolve, win32 } from "node:path";
|
|
24
24
|
import { parseDocument } from "yaml";
|
|
25
25
|
import { z } from "zod";
|
|
26
|
-
import { isIP } from "node:net";
|
|
26
|
+
import { connect, isIP } from "node:net";
|
|
27
27
|
import { createHash, randomUUID } from "node:crypto";
|
|
28
28
|
import { spawn } from "node:child_process";
|
|
29
29
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
@@ -84,21 +84,43 @@ function decideState(json, files) {
|
|
|
84
84
|
}
|
|
85
85
|
//#endregion
|
|
86
86
|
//#region src/contract/judge.ts
|
|
87
|
-
|
|
87
|
+
/** The inputs every judge call and question shares; `c.judge()` adds `contents` and its own `artifacts`. */
|
|
88
|
+
const sharedInputFields = {
|
|
88
89
|
task: z.boolean().optional(),
|
|
89
90
|
output: z.boolean().optional(),
|
|
90
91
|
files: z.boolean().optional(),
|
|
91
92
|
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
92
|
-
contents: z.array(nonEmpty()).optional(),
|
|
93
93
|
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
94
|
-
turns: z.enum(["last", "all"]).optional(),
|
|
95
94
|
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
95
|
+
};
|
|
96
|
+
const turnsSchema = z.enum(["last", "all"]);
|
|
97
|
+
const { task, output, files, diff, transcript, checks } = sharedInputFields;
|
|
98
|
+
const judgeInputsSchema = z.strictObject({
|
|
99
|
+
task,
|
|
100
|
+
output,
|
|
101
|
+
files,
|
|
102
|
+
diff,
|
|
103
|
+
contents: z.array(nonEmpty()).optional(),
|
|
104
|
+
transcript,
|
|
105
|
+
checks,
|
|
106
|
+
artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
|
|
96
107
|
});
|
|
108
|
+
const judgeSubjectSchema = z.strictObject({
|
|
109
|
+
on: z.custom().optional(),
|
|
110
|
+
turns: turnsSchema.optional()
|
|
111
|
+
});
|
|
112
|
+
const INPUT_KEYS = Object.keys(judgeInputsSchema.shape);
|
|
113
|
+
/** True when an input's value turns it on: not `false`, unset or an empty list. */
|
|
114
|
+
const inputOn = (value) => value !== void 0 && value !== false && !(Array.isArray(value) && value.length === 0);
|
|
115
|
+
/** The input keys an options object turns on, in `keys`' order. */
|
|
116
|
+
const inputsOn = (value, keys) => keys.filter((k) => inputOn(value[k]));
|
|
117
|
+
/** `a`, `b` and `c` as prose: "a, b or c". */
|
|
118
|
+
const orList = (items) => items.length < 2 ? items[0] ?? "" : `${items.slice(0, -1).join(", ")} or ${items.at(-1)}`;
|
|
119
|
+
const NO_JUDGE_INPUT = `c.judge() shows the judge nothing: turn on at least one input, such as { diff: true } or { contents: ['src/**'] }. The inputs are ${orList(INPUT_KEYS)}.`;
|
|
97
120
|
const judgeFields = {
|
|
98
121
|
threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional(),
|
|
99
122
|
providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional(),
|
|
100
123
|
maxConcurrent: z.number().int().min(1).optional(),
|
|
101
|
-
state: judgeStateSchema.optional(),
|
|
102
124
|
gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional()
|
|
103
125
|
};
|
|
104
126
|
const fakeJudgeDataSchema = z.strictObject({
|
|
@@ -114,18 +136,13 @@ const judgeOptionsSchema = z.strictObject({
|
|
|
114
136
|
model: z.custom().optional(),
|
|
115
137
|
...judgeFields
|
|
116
138
|
});
|
|
117
|
-
const judgeCallSchema = judgeOptionsSchema.extend(
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
shows: "state",
|
|
125
|
-
show: "state",
|
|
126
|
-
input: "state"
|
|
127
|
-
};
|
|
128
|
-
const VALUES = {
|
|
139
|
+
const judgeCallSchema = judgeOptionsSchema.extend(judgeInputsSchema.shape).extend(judgeSubjectSchema.shape);
|
|
140
|
+
const CALL_ONLY_KEYS = [
|
|
141
|
+
...INPUT_KEYS,
|
|
142
|
+
"on",
|
|
143
|
+
"turns"
|
|
144
|
+
];
|
|
145
|
+
const VALUES$1 = {
|
|
129
146
|
task: "true or false",
|
|
130
147
|
output: "true or false",
|
|
131
148
|
files: "true or false",
|
|
@@ -162,7 +179,10 @@ function judgeProblem(judge, site) {
|
|
|
162
179
|
return `${key ? `${base}.${key}` : base} in ${site.at === "config" ? site.file : site.where}`;
|
|
163
180
|
};
|
|
164
181
|
const loc = site.at === "call" ? " in c.judge()" : site.at === "options" ? ` in ${site.where}` : ` in ${site.path === "judge" ? "" : `${site.path.replace(/\.judge$/, "")} in `}${site.file}`;
|
|
165
|
-
if (typeof judge !== "object" || judge === null || Array.isArray(judge))
|
|
182
|
+
if (typeof judge !== "object" || judge === null || Array.isArray(judge)) {
|
|
183
|
+
const example = site.at === "call" ? "{ diff: true }" : "{ threshold: 0.8 }";
|
|
184
|
+
return `${field()} must be an object such as ${example} (got ${shown(judge)})`;
|
|
185
|
+
}
|
|
166
186
|
const value = judge;
|
|
167
187
|
const model = value.model;
|
|
168
188
|
if (site.at === "config" && typeof model === "string") {
|
|
@@ -171,7 +191,7 @@ function judgeProblem(judge, site) {
|
|
|
171
191
|
}
|
|
172
192
|
const schema = site.at === "call" ? judgeCallSchema : site.at === "config" ? configJudgeDataSchema : judgeOptionsSchema;
|
|
173
193
|
const parsed = schema.safeParse(judge);
|
|
174
|
-
if (parsed.success) return void 0;
|
|
194
|
+
if (parsed.success) return site.at === "call" && !inputsOn(value, INPUT_KEYS).length ? NO_JUDGE_INPUT : void 0;
|
|
175
195
|
const numericFake = site.at === "config" && typeof model === "object" && model?.kind === "fake-judge" && typeof model.score === "number";
|
|
176
196
|
const issues = parsed.error.issues.filter((i) => !(numericFake && i.path[0] === "model"));
|
|
177
197
|
if (!issues.length) return void 0;
|
|
@@ -194,29 +214,22 @@ function judgeProblem(judge, site) {
|
|
|
194
214
|
const name = issues.find((i) => i.code === "unrecognized_keys" && i.path.length === 0)?.keys[0];
|
|
195
215
|
if (name === void 0) return void 0;
|
|
196
216
|
const allowed = Object.keys(schema.shape);
|
|
197
|
-
const hint =
|
|
198
|
-
const why = site.at !== "call" && (name
|
|
217
|
+
const hint = nearest(name, allowed);
|
|
218
|
+
const why = site.at !== "call" && CALL_ONLY_KEYS.includes(name) ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
|
|
199
219
|
return `Unknown judge option "${name}"${loc}.${why}`;
|
|
200
220
|
};
|
|
201
|
-
const
|
|
202
|
-
const issue = issues.find((i) => i.path[0] === "
|
|
221
|
+
const inputProblem = () => {
|
|
222
|
+
const issue = issues.find((i) => typeof i.path[0] === "string" && i.path[0] in VALUES$1);
|
|
203
223
|
if (!issue) return void 0;
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
if (issue.code === "unrecognized_keys" && issue.path.length === 1) {
|
|
207
|
-
const key = issue.keys[0] ?? "";
|
|
208
|
-
const hint = nearest(key, STATE_KEYS$1);
|
|
209
|
-
return `Unknown judge.state key "${key}"${loc}.${hint ? ` Did you mean "${hint}"?` : ""}`;
|
|
210
|
-
}
|
|
211
|
-
const key = issue.path[1];
|
|
212
|
-
return `${field(`state.${String(key)}`)} must be ${VALUES[key]} (got ${shown(state[key])})`;
|
|
224
|
+
const key = issue.path[0];
|
|
225
|
+
return `${field(key)} must be ${VALUES$1[key]} (got ${shown(value[key])})`;
|
|
213
226
|
};
|
|
214
227
|
const providerOptionsProblem = () => at("providerOptions") ? `${field("providerOptions")} must map each provider to an object of JSON values, e.g. { google: { thinkingConfig: { thinkingBudget: 0 } } } (got ${shown(value.providerOptions)})` : void 0;
|
|
215
228
|
const artifactsProblem = () => at("artifacts") ? `c.judge() artifacts ${badJudgeArtifacts(value.artifacts)}` : void 0;
|
|
216
229
|
const gatewayProblem = () => at("gateway") ? `${field("gateway")} names a gateway in the config's gateway list: ${GATEWAY_NAMES.map((g) => `'${g}'`).join(" or ")}, or is its helper (got ${shown(value.gateway)})` : void 0;
|
|
217
|
-
if (site.at === "config") order.push(modelProblem, keyProblem,
|
|
218
|
-
else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem,
|
|
219
|
-
else order.push(maxConcurrentProblem, thresholdProblem, keyProblem,
|
|
230
|
+
if (site.at === "config") order.push(modelProblem, keyProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
|
|
231
|
+
else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, inputProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
|
|
232
|
+
else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, providerOptionsProblem, gatewayProblem);
|
|
220
233
|
for (const check of order) {
|
|
221
234
|
const problem = check();
|
|
222
235
|
if (problem) return problem;
|
|
@@ -228,22 +241,15 @@ function judgeProblem(judge, site) {
|
|
|
228
241
|
const pairFactorSchema = z.enum(["environment", "harness"]);
|
|
229
242
|
/** Evaluation models score yes/no questions only, so they can't answer a question. */
|
|
230
243
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
231
|
-
const questionStateSchema = z.strictObject({
|
|
232
|
-
task: z.boolean().optional(),
|
|
233
|
-
output: z.boolean().optional(),
|
|
234
|
-
files: z.boolean().optional(),
|
|
235
|
-
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
236
|
-
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
237
|
-
turns: z.enum(["last", "all"]).optional(),
|
|
238
|
-
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
239
|
-
});
|
|
240
244
|
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
|
|
241
|
-
const
|
|
242
|
-
|
|
243
|
-
model: questionModelSchema.optional(),
|
|
244
|
-
state: questionStateSchema.optional(),
|
|
245
|
+
const questionInputFields = {
|
|
246
|
+
...sharedInputFields,
|
|
245
247
|
artifacts: questionArtifactsSchema.optional()
|
|
246
|
-
}
|
|
248
|
+
};
|
|
249
|
+
z.strictObject(questionInputFields);
|
|
250
|
+
const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
|
|
251
|
+
const questionOptionsSchema = z.strictObject({ model: questionModelSchema.optional() });
|
|
252
|
+
const QUESTION_INPUT_KEYS = Object.keys(questionInputFields);
|
|
247
253
|
/** The first item a list holds twice. */
|
|
248
254
|
const twice = (items) => items.find((k, i) => items.indexOf(k) !== i);
|
|
249
255
|
const unique = (list, ctx) => {
|
|
@@ -254,35 +260,38 @@ const unique = (list, ctx) => {
|
|
|
254
260
|
};
|
|
255
261
|
const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
|
|
256
262
|
const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
|
|
257
|
-
const
|
|
263
|
+
const questionSchema = z.strictObject({
|
|
258
264
|
question: nonEmpty(),
|
|
259
265
|
model: questionModelSchema.optional(),
|
|
260
266
|
across: acrossSchema.optional(),
|
|
261
267
|
choices: choicesSchema.optional(),
|
|
262
|
-
|
|
263
|
-
|
|
268
|
+
turns: turnsSchema.optional(),
|
|
269
|
+
...questionInputFields
|
|
264
270
|
}).superRefine((q, ctx) => {
|
|
265
271
|
if (q.across !== void 0 && q.choices !== void 0) ctx.addIssue({
|
|
266
272
|
code: "custom",
|
|
267
273
|
path: ["choices"],
|
|
268
274
|
message: "with across"
|
|
269
275
|
});
|
|
276
|
+
if (!inputsOn(q, QUESTION_INPUT_KEYS).length) ctx.addIssue({
|
|
277
|
+
code: "custom",
|
|
278
|
+
message: "no input"
|
|
279
|
+
});
|
|
270
280
|
});
|
|
271
|
-
const
|
|
272
|
-
const questionText = (q) => typeof q === "string" ? q : q.question;
|
|
281
|
+
const questionText = (q) => q.question;
|
|
273
282
|
const questionsSchema = z.array(questionSchema).superRefine((list, ctx) => {
|
|
274
283
|
if (twice(list.map(questionText)) !== void 0) ctx.addIssue({
|
|
275
284
|
code: "custom",
|
|
276
285
|
message: "twice"
|
|
277
286
|
});
|
|
278
287
|
});
|
|
279
|
-
const QUESTION_EXAMPLE = `questions: ['How did the agent find the failing test?']`;
|
|
288
|
+
const QUESTION_EXAMPLE = `questions: [{ question: 'How did the agent find the failing test?', transcript: true }]`;
|
|
280
289
|
const ACROSS_HINT = `'environment' (vary the environment, same harness) or 'harness' (vary the harness, same environment)`;
|
|
281
290
|
const ACROSS_MEAN = " Did you mean \"across\"? It is the factor a pairwise question varies: across: 'environment'.";
|
|
282
|
-
const QUESTION_KEYS = Object.keys(
|
|
283
|
-
const
|
|
284
|
-
const
|
|
285
|
-
const
|
|
291
|
+
const QUESTION_KEYS = Object.keys(questionSchema.shape);
|
|
292
|
+
const INPUTS = orList(QUESTION_INPUT_KEYS);
|
|
293
|
+
const QUESTION_SHAPE = `{ question, model?, across?, choices?, turns? } with at least one of ${INPUTS}`;
|
|
294
|
+
const VALUES = {
|
|
286
295
|
task: "true or false",
|
|
287
296
|
output: "true or false",
|
|
288
297
|
files: "true or false",
|
|
@@ -299,41 +308,31 @@ const describe$2 = (value) => {
|
|
|
299
308
|
}
|
|
300
309
|
return JSON.stringify(value) ?? String(value);
|
|
301
310
|
};
|
|
302
|
-
/**
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
const { state, artifacts } = value;
|
|
309
|
-
if (state !== void 0) {
|
|
310
|
-
if (!isObjectLike(state) || Array.isArray(state)) return `${name("state")} in ${where} must be an object such as { transcript: 'tools' } (got ${describe$2(state)})`;
|
|
311
|
-
for (const [key, v] of Object.entries(state)) {
|
|
312
|
-
if (key === "contents") return `${name("state.contents")} in ${where} can't be set: questions are asked after the attempts, when their workspaces are gone. Narrow state.diff instead.`;
|
|
313
|
-
const schema = questionStateSchema.shape[key];
|
|
314
|
-
if (!schema) {
|
|
315
|
-
const hint = closest(key, STATE_KEYS);
|
|
316
|
-
return `Unknown ${name("state")} key "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""}`;
|
|
317
|
-
}
|
|
318
|
-
if (!schema.safeParse(v).success) return `${name(`state.${key}`)} in ${where} must be ${STATE_VALUES[key]} (got ${describe$2(v)})`;
|
|
311
|
+
/** What's wrong with an entry's inputs and `turns`, as one sentence, or undefined. */
|
|
312
|
+
function questionInputsProblem(value, at, where) {
|
|
313
|
+
for (const [key, v] of Object.entries(value)) {
|
|
314
|
+
if (key === "artifacts") {
|
|
315
|
+
if (!questionArtifactsSchema.safeParse(v).success) return `${at} artifacts in ${where} must be true or a non-empty list of capture names (got ${describe$2(v)})`;
|
|
316
|
+
continue;
|
|
319
317
|
}
|
|
318
|
+
if (!(key in VALUES)) continue;
|
|
319
|
+
if (!(key === "turns" ? turnsSchema : sharedInputFields[key]).safeParse(v).success) return `${at} ${key} in ${where} must be ${VALUES[key]} (got ${describe$2(v)})`;
|
|
320
320
|
}
|
|
321
|
-
if (
|
|
321
|
+
if (!inputsOn(value, QUESTION_INPUT_KEYS).length) return `${at} in ${where} shows the model nothing: turn on at least one of ${INPUTS}, e.g. { question: '…', diff: true }.`;
|
|
322
322
|
}
|
|
323
323
|
/** What's wrong with the `question` option set in `where`, as one sentence, or undefined. */
|
|
324
324
|
function questionOptionsProblem(value, where) {
|
|
325
|
-
if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model
|
|
325
|
+
if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
|
|
326
326
|
const keys = Object.keys(questionOptionsSchema.shape);
|
|
327
327
|
for (const key of Object.keys(value)) {
|
|
328
328
|
if (keys.includes(key)) continue;
|
|
329
|
-
if (key === "across" || key === "choices") return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }:
|
|
329
|
+
if (key === "across" || key === "choices" || key === "turns" || key in questionInputFields) return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: what a question is and what it's shown are its own.`;
|
|
330
330
|
const hint = closest(key, keys);
|
|
331
|
-
return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model
|
|
331
|
+
return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model.`;
|
|
332
332
|
}
|
|
333
333
|
const { model } = value;
|
|
334
334
|
if (model !== void 0 && typeof model === "string" && isEvaluationModel(model)) return `question.model in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
|
|
335
335
|
if (model !== void 0 && !questionModelSchema.safeParse(model).success) return `question.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
|
|
336
|
-
return questionPartsProblem(value, (key) => `question.${key}`, where);
|
|
337
336
|
}
|
|
338
337
|
/** What's wrong with an entry's `across`, or undefined. */
|
|
339
338
|
function acrossProblem(question, across, where) {
|
|
@@ -352,11 +351,11 @@ function choicesProblem(question, choices, where) {
|
|
|
352
351
|
if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
|
|
353
352
|
return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
|
|
354
353
|
}
|
|
355
|
-
/** One question's sentence
|
|
354
|
+
/** One question's sentence. */
|
|
356
355
|
function questionProblem(item, where) {
|
|
357
|
-
if (typeof item
|
|
358
|
-
if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes strings or ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
|
|
356
|
+
if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
|
|
359
357
|
const unknown = Object.keys(item).filter((k) => !QUESTION_KEYS.includes(k));
|
|
358
|
+
if (unknown.includes("contents")) return `questions in ${where} can't set contents: questions are asked after the attempts, when their workspaces are gone. Narrow diff instead.`;
|
|
360
359
|
if (unknown.length) {
|
|
361
360
|
const hint = unknown.some((k) => k === "cut" || k === "cuts") ? ACROSS_MEAN : "";
|
|
362
361
|
return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
|
|
@@ -373,7 +372,7 @@ function questionProblem(item, where) {
|
|
|
373
372
|
if (problem) return problem;
|
|
374
373
|
}
|
|
375
374
|
if (across !== void 0 && choices !== void 0) return `questions "${text}" in ${where} sets both across and choices. choices are the answers a question asked of each attempt picks from, and across makes it pick between two attempts instead: drop choices to vote between pairs, or drop across to ask each attempt.`;
|
|
376
|
-
const parts =
|
|
375
|
+
const parts = questionInputsProblem(item, `questions "${text}"`, where);
|
|
377
376
|
if (parts) return parts;
|
|
378
377
|
if (typeof model === "string" && isEvaluationModel(model)) return `questions "${text}" in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
|
|
379
378
|
return `questions "${text}" in ${where}: model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
|
|
@@ -389,6 +388,8 @@ function questionsProblem(value, where) {
|
|
|
389
388
|
//#endregion
|
|
390
389
|
//#region src/contract/config.ts
|
|
391
390
|
const count = () => z.number().int().min(1);
|
|
391
|
+
/** `runs`: a count, or `{ bestOf }` with an odd count of 3 or more. */
|
|
392
|
+
const runs = () => z.union([count(), z.strictObject({ bestOf: z.number().int().min(3).refine((n) => n % 2 === 1, "must be odd") })]);
|
|
392
393
|
const captureWindowSchema = z.strictObject({
|
|
393
394
|
width: count(),
|
|
394
395
|
height: count()
|
|
@@ -421,14 +422,15 @@ const profileDefaultsDataSchema = z.strictObject({
|
|
|
421
422
|
harness: harnessList.optional(),
|
|
422
423
|
sandbox: sandboxDescriptorSchema.optional(),
|
|
423
424
|
environment: environmentDataSchema.optional(),
|
|
424
|
-
runs:
|
|
425
|
+
runs: runs().optional(),
|
|
425
426
|
gate: gatePolicySchema.optional(),
|
|
426
427
|
env: envVarsSchema.optional(),
|
|
427
428
|
egress: egressPolicySchema.optional(),
|
|
428
429
|
judge: configJudgeDataSchema.optional(),
|
|
429
430
|
timeout: durationSchema.optional(),
|
|
430
431
|
capture: captureOptionsSchema.optional(),
|
|
431
|
-
question: questionOptionsSchema.optional()
|
|
432
|
+
question: questionOptionsSchema.optional(),
|
|
433
|
+
snapshot: snapshotOptionSchema.optional()
|
|
432
434
|
});
|
|
433
435
|
const profileLimitSchema = z.strictObject({
|
|
434
436
|
runs: count().optional(),
|
|
@@ -481,7 +483,7 @@ const reevalConfigDataSchema = z.strictObject({
|
|
|
481
483
|
harness: harnessList.optional(),
|
|
482
484
|
sandbox: sandboxDescriptorSchema.optional(),
|
|
483
485
|
environment: environmentDataSchema.optional(),
|
|
484
|
-
runs:
|
|
486
|
+
runs: runs().optional(),
|
|
485
487
|
gate: gatePolicySchema.optional(),
|
|
486
488
|
env: envVarsSchema.optional(),
|
|
487
489
|
egress: egressPolicySchema.optional(),
|
|
@@ -489,6 +491,7 @@ const reevalConfigDataSchema = z.strictObject({
|
|
|
489
491
|
timeout: durationSchema.optional(),
|
|
490
492
|
capture: captureOptionsSchema.optional(),
|
|
491
493
|
question: questionOptionsSchema.optional(),
|
|
494
|
+
snapshot: snapshotOptionSchema.optional(),
|
|
492
495
|
concurrency: count().optional(),
|
|
493
496
|
record: recordMode.optional(),
|
|
494
497
|
history: historyConfigSchema.optional(),
|
|
@@ -643,12 +646,18 @@ const CURSOR_EXCHANGE = {
|
|
|
643
646
|
path: { exact: "/auth/exchange_user_api_key" },
|
|
644
647
|
method: ["POST"]
|
|
645
648
|
};
|
|
646
|
-
/**
|
|
647
|
-
const
|
|
649
|
+
/** Codex's model calls: the Responses API, whose response names the served model. */
|
|
650
|
+
const OPENAI_MODEL_ENDPOINTS = [{
|
|
648
651
|
host: "api.openai.com",
|
|
649
652
|
path: { exact: "/v1/responses" },
|
|
650
653
|
method: ["POST"]
|
|
651
654
|
}];
|
|
655
|
+
/** What Codex sends the key on: its model calls, and its web search tool's own endpoint. */
|
|
656
|
+
const OPENAI_ENDPOINTS = [...OPENAI_MODEL_ENDPOINTS, {
|
|
657
|
+
host: "api.openai.com",
|
|
658
|
+
path: { exact: "/v1/alpha/search" },
|
|
659
|
+
method: ["POST"]
|
|
660
|
+
}];
|
|
652
661
|
/**
|
|
653
662
|
* The model the Responses API says served a request, from its response (a dated snapshot for some
|
|
654
663
|
* models), and the reasoning effort Codex resolved for it, e.g. `gpt-5.6-luna[effort=medium]`.
|
|
@@ -1125,7 +1134,7 @@ const AGENTS = {
|
|
|
1125
1134
|
},
|
|
1126
1135
|
unreported: cursorUnreported,
|
|
1127
1136
|
mcpServers: cursorMcpServers,
|
|
1128
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-
|
|
1137
|
+
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-C9EHSYyP.js")).createCursorSdk({
|
|
1129
1138
|
auth,
|
|
1130
1139
|
...mcpServers ? { mcpServers } : {},
|
|
1131
1140
|
...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
|
|
@@ -1192,7 +1201,7 @@ const AGENTS = {
|
|
|
1192
1201
|
},
|
|
1193
1202
|
servedModel: geminiServedModel,
|
|
1194
1203
|
mcpServers: antigravityMcpServers,
|
|
1195
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-
|
|
1204
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-Bp6X4193.js")).createAntigravity({
|
|
1196
1205
|
auth,
|
|
1197
1206
|
...vertex ? { vertex } : {},
|
|
1198
1207
|
...mcpServers ? { mcpServers } : {},
|
|
@@ -1219,7 +1228,7 @@ const AGENTS = {
|
|
|
1219
1228
|
servedModel: geminiServedModel,
|
|
1220
1229
|
forModel: opencodeForModel,
|
|
1221
1230
|
mcpServers: opencodeMcpServers,
|
|
1222
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-
|
|
1231
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-C5q3_fRV.js")).createOpenCode({
|
|
1223
1232
|
auth,
|
|
1224
1233
|
model,
|
|
1225
1234
|
...vertex ? { vertex } : { vars: opencodeVars(model) },
|
|
@@ -1241,14 +1250,14 @@ const AGENTS = {
|
|
|
1241
1250
|
},
|
|
1242
1251
|
servedModel: geminiServedModel,
|
|
1243
1252
|
mcpServers: gooseMcpServers,
|
|
1244
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-
|
|
1253
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-OOKE-5GA.js")).createGoose({
|
|
1245
1254
|
...vertex ? { vertex } : {},
|
|
1246
1255
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1247
1256
|
auth,
|
|
1248
|
-
provider: (await import("./src-
|
|
1257
|
+
provider: (await import("./src-OOKE-5GA.js")).AI_GATEWAY_PROVIDER
|
|
1249
1258
|
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1250
1259
|
auth,
|
|
1251
|
-
provider: (await import("./src-
|
|
1260
|
+
provider: (await import("./src-OOKE-5GA.js")).OPENROUTER_PROVIDER
|
|
1252
1261
|
} : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
|
|
1253
1262
|
model,
|
|
1254
1263
|
...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
@@ -1269,13 +1278,13 @@ const AGENTS = {
|
|
|
1269
1278
|
endpoints: GEMINI_ENDPOINTS,
|
|
1270
1279
|
servedModel: geminiServedModel,
|
|
1271
1280
|
mcpServers: piMcpServers,
|
|
1272
|
-
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-
|
|
1281
|
+
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-D4CZjMQ4.js")).createPi({
|
|
1273
1282
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1274
1283
|
auth,
|
|
1275
|
-
gateway: (await import("./src-
|
|
1284
|
+
gateway: (await import("./src-D4CZjMQ4.js")).PI_AI_GATEWAY
|
|
1276
1285
|
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1277
1286
|
auth,
|
|
1278
|
-
gateway: (await import("./src-
|
|
1287
|
+
gateway: (await import("./src-D4CZjMQ4.js")).PI_OPENROUTER
|
|
1279
1288
|
} : {
|
|
1280
1289
|
auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
|
|
1281
1290
|
...vertex ? { vertex } : {}
|
|
@@ -1296,7 +1305,7 @@ const AGENTS = {
|
|
|
1296
1305
|
factory: "codex",
|
|
1297
1306
|
credential: "OPENAI_API_KEY",
|
|
1298
1307
|
endpoints: OPENAI_ENDPOINTS,
|
|
1299
|
-
modelResponses:
|
|
1308
|
+
modelResponses: OPENAI_MODEL_ENDPOINTS,
|
|
1300
1309
|
expectedBlocked: {
|
|
1301
1310
|
hosts: [
|
|
1302
1311
|
"github.com",
|
|
@@ -1308,9 +1317,9 @@ const AGENTS = {
|
|
|
1308
1317
|
},
|
|
1309
1318
|
servedModel: openaiServedModel,
|
|
1310
1319
|
mcpServers: codexMcpServers,
|
|
1311
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-
|
|
1320
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-CJdpsrXi.js")).createCodex({
|
|
1312
1321
|
auth,
|
|
1313
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-
|
|
1322
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-CJdpsrXi.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-CJdpsrXi.js")).OPENROUTER_PROVIDER } : {},
|
|
1314
1323
|
...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1315
1324
|
...mcpServers ? { mcpServers } : {},
|
|
1316
1325
|
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
@@ -1328,7 +1337,7 @@ const AGENTS = {
|
|
|
1328
1337
|
...fxForModel("provider/model"),
|
|
1329
1338
|
forModel: fxForModel,
|
|
1330
1339
|
mcpServers: acpMcpServers,
|
|
1331
|
-
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-
|
|
1340
|
+
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-BZlhaDhE.js")).createFx({
|
|
1332
1341
|
auth,
|
|
1333
1342
|
model,
|
|
1334
1343
|
...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
|
|
@@ -1353,7 +1362,7 @@ const AGENTS = {
|
|
|
1353
1362
|
servedModel: xaiServedModel,
|
|
1354
1363
|
mcpServers: acpMcpServers,
|
|
1355
1364
|
adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
|
|
1356
|
-
const grok = await import("./src-
|
|
1365
|
+
const grok = await import("./src-GkJ-1Rdy.js");
|
|
1357
1366
|
return grok.createGrokBuild({
|
|
1358
1367
|
...vertexChat ? {
|
|
1359
1368
|
auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
|
|
@@ -1661,7 +1670,8 @@ function harnessEffort(h) {
|
|
|
1661
1670
|
}
|
|
1662
1671
|
/**
|
|
1663
1672
|
* The reasoning effort a served-model label recorded, e.g. `gpt-5.6-luna[effort=medium]` →
|
|
1664
|
-
* `medium`. Fast (`[fast=true]`)
|
|
1673
|
+
* `medium`, or Cursor's `[reasoning=medium]`. Fast (`[fast=true]`) and an on/off `thinking=true`
|
|
1674
|
+
* are not effort and are ignored.
|
|
1665
1675
|
*/
|
|
1666
1676
|
function effortFromModelLabel(model) {
|
|
1667
1677
|
const inner = model?.match(/\[([^\]]+)]/)?.[1];
|
|
@@ -1671,9 +1681,9 @@ function effortFromModelLabel(model) {
|
|
|
1671
1681
|
if (eq < 0) continue;
|
|
1672
1682
|
const key = part.slice(0, eq).trim();
|
|
1673
1683
|
if (key === "fast") continue;
|
|
1674
|
-
if (/^(?:
|
|
1684
|
+
if (/^(?:reasoning[_-]?)?effort$|^reasoning$|^thinking$/.test(key)) {
|
|
1675
1685
|
const value = part.slice(eq + 1).trim();
|
|
1676
|
-
if (value) return value;
|
|
1686
|
+
if (value && value !== "true" && value !== "false") return value;
|
|
1677
1687
|
}
|
|
1678
1688
|
}
|
|
1679
1689
|
}
|
|
@@ -1704,11 +1714,10 @@ function resolvedEffortOf(models) {
|
|
|
1704
1714
|
/** The error a per-attempt answer carries when the run stopped before a model answered it. */
|
|
1705
1715
|
const STOPPED_FIRST = "the run stopped before it was answered";
|
|
1706
1716
|
/**
|
|
1707
|
-
* What
|
|
1708
|
-
* turn's prompt.
|
|
1709
|
-
* win a side-by-side pick on its wording; the changed files repeat what the diff shows.
|
|
1717
|
+
* What a recorded pairwise answer without `stateOptions` showed each side: the task and the diff,
|
|
1718
|
+
* every turn's prompt.
|
|
1710
1719
|
*/
|
|
1711
|
-
const
|
|
1720
|
+
const UNRECORDED_QUESTION_STATE = {
|
|
1712
1721
|
task: true,
|
|
1713
1722
|
output: false,
|
|
1714
1723
|
files: false,
|
|
@@ -1717,62 +1726,39 @@ const DEFAULT_QUESTION_STATE = {
|
|
|
1717
1726
|
turns: "all",
|
|
1718
1727
|
checks: false
|
|
1719
1728
|
};
|
|
1729
|
+
const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
|
|
1720
1730
|
/**
|
|
1721
|
-
*
|
|
1722
|
-
*
|
|
1723
|
-
* agent went about the work is the point.
|
|
1731
|
+
* Every key set, in the record's key order, so equal selections serialize the same: an input the
|
|
1732
|
+
* question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
|
|
1724
1733
|
*/
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
const defaults = defaultQuestionState(pairwise);
|
|
1736
|
-
const out = {};
|
|
1737
|
-
for (const key of KEYS) out[key] = state?.[key] ?? defaults[key];
|
|
1738
|
-
return out;
|
|
1734
|
+
function resolveQuestionState(state) {
|
|
1735
|
+
return {
|
|
1736
|
+
task: state?.task ?? false,
|
|
1737
|
+
output: state?.output ?? false,
|
|
1738
|
+
files: state?.files ?? false,
|
|
1739
|
+
diff: state?.diff ?? false,
|
|
1740
|
+
transcript: state?.transcript ?? false,
|
|
1741
|
+
turns: state?.turns ?? "all",
|
|
1742
|
+
checks: state?.checks ?? false
|
|
1743
|
+
};
|
|
1739
1744
|
}
|
|
1740
|
-
/** `inner` over `outer
|
|
1745
|
+
/** `inner` over `outer`, field by field. */
|
|
1741
1746
|
function mergeQuestion(outer, inner) {
|
|
1742
|
-
|
|
1743
|
-
const out = {
|
|
1747
|
+
return inner ? {
|
|
1744
1748
|
...outer,
|
|
1745
1749
|
...inner
|
|
1746
|
-
};
|
|
1747
|
-
if (outer.state || inner.state) out.state = {
|
|
1748
|
-
...outer.state,
|
|
1749
|
-
...inner.state
|
|
1750
|
-
};
|
|
1751
|
-
return out;
|
|
1750
|
+
} : outer;
|
|
1752
1751
|
}
|
|
1753
|
-
/** A planned question
|
|
1752
|
+
/** A planned question under the cascaded `question` defaults: its own `model`, else theirs. */
|
|
1754
1753
|
function withQuestionDefaults(q, defaults) {
|
|
1755
|
-
const { state, artifacts } = mergeQuestion(defaults, {
|
|
1756
|
-
...q.state ? { state: q.state } : {},
|
|
1757
|
-
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
1758
|
-
});
|
|
1759
|
-
const { state: _state, artifacts: _artifacts, ...rest } = q;
|
|
1760
1754
|
const model = q.model ?? defaults.model;
|
|
1761
|
-
return {
|
|
1762
|
-
...
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
...artifacts !== void 0 ? { artifacts } : {}
|
|
1766
|
-
};
|
|
1755
|
+
return model !== void 0 ? {
|
|
1756
|
+
...q,
|
|
1757
|
+
model
|
|
1758
|
+
} : q;
|
|
1767
1759
|
}
|
|
1768
1760
|
const isOff = (value) => value === false || Array.isArray(value) && !value.length;
|
|
1769
|
-
/**
|
|
1770
|
-
const showsNothing = (q) => {
|
|
1771
|
-
if (q.artifacts !== void 0) return false;
|
|
1772
|
-
const s = resolveQuestionState(q.state, q.across !== void 0);
|
|
1773
|
-
return KEYS.every((k) => k === "turns" || isOff(s[k]));
|
|
1774
|
-
};
|
|
1775
|
-
/** The state keys a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1761
|
+
/** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1776
1762
|
function questionShows(state, artifacts) {
|
|
1777
1763
|
const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
|
|
1778
1764
|
return artifacts === void 0 ? on : [...on, "artifacts"];
|
|
@@ -1838,6 +1824,189 @@ function compareProblem(value, where, environment) {
|
|
|
1838
1824
|
if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness? }.`;
|
|
1839
1825
|
return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
|
|
1840
1826
|
}
|
|
1827
|
+
const POLL_MS = 200;
|
|
1828
|
+
/**
|
|
1829
|
+
* Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
|
|
1830
|
+
* (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
|
|
1831
|
+
*/
|
|
1832
|
+
function probeScript(pid, tables = true) {
|
|
1833
|
+
return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
|
|
1834
|
+
}
|
|
1835
|
+
/** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
|
|
1836
|
+
function procAddress(hex) {
|
|
1837
|
+
const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
|
|
1838
|
+
if (bytes.length === 4) return bytes.join(".");
|
|
1839
|
+
if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
|
|
1840
|
+
const groups = [];
|
|
1841
|
+
for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
|
|
1842
|
+
const full = groups.join(":");
|
|
1843
|
+
const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
|
|
1844
|
+
return longest ? full.replace(longest, "::") : full;
|
|
1845
|
+
}
|
|
1846
|
+
/** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
|
|
1847
|
+
const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
|
|
1848
|
+
function parseProbe(stdout) {
|
|
1849
|
+
const lines = stdout.split("\n");
|
|
1850
|
+
const alive = lines.includes("alive");
|
|
1851
|
+
if (lines.includes("noproc")) return {
|
|
1852
|
+
proc: false,
|
|
1853
|
+
alive
|
|
1854
|
+
};
|
|
1855
|
+
const listeners = [];
|
|
1856
|
+
for (const l of lines) {
|
|
1857
|
+
const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
|
|
1858
|
+
const address = m ? procAddress(m[1]) : "";
|
|
1859
|
+
if (m && reachable(address)) listeners.push({
|
|
1860
|
+
port: Number.parseInt(m[2], 16),
|
|
1861
|
+
address
|
|
1862
|
+
});
|
|
1863
|
+
}
|
|
1864
|
+
return {
|
|
1865
|
+
proc: true,
|
|
1866
|
+
alive,
|
|
1867
|
+
listeners
|
|
1868
|
+
};
|
|
1869
|
+
}
|
|
1870
|
+
/** The ports listening now that weren't before the command started, ascending. */
|
|
1871
|
+
const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
|
|
1872
|
+
/** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
|
|
1873
|
+
const addressFor = (listeners, port) => {
|
|
1874
|
+
const all = listeners.filter((l) => l.port === port).map((l) => l.address);
|
|
1875
|
+
return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
|
|
1876
|
+
};
|
|
1877
|
+
/** `want` is taken whoever listens on it: reeval allocated it, or the eval named it. */
|
|
1878
|
+
function decide(s) {
|
|
1879
|
+
const { listeners, alive } = s.probe;
|
|
1880
|
+
const ports = newPorts(listeners, s.before);
|
|
1881
|
+
if (listeners.some((l) => l.port === s.want)) return {
|
|
1882
|
+
kind: "port",
|
|
1883
|
+
port: s.want,
|
|
1884
|
+
address: addressFor(listeners, s.want)
|
|
1885
|
+
};
|
|
1886
|
+
const late = s.elapsedMs >= s.timeoutMs;
|
|
1887
|
+
if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
|
|
1888
|
+
if (ports.length === 1) {
|
|
1889
|
+
const port = ports[0];
|
|
1890
|
+
return {
|
|
1891
|
+
kind: "port",
|
|
1892
|
+
port,
|
|
1893
|
+
address: addressFor(listeners, port)
|
|
1894
|
+
};
|
|
1895
|
+
}
|
|
1896
|
+
return {
|
|
1897
|
+
kind: "several",
|
|
1898
|
+
ports
|
|
1899
|
+
};
|
|
1900
|
+
}
|
|
1901
|
+
if (!alive) return { kind: "exited" };
|
|
1902
|
+
return late ? {
|
|
1903
|
+
kind: "timeout",
|
|
1904
|
+
ports
|
|
1905
|
+
} : { kind: "wait" };
|
|
1906
|
+
}
|
|
1907
|
+
const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
|
|
1908
|
+
/** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
|
|
1909
|
+
function printedUrls(output, ports) {
|
|
1910
|
+
const found = /* @__PURE__ */ new Map();
|
|
1911
|
+
for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
|
|
1912
|
+
const port = Number(m[1]);
|
|
1913
|
+
if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
|
|
1914
|
+
}
|
|
1915
|
+
return [...found].map(([url, port]) => ({
|
|
1916
|
+
url,
|
|
1917
|
+
port
|
|
1918
|
+
}));
|
|
1919
|
+
}
|
|
1920
|
+
/** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
|
|
1921
|
+
async function hostListening(port) {
|
|
1922
|
+
for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
|
|
1923
|
+
const socket = connect({
|
|
1924
|
+
port,
|
|
1925
|
+
host
|
|
1926
|
+
});
|
|
1927
|
+
socket.once("connect", () => {
|
|
1928
|
+
socket.destroy();
|
|
1929
|
+
resolve(true);
|
|
1930
|
+
});
|
|
1931
|
+
socket.once("error", () => resolve(false));
|
|
1932
|
+
})) return host;
|
|
1933
|
+
}
|
|
1934
|
+
const sleep = (ms, signal) => new Promise((resolve) => {
|
|
1935
|
+
const done = () => {
|
|
1936
|
+
clearTimeout(timer);
|
|
1937
|
+
signal?.removeEventListener("abort", done);
|
|
1938
|
+
resolve();
|
|
1939
|
+
};
|
|
1940
|
+
const timer = setTimeout(done, ms);
|
|
1941
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
1942
|
+
});
|
|
1943
|
+
const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
|
|
1944
|
+
/**
|
|
1945
|
+
* Polls until something listens on `want`, or, without a given port, until the ports that are
|
|
1946
|
+
* new since the command started have settled on one. Throws, with the command's last output,
|
|
1947
|
+
* when they settle on several, it exits, or nothing listens in time.
|
|
1948
|
+
*/
|
|
1949
|
+
async function awaitListening(command, w) {
|
|
1950
|
+
const timeoutMs = w.timeoutMs ?? 12e4;
|
|
1951
|
+
const started = Date.now();
|
|
1952
|
+
let last = "";
|
|
1953
|
+
let changed = started;
|
|
1954
|
+
const output = async () => plain(await w.output().catch(() => ""));
|
|
1955
|
+
const fail = async (why, out) => {
|
|
1956
|
+
const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
|
|
1957
|
+
return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
|
|
1958
|
+
};
|
|
1959
|
+
for (;;) {
|
|
1960
|
+
w.signal?.throwIfAborted();
|
|
1961
|
+
const probe = await w.probe();
|
|
1962
|
+
const now = Date.now();
|
|
1963
|
+
if (!probe.proc) {
|
|
1964
|
+
const address = await (w.hostListening ?? hostListening)(w.want);
|
|
1965
|
+
if (address !== void 0) return {
|
|
1966
|
+
port: w.want,
|
|
1967
|
+
address,
|
|
1968
|
+
detected: false,
|
|
1969
|
+
readyMs: now - started
|
|
1970
|
+
};
|
|
1971
|
+
if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
|
|
1972
|
+
if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
|
|
1973
|
+
} else {
|
|
1974
|
+
const key = newPorts(probe.listeners, w.before).join(",");
|
|
1975
|
+
if (key !== last) {
|
|
1976
|
+
last = key;
|
|
1977
|
+
changed = now;
|
|
1978
|
+
}
|
|
1979
|
+
const d = decide({
|
|
1980
|
+
want: w.want,
|
|
1981
|
+
given: w.given,
|
|
1982
|
+
probe,
|
|
1983
|
+
before: w.before,
|
|
1984
|
+
stableMs: now - changed,
|
|
1985
|
+
elapsedMs: now - started,
|
|
1986
|
+
timeoutMs
|
|
1987
|
+
});
|
|
1988
|
+
if (d.kind === "port") return {
|
|
1989
|
+
port: d.port,
|
|
1990
|
+
address: d.address,
|
|
1991
|
+
detected: d.port !== w.want,
|
|
1992
|
+
readyMs: now - started
|
|
1993
|
+
};
|
|
1994
|
+
if (d.kind === "several") {
|
|
1995
|
+
const out = await output();
|
|
1996
|
+
const urls = printedUrls(out, d.ports);
|
|
1997
|
+
const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
|
|
1998
|
+
const hint = urls[0]?.port ?? d.ports[0];
|
|
1999
|
+
throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
|
|
2000
|
+
}
|
|
2001
|
+
if (d.kind === "exited") throw await fail("exited before it listened on a port");
|
|
2002
|
+
if (d.kind === "timeout") {
|
|
2003
|
+
const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
|
|
2004
|
+
throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
|
|
2005
|
+
}
|
|
2006
|
+
}
|
|
2007
|
+
await sleep(POLL_MS, w.signal);
|
|
2008
|
+
}
|
|
2009
|
+
}
|
|
1841
2010
|
//#endregion
|
|
1842
2011
|
//#region src/sandbox/fake.ts
|
|
1843
2012
|
/** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
|
|
@@ -1982,7 +2151,7 @@ async function createFakeSandbox(descriptor, env = {}) {
|
|
|
1982
2151
|
},
|
|
1983
2152
|
home: base,
|
|
1984
2153
|
readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
|
|
1985
|
-
getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}
|
|
2154
|
+
getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
|
|
1986
2155
|
async writeFile(path, content) {
|
|
1987
2156
|
const target = resolve(path);
|
|
1988
2157
|
await mkdir(dirname(target), { recursive: true });
|
|
@@ -2063,13 +2232,16 @@ function resolveOutput(input) {
|
|
|
2063
2232
|
"errors-only",
|
|
2064
2233
|
"none"
|
|
2065
2234
|
]);
|
|
2235
|
+
const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only";
|
|
2236
|
+
const color = env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions;
|
|
2066
2237
|
return {
|
|
2067
2238
|
mode,
|
|
2068
2239
|
statusLine,
|
|
2069
2240
|
logOrder,
|
|
2070
2241
|
logPrefix,
|
|
2071
|
-
outputLogs
|
|
2072
|
-
color
|
|
2242
|
+
outputLogs,
|
|
2243
|
+
color,
|
|
2244
|
+
rich: statusLine && color,
|
|
2073
2245
|
githubActions,
|
|
2074
2246
|
heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
|
|
2075
2247
|
};
|
|
@@ -2200,34 +2372,28 @@ function egressLabel({ policy, auto }) {
|
|
|
2200
2372
|
}
|
|
2201
2373
|
//#endregion
|
|
2202
2374
|
//#region src/judge/state-options.ts
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2375
|
+
/**
|
|
2376
|
+
* What one call turned on, every key set (`false` when off), in the record's key order, so equal
|
|
2377
|
+
* selections serialize the same.
|
|
2378
|
+
*/
|
|
2379
|
+
function resolveStateOptions(call) {
|
|
2380
|
+
return {
|
|
2381
|
+
task: call.task ?? false,
|
|
2382
|
+
output: call.output ?? false,
|
|
2383
|
+
files: call.files ?? false,
|
|
2384
|
+
diff: call.diff ?? false,
|
|
2385
|
+
contents: call.contents ?? [],
|
|
2386
|
+
transcript: call.transcript ?? false,
|
|
2387
|
+
turns: call.turns ?? "last",
|
|
2388
|
+
checks: call.checks ?? false
|
|
2389
|
+
};
|
|
2218
2390
|
}
|
|
2219
|
-
/** `inner` over `outer`, field by field
|
|
2391
|
+
/** `inner` over `outer`, field by field. */
|
|
2220
2392
|
function mergeJudge(outer, inner) {
|
|
2221
|
-
|
|
2222
|
-
const out = {
|
|
2393
|
+
return inner ? {
|
|
2223
2394
|
...outer,
|
|
2224
2395
|
...inner
|
|
2225
|
-
};
|
|
2226
|
-
if (outer.state || inner.state) out.state = {
|
|
2227
|
-
...outer.state,
|
|
2228
|
-
...inner.state
|
|
2229
|
-
};
|
|
2230
|
-
return out;
|
|
2396
|
+
} : outer;
|
|
2231
2397
|
}
|
|
2232
2398
|
const KNOWN = [...[
|
|
2233
2399
|
"harness",
|
|
@@ -2243,6 +2409,7 @@ const KNOWN = [...[
|
|
|
2243
2409
|
"question",
|
|
2244
2410
|
"questions",
|
|
2245
2411
|
"timeout",
|
|
2412
|
+
"snapshot",
|
|
2246
2413
|
"tags",
|
|
2247
2414
|
"revision",
|
|
2248
2415
|
"pin"
|
|
@@ -2257,6 +2424,7 @@ const SUGGEST = {
|
|
|
2257
2424
|
repeats: "runs",
|
|
2258
2425
|
iterations: "runs",
|
|
2259
2426
|
trials: "runs",
|
|
2427
|
+
bestOf: "runs",
|
|
2260
2428
|
secrets: "env",
|
|
2261
2429
|
vars: "env",
|
|
2262
2430
|
network: "egress",
|
|
@@ -2265,9 +2433,26 @@ const SUGGEST = {
|
|
|
2265
2433
|
version: "revision",
|
|
2266
2434
|
rev: "revision",
|
|
2267
2435
|
pins: "pin",
|
|
2436
|
+
workspace: "snapshot",
|
|
2437
|
+
snapshots: "snapshot",
|
|
2268
2438
|
passRate: "gate",
|
|
2269
2439
|
threshold: "gate"
|
|
2270
2440
|
};
|
|
2441
|
+
/**
|
|
2442
|
+
* `snapshot` over the levels above: the shorthand sets `when`, the object sets the fields it
|
|
2443
|
+
* names, and `compress` carries through either.
|
|
2444
|
+
*/
|
|
2445
|
+
function mergeSnapshot(outer, inner) {
|
|
2446
|
+
if (inner === void 0) return outer;
|
|
2447
|
+
if (typeof inner !== "object") return {
|
|
2448
|
+
...outer,
|
|
2449
|
+
when: inner
|
|
2450
|
+
};
|
|
2451
|
+
return {
|
|
2452
|
+
when: inner.when ?? outer.when,
|
|
2453
|
+
compress: inner.compress ?? outer.compress
|
|
2454
|
+
};
|
|
2455
|
+
}
|
|
2271
2456
|
/** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
|
|
2272
2457
|
function checkGate(value, where) {
|
|
2273
2458
|
const problem = gateProblem(value, where);
|
|
@@ -2310,10 +2495,12 @@ function sandboxId(s) {
|
|
|
2310
2495
|
return image ? `${engine}:${sanitize(image)}${install}` : `${engine}${install}`;
|
|
2311
2496
|
}
|
|
2312
2497
|
/** The planned attempts' sandboxes by id, once each, or `fallback`'s when none is planned. */
|
|
2313
|
-
function
|
|
2498
|
+
function plannedSandboxIds(planned, fallback) {
|
|
2314
2499
|
const ids = new Set(planned.map(sandboxId));
|
|
2315
|
-
return ids.size ? [...ids]
|
|
2500
|
+
return ids.size ? [...ids] : [sandboxId(fallback)];
|
|
2316
2501
|
}
|
|
2502
|
+
/** {@link plannedSandboxIds}, comma-separated. */
|
|
2503
|
+
const plannedSandboxes = (planned, fallback) => plannedSandboxIds(planned, fallback).join(", ");
|
|
2317
2504
|
const cellKey = (harness, environment, sandbox) => `${harness}/${environment}/${sandbox}`;
|
|
2318
2505
|
const isHarness = (v) => isObjectLike(v) && (v.kind === "fake" || isAgentKind(v.kind));
|
|
2319
2506
|
function checkMaxConcurrent(value, where) {
|
|
@@ -2335,19 +2522,44 @@ function refuseIllegalCompare(options, where) {
|
|
|
2335
2522
|
if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
|
|
2336
2523
|
}
|
|
2337
2524
|
const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
|
|
2525
|
+
/** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
|
|
2526
|
+
function checkRuns(runs, where) {
|
|
2527
|
+
if (typeof runs === "number" && Number.isInteger(runs) && runs >= 1) return;
|
|
2528
|
+
if (typeof runs === "number") throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
|
|
2529
|
+
if (!isObjectLike(runs)) throw new UsageError(`runs in ${where} must be a whole number of at least 1, or { bestOf: n } (got ${JSON.stringify(runs)})`);
|
|
2530
|
+
const extra = Object.keys(runs).find((k) => k !== "bestOf");
|
|
2531
|
+
if (extra !== void 0) throw new UsageError(`runs in ${where} takes { bestOf: n } only (got the key "${extra}"), e.g. runs: { bestOf: 3 }`);
|
|
2532
|
+
const { bestOf } = runs;
|
|
2533
|
+
if (!(Number.isInteger(bestOf) && bestOf >= 3 && bestOf % 2 === 1)) throw new UsageError(`runs.bestOf in ${where} must be an odd whole number of 3 or more, so one side always has a majority, e.g. runs: { bestOf: 3 } (got ${JSON.stringify(bestOf)})`);
|
|
2534
|
+
}
|
|
2535
|
+
/** The most attempts `runs` plans a cell, and its `bestOf` when it has one. */
|
|
2536
|
+
const runsOf = (runs) => typeof runs === "number" ? { runs } : {
|
|
2537
|
+
runs: runs.bestOf,
|
|
2538
|
+
bestOf: runs.bestOf
|
|
2539
|
+
};
|
|
2540
|
+
/**
|
|
2541
|
+
* A best-of cell's gate: half or more of its counted attempts pass, which is its majority once it
|
|
2542
|
+
* has one (an infra error counts on neither side).
|
|
2543
|
+
*/
|
|
2544
|
+
const MAJORITY_GATE = { passRate: .5 };
|
|
2545
|
+
/** How many of a best-of cell's attempts always run: the majority. */
|
|
2546
|
+
const majorityOf = (bestOf) => (bestOf + 1) / 2;
|
|
2338
2547
|
/** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
|
|
2339
2548
|
function checkOptions(options, where, allowed = KNOWN) {
|
|
2340
2549
|
for (const key of Object.keys(options)) {
|
|
2341
2550
|
if (allowed.includes(key)) continue;
|
|
2342
2551
|
const near = SUGGEST[key] ?? closest(key, allowed);
|
|
2343
|
-
const
|
|
2552
|
+
const value = options[key];
|
|
2553
|
+
const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
|
|
2344
2554
|
throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
|
|
2345
2555
|
}
|
|
2346
2556
|
const o = options;
|
|
2347
|
-
if (o.runs !== void 0
|
|
2557
|
+
if (o.runs !== void 0) checkRuns(o.runs, where);
|
|
2348
2558
|
if (o.gate !== void 0) checkGate(o.gate, where);
|
|
2349
2559
|
if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
|
|
2350
2560
|
if (o.revision !== void 0 && !isRevision(o.revision)) throw new UsageError(`revision in ${where} must be a whole number or a non-empty string, e.g. revision: 2 (got ${JSON.stringify(o.revision)})`);
|
|
2561
|
+
const snapshotIssue = snapshotProblem(o.snapshot, where);
|
|
2562
|
+
if (snapshotIssue) throw new UsageError(snapshotIssue);
|
|
2351
2563
|
if (o.env !== void 0) checkEnv(o.env, where, UsageError);
|
|
2352
2564
|
if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
|
|
2353
2565
|
const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
|
|
@@ -2478,17 +2690,18 @@ function checkUnique(list, where) {
|
|
|
2478
2690
|
seen.add(key);
|
|
2479
2691
|
}
|
|
2480
2692
|
}
|
|
2481
|
-
const plannedQuestions = (value) => value.map((q, i) =>
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2693
|
+
const plannedQuestions = (value) => value.map((q, i) => {
|
|
2694
|
+
const { question, model, across, choices, artifacts, ...rest } = q;
|
|
2695
|
+
const state = Object.fromEntries(Object.entries(rest).filter(([, v]) => v !== void 0));
|
|
2696
|
+
return {
|
|
2697
|
+
n: i + 1,
|
|
2698
|
+
question,
|
|
2699
|
+
...model !== void 0 ? { model } : {},
|
|
2700
|
+
...across !== void 0 ? { across } : {},
|
|
2701
|
+
...choices ? { choices } : {},
|
|
2702
|
+
state,
|
|
2703
|
+
...artifacts !== void 0 ? { artifacts } : {}
|
|
2704
|
+
};
|
|
2492
2705
|
});
|
|
2493
2706
|
/** Whether the eval's matrix varies each factor a pairwise question can vary. */
|
|
2494
2707
|
const variesBy = (r) => ({
|
|
@@ -2529,9 +2742,14 @@ function applyLevel(out, options, where, egress = "own") {
|
|
|
2529
2742
|
if (options.sandbox !== void 0) out.sandbox = options.sandbox;
|
|
2530
2743
|
if (options.environment !== void 0) out.environment = options.environment;
|
|
2531
2744
|
if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
|
|
2532
|
-
if (options.runs !== void 0)
|
|
2745
|
+
if (options.runs !== void 0) {
|
|
2746
|
+
const { runs, bestOf } = runsOf(options.runs);
|
|
2747
|
+
out.runs = runs;
|
|
2748
|
+
out.bestOf = bestOf;
|
|
2749
|
+
}
|
|
2533
2750
|
if (options.gate !== void 0) out.gate = options.gate;
|
|
2534
2751
|
if (options.timeout !== void 0) out.timeout = options.timeout;
|
|
2752
|
+
out.snapshot = mergeSnapshot(out.snapshot, options.snapshot);
|
|
2535
2753
|
if (options.revision !== void 0) out.revision = options.revision;
|
|
2536
2754
|
Object.assign(out.env, options.env);
|
|
2537
2755
|
out.judge = mergeJudge(out.judge, options.judge);
|
|
@@ -2578,11 +2796,7 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2578
2796
|
baseline: out.environment,
|
|
2579
2797
|
from: "environment"
|
|
2580
2798
|
};
|
|
2581
|
-
if (out.questions.length)
|
|
2582
|
-
out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
|
|
2583
|
-
const blank = out.questions.find(showsNothing);
|
|
2584
|
-
if (blank) throw new UsageError(`questions "${blank.question}" in ${where} shows the model nothing: every state key is off and it sends no artifacts. Turn on a state key, such as state: { diff: true }, or send artifacts.`);
|
|
2585
|
-
}
|
|
2799
|
+
if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
|
|
2586
2800
|
checkUnique(out.harness, where);
|
|
2587
2801
|
const harnessFromProfile = profile?.defaults.harness !== void 0 && !levels.some((l) => l.options.harness);
|
|
2588
2802
|
const authored = variesBy(out);
|
|
@@ -2648,8 +2862,10 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2648
2862
|
}
|
|
2649
2863
|
defaultRuns(out, notes, options);
|
|
2650
2864
|
if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && (out.runs ?? 1) > limit.runs) {
|
|
2651
|
-
|
|
2865
|
+
const was = out.bestOf !== void 0 ? `best of ${out.bestOf}` : String(out.runs);
|
|
2866
|
+
notes.push(`runs ${was} → ${limit.runs} (${tag})`);
|
|
2652
2867
|
out.runs = limit.runs;
|
|
2868
|
+
out.bestOf = void 0;
|
|
2653
2869
|
out.sources = {
|
|
2654
2870
|
...out.sources,
|
|
2655
2871
|
runs: `profiles.${profile.name}.limit`
|
|
@@ -2678,6 +2894,16 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2678
2894
|
if (narrowed) notes.push(`${plural(narrowed)} narrowed (profile ${profile.name})`);
|
|
2679
2895
|
out.questions = kept;
|
|
2680
2896
|
}
|
|
2897
|
+
if (options.runs !== void 0) out.bestOf = void 0;
|
|
2898
|
+
if (out.bestOf !== void 0 && out.compare?.variants.length) {
|
|
2899
|
+
const from = out.sources?.runs ?? "the config";
|
|
2900
|
+
const set = from === where ? "" : ` from ${from}`;
|
|
2901
|
+
throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
|
|
2902
|
+
}
|
|
2903
|
+
if (out.bestOf !== void 0) {
|
|
2904
|
+
if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
|
|
2905
|
+
out.gate = MAJORITY_GATE;
|
|
2906
|
+
}
|
|
2681
2907
|
defaultRuns(out, notes, options);
|
|
2682
2908
|
return {
|
|
2683
2909
|
resolved: {
|
|
@@ -2813,6 +3039,7 @@ function expand(resolved, vertex) {
|
|
|
2813
3039
|
egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers, onVertex),
|
|
2814
3040
|
gate,
|
|
2815
3041
|
...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
|
|
3042
|
+
...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
|
|
2816
3043
|
n,
|
|
2817
3044
|
label,
|
|
2818
3045
|
...onVertex ? { vertex: onVertex } : {}
|
|
@@ -2941,6 +3168,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
2941
3168
|
} } : {},
|
|
2942
3169
|
...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
|
|
2943
3170
|
runs: resolved.runs,
|
|
3171
|
+
...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
|
|
2944
3172
|
...resolved.gate !== void 0 ? { gate: resolved.gate } : {},
|
|
2945
3173
|
attempts: skip ? 0 : attempts.length,
|
|
2946
3174
|
notes,
|
|
@@ -2964,7 +3192,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
2964
3192
|
/** A pairwise question that shows each side's transcript, asked of two harnesses: tool names unblind it. */
|
|
2965
3193
|
function questionWarnings(resolved) {
|
|
2966
3194
|
if (resolved.harness.length < 2) return [];
|
|
2967
|
-
return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows
|
|
3195
|
+
return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows the transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
|
|
2968
3196
|
}
|
|
2969
3197
|
function captureNote(c) {
|
|
2970
3198
|
const parts = [];
|
|
@@ -3011,6 +3239,16 @@ function fxEffortRefusal(h, gateway) {
|
|
|
3011
3239
|
if (h.kind !== "fx" || effort === void 0) return void 0;
|
|
3012
3240
|
return `fx sends no effort of its own, and reeval adds fx's effort (${JSON.stringify(effort)}) only to its calls on Vertex AI, not on ${gateway}: drop effort or run fx on vertex()`;
|
|
3013
3241
|
}
|
|
3242
|
+
/**
|
|
3243
|
+
* Why `h` takes no effort on `gateway`, where an effort it set would be refused, or undefined
|
|
3244
|
+
* when it can take one there.
|
|
3245
|
+
*/
|
|
3246
|
+
function effortRefusedOn(h, gateway) {
|
|
3247
|
+
if (h.kind === "fx" && gateway !== "vertex") return `fx sends no effort of its own, and reeval adds one only on vertex()`;
|
|
3248
|
+
if (h.kind === "fake" || !isRoutingGateway(gateway)) return void 0;
|
|
3249
|
+
const { name, unprovenEffort } = ROUTING_GATEWAYS[gateway];
|
|
3250
|
+
return unprovenEffort[h.kind]?.(h.model) ? `reeval doesn't send ${agentName(h)}'s effort to ${h.model} through ${name} yet` : void 0;
|
|
3251
|
+
}
|
|
3014
3252
|
/** A routing gateway's entry: its route table decides whom it serves, and with what effort. */
|
|
3015
3253
|
function routingEntry(name) {
|
|
3016
3254
|
const gateway = ROUTING_GATEWAYS[name];
|
|
@@ -3157,6 +3395,7 @@ const CONFIG_KEYS = [
|
|
|
3157
3395
|
"capture",
|
|
3158
3396
|
"question",
|
|
3159
3397
|
"timeout",
|
|
3398
|
+
"snapshot",
|
|
3160
3399
|
"concurrency",
|
|
3161
3400
|
"record",
|
|
3162
3401
|
"history",
|
|
@@ -3187,7 +3426,8 @@ const DEFAULTS_KEYS = [
|
|
|
3187
3426
|
"judge",
|
|
3188
3427
|
"capture",
|
|
3189
3428
|
"question",
|
|
3190
|
-
"timeout"
|
|
3429
|
+
"timeout",
|
|
3430
|
+
"snapshot"
|
|
3191
3431
|
];
|
|
3192
3432
|
const LIMIT_KEYS = [
|
|
3193
3433
|
"runs",
|
|
@@ -3543,7 +3783,7 @@ function selectProfile(config, flag, env = process.env) {
|
|
|
3543
3783
|
function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
3544
3784
|
const user = withHostPaths(input, root, file);
|
|
3545
3785
|
if ("compare" in user) throw new UsageError(`compare can't be set in ${file}: it would multiply every eval. Put it on the group or eval that asks the question.`);
|
|
3546
|
-
if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model
|
|
3786
|
+
if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model } in ${file} sets every question's model.`);
|
|
3547
3787
|
checkOptions(user, file, CONFIG_KEYS);
|
|
3548
3788
|
const include = user.include ?? defaultIncludeGlobs();
|
|
3549
3789
|
if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
|
|
@@ -3588,6 +3828,10 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3588
3828
|
capture: user.capture ?? {},
|
|
3589
3829
|
question: user.question ?? {},
|
|
3590
3830
|
timeoutMs: parseDuration(user.timeout ?? "15m"),
|
|
3831
|
+
snapshot: mergeSnapshot({
|
|
3832
|
+
when: true,
|
|
3833
|
+
compress: false
|
|
3834
|
+
}, user.snapshot),
|
|
3591
3835
|
concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
|
|
3592
3836
|
record,
|
|
3593
3837
|
history: {
|
|
@@ -3772,6 +4016,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
|
|
|
3772
4016
|
});
|
|
3773
4017
|
}
|
|
3774
4018
|
//#endregion
|
|
3775
|
-
export {
|
|
4019
|
+
export { ciRunner as $, expand as A, captureWindowSchema as At, refuseIllegalCompare as B, artifactRefsOf as Bt, resolveGateway as C, credentialOf as Ct, checkLimitHarness as D, refusedMcp as Dt, cascade as E, layoutOf as Et, plainHeaderError as F, judgeFields as Ft, secretVarsIn as G, sandboxId as H, decideState as Ht, plainHeaders as I, judgeModelNameSchema as It, mergeJudge as J, unbrokered as K, plannedSandboxIds as L, judgeProblem as Lt, levelOf as M, NO_JUDGE_INPUT as Mt, majorityOf as N, fakeJudgeDataSchema as Nt, checkOptions as O, refusedPlugins as Ot, matchesTags as P, inputsOn as Pt, allowedHosts as Q, plannedSandboxes as R, sharedInputFields as Rt, fallbackHint as S, claudeMcpServers as St, brokeredIn as T, isAgentKind as Tt, sandboxProvider as U, filePartsOf as Ut, runsOf as V, badJudgeArtifacts as Vt, sandboxSetBy as W, resolveJudgeArtifacts as Wt, DEFAULT_EGRESS as X, resolveStateOptions as Y, DEFAULT_EGRESS_ALLOW as Z, fellBackToDirect as _, agentAuth as _t, configPathFor as a, resolveOutput as at, resolveModelGateway as b, authHint as bt, findConfig as c, parseProbe as ct, malformedCredentials as d, UNRECORDED_QUESTION_STATE as dt, directOnCloud as et, missingCredentials as f, questionShows as ft, selectProfile as g, servedEffortOf as gt, runJudgeModel as h, resolvedEffortOf as ht, configLabel as i, unenforceable as it, harnessId as j, INPUT_KEYS as jt, evalPlan as k, writePluginFiles as kt, hostCapacity as l, probeScript as lt, resolveConcurrency as m, harnessEffort as mt, analysisModelOf as n, gatewayPolicy as nt, defaultConcurrency as o, createFakeSandbox as ot, requireConfig as p, resolveQuestionState as pt, unbrokeredError as q, configFilesPhrase as r, isDirect as rt, defineConfig as s, awaitListening as st, CONFIG_FILES as t, egressLabel as tt, loadConfig as u, STOPPED_FIRST as ut, gatewayScope as v, agentFor as vt, routedHarness as w, expectedBlockedFor as wt, effortRefusedOn as x, brokeringFor as xt, modelGatewayOrNone as y, agentMcp as yt, profileJudge as z, turnsSchema as zt };
|
|
3776
4020
|
|
|
3777
|
-
//# sourceMappingURL=config-
|
|
4021
|
+
//# sourceMappingURL=config-BjjDryd5.js.map
|