reeval 0.15.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -14
- package/dist/{adapter-CbqbMR7y.js → adapter-CqUt4A0w.js} +3 -3
- package/dist/{adapter-CbqbMR7y.js.map → adapter-CqUt4A0w.js.map} +1 -1
- package/dist/{agent-Zgn-kiqq.js → agent-BtAbRTFq.js} +6 -6
- package/dist/{agent-Zgn-kiqq.js.map → agent-BtAbRTFq.js.map} +1 -1
- package/dist/{app-files-ChbvRx3P.js → app-files-CDttHPOz.js} +3 -3
- package/dist/app-files-CDttHPOz.js.map +1 -0
- package/dist/assemble-Bb7zqpRw.js +1 -0
- package/dist/{assemble-SKQGW_QF.js → assemble-J9rkE29m.js} +114 -31
- package/dist/assemble-J9rkE29m.js.map +1 -0
- package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
- package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{attempt-host-BOp8JklD.js → attempt-host-DDedP7Qv.js} +17 -8
- package/dist/attempt-host-DDedP7Qv.js.map +1 -0
- package/dist/capture-Bvc4FtWT.js +15 -0
- package/dist/capture-Bvc4FtWT.js.map +1 -0
- package/dist/capture-D68TiQQy.js +1 -0
- package/dist/{capture-BM_3100g.js → capture-DheNqeYS.js} +1495 -203
- package/dist/capture-DheNqeYS.js.map +1 -0
- package/dist/{cli-PdYrcoKM.js → cli-KTHjNFoo.js} +329 -74
- package/dist/cli-KTHjNFoo.js.map +1 -0
- package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{colors-QV9dFVhN.js → colors-Y6O3mtl_.js} +20 -2
- package/dist/colors-Y6O3mtl_.js.map +1 -0
- package/dist/{command-9xm5ImqI.js → command-1FBECMs-.js} +16 -16
- package/dist/{command-9xm5ImqI.js.map → command-1FBECMs-.js.map} +1 -1
- package/dist/{command-BqBxLgTc.js → command-Ce03LLtg.js} +14 -14
- package/dist/{command-BqBxLgTc.js.map → command-Ce03LLtg.js.map} +1 -1
- package/dist/{command-4pJYVCT4.js → command-CztRuwUM.js} +12 -12
- package/dist/{command-4pJYVCT4.js.map → command-CztRuwUM.js.map} +1 -1
- package/dist/{command-DbSMcxZL.js → command-nNN0mx65.js} +7 -7
- package/dist/{command-DbSMcxZL.js.map → command-nNN0mx65.js.map} +1 -1
- package/dist/{commands-BU6YJCz9.js → commands-DP13JX9i.js} +24 -12
- package/dist/commands-DP13JX9i.js.map +1 -0
- package/dist/{commands-Bi2SEQtr.js → commands-ZwQUezI1.js} +1 -1
- package/dist/{config-k6m7XbAz.js → config-BjjDryd5.js} +464 -220
- package/dist/config-BjjDryd5.js.map +1 -0
- package/dist/{contrasts-DLZF4dGF.js → contrasts-DSsRZpR8.js} +2 -2
- package/dist/{contrasts-DLZF4dGF.js.map → contrasts-DSsRZpR8.js.map} +1 -1
- package/dist/cursor-catalog-D7bFLEAD.js +143 -0
- package/dist/cursor-catalog-D7bFLEAD.js.map +1 -0
- package/dist/{db-CpsifVln.js → db-B9Ityuit.js} +8 -8
- package/dist/{db-CpsifVln.js.map → db-B9Ityuit.js.map} +1 -1
- package/dist/db-Bk6LWwza.js +2 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/default-effort-DK8gospa.js +91 -0
- package/dist/default-effort-DK8gospa.js.map +1 -0
- package/dist/default-effort-DmXHpqo9.js +1 -0
- package/dist/{docker-YFj4H1Cr.js → docker-BLCNfUie.js} +1 -1
- package/dist/{docker-B4-rx4dA.js → docker-SKpYOlGi.js} +5 -5
- package/dist/{docker-B4-rx4dA.js.map → docker-SKpYOlGi.js.map} +1 -1
- package/dist/{docker-51WvhN_X.js → docker-hwp3EucJ.js} +9 -5
- package/dist/docker-hwp3EucJ.js.map +1 -0
- package/dist/{doctor-BuQnfz4e.js → doctor-CT0Sgjng.js} +16 -16
- package/dist/{doctor-BuQnfz4e.js.map → doctor-CT0Sgjng.js.map} +1 -1
- package/dist/{environment-BOFgzFDg.js → environment-CZdercoj.js} +3 -3
- package/dist/{environment-BOFgzFDg.js.map → environment-CZdercoj.js.map} +1 -1
- package/dist/{errors-UjcLWelF.js → errors-BZVKVKYd.js} +15 -4
- package/dist/errors-BZVKVKYd.js.map +1 -0
- package/dist/{eval-source-Dje3rERU.js → eval-source-nZD7-FSz.js} +9 -8
- package/dist/eval-source-nZD7-FSz.js.map +1 -0
- package/dist/{evaluate-dqXccn4k.d.ts → evaluate-CNi0lZ7n.d.ts} +7 -6
- package/dist/{evaluate-BPP9s3s6.js → evaluate-fhC1PCd_.js} +345 -47
- package/dist/evaluate-fhC1PCd_.js.map +1 -0
- package/dist/{files-Clipx4OW.js → files-BGy05Dlr.js} +4 -4
- package/dist/{files-Clipx4OW.js.map → files-BGy05Dlr.js.map} +1 -1
- package/dist/{files-Bi461Be2.js → files-BIGxpL08.js} +25 -16
- package/dist/{files-Bi461Be2.js.map → files-BIGxpL08.js.map} +1 -1
- package/dist/files-C2RC7K4m.js +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/{files-H24pz0hL.js → files-I1i6Xyh1.js} +4 -4
- package/dist/{files-H24pz0hL.js.map → files-I1i6Xyh1.js.map} +1 -1
- package/dist/files-dJmrfmOM.js +1 -0
- package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-CsVZkNNL.js → git-DR02tHtU.js} +2 -2
- package/dist/{git-CsVZkNNL.js.map → git-DR02tHtU.js.map} +1 -1
- package/dist/{harness-3NVyTSGK.js → harness-ylJLvrbp.js} +39 -5
- package/dist/{harness-3NVyTSGK.js.map → harness-ylJLvrbp.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{images-4mwhNkvh.js → images---X8Wu9d.js} +8 -8
- package/dist/{images-4mwhNkvh.js.map → images---X8Wu9d.js.map} +1 -1
- package/dist/index.d.ts +71 -16
- package/dist/index.js +5 -5
- package/dist/init-BNTXrtMO.js +1 -0
- package/dist/{init-cOk3D44b.js → init-CA95ZnkV.js} +4 -4
- package/dist/{init-cOk3D44b.js.map → init-CA95ZnkV.js.map} +1 -1
- package/dist/{invocations-DW-lTk2f.js → invocations-BdoROAfb.js} +3 -3
- package/dist/{invocations-DW-lTk2f.js.map → invocations-BdoROAfb.js.map} +1 -1
- package/dist/{judge-CKdgEXN7.js → judge-Dp7FPJ1P.js} +3 -3
- package/dist/{judge-CKdgEXN7.js.map → judge-Dp7FPJ1P.js.map} +1 -1
- package/dist/judge.d.ts +2 -2
- package/dist/judge.js +1 -1
- package/dist/{live-XJQ6nqK4.js → live-CBFkdK1K.js} +3 -3
- package/dist/{live-XJQ6nqK4.js.map → live-CBFkdK1K.js.map} +1 -1
- package/dist/{matchers-DT2TaGix.js → matchers-Cv74mJRJ.js} +43 -13
- package/dist/matchers-Cv74mJRJ.js.map +1 -0
- package/dist/{model-Bq3G0OG1.js → model-6dfKFkQd.js} +4 -4
- package/dist/{model-Bq3G0OG1.js.map → model-6dfKFkQd.js.map} +1 -1
- package/dist/{model-CTnyvh7H.js → model-Be7fPvgi.js} +4 -51
- package/dist/model-Be7fPvgi.js.map +1 -0
- package/dist/{model-g3XnzUv0.js → model-CN4ccjNz.js} +1 -1
- package/dist/model-C_5TspZS.js +1 -0
- package/dist/{model-vRiUycWh.js → model-DkJVmVd2.js} +4 -4
- package/dist/{model-vRiUycWh.js.map → model-DkJVmVd2.js.map} +1 -1
- package/dist/{node-BVbbnFtv.js → node-D_Zkwhq2.js} +2 -2
- package/dist/{node-BVbbnFtv.js.map → node-D_Zkwhq2.js.map} +1 -1
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/{opencode-providers-Bz0l7OZI.js → opencode-providers-DPgdqRfg.js} +26 -2
- package/dist/{opencode-providers-Bz0l7OZI.js.map → opencode-providers-DPgdqRfg.js.map} +1 -1
- package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-CR00FWAa.js} +2 -2
- package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-CR00FWAa.js.map} +1 -1
- package/dist/{patch-DKJiC5Sq.js → patch-BJNdOZsf.js} +4 -4
- package/dist/patch-BJNdOZsf.js.map +1 -0
- package/dist/{plan-DdWkM6SW.js → plan-C1UpGZTk.js} +76 -12
- package/dist/plan-C1UpGZTk.js.map +1 -0
- package/dist/{plugins-D-CP6mMd.js → plugins-Cx6ReJWE.js} +24 -3
- package/dist/plugins-Cx6ReJWE.js.map +1 -0
- package/dist/{preflight-9ixuXzsx.js → preflight-BZ3L7P3a.js} +3 -3
- package/dist/{preflight-9ixuXzsx.js.map → preflight-BZ3L7P3a.js.map} +1 -1
- package/dist/{preflight-DTiVv0gY.js → preflight-CUooKgtT.js} +1 -1
- package/dist/{prepare-BfyikQit.js → prepare-BbBoTeDj.js} +7 -7
- package/dist/{prepare-BfyikQit.js.map → prepare-BbBoTeDj.js.map} +1 -1
- package/dist/{prepare-tmBBf4W8.js → prepare-CEZaEwaq.js} +2 -2
- package/dist/{provenance-BizwUn3h.js → provenance-BWvTIlQM.js} +2 -2
- package/dist/{provenance-BizwUn3h.js.map → provenance-BWvTIlQM.js.map} +1 -1
- package/dist/{providers-B6b9Tnyi.js → providers-B3yqakc7.js} +5 -5
- package/dist/{providers-B6b9Tnyi.js.map → providers-B3yqakc7.js.map} +1 -1
- package/dist/{providers-DkOupEBl.js → providers-DdyZW_ru.js} +1 -1
- package/dist/query-BkwQ0W-c.js +1 -0
- package/dist/{query-DyM-VExS.js → query-CZcqpOI9.js} +2 -2
- package/dist/{query-DyM-VExS.js.map → query-CZcqpOI9.js.map} +1 -1
- package/dist/{questions-BRhU8MzB.js → questions-CFU7-crW.js} +20 -19
- package/dist/questions-CFU7-crW.js.map +1 -0
- package/dist/{record-DZBCjuwm.js → record-Bu1Xi6MQ.js} +8 -4
- package/dist/record-Bu1Xi6MQ.js.map +1 -0
- package/dist/{remarks-C7i2LMRb.js → remarks-A3j-SkJ_.js} +42 -8
- package/dist/remarks-A3j-SkJ_.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{remarks-D_3qhBWU.js → remarks-Dj_x9l5n.js} +4 -3
- package/dist/{remarks-D_3qhBWU.js.map → remarks-Dj_x9l5n.js.map} +1 -1
- package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-DFIzcx_6.js → remote-EIFHdMd8.js} +5 -5
- package/dist/{remote-DFIzcx_6.js.map → remote-EIFHdMd8.js.map} +1 -1
- package/dist/{resolve-DWSJpKHX.js → resolve-8H5rz8mi.js} +5 -5
- package/dist/{resolve-DWSJpKHX.js.map → resolve-8H5rz8mi.js.map} +1 -1
- package/dist/resolve-Tm9pX7Hv.js +1 -0
- package/dist/{results-BC3pcG_Z.js → results-E7JdVjq2.js} +9 -9
- package/dist/{results-BC3pcG_Z.js.map → results-E7JdVjq2.js.map} +1 -1
- package/dist/{resume-j-rcIryi.js → resume-Blu4yIcq.js} +4 -4
- package/dist/{resume-j-rcIryi.js.map → resume-Blu4yIcq.js.map} +1 -1
- package/dist/{runtime-d56Gg3J5.js → runtime-CNbsMJQ7.js} +111 -37
- package/dist/runtime-CNbsMJQ7.js.map +1 -0
- package/dist/{sandbox-Cmln11gU.js → sandbox-RButGfIE.js} +2 -2
- package/dist/{sandbox-Cmln11gU.js.map → sandbox-RButGfIE.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-Dqrq-Sia.js +1 -0
- package/dist/{server-BOoH10J8.js → server-XgCYs_MF.js} +35 -11
- package/dist/server-XgCYs_MF.js.map +1 -0
- package/dist/{source-wD04-rRG.js → source-BOkbJCD2.js} +17 -3
- package/dist/{source-wD04-rRG.js.map → source-BOkbJCD2.js.map} +1 -1
- package/dist/source-DooSTtyz.js +1 -0
- package/dist/src-1M0srjji.js +1 -0
- package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
- package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-D91Hd6hT.js → src-BJCKWacl.js} +38 -18
- package/dist/src-BJCKWacl.js.map +1 -0
- package/dist/src-BZlhaDhE.js +2 -0
- package/dist/{src-BO8AYqJ5.js → src-BkhEPVJZ.js} +12 -3
- package/dist/{src-BO8AYqJ5.js.map → src-BkhEPVJZ.js.map} +1 -1
- package/dist/{src-C7upnZSd.js → src-Bp6X4193.js} +1 -1
- package/dist/{src-BYLM3hEz.js → src-C2Japg_I.js} +21 -10
- package/dist/{src-BYLM3hEz.js.map → src-C2Japg_I.js.map} +1 -1
- package/dist/src-C5q3_fRV.js +3 -0
- package/dist/src-C9EHSYyP.js +2 -0
- package/dist/src-CJdpsrXi.js +2 -0
- package/dist/{src-1lpTocbd.js → src-D1ET6oPL.js} +3 -27
- package/dist/{src-1lpTocbd.js.map → src-D1ET6oPL.js.map} +1 -1
- package/dist/{src-Bp4aN3rA.js → src-D4CZjMQ4.js} +1 -1
- package/dist/{src-BYPMM1x7.js → src-GkJ-1Rdy.js} +1 -1
- package/dist/{src-kYeaJJrl.js → src-OOKE-5GA.js} +1 -1
- package/dist/state-BCyNaz13.js +1 -0
- package/dist/{state-DrJ687ut.js → state-D6-tvqtp.js} +8 -8
- package/dist/{state-DrJ687ut.js.map → state-D6-tvqtp.js.map} +1 -1
- package/dist/suite-BiWajSlA.js +2 -0
- package/dist/{suite-Co8cwelb.js → suite-DYI1k2Ie.js} +22 -5
- package/dist/suite-DYI1k2Ie.js.map +1 -0
- package/dist/{summary-DLFHVoux.js → summary-CN2aqAUz.js} +59 -17
- package/dist/summary-CN2aqAUz.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +96 -44
- package/dist/testing.js +4 -4
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
- package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
- package/dist/{types-coMWHaU2.d.ts → types-DDeCnNmp.d.ts} +437 -181
- package/dist/view/index.d.ts +34 -5
- package/dist/view/index.js +8 -7
- package/dist/views-7bNY17Z6.js +2 -0
- package/dist/{views-BFO1BLKu.js → views-Bd-H71_t.js} +5 -5
- package/dist/{views-BFO1BLKu.js.map → views-Bd-H71_t.js.map} +1 -1
- package/dist/{worker-entry-CJtdsKBP.js → worker-entry-BIHJqE_z.js} +34 -13
- package/dist/worker-entry-BIHJqE_z.js.map +1 -0
- package/dist/workspace-CxwHv8mG.js +1 -0
- package/dist/{workspace-BRxjfa8N.js → workspace-Sy5g8zW9.js} +4 -4
- package/dist/{workspace-BRxjfa8N.js.map → workspace-Sy5g8zW9.js.map} +1 -1
- package/package.json +1 -1
- package/skills/eval-rubric/SKILL.md +11 -0
- package/dist/app-files-ChbvRx3P.js.map +0 -1
- package/dist/assemble-SKQGW_QF.js.map +0 -1
- package/dist/assemble-XBWuK3Ed.js +0 -1
- package/dist/attempt-host-BOp8JklD.js.map +0 -1
- package/dist/capture-BM_3100g.js.map +0 -1
- package/dist/capture-DSK8O5bA.js +0 -1
- package/dist/cli-PdYrcoKM.js.map +0 -1
- package/dist/cli-spec-D0qyrPZi.js.map +0 -1
- package/dist/client-Ct6s6OnF.js.map +0 -1
- package/dist/colors-QV9dFVhN.js.map +0 -1
- package/dist/commands-BU6YJCz9.js.map +0 -1
- package/dist/config-k6m7XbAz.js.map +0 -1
- package/dist/db-DkD-F-X-.js +0 -2
- package/dist/debug-BhizgAsU.js.map +0 -1
- package/dist/debug-DeM5pswK.js +0 -1
- package/dist/docker-51WvhN_X.js.map +0 -1
- package/dist/errors-UjcLWelF.js.map +0 -1
- package/dist/eval-source-Dje3rERU.js.map +0 -1
- package/dist/evaluate-BPP9s3s6.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-C2h2Cw4F.js +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-DmUrRmmL.js +0 -1
- package/dist/init-BDEOopK7.js +0 -1
- package/dist/matchers-DT2TaGix.js.map +0 -1
- package/dist/model-CTnyvh7H.js.map +0 -1
- package/dist/model-D-SWrLZG.js +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/pairs-puYi-zj2.js.map +0 -1
- package/dist/patch-DKJiC5Sq.js.map +0 -1
- package/dist/plan-DdWkM6SW.js.map +0 -1
- package/dist/plugins-D-CP6mMd.js.map +0 -1
- package/dist/query-Clm2mNSm.js +0 -1
- package/dist/questions-BRhU8MzB.js.map +0 -1
- package/dist/record-DZBCjuwm.js.map +0 -1
- package/dist/remarks-BwJ8pcTQ.js +0 -1
- package/dist/remarks-C7i2LMRb.js.map +0 -1
- package/dist/remarks-dA8LyoPc.js.map +0 -1
- package/dist/resolve--rQZ8u6Z.js +0 -1
- package/dist/runtime-d56Gg3J5.js.map +0 -1
- package/dist/server-BOoH10J8.js.map +0 -1
- package/dist/server-DpuVAbrO.js +0 -1
- package/dist/source-BFd6f0-R.js +0 -1
- package/dist/src-6_zz7oJs.js +0 -2
- package/dist/src-D91Hd6hT.js.map +0 -1
- package/dist/src-DH-9IO7f.js +0 -1
- package/dist/src-HSxBazcg.js +0 -3
- package/dist/src-LD1BAJFa.js +0 -2
- package/dist/src-dlX-Lmxg.js +0 -2
- package/dist/state-B5PLm5qY.js +0 -1
- package/dist/suite-22Kt9Grx.js +0 -2
- package/dist/suite-Co8cwelb.js.map +0 -1
- package/dist/summary-DLFHVoux.js.map +0 -1
- package/dist/views-BOf8qEL8.js +0 -2
- package/dist/worker-entry-CJtdsKBP.js.map +0 -1
- package/dist/workspace-DDHzkMjm.js +0 -1
|
@@ -1,36 +1,40 @@
|
|
|
1
|
-
import { t as isObjectLike } from "./object-
|
|
2
|
-
import { r as withLease } from "./client-
|
|
1
|
+
import { t as isObjectLike } from "./object-BkXGPKWY.js";
|
|
2
|
+
import { r as withLease } from "./client-DMmOkbRA.js";
|
|
3
3
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
|
|
4
4
|
import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
|
|
5
|
-
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-
|
|
6
|
-
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-
|
|
5
|
+
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-CZdercoj.js";
|
|
6
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-Cx6ReJWE.js";
|
|
7
7
|
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
|
|
8
|
-
import {
|
|
9
|
-
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-
|
|
8
|
+
import { A as expand, B as refuseIllegalCompare, Bt as artifactRefsOf, C as resolveGateway, Dt as refusedMcp, E as cascade, Et as layoutOf, F as plainHeaderError, G as secretVarsIn, H as sandboxId, Ht as decideState, I as plainHeaders, J as mergeJudge, K as unbrokered, Lt as judgeProblem, Mt as NO_JUDGE_INPUT, N as majorityOf, O as checkOptions, Ot as refusedPlugins, P as matchesTags, Pt as inputsOn, Q as allowedHosts, St as claudeMcpServers, T as brokeredIn, U as sandboxProvider, Ut as filePartsOf, V as runsOf, Vt as badJudgeArtifacts, W as sandboxSetBy, Wt as resolveJudgeArtifacts, Y as resolveStateOptions, _ as fellBackToDirect, b as resolveModelGateway, it as unenforceable, j as harnessId, jt as INPUT_KEYS, k as evalPlan, kt as writePluginFiles, ot as createFakeSandbox, q as unbrokeredError, rt as isDirect, tt as egressLabel, v as gatewayScope, wt as expectedBlockedFor, xt as brokeringFor, yt as agentMcp } from "./config-BjjDryd5.js";
|
|
9
|
+
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-Dp7FPJ1P.js";
|
|
10
10
|
import { v as vertexSettings } from "./vertex-DypYjx71.js";
|
|
11
|
-
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
11
|
+
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-DVIaahLv.js";
|
|
12
12
|
import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
|
|
13
13
|
import { t as inCI } from "./env-CARj5Axq.js";
|
|
14
|
-
import {
|
|
14
|
+
import { pt as shellQuote, t as MCP_TIMER, ut as OPAQUE_OUTPUT } from "./src-BJCKWacl.js";
|
|
15
15
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
16
|
-
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-
|
|
17
|
-
import {
|
|
18
|
-
import { t as
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
16
|
+
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CNbsMJQ7.js";
|
|
17
|
+
import { c as formatBytes } from "./debug-CgG3T071.js";
|
|
18
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CpbZ9arV.js";
|
|
19
|
+
import { t as attemptDebugDir } from "./files-D1y8Pg4Z.js";
|
|
20
|
+
import { r as openRouterFallbacks } from "./openrouter-models-CR00FWAa.js";
|
|
21
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-DheNqeYS.js";
|
|
21
22
|
import { t as contextTaker } from "./context-Dgn48BqT.js";
|
|
22
|
-
import { a as skillFiles, i as resolveEnvironment } from "./resolve-
|
|
23
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-8H5rz8mi.js";
|
|
23
24
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
24
|
-
import { i as preparedWorkspace } from "./workspace-
|
|
25
|
+
import { i as preparedWorkspace } from "./workspace-Sy5g8zW9.js";
|
|
25
26
|
import { t as patchPath } from "./patch-CquGAGX5.js";
|
|
26
|
-
import { n as judgeDiff } from "./patch-
|
|
27
|
-
import { t as transcriptForJudge } from "./transcript-
|
|
28
|
-
import { i as
|
|
27
|
+
import { n as judgeDiff } from "./patch-BJNdOZsf.js";
|
|
28
|
+
import { t as transcriptForJudge } from "./transcript-BJrKfgRW.js";
|
|
29
|
+
import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CHcahyNK.js";
|
|
30
|
+
import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-nZD7-FSz.js";
|
|
31
|
+
import { i as withDefaultEffort } from "./default-effort-DK8gospa.js";
|
|
29
32
|
import { dirname, join } from "node:path";
|
|
30
33
|
import { parse, stringify } from "yaml";
|
|
31
34
|
import { z } from "zod";
|
|
32
35
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
33
|
-
import {
|
|
36
|
+
import { gunzipSync, gzipSync } from "node:zlib";
|
|
37
|
+
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
34
38
|
//#region src/attempt/assertions.ts
|
|
35
39
|
const MESSAGE_LIMIT = 1024;
|
|
36
40
|
const LABEL_LIMIT = 60;
|
|
@@ -76,6 +80,7 @@ function assertionRecords(raw) {
|
|
|
76
80
|
matcher: r.matcher,
|
|
77
81
|
status: r.status,
|
|
78
82
|
soft: r.soft,
|
|
83
|
+
...r.at ? { at: r.at } : {},
|
|
79
84
|
...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
|
|
80
85
|
...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
|
|
81
86
|
};
|
|
@@ -84,7 +89,7 @@ function assertionRecords(raw) {
|
|
|
84
89
|
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
85
90
|
/**
|
|
86
91
|
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
87
|
-
* that didn't pass, an `expect
|
|
92
|
+
* that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
|
|
88
93
|
* `CAUGHT_HINT` after its message.
|
|
89
94
|
*/
|
|
90
95
|
function checkFailure(raw) {
|
|
@@ -436,7 +441,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
436
441
|
//#region src/harness/index.ts
|
|
437
442
|
async function createHarness(descriptor, vertex) {
|
|
438
443
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
439
|
-
const { createAgentHarness } = await import("./agent-
|
|
444
|
+
const { createAgentHarness } = await import("./agent-BtAbRTFq.js");
|
|
440
445
|
return createAgentHarness(descriptor, vertex);
|
|
441
446
|
}
|
|
442
447
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
@@ -466,8 +471,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
466
471
|
if (debug) traceEnvironment(debug, spec, env);
|
|
467
472
|
record.image = null;
|
|
468
473
|
if (spec.sandbox.kind === "docker") {
|
|
469
|
-
const { preparedFor } = await import("./preflight-
|
|
470
|
-
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-
|
|
474
|
+
const { preparedFor } = await import("./preflight-CUooKgtT.js");
|
|
475
|
+
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DdyZW_ru.js");
|
|
471
476
|
signal.throwIfAborted();
|
|
472
477
|
const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
|
|
473
478
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -536,16 +541,16 @@ function describeSize(state, question) {
|
|
|
536
541
|
return `${p.name} ${k(tokensOf(p.chars))}${top.length ? `, of which ${top.join(", ")}` : ""}`;
|
|
537
542
|
}).join("; ");
|
|
538
543
|
}
|
|
539
|
-
/** `Narrow
|
|
544
|
+
/** `Narrow diff, transcript or output.`: the inputs that were on, largest first. */
|
|
540
545
|
function stateHint(state, question) {
|
|
541
|
-
const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) =>
|
|
546
|
+
const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => p.name);
|
|
542
547
|
if (!keys.length) return "";
|
|
543
548
|
const last = keys.pop();
|
|
544
549
|
return ` Narrow ${keys.length ? `${keys.join(", ")} or ${last}` : last}.`;
|
|
545
550
|
}
|
|
546
551
|
/**
|
|
547
552
|
* A provider's error, as the provider worded it, then what the judge's input held: its estimated
|
|
548
|
-
* size and biggest parts and files, and the
|
|
553
|
+
* size and biggest parts and files, and the inputs that were on, largest first. reeval knows
|
|
549
554
|
* no model's limit, so this follows every provider error.
|
|
550
555
|
*/
|
|
551
556
|
const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
|
|
@@ -585,7 +590,7 @@ const workspacePath = (path) => path.replace(/^\.\//, "");
|
|
|
585
590
|
/** A `contents` entry that names one file rather than matching many. */
|
|
586
591
|
const isExactPath = (entry) => !entry.startsWith("!") && !/[*?[\]{}]/.test(entry);
|
|
587
592
|
/**
|
|
588
|
-
* The judge's state for
|
|
593
|
+
* The judge's state for the inputs the call turned on: the graded turn's task and output (or every turn's),
|
|
589
594
|
* its changed files and diff as `judgeDiff` shows them, files whole, `c.exec` results and the
|
|
590
595
|
* transcript.
|
|
591
596
|
*/
|
|
@@ -633,7 +638,7 @@ function checksShown(which, ran) {
|
|
|
633
638
|
if (which === false) return void 0;
|
|
634
639
|
if (which !== true) {
|
|
635
640
|
const missing = which.find((command) => !ran.some((r) => r.command === command));
|
|
636
|
-
if (missing !== void 0) throw new Error(`c.judge()
|
|
641
|
+
if (missing !== void 0) throw new Error(`c.judge() checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
|
|
637
642
|
}
|
|
638
643
|
return ran.filter((r) => which === true || which.includes(r.command)).map(({ command, exitCode, stdout, stderr }) => ({
|
|
639
644
|
command,
|
|
@@ -657,24 +662,24 @@ function placeFor(calls, order) {
|
|
|
657
662
|
* checks after it still run.
|
|
658
663
|
*/
|
|
659
664
|
function judgeCaller(a, readFile) {
|
|
660
|
-
const ask = async (order, question, { on, ...call }
|
|
661
|
-
if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
|
|
665
|
+
const ask = async (order, question, { on, ...call }) => {
|
|
662
666
|
const problem = judgeProblem(call, { at: "call" });
|
|
663
667
|
if (problem) throw new Error(problem);
|
|
664
668
|
const badArts = badJudgeArtifacts(call.artifacts);
|
|
665
669
|
if (badArts) throw new Error(`c.judge() artifacts ${badArts}`);
|
|
666
670
|
const subject = on && "turn" in on ? a.turns[on.turn] : a.turns.at(-1);
|
|
667
671
|
if (!subject) throw new Error(on && "turn" in on ? `c.judge() on names turn ${on.turn}, but this attempt has ${a.turns.length}` : "c.judge() has no turn to grade: call c.send() first");
|
|
668
|
-
const
|
|
672
|
+
const wanted = resolveStateOptions(call);
|
|
673
|
+
if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives a value to grade as the output, but output is off. Pass output: true.");
|
|
674
|
+
const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, artifacts, ...callJudge } = call;
|
|
675
|
+
const options = mergeJudge(a.spec.judge, callJudge);
|
|
669
676
|
const { model } = options;
|
|
670
677
|
if (model === void 0) {
|
|
671
678
|
const message = noJudgeModel(getRunContext().config.file);
|
|
672
679
|
a.judgeThrown.add(message);
|
|
673
680
|
throw new Error(message);
|
|
674
681
|
}
|
|
675
|
-
const
|
|
676
|
-
if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
|
|
677
|
-
const { state: _state, maxConcurrent, ...judgeOptions } = options;
|
|
682
|
+
const { maxConcurrent, ...judgeOptions } = options;
|
|
678
683
|
const output = on && "value" in on ? on.value : subject.text;
|
|
679
684
|
const { draft, dropped } = await draftState({
|
|
680
685
|
turns: a.turns,
|
|
@@ -687,7 +692,8 @@ function judgeCaller(a, readFile) {
|
|
|
687
692
|
});
|
|
688
693
|
const name = judgeModelName(model);
|
|
689
694
|
const state = redactValue(stateOf(draft));
|
|
690
|
-
const resolved = resolveJudgeArtifacts(
|
|
695
|
+
const resolved = resolveJudgeArtifacts(artifacts, a.artifactBlobs);
|
|
696
|
+
if (artifacts === true && !resolved.length && inputsOn(call, INPUT_KEYS).length === 1) throw new Error("c.judge() artifacts: true shows the judge nothing yet: this attempt has no captures. Call c.capture() first, or turn on another input.");
|
|
691
697
|
const refs = artifactRefsOf(resolved);
|
|
692
698
|
const decide = decideState(state, filePartsOf(resolved));
|
|
693
699
|
const input = {
|
|
@@ -789,7 +795,10 @@ function judgeCaller(a, readFile) {
|
|
|
789
795
|
return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
|
|
790
796
|
}
|
|
791
797
|
};
|
|
792
|
-
return async (question,
|
|
798
|
+
return async (question, given) => {
|
|
799
|
+
if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
|
|
800
|
+
if (given === null || typeof given !== "object" || Array.isArray(given)) throw new Error(judgeProblem(given, { at: "call" }) ?? NO_JUDGE_INPUT);
|
|
801
|
+
const { on, ...options } = given;
|
|
793
802
|
const order = a.judgeCalls++;
|
|
794
803
|
const call = on === void 0 ? options : {
|
|
795
804
|
...options,
|
|
@@ -805,7 +814,7 @@ async function createSandbox(descriptor, options) {
|
|
|
805
814
|
switch (descriptor.kind) {
|
|
806
815
|
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
807
816
|
case "docker": {
|
|
808
|
-
const { createDockerSandbox } = await import("./docker-
|
|
817
|
+
const { createDockerSandbox } = await import("./docker-BLCNfUie.js");
|
|
809
818
|
return await createDockerSandbox(descriptor, options);
|
|
810
819
|
}
|
|
811
820
|
}
|
|
@@ -849,8 +858,8 @@ function isJudges(a, error, raw) {
|
|
|
849
858
|
}
|
|
850
859
|
/**
|
|
851
860
|
* The error that fails the attempt: a judge's gives way to the first other check that didn't hold
|
|
852
|
-
* (an `expect
|
|
853
|
-
* as that check.
|
|
861
|
+
* (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
|
|
862
|
+
* fails as that check.
|
|
854
863
|
*/
|
|
855
864
|
function attemptError(a, error, raw) {
|
|
856
865
|
if (!isJudges(a, error, raw)) return error;
|
|
@@ -954,7 +963,7 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
954
963
|
const { record, spec } = a;
|
|
955
964
|
const recordPath = join(stagingDir, "evals", recordFileName(attemptKey(id, spec.cell.key, spec.n)));
|
|
956
965
|
await mkdir(join(stagingDir, "evals"), { recursive: true });
|
|
957
|
-
await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
|
|
966
|
+
await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), new Map([...a.artifactBlobs, ...a.stepBlobs]), a.capturePages);
|
|
958
967
|
const tmp = `${recordPath}.${process.pid}.tmp`;
|
|
959
968
|
await writeFile(tmp, stagedRecordJson(record));
|
|
960
969
|
await rename(tmp, recordPath);
|
|
@@ -968,12 +977,23 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
968
977
|
question: clip(j.question, 200)
|
|
969
978
|
})),
|
|
970
979
|
turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
|
|
980
|
+
...failedChecks(record),
|
|
971
981
|
...record.error?.prepare ? { prepare: {
|
|
972
982
|
ref: record.error.prepare.ref,
|
|
973
983
|
stage: record.error.prepare.stage
|
|
974
984
|
} } : {}
|
|
975
985
|
};
|
|
976
986
|
}
|
|
987
|
+
/** The record's checks that didn't pass, for the terminal; unset when all did. */
|
|
988
|
+
function failedChecks(record) {
|
|
989
|
+
const failed = (record.assertions ?? []).filter((x) => x.status !== "pass");
|
|
990
|
+
if (!failed.length) return {};
|
|
991
|
+
return { failedChecks: failed.map((x) => ({
|
|
992
|
+
label: x.label,
|
|
993
|
+
...x.at ? { at: x.at } : {},
|
|
994
|
+
message: clip(x.message?.split("\n")[0] ?? "", 500)
|
|
995
|
+
})) };
|
|
996
|
+
}
|
|
977
997
|
/** `turn N error: <message>` for each turn with an error event. */
|
|
978
998
|
function turnErrors(transcripts) {
|
|
979
999
|
return transcripts.flatMap((events, i) => {
|
|
@@ -1190,7 +1210,7 @@ const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
|
|
|
1190
1210
|
//#region src/attempt/state.ts
|
|
1191
1211
|
/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
|
|
1192
1212
|
const SETUP_BUDGET_MS = 12e5;
|
|
1193
|
-
function createAttempt(spec, signal, progress, start) {
|
|
1213
|
+
function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
1194
1214
|
const record = {
|
|
1195
1215
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1196
1216
|
outcome: "pass",
|
|
@@ -1245,7 +1265,10 @@ function createAttempt(spec, signal, progress, start) {
|
|
|
1245
1265
|
judgeInputCalls: [],
|
|
1246
1266
|
captureNames: /* @__PURE__ */ new Set(),
|
|
1247
1267
|
backgrounds: [],
|
|
1248
|
-
artifactBlobs: /* @__PURE__ */ new Map()
|
|
1268
|
+
artifactBlobs: /* @__PURE__ */ new Map(),
|
|
1269
|
+
capturePages: /* @__PURE__ */ new Map(),
|
|
1270
|
+
stepBlobs: /* @__PURE__ */ new Map(),
|
|
1271
|
+
snapshotDir
|
|
1249
1272
|
};
|
|
1250
1273
|
return a;
|
|
1251
1274
|
}
|
|
@@ -1638,6 +1661,237 @@ async function startSession(a, runId) {
|
|
|
1638
1661
|
return [box, session];
|
|
1639
1662
|
}
|
|
1640
1663
|
//#endregion
|
|
1664
|
+
//#region src/attempt/snapshot.ts
|
|
1665
|
+
/** The most a snapshot directory's files may add up to; a larger one isn't taken. */
|
|
1666
|
+
const SNAPSHOT_DIR_LIMIT = 50 * 2 ** 20;
|
|
1667
|
+
/** The most a compressed snapshot may be. */
|
|
1668
|
+
const SNAPSHOT_TAR_LIMIT = 10 * 2 ** 20;
|
|
1669
|
+
/** The most a compressed snapshot may be uncompressed, which the host holds in memory to mask. */
|
|
1670
|
+
const SNAPSHOT_RAW_LIMIT = 200 * 2 ** 20;
|
|
1671
|
+
/** How long one snapshot may take before the attempt goes on without it. */
|
|
1672
|
+
const SNAPSHOT_TIMEOUT_MS = 12e4;
|
|
1673
|
+
/** Installed dependencies, caches and git's objects: left out by directory name, at any depth. */
|
|
1674
|
+
const EXCLUDED_DIRS = [
|
|
1675
|
+
".git",
|
|
1676
|
+
"node_modules",
|
|
1677
|
+
".pnpm-store",
|
|
1678
|
+
".venv",
|
|
1679
|
+
"venv",
|
|
1680
|
+
".tox",
|
|
1681
|
+
"__pycache__",
|
|
1682
|
+
".pytest_cache",
|
|
1683
|
+
".mypy_cache",
|
|
1684
|
+
".ruff_cache",
|
|
1685
|
+
".next",
|
|
1686
|
+
".nuxt",
|
|
1687
|
+
".svelte-kit",
|
|
1688
|
+
".turbo",
|
|
1689
|
+
".parcel-cache",
|
|
1690
|
+
".cache",
|
|
1691
|
+
".dart_tool",
|
|
1692
|
+
".gradle"
|
|
1693
|
+
];
|
|
1694
|
+
const LISTED = 50;
|
|
1695
|
+
const SIZE_MARK = "--reeval-size--";
|
|
1696
|
+
const excluded = (path) => path.split("/").some((part) => EXCLUDED_DIRS.includes(part));
|
|
1697
|
+
const utf8 = new TextDecoder("utf-8", { fatal: true });
|
|
1698
|
+
function asText(data) {
|
|
1699
|
+
if (data.subarray(0, 8192).includes(0)) return null;
|
|
1700
|
+
try {
|
|
1701
|
+
return utf8.decode(data);
|
|
1702
|
+
} catch {
|
|
1703
|
+
return null;
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
/** The sandbox's archive, with the excluded directories gone and secret-looking values masked. */
|
|
1707
|
+
function cleanArchive(tar) {
|
|
1708
|
+
const entries = [];
|
|
1709
|
+
const masked = [];
|
|
1710
|
+
const dropped = [];
|
|
1711
|
+
const left = /* @__PURE__ */ new Set();
|
|
1712
|
+
for (const e of readTar(tar)) {
|
|
1713
|
+
if (excluded(e.path)) {
|
|
1714
|
+
const parts = e.path.split("/");
|
|
1715
|
+
const at = parts.findIndex((p) => EXCLUDED_DIRS.includes(p));
|
|
1716
|
+
left.add(redactSecrets(`${parts.slice(0, at + 1).join("/")}/`));
|
|
1717
|
+
continue;
|
|
1718
|
+
}
|
|
1719
|
+
if (findSecrets(e.path).length || e.type === "symlink" && findSecrets(e.target).length) {
|
|
1720
|
+
dropped.push(redactSecrets(e.path));
|
|
1721
|
+
continue;
|
|
1722
|
+
}
|
|
1723
|
+
if (e.type !== "file") {
|
|
1724
|
+
entries.push(e);
|
|
1725
|
+
continue;
|
|
1726
|
+
}
|
|
1727
|
+
const text = asText(e.data);
|
|
1728
|
+
if (text === null) {
|
|
1729
|
+
if (findSecrets(e.data.toString("latin1")).length) dropped.push(e.path);
|
|
1730
|
+
else entries.push(e);
|
|
1731
|
+
continue;
|
|
1732
|
+
}
|
|
1733
|
+
const clean = redactSecrets(text);
|
|
1734
|
+
if (clean === text) entries.push(e);
|
|
1735
|
+
else {
|
|
1736
|
+
masked.push(e.path);
|
|
1737
|
+
entries.push({
|
|
1738
|
+
...e,
|
|
1739
|
+
data: Buffer.from(clean)
|
|
1740
|
+
});
|
|
1741
|
+
}
|
|
1742
|
+
}
|
|
1743
|
+
return {
|
|
1744
|
+
entries,
|
|
1745
|
+
masked,
|
|
1746
|
+
dropped,
|
|
1747
|
+
excluded: [...left]
|
|
1748
|
+
};
|
|
1749
|
+
}
|
|
1750
|
+
const tarCommand = (workdir) => {
|
|
1751
|
+
const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
|
|
1752
|
+
return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
|
|
1753
|
+
};
|
|
1754
|
+
const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
|
|
1755
|
+
const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
|
|
1756
|
+
const listed = (paths) => paths.slice(0, LISTED);
|
|
1757
|
+
const LIMITS = {
|
|
1758
|
+
dir: SNAPSHOT_DIR_LIMIT,
|
|
1759
|
+
tar: SNAPSHOT_TAR_LIMIT,
|
|
1760
|
+
raw: SNAPSHOT_RAW_LIMIT
|
|
1761
|
+
};
|
|
1762
|
+
/**
|
|
1763
|
+
* Archives the sandbox's workspace and returns what to record and the masked entries to write,
|
|
1764
|
+
* or why none: over its form's cap, or an archive the sandbox couldn't make.
|
|
1765
|
+
*/
|
|
1766
|
+
async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
|
|
1767
|
+
const none = (cause, message, dirs = []) => ({
|
|
1768
|
+
taken: {
|
|
1769
|
+
bytes: 0,
|
|
1770
|
+
files: 0,
|
|
1771
|
+
excluded: listed(dirs),
|
|
1772
|
+
masked: [],
|
|
1773
|
+
dropped: [],
|
|
1774
|
+
skipped: {
|
|
1775
|
+
cause,
|
|
1776
|
+
message
|
|
1777
|
+
}
|
|
1778
|
+
},
|
|
1779
|
+
entries: null
|
|
1780
|
+
});
|
|
1781
|
+
const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
|
|
1782
|
+
const tar = tarCommand(box.workdir);
|
|
1783
|
+
const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
|
|
1784
|
+
const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
|
|
1785
|
+
const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
|
|
1786
|
+
const rawBytes = Number(raw.trim());
|
|
1787
|
+
const gzBytes = Number(size.trim());
|
|
1788
|
+
if (sized.exitCode !== 0 || !(rawBytes > 0) || !(gzBytes > 0)) return none("error", `couldn't archive the workspace: ${(sized.stderr || sized.stdout).trim()}`);
|
|
1789
|
+
const without = dirs.length ? dirs.join(", ") : "its dependency and cache directories";
|
|
1790
|
+
const rawLimit = compress ? limits.raw : limits.dir;
|
|
1791
|
+
if (rawBytes > rawLimit) return none("size", `the workspace is ${formatBytes(rawBytes)} without ${without}, over the ${formatBytes(rawLimit)} cap${compress ? "" : ` (compressed, the cap is ${formatBytes(limits.tar)})`}`, dirs);
|
|
1792
|
+
if (compress && gzBytes > limits.tar) return none("size", `the workspace is ${formatBytes(gzBytes)} compressed without ${without}, over the ${formatBytes(limits.tar)} cap`, dirs);
|
|
1793
|
+
const r = await box.execOwn(`${NEEDS_TAR}; echo ${OPAQUE_OUTPUT}; ${tar} | { ${GZIP}; } | base64`, { signal });
|
|
1794
|
+
const encoded = r.stdout.startsWith(`--opaque-output--\n`) ? r.stdout.slice(OPAQUE_OUTPUT.length + 1) : "";
|
|
1795
|
+
const fetched = Buffer.from(encoded, "base64");
|
|
1796
|
+
if (r.exitCode !== 0 || fetched.length === 0) return none("error", `couldn't archive the workspace: ${r.stderr.trim() || "no output"}`);
|
|
1797
|
+
const clean = cleanArchive(fetched[0] === 31 && fetched[1] === 139 ? gunzipSync(fetched, { maxOutputLength: rawLimit + 2 ** 20 }) : fetched);
|
|
1798
|
+
const all = [.../* @__PURE__ */ new Set([...dirs, ...clean.excluded])].sort();
|
|
1799
|
+
return {
|
|
1800
|
+
taken: {
|
|
1801
|
+
bytes: 0,
|
|
1802
|
+
files: clean.entries.filter((e) => e.type !== "dir").length,
|
|
1803
|
+
excluded: listed(all),
|
|
1804
|
+
masked: listed(clean.masked),
|
|
1805
|
+
dropped: listed(clean.dropped),
|
|
1806
|
+
skipped: null
|
|
1807
|
+
},
|
|
1808
|
+
entries: clean.entries
|
|
1809
|
+
};
|
|
1810
|
+
}
|
|
1811
|
+
/** Where an attempt's snapshots wait, beside its staging record, until the run is assembled. */
|
|
1812
|
+
const snapshotSidecar = (stagingDir, key) => join(stagingDir, "evals", recordFileName(key).replace(/\.json$/, ".snapshots"));
|
|
1813
|
+
/**
|
|
1814
|
+
* Takes snapshot `name` of the workspace into the attempt's sidecar and adds it to the record;
|
|
1815
|
+
* never fails the attempt. Adds nothing when the attempt's signal stopped it.
|
|
1816
|
+
*/
|
|
1817
|
+
async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SNAPSHOT_TIMEOUT_MS) {
|
|
1818
|
+
const { compress } = a.spec.snapshot;
|
|
1819
|
+
const form = compress ? "tar.gz" : "dir";
|
|
1820
|
+
const dir = a.snapshotDir;
|
|
1821
|
+
const target = join(dir, compress ? `${name}.tar.gz` : name);
|
|
1822
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
1823
|
+
const add = (taken) => {
|
|
1824
|
+
a.record.snapshots ??= [];
|
|
1825
|
+
a.record.snapshots.push({
|
|
1826
|
+
name,
|
|
1827
|
+
form,
|
|
1828
|
+
path: null,
|
|
1829
|
+
...taken
|
|
1830
|
+
});
|
|
1831
|
+
};
|
|
1832
|
+
const failed = (message) => {
|
|
1833
|
+
debugLog(phase, `no ${name} snapshot: ${message}`);
|
|
1834
|
+
add({
|
|
1835
|
+
bytes: 0,
|
|
1836
|
+
files: 0,
|
|
1837
|
+
excluded: [],
|
|
1838
|
+
masked: [],
|
|
1839
|
+
dropped: [],
|
|
1840
|
+
skipped: {
|
|
1841
|
+
cause: "error",
|
|
1842
|
+
message: `couldn't archive the workspace: ${message}`
|
|
1843
|
+
}
|
|
1844
|
+
});
|
|
1845
|
+
};
|
|
1846
|
+
try {
|
|
1847
|
+
const { taken, entries } = await archiveWorkspace(box, AbortSignal.any([a.signal, timeout]), compress, limits);
|
|
1848
|
+
if (!entries) {
|
|
1849
|
+
debugLog(phase, `no ${name} snapshot: ${taken.skipped?.message}`);
|
|
1850
|
+
add(taken);
|
|
1851
|
+
return;
|
|
1852
|
+
}
|
|
1853
|
+
await rm(target, {
|
|
1854
|
+
recursive: true,
|
|
1855
|
+
force: true
|
|
1856
|
+
});
|
|
1857
|
+
await mkdir(dir, { recursive: true });
|
|
1858
|
+
if (compress) {
|
|
1859
|
+
const archive = gzipSync(writeTar(entries));
|
|
1860
|
+
if (archive.length > limits.tar) {
|
|
1861
|
+
const message = `the workspace is ${formatBytes(archive.length)} compressed, over the ${formatBytes(limits.tar)} cap`;
|
|
1862
|
+
debugLog(phase, `no ${name} snapshot: ${message}`);
|
|
1863
|
+
add({
|
|
1864
|
+
...taken,
|
|
1865
|
+
files: 0,
|
|
1866
|
+
skipped: {
|
|
1867
|
+
cause: "size",
|
|
1868
|
+
message
|
|
1869
|
+
}
|
|
1870
|
+
});
|
|
1871
|
+
return;
|
|
1872
|
+
}
|
|
1873
|
+
await writeFile(target, archive);
|
|
1874
|
+
taken.bytes = archive.length;
|
|
1875
|
+
} else {
|
|
1876
|
+
const skipped = await extractTar(entries, target);
|
|
1877
|
+
taken.bytes = entries.reduce((n, e) => n + (e.type === "file" ? e.data.length : 0), 0);
|
|
1878
|
+
taken.dropped = listed([...taken.dropped, ...skipped]);
|
|
1879
|
+
}
|
|
1880
|
+
const masked = taken.masked.length ? ` · masked ${taken.masked.length}` : "";
|
|
1881
|
+
const left = taken.excluded.length ? ` · left out ${taken.excluded.join(", ")}` : "";
|
|
1882
|
+
debugLog(phase, `${name} snapshot: ${taken.files} files, ${formatBytes(taken.bytes)} (${form})${masked}${left}`);
|
|
1883
|
+
add(taken);
|
|
1884
|
+
} catch (error) {
|
|
1885
|
+
await rm(target, {
|
|
1886
|
+
recursive: true,
|
|
1887
|
+
force: true
|
|
1888
|
+
}).catch(() => {});
|
|
1889
|
+
if (a.signal.aborted) return;
|
|
1890
|
+
if (timeout.aborted) return failed(`it took over ${Math.round(timeoutMs / 1e3)}s`);
|
|
1891
|
+
failed(error instanceof Error ? error.message : String(error));
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
//#endregion
|
|
1641
1895
|
//#region src/attempt/turns.ts
|
|
1642
1896
|
/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
|
|
1643
1897
|
const TURN_STOP_GRACE_MS = 3e3;
|
|
@@ -1820,6 +2074,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
1820
2074
|
a.signal.throwIfAborted();
|
|
1821
2075
|
const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
|
|
1822
2076
|
a.record.diff = diff;
|
|
2077
|
+
if (a.spec.snapshot.when === "turns") await takeSnapshot(a, box, `turn-${turnNumber}`, "harness");
|
|
1823
2078
|
const data = {
|
|
1824
2079
|
prompt,
|
|
1825
2080
|
...result,
|
|
@@ -1863,7 +2118,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1863
2118
|
} finally {
|
|
1864
2119
|
clock?.resume();
|
|
1865
2120
|
}
|
|
1866
|
-
});
|
|
2121
|
+
}, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
|
|
1867
2122
|
const destroy = () => void a.sandbox?.destroy();
|
|
1868
2123
|
testSignal.addEventListener("abort", destroy, { once: true });
|
|
1869
2124
|
const takeContext = contextTaker(a);
|
|
@@ -1932,6 +2187,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1932
2187
|
if (a.session) debug.log("teardown", "harness session closed");
|
|
1933
2188
|
recordEgress(a, sandbox);
|
|
1934
2189
|
oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
|
|
2190
|
+
if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
|
|
1935
2191
|
await sandbox?.destroy();
|
|
1936
2192
|
if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
|
|
1937
2193
|
const { record } = a;
|
|
@@ -1953,6 +2209,30 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1953
2209
|
if (failure) throw failure.error;
|
|
1954
2210
|
}
|
|
1955
2211
|
//#endregion
|
|
2212
|
+
//#region src/best-of.ts
|
|
2213
|
+
/** The side an attempt that settled counts on: its record's outcome, else the engine's state. */
|
|
2214
|
+
function sideOf({ state, meta }) {
|
|
2215
|
+
const outcome = meta?.outcome ?? (state === "passed" ? "pass" : state === "failed" ? "fail" : null);
|
|
2216
|
+
return outcome === "pass" || outcome === "fail" ? outcome : null;
|
|
2217
|
+
}
|
|
2218
|
+
/** Whether a best-of cell still needs another attempt after these. */
|
|
2219
|
+
function undecided(sides, bestOf) {
|
|
2220
|
+
const majority = majorityOf(bestOf);
|
|
2221
|
+
const pass = sides.filter((s) => s === "pass").length;
|
|
2222
|
+
const fail = sides.filter((s) => s === "fail").length;
|
|
2223
|
+
return pass < majority && fail < majority;
|
|
2224
|
+
}
|
|
2225
|
+
/** One best-of cell: the `ready` and `settled` hooks of its attempt `n`, from 1 to `bestOf`. */
|
|
2226
|
+
function bestOfCell(bestOf) {
|
|
2227
|
+
const sides = [];
|
|
2228
|
+
const settle = [];
|
|
2229
|
+
for (let n = 1; n <= bestOf; n++) sides.push(new Promise((resolve) => settle.push(resolve)));
|
|
2230
|
+
return (n) => ({
|
|
2231
|
+
...n > majorityOf(bestOf) ? { ready: async () => undecided(await Promise.all(sides.slice(0, n - 1)), bestOf) } : {},
|
|
2232
|
+
settled: (settled) => settle[n - 1]?.(sideOf(settled))
|
|
2233
|
+
});
|
|
2234
|
+
}
|
|
2235
|
+
//#endregion
|
|
1956
2236
|
//#region src/contract/declaration.ts
|
|
1957
2237
|
/** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
|
|
1958
2238
|
const evalOptionsDataSchema = z.record(z.string(), z.unknown());
|
|
@@ -2014,12 +2294,14 @@ const declarationProblem = (children) => dataProblem(declarationTreeSchema, { ch
|
|
|
2014
2294
|
/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
|
|
2015
2295
|
const GRACE_MS = 5e3;
|
|
2016
2296
|
function configLevel(config) {
|
|
2297
|
+
const { runs, bestOf } = config.runs !== void 0 ? runsOf(config.runs) : { runs: void 0 };
|
|
2017
2298
|
return {
|
|
2018
2299
|
harness: config.harness,
|
|
2019
2300
|
sandbox: config.sandbox,
|
|
2020
2301
|
environment: config.environment,
|
|
2021
2302
|
compare: void 0,
|
|
2022
|
-
runs
|
|
2303
|
+
runs,
|
|
2304
|
+
bestOf,
|
|
2023
2305
|
gate: config.gate,
|
|
2024
2306
|
env: config.env,
|
|
2025
2307
|
egress: config.egress !== void 0 ? { inherited: config.egress } : {},
|
|
@@ -2028,6 +2310,7 @@ function configLevel(config) {
|
|
|
2028
2310
|
question: config.question,
|
|
2029
2311
|
questions: [],
|
|
2030
2312
|
timeout: void 0,
|
|
2313
|
+
snapshot: config.snapshot,
|
|
2031
2314
|
tags: [],
|
|
2032
2315
|
revision: void 0,
|
|
2033
2316
|
pin: [],
|
|
@@ -2101,7 +2384,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2101
2384
|
...harness,
|
|
2102
2385
|
gateway: route.name
|
|
2103
2386
|
};
|
|
2104
|
-
});
|
|
2387
|
+
}).map((harness) => withDefaultEffort(harness, ctx.cursorEfforts));
|
|
2105
2388
|
const plannedAttempts = resolvedEval.skip ? [] : expand({
|
|
2106
2389
|
...resolved,
|
|
2107
2390
|
harness: routed
|
|
@@ -2210,6 +2493,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2210
2493
|
setBy,
|
|
2211
2494
|
gate: planned.gate,
|
|
2212
2495
|
...planned.minLift !== void 0 ? { minLift: planned.minLift } : {},
|
|
2496
|
+
...planned.bestOf !== void 0 ? { bestOf: planned.bestOf } : {},
|
|
2213
2497
|
brokered: brokeredIn(resolved.env, planned),
|
|
2214
2498
|
secretVars: secretVarsIn(resolved.env, planned),
|
|
2215
2499
|
egress: planned.egress,
|
|
@@ -2249,6 +2533,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2249
2533
|
env: resolved.env,
|
|
2250
2534
|
judge: resolved.judge,
|
|
2251
2535
|
capture: resolved.capture,
|
|
2536
|
+
snapshot: resolved.snapshot,
|
|
2252
2537
|
sources: {
|
|
2253
2538
|
...resolved.sources,
|
|
2254
2539
|
sandbox: setBy.sandbox
|
|
@@ -2487,7 +2772,20 @@ function register(mode, name, own, fn, source, hint = "", family) {
|
|
|
2487
2772
|
declareHarnesses(harnesses);
|
|
2488
2773
|
if (!tests) return;
|
|
2489
2774
|
registered++;
|
|
2490
|
-
|
|
2775
|
+
const bestOfCells = /* @__PURE__ */ new Map();
|
|
2776
|
+
for (const { name: testName, spec, ...options } of tests) {
|
|
2777
|
+
const { cell, n, bestOf } = options.attempt;
|
|
2778
|
+
let hooks = {};
|
|
2779
|
+
if (bestOf !== void 0) {
|
|
2780
|
+
const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
|
|
2781
|
+
bestOfCells.set(cell.key, of);
|
|
2782
|
+
hooks = of(n);
|
|
2783
|
+
}
|
|
2784
|
+
registerTest(testName, {
|
|
2785
|
+
...options,
|
|
2786
|
+
...hooks
|
|
2787
|
+
}, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
2788
|
+
}
|
|
2491
2789
|
}
|
|
2492
2790
|
/** Evals registered so far in this worker, for the scheduler's tie-break. */
|
|
2493
2791
|
let registered = 0;
|
|
@@ -2601,4 +2899,4 @@ function optionsFrom(written, where, base) {
|
|
|
2601
2899
|
//#endregion
|
|
2602
2900
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
2603
2901
|
|
|
2604
|
-
//# sourceMappingURL=evaluate-
|
|
2902
|
+
//# sourceMappingURL=evaluate-fhC1PCd_.js.map
|