reeval 0.15.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -14
- package/dist/{adapter-CbqbMR7y.js → adapter-CqUt4A0w.js} +3 -3
- package/dist/{adapter-CbqbMR7y.js.map → adapter-CqUt4A0w.js.map} +1 -1
- package/dist/{agent-Zgn-kiqq.js → agent-BtAbRTFq.js} +6 -6
- package/dist/{agent-Zgn-kiqq.js.map → agent-BtAbRTFq.js.map} +1 -1
- package/dist/{app-files-ChbvRx3P.js → app-files-CDttHPOz.js} +3 -3
- package/dist/app-files-CDttHPOz.js.map +1 -0
- package/dist/assemble-Bb7zqpRw.js +1 -0
- package/dist/{assemble-SKQGW_QF.js → assemble-J9rkE29m.js} +114 -31
- package/dist/assemble-J9rkE29m.js.map +1 -0
- package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
- package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{attempt-host-BOp8JklD.js → attempt-host-DDedP7Qv.js} +17 -8
- package/dist/attempt-host-DDedP7Qv.js.map +1 -0
- package/dist/capture-Bvc4FtWT.js +15 -0
- package/dist/capture-Bvc4FtWT.js.map +1 -0
- package/dist/capture-D68TiQQy.js +1 -0
- package/dist/{capture-BM_3100g.js → capture-DheNqeYS.js} +1495 -203
- package/dist/capture-DheNqeYS.js.map +1 -0
- package/dist/{cli-PdYrcoKM.js → cli-KTHjNFoo.js} +329 -74
- package/dist/cli-KTHjNFoo.js.map +1 -0
- package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{colors-QV9dFVhN.js → colors-Y6O3mtl_.js} +20 -2
- package/dist/colors-Y6O3mtl_.js.map +1 -0
- package/dist/{command-9xm5ImqI.js → command-1FBECMs-.js} +16 -16
- package/dist/{command-9xm5ImqI.js.map → command-1FBECMs-.js.map} +1 -1
- package/dist/{command-BqBxLgTc.js → command-Ce03LLtg.js} +14 -14
- package/dist/{command-BqBxLgTc.js.map → command-Ce03LLtg.js.map} +1 -1
- package/dist/{command-4pJYVCT4.js → command-CztRuwUM.js} +12 -12
- package/dist/{command-4pJYVCT4.js.map → command-CztRuwUM.js.map} +1 -1
- package/dist/{command-DbSMcxZL.js → command-nNN0mx65.js} +7 -7
- package/dist/{command-DbSMcxZL.js.map → command-nNN0mx65.js.map} +1 -1
- package/dist/{commands-BU6YJCz9.js → commands-DP13JX9i.js} +24 -12
- package/dist/commands-DP13JX9i.js.map +1 -0
- package/dist/{commands-Bi2SEQtr.js → commands-ZwQUezI1.js} +1 -1
- package/dist/{config-k6m7XbAz.js → config-BjjDryd5.js} +464 -220
- package/dist/config-BjjDryd5.js.map +1 -0
- package/dist/{contrasts-DLZF4dGF.js → contrasts-DSsRZpR8.js} +2 -2
- package/dist/{contrasts-DLZF4dGF.js.map → contrasts-DSsRZpR8.js.map} +1 -1
- package/dist/cursor-catalog-D7bFLEAD.js +143 -0
- package/dist/cursor-catalog-D7bFLEAD.js.map +1 -0
- package/dist/{db-CpsifVln.js → db-B9Ityuit.js} +8 -8
- package/dist/{db-CpsifVln.js.map → db-B9Ityuit.js.map} +1 -1
- package/dist/db-Bk6LWwza.js +2 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/default-effort-DK8gospa.js +91 -0
- package/dist/default-effort-DK8gospa.js.map +1 -0
- package/dist/default-effort-DmXHpqo9.js +1 -0
- package/dist/{docker-YFj4H1Cr.js → docker-BLCNfUie.js} +1 -1
- package/dist/{docker-B4-rx4dA.js → docker-SKpYOlGi.js} +5 -5
- package/dist/{docker-B4-rx4dA.js.map → docker-SKpYOlGi.js.map} +1 -1
- package/dist/{docker-51WvhN_X.js → docker-hwp3EucJ.js} +9 -5
- package/dist/docker-hwp3EucJ.js.map +1 -0
- package/dist/{doctor-BuQnfz4e.js → doctor-CT0Sgjng.js} +16 -16
- package/dist/{doctor-BuQnfz4e.js.map → doctor-CT0Sgjng.js.map} +1 -1
- package/dist/{environment-BOFgzFDg.js → environment-CZdercoj.js} +3 -3
- package/dist/{environment-BOFgzFDg.js.map → environment-CZdercoj.js.map} +1 -1
- package/dist/{errors-UjcLWelF.js → errors-BZVKVKYd.js} +15 -4
- package/dist/errors-BZVKVKYd.js.map +1 -0
- package/dist/{eval-source-Dje3rERU.js → eval-source-nZD7-FSz.js} +9 -8
- package/dist/eval-source-nZD7-FSz.js.map +1 -0
- package/dist/{evaluate-dqXccn4k.d.ts → evaluate-CNi0lZ7n.d.ts} +7 -6
- package/dist/{evaluate-BPP9s3s6.js → evaluate-fhC1PCd_.js} +345 -47
- package/dist/evaluate-fhC1PCd_.js.map +1 -0
- package/dist/{files-Clipx4OW.js → files-BGy05Dlr.js} +4 -4
- package/dist/{files-Clipx4OW.js.map → files-BGy05Dlr.js.map} +1 -1
- package/dist/{files-Bi461Be2.js → files-BIGxpL08.js} +25 -16
- package/dist/{files-Bi461Be2.js.map → files-BIGxpL08.js.map} +1 -1
- package/dist/files-C2RC7K4m.js +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/{files-H24pz0hL.js → files-I1i6Xyh1.js} +4 -4
- package/dist/{files-H24pz0hL.js.map → files-I1i6Xyh1.js.map} +1 -1
- package/dist/files-dJmrfmOM.js +1 -0
- package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-CsVZkNNL.js → git-DR02tHtU.js} +2 -2
- package/dist/{git-CsVZkNNL.js.map → git-DR02tHtU.js.map} +1 -1
- package/dist/{harness-3NVyTSGK.js → harness-ylJLvrbp.js} +39 -5
- package/dist/{harness-3NVyTSGK.js.map → harness-ylJLvrbp.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{images-4mwhNkvh.js → images---X8Wu9d.js} +8 -8
- package/dist/{images-4mwhNkvh.js.map → images---X8Wu9d.js.map} +1 -1
- package/dist/index.d.ts +71 -16
- package/dist/index.js +5 -5
- package/dist/init-BNTXrtMO.js +1 -0
- package/dist/{init-cOk3D44b.js → init-CA95ZnkV.js} +4 -4
- package/dist/{init-cOk3D44b.js.map → init-CA95ZnkV.js.map} +1 -1
- package/dist/{invocations-DW-lTk2f.js → invocations-BdoROAfb.js} +3 -3
- package/dist/{invocations-DW-lTk2f.js.map → invocations-BdoROAfb.js.map} +1 -1
- package/dist/{judge-CKdgEXN7.js → judge-Dp7FPJ1P.js} +3 -3
- package/dist/{judge-CKdgEXN7.js.map → judge-Dp7FPJ1P.js.map} +1 -1
- package/dist/judge.d.ts +2 -2
- package/dist/judge.js +1 -1
- package/dist/{live-XJQ6nqK4.js → live-CBFkdK1K.js} +3 -3
- package/dist/{live-XJQ6nqK4.js.map → live-CBFkdK1K.js.map} +1 -1
- package/dist/{matchers-DT2TaGix.js → matchers-Cv74mJRJ.js} +43 -13
- package/dist/matchers-Cv74mJRJ.js.map +1 -0
- package/dist/{model-Bq3G0OG1.js → model-6dfKFkQd.js} +4 -4
- package/dist/{model-Bq3G0OG1.js.map → model-6dfKFkQd.js.map} +1 -1
- package/dist/{model-CTnyvh7H.js → model-Be7fPvgi.js} +4 -51
- package/dist/model-Be7fPvgi.js.map +1 -0
- package/dist/{model-g3XnzUv0.js → model-CN4ccjNz.js} +1 -1
- package/dist/model-C_5TspZS.js +1 -0
- package/dist/{model-vRiUycWh.js → model-DkJVmVd2.js} +4 -4
- package/dist/{model-vRiUycWh.js.map → model-DkJVmVd2.js.map} +1 -1
- package/dist/{node-BVbbnFtv.js → node-D_Zkwhq2.js} +2 -2
- package/dist/{node-BVbbnFtv.js.map → node-D_Zkwhq2.js.map} +1 -1
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/{opencode-providers-Bz0l7OZI.js → opencode-providers-DPgdqRfg.js} +26 -2
- package/dist/{opencode-providers-Bz0l7OZI.js.map → opencode-providers-DPgdqRfg.js.map} +1 -1
- package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-CR00FWAa.js} +2 -2
- package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-CR00FWAa.js.map} +1 -1
- package/dist/{patch-DKJiC5Sq.js → patch-BJNdOZsf.js} +4 -4
- package/dist/patch-BJNdOZsf.js.map +1 -0
- package/dist/{plan-DdWkM6SW.js → plan-C1UpGZTk.js} +76 -12
- package/dist/plan-C1UpGZTk.js.map +1 -0
- package/dist/{plugins-D-CP6mMd.js → plugins-Cx6ReJWE.js} +24 -3
- package/dist/plugins-Cx6ReJWE.js.map +1 -0
- package/dist/{preflight-9ixuXzsx.js → preflight-BZ3L7P3a.js} +3 -3
- package/dist/{preflight-9ixuXzsx.js.map → preflight-BZ3L7P3a.js.map} +1 -1
- package/dist/{preflight-DTiVv0gY.js → preflight-CUooKgtT.js} +1 -1
- package/dist/{prepare-BfyikQit.js → prepare-BbBoTeDj.js} +7 -7
- package/dist/{prepare-BfyikQit.js.map → prepare-BbBoTeDj.js.map} +1 -1
- package/dist/{prepare-tmBBf4W8.js → prepare-CEZaEwaq.js} +2 -2
- package/dist/{provenance-BizwUn3h.js → provenance-BWvTIlQM.js} +2 -2
- package/dist/{provenance-BizwUn3h.js.map → provenance-BWvTIlQM.js.map} +1 -1
- package/dist/{providers-B6b9Tnyi.js → providers-B3yqakc7.js} +5 -5
- package/dist/{providers-B6b9Tnyi.js.map → providers-B3yqakc7.js.map} +1 -1
- package/dist/{providers-DkOupEBl.js → providers-DdyZW_ru.js} +1 -1
- package/dist/query-BkwQ0W-c.js +1 -0
- package/dist/{query-DyM-VExS.js → query-CZcqpOI9.js} +2 -2
- package/dist/{query-DyM-VExS.js.map → query-CZcqpOI9.js.map} +1 -1
- package/dist/{questions-BRhU8MzB.js → questions-CFU7-crW.js} +20 -19
- package/dist/questions-CFU7-crW.js.map +1 -0
- package/dist/{record-DZBCjuwm.js → record-Bu1Xi6MQ.js} +8 -4
- package/dist/record-Bu1Xi6MQ.js.map +1 -0
- package/dist/{remarks-C7i2LMRb.js → remarks-A3j-SkJ_.js} +42 -8
- package/dist/remarks-A3j-SkJ_.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{remarks-D_3qhBWU.js → remarks-Dj_x9l5n.js} +4 -3
- package/dist/{remarks-D_3qhBWU.js.map → remarks-Dj_x9l5n.js.map} +1 -1
- package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-DFIzcx_6.js → remote-EIFHdMd8.js} +5 -5
- package/dist/{remote-DFIzcx_6.js.map → remote-EIFHdMd8.js.map} +1 -1
- package/dist/{resolve-DWSJpKHX.js → resolve-8H5rz8mi.js} +5 -5
- package/dist/{resolve-DWSJpKHX.js.map → resolve-8H5rz8mi.js.map} +1 -1
- package/dist/resolve-Tm9pX7Hv.js +1 -0
- package/dist/{results-BC3pcG_Z.js → results-E7JdVjq2.js} +9 -9
- package/dist/{results-BC3pcG_Z.js.map → results-E7JdVjq2.js.map} +1 -1
- package/dist/{resume-j-rcIryi.js → resume-Blu4yIcq.js} +4 -4
- package/dist/{resume-j-rcIryi.js.map → resume-Blu4yIcq.js.map} +1 -1
- package/dist/{runtime-d56Gg3J5.js → runtime-CNbsMJQ7.js} +111 -37
- package/dist/runtime-CNbsMJQ7.js.map +1 -0
- package/dist/{sandbox-Cmln11gU.js → sandbox-RButGfIE.js} +2 -2
- package/dist/{sandbox-Cmln11gU.js.map → sandbox-RButGfIE.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-Dqrq-Sia.js +1 -0
- package/dist/{server-BOoH10J8.js → server-XgCYs_MF.js} +35 -11
- package/dist/server-XgCYs_MF.js.map +1 -0
- package/dist/{source-wD04-rRG.js → source-BOkbJCD2.js} +17 -3
- package/dist/{source-wD04-rRG.js.map → source-BOkbJCD2.js.map} +1 -1
- package/dist/source-DooSTtyz.js +1 -0
- package/dist/src-1M0srjji.js +1 -0
- package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
- package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-D91Hd6hT.js → src-BJCKWacl.js} +38 -18
- package/dist/src-BJCKWacl.js.map +1 -0
- package/dist/src-BZlhaDhE.js +2 -0
- package/dist/{src-BO8AYqJ5.js → src-BkhEPVJZ.js} +12 -3
- package/dist/{src-BO8AYqJ5.js.map → src-BkhEPVJZ.js.map} +1 -1
- package/dist/{src-C7upnZSd.js → src-Bp6X4193.js} +1 -1
- package/dist/{src-BYLM3hEz.js → src-C2Japg_I.js} +21 -10
- package/dist/{src-BYLM3hEz.js.map → src-C2Japg_I.js.map} +1 -1
- package/dist/src-C5q3_fRV.js +3 -0
- package/dist/src-C9EHSYyP.js +2 -0
- package/dist/src-CJdpsrXi.js +2 -0
- package/dist/{src-1lpTocbd.js → src-D1ET6oPL.js} +3 -27
- package/dist/{src-1lpTocbd.js.map → src-D1ET6oPL.js.map} +1 -1
- package/dist/{src-Bp4aN3rA.js → src-D4CZjMQ4.js} +1 -1
- package/dist/{src-BYPMM1x7.js → src-GkJ-1Rdy.js} +1 -1
- package/dist/{src-kYeaJJrl.js → src-OOKE-5GA.js} +1 -1
- package/dist/state-BCyNaz13.js +1 -0
- package/dist/{state-DrJ687ut.js → state-D6-tvqtp.js} +8 -8
- package/dist/{state-DrJ687ut.js.map → state-D6-tvqtp.js.map} +1 -1
- package/dist/suite-BiWajSlA.js +2 -0
- package/dist/{suite-Co8cwelb.js → suite-DYI1k2Ie.js} +22 -5
- package/dist/suite-DYI1k2Ie.js.map +1 -0
- package/dist/{summary-DLFHVoux.js → summary-CN2aqAUz.js} +59 -17
- package/dist/summary-CN2aqAUz.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +96 -44
- package/dist/testing.js +4 -4
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
- package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
- package/dist/{types-coMWHaU2.d.ts → types-DDeCnNmp.d.ts} +437 -181
- package/dist/view/index.d.ts +34 -5
- package/dist/view/index.js +8 -7
- package/dist/views-7bNY17Z6.js +2 -0
- package/dist/{views-BFO1BLKu.js → views-Bd-H71_t.js} +5 -5
- package/dist/{views-BFO1BLKu.js.map → views-Bd-H71_t.js.map} +1 -1
- package/dist/{worker-entry-CJtdsKBP.js → worker-entry-BIHJqE_z.js} +34 -13
- package/dist/worker-entry-BIHJqE_z.js.map +1 -0
- package/dist/workspace-CxwHv8mG.js +1 -0
- package/dist/{workspace-BRxjfa8N.js → workspace-Sy5g8zW9.js} +4 -4
- package/dist/{workspace-BRxjfa8N.js.map → workspace-Sy5g8zW9.js.map} +1 -1
- package/package.json +1 -1
- package/skills/eval-rubric/SKILL.md +11 -0
- package/dist/app-files-ChbvRx3P.js.map +0 -1
- package/dist/assemble-SKQGW_QF.js.map +0 -1
- package/dist/assemble-XBWuK3Ed.js +0 -1
- package/dist/attempt-host-BOp8JklD.js.map +0 -1
- package/dist/capture-BM_3100g.js.map +0 -1
- package/dist/capture-DSK8O5bA.js +0 -1
- package/dist/cli-PdYrcoKM.js.map +0 -1
- package/dist/cli-spec-D0qyrPZi.js.map +0 -1
- package/dist/client-Ct6s6OnF.js.map +0 -1
- package/dist/colors-QV9dFVhN.js.map +0 -1
- package/dist/commands-BU6YJCz9.js.map +0 -1
- package/dist/config-k6m7XbAz.js.map +0 -1
- package/dist/db-DkD-F-X-.js +0 -2
- package/dist/debug-BhizgAsU.js.map +0 -1
- package/dist/debug-DeM5pswK.js +0 -1
- package/dist/docker-51WvhN_X.js.map +0 -1
- package/dist/errors-UjcLWelF.js.map +0 -1
- package/dist/eval-source-Dje3rERU.js.map +0 -1
- package/dist/evaluate-BPP9s3s6.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-C2h2Cw4F.js +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-DmUrRmmL.js +0 -1
- package/dist/init-BDEOopK7.js +0 -1
- package/dist/matchers-DT2TaGix.js.map +0 -1
- package/dist/model-CTnyvh7H.js.map +0 -1
- package/dist/model-D-SWrLZG.js +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/pairs-puYi-zj2.js.map +0 -1
- package/dist/patch-DKJiC5Sq.js.map +0 -1
- package/dist/plan-DdWkM6SW.js.map +0 -1
- package/dist/plugins-D-CP6mMd.js.map +0 -1
- package/dist/query-Clm2mNSm.js +0 -1
- package/dist/questions-BRhU8MzB.js.map +0 -1
- package/dist/record-DZBCjuwm.js.map +0 -1
- package/dist/remarks-BwJ8pcTQ.js +0 -1
- package/dist/remarks-C7i2LMRb.js.map +0 -1
- package/dist/remarks-dA8LyoPc.js.map +0 -1
- package/dist/resolve--rQZ8u6Z.js +0 -1
- package/dist/runtime-d56Gg3J5.js.map +0 -1
- package/dist/server-BOoH10J8.js.map +0 -1
- package/dist/server-DpuVAbrO.js +0 -1
- package/dist/source-BFd6f0-R.js +0 -1
- package/dist/src-6_zz7oJs.js +0 -2
- package/dist/src-D91Hd6hT.js.map +0 -1
- package/dist/src-DH-9IO7f.js +0 -1
- package/dist/src-HSxBazcg.js +0 -3
- package/dist/src-LD1BAJFa.js +0 -2
- package/dist/src-dlX-Lmxg.js +0 -2
- package/dist/state-B5PLm5qY.js +0 -1
- package/dist/suite-22Kt9Grx.js +0 -2
- package/dist/suite-Co8cwelb.js.map +0 -1
- package/dist/summary-DLFHVoux.js.map +0 -1
- package/dist/views-BOf8qEL8.js +0 -2
- package/dist/worker-entry-CJtdsKBP.js.map +0 -1
- package/dist/workspace-DDHzkMjm.js +0 -1
|
@@ -187,7 +187,7 @@ type FileRef = {
|
|
|
187
187
|
sha256: string;
|
|
188
188
|
bytes: number;
|
|
189
189
|
};
|
|
190
|
-
/** A generic attempt file from `c.capture()
|
|
190
|
+
/** A generic attempt file from `c.capture()`. */
|
|
191
191
|
type ArtifactRef = FileRef & {
|
|
192
192
|
name: string;
|
|
193
193
|
mediaType: 'image/png';
|
|
@@ -208,6 +208,14 @@ type ArtifactRef = FileRef & {
|
|
|
208
208
|
model: string;
|
|
209
209
|
moves: number;
|
|
210
210
|
};
|
|
211
|
+
/** The HTTP status of the page's document, after redirects; absent when no response came. */
|
|
212
|
+
status?: number;
|
|
213
|
+
/** How many uncaught exceptions and unhandled rejections the page had; absent when none. */
|
|
214
|
+
pageErrors?: number;
|
|
215
|
+
/** `artifacts/<name>.capture.json` beside the PNG: what the browser saw, a `CapturePage`. */
|
|
216
|
+
page?: FileRef;
|
|
217
|
+
/** Set on the screenshot taken as one of the capture's instructions ended. */
|
|
218
|
+
step?: CaptureStep;
|
|
211
219
|
};
|
|
212
220
|
};
|
|
213
221
|
type GitInfo = {
|
|
@@ -217,15 +225,18 @@ type GitInfo = {
|
|
|
217
225
|
*/
|
|
218
226
|
sha: string;
|
|
219
227
|
branch: string | null;
|
|
220
|
-
/** Uncommitted changes, outside `.reeval/` and the history folder, when the run started. */
|
|
228
|
+
/** Uncommitted changes, outside the `.reeval/` directory and the history folder, when the run started. */
|
|
221
229
|
dirty: boolean;
|
|
222
230
|
/** sha256 of the uncommitted changes (tracked and untracked, as a binary patch), when dirty. */
|
|
223
231
|
diffHash: string | null;
|
|
224
|
-
/**
|
|
232
|
+
/**
|
|
233
|
+
* Those changes are stored as the `worktree.patch` file in the artifact directory. Applying it
|
|
234
|
+
* to `sha` reproduces the workspace.
|
|
235
|
+
*/
|
|
225
236
|
worktreePatch: FileRef | null;
|
|
226
237
|
/**
|
|
227
|
-
*
|
|
228
|
-
* a secret (then where, and which rule, never the value).
|
|
238
|
+
* This explains why a dirty run has no `worktree.patch` file: it was too large, `artifacts` was
|
|
239
|
+
* `'none'`, or it looked like it held a secret (then where, and which rule, never the value).
|
|
229
240
|
*/
|
|
230
241
|
worktreePatchNote: string | null;
|
|
231
242
|
commitTime: string;
|
|
@@ -267,7 +278,7 @@ type JudgmentRecord = {
|
|
|
267
278
|
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
268
279
|
costUsd?: number;
|
|
269
280
|
/**
|
|
270
|
-
* `<index>:<sha256(question) prefix>`: the same question at the same place in the
|
|
281
|
+
* `<index>:<sha256(question) prefix>`: the same question at the same place in the evaluation, so a
|
|
271
282
|
* verdict can be followed across runs (and an edited question starts a new series). `<index>`
|
|
272
283
|
* is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
|
|
273
284
|
* whichever finishes first.
|
|
@@ -279,14 +290,17 @@ type JudgmentRecord = {
|
|
|
279
290
|
artifacts?: JudgeArtifactRef[];
|
|
280
291
|
};
|
|
281
292
|
type AttemptFiles = {
|
|
282
|
-
/** Each environment setup step's command, exit code, duration and output. */
|
|
293
|
+
/** Each environment setup step's command, exit code, duration, and output. */
|
|
283
294
|
setupLog?: FileRef;
|
|
284
295
|
transcript?: FileRef;
|
|
285
296
|
/** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
|
|
286
297
|
rawTranscript?: FileRef;
|
|
287
298
|
diff?: FileRef;
|
|
288
299
|
judgeInput?: FileRef;
|
|
289
|
-
/**
|
|
300
|
+
/**
|
|
301
|
+
* PNGs from `c.capture()`, under the `artifacts/<name>.png` path, then the page as each of its
|
|
302
|
+
* instructions ended, `artifacts/<name>.step-<n>.png`, kept when the capture fails too.
|
|
303
|
+
*/
|
|
290
304
|
artifacts?: ArtifactRef[];
|
|
291
305
|
};
|
|
292
306
|
type ContextFile = {
|
|
@@ -301,16 +315,22 @@ type UpstreamCount = {
|
|
|
301
315
|
type AttemptRecord = {
|
|
302
316
|
/** `<eval key> # <cell key>#<n>`: unique in the run. */
|
|
303
317
|
key: string;
|
|
304
|
-
/** `<cell key>#<n>`: unique in its
|
|
318
|
+
/** `<cell key>#<n>`: unique in its evaluation. */
|
|
305
319
|
id: string;
|
|
306
320
|
/** The cell's key; see `EvalEntry.cells`. */
|
|
307
321
|
cell: string;
|
|
308
322
|
/** Run number within the cell, from 1. */
|
|
309
323
|
n: number;
|
|
324
|
+
/**
|
|
325
|
+
* In a `runs: { bestOf }` cell: `added` when it ran past the majority's first attempts (the
|
|
326
|
+
* earlier ones split, or an infra error left them short), `skipped` when it didn't run because
|
|
327
|
+
* the earlier ones already had a majority. Unset on the attempts that always run.
|
|
328
|
+
*/
|
|
329
|
+
bestOf?: 'added' | 'skipped';
|
|
310
330
|
/** Start order across the run, from 1; null for attempts that never started. */
|
|
311
331
|
seq: number | null;
|
|
312
332
|
startedAt: string | null;
|
|
313
|
-
/** The image the container started from, in `stack.images`; null
|
|
333
|
+
/** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
|
|
314
334
|
imageId: string | null;
|
|
315
335
|
/** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
|
|
316
336
|
env: {
|
|
@@ -318,16 +338,17 @@ type AttemptRecord = {
|
|
|
318
338
|
secrets: string[];
|
|
319
339
|
brokered: string[];
|
|
320
340
|
};
|
|
321
|
-
/** Skills, rules and MCP config visible to the agent in its sandbox, by content hash. */
|
|
341
|
+
/** Skills, rules, and MCP config visible to the agent in its sandbox, by content hash. */
|
|
322
342
|
context: ContextFile[];
|
|
323
343
|
/** The egress policy and, when the sandbox records it, the hosts contacted and refused; null if it never ran. */
|
|
324
344
|
egress: EgressRecord | null;
|
|
325
345
|
/**
|
|
326
|
-
* The models the harness's
|
|
327
|
-
*
|
|
328
|
-
*
|
|
329
|
-
*
|
|
330
|
-
*
|
|
346
|
+
* The models served by the harness's API calls, as observed by the egress gateway. Values may
|
|
347
|
+
* come from requests or responses and include reported effort; null when no model was named or
|
|
348
|
+
* recorded. The gateway reads the request URL for Antigravity, Goose, pi, and OpenCode on a
|
|
349
|
+
* `google/` model, so a request for `<model>-low` records `<model>`, and the API response for
|
|
350
|
+
* Codex, Grok Build, and OpenCode on another provider. Null for Cursor and Claude Code, whose
|
|
351
|
+
* API requests name no model.
|
|
331
352
|
*/
|
|
332
353
|
servedModels: string[] | null;
|
|
333
354
|
/**
|
|
@@ -348,12 +369,14 @@ type AttemptRecord = {
|
|
|
348
369
|
*/
|
|
349
370
|
reportedCostUsd?: number;
|
|
350
371
|
/**
|
|
351
|
-
* The reasoning effort the harness
|
|
352
|
-
*
|
|
372
|
+
* The reasoning effort reeval asked the harness for: the factory's
|
|
373
|
+
* (`claudeCode({ effort: 'high' })`), or `medium` where it set none and the harness and model
|
|
374
|
+
* take one; null when reeval asked for none, leaving the CLI's own default. Fast is not
|
|
375
|
+
* this.
|
|
353
376
|
*/
|
|
354
377
|
effort: string | null;
|
|
355
378
|
/**
|
|
356
|
-
* The reasoning effort the harness's API said it ran at, from `servedModels`
|
|
379
|
+
* The reasoning effort the harness's API said it ran at, from `servedModels` or
|
|
357
380
|
* each turn's `model` (`medium` from `gpt-5.6-luna[effort=medium]`); null when those
|
|
358
381
|
* labels name none. Fast (`[fast=true]`) is not this.
|
|
359
382
|
*/
|
|
@@ -362,7 +385,8 @@ type AttemptRecord = {
|
|
|
362
385
|
error: AttemptError | null;
|
|
363
386
|
durationMs: number;
|
|
364
387
|
/**
|
|
365
|
-
* `text` is the agent's final message, cut at 8 KB; the whole of it is in `transcript.jsonl
|
|
388
|
+
* `text` is the agent's final message, cut at 8 KB; the whole of it is in the `transcript.jsonl`
|
|
389
|
+
* file.
|
|
366
390
|
* `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
|
|
367
391
|
*/
|
|
368
392
|
turns: Array<{
|
|
@@ -385,8 +409,8 @@ type AttemptRecord = {
|
|
|
385
409
|
/** Tokens across the attempt's turns; null when the harness reported none. */
|
|
386
410
|
usage: TokenUsage | null;
|
|
387
411
|
/**
|
|
388
|
-
* Tokens the capture vision loop used. Infrastructure only: never
|
|
389
|
-
*
|
|
412
|
+
* Tokens the capture vision loop used. Infrastructure only: never included
|
|
413
|
+
* in `usage`, effort, compare, or lift.
|
|
390
414
|
*/
|
|
391
415
|
captureUse?: {
|
|
392
416
|
model: string;
|
|
@@ -394,7 +418,7 @@ type AttemptRecord = {
|
|
|
394
418
|
usage: TokenUsage | null;
|
|
395
419
|
};
|
|
396
420
|
timings: PhaseTimings;
|
|
397
|
-
/** Counts from the transcript, so
|
|
421
|
+
/** Counts from the transcript, so the attempt's activity is available without reading the transcript. `webSearches` includes refused searches. */
|
|
398
422
|
activity: {
|
|
399
423
|
tools: number;
|
|
400
424
|
fileEdits: number;
|
|
@@ -419,6 +443,22 @@ type AttemptRecord = {
|
|
|
419
443
|
* names. Absent when there were none.
|
|
420
444
|
*/
|
|
421
445
|
setupProcesses?: SetupProcess[];
|
|
446
|
+
/**
|
|
447
|
+
* Copies of the attempt's workspace in its directory in `.reeval/runs` on the machine that ran
|
|
448
|
+
* it (never in the history folder), in the order taken: `turn-<n>` after each `c.send` with
|
|
449
|
+
* `snapshot: 'turns'`, then `final` after the checks. Each has its form, path and size, its
|
|
450
|
+
* file count, the directories left out by name, the files whose secret-looking values were
|
|
451
|
+
* masked and the binary ones left out for holding one, or why it wasn't taken. Absent with
|
|
452
|
+
* `snapshot: false`, and when the sandbox was gone first (a timeout or a cancel).
|
|
453
|
+
*/
|
|
454
|
+
snapshots?: Snapshot[];
|
|
455
|
+
/**
|
|
456
|
+
* Each `c.serve` call that listened, in call order: `{ command, port, portEnv, detected,
|
|
457
|
+
* address, readyMs }`. `port` is where it listens; `detected` is true when that isn't the
|
|
458
|
+
* `PORT` it was given (`portEnv`), so reeval took the one port that started listening after
|
|
459
|
+
* the command did. Absent when the attempt served nothing.
|
|
460
|
+
*/
|
|
461
|
+
serves?: ServeRecord[];
|
|
422
462
|
/**
|
|
423
463
|
* Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
|
|
424
464
|
* `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
|
|
@@ -433,9 +473,9 @@ type AttemptRecord = {
|
|
|
433
473
|
/** Every try, in order, when it was given `retries`; its log has each try's output. */
|
|
434
474
|
tries?: ExecTry[];
|
|
435
475
|
}>;
|
|
436
|
-
/** Every `expect` and `expect.
|
|
476
|
+
/** Every `expect` and `expect.require` matcher call that settled, in call order; unset with none. */
|
|
437
477
|
assertions?: AssertionRecord[];
|
|
438
|
-
/** Files under a dependency directory
|
|
478
|
+
/** Files under a dependency directory, such as the `node_modules/` directory or `.venv/` directory, collapse to one entry with `count`. */
|
|
439
479
|
changedFiles: Array<FileChange & {
|
|
440
480
|
count?: number;
|
|
441
481
|
}>;
|
|
@@ -451,7 +491,7 @@ type AttemptRecord = {
|
|
|
451
491
|
type EvalEntry = {
|
|
452
492
|
/** `file › group › … › name`, as shown. */
|
|
453
493
|
id: string;
|
|
454
|
-
/**
|
|
494
|
+
/** Used to match an evaluation across runs; equal to `id`. */
|
|
455
495
|
key: string;
|
|
456
496
|
/** Directory name inside the artifact directory, from `key`. */
|
|
457
497
|
slug: string;
|
|
@@ -461,8 +501,8 @@ type EvalEntry = {
|
|
|
461
501
|
/** Results are comparable only when fingerprints match. */
|
|
462
502
|
fingerprint: string;
|
|
463
503
|
/**
|
|
464
|
-
* The
|
|
465
|
-
* the local files it imported, and were fingerprinted over them.
|
|
504
|
+
* The evaluation file, by content hash: where the evaluation lives. Records from before `source`
|
|
505
|
+
* also list the local files it imported, and were fingerprinted over them.
|
|
466
506
|
*/
|
|
467
507
|
sources: Array<{
|
|
468
508
|
path: string;
|
|
@@ -476,12 +516,12 @@ type EvalEntry = {
|
|
|
476
516
|
/** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
|
|
477
517
|
revision?: Revision;
|
|
478
518
|
tags: string[];
|
|
479
|
-
/** The baseline environment, when the
|
|
519
|
+
/** The baseline environment, when the evaluation uses `compare`. */
|
|
480
520
|
compare?: EvalCompareMeta;
|
|
481
521
|
/**
|
|
482
|
-
* Over gated cells
|
|
483
|
-
* only when nothing counted), else `skipped` if all were, else `pass`. An ungated
|
|
484
|
-
* failures don't fail the
|
|
522
|
+
* Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
|
|
523
|
+
* `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
|
|
524
|
+
* cell's failures don't fail the evaluation.
|
|
485
525
|
*/
|
|
486
526
|
outcome: Outcome;
|
|
487
527
|
/** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
|
|
@@ -494,9 +534,9 @@ type EvalEntry = {
|
|
|
494
534
|
* The eval's pairwise questions (those with `across`), answered after its attempts by votes,
|
|
495
535
|
* one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
496
536
|
* may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
|
|
497
|
-
* (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
|
|
498
|
-
* `<eval-slug>/questions/<n>.json
|
|
499
|
-
* masked and screenshots as refs; unset where the
|
|
537
|
+
* (rates by cut, votes, ties, a failed call, and a sample of reasons). Each answer's `input` is
|
|
538
|
+
* the `<eval-slug>/questions/<n>.json` file: every call its model was sent, per pair and order,
|
|
539
|
+
* with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
|
|
500
540
|
*/
|
|
501
541
|
answers?: CompareAnswer[];
|
|
502
542
|
/** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
|
|
@@ -513,7 +553,7 @@ type FamilyInterval = {
|
|
|
513
553
|
hi: number;
|
|
514
554
|
level: 0.95;
|
|
515
555
|
/**
|
|
516
|
-
* `wilson`
|
|
556
|
+
* `wilson` or `newcombe-paired`: one counted attempt a case and side, so each case is one
|
|
517
557
|
* Bernoulli trial (or pair). `t-cases`: several, so a t interval over the cases' pass@1, with
|
|
518
558
|
* attempts within a case never counted as independent.
|
|
519
559
|
*/
|
|
@@ -588,17 +628,17 @@ type FamilyLift = {
|
|
|
588
628
|
outcome: 'pass' | 'fail';
|
|
589
629
|
effort?: FamilyEffort;
|
|
590
630
|
};
|
|
591
|
-
/** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an
|
|
631
|
+
/** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an evaluation. */
|
|
592
632
|
type FamilyRecord = {
|
|
593
633
|
/** `file › group › … › template`. */
|
|
594
634
|
id: string;
|
|
595
635
|
file: string;
|
|
596
636
|
group: string[];
|
|
597
|
-
/** The name template,
|
|
637
|
+
/** The name template, for example `fixes $id`. */
|
|
598
638
|
name: string;
|
|
599
639
|
/** Cases in the list. */
|
|
600
640
|
cases: number;
|
|
601
|
-
/** Cases with an
|
|
641
|
+
/** Cases with an evaluation in this run; fewer when a filter or tag left some out. */
|
|
602
642
|
ran: number;
|
|
603
643
|
/** One per cell. */
|
|
604
644
|
rates: FamilyRate[];
|
|
@@ -643,7 +683,7 @@ type FingerprintParts = {
|
|
|
643
683
|
environment: string | null;
|
|
644
684
|
image: string | null;
|
|
645
685
|
agentCli: string | null;
|
|
646
|
-
/** Over the judge's model, threshold and version
|
|
686
|
+
/** Over the judge's model, threshold and version. */
|
|
647
687
|
judge: string;
|
|
648
688
|
/** Over the attempt's context files. */
|
|
649
689
|
context: string;
|
|
@@ -655,7 +695,7 @@ type FingerprintParts = {
|
|
|
655
695
|
*/
|
|
656
696
|
gate?: string;
|
|
657
697
|
};
|
|
658
|
-
/** One harness and environment of an
|
|
698
|
+
/** One harness and environment of an evaluation, and how its attempts went. */
|
|
659
699
|
type CellRecord = Cell & {
|
|
660
700
|
/** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
|
|
661
701
|
fingerprint: string;
|
|
@@ -674,14 +714,20 @@ type CellRecord = Cell & {
|
|
|
674
714
|
/** The configured harness model, when it has one. */
|
|
675
715
|
model: string | null;
|
|
676
716
|
/**
|
|
677
|
-
* The reasoning effort the harness
|
|
678
|
-
*
|
|
717
|
+
* The reasoning effort reeval asked the harness for: the factory's, or `medium` where it set
|
|
718
|
+
* none and the harness and model take one; null when reeval asked for none, leaving the CLI's
|
|
719
|
+
* own default. Fast is not this.
|
|
679
720
|
*/
|
|
680
721
|
effort: string | null;
|
|
681
|
-
/** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the
|
|
722
|
+
/** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the evaluation. */
|
|
682
723
|
gate: CellGate;
|
|
683
724
|
/** Environment keys the variant replaced on the control's (or a child on its parent's). */
|
|
684
725
|
overrides: string[];
|
|
726
|
+
/**
|
|
727
|
+
* Set by `runs: { bestOf }`: the cell stopped once a majority of this many attempts agreed, and
|
|
728
|
+
* gates on that majority. `runs` still counts every planned attempt, the skipped ones included.
|
|
729
|
+
*/
|
|
730
|
+
bestOf?: number;
|
|
685
731
|
runs: number;
|
|
686
732
|
pass: number;
|
|
687
733
|
fail: number;
|
|
@@ -697,7 +743,7 @@ type CellRecord = Cell & {
|
|
|
697
743
|
outcome: Outcome;
|
|
698
744
|
lift?: Lift;
|
|
699
745
|
/**
|
|
700
|
-
* Each
|
|
746
|
+
* Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
|
|
701
747
|
* attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
|
|
702
748
|
*/
|
|
703
749
|
checks?: CellCheck[];
|
|
@@ -710,10 +756,10 @@ type CheckTally = {
|
|
|
710
756
|
notRun: number;
|
|
711
757
|
};
|
|
712
758
|
/**
|
|
713
|
-
* One check over a cell's counted attempts. `verdict` (variant cells)
|
|
714
|
-
*
|
|
715
|
-
*
|
|
716
|
-
*
|
|
759
|
+
* One check over a cell's counted attempts. `verdict` (variant cells) is `discriminates` when
|
|
760
|
+
* every answered instance passes on one side and fails on the other, `both-pass` when every
|
|
761
|
+
* answered instance passes on both sides, `both-fail` when every answered instance fails on both
|
|
762
|
+
* sides, or `mixed` otherwise. It is absent when either side has no pass or fail.
|
|
717
763
|
*/
|
|
718
764
|
type CellCheck = CheckTally & {
|
|
719
765
|
label: string;
|
|
@@ -773,7 +819,10 @@ type RunRecord = {
|
|
|
773
819
|
harness?: string[];
|
|
774
820
|
environment?: string;
|
|
775
821
|
sandbox?: string;
|
|
822
|
+
/** With `bestOf`, the most attempts a cell runs. */
|
|
776
823
|
runs?: number;
|
|
824
|
+
/** Set by `defaults.runs: { bestOf }`. */
|
|
825
|
+
bestOf?: number;
|
|
777
826
|
timeout?: string | number;
|
|
778
827
|
env?: string[];
|
|
779
828
|
};
|
|
@@ -807,8 +856,13 @@ type RunRecord = {
|
|
|
807
856
|
config: {
|
|
808
857
|
sandbox: string;
|
|
809
858
|
environment: string | null;
|
|
810
|
-
/**
|
|
859
|
+
/**
|
|
860
|
+
* The config's `runs` (with `bestOf`, the most a cell runs); 1 when it sets none, though a
|
|
861
|
+
* compare's cells then run 4 each.
|
|
862
|
+
*/
|
|
811
863
|
runs: number;
|
|
864
|
+
/** Set by the config's `runs: { bestOf }`. */
|
|
865
|
+
bestOf?: number;
|
|
812
866
|
judgeModel: string | null;
|
|
813
867
|
threshold: number;
|
|
814
868
|
timeoutMs: number;
|
|
@@ -827,28 +881,26 @@ type RunRecord = {
|
|
|
827
881
|
/** Each pooled `evaluate.each` that ran; unset when none did. */
|
|
828
882
|
families?: FamilyRecord[];
|
|
829
883
|
/**
|
|
830
|
-
*
|
|
831
|
-
* attempt awaited, a worker that died. Any one fails the run. Unset when there
|
|
884
|
+
* These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
|
|
885
|
+
* or throw no attempt awaited, or a worker that died. Any one fails the run. Unset when there
|
|
886
|
+
* were none.
|
|
832
887
|
*/
|
|
833
888
|
errors?: string[];
|
|
834
889
|
};
|
|
835
|
-
/**
|
|
890
|
+
/** These fields describe what ran and graded the agents: the machine, images, harnesses, and judge. */
|
|
836
891
|
type Stack = {
|
|
837
892
|
/** The reeval version that wrote the run. */
|
|
838
893
|
reeval: string;
|
|
839
|
-
/** `process.versions.node
|
|
894
|
+
/** `process.versions.node`, or the compatible Node version reported by another runtime. */
|
|
840
895
|
node: string;
|
|
841
|
-
/**
|
|
842
|
-
* The runtime reeval ran on, `node`, `bun`, `deno` or another, and its own version. Absent in
|
|
843
|
-
* runs recorded before it, which ran on Node.
|
|
844
|
-
*/
|
|
896
|
+
/** The runtime reeval ran on, `node`, `bun`, `deno`, or another, and its version. */
|
|
845
897
|
runtime?: {
|
|
846
898
|
name: string;
|
|
847
899
|
version: string;
|
|
848
900
|
};
|
|
849
901
|
/**
|
|
850
|
-
* The language
|
|
851
|
-
*
|
|
902
|
+
* The evaluation language, `typescript` for `*.eval.ts` files, and the version of its toolchain
|
|
903
|
+
* when the runner reports one.
|
|
852
904
|
*/
|
|
853
905
|
language?: {
|
|
854
906
|
name: string;
|
|
@@ -856,16 +908,17 @@ type Stack = {
|
|
|
856
908
|
};
|
|
857
909
|
os: string;
|
|
858
910
|
arch: string;
|
|
859
|
-
/** Every image an attempt started from. `id` is content-addressed; `repoDigests` only
|
|
911
|
+
/** Every image an attempt started from. `id` is content-addressed; `repoDigests` exist only after the image is pushed. */
|
|
860
912
|
images: Array<AttemptImage & {
|
|
861
913
|
repoDigests: string[];
|
|
862
914
|
}>;
|
|
863
915
|
/** Exact versions of the packages that run and grade the agent. */
|
|
864
916
|
packages: Record<string, string>;
|
|
865
917
|
/**
|
|
866
|
-
*
|
|
867
|
-
* reported serving, with the variant it resolved), and
|
|
868
|
-
* suffixes those labels named). Fast (`[fast=true]`) is not
|
|
918
|
+
* For each harness, the record includes its configured `model` and the `effort` it ran at,
|
|
919
|
+
* `resolvedModels` (what it reported serving, with the variant it resolved), and
|
|
920
|
+
* `resolvedEffort` (reasoning-effort suffixes those labels named). Fast (`[fast=true]`) is not
|
|
921
|
+
* `resolvedEffort`.
|
|
869
922
|
*/
|
|
870
923
|
harnesses: Array<{
|
|
871
924
|
id: string;
|
|
@@ -873,8 +926,9 @@ type Stack = {
|
|
|
873
926
|
/** The configured model, when the harness has one. */
|
|
874
927
|
model: string | null;
|
|
875
928
|
/**
|
|
876
|
-
* The reasoning effort the harness
|
|
877
|
-
*
|
|
929
|
+
* The reasoning effort reeval asked the harness for: the factory's, or `medium` where it set
|
|
930
|
+
* none and the harness and model take one; null when reeval asked for none, leaving the
|
|
931
|
+
* CLI's own default. Fast is not this.
|
|
878
932
|
*/
|
|
879
933
|
effort: string | null;
|
|
880
934
|
/**
|
|
@@ -901,12 +955,12 @@ type Stack = {
|
|
|
901
955
|
threshold: number;
|
|
902
956
|
/** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
|
|
903
957
|
version: string;
|
|
904
|
-
/**
|
|
958
|
+
/** The config's default judge inputs, every key resolved. Unset when the config names none, which reeval requires: each `c.judge()` call names its own. */
|
|
905
959
|
state?: ResolvedJudgeStateOptions;
|
|
906
960
|
};
|
|
907
961
|
/** Union of every attempt's context files. */
|
|
908
962
|
context: ContextFile[];
|
|
909
|
-
/** What
|
|
963
|
+
/** What evaluation fingerprints hash; fingerprints from different scopes aren't comparable. */
|
|
910
964
|
fingerprintScope: string[];
|
|
911
965
|
};
|
|
912
966
|
/** The parts of a run's stack that `reeval compare` checks for drift. */
|
|
@@ -1108,16 +1162,22 @@ type Wait = CaptureWait | {
|
|
|
1108
1162
|
type CaptureInstruction = string | {
|
|
1109
1163
|
wait: Wait;
|
|
1110
1164
|
};
|
|
1111
|
-
|
|
1165
|
+
/** The keys a capture's driver presses. In a focused `<select>`, the arrows change the choice. */
|
|
1166
|
+
type CaptureKey = 'Tab' | 'Enter' | 'Escape' | 'Backspace' | 'Space' | 'ArrowUp' | 'ArrowDown' | 'ArrowLeft' | 'ArrowRight';
|
|
1167
|
+
/** A move on the page. */
|
|
1168
|
+
type CaptureUseMove = ({
|
|
1112
1169
|
type: 'click';
|
|
1113
1170
|
x: number;
|
|
1114
1171
|
y: number;
|
|
1115
|
-
} |
|
|
1172
|
+
} |
|
|
1173
|
+
/** `replace` selects what the focused field holds first, so the text replaces it. */
|
|
1174
|
+
{
|
|
1116
1175
|
type: 'type';
|
|
1117
1176
|
text: string;
|
|
1177
|
+
replace?: boolean;
|
|
1118
1178
|
} | {
|
|
1119
1179
|
type: 'key';
|
|
1120
|
-
key:
|
|
1180
|
+
key: CaptureKey;
|
|
1121
1181
|
} | {
|
|
1122
1182
|
type: 'scroll';
|
|
1123
1183
|
x: number;
|
|
@@ -1133,7 +1193,15 @@ type CaptureUseAction = {
|
|
|
1133
1193
|
type: 'hover';
|
|
1134
1194
|
x: number;
|
|
1135
1195
|
y: number;
|
|
1136
|
-
}
|
|
1196
|
+
}) & {
|
|
1197
|
+
/**
|
|
1198
|
+
* The move finishes its instruction: once the page settles, the journey goes on to the next.
|
|
1199
|
+
* A model's final move that changed nothing on the page is asked about again instead.
|
|
1200
|
+
*/
|
|
1201
|
+
final?: boolean;
|
|
1202
|
+
};
|
|
1203
|
+
/** A move, or `done`: the instruction's goal is met without another move. */
|
|
1204
|
+
type CaptureUseAction = CaptureUseMove | {
|
|
1137
1205
|
type: 'done';
|
|
1138
1206
|
};
|
|
1139
1207
|
/** One list per instruction string. Tests; never downloads a model. */
|
|
@@ -1152,7 +1220,7 @@ type CaptureUse = {
|
|
|
1152
1220
|
type CaptureOptions = {
|
|
1153
1221
|
/** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
|
|
1154
1222
|
viewport?: Viewport;
|
|
1155
|
-
/** How long to poll before failing. Default `'60s'`. */
|
|
1223
|
+
/** How long to poll before failing; a 502, 503 or 504 is polled past until then. Default `'60s'`. */
|
|
1156
1224
|
timeout?: Duration;
|
|
1157
1225
|
/** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
|
|
1158
1226
|
browser?: boolean;
|
|
@@ -1166,12 +1234,15 @@ type CaptureOptions = {
|
|
|
1166
1234
|
type BackgroundHandle = {
|
|
1167
1235
|
kill(): Promise<void>;
|
|
1168
1236
|
};
|
|
1169
|
-
/** `c.serve`: a background process plus the port it
|
|
1237
|
+
/** `c.serve`: a background process plus the port it listens on. */
|
|
1170
1238
|
type ServeHandle = BackgroundHandle & {
|
|
1171
1239
|
port: number;
|
|
1172
1240
|
};
|
|
1173
|
-
/**
|
|
1174
|
-
|
|
1241
|
+
/**
|
|
1242
|
+
* What `c.capture()` returns: the PNG is also in the attempt record under `artifacts/`, and the
|
|
1243
|
+
* page facts in `artifacts/<name>.capture.json` beside it.
|
|
1244
|
+
*/
|
|
1245
|
+
type Capture = CapturePage & {
|
|
1175
1246
|
name: string;
|
|
1176
1247
|
/** Container port that was tunneled. */
|
|
1177
1248
|
port: number;
|
|
@@ -1186,6 +1257,61 @@ type Capture = {
|
|
|
1186
1257
|
bytes: number;
|
|
1187
1258
|
};
|
|
1188
1259
|
};
|
|
1260
|
+
/** One message the page's console showed while it was captured. */
|
|
1261
|
+
type CaptureConsoleMessage = {
|
|
1262
|
+
level: 'error' | 'warning' | 'info' | 'log' | 'debug';
|
|
1263
|
+
/** The message, at most 2000 characters, with the app's own URLs as paths. */
|
|
1264
|
+
text: string;
|
|
1265
|
+
/**
|
|
1266
|
+
* `console-api` for the page's `console.*` calls; otherwise the browser's own source, such as
|
|
1267
|
+
* `network` (a request that failed), `javascript`, `security` or `deprecation`.
|
|
1268
|
+
*/
|
|
1269
|
+
source: string;
|
|
1270
|
+
/** The script or resource it came from, as a path when it's on the app. */
|
|
1271
|
+
url?: string;
|
|
1272
|
+
};
|
|
1273
|
+
/** A request the page made that failed or answered 400 or above. */
|
|
1274
|
+
type CaptureFailedRequest = {
|
|
1275
|
+
method: string;
|
|
1276
|
+
/** As a path when it's on the app. */
|
|
1277
|
+
url: string;
|
|
1278
|
+
/** The response's status; absent when no response came. */
|
|
1279
|
+
status?: number;
|
|
1280
|
+
/** The browser's error, such as `net::ERR_CONNECTION_REFUSED`; absent when a response came. */
|
|
1281
|
+
error?: string;
|
|
1282
|
+
};
|
|
1283
|
+
/** A class of HTTP status for `toHaveStatus()`: `'2xx'` is 200 to 299. */
|
|
1284
|
+
type StatusRange = `${1 | 2 | 3 | 4 | 5}xx`;
|
|
1285
|
+
/**
|
|
1286
|
+
* What the browser saw while it loaded and captured a page: `<name>.capture.json` beside the
|
|
1287
|
+
* capture's PNG, and these fields of the `Capture` that `c.capture()` returns. On the fake
|
|
1288
|
+
* sandbox, which has no browser, `status` and `finalUrl` come from an HTTP GET and the lists are
|
|
1289
|
+
* empty.
|
|
1290
|
+
*/
|
|
1291
|
+
type CapturePage = {
|
|
1292
|
+
/**
|
|
1293
|
+
* The HTTP status of the main frame's latest document, after redirects. A full navigation
|
|
1294
|
+
* replaces it; a client-side route change keeps it. Absent when no response came.
|
|
1295
|
+
*/
|
|
1296
|
+
status?: number;
|
|
1297
|
+
/** Where the page ended up, after redirects and client-side routing: a path on the app. */
|
|
1298
|
+
finalUrl: string;
|
|
1299
|
+
/** Each message the console showed, in order; at most 100 of each level. */
|
|
1300
|
+
console: CaptureConsoleMessage[];
|
|
1301
|
+
/**
|
|
1302
|
+
* The page errors: each uncaught exception and unhandled promise rejection, with its stack, at
|
|
1303
|
+
* most 100.
|
|
1304
|
+
*/
|
|
1305
|
+
errors: string[];
|
|
1306
|
+
/** Requests that failed or answered 400 or above, in order, at most 100. */
|
|
1307
|
+
failedRequests: CaptureFailedRequest[];
|
|
1308
|
+
/** How many messages, errors and requests the caps left out; absent when none. */
|
|
1309
|
+
dropped?: number;
|
|
1310
|
+
};
|
|
1311
|
+
/** A capture's `artifacts/<name>.capture.json`: its page facts, versioned. */
|
|
1312
|
+
type CapturePageFile = {
|
|
1313
|
+
schemaVersion: 1;
|
|
1314
|
+
} & CapturePage;
|
|
1189
1315
|
/**
|
|
1190
1316
|
* Optional camera and path for `c.capture(port | ServeHandle, options?)`.
|
|
1191
1317
|
* `port` is the first argument, not a field here. Default `path` is `/`;
|
|
@@ -1279,10 +1405,10 @@ type JudgeState = JudgeJsonState | Array<{
|
|
|
1279
1405
|
type: 'json';
|
|
1280
1406
|
value: JudgeJsonState;
|
|
1281
1407
|
} | JudgeFilePart>;
|
|
1282
|
-
/** A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png
|
|
1408
|
+
/** A capture this judge call sent as a file part. Bytes live in the `artifacts/<name>.png` file. */
|
|
1283
1409
|
type JudgeArtifactRef = {
|
|
1284
1410
|
name: string;
|
|
1285
|
-
/** `artifacts/<name>.png` under the attempt directory. */
|
|
1411
|
+
/** The `artifacts/<name>.png` path under the attempt directory. */
|
|
1286
1412
|
path: string;
|
|
1287
1413
|
sha256: string;
|
|
1288
1414
|
bytes: number;
|
|
@@ -1314,11 +1440,6 @@ type JudgeOptions = {
|
|
|
1314
1440
|
providerOptions?: Record<string, Record<string, JSONValue>>;
|
|
1315
1441
|
/** The most judge calls to this model's provider at once, across the run. */
|
|
1316
1442
|
maxConcurrent?: number;
|
|
1317
|
-
/**
|
|
1318
|
-
* The judge's state: what the judge is given beside the question, not the workspace's or the
|
|
1319
|
-
* attempt's state. Merged key by key down the cascade.
|
|
1320
|
-
*/
|
|
1321
|
-
state?: JudgeStateOptions;
|
|
1322
1443
|
/**
|
|
1323
1444
|
* The gateway the judge's calls go through, by name or by its helper. It skips the config's
|
|
1324
1445
|
* `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
|
|
@@ -1326,43 +1447,61 @@ type JudgeOptions = {
|
|
|
1326
1447
|
gateway?: GatewayName | GatewayDescriptor;
|
|
1327
1448
|
};
|
|
1328
1449
|
/**
|
|
1329
|
-
* What
|
|
1330
|
-
*
|
|
1450
|
+
* What one `c.judge()` call shows the judge beside the question. Every input is off unless the
|
|
1451
|
+
* call turns it on, and a call turns on at least one; each is a key of the state in
|
|
1452
|
+
* `judge-input.json`, or the captures sent as file parts.
|
|
1331
1453
|
*/
|
|
1332
|
-
type
|
|
1333
|
-
/** The graded turn's prompt.
|
|
1454
|
+
type JudgeInputs = {
|
|
1455
|
+
/** The graded turn's prompt. */
|
|
1334
1456
|
task?: boolean;
|
|
1335
|
-
/** The graded turn's final message, or `on`'s value.
|
|
1457
|
+
/** The graded turn's final message, or `on`'s value. */
|
|
1336
1458
|
output?: boolean;
|
|
1337
|
-
/** `status path` lines for the changed files, under `diff`'s patterns.
|
|
1459
|
+
/** `status path` lines for the changed files, under `diff`'s patterns. */
|
|
1338
1460
|
files?: boolean;
|
|
1339
1461
|
/**
|
|
1340
1462
|
* The diff since the eval started. A list is gitignore-style patterns over workspace paths:
|
|
1341
|
-
* the last matching pattern wins and `!` excludes.
|
|
1463
|
+
* the last matching pattern wins and `!` excludes.
|
|
1342
1464
|
*/
|
|
1343
1465
|
diff?: boolean | string[];
|
|
1344
1466
|
/**
|
|
1345
1467
|
* Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
|
|
1346
1468
|
* patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
|
|
1347
1469
|
* the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
|
|
1348
|
-
* These files leave the diff.
|
|
1470
|
+
* These files leave the diff.
|
|
1349
1471
|
*/
|
|
1350
1472
|
contents?: string[];
|
|
1351
1473
|
/**
|
|
1352
1474
|
* The graded turn's transcript as text lines: `true` is messages and tool calls with tool
|
|
1353
1475
|
* output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
|
|
1354
|
-
* Default false.
|
|
1355
1476
|
*/
|
|
1356
1477
|
transcript?: boolean | 'tools' | 'full';
|
|
1478
|
+
/** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
|
|
1479
|
+
checks?: boolean | string[];
|
|
1480
|
+
/**
|
|
1481
|
+
* Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
|
|
1482
|
+
* capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
|
|
1483
|
+
*/
|
|
1484
|
+
artifacts?: true | Array<Capture | string>;
|
|
1485
|
+
};
|
|
1486
|
+
/** What a judge call's inputs read: which turn, and whether the turns before it too. */
|
|
1487
|
+
type JudgeSubject = {
|
|
1488
|
+
/** The turn `task`, `output`, `transcript` and `diff` read instead of the latest, or a value used as the output. */
|
|
1489
|
+
on?: Turn | JSONValue;
|
|
1357
1490
|
/**
|
|
1358
1491
|
* `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
|
|
1359
1492
|
* `transcript`) transcript. Default `'last'`.
|
|
1360
1493
|
*/
|
|
1361
1494
|
turns?: 'last' | 'all';
|
|
1362
|
-
/** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
|
|
1363
|
-
checks?: boolean | string[];
|
|
1364
1495
|
};
|
|
1365
|
-
/** `
|
|
1496
|
+
/** One input turned on: `true`, a mode or a non-empty selection, not `false`. */
|
|
1497
|
+
type OneInput<I, K extends keyof I> = { [P in K]-?: Exclude<I[P], false | undefined>; };
|
|
1498
|
+
/** At least one of `I`'s keys turned on. */
|
|
1499
|
+
type AtLeastOneInput<I> = { [K in keyof I]-?: OneInput<I, K>; }[keyof I];
|
|
1500
|
+
/** `c.judge()`'s options, before the at-least-one-input rule that `JudgeCallOptions` adds. */
|
|
1501
|
+
type JudgeCallFields = JudgeOptions & JudgeInputs & JudgeSubject;
|
|
1502
|
+
/** `c.judge()`'s options: the judge options, the inputs (at least one) and what they read. */
|
|
1503
|
+
type JudgeCallOptions = JudgeCallFields & AtLeastOneInput<JudgeInputs>;
|
|
1504
|
+
/** The inputs one judge call saw, every key set (`false` when off); `artifacts` is recorded apart. */
|
|
1366
1505
|
type ResolvedJudgeStateOptions = {
|
|
1367
1506
|
task: boolean;
|
|
1368
1507
|
output: boolean;
|
|
@@ -1373,16 +1512,6 @@ type ResolvedJudgeStateOptions = {
|
|
|
1373
1512
|
turns: 'last' | 'all';
|
|
1374
1513
|
checks: boolean | string[];
|
|
1375
1514
|
};
|
|
1376
|
-
type JudgeCallOptions = JudgeOptions & {
|
|
1377
|
-
/** What to grade instead of the latest turn: another turn, or a value used as the output. */
|
|
1378
|
-
on?: Turn | JSONValue;
|
|
1379
|
-
/**
|
|
1380
|
-
* Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
|
|
1381
|
-
* `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
|
|
1382
|
-
* captures; a provider that cannot take files fails the call.
|
|
1383
|
-
*/
|
|
1384
|
-
artifacts?: true | Array<Capture | string>;
|
|
1385
|
-
};
|
|
1386
1515
|
type Verdict = {
|
|
1387
1516
|
verdict: 'pass' | 'fail';
|
|
1388
1517
|
/** The judge's probability that the answer is yes, from 0 to 1. */
|
|
@@ -1418,10 +1547,11 @@ type AttemptContext = {
|
|
|
1418
1547
|
/** Sends the next turn of this eval's agent session. */
|
|
1419
1548
|
send(prompt: string): Promise<Turn>;
|
|
1420
1549
|
/**
|
|
1421
|
-
* Asks the judge a yes/no question about
|
|
1422
|
-
*
|
|
1550
|
+
* Asks the judge a yes/no question about what `options` turns on, and nothing else: at least
|
|
1551
|
+
* one of `task`, `output`, `files`, `diff`, `contents`, `transcript`, `checks` or `artifacts`.
|
|
1552
|
+
* They read the latest turn, or `options.on`.
|
|
1423
1553
|
*/
|
|
1424
|
-
judge(question: string, options
|
|
1554
|
+
judge(question: string, options: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
|
|
1425
1555
|
/**
|
|
1426
1556
|
* Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
|
|
1427
1557
|
* many extra times, `interval` ms apart; it resolves with the last try.
|
|
@@ -1436,8 +1566,10 @@ type AttemptContext = {
|
|
|
1436
1566
|
background?: boolean;
|
|
1437
1567
|
}): Promise<ExecResult | BackgroundHandle>;
|
|
1438
1568
|
/**
|
|
1439
|
-
* Starts `command` in the background
|
|
1440
|
-
* and
|
|
1569
|
+
* Starts `command` in the background with `PORT` set to `port`, or to a port reeval allocates,
|
|
1570
|
+
* and returns once it listens. Without `port`, a command that listens on one new port instead
|
|
1571
|
+
* (Vite's 5173) gets that one; several new ports, none of them `PORT`, or an exit first, throw
|
|
1572
|
+
* with its last output.
|
|
1441
1573
|
*/
|
|
1442
1574
|
serve(command: string, port?: number): Promise<ServeHandle>;
|
|
1443
1575
|
/**
|
|
@@ -2378,9 +2510,11 @@ type CompareOptions<E = Environment> = {
|
|
|
2378
2510
|
* With `across`, it picks between two attempts that differ only in that factor, in both orders;
|
|
2379
2511
|
* without it, it's asked of each attempt on its own. Without a model, the cascaded
|
|
2380
2512
|
* `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
|
|
2381
|
-
*
|
|
2513
|
+
* It's shown only the inputs it turns on, at least one.
|
|
2382
2514
|
*/
|
|
2383
|
-
type Question =
|
|
2515
|
+
type Question = QuestionFields & AtLeastOneInput<QuestionInputs>;
|
|
2516
|
+
/** A question's options, before the at-least-one-input rule that `Question` adds. */
|
|
2517
|
+
type QuestionFields = QuestionInputs & {
|
|
2384
2518
|
question: string;
|
|
2385
2519
|
/** `'provider/model'` that answers it, over the cascaded `question.model`. */
|
|
2386
2520
|
model?: string;
|
|
@@ -2391,43 +2525,39 @@ type Question = string | {
|
|
|
2391
2525
|
across?: PairFactor | PairFactor[];
|
|
2392
2526
|
/** The answers a question asked of each attempt picks from, with a short reason. */
|
|
2393
2527
|
choices?: string[];
|
|
2394
|
-
/**
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2528
|
+
/**
|
|
2529
|
+
* `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
|
|
2530
|
+
* transcript; `'last'`: the last turn only. Default `'all'`.
|
|
2531
|
+
*/
|
|
2532
|
+
turns?: 'last' | 'all';
|
|
2398
2533
|
};
|
|
2399
2534
|
/**
|
|
2400
|
-
* What a question is shown of each attempt:
|
|
2401
|
-
*
|
|
2402
|
-
*
|
|
2403
|
-
* message and the transcript.
|
|
2535
|
+
* What a question is shown of each attempt: `c.judge()`'s inputs but `contents` (the attempts'
|
|
2536
|
+
* workspaces are gone when questions are asked). Every input is off unless the question turns it
|
|
2537
|
+
* on, and a question turns on at least one.
|
|
2404
2538
|
*/
|
|
2405
|
-
type
|
|
2406
|
-
/** The prompts the attempt was sent, once for both sides when they match.
|
|
2539
|
+
type QuestionInputs = {
|
|
2540
|
+
/** The prompts the attempt was sent, once for both sides when they match. */
|
|
2407
2541
|
task?: boolean;
|
|
2408
|
-
/** The final message: the agent's own account of its work.
|
|
2542
|
+
/** The final message: the agent's own account of its work. */
|
|
2409
2543
|
output?: boolean;
|
|
2410
|
-
/** `status path` lines for the changed files, under `diff`'s patterns.
|
|
2544
|
+
/** `status path` lines for the changed files, under `diff`'s patterns. */
|
|
2411
2545
|
files?: boolean;
|
|
2412
2546
|
/**
|
|
2413
2547
|
* The diff since the eval started. A list is gitignore-style patterns over workspace paths:
|
|
2414
|
-
* the last matching pattern wins and `!` excludes.
|
|
2548
|
+
* the last matching pattern wins and `!` excludes.
|
|
2415
2549
|
*/
|
|
2416
2550
|
diff?: boolean | string[];
|
|
2417
2551
|
/**
|
|
2418
2552
|
* The transcript as text lines: `true` is messages and tool calls with tool output clipped,
|
|
2419
2553
|
* `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
|
|
2420
|
-
* harness's tools, so a pair of two harnesses isn't blind.
|
|
2421
|
-
* attempt.
|
|
2554
|
+
* harness's tools, so a pair of two harnesses isn't blind.
|
|
2422
2555
|
*/
|
|
2423
2556
|
transcript?: boolean | 'tools' | 'full';
|
|
2424
|
-
/**
|
|
2425
|
-
* `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
|
|
2426
|
-
* transcript; `'last'`: the last turn only. Default `'all'`.
|
|
2427
|
-
*/
|
|
2428
|
-
turns?: 'last' | 'all';
|
|
2429
|
-
/** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
|
|
2557
|
+
/** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
|
|
2430
2558
|
checks?: boolean | string[];
|
|
2559
|
+
/** Each attempt's own captures, by name; `true` sends all of them. */
|
|
2560
|
+
artifacts?: true | string[];
|
|
2431
2561
|
};
|
|
2432
2562
|
/** `QuestionStateOptions` with every key set: what each side of one question showed. */
|
|
2433
2563
|
type ResolvedQuestionStateOptions = {
|
|
@@ -2439,14 +2569,10 @@ type ResolvedQuestionStateOptions = {
|
|
|
2439
2569
|
turns: 'last' | 'all';
|
|
2440
2570
|
checks: boolean | string[];
|
|
2441
2571
|
};
|
|
2442
|
-
/** Defaults for every question
|
|
2572
|
+
/** Defaults for every question below it. What a question is shown is its own, never a default. */
|
|
2443
2573
|
type QuestionOptions = {
|
|
2444
2574
|
/** `'provider/model'` that answers every question below it without its own `model`. */
|
|
2445
2575
|
model?: string;
|
|
2446
|
-
/** What it's shown of each attempt. */
|
|
2447
|
-
state?: QuestionStateOptions;
|
|
2448
|
-
/** Each attempt's own captures, by name; `true` sends all of them. Default none. */
|
|
2449
|
-
artifacts?: true | string[];
|
|
2450
2576
|
};
|
|
2451
2577
|
/** A matrix factor a pairwise question varies; the others are held. */
|
|
2452
2578
|
type PairFactor = 'environment' | 'harness';
|
|
@@ -2484,7 +2610,7 @@ type CompareAnswer = {
|
|
|
2484
2610
|
model: string;
|
|
2485
2611
|
/** The model id the provider says served the calls, when it reports one. */
|
|
2486
2612
|
resolvedModel?: string;
|
|
2487
|
-
/**
|
|
2613
|
+
/** The inputs each side showed, every key resolved; unset on answers recorded before reeval wrote it, which showed the task and the diff. */
|
|
2488
2614
|
stateOptions?: ResolvedQuestionStateOptions;
|
|
2489
2615
|
/** The captures it sent of each side, as asked: `true` for all, or their names. */
|
|
2490
2616
|
artifacts?: true | string[];
|
|
@@ -2537,8 +2663,21 @@ type GatePolicy = 'all' | {
|
|
|
2537
2663
|
type CellGate = GatePolicy | 'none';
|
|
2538
2664
|
/** A compare object: `baseline` and `variants`. Same as `CompareOptions`. */
|
|
2539
2665
|
type CompareSpec<E = Environment> = CompareOptions<E>;
|
|
2666
|
+
type OddDigit = '1' | '3' | '5' | '7' | '9';
|
|
2667
|
+
/**
|
|
2668
|
+
* `N` when it's an odd whole number of 3 or more, else a sentence the compiler shows as the type
|
|
2669
|
+
* it expected. A `number` that isn't a literal stays `number`: only planning can check it.
|
|
2670
|
+
*/
|
|
2671
|
+
type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${string}` | `${string}.${string}` | `${string}e${string}` | '1' ? 'an odd whole number of 3 or more, e.g. 3 or 5' : `${N}` extends `${string}${OddDigit}` ? N : 'an odd whole number of 3 or more, e.g. 3 or 5';
|
|
2672
|
+
/**
|
|
2673
|
+
* Attempts per cell: a fixed count, or `{ bestOf }`, an odd number of 3 or more, which stops a
|
|
2674
|
+
* cell once one side (pass or fail) has a majority: `{ bestOf: 5 }` runs 3 to 5 attempts.
|
|
2675
|
+
*/
|
|
2676
|
+
type Runs<N extends number = number> = number | {
|
|
2677
|
+
bestOf: OddAtLeast3<N>;
|
|
2678
|
+
};
|
|
2540
2679
|
/** Options other than `environment` and `compare`, which are exclusive of each other. */
|
|
2541
|
-
type EvalOptionFields = {
|
|
2680
|
+
type EvalOptionFields<N extends number = number> = {
|
|
2542
2681
|
/** Which agents run the eval; each harness is its own cell. */
|
|
2543
2682
|
harness?: HarnessOption;
|
|
2544
2683
|
/**
|
|
@@ -2547,11 +2686,12 @@ type EvalOptionFields = {
|
|
|
2547
2686
|
*/
|
|
2548
2687
|
sandbox?: SandboxDescriptor;
|
|
2549
2688
|
/**
|
|
2550
|
-
* Attempts per cell
|
|
2551
|
-
*
|
|
2552
|
-
*
|
|
2689
|
+
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
|
|
2690
|
+
* a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
|
|
2691
|
+
* Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
|
|
2692
|
+
* a verdict, and anything else runs once, a pooled `evaluate.each` case included.
|
|
2553
2693
|
*/
|
|
2554
|
-
runs?:
|
|
2694
|
+
runs?: Runs<N>;
|
|
2555
2695
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
2556
2696
|
gate?: GatePolicy;
|
|
2557
2697
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
@@ -2565,7 +2705,7 @@ type EvalOptionFields = {
|
|
|
2565
2705
|
judge?: JudgeOptions;
|
|
2566
2706
|
/** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
|
|
2567
2707
|
capture?: CaptureOptions;
|
|
2568
|
-
/**
|
|
2708
|
+
/** The model every question that names none of its own is answered by, down the cascade. */
|
|
2569
2709
|
question?: QuestionOptions;
|
|
2570
2710
|
/**
|
|
2571
2711
|
* What a language model is asked about the attempts, always a list; they never change pass,
|
|
@@ -2576,6 +2716,11 @@ type EvalOptionFields = {
|
|
|
2576
2716
|
questions?: Question[];
|
|
2577
2717
|
/** Per attempt. */
|
|
2578
2718
|
timeout?: Duration;
|
|
2719
|
+
/**
|
|
2720
|
+
* When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
|
|
2721
|
+
* the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
|
|
2722
|
+
*/
|
|
2723
|
+
snapshot?: SnapshotOption;
|
|
2579
2724
|
/** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
|
|
2580
2725
|
tags?: string[];
|
|
2581
2726
|
/**
|
|
@@ -2622,8 +2767,9 @@ type EnvironmentOrCompare<E> = {
|
|
|
2622
2767
|
baseline?: undefined;
|
|
2623
2768
|
};
|
|
2624
2769
|
};
|
|
2625
|
-
|
|
2626
|
-
type
|
|
2770
|
+
/** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
|
|
2771
|
+
type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
|
|
2772
|
+
type GroupOptions<N extends number = number> = EvalOptions<N>;
|
|
2627
2773
|
/**
|
|
2628
2774
|
* `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
|
|
2629
2775
|
* by name or a fake judge, and environments without `extend`. The same `environment` / compare
|
|
@@ -2663,8 +2809,8 @@ type ProfileDefaults = {
|
|
|
2663
2809
|
sandbox?: SandboxDescriptor;
|
|
2664
2810
|
/** What every eval runs in when it names no environment. */
|
|
2665
2811
|
environment?: Environment;
|
|
2666
|
-
/** Attempts per cell. */
|
|
2667
|
-
runs?:
|
|
2812
|
+
/** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
|
|
2813
|
+
runs?: Runs;
|
|
2668
2814
|
/** How each cell's attempts decide its outcome. */
|
|
2669
2815
|
gate?: GatePolicy;
|
|
2670
2816
|
/** Variables and secrets for the sandbox, merged by name over the config's. */
|
|
@@ -2679,6 +2825,8 @@ type ProfileDefaults = {
|
|
|
2679
2825
|
capture?: CaptureOptions;
|
|
2680
2826
|
/** Defaults for every question, merged over the config's. */
|
|
2681
2827
|
question?: QuestionOptions;
|
|
2828
|
+
/** When each attempt's workspace is copied; merged over the config's. */
|
|
2829
|
+
snapshot?: SnapshotOption;
|
|
2682
2830
|
};
|
|
2683
2831
|
/** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
|
|
2684
2832
|
type ProfileLimit = {
|
|
@@ -2755,15 +2903,15 @@ type BlindAttempt = {
|
|
|
2755
2903
|
choices?: string[];
|
|
2756
2904
|
attempt: BlindSide;
|
|
2757
2905
|
};
|
|
2758
|
-
/** A turn of one side, with the
|
|
2906
|
+
/** A turn of one side, with the inputs its question turned on. */
|
|
2759
2907
|
type BlindTurn = {
|
|
2760
2908
|
task?: string;
|
|
2761
2909
|
output?: string;
|
|
2762
2910
|
transcript?: string;
|
|
2763
2911
|
};
|
|
2764
2912
|
/**
|
|
2765
|
-
* One side of a blinded pair, or the attempt a per-attempt question is asked of: only the
|
|
2766
|
-
* its question
|
|
2913
|
+
* One side of a blinded pair, or the attempt a per-attempt question is asked of: only the inputs
|
|
2914
|
+
* its question turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
|
|
2767
2915
|
* under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
|
|
2768
2916
|
*/
|
|
2769
2917
|
type BlindSide = BlindTurn & {
|
|
@@ -2817,8 +2965,11 @@ type ReevalConfig = {
|
|
|
2817
2965
|
sandbox?: SandboxDescriptor;
|
|
2818
2966
|
/** What every eval runs in when it names no environment, and a baseline-less compare's control. */
|
|
2819
2967
|
environment?: Environment;
|
|
2820
|
-
/**
|
|
2821
|
-
|
|
2968
|
+
/**
|
|
2969
|
+
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
|
|
2970
|
+
* variants runs 4 a side and anything else once.
|
|
2971
|
+
*/
|
|
2972
|
+
runs?: Runs;
|
|
2822
2973
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
2823
2974
|
gate?: GatePolicy;
|
|
2824
2975
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
@@ -2836,6 +2987,11 @@ type ReevalConfig = {
|
|
|
2836
2987
|
capture?: CaptureOptions;
|
|
2837
2988
|
/** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
|
|
2838
2989
|
question?: QuestionOptions;
|
|
2990
|
+
/**
|
|
2991
|
+
* When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
|
|
2992
|
+
* the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
|
|
2993
|
+
*/
|
|
2994
|
+
snapshot?: SnapshotOption;
|
|
2839
2995
|
/** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
|
|
2840
2996
|
concurrency?: number;
|
|
2841
2997
|
/** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
|
|
@@ -2920,9 +3076,9 @@ type AttemptError = {
|
|
|
2920
3076
|
};
|
|
2921
3077
|
/**
|
|
2922
3078
|
* One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
|
|
2923
|
-
* reeval's MCP timer, which every stdio server runs through. An agent CLI that
|
|
2924
|
-
* turn
|
|
2925
|
-
*
|
|
3079
|
+
* reeval's MCP timer, which every stdio server runs through. An agent CLI that starts one process
|
|
3080
|
+
* per turn starts its servers on every turn; a CLI that keeps a session starts them on the first
|
|
3081
|
+
* turn only. A server that doesn't answer within 60s fails the attempt as an infra
|
|
2926
3082
|
* error naming it; one whose client closes it first doesn't. `mcpStartup` is absent when the
|
|
2927
3083
|
* environment has no stdio MCP server, or the sandbox has no `node` to run the timer.
|
|
2928
3084
|
*/
|
|
@@ -2935,7 +3091,7 @@ type McpStartup = {
|
|
|
2935
3091
|
*/
|
|
2936
3092
|
ms: number | null;
|
|
2937
3093
|
};
|
|
2938
|
-
/**
|
|
3094
|
+
/** The duration of each evaluation phase, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
|
|
2939
3095
|
type PhaseTimings = {
|
|
2940
3096
|
setupMs: number;
|
|
2941
3097
|
agentMs: number;
|
|
@@ -2945,16 +3101,28 @@ type PhaseTimings = {
|
|
|
2945
3101
|
};
|
|
2946
3102
|
type CellRole = 'control' | 'variant';
|
|
2947
3103
|
/**
|
|
2948
|
-
*
|
|
2949
|
-
*
|
|
2950
|
-
*
|
|
2951
|
-
*
|
|
3104
|
+
* A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
|
|
3105
|
+
* environment's sandbox id. With `compare`, the control is `baseline` (or the cascaded
|
|
3106
|
+
* environment, or an empty `baseline`) and each variant is that environment on its own,
|
|
3107
|
+
* including its sandbox.
|
|
2952
3108
|
*/
|
|
2953
3109
|
type Cell = {
|
|
3110
|
+
/** The harness id, the environment name, and the sandbox id, joined with `/`. */
|
|
2954
3111
|
key: string;
|
|
3112
|
+
/**
|
|
3113
|
+
* The harness id: `id` when set, otherwise `<kind>:<model>` (`<kind>` alone without a model),
|
|
3114
|
+
* with `[effort=<value>]` appended whenever `effort` is set.
|
|
3115
|
+
*/
|
|
2955
3116
|
harness: string;
|
|
2956
|
-
/** The environment's own name, also on a variant cell (`with-
|
|
3117
|
+
/** The environment's own name, also on a variant cell (`with-skill`, with no prefix). */
|
|
2957
3118
|
environment: string;
|
|
3119
|
+
/**
|
|
3120
|
+
* The sandbox id is `id` when set, or `fake` for the fake sandbox. A Docker sandbox starts with
|
|
3121
|
+
* the engine (`docker` by default). It adds `:` and the sanitized final component of `image` or
|
|
3122
|
+
* `from`, or for `dockerfile` the sanitized filename (the parent directory name when that
|
|
3123
|
+
* filename is `Dockerfile` and a parent is present). Non-empty `install` steps append `+` and an
|
|
3124
|
+
* eight-character hash.
|
|
3125
|
+
*/
|
|
2958
3126
|
sandbox: string;
|
|
2959
3127
|
role: CellRole;
|
|
2960
3128
|
/** The variant's name, for variant cells. */
|
|
@@ -2997,9 +3165,9 @@ type EnvironmentRecord = {
|
|
|
2997
3165
|
name: string;
|
|
2998
3166
|
/** The previous `extendEnvironment` layer's name, else null. */
|
|
2999
3167
|
parent: string | null;
|
|
3000
|
-
/** Keys a later layer replaced,
|
|
3168
|
+
/** Keys a later layer replaced, for example `env.KEY`, `skills.typescript`, `mcp.docs`, or `rules`. */
|
|
3001
3169
|
overrides: string[];
|
|
3002
|
-
/** Covers every field but the rules file's path; secret names are included
|
|
3170
|
+
/** Covers every field but the rules file's path; secret names are included; values never are. The image is the sandbox's. */
|
|
3003
3171
|
digest: string;
|
|
3004
3172
|
setup: string[];
|
|
3005
3173
|
skills: Array<{
|
|
@@ -3013,7 +3181,7 @@ type EnvironmentRecord = {
|
|
|
3013
3181
|
/** The plugin the skill came with; unset for a skill from `skills`. */
|
|
3014
3182
|
plugin?: string;
|
|
3015
3183
|
}>;
|
|
3016
|
-
/**
|
|
3184
|
+
/** The names of the MCP servers the agent was given. */
|
|
3017
3185
|
mcp: string[];
|
|
3018
3186
|
/** The plugins placed, each with what it placed; in the digest. Unset when there are none. */
|
|
3019
3187
|
plugins?: PluginRecord[];
|
|
@@ -3038,9 +3206,9 @@ type EnvironmentRecord = {
|
|
|
3038
3206
|
/** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
|
|
3039
3207
|
egress: EgressPolicy;
|
|
3040
3208
|
/**
|
|
3041
|
-
*
|
|
3042
|
-
* project root), the ref and path asked for (null when unset), the commit the
|
|
3043
|
-
* and the hash of the tree that was copied in, which is in the digest.
|
|
3209
|
+
* This records the Git repository the workspace started as: the repo as written (a local one
|
|
3210
|
+
* relative to the project root), the ref and path asked for (null when unset), the commit the
|
|
3211
|
+
* ref resolved to, and the hash of the tree that was copied in, which is in the digest.
|
|
3044
3212
|
*/
|
|
3045
3213
|
workspace?: {
|
|
3046
3214
|
source: string;
|
|
@@ -3057,8 +3225,8 @@ type EnvironmentRecord = {
|
|
|
3057
3225
|
* and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
|
|
3058
3226
|
* can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
|
|
3059
3227
|
* the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
|
|
3060
|
-
* harness always tries (its CLI's telemetry, say), kept out of `blocked
|
|
3061
|
-
*
|
|
3228
|
+
* harness always tries (its CLI's telemetry, say), kept out of `blocked`, so `blocked` contains
|
|
3229
|
+
* only unexpected refusals.
|
|
3062
3230
|
*/
|
|
3063
3231
|
type EgressRecord = {
|
|
3064
3232
|
policy: EgressPolicy;
|
|
@@ -3096,7 +3264,7 @@ type AttemptImage = {
|
|
|
3096
3264
|
/** Content-addressed image id (`sha256:…`). */
|
|
3097
3265
|
id: string;
|
|
3098
3266
|
agentCli: string | null;
|
|
3099
|
-
/** The harness image `install`
|
|
3267
|
+
/** The harness image the `install` layers were baked onto: `ref` itself without any. */
|
|
3100
3268
|
base: string;
|
|
3101
3269
|
/** The `docker({ install })` commands, in order. */
|
|
3102
3270
|
install: string[];
|
|
@@ -3131,11 +3299,26 @@ type SetupProcess = {
|
|
|
3131
3299
|
/** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
|
|
3132
3300
|
secrets: string[];
|
|
3133
3301
|
};
|
|
3134
|
-
/** One `
|
|
3302
|
+
/** One `c.serve` call that listened. */
|
|
3303
|
+
type ServeRecord = {
|
|
3304
|
+
/** The command as the eval passed it. */
|
|
3305
|
+
command: string;
|
|
3306
|
+
/** The port it listens on, which the handle and `c.capture` use. */
|
|
3307
|
+
port: number;
|
|
3308
|
+
/** The `PORT` it was given: the port the eval passed, or the one reeval allocated. */
|
|
3309
|
+
portEnv: number;
|
|
3310
|
+
/** It listened on a port other than `PORT`, the one that started listening after it did. */
|
|
3311
|
+
detected: boolean;
|
|
3312
|
+
/** The address it listens on there, e.g. `127.0.0.1`, `::1` or `0.0.0.0`; empty when the sandbox can't tell. */
|
|
3313
|
+
address: string;
|
|
3314
|
+
/** From the command's start until it listened. */
|
|
3315
|
+
readyMs: number;
|
|
3316
|
+
};
|
|
3317
|
+
/** One `expect` or `expect.require` matcher call an attempt made, or a check another language made. */
|
|
3135
3318
|
type AssertionRecord = {
|
|
3136
3319
|
/**
|
|
3137
|
-
* `
|
|
3138
|
-
*
|
|
3320
|
+
* `label` is the supplied `expect` label. Without one, it is the matcher plus its first
|
|
3321
|
+
* argument or named subject, cut at 60 characters. Repeats get ` #2`, ` #3`, and later suffixes.
|
|
3139
3322
|
*/
|
|
3140
3323
|
label: string;
|
|
3141
3324
|
/** The eval gave the label. */
|
|
@@ -3144,12 +3327,81 @@ type AssertionRecord = {
|
|
|
3144
3327
|
matcher: string;
|
|
3145
3328
|
/** `error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report). */
|
|
3146
3329
|
status: 'pass' | 'fail' | 'error';
|
|
3330
|
+
/**
|
|
3331
|
+
* A failure didn't end the eval's body: an `expect`. False for `expect.require` and for a check
|
|
3332
|
+
* that threw, such as Python's `assert`, which end the body when they fail.
|
|
3333
|
+
*/
|
|
3147
3334
|
soft: boolean;
|
|
3335
|
+
/** `path:line` of the call in the eval file, relative to the project root, when reeval saw it. */
|
|
3336
|
+
at?: string;
|
|
3148
3337
|
/** Why it failed or couldn't answer, cut at 1 KB. */
|
|
3149
3338
|
message?: string;
|
|
3150
3339
|
/** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
|
|
3151
3340
|
durationMs?: number;
|
|
3152
3341
|
};
|
|
3342
|
+
/**
|
|
3343
|
+
* A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
|
|
3344
|
+
* folder), or why it wasn't.
|
|
3345
|
+
*/
|
|
3346
|
+
type Snapshot = {
|
|
3347
|
+
/** `final` after the checks, or `turn-<n>` after the nth `c.send`. */
|
|
3348
|
+
name: string;
|
|
3349
|
+
/** A plain directory, or a gzipped tarball with `snapshot: { compress: true }`. */
|
|
3350
|
+
form: 'dir' | 'tar.gz';
|
|
3351
|
+
/**
|
|
3352
|
+
* Where it is, relative to the run's directory in `.reeval/runs`:
|
|
3353
|
+
* `<eval>/<cell>/<n>/snapshots/<name>` (with `.tar.gz` when compressed); null when it wasn't
|
|
3354
|
+
* taken.
|
|
3355
|
+
*/
|
|
3356
|
+
path: string | null;
|
|
3357
|
+
/** Its size on disk: the directory's files, or the tarball; 0 when it wasn't taken. */
|
|
3358
|
+
bytes: number;
|
|
3359
|
+
/** Regular files and symbolic links in it. */
|
|
3360
|
+
files: number;
|
|
3361
|
+
/** Directories left out by name, such as `node_modules/` and `.git/`, at most 50. */
|
|
3362
|
+
excluded: string[];
|
|
3363
|
+
/** Text files whose secret-looking values were masked, at most 50. */
|
|
3364
|
+
masked: string[];
|
|
3365
|
+
/** Binary files left out because they held a secret-looking value, at most 50. */
|
|
3366
|
+
dropped: string[];
|
|
3367
|
+
/**
|
|
3368
|
+
* Why it wasn't taken: `size` when the workspace was over its form's cap, `error` when the
|
|
3369
|
+
* sandbox couldn't archive it; null when it was.
|
|
3370
|
+
*/
|
|
3371
|
+
skipped: {
|
|
3372
|
+
cause: 'size' | 'error';
|
|
3373
|
+
message: string;
|
|
3374
|
+
} | null;
|
|
3375
|
+
};
|
|
3376
|
+
/** `snapshot`: the shorthand, or the object that also picks the form. */
|
|
3377
|
+
type SnapshotOption = boolean | 'turns' | SnapshotOptions;
|
|
3378
|
+
/** `snapshot` as an object, merged field by field over the levels above. */
|
|
3379
|
+
type SnapshotOptions = {
|
|
3380
|
+
/** `true` after the checks, `'turns'` after each `c.send` too, `false` never. Default `true`. */
|
|
3381
|
+
when?: boolean | 'turns';
|
|
3382
|
+
/** Write each as `<name>.tar.gz` instead of a directory. Default `false`. */
|
|
3383
|
+
compress?: boolean;
|
|
3384
|
+
};
|
|
3385
|
+
/**
|
|
3386
|
+
* One instruction of a capture, as it ended. Its screenshot is `artifacts/<name>.step-<index>.png`,
|
|
3387
|
+
* beside the capture's own `artifacts/<name>.png`.
|
|
3388
|
+
*/
|
|
3389
|
+
type CaptureStep = {
|
|
3390
|
+
/** The capture's name. */
|
|
3391
|
+
of: string;
|
|
3392
|
+
/** Its place among the capture's goal strings, from 1. */
|
|
3393
|
+
index: number;
|
|
3394
|
+
instruction: string;
|
|
3395
|
+
/** Moves made on the page for it. */
|
|
3396
|
+
moves: number;
|
|
3397
|
+
/** Milliseconds from its first look at the page to its screenshot. */
|
|
3398
|
+
ms: number;
|
|
3399
|
+
/**
|
|
3400
|
+
* `final`: the model marked a move as finishing it. `done`: the model said the goal was met.
|
|
3401
|
+
* `stopped`: it stopped the capture (its timeout, the move cap, a repeated move or an error).
|
|
3402
|
+
*/
|
|
3403
|
+
ended: 'final' | 'done' | 'stopped';
|
|
3404
|
+
};
|
|
3153
3405
|
/** Which environment ran as the control, so the studio can show it. */
|
|
3154
3406
|
type EvalCompareMeta = {
|
|
3155
3407
|
baseline: string;
|
|
@@ -3160,16 +3412,20 @@ type EvalCompareMeta = {
|
|
|
3160
3412
|
from?: 'environment' | 'empty';
|
|
3161
3413
|
};
|
|
3162
3414
|
/**
|
|
3163
|
-
* A sandbox with more privilege than the default `reeval` user:
|
|
3164
|
-
* `docker({ capabilities })`. One per level that set such a sandbox,
|
|
3415
|
+
* A `SandboxWarning` describes a sandbox with more privilege than the default `reeval` user:
|
|
3416
|
+
* `docker({ root: true })` or `docker({ capabilities })`. One per level that set such a sandbox,
|
|
3417
|
+
* however many evaluations inherit it.
|
|
3165
3418
|
*/
|
|
3166
3419
|
type SandboxWarning = {
|
|
3167
3420
|
kind: 'root' | 'capabilities';
|
|
3168
3421
|
/** Which level set the sandbox. */
|
|
3169
3422
|
level: 'config' | 'profile' | 'group' | 'eval';
|
|
3170
|
-
/**
|
|
3423
|
+
/**
|
|
3424
|
+
* The value is the level as reeval names it: the `reeval.config.ts` file, `profiles.pr.defaults`,
|
|
3425
|
+
* `group("net")`, or `evaluate("installs deps")`.
|
|
3426
|
+
*/
|
|
3171
3427
|
where: string;
|
|
3172
|
-
/** The
|
|
3428
|
+
/** The evaluation file that sets it, for a group or evaluation; null for the config or a profile. */
|
|
3173
3429
|
file: string | null;
|
|
3174
3430
|
/** The capabilities added back, for `capabilities`. */
|
|
3175
3431
|
capabilities?: string[];
|
|
@@ -3185,5 +3441,5 @@ type ResolvedGateway = {
|
|
|
3185
3441
|
}>;
|
|
3186
3442
|
};
|
|
3187
3443
|
//#endregion
|
|
3188
|
-
export {
|
|
3189
|
-
//# sourceMappingURL=types-
|
|
3444
|
+
export { FakeAnswer as $, tokenValues as $n, TranscriptEvent as $t, CompareCut as A, EFFORT_MIN_RUNS as An, PairFactor as At, EgressPolicy as B, cellEffort as Bn, QuestionInputs as Bt, CaptureWindow as C, Lift as Cn, JudgeOptions as Ct, ClaudeCodeHarnessDescriptor as D, CellEffort as Dn, OpenCodeHarnessDescriptor as Dt, ClaudeCodeEffort as E, UpstreamCount as En, McpServer as Et, Diff as F, Spread as Fn, PluginSource as Ft, EnvironmentInput as G, effortLabel as Gn, SandboxWarning as Gt, EnvVars as H, contrastEffort as Hn, ReevalConfig as Ht, DirectGatewayDescriptor as I, TOKENS_UNREPORTED as In, Profile as It, EvalFn as J, effortSummary as Jn, ServeHandle as Jt, EnvironmentSpec as K, effortOf as Kn, Secret as Kt, DirectoryOptions as L, VERDICT_TEXT as Ln, ProfileDefaults as Lt, ConfigHelpers as M, EffortMetric as Mn, PiHarnessDescriptor as Mt, ConfigJudgeOptions as N, EffortSample as Nn, PinnableKey as Nt, CodexHarnessDescriptor as O, DIRECTIONAL_MIN as On, OpenRouterGatewayDescriptor as Ot, CursorHarnessDescriptor as P, EffortVerdict as Pn, PluginPart as Pt, FakeAnalysisModel as Q, formatSeconds as Qn, TokenUsage as Qt, DockerSandboxDescriptor as R, bothUnderATenth as Rn, ProfileLimit as Rt, CaptureWait as S, JudgmentRecord as Sn, JudgeModel as St, CellRole as T, StackSummary as Tn, JudgeSubject as Tt, Environment as U, directional as Un, ResolvedGateway as Ut, EgressRecord as V, cellSamples as Vn, QuestionOptions as Vt, EnvironmentHelpers as W, effortChangeLabel as Wn, SandboxDescriptor as Wt, ExecResult as X, formatCount as Xn, SkillSource as Xt, EvalOptions as Y, effortVerdict as Yn, SetupStep as Yt, ExecRetry as Z, formatRatio as Zn, StatusRange as Zt, CapturePage as _, FamilyRate as _n, JSONValue as _t, AttemptContext as a, VertexGatewayDescriptor as an, FxHarnessDescriptor as at, CaptureUseAction as b, HistoryCell as bn, JudgeFailure as bt, BlindPair as c, Wait as cn, GooseHarnessDescriptor as ct, CaptureCall as d, AttemptRecord as dn, HarnessData as dt, Turn as en, DeclarationTree as er, FakeHarnessDescriptor as et, CaptureConsoleMessage as f, CellRecord as fn, HarnessDescriptor as ft, CaptureOptions as g, FamilyLiftSummary as gn, HostFile as gt, CaptureKey as h, FamilyLift as hn, HistoryConfig as ht, AttemptAnswer as i, Verdict as in, FakeVote as it, CompareOptions as j, EffortContrast as jn, PairVote as jt, CompareAnswer as k, EFFORT_ALPHA as kn, Outcome as kt, Budget as l, WebEventType as ln, GrokBuildHarnessDescriptor as lt, CaptureInstruction as m, EvalEntry as mn, HarnessOption as mt, AnalysisConfig as n, TurnStatus as nn, FakeSandboxDescriptor as nt, BackgroundHandle as o, VertexPublisher as on, GatewayDescriptor as ot, CaptureFailedRequest as p, CheckTally as pn, HarnessEdit as pt, ErrorKind as q, effortSample as qn, SecretTemplate as qt, AntigravityHarnessDescriptor as r, VercelGatewayDescriptor as rn, FakeScriptStep as rt, BlindAttempt as s, Viewport as sn, GatewayName as st, AgentHarnessDescriptor as t, TurnData as tn, DeclaredNode as tr, FakeJudgeModel as tt, Capture as u, ArtifactRef as un, GroupOptions as ut, CapturePageFile as v, FamilyRecord as vn, JudgeArtifactRef as vt, Cell as w, RunRecord as wn, JudgeState as wt, CaptureUseScript as x, HistoryEntry as xn, JudgeInputs as xt, CaptureUse as y, FileRef as yn, JudgeCallOptions as yt, Duration as z, cellContrast as zn, Question as zt };
|
|
3445
|
+
//# sourceMappingURL=types-DDeCnNmp.d.ts.map
|