reeval 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +803 -0
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C8xPuXYq.js → adapter-BdzhdRYo.js} +8 -6
- package/dist/adapter-BdzhdRYo.js.map +1 -0
- package/dist/{agent-CBoLndM7.js → agent-Dx-bDmlV.js} +179 -31
- package/dist/agent-Dx-bDmlV.js.map +1 -0
- package/dist/{agents-C6zhkxp3.js → agents-l0-k7vw3.js} +362 -102
- package/dist/agents-l0-k7vw3.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
- package/dist/app-files-B-CQYB0c.js.map +1 -0
- package/dist/{assemble-UPCRrtOz.js → assemble-CYMpN8LC.js} +1 -1
- package/dist/{assemble-t6h018An.js → assemble-CsZIcJo8.js} +171 -18
- package/dist/assemble-CsZIcJo8.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
- package/dist/{attempt-host-CIm1-11m.js → attempt-host-BwSmNYkX.js} +17 -8
- package/dist/attempt-host-BwSmNYkX.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
- package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
- package/dist/{boat-CJCtzfB1.js → boat-su1Y7LPd.js} +17 -14
- package/dist/boat-su1Y7LPd.js.map +1 -0
- package/dist/bridge/index.mjs +113 -8
- package/dist/{capture-iBzFiC0j.js → capture-BqIAeFyC.js} +55 -29
- package/dist/{capture-iBzFiC0j.js.map → capture-BqIAeFyC.js.map} +1 -1
- package/dist/capture-D0Ln5aCP.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-20Fpvs5H.js → cli-s_3opn_I.js} +219 -109
- package/dist/cli-s_3opn_I.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
- package/dist/cli-spec-DxzIjmq_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-C5fIzHQa.js +803 -0
- package/dist/client-C5fIzHQa.js.map +1 -0
- package/dist/{command-CRh-cb6Z.js → command-B5OAIjp7.js} +25 -21
- package/dist/command-B5OAIjp7.js.map +1 -0
- package/dist/{command-BLNVDiIg.js → command-BJ7KqczP.js} +10 -10
- package/dist/{command-BLNVDiIg.js.map → command-BJ7KqczP.js.map} +1 -1
- package/dist/{command-aUJvENwy.js → command-BXLZsEwQ.js} +16 -16
- package/dist/{command-aUJvENwy.js.map → command-BXLZsEwQ.js.map} +1 -1
- package/dist/{command-B5OyHpOu.js → command-DA_SHQUQ.js} +16 -16
- package/dist/{command-B5OyHpOu.js.map → command-DA_SHQUQ.js.map} +1 -1
- package/dist/{commands-CWy-WndG.js → commands-BKBamOcX.js} +174 -40
- package/dist/commands-BKBamOcX.js.map +1 -0
- package/dist/{commands-CfdBX-mc.js → commands-BPvAOEmb.js} +1 -1
- package/dist/{config-B9mB1kF5.js → config-C0URar0t.js} +255 -89
- package/dist/config-C0URar0t.js.map +1 -0
- package/dist/{context-Bkv3ndjq.js → context-Sj58aJEu.js} +2 -2
- package/dist/{context-Bkv3ndjq.js.map → context-Sj58aJEu.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-BSn9xjmo.js → cursor-catalog-FWgSo-T1.js} +4 -4
- package/dist/{cursor-catalog-BSn9xjmo.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
- package/dist/db-Bc4L-Wcr.js +2 -0
- package/dist/{db-zgC1tvKB.js → db-BjAWxVAG.js} +142 -248
- package/dist/db-BjAWxVAG.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BAy8ixd1.js → default-effort-CWlhZ6U1.js} +1 -1
- package/dist/{default-effort-DGhQmu95.js → default-effort-Ck2GYIxX.js} +4 -4
- package/dist/{default-effort-DGhQmu95.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
- package/dist/{dist-CAFD1yfv.js → dist-DeqeCUZe.js} +15 -8
- package/dist/dist-DeqeCUZe.js.map +1 -0
- package/dist/{docker-DAMnm3L9.js → docker-CDdK6Ity.js} +49 -22
- package/dist/docker-CDdK6Ity.js.map +1 -0
- package/dist/{docker-Cqw-CyWz.js → docker-D74l67k8.js} +4 -4
- package/dist/{docker-Cqw-CyWz.js.map → docker-D74l67k8.js.map} +1 -1
- package/dist/{docker-Dr4XeDc8.js → docker-DMug3l-a.js} +1 -1
- package/dist/{doctor-B2PohneB.js → doctor-eFxyn6at.js} +85 -46
- package/dist/doctor-eFxyn6at.js.map +1 -0
- package/dist/{environment-DB8o3hGi.js → environment-CJIKz-Ud.js} +49 -11
- package/dist/environment-CJIKz-Ud.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-D3sbAzt-.js → eval-source-CXIRJ91v.js} +24 -12
- package/dist/eval-source-CXIRJ91v.js.map +1 -0
- package/dist/{evaluate-CwKeN_fo.js → evaluate-DA4QsrHO.js} +792 -281
- package/dist/evaluate-DA4QsrHO.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
- package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
- package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
- package/dist/{files-BiFU16BW.js → files-BVVWAS0A.js} +6 -6
- package/dist/{files-BiFU16BW.js.map → files-BVVWAS0A.js.map} +1 -1
- package/dist/files-BaqXvB88.js +1 -0
- package/dist/{files-BEwfjD8J.js → files-Bey1VAex.js} +3 -3
- package/dist/{files-BEwfjD8J.js.map → files-Bey1VAex.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/files-CNHhSyb3.js +1 -0
- package/dist/files-CW1lChNh.js +1 -0
- package/dist/{files-BRzV7aPh.js → files-D6j-XpaK.js} +5 -5
- package/dist/{files-BRzV7aPh.js.map → files-D6j-XpaK.js.map} +1 -1
- package/dist/{files-Cl7BQ5gE.js → files-F44NRCN0.js} +143 -20
- package/dist/files-F44NRCN0.js.map +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-fDiJ8UIV.js → git-Jqkf9SBa.js} +3 -3
- package/dist/{git-fDiJ8UIV.js.map → git-Jqkf9SBa.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-BNgcR77K.js → images-Dsc4Wi-F.js} +65 -30
- package/dist/images-Dsc4Wi-F.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/init-C0oUG3jh.js +1 -0
- package/dist/{init-DbuZVRrW.js → init-QB5ecPDA.js} +33 -14
- package/dist/init-QB5ecPDA.js.map +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
- package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
- package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
- package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
- package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
- package/dist/layout-DfQa4mJs.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
- package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
- package/dist/live-DwjWJTdn.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
- package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
- package/dist/{live-CnGrRM6j.js → live-y1p3oNXH.js} +9 -9
- package/dist/{live-CnGrRM6j.js.map → live-y1p3oNXH.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
- package/dist/model-3pZZzloU.js.map +1 -0
- package/dist/{model-047Dcd-a.js → model-DEjwetkJ.js} +1 -1
- package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{model-DJjLw-0r.js → model-jRKSQBg6.js} +5 -5
- package/dist/{model-DJjLw-0r.js.map → model-jRKSQBg6.js.map} +1 -1
- package/dist/model-nfewGTzB.js +1 -0
- package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
- package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
- package/dist/options-DpsJ_5CU.js +1 -0
- package/dist/{patch-DH4e_Gkd.js → patch-Bx-qp8Li.js} +2 -2
- package/dist/{patch-DH4e_Gkd.js.map → patch-Bx-qp8Li.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-Cgq9NCEX.js → plan-ChO0A-xn.js} +63 -12
- package/dist/plan-ChO0A-xn.js.map +1 -0
- package/dist/{plugins-BkmPxDiY.js → plugins-BioMWncI.js} +636 -9
- package/dist/plugins-BioMWncI.js.map +1 -0
- package/dist/{preflight-R-InZCQb.js → preflight-BUjk_6wT.js} +1 -1
- package/dist/{preflight-CtlloeOQ.js → preflight-CJSGzMzc.js} +8 -9
- package/dist/preflight-CJSGzMzc.js.map +1 -0
- package/dist/preflight-DD3fI4Hz.js +1 -0
- package/dist/{preflight-BElE_q0u.js → preflight-qFCXmxha.js} +4 -4
- package/dist/{preflight-BElE_q0u.js.map → preflight-qFCXmxha.js.map} +1 -1
- package/dist/{prepare-C7f5mPl5.js → prepare-D7oSSFNm.js} +9 -9
- package/dist/{prepare-C7f5mPl5.js.map → prepare-D7oSSFNm.js.map} +1 -1
- package/dist/{prepare-CzkAdbXm.js → prepare-xHU2BpN-.js} +2 -2
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-zsc4--dP.js → provenance-B-flBAzR.js} +5 -5
- package/dist/{provenance-zsc4--dP.js.map → provenance-B-flBAzR.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/{providers-B8xYMeah.js → providers-B6StVyfG.js} +215 -48
- package/dist/providers-B6StVyfG.js.map +1 -0
- package/dist/providers-Dljf6al6.js +1 -0
- package/dist/query-BAc2tQZj.js +1 -0
- package/dist/{query-Bb3SYICk.js → query-CkLmJv7R.js} +2 -2
- package/dist/{query-Bb3SYICk.js.map → query-CkLmJv7R.js.map} +1 -1
- package/dist/{questions-CWlR9ffu.js → questions-CJIh2FSX.js} +7 -7
- package/dist/{questions-CWlR9ffu.js.map → questions-CJIh2FSX.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
- package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
- package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
- package/dist/redact-DL2BTykw.js.map +1 -0
- package/dist/{remarks-vtkPmQb9.js → remarks-B3wjKscW.js} +46 -11
- package/dist/remarks-B3wjKscW.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-BgYWqogI.js → remote-C88amFRN.js} +6 -6
- package/dist/{remote-BgYWqogI.js.map → remote-C88amFRN.js.map} +1 -1
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/resolve-DW92wpIZ.js +1 -0
- package/dist/{resolve-Do0ZaYH6.js → resolve-G576PiFJ.js} +14 -11
- package/dist/resolve-G576PiFJ.js.map +1 -0
- package/dist/{results-JyuM64gy.js → results-zVHIW6xw.js} +9 -22
- package/dist/results-zVHIW6xw.js.map +1 -0
- package/dist/{resume-AW5oXzUc.js → resume-D_PcUXfG.js} +94 -16
- package/dist/resume-D_PcUXfG.js.map +1 -0
- package/dist/{retry-ChzISxmy.js → retry-BvTzCxdc.js} +4 -4
- package/dist/{retry-ChzISxmy.js.map → retry-BvTzCxdc.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
- package/dist/routes-B0F2R-K7.js.map +1 -0
- package/dist/{runtime-BpW1f1Mx.js → runtime-B3r_pAq6.js} +4 -4
- package/dist/{runtime-BpW1f1Mx.js.map → runtime-B3r_pAq6.js.map} +1 -1
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-kSgr7vo5.js → sandbox-Duh3z1TV.js} +3 -3
- package/dist/{sandbox-kSgr7vo5.js.map → sandbox-Duh3z1TV.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CGEsvVRG.js +1 -0
- package/dist/{server-Cn8VqO9S.js → server-DUWMRKik.js} +8 -8
- package/dist/{server-Cn8VqO9S.js.map → server-DUWMRKik.js.map} +1 -1
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-CYOhR98B.js → source-CHuN4Nfg.js} +2 -2
- package/dist/{source-CYOhR98B.js.map → source-CHuN4Nfg.js.map} +1 -1
- package/dist/source-Wz1Nrfue.js +1 -0
- package/dist/src--yl1OypK.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
- package/dist/src-B-HuDqpH.js.map +1 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Ck2r_EMT.js → src-BHJ4MWvY.js} +504 -119
- package/dist/src-BHJ4MWvY.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
- package/dist/src-BI0v8qlZ.js.map +1 -0
- package/dist/src-BL-_0QoE.js +1 -0
- package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
- package/dist/src-BqNzHHWn.js.map +1 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C-mOSczH.js +2 -0
- package/dist/src-CZaYu6ER.js +3 -0
- package/dist/src-CeBhwD8k.js +2 -0
- package/dist/src-CkEiiyON.js +2 -0
- package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
- package/dist/src-CmA5Yev6.js.map +1 -0
- package/dist/src-CwH_nEaw.js +2 -0
- package/dist/src-DA95rM6v.js +4 -0
- package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
- package/dist/src-DL-EE4DU.js.map +1 -0
- package/dist/src-De0KY5S_.js +2 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/state-B3jex5KU.js +1 -0
- package/dist/{state-BOLt8pJg.js → state-DUBRDyMk.js} +22 -22
- package/dist/{state-BOLt8pJg.js.map → state-DUBRDyMk.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
- package/dist/suite-Bhs9-Z7O.js +2 -0
- package/dist/{summary-DU5zaibx.js → summary-CSp4cfnr.js} +25 -7
- package/dist/{summary-DU5zaibx.js.map → summary-CSp4cfnr.js.map} +1 -1
- package/dist/sync-CxAY6Dki.js +559 -0
- package/dist/sync-CxAY6Dki.js.map +1 -0
- package/dist/tart-CcrrtSrp.js +704 -0
- package/dist/tart-CcrrtSrp.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-F-6seRC9.js +192 -0
- package/dist/tart-bake-F-6seRC9.js.map +1 -0
- package/dist/tart-bake-RK7RaT9E.js +1 -0
- package/dist/tart-cli-B86Jqo1t.js +89 -0
- package/dist/tart-cli-B86Jqo1t.js.map +1 -0
- package/dist/tart-cli-ClvRniLQ.js +1 -0
- package/dist/tart-hosts-Brii7zHM.js +1 -0
- package/dist/tart-hosts-DPyZsg-1.js +313 -0
- package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
- package/dist/tart-images-CWSf03w8.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-CVDdJy0H.js +433 -0
- package/dist/tool-class-CVDdJy0H.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
- package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +27 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-DAIiLiV3.js → views-DRxXCyIH.js} +6 -6
- package/dist/{views-DAIiLiV3.js.map → views-DRxXCyIH.js.map} +1 -1
- package/dist/views-i67V1ly3.js +2 -0
- package/dist/{worker-entry-BIROTPX1.js → worker-entry-C3Q4U63F.js} +12 -9
- package/dist/worker-entry-C3Q4U63F.js.map +1 -0
- package/dist/workspace-DhUOW8if.js +1 -0
- package/dist/{workspace-uuE51x35.js → workspace-P72SXLXr.js} +7 -7
- package/dist/{workspace-uuE51x35.js.map → workspace-P72SXLXr.js.map} +1 -1
- package/package.json +21 -10
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C8xPuXYq.js.map +0 -1
- package/dist/agent-CBoLndM7.js.map +0 -1
- package/dist/agents-C6zhkxp3.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-t6h018An.js.map +0 -1
- package/dist/attempt-host-CIm1-11m.js.map +0 -1
- package/dist/boat-CJCtzfB1.js.map +0 -1
- package/dist/capture-DoxFUN3J.js +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-20Fpvs5H.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-CRh-cb6Z.js.map +0 -1
- package/dist/commands-CWy-WndG.js.map +0 -1
- package/dist/config-B9mB1kF5.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-P2fXw4nx.js +0 -2
- package/dist/db-zgC1tvKB.js.map +0 -1
- package/dist/dist-CAFD1yfv.js.map +0 -1
- package/dist/docker-DAMnm3L9.js.map +0 -1
- package/dist/doctor-B2PohneB.js.map +0 -1
- package/dist/environment-DB8o3hGi.js.map +0 -1
- package/dist/eval-source-D3sbAzt-.js.map +0 -1
- package/dist/evaluate-CwKeN_fo.js.map +0 -1
- package/dist/files-B4KQUiM7.js +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-C3mCC8Mn.js +0 -1
- package/dist/files-Cl7BQ5gE.js.map +0 -1
- package/dist/files-DnlA1A7N.js +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-BNgcR77K.js.map +0 -1
- package/dist/init-4RPeD-K-.js +0 -1
- package/dist/init-DbuZVRrW.js.map +0 -1
- package/dist/kinds-C1oY3qyC.js +0 -697
- package/dist/kinds-C1oY3qyC.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CJCK11nh.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/options-CHubslQf.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-Cgq9NCEX.js.map +0 -1
- package/dist/plugins-BkmPxDiY.js.map +0 -1
- package/dist/preflight-4gWSy0Pe.js +0 -1
- package/dist/preflight-CtlloeOQ.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-B8xYMeah.js.map +0 -1
- package/dist/providers-CVu8e0xS.js +0 -1
- package/dist/query-DIfkUcPh.js +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/remarks-vtkPmQb9.js.map +0 -1
- package/dist/resolve-Do0ZaYH6.js.map +0 -1
- package/dist/resolve-amw_2Odz.js +0 -1
- package/dist/results-JyuM64gy.js.map +0 -1
- package/dist/resume-AW5oXzUc.js.map +0 -1
- package/dist/server-2_gUnVvf.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-CY3etKzw.js +0 -1
- package/dist/src-6Zzyg4Qd.js +0 -2
- package/dist/src-BDhcPtnI.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BRMsk9nh.js +0 -2
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-Ck2r_EMT.js.map +0 -1
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DB7N8TYJ.js +0 -2
- package/dist/src-DPDaAYwh.js +0 -2
- package/dist/src-Dbk19BiP.js +0 -2
- package/dist/src-DtodMT59.js +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-T-OeuJLr.js +0 -2
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-zntEDlnx.js +0 -3
- package/dist/state-kimtIhBP.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/tart-BTVXp8rz.js +0 -373
- package/dist/tart-BTVXp8rz.js.map +0 -1
- package/dist/tart-cli-BIC9HqPg.js +0 -37
- package/dist/tart-cli-BIC9HqPg.js.map +0 -1
- package/dist/tart-cli-BMrkmBjI.js +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-DtRhj2Tn.js +0 -2
- package/dist/worker-entry-BIROTPX1.js.map +0 -1
- package/dist/workspace-CcQRT31C.js +0 -1
|
@@ -11,6 +11,10 @@ type EffortSample = {
|
|
|
11
11
|
filesChanged: number;
|
|
12
12
|
/** Total tokens when this attempt reported usage; omitted when the harness dropped it. */
|
|
13
13
|
tokens?: number;
|
|
14
|
+
/** Where its agent time went; omitted for an attempt recorded without it. */
|
|
15
|
+
split?: TimeSplit;
|
|
16
|
+
/** The agent's cost in US dollars (`cost.usd`); omitted when the attempt has none. */
|
|
17
|
+
usd?: number;
|
|
14
18
|
};
|
|
15
19
|
type Spread = {
|
|
16
20
|
median: number;
|
|
@@ -28,6 +32,13 @@ type CellEffort = {
|
|
|
28
32
|
filesChanged: number;
|
|
29
33
|
/** Over the counted attempts that reported usage; omitted when none did. */
|
|
30
34
|
tokens?: Spread;
|
|
35
|
+
/**
|
|
36
|
+
* Median model, tool, waiting and other time over the counted attempts that have a split;
|
|
37
|
+
* omitted when none does. `modelMs` is null when any of them saw no model calls.
|
|
38
|
+
*/
|
|
39
|
+
split?: TimeSplit;
|
|
40
|
+
/** The agent's cost in US dollars over the counted attempts that have one; omitted when none does. */
|
|
41
|
+
usd?: Spread;
|
|
31
42
|
};
|
|
32
43
|
/**
|
|
33
44
|
* `fewer` / `more`: the exact rank test's p is below `EFFORT_ALPHA`. `noise`: it isn't.
|
|
@@ -67,11 +78,13 @@ type EffortContrast = {
|
|
|
67
78
|
/** Set when some counted attempts on either side reported no usage. Not a verdict. */
|
|
68
79
|
tokensNote?: typeof TOKENS_UNREPORTED;
|
|
69
80
|
};
|
|
70
|
-
declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings' | 'usage' | 'changedFiles'>) => EffortSample;
|
|
81
|
+
declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings' | 'usage' | 'changedFiles' | 'timeSplit' | 'cost'>) => EffortSample;
|
|
71
82
|
/** The files the workspace diff changed, a collapsed directory counting each of its files. */
|
|
72
83
|
declare const filesChanged: (files: AttemptRecord['changedFiles']) => number;
|
|
73
84
|
/** Token totals from the attempts that reported usage; empty when none did. */
|
|
74
85
|
declare const tokenValues: (samples: EffortSample[]) => number[];
|
|
86
|
+
/** The samples' median split, `modelMs` null when any sample's is; undefined when none has one. */
|
|
87
|
+
declare function splitOf(samples: EffortSample[]): TimeSplit | undefined;
|
|
75
88
|
declare function effortOf(samples: EffortSample[]): CellEffort | null;
|
|
76
89
|
/**
|
|
77
90
|
* The verdict of an exact rank test of `xs` (variant) against `ys` (control). Its direction is
|
|
@@ -109,6 +122,11 @@ declare const formatRatio: (r: number | null) => string;
|
|
|
109
122
|
declare const bothUnderATenth: (a: number, b: number) => boolean;
|
|
110
123
|
/** Seconds for agent time: `<0.1s`, `0.4s`, `21s`, `1m 05s`. */
|
|
111
124
|
declare function formatSeconds(ms: number): string;
|
|
125
|
+
/**
|
|
126
|
+
* `model 1m 05s · tools 12s · waiting 1m 21s · other 3m 22s`: model left out when unknown,
|
|
127
|
+
* waiting when there was none.
|
|
128
|
+
*/
|
|
129
|
+
declare const splitLabel: (s: TimeSplit) => string;
|
|
112
130
|
declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
|
|
113
131
|
/**
|
|
114
132
|
* `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
|
|
@@ -118,10 +136,16 @@ declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
|
|
|
118
136
|
* times both under a tenth of a second have no ratio: `agent <0.1s vs <0.1s`.
|
|
119
137
|
*/
|
|
120
138
|
declare function effortLabel(c: EffortContrast): string;
|
|
121
|
-
/**
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
139
|
+
/**
|
|
140
|
+
* One cell's effort on its own: `tools 43 · agent 46s`, plus tokens and cost when any attempt
|
|
141
|
+
* reported them.
|
|
142
|
+
*/
|
|
143
|
+
declare const effortSummary: (e: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens' | 'usd'>) => string;
|
|
144
|
+
/**
|
|
145
|
+
* The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`, then
|
|
146
|
+
* `waiting 0s → 1m 21s` when either side waited on a background job.
|
|
147
|
+
*/
|
|
148
|
+
declare function effortChangeLabel(base: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens' | 'split' | 'usd'>, head: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens' | 'split' | 'usd'>): string;
|
|
125
149
|
/** Stays 1 while reeval is pre-release: the record changes in place. */
|
|
126
150
|
declare const SCHEMA_VERSION = 1;
|
|
127
151
|
type ArtifactLevel = 'all' | 'failures' | 'none';
|
|
@@ -132,6 +156,16 @@ type FileRef = {
|
|
|
132
156
|
sha256: string;
|
|
133
157
|
bytes: number;
|
|
134
158
|
};
|
|
159
|
+
type RunDebug = {
|
|
160
|
+
/** `requests`: each attempt's `requests.ndjson`; `all`: its `debug.log` and `trace.ndjson` too. */
|
|
161
|
+
level: 'requests' | 'all';
|
|
162
|
+
files: FileRef[];
|
|
163
|
+
/** Files left out because the secret scan found something in them, with what it found. */
|
|
164
|
+
dropped?: Array<{
|
|
165
|
+
path: string;
|
|
166
|
+
note: string;
|
|
167
|
+
}>;
|
|
168
|
+
};
|
|
135
169
|
/** A generic attempt file from `c.capture()`. */
|
|
136
170
|
type ArtifactRef = FileRef & {
|
|
137
171
|
name: string;
|
|
@@ -222,6 +256,10 @@ type JudgmentRecord = {
|
|
|
222
256
|
upstream?: string;
|
|
223
257
|
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
224
258
|
costUsd?: number;
|
|
259
|
+
/** The call's tokens; unset when the provider reported none, and for the fake judge. */
|
|
260
|
+
usage?: TokenUsage;
|
|
261
|
+
/** `costUsd` as reported, else the call's tokens at the judge model's list price; unset when neither is known. */
|
|
262
|
+
cost?: CostUsd;
|
|
225
263
|
/**
|
|
226
264
|
* `<index>:<sha256(question) prefix>`: the same question at the same place in the evaluation, so a
|
|
227
265
|
* verdict can be followed across runs (and an edited question starts a new series). `<index>`
|
|
@@ -348,8 +386,7 @@ type AttemptRecord = {
|
|
|
348
386
|
* `text` is the agent's final message, cut at 8 KB; the whole of it is in the `transcript.jsonl`
|
|
349
387
|
* file.
|
|
350
388
|
* `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
|
|
351
|
-
* `reasoningMs` is how long the turn's
|
|
352
|
-
* time it.
|
|
389
|
+
* `reasoningMs` is how long the turn's reasoning took, null when the harness can't time it.
|
|
353
390
|
*/
|
|
354
391
|
turns: Array<{
|
|
355
392
|
prompt: string;
|
|
@@ -361,6 +398,11 @@ type AttemptRecord = {
|
|
|
361
398
|
usage: TokenUsage | null;
|
|
362
399
|
/** The model the harness says served the turn; null when it reports none. */
|
|
363
400
|
model: string | null;
|
|
401
|
+
/**
|
|
402
|
+
* What the harness says the turn cost (Claude Code, pi), else its tokens at the list price of
|
|
403
|
+
* its model (or the cell's); unset when neither is known.
|
|
404
|
+
*/
|
|
405
|
+
cost?: CostUsd;
|
|
364
406
|
/** Event types the harness can't report; see `Turn.unreported`. */
|
|
365
407
|
unreported: WebEventType[];
|
|
366
408
|
/** The turn's first error, such as a provider's billing or auth error, cut at 1 KB; else null. */
|
|
@@ -377,9 +419,25 @@ type AttemptRecord = {
|
|
|
377
419
|
* script's.
|
|
378
420
|
*/
|
|
379
421
|
reasoningMs?: number | null;
|
|
422
|
+
/**
|
|
423
|
+
* Where the turn's time went: its model calls, tool calls, the background jobs it waited on,
|
|
424
|
+
* and the rest, summing to `durationMs` (`TimeSplit`). Absent in records from before reeval
|
|
425
|
+
* split it.
|
|
426
|
+
*/
|
|
427
|
+
timeSplit?: TimeSplit;
|
|
380
428
|
}>;
|
|
429
|
+
/**
|
|
430
|
+
* The turns' `timeSplit` added up, its `modelMs` null when any turn's is. Absent when no turn
|
|
431
|
+
* has one.
|
|
432
|
+
*/
|
|
433
|
+
timeSplit?: TimeSplit;
|
|
381
434
|
/** Tokens across the attempt's turns; null when the harness reported none. */
|
|
382
435
|
usage: TokenUsage | null;
|
|
436
|
+
/**
|
|
437
|
+
* The gateway's billed `reportedCostUsd`, else the turns' costs added up; unset when a turn with
|
|
438
|
+
* tokens has none.
|
|
439
|
+
*/
|
|
440
|
+
cost?: CostUsd;
|
|
383
441
|
/**
|
|
384
442
|
* Tokens the capture vision loop used. Infrastructure only: never included
|
|
385
443
|
* in `usage`, effort, compare, or lift.
|
|
@@ -418,7 +476,10 @@ type AttemptRecord = {
|
|
|
418
476
|
maxToolMs: number | null;
|
|
419
477
|
};
|
|
420
478
|
judgements: JudgmentRecord[];
|
|
421
|
-
/**
|
|
479
|
+
/**
|
|
480
|
+
* Harness session starts that didn't become ready within `startTimeout`, or failed on a transient
|
|
481
|
+
* error (a 5xx or a network error), and were started again (at most one).
|
|
482
|
+
*/
|
|
422
483
|
startRetries?: StartRetry[];
|
|
423
484
|
/**
|
|
424
485
|
* The infra errors this attempt ended with before `reeval run --retry-errors` or `--resume` ran
|
|
@@ -810,6 +871,11 @@ type CellRecord = Cell & {
|
|
|
810
871
|
* gates on that majority. `runs` still counts every planned attempt, the skipped ones included.
|
|
811
872
|
*/
|
|
812
873
|
bestOf?: number;
|
|
874
|
+
/** The cell's attempts' agent and judge costs added up; unset where none has one. */
|
|
875
|
+
cost?: {
|
|
876
|
+
agent?: CostTotal;
|
|
877
|
+
judge?: CostTotal;
|
|
878
|
+
};
|
|
813
879
|
runs: number;
|
|
814
880
|
pass: number;
|
|
815
881
|
fail: number;
|
|
@@ -867,7 +933,7 @@ type RunSummary = {
|
|
|
867
933
|
infraError: number;
|
|
868
934
|
skipped: number;
|
|
869
935
|
};
|
|
870
|
-
/** The tokens the run used, as the harnesses count them
|
|
936
|
+
/** The tokens the run used, as the harnesses count them, and what they cost. */
|
|
871
937
|
type Cost = {
|
|
872
938
|
/** Prompt tokens, cache reads and writes included. */
|
|
873
939
|
inputTokens: number | null;
|
|
@@ -875,6 +941,10 @@ type Cost = {
|
|
|
875
941
|
/** The same totals split as each turn reports them. */
|
|
876
942
|
usage: TokenUsage | null;
|
|
877
943
|
source: 'reported' | 'unreported';
|
|
944
|
+
/** The attempts' costs added up; unset when none has one. */
|
|
945
|
+
agentUsd?: CostTotal;
|
|
946
|
+
/** The judge calls' costs added up; unset when none has one. */
|
|
947
|
+
judgeUsd?: CostTotal;
|
|
878
948
|
};
|
|
879
949
|
/**
|
|
880
950
|
* The `reeval run` settings that shaped a run's plan, as it was first started. A resume keeps
|
|
@@ -988,6 +1058,11 @@ type RunRecord = {
|
|
|
988
1058
|
cost: Cost;
|
|
989
1059
|
/** Which artifacts sit next to this record. Refs to files that weren't kept are removed. */
|
|
990
1060
|
artifacts: ArtifactLevel;
|
|
1061
|
+
/**
|
|
1062
|
+
* The attempts' debug data uploaded with the run to its store (`history.debug`), each file by
|
|
1063
|
+
* its path in `.reeval/debug/<run>/` (`<eval>/<cell>/<n>/requests.ndjson`); absent when none was.
|
|
1064
|
+
*/
|
|
1065
|
+
debug?: RunDebug;
|
|
991
1066
|
redaction: {
|
|
992
1067
|
version: 1;
|
|
993
1068
|
};
|
|
@@ -1205,6 +1280,11 @@ type TranscriptEvent = EventTime & ({
|
|
|
1205
1280
|
durationMs: number | null;
|
|
1206
1281
|
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
1207
1282
|
stalled?: boolean;
|
|
1283
|
+
/**
|
|
1284
|
+
* Set when the harness never reported the call's end, so `durationMs` runs to the next
|
|
1285
|
+
* event's `atMs` (the turn's end for the last).
|
|
1286
|
+
*/
|
|
1287
|
+
durationInferred?: true;
|
|
1208
1288
|
/** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
|
|
1209
1289
|
subagent?: string;
|
|
1210
1290
|
}) | {
|
|
@@ -1221,6 +1301,11 @@ type TranscriptEvent = EventTime & ({
|
|
|
1221
1301
|
durationMs: number | null;
|
|
1222
1302
|
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
1223
1303
|
stalled?: boolean;
|
|
1304
|
+
/**
|
|
1305
|
+
* Set when the harness never reported the call's end, so `durationMs` runs to the next
|
|
1306
|
+
* event's `atMs` (the turn's end for the last).
|
|
1307
|
+
*/
|
|
1308
|
+
durationInferred?: true;
|
|
1224
1309
|
/** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
|
|
1225
1310
|
subagent?: string;
|
|
1226
1311
|
} | {
|
|
@@ -1232,6 +1317,11 @@ type TranscriptEvent = EventTime & ({
|
|
|
1232
1317
|
durationMs: number | null;
|
|
1233
1318
|
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
1234
1319
|
stalled?: boolean;
|
|
1320
|
+
/**
|
|
1321
|
+
* Set when the harness never reported the call's end, so `durationMs` runs to the next
|
|
1322
|
+
* event's `atMs` (the turn's end for the last).
|
|
1323
|
+
*/
|
|
1324
|
+
durationInferred?: true;
|
|
1235
1325
|
/** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
|
|
1236
1326
|
subagent?: string;
|
|
1237
1327
|
} | {
|
|
@@ -1240,11 +1330,31 @@ type TranscriptEvent = EventTime & ({
|
|
|
1240
1330
|
tool: string;
|
|
1241
1331
|
input: unknown;
|
|
1242
1332
|
output: unknown;
|
|
1333
|
+
/**
|
|
1334
|
+
* What `output` holds: `model`, the result as the harness gave it to the model; `full`,
|
|
1335
|
+
* the tool's whole output, which can be more than the model saw (the harness cut it for
|
|
1336
|
+
* the model, or saved it to a file and gave the model a pointer, which reeval read).
|
|
1337
|
+
*/
|
|
1338
|
+
outputView?: 'model' | 'full';
|
|
1339
|
+
/**
|
|
1340
|
+
* Set when `output` is only part of the output, with a line where the rest was cut:
|
|
1341
|
+
* `transcript.jsonl` keeps the first 96K and last 16K characters of one past 128K (of its
|
|
1342
|
+
* JSON, for structured output), and the harness adapter keeps the first and last 512 KiB
|
|
1343
|
+
* of a saved output past 1 MiB. `bytes` is the whole output's size in UTF-8.
|
|
1344
|
+
*/
|
|
1345
|
+
outputTruncated?: {
|
|
1346
|
+
bytes: number;
|
|
1347
|
+
};
|
|
1243
1348
|
/** Whether the call succeeded; null when the harness never reported its result. */
|
|
1244
1349
|
ok: boolean | null;
|
|
1245
1350
|
durationMs: number | null;
|
|
1246
1351
|
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
1247
1352
|
stalled?: boolean;
|
|
1353
|
+
/**
|
|
1354
|
+
* Set when the harness never reported the call's end, so `durationMs` runs to the next
|
|
1355
|
+
* event's `atMs` (the turn's end for the last).
|
|
1356
|
+
*/
|
|
1357
|
+
durationInferred?: true;
|
|
1248
1358
|
/** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
|
|
1249
1359
|
subagent?: string;
|
|
1250
1360
|
} | {
|
|
@@ -1255,7 +1365,40 @@ type TranscriptEvent = EventTime & ({
|
|
|
1255
1365
|
{
|
|
1256
1366
|
type: 'retry';
|
|
1257
1367
|
message: string;
|
|
1258
|
-
});
|
|
1368
|
+
} | ModelCallEvent);
|
|
1369
|
+
/**
|
|
1370
|
+
* One of the agent's calls to its model API, as the egress gateway saw it (in a sandbox behind
|
|
1371
|
+
* it, such as Docker's): when it started, when its response's first byte came (on a streamed call,
|
|
1372
|
+
* about its first token) and when it ended, with the tokens and the model its response named.
|
|
1373
|
+
* Cursor sends a whole turn over one long stream to `api2.cursor.sh` (`AgentService/RunSSE`),
|
|
1374
|
+
* so a Cursor turn has one call, with `stream: true`, its tool calls inside it and its tokens
|
|
1375
|
+
* only in the turn's `usage`.
|
|
1376
|
+
*/
|
|
1377
|
+
type ModelCallEvent = {
|
|
1378
|
+
type: 'model-call';
|
|
1379
|
+
/** `POST api.anthropic.com/v1/messages`: the method, host and path. */
|
|
1380
|
+
request: string;
|
|
1381
|
+
/** The response's HTTP status; null when none came. */
|
|
1382
|
+
status: number | null;
|
|
1383
|
+
durationMs: number;
|
|
1384
|
+
/** Milliseconds from the call's start to its response body's first byte; null when none came. */
|
|
1385
|
+
firstByteMs: number | null;
|
|
1386
|
+
/** The tokens its response body named; null when it named none (Cursor's are protobuf). */
|
|
1387
|
+
usage: TokenUsage | null;
|
|
1388
|
+
/** The model that served it, from its request or response; null when neither named one. */
|
|
1389
|
+
model: string | null;
|
|
1390
|
+
/** Set on a call that carries the whole turn (Cursor's), so it isn't counted as model time. */
|
|
1391
|
+
stream?: true;
|
|
1392
|
+
/**
|
|
1393
|
+
* `main` for one of the agent's own steps, `side` for a call its CLI made around them: a
|
|
1394
|
+
* session title, a web search's own model call, Antigravity's Flash Lite calls. A call is
|
|
1395
|
+
* `main` when its tokens are one of the turn's steps' as the harness reported them. Otherwise
|
|
1396
|
+
* it's `side` when its request declared at most one tool while another call in the attempt
|
|
1397
|
+
* declared more, or when another model served it than served the turn's other calls, and `main`
|
|
1398
|
+
* when neither holds. Absent in records from before roles.
|
|
1399
|
+
*/
|
|
1400
|
+
role?: 'main' | 'side';
|
|
1401
|
+
};
|
|
1259
1402
|
/** The transcript events a harness may be unable to report. */
|
|
1260
1403
|
type WebEventType = 'web-search' | 'web-fetch';
|
|
1261
1404
|
type ExecResult = {
|
|
@@ -1270,6 +1413,13 @@ type ExecRetry = {
|
|
|
1270
1413
|
/** Milliseconds to wait before each extra try. Default 1000. */
|
|
1271
1414
|
interval?: number;
|
|
1272
1415
|
};
|
|
1416
|
+
/**
|
|
1417
|
+
* Variables `c.exec` sets for one command, over the environment's own. A value whose name looks
|
|
1418
|
+
* like a secret's (`KEY`, `TOKEN`, `SECRET`, `PASSWORD`, `CREDENTIAL`) is masked in everything
|
|
1419
|
+
* reeval writes, as a host variable's is. The host value of a secret the environment brokers
|
|
1420
|
+
* goes in as its placeholder, which the gateway swaps back on that secret's hosts.
|
|
1421
|
+
*/
|
|
1422
|
+
type ExecEnv = Record<string, string>;
|
|
1273
1423
|
/** One run of a `c.exec` command that had `retries`. */
|
|
1274
1424
|
type ExecTry = {
|
|
1275
1425
|
exitCode: number;
|
|
@@ -1487,6 +1637,45 @@ type TokenUsage = {
|
|
|
1487
1637
|
cacheWrite: number;
|
|
1488
1638
|
reasoning: number | null;
|
|
1489
1639
|
};
|
|
1640
|
+
/**
|
|
1641
|
+
* Where a turn's time went, in ms, summing to its `durationMs`. Each moment counts once, as the
|
|
1642
|
+
* first of: `waitingMs`, a background job the agent waited on (from the call that started it
|
|
1643
|
+
* through the polls that checked on it), `toolMs`, a tool call, `modelMs`, a model call, and
|
|
1644
|
+
* `otherMs`, the rest (the harness's own work, and time between calls). `modelMs` is null when
|
|
1645
|
+
* the turn's model calls weren't seen one by one: on a sandbox without the egress gateway, and
|
|
1646
|
+
* on Cursor, whose turn is one stream; that time is in `otherMs`. Which calls wait is a heuristic
|
|
1647
|
+
* over the transcript: a shell call run in the background (`run_in_background`, a trailing `&`,
|
|
1648
|
+
* `nohup`, or the command a Cursor terminal file names), then `sleep` and `wait` commands, output
|
|
1649
|
+
* tools (`BashOutput`, `TaskOutput`) and reads of a Cursor terminal file.
|
|
1650
|
+
*/
|
|
1651
|
+
type TimeSplit = {
|
|
1652
|
+
modelMs: number | null;
|
|
1653
|
+
toolMs: number;
|
|
1654
|
+
waitingMs: number;
|
|
1655
|
+
otherMs: number;
|
|
1656
|
+
};
|
|
1657
|
+
/** A dollar figure and where it came from. */
|
|
1658
|
+
type CostUsd = {
|
|
1659
|
+
/** US dollars. */
|
|
1660
|
+
usd: number;
|
|
1661
|
+
/**
|
|
1662
|
+
* `reported`: what the harness (Claude Code, pi) or the gateway (OpenRouter) said it cost;
|
|
1663
|
+
* `estimated`: the tokens at list prices, which leave out discounts and long-context rates.
|
|
1664
|
+
*/
|
|
1665
|
+
basis: 'reported' | 'estimated';
|
|
1666
|
+
/**
|
|
1667
|
+
* Who reported it (`claude-code`, `pi`, `openrouter`, `fake`), or the prices it was estimated
|
|
1668
|
+
* at (`models.dev 2026-11-05`, a `REEVAL_PRICES` file); several, joined with `, `, in a sum.
|
|
1669
|
+
*/
|
|
1670
|
+
source: string;
|
|
1671
|
+
};
|
|
1672
|
+
/**
|
|
1673
|
+
* Costs added up: `basis` is `reported` only when every part was. `unpriced` counts the parts
|
|
1674
|
+
* (attempts, or judge calls) that used tokens but have no cost, which `usd` leaves out.
|
|
1675
|
+
*/
|
|
1676
|
+
type CostTotal = CostUsd & {
|
|
1677
|
+
unpriced?: number;
|
|
1678
|
+
};
|
|
1490
1679
|
/** One agent turn. Immutable once `c.send()` resolves. */
|
|
1491
1680
|
/** A turn as data, as a record or another language's runner carries it: `Turn` without its methods. */
|
|
1492
1681
|
type TurnData = {
|
|
@@ -1517,7 +1706,9 @@ type TurnData = {
|
|
|
1517
1706
|
};
|
|
1518
1707
|
type Turn = TurnData & {
|
|
1519
1708
|
/** Runs in the eval's sandbox. Same as `c.exec`. */
|
|
1520
|
-
exec(command: string, options?: ExecRetry
|
|
1709
|
+
exec(command: string, options?: ExecRetry & {
|
|
1710
|
+
env?: ExecEnv;
|
|
1711
|
+
}): Promise<ExecResult>;
|
|
1521
1712
|
readFile(path: string): Promise<string | null>;
|
|
1522
1713
|
};
|
|
1523
1714
|
/** Any JSON value. */
|
|
@@ -1695,15 +1886,19 @@ type AttemptContext = {
|
|
|
1695
1886
|
judge(question: string, options: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
|
|
1696
1887
|
/**
|
|
1697
1888
|
* Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
|
|
1698
|
-
* many extra times, `interval` ms apart; it resolves with the last try.
|
|
1889
|
+
* many extra times, `interval` ms apart; it resolves with the last try. `env` adds variables
|
|
1890
|
+
* for this command only.
|
|
1699
1891
|
*/
|
|
1700
1892
|
exec(command: string, options?: ExecRetry & {
|
|
1893
|
+
env?: ExecEnv;
|
|
1701
1894
|
background?: false;
|
|
1702
1895
|
}): Promise<ExecResult>;
|
|
1703
1896
|
exec(command: string, options: {
|
|
1704
1897
|
background: true;
|
|
1898
|
+
env?: ExecEnv;
|
|
1705
1899
|
}): Promise<BackgroundHandle>;
|
|
1706
1900
|
exec(command: string, options?: ExecRetry & {
|
|
1901
|
+
env?: ExecEnv;
|
|
1707
1902
|
background?: boolean;
|
|
1708
1903
|
}): Promise<ExecResult | BackgroundHandle>;
|
|
1709
1904
|
/**
|
|
@@ -1815,6 +2010,12 @@ type FakeHarnessDescriptor = {
|
|
|
1815
2010
|
failWith?: string;
|
|
1816
2011
|
/** The first this many session starts of an attempt never become ready, even when aborted (`true`: every one). */
|
|
1817
2012
|
startHangs?: number | true;
|
|
2013
|
+
/**
|
|
2014
|
+
* A shell command the fake agent runs as its session starts, once per sandbox, as a real
|
|
2015
|
+
* adapter's bootstrap installs its CLI: `tart()` bakes it into the image it clones, as it does a
|
|
2016
|
+
* real harness's, so an attempt there finds it done.
|
|
2017
|
+
*/
|
|
2018
|
+
bootstrap?: string;
|
|
1818
2019
|
/** A reasoning block at the start of every turn. */
|
|
1819
2020
|
reasoning?: string;
|
|
1820
2021
|
/**
|
|
@@ -1829,13 +2030,15 @@ type FakeHarnessDescriptor = {
|
|
|
1829
2030
|
effort?: string;
|
|
1830
2031
|
/**
|
|
1831
2032
|
* Tool calls the fake agent reports on every turn, before writing files. `ok` defaults to true;
|
|
1832
|
-
* null reports a call whose result never came.
|
|
2033
|
+
* null reports a call whose result never came. `durationMs` defaults to 0; null reports a call
|
|
2034
|
+
* whose end the harness never said.
|
|
1833
2035
|
*/
|
|
1834
2036
|
tools?: Array<{
|
|
1835
2037
|
tool: string;
|
|
1836
2038
|
input: unknown;
|
|
1837
2039
|
output: unknown;
|
|
1838
2040
|
ok?: boolean | null;
|
|
2041
|
+
durationMs?: number | null;
|
|
1839
2042
|
}>;
|
|
1840
2043
|
/** Files and message used instead when the environment gives the agent a skill. */
|
|
1841
2044
|
withSkill?: {
|
|
@@ -1844,6 +2047,8 @@ type FakeHarnessDescriptor = {
|
|
|
1844
2047
|
};
|
|
1845
2048
|
/** Chance (0–1) that an attempt writes no files and says so, for flaky pass rates. */
|
|
1846
2049
|
flakeRate?: number;
|
|
2050
|
+
/** US dollars the fake agent says every turn cost, as a harness that reports its cost does; 0 reports none. */
|
|
2051
|
+
costUsd?: number;
|
|
1847
2052
|
/** Tokens the fake agent reports for every turn, as `turn.usage`; without it, null. */
|
|
1848
2053
|
usage?: {
|
|
1849
2054
|
input: number;
|
|
@@ -1875,9 +2080,13 @@ type FakeHarnessDescriptor = {
|
|
|
1875
2080
|
layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex' | 'fx' | 'grok-build';
|
|
1876
2081
|
/**
|
|
1877
2082
|
* In Docker, broker the key of the agent `layout` names (`OPENAI_API_KEY` for `codex`, Cursor's
|
|
1878
|
-
* when unset) on that agent's endpoints, as its adapter does
|
|
2083
|
+
* when unset) on that agent's endpoints, as its adapter does; OpenCode, Goose and pi take their
|
|
2084
|
+
* model's provider's key, so with their layouts it's `FAKE_AGENT_API_KEY`, on no endpoint. The key stays on the host, and
|
|
1879
2085
|
* `exec` steps get a placeholder in its variable, which the egress gateway swaps in on those
|
|
1880
2086
|
* endpoints when sent as `Authorization: Bearer <placeholder>`, and refuses on any other request.
|
|
2087
|
+
* For an agent whose bridge is a server in the sandbox (Claude Code, OpenCode, Goose, fx, Grok
|
|
2088
|
+
* Build), it also serves on the session's first exposed port and reaches it from the host, as
|
|
2089
|
+
* that adapter's start does.
|
|
1881
2090
|
*/
|
|
1882
2091
|
brokers?: true;
|
|
1883
2092
|
};
|
|
@@ -1886,7 +2095,8 @@ type FakeHarnessDescriptor = {
|
|
|
1886
2095
|
* the command's stdout, `write` as file edits, and `reply` is the turn's final message. `mcp`
|
|
1887
2096
|
* (`'<server>/<tool>'`) starts the environment's stdio MCP server in the sandbox, as an agent
|
|
1888
2097
|
* would, and calls the tool with no arguments; it's reported as `mcp__<server>__<tool>`.
|
|
1889
|
-
* `error` ends the turn there with status `error` and that message, as a provider error would
|
|
2098
|
+
* `error` ends the turn there with status `error` and that message, as a provider error would; one
|
|
2099
|
+
* worded as a provider's credit, quota or rate limit makes the attempt an infra error.
|
|
1890
2100
|
* `refuseKey` ends it as the model API rejecting the key in that variable would, which makes the
|
|
1891
2101
|
* attempt an infra error.
|
|
1892
2102
|
*/
|
|
@@ -2021,8 +2231,9 @@ type OpenCodeHarnessDescriptor = {
|
|
|
2021
2231
|
startTimeout?: Duration;
|
|
2022
2232
|
};
|
|
2023
2233
|
/**
|
|
2024
|
-
* Goose through `@reeval/harness-goose` (`goose acp`), made by `goose()`. It runs
|
|
2025
|
-
*
|
|
2234
|
+
* Goose through `@reeval/harness-goose` (`goose acp`), made by `goose()`. It runs a model on its
|
|
2235
|
+
* provider's key, and reads each turn's usage from Goose's `usage_ledger` (its ACP server counts
|
|
2236
|
+
* one call).
|
|
2026
2237
|
*/
|
|
2027
2238
|
type GooseHarnessDescriptor = {
|
|
2028
2239
|
kind: 'goose';
|
|
@@ -2031,7 +2242,10 @@ type GooseHarnessDescriptor = {
|
|
|
2031
2242
|
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2032
2243
|
*/
|
|
2033
2244
|
gateway?: GatewayName;
|
|
2034
|
-
/**
|
|
2245
|
+
/**
|
|
2246
|
+
* `<provider>/<model>`: Goose's `google`, `anthropic`, `openai` or `xai` provider on its own key,
|
|
2247
|
+
* or the gateway's id on `vercel()` and `openrouter()`, e.g. `google/gemini-3.8-flash`.
|
|
2248
|
+
*/
|
|
2035
2249
|
model: string;
|
|
2036
2250
|
/**
|
|
2037
2251
|
* Reasoning effort as `GOOSE_THINKING_EFFORT`, e.g. `off`, `low`, `medium`, `high` or `max`.
|
|
@@ -2048,8 +2262,8 @@ type GooseHarnessDescriptor = {
|
|
|
2048
2262
|
startTimeout?: Duration;
|
|
2049
2263
|
};
|
|
2050
2264
|
/**
|
|
2051
|
-
* pi through `@reeval/harness-pi` (`pi --mode json` in the sandbox), made by `pi()`. It runs
|
|
2052
|
-
*
|
|
2265
|
+
* pi through `@reeval/harness-pi` (`pi --mode json` in the sandbox), made by `pi()`. It runs a
|
|
2266
|
+
* model on its provider's key and reports each model call's tokens. It takes stdio and
|
|
2053
2267
|
* HTTP MCP servers, not SSE ones.
|
|
2054
2268
|
*/
|
|
2055
2269
|
type PiHarnessDescriptor = {
|
|
@@ -2059,7 +2273,10 @@ type PiHarnessDescriptor = {
|
|
|
2059
2273
|
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2060
2274
|
*/
|
|
2061
2275
|
gateway?: GatewayName;
|
|
2062
|
-
/**
|
|
2276
|
+
/**
|
|
2277
|
+
* `<provider>/<model>`: one of pi's providers whose key it sends in a header on a fixed host, or
|
|
2278
|
+
* the gateway's id on `vercel()` and `openrouter()`, e.g. `google/gemini-3.8-flash`.
|
|
2279
|
+
*/
|
|
2063
2280
|
model: string;
|
|
2064
2281
|
/**
|
|
2065
2282
|
* Reasoning effort, passed as pi's `--thinking` (the CLI's name). Values are whatever that
|
|
@@ -2309,6 +2526,27 @@ type TartSandboxDescriptor = {
|
|
|
2309
2526
|
user?: string;
|
|
2310
2527
|
key: string;
|
|
2311
2528
|
};
|
|
2529
|
+
/**
|
|
2530
|
+
* Macs that run the VMs instead of this machine, each one plain `ssh` reaches without a prompt
|
|
2531
|
+
* (an `~/.ssh/config` alias, the agent or a key file; no passwords): `'mac-studio'`,
|
|
2532
|
+
* `'elliot@mac-mini.local'`, `'ssh://ci@10.0.0.5:2222'` or `{ host, user?, port?, identityFile?, vms? }`.
|
|
2533
|
+
* reeval runs `tart` on each over SSH and keeps the egress gateway, and every key, here. Each
|
|
2534
|
+
* host needs the image; preflight bakes the harness onto it there. This machine when unset.
|
|
2535
|
+
*/
|
|
2536
|
+
hosts?: TartHost[];
|
|
2537
|
+
};
|
|
2538
|
+
/** A Mac that runs `tart()`'s VMs over SSH: an SSH destination, or its parts. */
|
|
2539
|
+
type TartHost = string | {
|
|
2540
|
+
/** The host name, address or `~/.ssh/config` alias. */
|
|
2541
|
+
host: string;
|
|
2542
|
+
/** The login user; `ssh`'s own (the config's, or this machine's user) when unset. */
|
|
2543
|
+
user?: string;
|
|
2544
|
+
/** The SSH port; `ssh`'s own (the config's, or 22) when unset. */
|
|
2545
|
+
port?: number;
|
|
2546
|
+
/** The private key `ssh -i` takes: absolute, or `~/…`; the agent's or the config's when unset. */
|
|
2547
|
+
identityFile?: string;
|
|
2548
|
+
/** The most of reeval's macOS VMs it runs at once: 1 or 2, and 2 when unset. */
|
|
2549
|
+
vms?: number;
|
|
2312
2550
|
};
|
|
2313
2551
|
/** A boat.dev machine size: `small` is 2 vCPUs and 4 GB. */
|
|
2314
2552
|
type BoatMachineType = 'small' | 'default' | 'large' | 'xlarge';
|
|
@@ -2492,6 +2730,17 @@ type SetupStep = string | {
|
|
|
2492
2730
|
type EgressPolicy = 'proxy' | 'allow-all' | 'deny-all' | {
|
|
2493
2731
|
allow: readonly string[];
|
|
2494
2732
|
};
|
|
2733
|
+
/**
|
|
2734
|
+
* An egress policy per phase of the attempt: `setup` covers its files and setup steps, `agent`
|
|
2735
|
+
* the harness's start, every turn and every check after them. A phase it leaves unset keeps the
|
|
2736
|
+
* policy it inherits.
|
|
2737
|
+
*/
|
|
2738
|
+
type PhasedEgress = {
|
|
2739
|
+
setup?: EgressPolicy;
|
|
2740
|
+
agent?: EgressPolicy;
|
|
2741
|
+
};
|
|
2742
|
+
/** One policy for the whole attempt, or one per phase. */
|
|
2743
|
+
type Egress = EgressPolicy | PhasedEgress;
|
|
2495
2744
|
/**
|
|
2496
2745
|
* A git repository (or one directory of it) the workspace starts as, from the `git` helper. `url`
|
|
2497
2746
|
* is what reeval fetches: the clone URL, or an absolute path for a local repository.
|
|
@@ -2528,8 +2777,11 @@ type EnvironmentSpec = {
|
|
|
2528
2777
|
workspace?: GitWorkspace;
|
|
2529
2778
|
/** Variables and secrets for the sandbox, merged by name over the parent's. */
|
|
2530
2779
|
env?: EnvVars;
|
|
2531
|
-
/**
|
|
2532
|
-
|
|
2780
|
+
/**
|
|
2781
|
+
* Replaces the inherited policy, per phase: a plain policy sets both, `{ setup, agent }` the
|
|
2782
|
+
* phases it names. An `extend` child's replaces its parent's the same way.
|
|
2783
|
+
*/
|
|
2784
|
+
egress?: Egress;
|
|
2533
2785
|
/** Workspace-relative, or `~/` for home. */
|
|
2534
2786
|
files?: Record<string, string | HostFile>;
|
|
2535
2787
|
/** Project rules, written where the harness's agent reads them: `AGENTS.md`, or `CLAUDE.md` for Claude Code. */
|
|
@@ -2998,10 +3250,10 @@ type EvalOptionFields<N extends number = number> = {
|
|
|
2998
3250
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
2999
3251
|
env?: EnvVars;
|
|
3000
3252
|
/**
|
|
3001
|
-
* Where the sandbox may connect. Replaces the inherited policy: config, profile
|
|
3002
|
-
* environment, then groups and the eval.
|
|
3253
|
+
* Where the sandbox may connect. Replaces the inherited policy, per phase: config, profile
|
|
3254
|
+
* defaults, the environment, then groups and the eval.
|
|
3003
3255
|
*/
|
|
3004
|
-
egress?:
|
|
3256
|
+
egress?: Egress;
|
|
3005
3257
|
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
3006
3258
|
judge?: JudgeOptions;
|
|
3007
3259
|
/** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
|
|
@@ -3089,11 +3341,78 @@ type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
|
|
|
3089
3341
|
/** `'provider/model'`, or a fake judge. */
|
|
3090
3342
|
model?: string | FakeJudgeModel;
|
|
3091
3343
|
};
|
|
3092
|
-
|
|
3093
|
-
|
|
3344
|
+
/** A history folder committed with the evals: `local()` from `reeval/store/local`. */
|
|
3345
|
+
type LocalStoreDescriptor = {
|
|
3346
|
+
kind: 'local';
|
|
3347
|
+
/**
|
|
3348
|
+
* The folder, relative to the config's directory (default: `.history` in the directory the
|
|
3349
|
+
* evals live in).
|
|
3350
|
+
*/
|
|
3094
3351
|
dir?: string;
|
|
3095
|
-
|
|
3352
|
+
};
|
|
3353
|
+
/**
|
|
3354
|
+
* The host variables an `s3()` store reads. Each names a variable, never a value: credentials
|
|
3355
|
+
* come from the environment only.
|
|
3356
|
+
*/
|
|
3357
|
+
type S3StoreEnv = {
|
|
3358
|
+
/** The access key id (default `AWS_ACCESS_KEY_ID`). */
|
|
3359
|
+
accessKeyId?: string;
|
|
3360
|
+
/** The secret access key (default `AWS_SECRET_ACCESS_KEY`). */
|
|
3361
|
+
secretAccessKey?: string;
|
|
3362
|
+
/** A session token for temporary credentials (default `AWS_SESSION_TOKEN`), when it's set. */
|
|
3363
|
+
sessionToken?: string;
|
|
3364
|
+
/** The endpoint, when `endpoint` isn't set (default `AWS_ENDPOINT_URL_S3`, then `AWS_ENDPOINT_URL`). */
|
|
3365
|
+
endpoint?: string;
|
|
3366
|
+
/** The bucket, when `bucket` isn't set. */
|
|
3367
|
+
bucket?: string;
|
|
3368
|
+
/** The region, when `region` isn't set (default `AWS_REGION`, then `AWS_DEFAULT_REGION`). */
|
|
3369
|
+
region?: string;
|
|
3370
|
+
};
|
|
3371
|
+
/**
|
|
3372
|
+
* An S3-compatible bucket, such as AWS S3, Cloudflare R2 or MinIO: `s3()` from `reeval/store/s3`.
|
|
3373
|
+
* reeval keeps a copy of it in `.reeval/store/` that every history command reads.
|
|
3374
|
+
*/
|
|
3375
|
+
type S3StoreDescriptor = {
|
|
3376
|
+
kind: 's3';
|
|
3377
|
+
/** The bucket; `env.bucket` names a variable with it instead. */
|
|
3378
|
+
bucket?: string;
|
|
3379
|
+
/**
|
|
3380
|
+
* The endpoint's origin, such as `https://<account>.r2.cloudflarestorage.com` or
|
|
3381
|
+
* `http://127.0.0.1:9000`; without it, `env.endpoint`'s variable, else AWS S3 in the region.
|
|
3382
|
+
*/
|
|
3383
|
+
endpoint?: string;
|
|
3384
|
+
/** The signing region (default: `env.region`'s variable, else `auto`). */
|
|
3385
|
+
region?: string;
|
|
3386
|
+
/** A key prefix every object goes under, so several projects can share a bucket (default none). */
|
|
3387
|
+
prefix?: string;
|
|
3388
|
+
/** Which host variables hold the credentials, and the bucket, endpoint or region when they aren't set here. */
|
|
3389
|
+
env?: S3StoreEnv;
|
|
3390
|
+
};
|
|
3391
|
+
/** A history store as data, which `reeval.yaml` can name too. */
|
|
3392
|
+
type HistoryStoreDescriptor = LocalStoreDescriptor | S3StoreDescriptor;
|
|
3393
|
+
type HistoryConfig = {
|
|
3394
|
+
/**
|
|
3395
|
+
* Where recorded runs are kept: `local()` (the default, a folder committed with the evals) or
|
|
3396
|
+
* `s3()`.
|
|
3397
|
+
*/
|
|
3398
|
+
store?: HistoryStoreDescriptor;
|
|
3399
|
+
/**
|
|
3400
|
+
* Artifacts recorded next to each run: every attempt's, failed attempts' only (the default), or
|
|
3401
|
+
* none. The setup log and each `c.exec` log are an attempt's artifacts too.
|
|
3402
|
+
*/
|
|
3096
3403
|
artifacts?: ArtifactLevel;
|
|
3404
|
+
/**
|
|
3405
|
+
* Each attempt's debug data uploaded with the run to a remote store: none (the default), its
|
|
3406
|
+
* gateway requests (`requests.ndjson`, which holds prompts and responses), or all of it
|
|
3407
|
+
* (`debug.log` and `trace.ndjson` too). Rescanned for secrets first; `local()` takes `'none'` only.
|
|
3408
|
+
*/
|
|
3409
|
+
debug?: DebugUpload;
|
|
3410
|
+
};
|
|
3411
|
+
type DebugUpload = 'none' | 'requests' | 'all';
|
|
3412
|
+
/** `HistoryConfig` as data: the store is `local` or `s3`. */
|
|
3413
|
+
type HistoryConfigData = Omit<HistoryConfig, 'store'> & {
|
|
3414
|
+
/** Where recorded runs are kept: `{ kind: local }` (the default) or `{ kind: s3, … }`. */
|
|
3415
|
+
store?: HistoryStoreDescriptor;
|
|
3097
3416
|
};
|
|
3098
3417
|
/** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
|
|
3099
3418
|
type Budget = {
|
|
@@ -3116,8 +3435,8 @@ type ProfileDefaults = {
|
|
|
3116
3435
|
gate?: GatePolicy;
|
|
3117
3436
|
/** Variables and secrets for the sandbox, merged by name over the config's. */
|
|
3118
3437
|
env?: EnvVars;
|
|
3119
|
-
/** Where the sandbox may connect; replaces the config's policy. */
|
|
3120
|
-
egress?:
|
|
3438
|
+
/** Where the sandbox may connect; replaces the config's policy, per phase. */
|
|
3439
|
+
egress?: Egress;
|
|
3121
3440
|
/** Defaults for `c.judge()`, merged field by field over the config's. */
|
|
3122
3441
|
judge?: ConfigJudgeOptions;
|
|
3123
3442
|
/** Per attempt. */
|
|
@@ -3158,6 +3477,11 @@ type Profile = {
|
|
|
3158
3477
|
budget?: Budget;
|
|
3159
3478
|
/** When this profile's runs are copied into the history folder. */
|
|
3160
3479
|
record?: RecordMode;
|
|
3480
|
+
/**
|
|
3481
|
+
* The history this profile records to and every history command reads with `--profile`, merged
|
|
3482
|
+
* over the config's `history` key by key.
|
|
3483
|
+
*/
|
|
3484
|
+
history?: HistoryConfig;
|
|
3161
3485
|
/** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
|
|
3162
3486
|
tags?: string[];
|
|
3163
3487
|
/** Replaces the config's `gateway` list for this profile's runs. */
|
|
@@ -3292,8 +3616,8 @@ type ReevalConfig = {
|
|
|
3292
3616
|
gate?: GatePolicy;
|
|
3293
3617
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
3294
3618
|
env?: EnvVars;
|
|
3295
|
-
/** Where the sandbox may connect; default `'proxy'
|
|
3296
|
-
egress?:
|
|
3619
|
+
/** Where the sandbox may connect; default `'proxy'` in both phases. */
|
|
3620
|
+
egress?: Egress;
|
|
3297
3621
|
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
3298
3622
|
judge?: ConfigJudgeOptions;
|
|
3299
3623
|
/** Per attempt; default `'15m'`. */
|
|
@@ -3314,7 +3638,7 @@ type ReevalConfig = {
|
|
|
3314
3638
|
concurrency?: number;
|
|
3315
3639
|
/** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
|
|
3316
3640
|
record?: RecordMode;
|
|
3317
|
-
/**
|
|
3641
|
+
/** Where recorded runs are kept, and which attempts keep their artifacts there. */
|
|
3318
3642
|
history?: HistoryConfig;
|
|
3319
3643
|
/** Caps on a run; a profile's `budget` is merged over it. */
|
|
3320
3644
|
budget?: Budget;
|
|
@@ -3325,7 +3649,7 @@ type ReevalConfig = {
|
|
|
3325
3649
|
ci?: string;
|
|
3326
3650
|
local?: string;
|
|
3327
3651
|
};
|
|
3328
|
-
/** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers written as plain strings, instead of warning (default `'warn'`). */
|
|
3652
|
+
/** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers and `env` values written as plain strings, instead of warning (default `'warn'`). */
|
|
3329
3653
|
secrets?: {
|
|
3330
3654
|
unbrokered?: 'warn' | 'error';
|
|
3331
3655
|
};
|
|
@@ -3382,15 +3706,26 @@ type ResourceFailureRecord = {
|
|
|
3382
3706
|
cause: 'oom';
|
|
3383
3707
|
kills: number;
|
|
3384
3708
|
};
|
|
3709
|
+
/**
|
|
3710
|
+
* The model provider ended a turn, not the agent: out of credit, over its quota, rate limited or
|
|
3711
|
+
* overloaded, with the HTTP status of the last model call the egress gateway brokered when that
|
|
3712
|
+
* call failed (`null` when it didn't, as in a stream that failed inside an HTTP 200, or none was).
|
|
3713
|
+
* The attempt is then an infra error, so `--retry-errors` runs it again.
|
|
3714
|
+
*/
|
|
3715
|
+
type ProviderStopRecord = {
|
|
3716
|
+
cause: 'credit' | 'quota' | 'rate_limit' | 'overloaded';
|
|
3717
|
+
status: number | null;
|
|
3718
|
+
};
|
|
3385
3719
|
/**
|
|
3386
3720
|
* Why an attempt failed; `prepare` when its sandbox couldn't be prepared, `resource` when it ran
|
|
3387
|
-
* out of one.
|
|
3721
|
+
* out of one, `provider` when the model provider stopped a turn.
|
|
3388
3722
|
*/
|
|
3389
3723
|
type AttemptError = {
|
|
3390
3724
|
kind: ErrorKind;
|
|
3391
3725
|
message: string;
|
|
3392
3726
|
prepare?: PrepareFailureRecord;
|
|
3393
3727
|
resource?: ResourceFailureRecord;
|
|
3728
|
+
provider?: ProviderStopRecord;
|
|
3394
3729
|
};
|
|
3395
3730
|
/**
|
|
3396
3731
|
* One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
|
|
@@ -3527,8 +3862,16 @@ type EnvironmentRecord = {
|
|
|
3527
3862
|
brokered: string[];
|
|
3528
3863
|
/** Optional secrets whose host variable was unset, so they weren't injected. */
|
|
3529
3864
|
unset: string[];
|
|
3530
|
-
/**
|
|
3865
|
+
/**
|
|
3866
|
+
* The policy the attempt's agent phase ran under (the harness's start, its turns and the checks),
|
|
3867
|
+
* from the options and the environment; in the digest unless `proxy`.
|
|
3868
|
+
*/
|
|
3531
3869
|
egress: EgressPolicy;
|
|
3870
|
+
/**
|
|
3871
|
+
* The policy the setup phase (files and setup steps) ran under, when it wasn't the agent's; in
|
|
3872
|
+
* the digest when set.
|
|
3873
|
+
*/
|
|
3874
|
+
setupEgress?: EgressPolicy;
|
|
3532
3875
|
/**
|
|
3533
3876
|
* This records the Git repository the workspace started as: the repo as written (a local one
|
|
3534
3877
|
* relative to the project root), the ref and path asked for (null when unset), the commit the
|
|
@@ -3553,11 +3896,21 @@ type EnvironmentRecord = {
|
|
|
3553
3896
|
* contains only unexpected refusals. A request that carries the harness's brokered key off the
|
|
3554
3897
|
* endpoints reeval brokers it for is refused on a host the policy let through: it's in `blocked`
|
|
3555
3898
|
* once per method and path, its `reason` names them, and `connections` counts its requests. Past
|
|
3556
|
-
* the first 20 pairs, the rest share one entry per host.
|
|
3899
|
+
* the first 20 pairs, the rest share one entry per host. With a setup policy of its own, every
|
|
3900
|
+
* entry names the `phase` it happened in.
|
|
3557
3901
|
*/
|
|
3558
3902
|
type EgressRecord = {
|
|
3903
|
+
/** The agent phase's policy: the harness's start, its turns and the checks. */
|
|
3559
3904
|
policy: EgressPolicy;
|
|
3560
3905
|
allowed: string[] | null;
|
|
3906
|
+
/**
|
|
3907
|
+
* The setup phase's policy (files and setup steps) and the hosts it allowed, when it wasn't the
|
|
3908
|
+
* agent's.
|
|
3909
|
+
*/
|
|
3910
|
+
setup?: {
|
|
3911
|
+
policy: EgressPolicy;
|
|
3912
|
+
allowed: string[] | null;
|
|
3913
|
+
};
|
|
3561
3914
|
recorded: boolean;
|
|
3562
3915
|
/**
|
|
3563
3916
|
* True when traffic could leave around the gateway (`allow-all` in a container that kept its
|
|
@@ -3571,23 +3924,29 @@ type EgressRecord = {
|
|
|
3571
3924
|
connections: number;
|
|
3572
3925
|
sent: number;
|
|
3573
3926
|
received: number;
|
|
3927
|
+
/** With a setup policy of its own: the phase the connections were made in. */
|
|
3928
|
+
phase?: 'setup' | 'agent';
|
|
3574
3929
|
}>;
|
|
3575
3930
|
/**
|
|
3576
3931
|
* Hosts the policy refused, and requests that carry the harness's brokered key off the endpoints
|
|
3577
3932
|
* reeval brokers it for (one entry per method and path), each with why. A name that didn't
|
|
3578
|
-
* resolve is in `unresolved` instead.
|
|
3933
|
+
* resolve is in `unresolved` instead. A tunnel setup opened to a host the agent's policy refuses
|
|
3934
|
+
* is closed as the agent phase starts, and is here with `closed: true`.
|
|
3579
3935
|
*/
|
|
3580
3936
|
blocked: Array<{
|
|
3581
3937
|
host: string;
|
|
3582
3938
|
port: number;
|
|
3583
3939
|
connections: number;
|
|
3584
3940
|
reason: string;
|
|
3941
|
+
phase?: 'setup' | 'agent';
|
|
3942
|
+
closed?: true;
|
|
3585
3943
|
}>;
|
|
3586
3944
|
expectedBlocked: Array<{
|
|
3587
3945
|
host: string;
|
|
3588
3946
|
port: number;
|
|
3589
3947
|
connections: number;
|
|
3590
3948
|
reason: string;
|
|
3949
|
+
phase?: 'setup' | 'agent';
|
|
3591
3950
|
}>;
|
|
3592
3951
|
/**
|
|
3593
3952
|
* Hosts the policy let through whose names didn't resolve (`metadata.google.internal` outside
|
|
@@ -3600,6 +3959,7 @@ type EgressRecord = {
|
|
|
3600
3959
|
port: number;
|
|
3601
3960
|
connections: number;
|
|
3602
3961
|
reason: string;
|
|
3962
|
+
phase?: 'setup' | 'agent';
|
|
3603
3963
|
}>;
|
|
3604
3964
|
};
|
|
3605
3965
|
/** The image an attempt's container started from, and how reeval made it. */
|
|
@@ -3633,11 +3993,15 @@ type VmImage = {
|
|
|
3633
3993
|
/** The OCI manifest digest (`sha256:…`) Tart had for it; null for a local VM, which has none. */
|
|
3634
3994
|
digest: string | null;
|
|
3635
3995
|
};
|
|
3636
|
-
/** A harness session start that timed out and was started again. */
|
|
3996
|
+
/** A harness session start that timed out, or failed on a transient error, and was started again. */
|
|
3637
3997
|
type StartRetry = {
|
|
3638
|
-
/**
|
|
3998
|
+
/**
|
|
3999
|
+
* `<harness kind>: session did not become ready within <startTimeout>`, or the transient error
|
|
4000
|
+
* the start failed on, such as `Cursor harness failed to start: Cursor's model catalog request
|
|
4001
|
+
* failed: HTTP 502 (tried 3 times)`.
|
|
4002
|
+
*/
|
|
3639
4003
|
error: string;
|
|
3640
|
-
/** How long it waited before giving up on that start. */
|
|
4004
|
+
/** How long it waited before giving up on that start, or until that start failed. */
|
|
3641
4005
|
waitedMs: number;
|
|
3642
4006
|
};
|
|
3643
4007
|
/**
|
|
@@ -3800,5 +4164,5 @@ type ResolvedGateway = {
|
|
|
3800
4164
|
why: string;
|
|
3801
4165
|
}>;
|
|
3802
4166
|
};
|
|
3803
|
-
export {
|
|
3804
|
-
//# sourceMappingURL=types-
|
|
4167
|
+
export { EnvironmentInput as $, EffortSample as $n, Question as $t, CellRole as A, CheckTally as An, JudgeArtifactRef as At, CostUsd as B, JudgmentRecord as Bn, Observation as Bt, CaptureUseAction as C, VertexPublisher as Cn, HarnessEdit as Ct, CaseFn as D, ArtifactRef as Dn, HistoryStoreDescriptor as Dt, CaptureWindow as E, WebEventType as En, HistoryConfigData as Et, CompareCut as F, FamilyRecord as Fn, JudgeOptions as Ft, DirectoryOptions as G, StackSummary as Gn, PairVote as Gt, DebugUpload as H, RunDebug as Hn, OpenRouterGatewayDescriptor as Ht, CompareOptions as I, FamilyVerdict as In, JudgeState as It, EgressPolicy as J, DIRECTIONAL_MIN as Jn, PluginPart as Jt, DockerSandboxDescriptor as K, UpstreamCount as Kn, PiHarnessDescriptor as Kt, ConfigHelpers as L, FileRef as Ln, JudgeSubject as Lt, ClaudeCodeHarnessDescriptor as M, FamilyLift as Mn, JudgeFailure as Mt, CodexHarnessDescriptor as N, FamilyLiftSummary as Nn, JudgeInputs as Nt, CaseOptions as O, AttemptRecord as On, HostFile as Ot, CompareAnswer as P, FamilyRate as Pn, JudgeModel as Pt, EnvironmentHelpers as Q, EffortMetric as Qn, ProfileLimit as Qt, ConfigJudgeOptions as R, HistoryCell as Rn, LocalStoreDescriptor as Rt, CaptureUse as S, VertexGatewayDescriptor as Sn, tokenValues as Sr, HarnessDescriptor as St, CaptureWait as T, Wait as Tn, HistoryConfig as Tt, Diff as U, RunInvocation as Un, Outcome as Ut, CursorHarnessDescriptor as V, Lift as Vn, OpenCodeHarnessDescriptor as Vt, DirectGatewayDescriptor as W, RunRecord as Wn, PairFactor as Wt, EnvVars as X, EFFORT_MIN_RUNS as Xn, Profile as Xt, EgressRecord as Y, EFFORT_ALPHA as Yn, PluginSource as Yt, Environment as Z, EffortContrast as Zn, ProfileDefaults as Zt, CaptureInstruction as _, Turn as _n, formatCount as _r, GatewayName as _t, AttemptContext as a, S3StoreEnv as an, cellContrast as ar, ExecEnv as at, CapturePage as b, VercelGatewayDescriptor as bn, splitLabel as br, GroupOptions as bt, BlindPair as c, Secret as cn, contrastEffort as cr, FakeAnalysisModel as ct, BoatSandboxDescriptor as d, SetupStep as dn, effortLabel as dr, FakeJudgeModel as dt, QuestionInputs as en, EffortVerdict as er, EnvironmentSpec as et, Budget as f, SkillSource as fn, effortOf as fr, FakeSandboxDescriptor as ft, CaptureFailedRequest as g, TranscriptEvent as gn, filesChanged as gr, GatewayDescriptor as gt, CaptureConsoleMessage as h, TokenUsage as hn, effortVerdict as hr, FxHarnessDescriptor as ht, AttemptAnswer as i, S3StoreDescriptor as in, bothUnderATenth as ir, EvalOptionsData as it, ClaudeCodeEffort as j, EvalEntry as jn, JudgeCallOptions as jt, Cell as k, CellRecord as kn, JSONValue as kt, BoatApiKey as l, SecretTemplate as ln, directional as lr, FakeAnswer as lt, CaptureCall as m, TartSandboxDescriptor as mn, effortSummary as mr, FakeVote as mt, AnalysisConfig as n, ReevalConfig as nn, TOKENS_UNREPORTED as nr, EvalFn as nt, BackgroundHandle as o, SandboxDescriptor as on, cellEffort as or, ExecResult as ot, Capture as p, StatusRange as pn, effortSample as pr, FakeScriptStep as pt, Duration as q, CellEffort as qn, PinnableKey as qt, AntigravityHarnessDescriptor as r, ResolvedGateway as rn, VERDICT_TEXT as rr, EvalOptions as rt, BlindAttempt as s, SandboxWarning as sn, cellSamples as sr, ExecRetry as st, AgentHarnessDescriptor as t, QuestionOptions as tn, Spread as tr, ErrorKind as tt, BoatMachineType as u, ServeHandle as un, effortChangeLabel as ur, FakeHarnessDescriptor as ut, CaptureKey as v, TurnData as vn, formatRatio as vr, GooseHarnessDescriptor as vt, CaptureUseScript as w, Viewport as wn, HarnessOption as wt, CapturePageFile as x, Verdict as xn, splitOf as xr, HarnessData as xt, CaptureOptions as y, TurnStatus as yn, formatSeconds as yr, GrokBuildHarnessDescriptor as yt, CostTotal as z, HistoryEntry as zn, McpServer as zt };
|
|
4168
|
+
//# sourceMappingURL=types-BJ0wh8wC.d.ts.map
|