reeval 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +803 -0
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C8xPuXYq.js → adapter-BdzhdRYo.js} +8 -6
- package/dist/adapter-BdzhdRYo.js.map +1 -0
- package/dist/{agent-CBoLndM7.js → agent-Dx-bDmlV.js} +179 -31
- package/dist/agent-Dx-bDmlV.js.map +1 -0
- package/dist/{agents-C6zhkxp3.js → agents-l0-k7vw3.js} +362 -102
- package/dist/agents-l0-k7vw3.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
- package/dist/app-files-B-CQYB0c.js.map +1 -0
- package/dist/{assemble-UPCRrtOz.js → assemble-CYMpN8LC.js} +1 -1
- package/dist/{assemble-t6h018An.js → assemble-CsZIcJo8.js} +171 -18
- package/dist/assemble-CsZIcJo8.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
- package/dist/{attempt-host-CIm1-11m.js → attempt-host-BwSmNYkX.js} +17 -8
- package/dist/attempt-host-BwSmNYkX.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
- package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
- package/dist/{boat-CJCtzfB1.js → boat-su1Y7LPd.js} +17 -14
- package/dist/boat-su1Y7LPd.js.map +1 -0
- package/dist/bridge/index.mjs +113 -8
- package/dist/{capture-iBzFiC0j.js → capture-BqIAeFyC.js} +55 -29
- package/dist/{capture-iBzFiC0j.js.map → capture-BqIAeFyC.js.map} +1 -1
- package/dist/capture-D0Ln5aCP.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-20Fpvs5H.js → cli-s_3opn_I.js} +219 -109
- package/dist/cli-s_3opn_I.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
- package/dist/cli-spec-DxzIjmq_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-C5fIzHQa.js +803 -0
- package/dist/client-C5fIzHQa.js.map +1 -0
- package/dist/{command-CRh-cb6Z.js → command-B5OAIjp7.js} +25 -21
- package/dist/command-B5OAIjp7.js.map +1 -0
- package/dist/{command-BLNVDiIg.js → command-BJ7KqczP.js} +10 -10
- package/dist/{command-BLNVDiIg.js.map → command-BJ7KqczP.js.map} +1 -1
- package/dist/{command-aUJvENwy.js → command-BXLZsEwQ.js} +16 -16
- package/dist/{command-aUJvENwy.js.map → command-BXLZsEwQ.js.map} +1 -1
- package/dist/{command-B5OyHpOu.js → command-DA_SHQUQ.js} +16 -16
- package/dist/{command-B5OyHpOu.js.map → command-DA_SHQUQ.js.map} +1 -1
- package/dist/{commands-CWy-WndG.js → commands-BKBamOcX.js} +174 -40
- package/dist/commands-BKBamOcX.js.map +1 -0
- package/dist/{commands-CfdBX-mc.js → commands-BPvAOEmb.js} +1 -1
- package/dist/{config-B9mB1kF5.js → config-C0URar0t.js} +255 -89
- package/dist/config-C0URar0t.js.map +1 -0
- package/dist/{context-Bkv3ndjq.js → context-Sj58aJEu.js} +2 -2
- package/dist/{context-Bkv3ndjq.js.map → context-Sj58aJEu.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-BSn9xjmo.js → cursor-catalog-FWgSo-T1.js} +4 -4
- package/dist/{cursor-catalog-BSn9xjmo.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
- package/dist/db-Bc4L-Wcr.js +2 -0
- package/dist/{db-zgC1tvKB.js → db-BjAWxVAG.js} +142 -248
- package/dist/db-BjAWxVAG.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BAy8ixd1.js → default-effort-CWlhZ6U1.js} +1 -1
- package/dist/{default-effort-DGhQmu95.js → default-effort-Ck2GYIxX.js} +4 -4
- package/dist/{default-effort-DGhQmu95.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
- package/dist/{dist-CAFD1yfv.js → dist-DeqeCUZe.js} +15 -8
- package/dist/dist-DeqeCUZe.js.map +1 -0
- package/dist/{docker-DAMnm3L9.js → docker-CDdK6Ity.js} +49 -22
- package/dist/docker-CDdK6Ity.js.map +1 -0
- package/dist/{docker-Cqw-CyWz.js → docker-D74l67k8.js} +4 -4
- package/dist/{docker-Cqw-CyWz.js.map → docker-D74l67k8.js.map} +1 -1
- package/dist/{docker-Dr4XeDc8.js → docker-DMug3l-a.js} +1 -1
- package/dist/{doctor-B2PohneB.js → doctor-eFxyn6at.js} +85 -46
- package/dist/doctor-eFxyn6at.js.map +1 -0
- package/dist/{environment-DB8o3hGi.js → environment-CJIKz-Ud.js} +49 -11
- package/dist/environment-CJIKz-Ud.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-D3sbAzt-.js → eval-source-CXIRJ91v.js} +24 -12
- package/dist/eval-source-CXIRJ91v.js.map +1 -0
- package/dist/{evaluate-CwKeN_fo.js → evaluate-DA4QsrHO.js} +792 -281
- package/dist/evaluate-DA4QsrHO.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
- package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
- package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
- package/dist/{files-BiFU16BW.js → files-BVVWAS0A.js} +6 -6
- package/dist/{files-BiFU16BW.js.map → files-BVVWAS0A.js.map} +1 -1
- package/dist/files-BaqXvB88.js +1 -0
- package/dist/{files-BEwfjD8J.js → files-Bey1VAex.js} +3 -3
- package/dist/{files-BEwfjD8J.js.map → files-Bey1VAex.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/files-CNHhSyb3.js +1 -0
- package/dist/files-CW1lChNh.js +1 -0
- package/dist/{files-BRzV7aPh.js → files-D6j-XpaK.js} +5 -5
- package/dist/{files-BRzV7aPh.js.map → files-D6j-XpaK.js.map} +1 -1
- package/dist/{files-Cl7BQ5gE.js → files-F44NRCN0.js} +143 -20
- package/dist/files-F44NRCN0.js.map +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-fDiJ8UIV.js → git-Jqkf9SBa.js} +3 -3
- package/dist/{git-fDiJ8UIV.js.map → git-Jqkf9SBa.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-BNgcR77K.js → images-Dsc4Wi-F.js} +65 -30
- package/dist/images-Dsc4Wi-F.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/init-C0oUG3jh.js +1 -0
- package/dist/{init-DbuZVRrW.js → init-QB5ecPDA.js} +33 -14
- package/dist/init-QB5ecPDA.js.map +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
- package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
- package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
- package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
- package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
- package/dist/layout-DfQa4mJs.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
- package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
- package/dist/live-DwjWJTdn.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
- package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
- package/dist/{live-CnGrRM6j.js → live-y1p3oNXH.js} +9 -9
- package/dist/{live-CnGrRM6j.js.map → live-y1p3oNXH.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
- package/dist/model-3pZZzloU.js.map +1 -0
- package/dist/{model-047Dcd-a.js → model-DEjwetkJ.js} +1 -1
- package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{model-DJjLw-0r.js → model-jRKSQBg6.js} +5 -5
- package/dist/{model-DJjLw-0r.js.map → model-jRKSQBg6.js.map} +1 -1
- package/dist/model-nfewGTzB.js +1 -0
- package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
- package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
- package/dist/options-DpsJ_5CU.js +1 -0
- package/dist/{patch-DH4e_Gkd.js → patch-Bx-qp8Li.js} +2 -2
- package/dist/{patch-DH4e_Gkd.js.map → patch-Bx-qp8Li.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-Cgq9NCEX.js → plan-ChO0A-xn.js} +63 -12
- package/dist/plan-ChO0A-xn.js.map +1 -0
- package/dist/{plugins-BkmPxDiY.js → plugins-BioMWncI.js} +636 -9
- package/dist/plugins-BioMWncI.js.map +1 -0
- package/dist/{preflight-R-InZCQb.js → preflight-BUjk_6wT.js} +1 -1
- package/dist/{preflight-CtlloeOQ.js → preflight-CJSGzMzc.js} +8 -9
- package/dist/preflight-CJSGzMzc.js.map +1 -0
- package/dist/preflight-DD3fI4Hz.js +1 -0
- package/dist/{preflight-BElE_q0u.js → preflight-qFCXmxha.js} +4 -4
- package/dist/{preflight-BElE_q0u.js.map → preflight-qFCXmxha.js.map} +1 -1
- package/dist/{prepare-C7f5mPl5.js → prepare-D7oSSFNm.js} +9 -9
- package/dist/{prepare-C7f5mPl5.js.map → prepare-D7oSSFNm.js.map} +1 -1
- package/dist/{prepare-CzkAdbXm.js → prepare-xHU2BpN-.js} +2 -2
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-zsc4--dP.js → provenance-B-flBAzR.js} +5 -5
- package/dist/{provenance-zsc4--dP.js.map → provenance-B-flBAzR.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/{providers-B8xYMeah.js → providers-B6StVyfG.js} +215 -48
- package/dist/providers-B6StVyfG.js.map +1 -0
- package/dist/providers-Dljf6al6.js +1 -0
- package/dist/query-BAc2tQZj.js +1 -0
- package/dist/{query-Bb3SYICk.js → query-CkLmJv7R.js} +2 -2
- package/dist/{query-Bb3SYICk.js.map → query-CkLmJv7R.js.map} +1 -1
- package/dist/{questions-CWlR9ffu.js → questions-CJIh2FSX.js} +7 -7
- package/dist/{questions-CWlR9ffu.js.map → questions-CJIh2FSX.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
- package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
- package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
- package/dist/redact-DL2BTykw.js.map +1 -0
- package/dist/{remarks-vtkPmQb9.js → remarks-B3wjKscW.js} +46 -11
- package/dist/remarks-B3wjKscW.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-BgYWqogI.js → remote-C88amFRN.js} +6 -6
- package/dist/{remote-BgYWqogI.js.map → remote-C88amFRN.js.map} +1 -1
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/resolve-DW92wpIZ.js +1 -0
- package/dist/{resolve-Do0ZaYH6.js → resolve-G576PiFJ.js} +14 -11
- package/dist/resolve-G576PiFJ.js.map +1 -0
- package/dist/{results-JyuM64gy.js → results-zVHIW6xw.js} +9 -22
- package/dist/results-zVHIW6xw.js.map +1 -0
- package/dist/{resume-AW5oXzUc.js → resume-D_PcUXfG.js} +94 -16
- package/dist/resume-D_PcUXfG.js.map +1 -0
- package/dist/{retry-ChzISxmy.js → retry-BvTzCxdc.js} +4 -4
- package/dist/{retry-ChzISxmy.js.map → retry-BvTzCxdc.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
- package/dist/routes-B0F2R-K7.js.map +1 -0
- package/dist/{runtime-BpW1f1Mx.js → runtime-B3r_pAq6.js} +4 -4
- package/dist/{runtime-BpW1f1Mx.js.map → runtime-B3r_pAq6.js.map} +1 -1
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-kSgr7vo5.js → sandbox-Duh3z1TV.js} +3 -3
- package/dist/{sandbox-kSgr7vo5.js.map → sandbox-Duh3z1TV.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CGEsvVRG.js +1 -0
- package/dist/{server-Cn8VqO9S.js → server-DUWMRKik.js} +8 -8
- package/dist/{server-Cn8VqO9S.js.map → server-DUWMRKik.js.map} +1 -1
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-CYOhR98B.js → source-CHuN4Nfg.js} +2 -2
- package/dist/{source-CYOhR98B.js.map → source-CHuN4Nfg.js.map} +1 -1
- package/dist/source-Wz1Nrfue.js +1 -0
- package/dist/src--yl1OypK.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
- package/dist/src-B-HuDqpH.js.map +1 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Ck2r_EMT.js → src-BHJ4MWvY.js} +504 -119
- package/dist/src-BHJ4MWvY.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
- package/dist/src-BI0v8qlZ.js.map +1 -0
- package/dist/src-BL-_0QoE.js +1 -0
- package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
- package/dist/src-BqNzHHWn.js.map +1 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C-mOSczH.js +2 -0
- package/dist/src-CZaYu6ER.js +3 -0
- package/dist/src-CeBhwD8k.js +2 -0
- package/dist/src-CkEiiyON.js +2 -0
- package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
- package/dist/src-CmA5Yev6.js.map +1 -0
- package/dist/src-CwH_nEaw.js +2 -0
- package/dist/src-DA95rM6v.js +4 -0
- package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
- package/dist/src-DL-EE4DU.js.map +1 -0
- package/dist/src-De0KY5S_.js +2 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/state-B3jex5KU.js +1 -0
- package/dist/{state-BOLt8pJg.js → state-DUBRDyMk.js} +22 -22
- package/dist/{state-BOLt8pJg.js.map → state-DUBRDyMk.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
- package/dist/suite-Bhs9-Z7O.js +2 -0
- package/dist/{summary-DU5zaibx.js → summary-CSp4cfnr.js} +25 -7
- package/dist/{summary-DU5zaibx.js.map → summary-CSp4cfnr.js.map} +1 -1
- package/dist/sync-CxAY6Dki.js +559 -0
- package/dist/sync-CxAY6Dki.js.map +1 -0
- package/dist/tart-CcrrtSrp.js +704 -0
- package/dist/tart-CcrrtSrp.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-F-6seRC9.js +192 -0
- package/dist/tart-bake-F-6seRC9.js.map +1 -0
- package/dist/tart-bake-RK7RaT9E.js +1 -0
- package/dist/tart-cli-B86Jqo1t.js +89 -0
- package/dist/tart-cli-B86Jqo1t.js.map +1 -0
- package/dist/tart-cli-ClvRniLQ.js +1 -0
- package/dist/tart-hosts-Brii7zHM.js +1 -0
- package/dist/tart-hosts-DPyZsg-1.js +313 -0
- package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
- package/dist/tart-images-CWSf03w8.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-CVDdJy0H.js +433 -0
- package/dist/tool-class-CVDdJy0H.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
- package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +27 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-DAIiLiV3.js → views-DRxXCyIH.js} +6 -6
- package/dist/{views-DAIiLiV3.js.map → views-DRxXCyIH.js.map} +1 -1
- package/dist/views-i67V1ly3.js +2 -0
- package/dist/{worker-entry-BIROTPX1.js → worker-entry-C3Q4U63F.js} +12 -9
- package/dist/worker-entry-C3Q4U63F.js.map +1 -0
- package/dist/workspace-DhUOW8if.js +1 -0
- package/dist/{workspace-uuE51x35.js → workspace-P72SXLXr.js} +7 -7
- package/dist/{workspace-uuE51x35.js.map → workspace-P72SXLXr.js.map} +1 -1
- package/package.json +21 -10
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C8xPuXYq.js.map +0 -1
- package/dist/agent-CBoLndM7.js.map +0 -1
- package/dist/agents-C6zhkxp3.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-t6h018An.js.map +0 -1
- package/dist/attempt-host-CIm1-11m.js.map +0 -1
- package/dist/boat-CJCtzfB1.js.map +0 -1
- package/dist/capture-DoxFUN3J.js +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-20Fpvs5H.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-CRh-cb6Z.js.map +0 -1
- package/dist/commands-CWy-WndG.js.map +0 -1
- package/dist/config-B9mB1kF5.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-P2fXw4nx.js +0 -2
- package/dist/db-zgC1tvKB.js.map +0 -1
- package/dist/dist-CAFD1yfv.js.map +0 -1
- package/dist/docker-DAMnm3L9.js.map +0 -1
- package/dist/doctor-B2PohneB.js.map +0 -1
- package/dist/environment-DB8o3hGi.js.map +0 -1
- package/dist/eval-source-D3sbAzt-.js.map +0 -1
- package/dist/evaluate-CwKeN_fo.js.map +0 -1
- package/dist/files-B4KQUiM7.js +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-C3mCC8Mn.js +0 -1
- package/dist/files-Cl7BQ5gE.js.map +0 -1
- package/dist/files-DnlA1A7N.js +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-BNgcR77K.js.map +0 -1
- package/dist/init-4RPeD-K-.js +0 -1
- package/dist/init-DbuZVRrW.js.map +0 -1
- package/dist/kinds-C1oY3qyC.js +0 -697
- package/dist/kinds-C1oY3qyC.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CJCK11nh.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/options-CHubslQf.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-Cgq9NCEX.js.map +0 -1
- package/dist/plugins-BkmPxDiY.js.map +0 -1
- package/dist/preflight-4gWSy0Pe.js +0 -1
- package/dist/preflight-CtlloeOQ.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-B8xYMeah.js.map +0 -1
- package/dist/providers-CVu8e0xS.js +0 -1
- package/dist/query-DIfkUcPh.js +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/remarks-vtkPmQb9.js.map +0 -1
- package/dist/resolve-Do0ZaYH6.js.map +0 -1
- package/dist/resolve-amw_2Odz.js +0 -1
- package/dist/results-JyuM64gy.js.map +0 -1
- package/dist/resume-AW5oXzUc.js.map +0 -1
- package/dist/server-2_gUnVvf.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-CY3etKzw.js +0 -1
- package/dist/src-6Zzyg4Qd.js +0 -2
- package/dist/src-BDhcPtnI.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BRMsk9nh.js +0 -2
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-Ck2r_EMT.js.map +0 -1
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DB7N8TYJ.js +0 -2
- package/dist/src-DPDaAYwh.js +0 -2
- package/dist/src-Dbk19BiP.js +0 -2
- package/dist/src-DtodMT59.js +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-T-OeuJLr.js +0 -2
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-zntEDlnx.js +0 -3
- package/dist/state-kimtIhBP.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/tart-BTVXp8rz.js +0 -373
- package/dist/tart-BTVXp8rz.js.map +0 -1
- package/dist/tart-cli-BIC9HqPg.js +0 -37
- package/dist/tart-cli-BIC9HqPg.js.map +0 -1
- package/dist/tart-cli-BMrkmBjI.js +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-DtRhj2Tn.js +0 -2
- package/dist/worker-entry-BIROTPX1.js.map +0 -1
- package/dist/workspace-CcQRT31C.js +0 -1
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: reeval
|
|
3
|
+
description: >
|
|
4
|
+
How to write, run and debug evals with reeval, the eval runner for coding agents (npm package
|
|
5
|
+
reeval; CLI `reeval`, alias `re`). Use whenever a task mentions reeval, reeval.config.ts,
|
|
6
|
+
*.eval.ts files, `reeval run`, evals for a coding agent, comparing skills, MCP servers or
|
|
7
|
+
docs, or lift. Not for changing reeval's own source: contributors follow the repository's
|
|
8
|
+
AGENTS.md.
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
# reeval
|
|
12
|
+
|
|
13
|
+
This skill is for writing evals with reeval, not for changing reeval itself: to contribute to
|
|
14
|
+
the invertase/reeval repository, follow its root `AGENTS.md` instead.
|
|
15
|
+
|
|
16
|
+
reeval runs a real coding agent (Cursor, Claude Code, Antigravity, OpenCode, Goose, pi,
|
|
17
|
+
Codex, fx or Grok Build) on a task in a fresh Docker sandbox, then checks the result with
|
|
18
|
+
deterministic assertions and an LLM judge. You write evals in TypeScript; `reeval run` runs them.
|
|
19
|
+
|
|
20
|
+
The package and CLI are `reeval`. Full docs: the `docs/` folder of github.com/invertase/reeval (a docs.page
|
|
21
|
+
site; its Reference tab has every option, flag and error).
|
|
22
|
+
|
|
23
|
+
This file covers the common work: set up a project, write an eval with checks and a judge,
|
|
24
|
+
compare a skill, fix a config, read a failure. Read a reference file when the task needs it:
|
|
25
|
+
|
|
26
|
+
| Read | When the task involves |
|
|
27
|
+
|---|---|
|
|
28
|
+
| [references/config.md](references/config.md) | Harness factories and their keys, `effort`, judge models, `reeval.yaml`, path rules, where the sandbox comes from, imports, the options cascade |
|
|
29
|
+
| [references/evals.md](references/evals.md) | `cases`, `caseKey` and `where`, `gate: 'observe'`, `toStayWithin`, `c.serve`, `c.writeFile` hidden tests, the judge's inputs, the judge's input size, judge errors, every matcher |
|
|
30
|
+
| [references/capture.md](references/capture.md) | Web UIs: `c.capture()` screenshots, `instructions` journeys, `artifacts` on judges and questions |
|
|
31
|
+
| [references/compare.md](references/compare.md) | `compare` beyond the example below: the control, gates, lift and effort, labeled checks, `questions`, `skill_files` |
|
|
32
|
+
| [references/environments.md](references/environments.md) | Environments: skills, files, rules, MCP servers, `git()` workspaces, `directory()`, Docker sandboxes, secrets, egress |
|
|
33
|
+
| [references/cli.md](references/cli.md) | The CLI beyond `run`: `query` views, `analyze`, `show`, `compare`, `record`, `studio`, `mcp`, filters, exit codes |
|
|
34
|
+
| [references/mistakes.md](references/mistakes.md) | An error message from reeval you need to map to its fix (the full table) |
|
|
35
|
+
| [../eval-rubric/SKILL.md](../eval-rubric/SKILL.md) | A `Worth knowing:` list after a run, or reviewing an eval's name, prompts and judge questions: one section per remark kind, with its fix |
|
|
36
|
+
|
|
37
|
+
Use reeval's terms in code and messages: eval (`evaluate`), attempt (one run of a cell;
|
|
38
|
+
`runs: n`), harness (`cursor()`, …), cell (harness × environment × sandbox), control and variant,
|
|
39
|
+
lift (variant pass@1 − control pass@1, 95% interval), gate, checks and judge, transcript, effort
|
|
40
|
+
(measured tool calls, agent time, tokens). Not "trial", "baseline/treatment" or "grader";
|
|
41
|
+
[references/vocabulary.md](references/vocabulary.md) maps other frameworks' words to these.
|
|
42
|
+
|
|
43
|
+
## Workflow
|
|
44
|
+
|
|
45
|
+
Setting reeval up for someone (new to reeval, or "set up reeval here")? Follow the setup
|
|
46
|
+
skill, [../setup/SKILL.md](../setup/SKILL.md), first: it asks what to evaluate and
|
|
47
|
+
writes the first evals. This section is the reference it builds on.
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
npm i -D reeval
|
|
51
|
+
npx reeval init --harness cursor # or --bare: the config, no example eval
|
|
52
|
+
npx reeval run --dry-run # the plan, no keys or Docker
|
|
53
|
+
npx reeval image build # once, for real agents
|
|
54
|
+
npx reeval doctor # Docker, keys and images, with fixes
|
|
55
|
+
npx reeval run
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
1. Install: `npm init -y` if there's no `package.json`, then `npm i -D reeval`
|
|
59
|
+
(or a tarball: `npm i -D /path/to/reeval.tgz`). Pre-release: pin the version.
|
|
60
|
+
Node 22.13 or later; npm, pnpm or Yarn.
|
|
61
|
+
2. `npx reeval init --harness <cursor|claude-code|antigravity|opencode|goose|pi|codex|fx|grok-build>`
|
|
62
|
+
(it only asks in a terminal) writes `reeval.config.ts` (the agent, `analysis.model:
|
|
63
|
+
'google/gemini-3.8-flash'` for `reeval analyze`, and `pr` and `nightly` profiles), `evals/environments.ts` (one `defineEnvironment`), `evals/greet.eval.ts` (an eval
|
|
64
|
+
in that environment), the `.gitignore` entries (`.reeval/`, `.env*`), an `eval` script and a
|
|
65
|
+
`tsconfig.json` that allows the evals' `.ts` imports, then says whether Docker is running.
|
|
66
|
+
`--bare` writes only the config, `.gitignore` and `package.json`. Where a config exists it
|
|
67
|
+
exits 2 before asking; `--force` replaces only `reeval.config.ts`. Otherwise it only adds:
|
|
68
|
+
it merges into `package.json` and `.gitignore` and skips any other file that exists. Or write
|
|
69
|
+
`reeval.config.ts` and `evals/**/*.eval.ts` (the default `include`) yourself.
|
|
70
|
+
3. `npx reeval run --dry-run` prints the plan (evals, cells, attempts, time and a cost range)
|
|
71
|
+
without running anything and needs no credentials or Docker. `--dry-run --json` for a program
|
|
72
|
+
to read.
|
|
73
|
+
4. For real agents, and before any `tart()` run (a run never pulls its macOS image; `tart()`
|
|
74
|
+
runs every real harness, and the build bakes each one's CLI into a copy of the image):
|
|
75
|
+
`npx reeval image build` once (and after upgrading reeval), keys in
|
|
76
|
+
`.env.local` next to `reeval.config.ts` (not the directory you run from), `npx reeval doctor`
|
|
77
|
+
(exit 1 names each problem and its fix; a missing `analysis.model` or its key,
|
|
78
|
+
`GOOGLE_GENERATIVE_AI_API_KEY`, is only a warning; `--json`, `--online` tries the keys), then
|
|
79
|
+
`npx reeval run`. Exit 0 means every gated cell passed. `npx reeval run --json` prints the
|
|
80
|
+
results as one JSON document on stdout (each check `ran` or `not run`, each variant's
|
|
81
|
+
`effect`), with the progress on stderr.
|
|
82
|
+
5. Read failures in the summary, then `.reeval/runs/<run>/summary.md` and each attempt's directory
|
|
83
|
+
(`transcript.jsonl`, `diff.patch`, `checks/<n>.log`, `judge-input.json`, `setup.log`).
|
|
84
|
+
`npx reeval run -v` prints a timestamped trace per attempt; `.reeval/debug/` keeps it.
|
|
85
|
+
Each attempt's whole final workspace is in its directory there, under `snapshots/final/`, to
|
|
86
|
+
read or run what the agent built.
|
|
87
|
+
`npx reeval query "SELECT … FROM failing_attempts"` names each failed attempt's first failed
|
|
88
|
+
check (more views in [references/cli.md](references/cli.md)).
|
|
89
|
+
|
|
90
|
+
Always rerun `--dry-run` after editing: config mistakes are exit-2 usage errors with a precise
|
|
91
|
+
message, one at a time.
|
|
92
|
+
|
|
93
|
+
## reeval.config.ts
|
|
94
|
+
|
|
95
|
+
```ts
|
|
96
|
+
import { defineConfig } from 'reeval'
|
|
97
|
+
import { cursor } from 'reeval/harness'
|
|
98
|
+
|
|
99
|
+
export default defineConfig({
|
|
100
|
+
include: ['evals/**/*.eval.ts'], // default
|
|
101
|
+
harnesses: cursor({ model: 'composer-2.5' }), // model required; CURSOR_API_KEY
|
|
102
|
+
// no default judge model; pin versions
|
|
103
|
+
judge: { model: 'typesafe-ai/jev-1.14.0', threshold: 0.8 },
|
|
104
|
+
// runs: attempts per cell; unset, 1, or 4 for a compare with variants
|
|
105
|
+
timeout: '15m', // per attempt
|
|
106
|
+
})
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
- Harnesses (`reeval/harness`), each with a required `model`; a list runs each as its own cell:
|
|
110
|
+
`cursor` (`CURSOR_API_KEY`), `claudeCode` (`ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`), `antigravity`
|
|
111
|
+
(`GEMINI_API_KEY`), `opencode` (the provider's key; model `<provider>/<model>`), `goose` and
|
|
112
|
+
`pi` (the provider's key; model `<provider>/<model>`, e.g. `google/gemini-3.8-flash` on `GOOGLE_GENERATIVE_AI_API_KEY`), `codex` (`OPENAI_API_KEY`), `fx` (`OPENAI_API_KEY` for a bare id,
|
|
113
|
+
`AI_GATEWAY_API_KEY` with `vercel()` or `OPENROUTER_API_KEY` with `openrouter()` in `gateway`), `grokBuild` (`XAI_API_KEY`), `fakeHarness`. Details
|
|
114
|
+
and `effort` in [references/config.md](references/config.md).
|
|
115
|
+
- Judge models: `'typesafe-ai/jev-1.14.0'` (`TYPESAFE_AI_API_KEY`, recommended),
|
|
116
|
+
`'google/gemini-3.8-flash'` (`GOOGLE_GENERATIVE_AI_API_KEY`), `anthropic/*` (`ANTHROPIC_API_KEY`)
|
|
117
|
+
and `'openai/gpt-6-luna'` (`OPENAI_API_KEY`; OpenAI's Decisions API takes only its decision
|
|
118
|
+
model, so not `openai/gpt-5.5`), each on its provider's own key. reeval uses no gateway by
|
|
119
|
+
default: `AI_GATEWAY_API_KEY` is read only with `gateway: [vercel()]` in the config (or a judge
|
|
120
|
+
`gateway: 'vercel'` pin, listed there), `OPENROUTER_API_KEY` only with `openrouter()`. As a
|
|
121
|
+
judge, `vercel()` serves decision models only (Jev, `openai/gpt-6-luna-decisions`), and
|
|
122
|
+
`openrouter()` only those on its Decisions API (`typesafe/jev-1.14`), never a language model:
|
|
123
|
+
pin a language-model judge to `gateway: 'direct'`. Always a provider prefix. Every `gateway`
|
|
124
|
+
list ends with `direct()`, so a cell no listed gateway serves falls back to its own key (the
|
|
125
|
+
plan's hint says so); `direct({ enabled: false })` refuses it instead.
|
|
126
|
+
- Imports: `reeval` (`defineConfig`, `defineEnvironment`, `group`, `evaluate`,
|
|
127
|
+
`expect`, `DEFAULT_EGRESS_ALLOW`, `DEFAULT_SKILL_EXCLUDE`, types), `/harness`, `/sandbox`
|
|
128
|
+
(`docker`, `fakeSandbox`), `/judge` (`fakeJudge`, `RECOMMENDED_JUDGE_MODEL`,
|
|
129
|
+
`DEFAULT_THRESHOLD`). There is no `test`, `describe`, `baseline`, `secret` or `skill` export.
|
|
130
|
+
- `defineConfig`, `group` and `evaluate` take the same options (`harnesses`, `sandbox`,
|
|
131
|
+
`environments`, `env`, `judge`, `runs`, `gate`, `timeout`, `tags`, …) and inner levels win;
|
|
132
|
+
`compare` is for groups and evals only. Options are plain objects, never functions. Unknown
|
|
133
|
+
keys are errors with a hint (`repeat` → "Did you mean runs?"), and type errors in a
|
|
134
|
+
`defineConfig` object.
|
|
135
|
+
- `runs: { bestOf: 5 }` (odd, 3 or more) stops each cell once pass or fail has a majority (3 to 5
|
|
136
|
+
attempts) and gates on it, replacing `gate`. Use it to settle a flaky pass/fail for less than a
|
|
137
|
+
fixed count; not on an eval with `compare` (refused: lift needs equal attempts), and a cell
|
|
138
|
+
only reaches `flaky_cells`' 6-attempt `flaky` in one run from `bestOf: 7`.
|
|
139
|
+
- An eval's fingerprint reads its own body, options, case and `revision`, not
|
|
140
|
+
helpers, constants outside the body or fixtures. After changing one of those in a way that
|
|
141
|
+
changes what the eval measures, bump `revision` (`{ revision: 2 }`, on the eval or its group)
|
|
142
|
+
so `reeval compare` says `can't compare … revision 1 → 2` instead of `regressed`.
|
|
143
|
+
|
|
144
|
+
### Without Docker or API keys: the fake trio
|
|
145
|
+
|
|
146
|
+
```ts
|
|
147
|
+
import { defineConfig } from 'reeval'
|
|
148
|
+
import { fakeHarness } from 'reeval/harness'
|
|
149
|
+
import { fakeJudge } from 'reeval/judge'
|
|
150
|
+
import { fakeSandbox } from 'reeval/sandbox'
|
|
151
|
+
|
|
152
|
+
export default defineConfig({
|
|
153
|
+
// A scripted "agent": writes these files every turn and replies with text.
|
|
154
|
+
harnesses: fakeHarness({
|
|
155
|
+
files: { 'hello.txt': 'hello\n' },
|
|
156
|
+
text: 'Created hello.txt.',
|
|
157
|
+
}),
|
|
158
|
+
sandbox: fakeSandbox(), // default machine; not an environment
|
|
159
|
+
judge: { model: fakeJudge({ score: 0.9 }) }, // fixed score, no model call
|
|
160
|
+
})
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
`fakeHarness` also takes `withSkill: { files, text }` (used when the environment gives the agent
|
|
164
|
+
a skill), `script` (one list of `{ exec }`, `{ write }`, `{ reply }`, `{ mcp: 'server/tool' }`
|
|
165
|
+
steps per turn), `layout` (an agent's id, such as `'claude-code'`), `failWith`, `delayMs`,
|
|
166
|
+
`turnMs` (each turn's agent time, exactly: effort comparisons without scheduling jitter),
|
|
167
|
+
`flakeRate`, `reasoning`, `tools` and `id`.
|
|
168
|
+
|
|
169
|
+
`fakeSandbox()` is for testing configs and reeval itself without Docker. It runs commands on
|
|
170
|
+
your host with your tools, brokers no secrets and enforces no egress, so an eval passing there says nothing about its checks in the eval's real sandbox.
|
|
171
|
+
Don't use it to validate an eval, and don't put it on an environment that real harnesses use:
|
|
172
|
+
real Cursor runs need Docker.
|
|
173
|
+
|
|
174
|
+
## Evals
|
|
175
|
+
|
|
176
|
+
```ts
|
|
177
|
+
// evals/hello.eval.ts
|
|
178
|
+
import { evaluate, expect, group } from 'reeval'
|
|
179
|
+
|
|
180
|
+
group('hello', () => {
|
|
181
|
+
evaluate('creates hello.txt', async (c) => {
|
|
182
|
+
const turn = await c.send('Create hello.txt containing "hello".')
|
|
183
|
+
await expect(turn).toHaveFile('hello.txt', /hello/) // async: await it
|
|
184
|
+
await expect(c.exec('grep -q hello hello.txt')).toSucceed()
|
|
185
|
+
const q = 'Does hello.txt contain the right content?'
|
|
186
|
+
expect(await c.judge(q, { contents: ['hello.txt'] })).toPass()
|
|
187
|
+
})
|
|
188
|
+
})
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
- `evaluate(name, [options,] fn)` and `group(name, [options,] fn)`; also `.skip`, `.only`
|
|
192
|
+
(an error in CI), `.todo`. `{ cases: [...] }` runs one eval over a list, with the body
|
|
193
|
+
`(c, item) => …` (cases pool into one pass rate, see [references/evals.md](references/evals.md)).
|
|
194
|
+
`gate: 'observe'` gates nothing: the eval records what agents do. The eval id is `file › group › name`; renaming an
|
|
195
|
+
eval restarts its history. Name the eval after the question it answers
|
|
196
|
+
(`recommends Genkit for a production AI feature`, not `genkit test 2`), so the results and
|
|
197
|
+
`reeval analyze` read in those terms.
|
|
198
|
+
- `await c.send(prompt)` runs one agent turn and returns a `Turn` (`text`, `status`, `diff`,
|
|
199
|
+
`transcript`). Call it again for multi-turn;
|
|
200
|
+
`c.judge(q, { on: firstTurn, task: true, output: true })` grades an earlier one.
|
|
201
|
+
- `await c.exec(cmd)` returns `{ exitCode, stdout, stderr }`. A check that can fail on a
|
|
202
|
+
transient (a hidden test that calls a model or a network service) gets
|
|
203
|
+
`c.exec(cmd, { retries: 2 })`: any non-zero exit runs it again, and the record and output say
|
|
204
|
+
`passed on try 2 of 3`. Don't roll a retry loop into the test itself.
|
|
205
|
+
`c.exec(cmd, { env: { EXPECTED: '42' } })` sets variables for that command only (a hidden
|
|
206
|
+
verify script's inputs), over the environment's; secret-named values are masked, and a brokered
|
|
207
|
+
secret's host value goes in as its placeholder.
|
|
208
|
+
`await c.readFile(path)` the text or `null` (`~/` for home). `await c.writeFile(path, text)` places a file after a turn (a hidden
|
|
209
|
+
test the agent never saw). Web apps: `c.serve` and `c.capture` in
|
|
210
|
+
[references/capture.md](references/capture.md).
|
|
211
|
+
- `await c.judge(question, options)`: the judge sees only the inputs the call turns on, and it
|
|
212
|
+
needs at least one (there are no defaults, here or in any config): `task`, `output`, `files`,
|
|
213
|
+
`diff` (`true` or patterns), `contents` (files whole), `transcript` (`true`, `'tools'`,
|
|
214
|
+
`'full'`), `checks`, `artifacts`. Pick what the question needs: code → `{ diff: ['src/**'] }`
|
|
215
|
+
or `{ contents: ['README.md'] }`, the agent's answer → `{ output: true }`, how it worked →
|
|
216
|
+
`{ transcript: 'tools' }`. `on` and `turns` only select; details in
|
|
217
|
+
[references/evals.md](references/evals.md). Prefer deterministic checks for what can be checked; use the judge for quality, with plain
|
|
218
|
+
yes/no questions that don't name one specific way of doing it.
|
|
219
|
+
- A failed judge call doesn't throw: `c.judge()` returns `{ verdict: 'error', score: null, error }`,
|
|
220
|
+
the later judges and checks still run, and the attempt fails as a judge error at the end, or
|
|
221
|
+
as an infra error (which `--retry-errors` reruns) when the provider never answered and nothing
|
|
222
|
+
else failed.
|
|
223
|
+
`expect(await c.judge(q, { diff: true }), name).toPass()` records each judge's result and carries on.
|
|
224
|
+
- Make the eval trustworthy. Before a real run, swap in the fake harness, and only the harness,
|
|
225
|
+
so the eval runs in its real sandbox (its `docker()` image, `install`, egress and brokered
|
|
226
|
+
secrets): an oracle `fakeHarness` that does the task right must pass, and a canary that cheats
|
|
227
|
+
or gets it wrong must fail. A canary that passes means a check isn't checking. Put them in the
|
|
228
|
+
one config as profiles that set `defaults.harnesses` (and `defaults.judge`), never
|
|
229
|
+
`defaults.sandbox`, then `reeval image build --profile oracle` and
|
|
230
|
+
`reeval run --profile oracle` / `--profile canary` (needs Docker, no keys); see
|
|
231
|
+
[references/evals.md](references/evals.md#trustworthy-evals). Not `fakeSandbox()`.
|
|
232
|
+
Keep grading out of the agent's reach: place hidden tests after the turn with `c.writeFile`, and guard seeded files
|
|
233
|
+
with `expect(await c.readFile(path)).toBe(original)` or `expect(c).not.toHaveChanged(['test/**'])`.
|
|
234
|
+
When the behavior is the outcome, judge the program's output, not the diff:
|
|
235
|
+
`c.judge(q, { on: (await c.exec('node app.js')).stdout, output: true })`.
|
|
236
|
+
- Give the environment a `.gitignore` in `files` (the project's isn't copied in): an agent that
|
|
237
|
+
runs a build otherwise fills the judge's diff with its output.
|
|
238
|
+
- Matchers: `await expect(turn).toHaveFile(path, pattern?)`,
|
|
239
|
+
`expect(await c.exec(cmd)).toSucceed()` (or `await expect(c.exec(cmd)).toSucceed()`),
|
|
240
|
+
`expect(await c.judge(q, { diff: true })).toPass()`, plus Vitest's. Each takes only its own subject:
|
|
241
|
+
`toPass()` on an exec result is a type error, so typecheck evals (`tsc --noEmit`, with
|
|
242
|
+
`typescript` installed).
|
|
243
|
+
Transcript matchers (sync, no `await`): `expect(turn | c).toHaveCalledSkill(name)`,
|
|
244
|
+
`toHaveCalledMcp(server, tool, input?)`, `toHaveCalledTool(name | RegExp, input?)`,
|
|
245
|
+
`toHaveRun(command?)`, `toHaveEdited(path?)`, `toHaveSearched(query?)`,
|
|
246
|
+
`toHaveFetched(url?)`. `toHaveEdited` counts edit tools' events and the files that changed
|
|
247
|
+
while the agent ran the turn, so a shell write counts and a read doesn't. `toHaveRun('genkit')`
|
|
248
|
+
matches anywhere in a command line (`cat genkit.config.ts` too); to ask "did it run the CLI",
|
|
249
|
+
use `toHaveRun({ program: 'genkit', args: /^flow:run\b/ })`, which matches the program each
|
|
250
|
+
segment ran after env assignments, `timeout`, `sudo`, `npx`, `pnpm dlx` and similar wrappers.
|
|
251
|
+
`shellCommands(c)` (from `reeval`) lists those `{ program, args, line, ok, turn }` to count. Guard files with
|
|
252
|
+
`expect(c).not.toHaveChanged(globs)`: its globs are exact, while `toHaveEdited`'s string
|
|
253
|
+
matches any path containing it. `toHaveEdited()` counts files that appear in the workspace
|
|
254
|
+
diff. Gitignored files don't appear there, so use `toHaveFile()` to check them. Label
|
|
255
|
+
requirements with
|
|
256
|
+
`expect`'s second argument: `await expect(c, 'exports greet').toHaveFile('greet.js', /export/)`.
|
|
257
|
+
- Checks don't stop the eval: a failed `expect` is recorded, the body carries on, and the attempt
|
|
258
|
+
fails at its end, with every failed check listed by label and line. Use `expect.require` for
|
|
259
|
+
gates, such as a build the later checks depend on: its failure ends the body there
|
|
260
|
+
(`await expect.require(c.exec('npm run build'), 'builds').toSucceed()`). A caught
|
|
261
|
+
`expect.require` still fails the attempt. Thrown errors (the body's own, a dead sandbox, a
|
|
262
|
+
timeout) still end the attempt. To wait for something, use
|
|
263
|
+
`expect.poll(() => c.exec(cmd), { timeout: 10_000 }).toSucceed()` (Python
|
|
264
|
+
`expect.poll(lambda: c.exec(cmd), timeout=10).to_succeed()`), never `expect` in a retry loop. A
|
|
265
|
+
caught plain Python `assert` records nothing, so `assert` or `if` in a loop also works.
|
|
266
|
+
|
|
267
|
+
## Environments and compare
|
|
268
|
+
|
|
269
|
+
An environment is what the agent runs *in*: skills, files, rules, MCP servers, env vars, setup
|
|
270
|
+
steps and the workspace's starting code. `compare` runs the same eval in a control and each
|
|
271
|
+
variant and reports lift.
|
|
272
|
+
|
|
273
|
+
```ts
|
|
274
|
+
import { defineEnvironment, evaluate, expect, group } from 'reeval'
|
|
275
|
+
|
|
276
|
+
const greetsAda = `node -e "import('./greet.js').then((m) =>
|
|
277
|
+
process.exit(m.greet('Ada') === 'Ahoy, Ada!' ? 0 : 1))"`
|
|
278
|
+
const plain = defineEnvironment('plain', {})
|
|
279
|
+
const houseStyle = plain.extendEnvironment('house-style', ({ skill }) => ({
|
|
280
|
+
skills: [skill.local('./skills/house-greeting')], // a dir with SKILL.md, relative to this file
|
|
281
|
+
}))
|
|
282
|
+
|
|
283
|
+
group(
|
|
284
|
+
'greetings',
|
|
285
|
+
{ compare: { control: plain, variants: [houseStyle] } },
|
|
286
|
+
() => {
|
|
287
|
+
evaluate('greets in the house style', async (c) => {
|
|
288
|
+
await c.send('Create greet.js exporting greet(name) in our house style.')
|
|
289
|
+
await expect(c.exec(greetsAda)).toSucceed()
|
|
290
|
+
})
|
|
291
|
+
},
|
|
292
|
+
)
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
- A spec, or a synchronous callback that destructures the helpers it needs (`skill`, `plugin`,
|
|
296
|
+
`file`, `directory`, `git`, `mcp`, `secret`): `files: { 'docs/api.md': file('../docs/api.md') }`,
|
|
297
|
+
`rules: file('../rules/house.md')` (AGENTS.md; CLAUDE.md for Claude Code),
|
|
298
|
+
`mcp: { docs: mcp.stdio('npx', ['-y', '@acme/docs-mcp@1.0.0']) }`,
|
|
299
|
+
`setup: [{ run: 'npm ci', timeout: '5m' }]`. More in
|
|
300
|
+
[references/environments.md](references/environments.md).
|
|
301
|
+
- Attempts = evals × harnesses × (1 control + variants) × runs. The example is 1 eval, 2 cells
|
|
302
|
+
(`<harness>/plain/<sandbox>` control, `<harness>/house-style/<sandbox>` variant), 8 attempts:
|
|
303
|
+
without `runs`, a compare runs 4 a cell, the fewest whose lift often holds (with `cases`, once a
|
|
304
|
+
case: the lift pairs cases). A `runs` below 4 shows
|
|
305
|
+
only the direction, with a `too-few-tries` remark; `runs: 1` at any level or a profile's `limit.runs: 1` keeps a smoke run cheap.
|
|
306
|
+
- Each variant runs as that environment on its own; `compare` does not layer, inherit or
|
|
307
|
+
extend. Share setup with `parent.extendEnvironment(name, spec)` before `compare`. `control` is
|
|
308
|
+
required, and `environments` beside `compare` is an error: the control goes in `compare`.
|
|
309
|
+
- Only variant cells gate by default; a control's misses don't fail the run. Below four tries a
|
|
310
|
+
side each variant's effect is only a direction; from four, the block adds an effort table
|
|
311
|
+
(median tools, time, tokens per arm), which often shows a skill's effect first. In a compare,
|
|
312
|
+
keep independent requirements as labeled `expect`s, not `expect.require`, so each gets a verdict.
|
|
313
|
+
- `questions: [{ question: 'How did the agent pick its greeting?', transcript: true }]` beside
|
|
314
|
+
`compare` asks a model about each attempt as it ends (`choices: ['yes', 'no']` for a pick); an
|
|
315
|
+
entry with `across: 'environment'` (and, say, `diff: true`) asks which side did better, pair by
|
|
316
|
+
pair, after the run. Each entry turns on at least one input, as on `c.judge()`, except `contents`. Questions
|
|
317
|
+
never gate and need a language model. Details, gates,
|
|
318
|
+
`minLift` and `reeval query skill_files` (which skill files each attempt opened) are in
|
|
319
|
+
[references/compare.md](references/compare.md).
|
|
320
|
+
|
|
321
|
+
## CLI
|
|
322
|
+
|
|
323
|
+
```text
|
|
324
|
+
reeval run [filter...] --dry-run [--json] --runs <n> --tag <t> --trait <k=v> --profile <p>
|
|
325
|
+
--config <path> --record --ui plain -v
|
|
326
|
+
reeval show [run] reeval history reeval compare [base] [head]
|
|
327
|
+
reeval query lift reeval query effort
|
|
328
|
+
reeval query "SELECT … FROM failing_attempts"
|
|
329
|
+
reeval analyze reeval studio reeval image build reeval doctor
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
| Exit code | Means |
|
|
333
|
+
|---|---|
|
|
334
|
+
| 0 | Pass, and `--dry-run` |
|
|
335
|
+
| 1 | A gated failure |
|
|
336
|
+
| 2 | A usage or config error, including a filter that matches nothing |
|
|
337
|
+
| 3 | An infra error, such as a sandbox that failed to start, a harness key the API rejected or a turn the provider stopped |
|
|
338
|
+
|
|
339
|
+
`reeval run` ends with a status list: each eval file, a line per eval under it with its icon and
|
|
340
|
+
`N of M runs passed`, and for a failure, a compare or an eval with remarks, a sentence for each
|
|
341
|
+
way it went (the harnesses, a variant's effect as `helped`, `made it worse` or `same`, the failed
|
|
342
|
+
checks with their messages and lines; for an agent, also the rest of each message, the snapshot
|
|
343
|
+
and each remark's whole sentence); `-v` expands every eval. Then the tally
|
|
344
|
+
(`Done in 12s · 1 failed · 2 passed`; an `error:` outside any eval, such as a stray rejection in an eval file, is
|
|
345
|
+
`1 run error` and fails the run; an eval with `gate: 'observe'` is `1 ran`, never a pass,
|
|
346
|
+
so read its arm rows for what passed). `Worth knowing:` under an eval names what about it needs work
|
|
347
|
+
(a flaky check, too few tries, a slow MCP server), with the line to fix, from fixed rules with no
|
|
348
|
+
model; they never change a verdict or the exit code, and go in `remarks.json` beside the run. The
|
|
349
|
+
eval rubric ([../eval-rubric/SKILL.md](../eval-rubric/SKILL.md)) explains each kind. `summary.md`
|
|
350
|
+
and `reeval show` print the full result blocks, a table per arm and check.
|
|
351
|
+
|
|
352
|
+
Every command uses the nearest `reeval.config.*` (or `reeval.yaml`) in the current directory or
|
|
353
|
+
above, or `--config <path>`. Runs land in `.reeval/runs/`. All views, `analyze` and the MCP
|
|
354
|
+
server: [references/cli.md](references/cli.md).
|
|
355
|
+
|
|
356
|
+
## Common mistakes (and the fix)
|
|
357
|
+
|
|
358
|
+
| Mistake | Fix |
|
|
359
|
+
|---|---|
|
|
360
|
+
| `compare` in `reeval.config.ts` | Put it on the group or eval |
|
|
361
|
+
| `environments: [a, b]` to measure b, or `compare: [b]` | `compare: { control: a, variants: [b] }` |
|
|
362
|
+
| `compare: { baseline: a, … }` | `compare: { control: a, … }` |
|
|
363
|
+
| `repeat: 3` / `times` | `runs: 3` |
|
|
364
|
+
| Judge model `'gemini-3.8-flash'` | `'google/gemini-3.8-flash'` |
|
|
365
|
+
| Forgetting `await` on `toHaveFile`, or on `toSucceed` given `c.exec()`'s promise | `await expect(...)` |
|
|
366
|
+
| `toPass()` on `c.exec()`'s result | `toSucceed()`; `toPass()` is for `await c.judge()` |
|
|
367
|
+
| `cursor()` with `fakeSandbox()` | Use `fakeHarness()` or `docker()` |
|
|
368
|
+
| Test file named `*.test.ts` | `*.eval.ts` under `evals/` |
|
|
369
|
+
|
|
370
|
+
Every error message and its fix: [references/mistakes.md](references/mistakes.md).
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Web UIs: serve, capture and artifacts
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
evaluate('builds the sign-in page', async (c) => {
|
|
5
|
+
await c.send('Build a sign-in page served by `npm start` on $PORT.')
|
|
6
|
+
const app = await c.serve('npm start')
|
|
7
|
+
await c.capture(app, { name: 'sign-in', path: '/login', wait: 'idle' })
|
|
8
|
+
const q = 'Does the screenshot show an email field and a password field?'
|
|
9
|
+
expect(await c.judge(q, { artifacts: ['sign-in'] })).toPass()
|
|
10
|
+
})
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
- `await c.serve(cmd, port?)` starts `cmd` in the background with `PORT` set (`$PORT` on a bash
|
|
14
|
+
string) and returns a `ServeHandle` once `cmd` listens. Without `port`, a server
|
|
15
|
+
that ignores `PORT` and listens on one new port of its own (`npm run dev` on Vite's 5173, IPv4 or
|
|
16
|
+
IPv6 loopback) gets that port as `handle.port`; no `--port` flags needed. Several new ports, none
|
|
17
|
+
of them `PORT`, throw with the list and the URLs it printed: pass the one to use, `c.serve('npm run dev', 5173)`. An
|
|
18
|
+
exit, or nothing listening in 120 s, throws with its last output. With `port`, only that
|
|
19
|
+
port counts, and a `port` something already listens on (the agent's own dev server, left
|
|
20
|
+
running) throws before `cmd` starts: pass none and serve on `PORT`. `c.exec(cmd, { background:
|
|
21
|
+
true })` starts a command without a port or the wait and returns `{ kill() }`.
|
|
22
|
+
- `await c.capture(port | server, { name?, path?,
|
|
23
|
+
wait?, instructions?, viewport?, model? })` screenshots the tunneled port into
|
|
24
|
+
`artifacts/<name>.png`. The first argument is the container port or a `ServeHandle`;
|
|
25
|
+
default `path` is `/`, default `name` comes from the path. A `name` is 1 to 100 of
|
|
26
|
+
`a-z`, `0-9`, `_` and `-`, starting with a letter or digit. `wait` is once, before the
|
|
27
|
+
first model shot or the saved PNG (`'load'`, `'idle'`, or `{ ms }`). `instructions` is
|
|
28
|
+
a journey of goal strings and `{ wait }` beats (`{ wait: 'idle' }` waits for the
|
|
29
|
+
network to go quiet from that beat, so it covers a click that only fetches) — no
|
|
30
|
+
selectors; the model clicks pixels. `viewport: 'full-page'` is the saved shot (default window + tall PNG).
|
|
31
|
+
- `capture.model` cascades like `question`; the call may override. Default is
|
|
32
|
+
`analysis.model`, else the attempt's judge model (a profile's, group's or eval's),
|
|
33
|
+
whichever is first a language model; never Jev. Gemini's 0–1000 points are scaled to the screenshot. Fixture evals import
|
|
34
|
+
`fakeInstructions` from `reeval/testing` and pass it as `instructions`.
|
|
35
|
+
The same `timeout` is the budget for the whole capture, its instructions included: if it
|
|
36
|
+
runs out the capture fails and there is no `<name>.png`, but the page as the instruction
|
|
37
|
+
stopped is in `artifacts/<name>.step-<n>.png`, which the error names. `capture.browser: false` or
|
|
38
|
+
`REEVAL_CAPTURE_BROWSER=0` makes `c.capture()` throw.
|
|
39
|
+
- The `capture` option shallow-merges camera defaults (`viewport`, `timeout`, `browser`,
|
|
40
|
+
`model`, the vision model) down the cascade; `viewport` replaces the value, and the port is
|
|
41
|
+
not an option.
|
|
42
|
+
- On a judge, `artifacts` is an input like `diff`, set on the call: `true` for every capture
|
|
43
|
+
so far, or `c.capture()` handles or names, sent as file parts. Alone, the judge sees only the
|
|
44
|
+
pictures. Ask about facts in the picture, not whether it "looks
|
|
45
|
+
great". Jev (`typesafe-ai/jev-1.14.0`) takes PNG captures; a provider that cannot take files
|
|
46
|
+
fails that call.
|
|
47
|
+
- On a question, `artifacts: ['sign-in']` (or `true`) sends each attempt's own capture of that
|
|
48
|
+
name as an image (see [compare.md](compare.md)).
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# The CLI, results and analysis
|
|
2
|
+
|
|
3
|
+
## Contents
|
|
4
|
+
|
|
5
|
+
- Commands
|
|
6
|
+
- From a failure to what the agent did
|
|
7
|
+
- `reeval analyze`
|
|
8
|
+
- Config discovery, filters and exit codes
|
|
9
|
+
- Recording runs
|
|
10
|
+
|
|
11
|
+
## Commands
|
|
12
|
+
|
|
13
|
+
```text
|
|
14
|
+
reeval run [filter...] --dry-run [--json] --runs <n> --tag <t> --trait <k=v> --profile <p>
|
|
15
|
+
--record --ui plain -v
|
|
16
|
+
reeval record reeval history reeval show [run]
|
|
17
|
+
reeval compare [base] [head] [--md] [--fail-on-regression] [-v]
|
|
18
|
+
reeval query lift reeval query effort
|
|
19
|
+
reeval query "SELECT … FROM failing_attempts"
|
|
20
|
+
reeval analyze one checked LLM reading of a run; --vs none for this
|
|
21
|
+
run only; --context for your own agent
|
|
22
|
+
reeval studio reeval studio build --out site
|
|
23
|
+
reeval image build [--dry-run] [--profile <p>] reeval clean
|
|
24
|
+
reeval mcp the history as read-only MCP tools
|
|
25
|
+
reeval doctor Docker, keys and images, with fixes (--json, --online)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## From a failure to what the agent did
|
|
29
|
+
|
|
30
|
+
- `reeval run`'s status list names each failed check with its message and line, and `reeval show` prints the result blocks. Then `.reeval/runs/<run>/summary.md` and each attempt's
|
|
31
|
+
directory (`transcript.jsonl`, `diff.patch`, `checks/<n>.log`, `judge-input.json`,
|
|
32
|
+
`setup.log`). `reeval run -v` prints a timestamped trace per attempt; `.reeval/debug/` keeps it.
|
|
33
|
+
- For a slow run, `reeval show <run> --trace` writes `.reeval/traces/<run>.json`, a Perfetto
|
|
34
|
+
timeline: attempts in lanes with their setup, turns, tool calls, checks and judge.
|
|
35
|
+
- Each attempt's directory in `.reeval/runs/<run>/` has `snapshots/final/`: its whole final
|
|
36
|
+
workspace after the checks (the app the agent built, not only its diff), to read or run. A failed
|
|
37
|
+
attempt's result block in `reeval show` prints its path. `snapshot: 'turns'` on an eval also keeps
|
|
38
|
+
`snapshots/turn-<n>/` after each `c.send`; `snapshot: { compress: true }` writes
|
|
39
|
+
`<name>.tar.gz` instead; `snapshot: false` turns it off. Snapshots leave out `node_modules/`,
|
|
40
|
+
`.git/` and caches, are capped (50 MB as a directory, 10 MB compressed), never go into the
|
|
41
|
+
history folder, and are kept for the last `debug.keep` runs (20 by default) until `reeval clean`.
|
|
42
|
+
An attempt that timed out or was canceled has none.
|
|
43
|
+
- `reeval query "SELECT … FROM failing_attempts"`: each failed attempt and infra error with its
|
|
44
|
+
cell, skill and `first_failed_check` (the label of the first check that failed).
|
|
45
|
+
- `reeval query lift` and `reeval query effort` list every variant cell's lift and effort from
|
|
46
|
+
recorded runs. `reeval query "SELECT eval_key, label, verdict FROM check_discrimination WHERE
|
|
47
|
+
verdict LIKE 'both-%'"` finds checks that don't tell a variant from its control, and
|
|
48
|
+
`reeval query "SELECT cell_key, role, rate FROM check_rates WHERE label = 'flow works'"` is one
|
|
49
|
+
check's pass rate per cell (`pass / (pass + fail)`; errors and not-run left out).
|
|
50
|
+
- `reeval query flaky_cells` lists cells that both passed and failed over comparable runs:
|
|
51
|
+
`verdict` is `flaky` from 6 counted attempts, else `mixed` (too few to call it flaky; raise
|
|
52
|
+
`runs` or record more runs before treating it as flaky).
|
|
53
|
+
- `reeval query skill_files` says which declared files (`SKILL.md`, references, rules) each
|
|
54
|
+
attempt opened, and `context_use` counts its tool calls as `context`, `dependency`
|
|
55
|
+
(`node_modules`, `--help`), `web`, `verify` or `workspace`: a heuristic, read it as counts.
|
|
56
|
+
- `reeval query --schema` lists every view and its columns; `--json` on any query for a program.
|
|
57
|
+
`--run last` (or a run id) binds `:run`: `reeval query failing_attempts --run last`, or
|
|
58
|
+
`reeval query "SELECT … FROM cells WHERE run_id = :run" --run last`. Without it a query spans
|
|
59
|
+
every run.
|
|
60
|
+
- If your client has a `reeval` MCP server (`reeval mcp`), prefer its tools for reading results:
|
|
61
|
+
`failures` is the quickest way from "it failed" to why (failed checks and judge scores, check
|
|
62
|
+
logs, the transcript's tail, and whether the cell failed in earlier runs), far smaller than
|
|
63
|
+
`show` or summary.md; then `transcript` or `compare`. The rest return the same JSON as `--json`.
|
|
64
|
+
|
|
65
|
+
## `reeval analyze`
|
|
66
|
+
|
|
67
|
+
`reeval analyze` gives a one-sentence TL;DR. Its confidence is capped after the citation and
|
|
68
|
+
command checks: not high if any check failed, low if a claim was dropped or a command rejected.
|
|
69
|
+
Anomalies are number-checked, and a cited judge score that disagrees is dropped.
|
|
70
|
+
|
|
71
|
+
When every cell is ungated (`gate: 'observe'`), it reads the run as a study: how the arms differ, check by
|
|
72
|
+
check, so label your expects. `analysis.instructions` is the question it answers.
|
|
73
|
+
|
|
74
|
+
`analysis.model` is the model `reeval analyze` uses (else the judge's, when it's a language
|
|
75
|
+
model), and the one questions without their own model use. `reeval init` sets it to `google/gemini-3.8-flash` (`GOOGLE_GENERATIVE_AI_API_KEY`;
|
|
76
|
+
`AI_GATEWAY_API_KEY` only with `gateway: [vercel()]`).
|
|
77
|
+
|
|
78
|
+
`reeval run` lists remarks under `Worth knowing:` beneath an eval that itself needs work, with no model: `flaky-check`,
|
|
79
|
+
`slow-mcp-start`, `proxy-only-egress`, `setup-process-secret`, `setup-tunnel-closed`, `snapshot-too-large`, `too-few-tries`,
|
|
80
|
+
`cant-tell-apart` and `observe-throws` (concerns, at most three), and
|
|
81
|
+
with `-v` `check-never-fails` (a suggestion). They go in `remarks.json` beside
|
|
82
|
+
the run, which `reeval record` copies into the history folder. Each names the source line to change (`Fix at <file:line>: …`).
|
|
83
|
+
Act on them: raise `runs`, make a task both arms always pass harder, tighten or remove the check, bake the MCP server's dependencies into
|
|
84
|
+
the image, or set `egress: 'allow-all'` on an eval that needs raw TCP. Each prints once per eval
|
|
85
|
+
version (`too-few-tries`, `slow-mcp-start`, `proxy-only-egress`, `setup-process-secret`, `setup-tunnel-closed` and `snapshot-too-large`
|
|
86
|
+
every run); an edit
|
|
87
|
+
re-arms it.
|
|
88
|
+
One the cap or a run without `-v` hid isn't spent: the next `reeval run -v` prints it.
|
|
89
|
+
|
|
90
|
+
## Config discovery, filters and exit codes
|
|
91
|
+
|
|
92
|
+
Filters: a part that is a path on disk (`evals/a.eval.ts`, `evals/slow/`) picks only the files it
|
|
93
|
+
names; one that names a file but isn't on disk (`skill`) also matches by name and cell key; any
|
|
94
|
+
other part picks evals by name or cell key (`reeval run skill antigravity`).
|
|
95
|
+
|
|
96
|
+
Every command uses the nearest `reeval.config.*` (or `reeval.yaml`) in the current directory or
|
|
97
|
+
above (paths resolve from its directory), or `--config <path>`. `reeval run` and `reeval image
|
|
98
|
+
build` fail with no config (`No reeval.config.ts … here or above. Run reeval init.`); there is
|
|
99
|
+
no hidden default harness or include.
|
|
100
|
+
|
|
101
|
+
| Exit code | Means |
|
|
102
|
+
|---|---|
|
|
103
|
+
| 0 | Pass, and `--dry-run` |
|
|
104
|
+
| 1 | A gated failure |
|
|
105
|
+
| 2 | A usage or config error, including a filter that matches nothing |
|
|
106
|
+
| 3 | An infra error, such as a sandbox that failed to start, a harness key the API rejected or a turn the provider stopped |
|
|
107
|
+
| 130, 143 | Ctrl-C, SIGTERM |
|
|
108
|
+
|
|
109
|
+
One Ctrl-C (or SIGTERM) cancels the run and still writes its record. A second one quits without a
|
|
110
|
+
record: `reeval run --resume <id>` keeps the finished attempts and writes it. pnpm passes Ctrl-C on
|
|
111
|
+
as a SIGTERM, which counts as the same press, and gives the terminal back: the run prints the
|
|
112
|
+
`kill <pid>` that quits it.
|
|
113
|
+
|
|
114
|
+
`--resume` plans with the run's own `--runs`, profile, filters, `--tag`, `--trait` and
|
|
115
|
+
`--concurrency`, so leave them out. Another `--runs` or profile exits 2; other filters, tags or
|
|
116
|
+
traits replace the run's for that resume. Attempts the run planned that they leave out stay in
|
|
117
|
+
the record as not run (or as they finished), the run stays `interrupted` (or `budget`), and a
|
|
118
|
+
later `--resume` runs them. A `.skip` or `.todo` never runs, so it counts as done. A resume
|
|
119
|
+
keeps passes and fails and runs infra errors again.
|
|
120
|
+
|
|
121
|
+
After an outage (a provider down, Docker out of disk), `reeval run --retry-errors [<id>]` runs a
|
|
122
|
+
finished run's infra-errored attempts again into the same run, with the same attempt ids; passes
|
|
123
|
+
and fails stay. It exits 2 if an eval's source, harness, judge, environment, the reeval version
|
|
124
|
+
or the plan changed since the run (start a new run instead). A run with no infra errors prints a
|
|
125
|
+
note and exits with its own outcome, so `reeval run || reeval run --retry-errors` stays red
|
|
126
|
+
after a real fail. Each retried attempt keeps its earlier errors in `retries`, and the run's exit
|
|
127
|
+
code is its outcome after the retry. A Ctrl-C or a kill stops the retry; `--resume` finishes
|
|
128
|
+
the retry, with the same refusals.
|
|
129
|
+
|
|
130
|
+
## Recording runs
|
|
131
|
+
|
|
132
|
+
Runs land in `.reeval/runs/`; `--record` (or `record: 'ci'` in CI) copies them to
|
|
133
|
+
`evals/.history/` to commit. That folder gets a `.gitignore` that un-ignores recorded `*.log`
|
|
134
|
+
files and a `.gitattributes` that marks recorded runs `linguist-generated`; `reeval record` warns if git still ignores a file it wrote.
|
|
135
|
+
|
|
136
|
+
`reeval compare` diffs head (default `latest`) against its nearest recorded ancestor. It prints a
|
|
137
|
+
block per eval that changed, then a tally; unchanged evals collapse to `+N same`:
|
|
138
|
+
|
|
139
|
+
```text
|
|
140
|
+
regressed greets in the house style
|
|
141
|
+
base 20261020T231407Z_ea2a649_ada-lovelace_8h1n
|
|
142
|
+
head 20261020T231408Z_b2c98b4_ada-lovelace_fypq
|
|
143
|
+
house greeting pass → fail
|
|
144
|
+
expected command to succeed, exit code 1
|
|
145
|
+
sign-off not run
|
|
146
|
+
+1 more passed
|
|
147
|
+
|
|
148
|
+
+2 same
|
|
149
|
+
…
|
|
150
|
+
1 regressed · 1 fixed · 1 can't compare · 1 new · 1 removed · 4 same
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
A check head never reached reads `not run`, not a failure. `can't compare` names what changed
|
|
154
|
+
(`changed environment`, `evals/helpers.ts`): that flip isn't a regression. A changed gate
|
|
155
|
+
(eval, config or profile `defaults`) keeps the eval comparable: base's attempts are re-scored
|
|
156
|
+
under head's gate, so a flip the gate alone explains is `gate changed`, not a regression, and
|
|
157
|
+
one base would still have passed under the new gate is `regressed`. An eval Ctrl-C
|
|
158
|
+
canceled is `can't compare` with that side `canceled`. An eval with `cases` is one row
|
|
159
|
+
with its cases' rows inside it (`cases` in `--json`); its pooled `minLift` or `passRate` gate
|
|
160
|
+
decides the row, so it can regress while every case passes. `--fail-on-regression` exits 1 on a
|
|
161
|
+
regressed eval, or on a new or can't-compare one that fails, unless head was canceled. `-v` adds each cell's
|
|
162
|
+
lift and effort, base → head.
|