reeval 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +803 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-BdzhdRYo.js} +8 -6
- package/dist/adapter-BdzhdRYo.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-Dx-bDmlV.js} +179 -31
- package/dist/agent-Dx-bDmlV.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-l0-k7vw3.js} +361 -101
- package/dist/agents-l0-k7vw3.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
- package/dist/app-files-B-CQYB0c.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CYMpN8LC.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-CsZIcJo8.js} +171 -18
- package/dist/assemble-CsZIcJo8.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-BwSmNYkX.js} +17 -8
- package/dist/attempt-host-BwSmNYkX.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
- package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
- package/dist/{boat-DSNgZt-t.js → boat-su1Y7LPd.js} +17 -14
- package/dist/boat-su1Y7LPd.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/{capture-Ccf0Hmyx.js → capture-BqIAeFyC.js} +55 -29
- package/dist/{capture-Ccf0Hmyx.js.map → capture-BqIAeFyC.js.map} +1 -1
- package/dist/capture-D0Ln5aCP.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-s_3opn_I.js} +219 -109
- package/dist/cli-s_3opn_I.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
- package/dist/cli-spec-DxzIjmq_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-C5fIzHQa.js +803 -0
- package/dist/client-C5fIzHQa.js.map +1 -0
- package/dist/{command-DSL7I-SI.js → command-B5OAIjp7.js} +25 -21
- package/dist/command-B5OAIjp7.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BJ7KqczP.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BJ7KqczP.js.map} +1 -1
- package/dist/{command-D8oMB2fa.js → command-BXLZsEwQ.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-BXLZsEwQ.js.map} +1 -1
- package/dist/{command-Ff0T-1Hi.js → command-DA_SHQUQ.js} +16 -16
- package/dist/{command-Ff0T-1Hi.js.map → command-DA_SHQUQ.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-BKBamOcX.js} +174 -40
- package/dist/commands-BKBamOcX.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-BPvAOEmb.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0URar0t.js} +255 -89
- package/dist/config-C0URar0t.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-Sj58aJEu.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-Sj58aJEu.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-FWgSo-T1.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
- package/dist/db-Bc4L-Wcr.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-BjAWxVAG.js} +142 -248
- package/dist/db-BjAWxVAG.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-CWlhZ6U1.js} +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-Ck2GYIxX.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DG_uFgn2.js → docker-CDdK6Ity.js} +41 -16
- package/dist/docker-CDdK6Ity.js.map +1 -0
- package/dist/{docker-DFrjoNs-.js → docker-D74l67k8.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-D74l67k8.js.map} +1 -1
- package/dist/{docker--mukta3L.js → docker-DMug3l-a.js} +1 -1
- package/dist/{doctor-DWK4ttex.js → doctor-eFxyn6at.js} +85 -46
- package/dist/doctor-eFxyn6at.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-CJIKz-Ud.js} +49 -11
- package/dist/environment-CJIKz-Ud.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CXIRJ91v.js} +24 -12
- package/dist/eval-source-CXIRJ91v.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-DA4QsrHO.js} +792 -281
- package/dist/evaluate-DA4QsrHO.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
- package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
- package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
- package/dist/{files-BOBdC0uI.js → files-BVVWAS0A.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BVVWAS0A.js.map} +1 -1
- package/dist/files-BaqXvB88.js +1 -0
- package/dist/{files-tiNWe8Ih.js → files-Bey1VAex.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-Bey1VAex.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/files-CNHhSyb3.js +1 -0
- package/dist/files-CW1lChNh.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-D6j-XpaK.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-D6j-XpaK.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-F44NRCN0.js} +143 -20
- package/dist/files-F44NRCN0.js.map +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-Jqkf9SBa.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-Jqkf9SBa.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Dsc4Wi-F.js} +65 -30
- package/dist/images-Dsc4Wi-F.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/init-C0oUG3jh.js +1 -0
- package/dist/{init-Bi1Q3lRi.js → init-QB5ecPDA.js} +33 -14
- package/dist/init-QB5ecPDA.js.map +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
- package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
- package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
- package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
- package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
- package/dist/layout-DfQa4mJs.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
- package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
- package/dist/live-DwjWJTdn.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
- package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
- package/dist/{live-DL0se4Zd.js → live-y1p3oNXH.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-y1p3oNXH.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
- package/dist/model-3pZZzloU.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-DEjwetkJ.js} +1 -1
- package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-jRKSQBg6.js} +5 -5
- package/dist/{model-cgt5uvu8.js.map → model-jRKSQBg6.js.map} +1 -1
- package/dist/model-nfewGTzB.js +1 -0
- package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
- package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
- package/dist/options-DpsJ_5CU.js +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-Bx-qp8Li.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-Bx-qp8Li.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-ChO0A-xn.js} +63 -12
- package/dist/plan-ChO0A-xn.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-BioMWncI.js} +636 -9
- package/dist/plugins-BioMWncI.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-BUjk_6wT.js} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-CJSGzMzc.js} +8 -9
- package/dist/preflight-CJSGzMzc.js.map +1 -0
- package/dist/preflight-DD3fI4Hz.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-qFCXmxha.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-qFCXmxha.js.map} +1 -1
- package/dist/{prepare-DQEDXoDp.js → prepare-D7oSSFNm.js} +9 -9
- package/dist/{prepare-DQEDXoDp.js.map → prepare-D7oSSFNm.js.map} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-xHU2BpN-.js} +2 -2
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-B-flBAzR.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-B-flBAzR.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-B6StVyfG.js} +215 -48
- package/dist/providers-B6StVyfG.js.map +1 -0
- package/dist/providers-Dljf6al6.js +1 -0
- package/dist/query-BAc2tQZj.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-CkLmJv7R.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-CkLmJv7R.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-CJIh2FSX.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-CJIh2FSX.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
- package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
- package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
- package/dist/redact-DL2BTykw.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-B3wjKscW.js} +46 -11
- package/dist/remarks-B3wjKscW.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-C88amFRN.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-C88amFRN.js.map} +1 -1
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/resolve-DW92wpIZ.js +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-G576PiFJ.js} +14 -11
- package/dist/resolve-G576PiFJ.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-zVHIW6xw.js} +9 -22
- package/dist/results-zVHIW6xw.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-D_PcUXfG.js} +94 -16
- package/dist/resume-D_PcUXfG.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-BvTzCxdc.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-BvTzCxdc.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
- package/dist/routes-B0F2R-K7.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-B3r_pAq6.js} +4 -4
- package/dist/{runtime-Xm9oeC7o.js.map → runtime-B3r_pAq6.js.map} +1 -1
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-Duh3z1TV.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-Duh3z1TV.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CGEsvVRG.js +1 -0
- package/dist/{server-C00zd6zl.js → server-DUWMRKik.js} +8 -8
- package/dist/{server-C00zd6zl.js.map → server-DUWMRKik.js.map} +1 -1
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-CHuN4Nfg.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-CHuN4Nfg.js.map} +1 -1
- package/dist/source-Wz1Nrfue.js +1 -0
- package/dist/src--yl1OypK.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
- package/dist/src-B-HuDqpH.js.map +1 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-y0nOeRRE.js → src-BHJ4MWvY.js} +494 -112
- package/dist/src-BHJ4MWvY.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
- package/dist/src-BI0v8qlZ.js.map +1 -0
- package/dist/src-BL-_0QoE.js +1 -0
- package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
- package/dist/src-BqNzHHWn.js.map +1 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C-mOSczH.js +2 -0
- package/dist/src-CZaYu6ER.js +3 -0
- package/dist/src-CeBhwD8k.js +2 -0
- package/dist/src-CkEiiyON.js +2 -0
- package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
- package/dist/src-CmA5Yev6.js.map +1 -0
- package/dist/src-CwH_nEaw.js +2 -0
- package/dist/src-DA95rM6v.js +4 -0
- package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
- package/dist/src-DL-EE4DU.js.map +1 -0
- package/dist/src-De0KY5S_.js +2 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/state-B3jex5KU.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUBRDyMk.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUBRDyMk.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
- package/dist/suite-Bhs9-Z7O.js +2 -0
- package/dist/{summary-DDy9DfyJ.js → summary-CSp4cfnr.js} +25 -7
- package/dist/{summary-DDy9DfyJ.js.map → summary-CSp4cfnr.js.map} +1 -1
- package/dist/sync-CxAY6Dki.js +559 -0
- package/dist/sync-CxAY6Dki.js.map +1 -0
- package/dist/{tart-dWoJvWHc.js → tart-CcrrtSrp.js} +179 -61
- package/dist/tart-CcrrtSrp.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-F-6seRC9.js +192 -0
- package/dist/tart-bake-F-6seRC9.js.map +1 -0
- package/dist/tart-bake-RK7RaT9E.js +1 -0
- package/dist/tart-cli-B86Jqo1t.js +89 -0
- package/dist/tart-cli-B86Jqo1t.js.map +1 -0
- package/dist/tart-cli-ClvRniLQ.js +1 -0
- package/dist/tart-hosts-Brii7zHM.js +1 -0
- package/dist/tart-hosts-DPyZsg-1.js +313 -0
- package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
- package/dist/tart-images-CWSf03w8.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-CVDdJy0H.js +433 -0
- package/dist/tool-class-CVDdJy0H.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
- package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +27 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-DRxXCyIH.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-DRxXCyIH.js.map} +1 -1
- package/dist/views-i67V1ly3.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-C3Q4U63F.js} +12 -9
- package/dist/worker-entry-C3Q4U63F.js.map +1 -0
- package/dist/workspace-DhUOW8if.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-P72SXLXr.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-P72SXLXr.js.map} +1 -1
- package/package.json +12 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# Evals: the attempt API, judges and matchers
|
|
2
|
+
|
|
3
|
+
## Contents
|
|
4
|
+
|
|
5
|
+
- Cases: `cases`, `caseKey`, `where`, and `gate: 'observe'`
|
|
6
|
+
- The attempt: `c.send`, `c.exec`, `c.serve`, files
|
|
7
|
+
- The judge: inputs, input size, errors
|
|
8
|
+
- Trustworthy evals
|
|
9
|
+
- Matchers
|
|
10
|
+
|
|
11
|
+
## Cases: `cases`, `caseKey`, `where`, and `gate: 'observe'`
|
|
12
|
+
|
|
13
|
+
- `evaluate(name, [options,] fn)` and `group(name, [options,] fn)`; also `.skip`, `.only`
|
|
14
|
+
(an error in CI), `.todo`. The eval id is `file › group › name`; renaming an eval restarts its history.
|
|
15
|
+
- `evaluate(name, { cases: [...] }, async (c, item) => …)` runs one eval over a list: one pass
|
|
16
|
+
rate over the cases and, with `compare`, one lift paired by case, which `minLift` gates (not
|
|
17
|
+
each case's). Prefer it to separate evals when the cases are one population: six cases at
|
|
18
|
+
`runs: 1` can reach a lift verdict where each case alone is within noise. Fewer than six
|
|
19
|
+
discordant cases shows only a direction (`insufficient` in `reeval query case_lift`), so
|
|
20
|
+
aim for six cases or more; the plan hints when there are fewer. No `runs: { bestOf }` beside it.
|
|
21
|
+
`gate: { passRate }` gates each cell's rate over the cases (the mean of each case's pass@1), so
|
|
22
|
+
one failing case of three passes `passRate: 0.6`; under `'all'` every case must pass, so a
|
|
23
|
+
failing case fails the eval. It is one eval: one row in `--dry-run` and the run output (a
|
|
24
|
+
case table under it when it fails, or with `-v`), one entry in the record with its cases
|
|
25
|
+
inside, counted once in the tally. The failing case still records `fail`.
|
|
26
|
+
- For evals that pass or fail on their own, don't use `cases`: write a `for` loop that calls
|
|
27
|
+
`evaluate` once per item, with the item in the name.
|
|
28
|
+
- Each case has a key, which names its row: a string or number is its own key, an object its
|
|
29
|
+
`id`, else its `name`; otherwise pass `caseKey: (item) => item.slug`. Keys are unique, at most
|
|
30
|
+
80 characters, with no `#` or `›`. `cases`, `caseKey` and `where` go on `evaluate`, never a
|
|
31
|
+
group or the config.
|
|
32
|
+
- Porting `evaluate.each(items)('name: %s', opts, fn)`: write `evaluate('name', { cases: items,
|
|
33
|
+
...opts }, fn)` with the old option names made plural (`harnesses`, `environments`,
|
|
34
|
+
`compare.control`). The old runs then read as this eval, keyed by what `%s` or `$id` held, so
|
|
35
|
+
`reeval compare` lines them up; a `caseKey` that changes a key starts that case's history again.
|
|
36
|
+
- `where: ({ case, environment, harness }) => boolean` (TypeScript only) picks which case runs
|
|
37
|
+
on which environment, at plan time, from each environment's `traits` (`defineEnvironment('next',
|
|
38
|
+
{ traits: { stack: 'next' } })`; never shown to the agent). Reading a trait no environment
|
|
39
|
+
defines is an error; with `compare` it sees the control, and its answer holds for the variants.
|
|
40
|
+
`--dry-run` notes `excluded on vite-app: q2`, the run's case table shows `–` there, and each case's record keeps its own in
|
|
41
|
+
`excluded`. A cell left with no cases is an error: drop that environment instead.
|
|
42
|
+
- `where` says which pairs fit, the same every run; to run only some environments today, use
|
|
43
|
+
`reeval run --trait stack=next` (repeat to AND) or a profile's
|
|
44
|
+
`limit: { traits: { stack: ['next', 'vite'] } }`. Both narrow after `where`, by the control's
|
|
45
|
+
traits under `compare`, and `--dry-run` notes `--trait stack=next leaves out vite-app`. Never
|
|
46
|
+
read a flag or a variable in `where` to do this.
|
|
47
|
+
- `evaluate(name, { gate: 'observe' }, fn)` records what agents do (with `c.observe` and
|
|
48
|
+
checks) and fails only on run or infra errors. Its block reads `ran`. `compare.minLift` and
|
|
49
|
+
`runs: { bestOf }` are errors beside it; `compare: { gate: 'observe' }` ungates a compare's cells.
|
|
50
|
+
Any gate but `'all'`, `{ passRate }` or `'observe'` is an error.
|
|
51
|
+
A comment on an eval describes the current check (why a canary fails, which path it reads),
|
|
52
|
+
not what used to be true or which product a prompt was ported from.
|
|
53
|
+
|
|
54
|
+
## The attempt: `c.send`, `c.exec`, `c.serve`, files
|
|
55
|
+
|
|
56
|
+
- `await c.send(prompt)` runs one agent turn and returns a `Turn` (`text`, `status`, `diff`,
|
|
57
|
+
`transcript`). Call it again for multi-turn;
|
|
58
|
+
`c.judge(q, { on: firstTurn, task: true, output: true })` grades an earlier one.
|
|
59
|
+
- `await c.exec(cmd)` returns `{ exitCode, stdout, stderr }`; `c.exec(cmd, { background: true })`
|
|
60
|
+
starts it without waiting and returns `{ kill() }`. `{ env: { NAME: 'value' } }` sets variables
|
|
61
|
+
for that command only, over the environment's (in the background too); a value named like a
|
|
62
|
+
secret (`*_TOKEN`, `*_KEY`, …) is masked in everything reeval writes, and a brokered secret's
|
|
63
|
+
host value goes in as its placeholder. `c.exec(cmd, { retries: 2, interval: 2000 })`
|
|
64
|
+
runs a command that exits non-zero again, up to 2 more times 2 s apart, and resolves with the
|
|
65
|
+
last try: for a check that can fail on a transient, such as a test calling a model. Any non-zero
|
|
66
|
+
exit retries (reeval doesn't read the output), and a last failed try is a failed check. `await c.serve(cmd, port?)` is that plus a
|
|
67
|
+
port (`PORT` / `$PORT` on a bash string), and returns once `cmd` listens, on `PORT` or the one
|
|
68
|
+
port it picked itself. A `port` something already listens on (the agent's dev server) throws:
|
|
69
|
+
pass none and serve on `PORT`. `c.capture()` screenshots it: see
|
|
70
|
+
[capture.md](capture.md).
|
|
71
|
+
- `await c.readFile(path)` the text or `null` (`~/` for home). `await c.writeFile(path, text)`
|
|
72
|
+
places a file after a turn (a hidden test the agent never saw) instead of seeding it in
|
|
73
|
+
`files`; later turns' `diff` leaves it out but shows any change the agent makes to it, even
|
|
74
|
+
under `.gitignore`. Don't place grading files with a `c.exec` heredoc: the next turn's diff
|
|
75
|
+
(and the judge) would count them as the agent's. The sandbox and session start on the first
|
|
76
|
+
call that needs them (`c.send`, `c.exec`, `c.readFile`, `c.writeFile`, `c.serve`,
|
|
77
|
+
`c.capture`), so a check before any turn sees what the environment set up.
|
|
78
|
+
`c.attempt.cell.role` is
|
|
79
|
+
`'control'` or `'variant'`.
|
|
80
|
+
- `c.observe(name, value)` records a JSON value about the attempt (which ORM it installed, how many
|
|
81
|
+
files it touched) that never affects pass or fail; the run output tallies it per cell and
|
|
82
|
+
`reeval query attempt_values` lists it. `c.judge()` and questions see it only with
|
|
83
|
+
`observations: true` (or a list of names). Use it for what you want to count, `expect` for what
|
|
84
|
+
must hold.
|
|
85
|
+
|
|
86
|
+
## The judge: inputs, input size, errors
|
|
87
|
+
|
|
88
|
+
- `await c.judge(question, options)`: the judge sees the question and only the inputs the call
|
|
89
|
+
turns on. Every input is off until the call sets it, and a call needs at least one (`true`, a
|
|
90
|
+
mode or a non-empty list); there are no defaults, and the config, profile, group and eval
|
|
91
|
+
`judge` take only `model`, `threshold`, `providerOptions`, `maxConcurrent` and `gateway`.
|
|
92
|
+
With none, `c.judge()` throws before any model call, and TypeScript refuses it.
|
|
93
|
+
|
|
94
|
+
| Input | Shows |
|
|
95
|
+
|---|---|
|
|
96
|
+
| `task: true` | the graded turn's prompt |
|
|
97
|
+
| `output: true` | the graded turn's final message, or `on`'s value |
|
|
98
|
+
| `files: true` | `status path` lines for the changed files, under `diff`'s patterns |
|
|
99
|
+
| `diff: true` / `['src/**', '!**/*.test.ts']` | the diff since the eval started (gitignore-style, last match wins) |
|
|
100
|
+
| `contents: ['README.md', 'src/**']` | files whole, changed or not; they leave the diff |
|
|
101
|
+
| `transcript: true` / `'tools'` / `'full'` | the turn's transcript: clipped, tool calls only, everything |
|
|
102
|
+
| `checks: true` / `['npm test']` | `c.exec` results, or those commands' |
|
|
103
|
+
| `artifacts: true` / `[shot, 'home']` | captures, sent as file parts |
|
|
104
|
+
|
|
105
|
+
Pick the inputs the question needs, usually one or two: a question about code reads `diff` or
|
|
106
|
+
`contents`, one about the agent's answer `output`, one about how it worked `transcript`, one
|
|
107
|
+
about a screenshot `artifacts`. Spell out `{ task: true, output: true, diff: true }` only when
|
|
108
|
+
the question needs all three. The diff leaves out dependencies, build output (`dist/`,
|
|
109
|
+
`.next/`, top-level `build/` and `out/`), lockfiles, `*.map`, `*.tsbuildinfo` and minified
|
|
110
|
+
files.
|
|
111
|
+
- `on` and `turns` select what `task`, `output`, `transcript` and `diff` read and show nothing alone:
|
|
112
|
+
`on: turn` grades an earlier turn, `on: value` grades any value as the output (with
|
|
113
|
+
`output: true`), and `turns: 'all'` adds every turn before the graded one. `judge-input.json`
|
|
114
|
+
records each call's `stateOptions` (every input, `false` when off) and the `state` it sent.
|
|
115
|
+
- Ask about facts in a captured picture, not whether it "looks great". Jev
|
|
116
|
+
(`typesafe-ai/jev-1.14.0`) takes PNG captures; a provider that cannot take files fails that
|
|
117
|
+
call. Prefer deterministic checks for what can be checked; use the judge for quality, with
|
|
118
|
+
plain yes/no questions that don't name one specific way of doing it.
|
|
119
|
+
- Give the environment a `.gitignore` in `files` (the project's isn't copied in): an agent that
|
|
120
|
+
runs a build otherwise fills the judge's diff with its output. reeval sends every input the
|
|
121
|
+
call turns on, uncut, and knows no model's limit: a provider error refusing it keeps the
|
|
122
|
+
provider's wording and adds one line with the input's estimated size, biggest files and the
|
|
123
|
+
inputs that were on, largest first; narrow one, e.g. `{ diff: ['src/'] }`.
|
|
124
|
+
- A failed judge call doesn't throw: `c.judge()` returns `{ verdict: 'error', score: null, error }`,
|
|
125
|
+
the later judges and checks still run, and the attempt fails as a judge error at the end, or
|
|
126
|
+
as an infra error (which `--retry-errors` reruns) when the provider never answered and nothing
|
|
127
|
+
else failed.
|
|
128
|
+
`expect(await c.judge(q, { diff: true }), name).toPass()` records each judge's result and
|
|
129
|
+
carries on.
|
|
130
|
+
|
|
131
|
+
## Trustworthy evals
|
|
132
|
+
|
|
133
|
+
Before a real run, prove each check can pass and fail with the fake harness in the eval's real
|
|
134
|
+
sandbox: an oracle `fakeHarness` that does the task right (must pass) and a canary that cheats
|
|
135
|
+
or gets it wrong (must fail). A canary that passes means a check isn't checking. Give each check
|
|
136
|
+
you rely on a canary that only it catches.
|
|
137
|
+
|
|
138
|
+
```ts
|
|
139
|
+
// reeval.config.ts: the real harness, plus two profiles that swap only the harness and judge
|
|
140
|
+
export default defineConfig({
|
|
141
|
+
harnesses: cursor({ model: 'composer-2.5' }),
|
|
142
|
+
profiles: {
|
|
143
|
+
oracle: {
|
|
144
|
+
defaults: {
|
|
145
|
+
harnesses: fakeHarness({ files: { 'greet.js': 'export const greet = …' } }),
|
|
146
|
+
judge: { model: fakeJudge({ score: 0.95 }) },
|
|
147
|
+
},
|
|
148
|
+
},
|
|
149
|
+
canary: {
|
|
150
|
+
defaults: {
|
|
151
|
+
harnesses: fakeHarness({ files: { 'greet.test.js': "test('greets', () => {})\n" } }),
|
|
152
|
+
judge: { model: fakeJudge({ score: 0.1 }) },
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
})
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
npx reeval image build --profile oracle # once: the eval's images
|
|
161
|
+
npx reeval run --profile oracle # every check passes, exit 0
|
|
162
|
+
npx reeval run --profile canary # the checks you meant fail, exit 1
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
- Set `defaults.harnesses`, never `defaults.sandbox`: the environment's `docker()` (image,
|
|
166
|
+
`install`, egress, brokered secrets) stays, so the checks run where they will for real. A
|
|
167
|
+
profile's `sandbox` only fills in for an environment that omits one, and the fake harness
|
|
168
|
+
without `layout` uses Cursor's image; `layout: 'claude-code'` (or another agent id) uses that
|
|
169
|
+
agent's image and reads rules and skills where it does.
|
|
170
|
+
- `script` steps (`{ exec }`, `{ write }`, `{ mcp }`) run inside the container as the agent, so an
|
|
171
|
+
oracle can run the real fix (`{ exec: 'node scripts/solve.mjs' }`) or call a brokered API.
|
|
172
|
+
- `fakeJudge` checks the wiring (a low score fails the check); keep the real `judge.model` on the
|
|
173
|
+
oracle, with its key, to check that a question passes good work.
|
|
174
|
+
- `fakeSandbox()` isn't the oracle's sandbox: it runs on the host, brokers nothing and enforces no
|
|
175
|
+
egress. It's for trying reeval and testing configs without Docker.
|
|
176
|
+
- An eval or group that sets its own `harnesses` keeps it (the plan notes `harnesses set by the
|
|
177
|
+
eval`), and an ungated cell's (`compare: { gate: 'observe' }`, a control's) failed checks don't change the
|
|
178
|
+
exit code: its row in the result block shows its count (`plain 0 of 4`).
|
|
179
|
+
|
|
180
|
+
Keep grading out of the
|
|
181
|
+
agent's reach: place hidden tests after the turn with `c.writeFile`, so the agent never sees
|
|
182
|
+
them, seed tests in `files` only when the agent should see them, and guard seeded files
|
|
183
|
+
with `expect(await c.readFile(path)).toBe(original)` or `expect(c).not.toHaveChanged(path)`.
|
|
184
|
+
When the behavior is the
|
|
185
|
+
outcome, judge the program's output, not the diff:
|
|
186
|
+
`c.judge(q, { on: (await c.exec('node app.js')).stdout, output: true })`. Checks run in the agent's
|
|
187
|
+
container as the same user, which suits AX evals but isn't a sealed verifier.
|
|
188
|
+
|
|
189
|
+
## Matchers
|
|
190
|
+
|
|
191
|
+
- `await expect(turn).toHaveFile(path, pattern?)`,
|
|
192
|
+
`await expect(c.exec(cmd)).toSucceed()`, `expect(verdict).toPass()`,
|
|
193
|
+
`expect(turn).toHaveSearched(query?)` / `toHaveFetched(url?)` (sync; they throw
|
|
194
|
+
`HarnessCapabilityError` when `turn.unreported` lists web events; a refused
|
|
195
|
+
search is still a `web-search` with `ok: false`; a call whose result never came has
|
|
196
|
+
`ok: null`, not `false`), plus Vitest's.
|
|
197
|
+
- Transcript matchers (sync, no `await`): `expect(turn | c).toHaveCalledSkill(name)`,
|
|
198
|
+
`toHaveCalledMcp(server, tool, input?)`, `toHaveCalledTool(name | RegExp, input?)`,
|
|
199
|
+
`toHaveRun(command?)`, `toHaveEdited(path?)`, `toHaveSearched(query?)`,
|
|
200
|
+
`toHaveFetched(url?)`. `.not` is Vitest's. `toHaveRun(string | RegExp)` matches the whole
|
|
201
|
+
command line, so `/genkit/` passes on `cat genkit.config.ts`; `toHaveRun({ program, args? })`
|
|
202
|
+
matches the program a segment (split on line breaks, `&&`, `||`, `;`, `|`, `&`) ran, by its
|
|
203
|
+
name without a directory, after `VAR=x`, `env`, `timeout`, `time`, `exec`, `sudo`, `nohup`,
|
|
204
|
+
`nice`, `command`, `if`/`then`/`do`, `bash -c`, `npx`, `bunx`, `npm exec`, `pnpm exec`/`dlx`
|
|
205
|
+
and `yarn dlx`; `args` (joined by spaces) is a substring or RegExp, e.g. `/^add\b/`.
|
|
206
|
+
`pnpm <bin>` and `npm run` aren't unwrapped, and a run counts even if its call failed.
|
|
207
|
+
`shellCommands(c | turn | c.turns)` returns those programs to count or filter. `toHaveCalledSkill`
|
|
208
|
+
is opened or loaded (a `skill` / `Skill` / `load_skill` tool, or a Read/`sed` of
|
|
209
|
+
`skills/<name>/`, on every harness), not "followed". MCP is `toHaveCalledMcp(server, tool)`
|
|
210
|
+
(the names in `mcp.<server>`); `toHaveCalledTool('mcp__<server>__<tool>')` is
|
|
211
|
+
the raw transcript name. After adapters and the normalizer, that string is
|
|
212
|
+
the same on every harness (stdio; Goose, OpenCode, pi and Codex refuse
|
|
213
|
+
`mcp.sse`). Use `toHaveEdited` / `toHaveSearched`, not
|
|
214
|
+
`toHaveCalledTool('edit')` or `'webSearch'`.
|
|
215
|
+
- Budgets: `expect(turn | c).toStayWithin({ tokens: 20_000, toolCalls: 30, duration: '5m' })`
|
|
216
|
+
(sync) caps the turns' summed tokens, tool calls (a number, or `{ bash: 5 }` per tool) and time.
|
|
217
|
+
Tokens include subagents. With `tokens`, a harness that reported no usage throws
|
|
218
|
+
`HarnessCapabilityError` (an error, not a pass). `.not`, an unknown key or a bad limit throws a
|
|
219
|
+
`TypeError` that ends the attempt as its error, not a failed check. Python:
|
|
220
|
+
`to_stay_within(tokens=, tool_calls=, duration=)`.
|
|
221
|
+
- Guard files the agent must leave alone with `expect(c).not.toHaveChanged(['test/**',
|
|
222
|
+
'package.json'])` (sync): it reads the workspace diff, so a shell or subprocess write counts
|
|
223
|
+
and a read doesn't. Strings are globs (a directory matches everything under it), RegExps work,
|
|
224
|
+
and a turn means up to that turn. `toHaveEdited` counts the files that changed while the agent
|
|
225
|
+
ran the turn (a turn only its own), so a shell write counts, but its string matches any path
|
|
226
|
+
containing it; guard with `toHaveChanged`'s globs. `toHaveEdited()` counts files that appear
|
|
227
|
+
in the workspace diff. Gitignored files don't appear there, so use `toHaveFile()` to check them.
|
|
228
|
+
- Checks don't stop the eval: a failed `expect` is recorded and the body carries on, so every
|
|
229
|
+
later check and judge still runs; the attempt fails at its end. Use `expect.require` for gates:
|
|
230
|
+
its failure ends the body there (`await expect.require(c.exec('npm run build'), 'builds')
|
|
231
|
+
.toSucceed()`, `expect.require.poll(…)` for one that waits). One the eval catches still fails
|
|
232
|
+
the attempt, and its error says the eval caught it. Only matcher failures carry on: an error
|
|
233
|
+
the body throws, a dead sandbox or a timeout ends the attempt. Wait or retry with `expect.poll`
|
|
234
|
+
(`await expect.poll(() => c.exec(cmd), { timeout: 10_000 }).toSucceed()`), which records one
|
|
235
|
+
check with its final outcome, not with `expect` in a `try` loop.
|
|
236
|
+
- **Label each requirement** with `expect`'s second argument:
|
|
237
|
+
`await expect(c, 'exports greet').toHaveFile('greet.js', /export/)`. Every matcher call
|
|
238
|
+
is recorded (`evals[].attempts[].assertions`; unlabeled ones get `<matcher> <first arg>`).
|
|
239
|
+
Changing a label changes the eval's fingerprint. Label judges too
|
|
240
|
+
(`expect(await c.judge(q, { diff: true }), 'Social OAuth').toPass()`): the label replaces the clipped
|
|
241
|
+
question in judge rates, `reeval show`, `summary.md` and the digest.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Common mistakes (and the fix)
|
|
2
|
+
|
|
3
|
+
Each row is a mistake, what reeval says and the fix.
|
|
4
|
+
|
|
5
|
+
| Mistake | reeval says | Fix |
|
|
6
|
+
|---|---|---|
|
|
7
|
+
| No `reeval.config.*` | `No reeval.config.ts … here or above. Run reeval init.` | `reeval init`, or pass `--config` |
|
|
8
|
+
| `compare` in `reeval.config.ts` | `compare can't be set in reeval.config.ts` | Put it on the group or eval |
|
|
9
|
+
| `compare: [b]` | `compare … must be an object, not a list` | `compare: { control: a, variants: [b] }` |
|
|
10
|
+
| `compare: { variants: [b] }` | `compare … has no control` | `compare: { control: a, variants: [b] }` |
|
|
11
|
+
| `compare: { baseline: a, … }` | `Unknown compare option "baseline" … Did you mean "control"?` | `compare: { control: a, … }` |
|
|
12
|
+
| `compare: { strategy: 'peer' }` | `Unknown compare option "strategy"` | Drop it. Variants already run on their own |
|
|
13
|
+
| `environments` beside `compare` | `… sets both environments and compare` | Move the environment into `compare.control` |
|
|
14
|
+
| `evaluate.each(cases)(name, fn)` | `evaluate.each is not a function` | `evaluate(name, { cases }, async (c, item) => …)` |
|
|
15
|
+
| `cases` for items that should each pass or fail alone | One failing case fails the whole eval | A loop: `for (const item of items) evaluate(nameOf(item), fn)` |
|
|
16
|
+
| `where` without `cases`, or on a group | `… sets where without cases` / `Unknown option "where" in group(…)` | Put `where` beside `cases` on the eval |
|
|
17
|
+
| `survey(…)`, or `gate: 'none'` | `survey is not a function` / `gate in … is 'all' …, { passRate } … or 'observe'` | `evaluate(name, { gate: 'observe' }, fn)` |
|
|
18
|
+
| `compare: false` on an eval or group | `compare: false can't be set` | Use a profile's `limit.compare: false` |
|
|
19
|
+
| A question and no language model | `questions "…" (…) needs a language model` (exit 2) | Set `analysis.model`, give the question its own `model`, or set `question: { model }` |
|
|
20
|
+
| `import { baseline }` | `reeval has no baseline` | Name the control in `compare: { control }`; it isn't an export |
|
|
21
|
+
| `repeat: 3` / `times` | `Unknown option "repeat" … Did you mean "runs"?` | `runs: 3` |
|
|
22
|
+
| `runs: { bestOf: 4 }` | `runs.bestOf … must be an odd whole number of 3 or more` | `runs: { bestOf: 3 }` or `5` |
|
|
23
|
+
| `runs: { bestOf: 3 }` with `compare` | `… compares environments, so it can't take runs: { bestOf: 3 }` | a fixed count, `runs: 4` |
|
|
24
|
+
| Judge model `'gemini-3.8-flash'` | needs a provider prefix | `'google/gemini-3.8-flash'` |
|
|
25
|
+
| `opencode({ model: 'gemini-3.8-flash' })` | `an OpenCode model id names its provider` (exit 2) | `'google/gemini-3.8-flash'` |
|
|
26
|
+
| No `judge.model` anywhere | `c.judge() has no model` | Set one, or `fakeJudge()` |
|
|
27
|
+
| The judge's provider refuses a big input | The provider's message, then `The judge's input was ~…k tokens (estimated): diff …, of which …` | Add a `.gitignore` to the environment, or narrow the call: `{ diff: ['src/'] }` |
|
|
28
|
+
| `c.judge(q)` with no inputs, expecting the judge to see the task, answer and diff by default | `c.judge() shows the judge nothing: turn on at least one input, …` (TypeScript refuses it too) | There are no default inputs: turn on what the question needs, `{ diff: true }`, `{ contents: ['README.md'] }`, `{ output: true }`, `{ transcript: 'tools' }` |
|
|
29
|
+
| `c.judge(q, { on: value })` alone | `c.judge() shows the judge nothing: …`, or with another input `… on gives a value to grade as the output, but output is off. Pass output: true.` | `{ on: value, output: true }`; `on` and `turns` only select |
|
|
30
|
+
| An input (`diff`, `contents`, …) on a config, profile, group or eval `judge` | `Unknown judge option "diff" in …. diff is per call: c.judge(question, { diff }).` | Put it on each `c.judge()` call |
|
|
31
|
+
| A question as a string, or with no input | `questions … takes { question, model?, across?, choices?, turns? } with at least one of task, output, …` / `… shows the model nothing: …` (exit 2) | `{ question: 'Which is cleaner?', across: 'environment', diff: true }` |
|
|
32
|
+
| `cursor()` with `fakeSandbox()` | real harnesses need `docker()` | Use `fakeHarness()` or `docker()` |
|
|
33
|
+
| Forgetting `await` on `toHaveFile`, or on `toSucceed` given `c.exec()`'s promise | ``expect(…).toHaveFile() at <file>:<line> wasn't awaited. Its check is async: write `await …` `` (the eval fails) | `await expect(...)`; `expect(await c.exec(cmd)).toSucceed()` needs no `await` |
|
|
34
|
+
| `toPass()` on an exec result, `toSucceed()` on a verdict | a type error, ``Type 'WrongSubject<"toPass() takes the verdict from await c.judge()">' has no call signatures`` | `toSucceed()` for `c.exec()`, `toPass()` for `await c.judge()` |
|
|
35
|
+
| Secret as a string literal | refused | `secret.fromEnv('NAME')` in a callback |
|
|
36
|
+
| `workspace: 'https://…'` | `must come from the git helper` | `({ git }) => ({ workspace: git('https://…', { ref }) })` |
|
|
37
|
+
| `async` environment callback | returned a Promise | Return synchronously; fetch in a `setup` step |
|
|
38
|
+
| Test file named `*.test.ts` | `No eval files in … match include: …` (exit 2) | `*.eval.ts` under `evals/` |
|
|
39
|
+
| `.env.local` outside the config's directory | `Missing credentials: …` | Put it next to `reeval.config.ts` |
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Vocabulary
|
|
2
|
+
|
|
3
|
+
reeval's terms, and what other eval frameworks (Inspect, Harbor, SWE-bench) call them. Use
|
|
4
|
+
reeval's word in code and messages.
|
|
5
|
+
|
|
6
|
+
| reeval | Elsewhere called |
|
|
7
|
+
|---|---|
|
|
8
|
+
| eval (`evaluate(name, fn)`) | task, test case |
|
|
9
|
+
| case (an item of `cases`) | sample, example, instance |
|
|
10
|
+
| attempt (one run of a cell; `runs: n`) | trial (Anthropic, Harbor), epoch (Inspect), repetition |
|
|
11
|
+
| harness (`cursor()`, …) | agent harness, scaffold, solver (reeval itself is the *evaluation* harness) |
|
|
12
|
+
| cell (harness × environment × sandbox) | a factorial-design cell |
|
|
13
|
+
| control, variant | baseline, treatment |
|
|
14
|
+
| lift (variant pass@1 − control pass@1, 95% interval) | treatment effect, delta, uplift |
|
|
15
|
+
| gate (how a cell's attempts decide its outcome) | `'all'` is pass^k, `{ passRate }` a pass@1 threshold; epoch reducer (Inspect), quality gate |
|
|
16
|
+
| pass@1 (passes over passes and fails; `pass_at_1` in `reeval query`) | success rate, resolve rate (SWE-bench) |
|
|
17
|
+
| checks, judge (`expect`, `c.exec`, `c.judge`) | grader, verifier, scorer, evaluator |
|
|
18
|
+
| transcript (`transcript.jsonl`) | trajectory, trace |
|
|
19
|
+
| effort (measured tool calls, agent time, tokens, cost) | efficiency |
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: setup
|
|
3
|
+
description: >
|
|
4
|
+
Read this first, before the reeval skill, whenever you're asked to set up, install, add,
|
|
5
|
+
try or get started with reeval (the eval runner for coding agents), in a project or an empty
|
|
6
|
+
directory, including "set up reeval here" and "add evals". It detects what's set up, asks what
|
|
7
|
+
to evaluate, installs reeval, writes one to three small evals that show its value (a skill with
|
|
8
|
+
and without, when the project has skills), runs `reeval doctor` for keys and Docker, and
|
|
9
|
+
explains the results.
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
# Set up reeval
|
|
13
|
+
|
|
14
|
+
Your job is the user's first good run: a few small evals that pass `--dry-run`, the keys and
|
|
15
|
+
Docker sorted, and a short explanation of what the evals show. Small and readable beats complete.
|
|
16
|
+
For the authoring API in depth, the reeval skill (`skills/reeval/SKILL.md` in
|
|
17
|
+
github.com/invertase/reeval) has it; this file has everything a first setup needs.
|
|
18
|
+
|
|
19
|
+
Rules that hold throughout:
|
|
20
|
+
|
|
21
|
+
- **Never overwrite or re-initialize.** Don't run `reeval init` where a `reeval.config.*` or
|
|
22
|
+
`reeval.yaml` exists, never pass `--force`, never edit or delete an existing eval, and add new
|
|
23
|
+
files beside the user's.
|
|
24
|
+
- **Never write, print or ask for a key's value.** Name the variable and the file it goes in.
|
|
25
|
+
- **Ask before you build,** then stop and wait (step 2): always, even when reeval is half set
|
|
26
|
+
up or the right eval looks obvious. Only a message that already says what to evaluate
|
|
27
|
+
("add an eval for the postgres skill") skips the question.
|
|
28
|
+
|
|
29
|
+
## 1. Look before you touch anything
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
ls -a; ls reeval.config.* reeval.yaml evals 2>/dev/null
|
|
33
|
+
grep -n '"reeval"' package.json 2>/dev/null
|
|
34
|
+
ls pnpm-lock.yaml yarn.lock bun.lock package-lock.json 2>/dev/null # the package manager
|
|
35
|
+
find . -name SKILL.md -not -path '*/node_modules/*' | head -20 # the project's skills
|
|
36
|
+
ls AGENTS.md CLAUDE.md GEMINI.md .cursor/rules .github/copilot-instructions.md 2>/dev/null # rules
|
|
37
|
+
ls CONTRIBUTING.md .mcp.json .cursor/mcp.json docs 2>/dev/null
|
|
38
|
+
grep -o '^[A-Z_]*=' .env.local .env 2>/dev/null # key names only
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
| You find | State | Then |
|
|
42
|
+
|---|---|---|
|
|
43
|
+
| A `reeval.config.*` or `reeval.yaml` | Set up | No init. Read the config and the evals and run `npx reeval run --dry-run`; after step 2, add evals beside them |
|
|
44
|
+
| `reeval` in `package.json`, no config | Installed | `npx reeval init --bare --harness <h>` |
|
|
45
|
+
| Neither | New | Install, then `init --bare` (step 3) |
|
|
46
|
+
|
|
47
|
+
The directory is a **project** if it has code, docs or skills (`package.json`, `src/`,
|
|
48
|
+
`README.md`, `skills/`, a `.git` with commits); it's **empty** if there's nothing but dotfiles.
|
|
49
|
+
|
|
50
|
+
## 2. Agree what to evaluate
|
|
51
|
+
|
|
52
|
+
"Set up reeval" says to set reeval up, not what to evaluate, so it doesn't answer this step,
|
|
53
|
+
however clear the choice looks from the repo. Your first reply is the proposal and the
|
|
54
|
+
question, with nothing installed or written yet: end your turn there and wait for the answer.
|
|
55
|
+
|
|
56
|
+
**In a project,** say in two or three sentences what you found and what you propose, then ask
|
|
57
|
+
and stop. For example: "This repo has two skills. I'd set up reeval with one eval that runs Cursor
|
|
58
|
+
on a billing task with and without `acme-billing`, to show what the skill changes. Shall I set
|
|
59
|
+
up evals for this project?" With no skills, propose the rules file or a task in the code
|
|
60
|
+
instead.
|
|
61
|
+
|
|
62
|
+
**In an empty directory, or when the user says no,** ask what they want to evaluate (an SDK, an
|
|
63
|
+
API, a library, docs or a skill they're writing, a kind of task) and stop. When they answer,
|
|
64
|
+
research it before you write a line: read its docs, changelog and README (search the web, fetch
|
|
65
|
+
the pages), note the current version, and find a few things a coding agent gets wrong without
|
|
66
|
+
help: a renamed API, a new default, a required option, a convention. Those are your evals.
|
|
67
|
+
|
|
68
|
+
## 3. Install and initialize
|
|
69
|
+
|
|
70
|
+
Use the project's package manager (its lockfile), and pick the harness the user has a key for:
|
|
71
|
+
|
|
72
|
+
| Harness | `--harness` | Key |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| Cursor (default) | `cursor` | `CURSOR_API_KEY` |
|
|
75
|
+
| Claude Code | `claude-code` | `ANTHROPIC_API_KEY` |
|
|
76
|
+
| Codex | `codex` | `OPENAI_API_KEY` |
|
|
77
|
+
| Antigravity | `antigravity` | `GEMINI_API_KEY` |
|
|
78
|
+
| OpenCode, Goose, pi | `opencode`, `goose`, `pi` | The model's provider's key, e.g. `GOOGLE_GENERATIVE_AI_API_KEY` for `google/…` |
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
npm init -y # only if there's no package.json
|
|
82
|
+
npm i -D reeval # pnpm add -D reeval · yarn add -D reeval
|
|
83
|
+
npx reeval init --bare --harness cursor # pnpm reeval init … · yarn reeval init …
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
`--bare` writes only `reeval.config.ts`, the `.gitignore` entries (`.reeval/`, `.env*`) and the
|
|
87
|
+
`eval` script; it merges into existing files and skips the rest. It exits 2 if reeval is set up
|
|
88
|
+
already: that's your cue to stop initializing, not to `--force`. The config sets
|
|
89
|
+
`analysis.model` for `reeval analyze` (needs `GOOGLE_GENERATIVE_AI_API_KEY`) and `pr` and `nightly`
|
|
90
|
+
profiles. Leave them.
|
|
91
|
+
|
|
92
|
+
## 4. Write one to three evals
|
|
93
|
+
|
|
94
|
+
Pick the low-hanging fruit, in this order:
|
|
95
|
+
|
|
96
|
+
1. **The project has skills:** a compare of one skill, with and without. The control is an empty
|
|
97
|
+
workspace, the variant the same plus the skill; the task is one the skill's own rules decide.
|
|
98
|
+
This is the eval that shows lift. Template A in
|
|
99
|
+
[references/templates.md](references/templates.md).
|
|
100
|
+
2. **No skills, but agent rules** (`AGENTS.md`, `CLAUDE.md`, `GEMINI.md`, `.cursor/rules`): a
|
|
101
|
+
compare of the rules file, with and without, on a small task in the real repo. Template D.
|
|
102
|
+
3. **Any other codebase (an app, a CLI, a library):** a small task in the real repo, checked by
|
|
103
|
+
its typecheck, its tests or a hidden test. Template B.
|
|
104
|
+
4. **An empty directory:** the researched subject, a compare of a short skill or doc you write
|
|
105
|
+
from the research (`skills/<subject>/SKILL.md`) against none, or a single eval. Template C.
|
|
106
|
+
|
|
107
|
+
Work in the user's code, not a copy you make up: seed the repo with `directory('..')` rather
|
|
108
|
+
than stub files, which hand the agent the convention you meant to test. Don't write a skill or
|
|
109
|
+
rules from what you already know to compare against: your checks would restate what you wrote,
|
|
110
|
+
so the compare measures nothing. Write one only from research (step 2, Template C).
|
|
111
|
+
|
|
112
|
+
Each eval:
|
|
113
|
+
|
|
114
|
+
- **Check rules only the skill teaches.** Read the skill (or rules file) and pick two or three
|
|
115
|
+
of its most specific rules that one task touches: a house convention, a helper or file it
|
|
116
|
+
names, a version-specific API, a non-obvious default. Ask of each: would a strong agent do
|
|
117
|
+
this unprompted? If yes (index a foreign key, deny public access, validate input), both arms
|
|
118
|
+
pass it; pick another. In a real repo, the agent copies what the code around the change
|
|
119
|
+
already shows (the error class, the logger), so prefer rules the code can't show: a changelog
|
|
120
|
+
entry, where tests go, a file to register in. Grep the repo for each rule: one that
|
|
121
|
+
`CONTRIBUTING.md` or the docs repeat reaches the control too, so leave those files out of
|
|
122
|
+
the control as well, or pick another rule. Check that the task triggers each rule: a rule for
|
|
123
|
+
user-facing changes needs a user-facing task. One labeled check per rule, so the check
|
|
124
|
+
table shows which ones the skill changed.
|
|
125
|
+
- **One task, one prompt, that doesn't name the answer.** Ask for the outcome in the user's
|
|
126
|
+
words ("an invoices table with its total"), never the rule the check looks for ("store the
|
|
127
|
+
total in cents"), and not in the rule's terms either ("a user-visible fix" hints at the
|
|
128
|
+
changelog). Reread the prompt: if it restates the rule, the control passes too. Name the
|
|
129
|
+
file the task is about (`src/projectId.ts`) so a check can read it, but not the files a rule
|
|
130
|
+
decides (where the test goes): check those with a glob or the one path the rule allows.
|
|
131
|
+
- **Deterministic checks first:** a file's content, a command's exit code, a script that imports
|
|
132
|
+
the agent's code. Label each requirement with `expect(x, 'label')`; a failed one is recorded and the eval carries on. Add a judge
|
|
133
|
+
(`c.judge`) only for what no check can see, with the inputs its question reads
|
|
134
|
+
(`{ diff: ['src/**'] }`, `{ output: true }`), and then add `judge:` to the config.
|
|
135
|
+
- **A name that states the question:** `stores invoice totals in cents with the billing skill`,
|
|
136
|
+
not `billing test`. The results and `reeval analyze` read in its terms.
|
|
137
|
+
- **Small:** leave `runs` unset: a compare runs 4 a cell (8 attempts), the fewest for a lift
|
|
138
|
+
verdict, and anything else once. Seed only the files the task
|
|
139
|
+
needs. If it needs the repo, seed it with `directory('..', { glob: ['!.env*', '!.reeval/',
|
|
140
|
+
'!evals/', '!reeval.config.*'] })`: `directory()` doesn't skip `.env*`, keys must not reach
|
|
141
|
+
the sandbox, and the agent must not read your checks or the oracle's answer in the config.
|
|
142
|
+
- **Eval files go in `evals/*.eval.ts`,** environments in `evals/environments.ts`. Paths in
|
|
143
|
+
`skill.local()`, `file()` and `directory()` are relative to the file that calls them. If
|
|
144
|
+
`evals/environments.ts` exists, append your environments below what's there and leave its
|
|
145
|
+
lines, comments included, as they are.
|
|
146
|
+
|
|
147
|
+
When a check is a regex or a script you wrote, add two fake-agent profiles (Template A shows
|
|
148
|
+
both): a **canary** that does the task wrong must fail every check, so the checks check; an
|
|
149
|
+
**oracle** that does it right, written another way than you'd expect (another helper name,
|
|
150
|
+
another order, schema-qualified `public.notes`), must pass, so the checks don't demand one
|
|
151
|
+
spelling: a regex that wants `on notes (` fails the agent that writes `on public.notes (`. A
|
|
152
|
+
check that runs a build or the project's tests needs no oracle.
|
|
153
|
+
|
|
154
|
+
## 5. Check it plans, then the machine
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
npx reeval run --dry-run # exit 0; the evals, cells and attempts you meant
|
|
158
|
+
npx reeval doctor # Docker, image, keys; exit 1 names each fix
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
- Rerun `--dry-run` after every edit. Exit 2 is a config or eval mistake with a precise
|
|
162
|
+
message; fix it, don't work around it. Check the counts: a compare is 2 cells.
|
|
163
|
+
- `doctor` lists each missing key as `fix: add NAME=... to .env.local`. Tell the user exactly
|
|
164
|
+
those lines, and that `.env.local` goes next to `reeval.config.ts` (it's gitignored). The
|
|
165
|
+
analysis key is a warning only: runs pass or fail the same without it.
|
|
166
|
+
- No Docker, or `doctor` says the image isn't built: the user installs or starts Docker, then
|
|
167
|
+
runs `npx reeval image build` once (minutes, cached).
|
|
168
|
+
- With Docker: `npx reeval run --profile canary <eval-name>` must fail every
|
|
169
|
+
check you wrote, and `--profile oracle` must pass them. Name the eval: a fake agent writes the
|
|
170
|
+
same files for every eval, so it answers only the one it was written for. Needs no keys,
|
|
171
|
+
seconds each. A canary that passes means a check isn't checking; an oracle that fails means a
|
|
172
|
+
check is too narrow. Fix the check.
|
|
173
|
+
|
|
174
|
+
Only start a real `npx reeval run` if the user asked for it or agrees: it runs a real agent on
|
|
175
|
+
their key. One compare (8 attempts) on Cursor is several minutes.
|
|
176
|
+
|
|
177
|
+
## 6. Hand over
|
|
178
|
+
|
|
179
|
+
End with a short message, not a report:
|
|
180
|
+
|
|
181
|
+
1. What you added, one line per file, and what each eval asks.
|
|
182
|
+
2. Each key the config needs by name (`CURSOR_API_KEY`, the analysis key) and whether `doctor`
|
|
183
|
+
found it; then what's missing, as commands: the `.env.local` lines, `npx reeval image
|
|
184
|
+
build`, Docker.
|
|
185
|
+
3. The command to run: `npx reeval run` (or `npx reeval run <eval-name>` for one).
|
|
186
|
+
4. How to read the output, in a few lines. `npx reeval run` prints a line per eval and a sentence
|
|
187
|
+
for each way one failed; `npx reeval show` prints its arms and checks. A real Cursor run of
|
|
188
|
+
Template A, from `npx reeval show`:
|
|
189
|
+
|
|
190
|
+
```text
|
|
191
|
+
pass stores invoice totals in cents with the billing skill
|
|
192
|
+
arm pass
|
|
193
|
+
plain 0 of 2
|
|
194
|
+
with-billing-skill 2 of 2 helped
|
|
195
|
+
check plain with-billing-skill
|
|
196
|
+
total in integer cents 0 of 2 2 of 2
|
|
197
|
+
a currency column 0 of 2 2 of 2
|
|
198
|
+
+1 more passed
|
|
199
|
+
|
|
200
|
+
1 pass
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
- Each arm's `N of M` is passing attempts. The variant's word is its effect against the control:
|
|
204
|
+
`helped`, `made it worse`, or `same`; on a small sample (below 4 runs a side, or a `same` at
|
|
205
|
+
2 of 4 against 2 of 4) a `too-few-tries` remark says it doesn't hold yet and names the `runs`. When
|
|
206
|
+
both arms pass every run, or both fail every run, at 4 runs a side or more, a `cant-tell-apart`
|
|
207
|
+
remark says more runs won't help: make the task harder, or check the eval. When both pass but
|
|
208
|
+
another measure differs, the line under the variant names it (`Same pass rate · 5× faster.`),
|
|
209
|
+
and a measure that only leans gets a `too-few-tries` remark with the runs that would confirm it.
|
|
210
|
+
The check table shows which check told the arms apart.
|
|
211
|
+
- `Worth knowing:` under an eval in `reeval run`'s list names what about it needs work, with the line to fix. Remarks never
|
|
212
|
+
change a verdict.
|
|
213
|
+
- Both arms passing is a result, not a broken eval: the agent already does this without the
|
|
214
|
+
skill. Say so, and suggest a harder task or a more specific rule from the skill next.
|
|
215
|
+
- A check that fails every attempt on both arms is more often a broken check than a hard task.
|
|
216
|
+
Open one attempt's `diff.patch` before you say the agent can't do it.
|
|
217
|
+
- Exit 0 means every gated cell passed; only the variant is gated in a compare.
|
|
218
|
+
- Details: `.reeval/runs/<run>/summary.md`, each attempt's `transcript.jsonl` and `diff.patch`;
|
|
219
|
+
`npx reeval studio` browses them; `npx reeval query effort` compares tools, time and tokens.
|