reeval 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +803 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-BdzhdRYo.js} +8 -6
- package/dist/adapter-BdzhdRYo.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-Dx-bDmlV.js} +179 -31
- package/dist/agent-Dx-bDmlV.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-l0-k7vw3.js} +361 -101
- package/dist/agents-l0-k7vw3.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
- package/dist/app-files-B-CQYB0c.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CYMpN8LC.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-CsZIcJo8.js} +171 -18
- package/dist/assemble-CsZIcJo8.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-BwSmNYkX.js} +17 -8
- package/dist/attempt-host-BwSmNYkX.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
- package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
- package/dist/{boat-DSNgZt-t.js → boat-su1Y7LPd.js} +17 -14
- package/dist/boat-su1Y7LPd.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/{capture-Ccf0Hmyx.js → capture-BqIAeFyC.js} +55 -29
- package/dist/{capture-Ccf0Hmyx.js.map → capture-BqIAeFyC.js.map} +1 -1
- package/dist/capture-D0Ln5aCP.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-s_3opn_I.js} +219 -109
- package/dist/cli-s_3opn_I.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
- package/dist/cli-spec-DxzIjmq_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-C5fIzHQa.js +803 -0
- package/dist/client-C5fIzHQa.js.map +1 -0
- package/dist/{command-DSL7I-SI.js → command-B5OAIjp7.js} +25 -21
- package/dist/command-B5OAIjp7.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BJ7KqczP.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BJ7KqczP.js.map} +1 -1
- package/dist/{command-D8oMB2fa.js → command-BXLZsEwQ.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-BXLZsEwQ.js.map} +1 -1
- package/dist/{command-Ff0T-1Hi.js → command-DA_SHQUQ.js} +16 -16
- package/dist/{command-Ff0T-1Hi.js.map → command-DA_SHQUQ.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-BKBamOcX.js} +174 -40
- package/dist/commands-BKBamOcX.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-BPvAOEmb.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0URar0t.js} +255 -89
- package/dist/config-C0URar0t.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-Sj58aJEu.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-Sj58aJEu.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-FWgSo-T1.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
- package/dist/db-Bc4L-Wcr.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-BjAWxVAG.js} +142 -248
- package/dist/db-BjAWxVAG.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-CWlhZ6U1.js} +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-Ck2GYIxX.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DG_uFgn2.js → docker-CDdK6Ity.js} +41 -16
- package/dist/docker-CDdK6Ity.js.map +1 -0
- package/dist/{docker-DFrjoNs-.js → docker-D74l67k8.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-D74l67k8.js.map} +1 -1
- package/dist/{docker--mukta3L.js → docker-DMug3l-a.js} +1 -1
- package/dist/{doctor-DWK4ttex.js → doctor-eFxyn6at.js} +85 -46
- package/dist/doctor-eFxyn6at.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-CJIKz-Ud.js} +49 -11
- package/dist/environment-CJIKz-Ud.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CXIRJ91v.js} +24 -12
- package/dist/eval-source-CXIRJ91v.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-DA4QsrHO.js} +792 -281
- package/dist/evaluate-DA4QsrHO.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
- package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
- package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
- package/dist/{files-BOBdC0uI.js → files-BVVWAS0A.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BVVWAS0A.js.map} +1 -1
- package/dist/files-BaqXvB88.js +1 -0
- package/dist/{files-tiNWe8Ih.js → files-Bey1VAex.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-Bey1VAex.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/files-CNHhSyb3.js +1 -0
- package/dist/files-CW1lChNh.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-D6j-XpaK.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-D6j-XpaK.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-F44NRCN0.js} +143 -20
- package/dist/files-F44NRCN0.js.map +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-Jqkf9SBa.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-Jqkf9SBa.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Dsc4Wi-F.js} +65 -30
- package/dist/images-Dsc4Wi-F.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/init-C0oUG3jh.js +1 -0
- package/dist/{init-Bi1Q3lRi.js → init-QB5ecPDA.js} +33 -14
- package/dist/init-QB5ecPDA.js.map +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
- package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
- package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
- package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
- package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
- package/dist/layout-DfQa4mJs.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
- package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
- package/dist/live-DwjWJTdn.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
- package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
- package/dist/{live-DL0se4Zd.js → live-y1p3oNXH.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-y1p3oNXH.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
- package/dist/model-3pZZzloU.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-DEjwetkJ.js} +1 -1
- package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-jRKSQBg6.js} +5 -5
- package/dist/{model-cgt5uvu8.js.map → model-jRKSQBg6.js.map} +1 -1
- package/dist/model-nfewGTzB.js +1 -0
- package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
- package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
- package/dist/options-DpsJ_5CU.js +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-Bx-qp8Li.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-Bx-qp8Li.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-ChO0A-xn.js} +63 -12
- package/dist/plan-ChO0A-xn.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-BioMWncI.js} +636 -9
- package/dist/plugins-BioMWncI.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-BUjk_6wT.js} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-CJSGzMzc.js} +8 -9
- package/dist/preflight-CJSGzMzc.js.map +1 -0
- package/dist/preflight-DD3fI4Hz.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-qFCXmxha.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-qFCXmxha.js.map} +1 -1
- package/dist/{prepare-DQEDXoDp.js → prepare-D7oSSFNm.js} +9 -9
- package/dist/{prepare-DQEDXoDp.js.map → prepare-D7oSSFNm.js.map} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-xHU2BpN-.js} +2 -2
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-B-flBAzR.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-B-flBAzR.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-B6StVyfG.js} +215 -48
- package/dist/providers-B6StVyfG.js.map +1 -0
- package/dist/providers-Dljf6al6.js +1 -0
- package/dist/query-BAc2tQZj.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-CkLmJv7R.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-CkLmJv7R.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-CJIh2FSX.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-CJIh2FSX.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
- package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
- package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
- package/dist/redact-DL2BTykw.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-B3wjKscW.js} +46 -11
- package/dist/remarks-B3wjKscW.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-C88amFRN.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-C88amFRN.js.map} +1 -1
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/resolve-DW92wpIZ.js +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-G576PiFJ.js} +14 -11
- package/dist/resolve-G576PiFJ.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-zVHIW6xw.js} +9 -22
- package/dist/results-zVHIW6xw.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-D_PcUXfG.js} +94 -16
- package/dist/resume-D_PcUXfG.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-BvTzCxdc.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-BvTzCxdc.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
- package/dist/routes-B0F2R-K7.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-B3r_pAq6.js} +4 -4
- package/dist/{runtime-Xm9oeC7o.js.map → runtime-B3r_pAq6.js.map} +1 -1
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-Duh3z1TV.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-Duh3z1TV.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CGEsvVRG.js +1 -0
- package/dist/{server-C00zd6zl.js → server-DUWMRKik.js} +8 -8
- package/dist/{server-C00zd6zl.js.map → server-DUWMRKik.js.map} +1 -1
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-CHuN4Nfg.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-CHuN4Nfg.js.map} +1 -1
- package/dist/source-Wz1Nrfue.js +1 -0
- package/dist/src--yl1OypK.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
- package/dist/src-B-HuDqpH.js.map +1 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-y0nOeRRE.js → src-BHJ4MWvY.js} +494 -112
- package/dist/src-BHJ4MWvY.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
- package/dist/src-BI0v8qlZ.js.map +1 -0
- package/dist/src-BL-_0QoE.js +1 -0
- package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
- package/dist/src-BqNzHHWn.js.map +1 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C-mOSczH.js +2 -0
- package/dist/src-CZaYu6ER.js +3 -0
- package/dist/src-CeBhwD8k.js +2 -0
- package/dist/src-CkEiiyON.js +2 -0
- package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
- package/dist/src-CmA5Yev6.js.map +1 -0
- package/dist/src-CwH_nEaw.js +2 -0
- package/dist/src-DA95rM6v.js +4 -0
- package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
- package/dist/src-DL-EE4DU.js.map +1 -0
- package/dist/src-De0KY5S_.js +2 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/state-B3jex5KU.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUBRDyMk.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUBRDyMk.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
- package/dist/suite-Bhs9-Z7O.js +2 -0
- package/dist/{summary-DDy9DfyJ.js → summary-CSp4cfnr.js} +25 -7
- package/dist/{summary-DDy9DfyJ.js.map → summary-CSp4cfnr.js.map} +1 -1
- package/dist/sync-CxAY6Dki.js +559 -0
- package/dist/sync-CxAY6Dki.js.map +1 -0
- package/dist/{tart-dWoJvWHc.js → tart-CcrrtSrp.js} +179 -61
- package/dist/tart-CcrrtSrp.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-F-6seRC9.js +192 -0
- package/dist/tart-bake-F-6seRC9.js.map +1 -0
- package/dist/tart-bake-RK7RaT9E.js +1 -0
- package/dist/tart-cli-B86Jqo1t.js +89 -0
- package/dist/tart-cli-B86Jqo1t.js.map +1 -0
- package/dist/tart-cli-ClvRniLQ.js +1 -0
- package/dist/tart-hosts-Brii7zHM.js +1 -0
- package/dist/tart-hosts-DPyZsg-1.js +313 -0
- package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
- package/dist/tart-images-CWSf03w8.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-CVDdJy0H.js +433 -0
- package/dist/tool-class-CVDdJy0H.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
- package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +27 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-DRxXCyIH.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-DRxXCyIH.js.map} +1 -1
- package/dist/views-i67V1ly3.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-C3Q4U63F.js} +12 -9
- package/dist/worker-entry-C3Q4U63F.js.map +1 -0
- package/dist/workspace-DhUOW8if.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-P72SXLXr.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-P72SXLXr.js.map} +1 -1
- package/package.json +12 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# compare, cells, lift and questions
|
|
2
|
+
|
|
3
|
+
## Contents
|
|
4
|
+
|
|
5
|
+
- What `compare` runs
|
|
6
|
+
- Gates and lift
|
|
7
|
+
- Effort and contrasts
|
|
8
|
+
- Labeled checks that discriminate
|
|
9
|
+
- `questions`
|
|
10
|
+
|
|
11
|
+
## What `compare` runs
|
|
12
|
+
|
|
13
|
+
`compare` answers "does this environment help?". `compare: { control, variants }` names
|
|
14
|
+
both sides: the control is what each variant is measured against, and `control` is required.
|
|
15
|
+
|
|
16
|
+
Each variant runs as that environment on its own, with the same prompt and checks. `compare`
|
|
17
|
+
does not layer, inherit or extend. Share setup first (the example is in SKILL.md).
|
|
18
|
+
|
|
19
|
+
- Attempts = evals × harnesses × (1 control + variants) × runs. The example is 1 eval, 2 cells
|
|
20
|
+
(`<harness>/plain/<sandbox>` control, `<harness>/house-style/<sandbox>` variant), 8 attempts.
|
|
21
|
+
- With `runs` unset at every level, a compare with variants runs 4 a cell, the fewest tries a
|
|
22
|
+
side a lift verdict needs (the plan notes `runs 4 (compare default)`); anything else runs once,
|
|
23
|
+
including a compare with two or more `cases`: its lift pairs cases, so add cases rather than
|
|
24
|
+
runs (the plan notes `runs 1 a case`).
|
|
25
|
+
Any level that sets `runs`, `runs: 1` and `--runs` included, replaces it, and a profile's
|
|
26
|
+
`limit.runs` caps it. A compare with `cases` runs each case once: its lift pools over the cases.
|
|
27
|
+
Don't set `runs: 1` in the config unless you mean every compare to run once.
|
|
28
|
+
- `parent.extendEnvironment(name, spec)` appends the child's spec as a new layer. The child wins
|
|
29
|
+
on shared keys; `setup` concatenates. Call it before `compare`. Two stacks that should not
|
|
30
|
+
share are two `defineEnvironment` calls, then `{ control, variants }`.
|
|
31
|
+
- `environments` beside `compare` (on the same object) is exit 2: move it to `compare.control`.
|
|
32
|
+
An inherited `environments` is not applied as files, skills or variables. A side that omits
|
|
33
|
+
`sandbox` uses the cascaded `sandbox` option, else implicit Docker; `.extend()` keeps the
|
|
34
|
+
parent env's sandbox when the child omits it.
|
|
35
|
+
- There is no `strategy` field. A profile's `limit.compare: false` strips variants and runs
|
|
36
|
+
the control, which then gates like an eval without compare (as does `maxVariants: 0`), whatever
|
|
37
|
+
`compare.gate` says. `compare: false` on `evaluate` or `group` is an error.
|
|
38
|
+
|
|
39
|
+
## Gates and lift
|
|
40
|
+
|
|
41
|
+
- Only variant cells gate by default (`gate?: 'variants' | 'all' | 'observe'`). A control's misses
|
|
42
|
+
show in its arm's count (`plain 0 of 2`) and never fail the run. `compare.harnesses: [cursor({ model: 'composer-2.5' })]` compares on some harnesses only.
|
|
43
|
+
- With `runs > 1`, a cell passes only if every counted attempt passes (pass^k), unless
|
|
44
|
+
`gate: { passRate: 0.8 }` (a pass@1 threshold; pass@1 is passes over passes and fails). Infra errors don't count; a cell is `infra_error`
|
|
45
|
+
only when nothing counted.
|
|
46
|
+
- **Lift** is variant pass@1 minus control pass@1, with a 95% interval. The result block
|
|
47
|
+
prints it as an effect word after the variant's count: `helped`, `made it worse`, or `can't
|
|
48
|
+
tell yet` below four tries a side or when the interval crosses 0; `-v` adds the difference
|
|
49
|
+
and interval. `reeval query factor_effects` and `reeval analyze` use the same four: below
|
|
50
|
+
it the lift is `insufficient` (not measured), never a results line. When the eval's sources changed across the compared runs, that pass@1
|
|
51
|
+
difference is `not comparable: the eval changed`, not an effect, and mixed counts such as
|
|
52
|
+
`6/10 vs 8/9` are not a results line.
|
|
53
|
+
- `compare: { control, variants: [x], minLift: 0 }` fails a variant whose interval's lower
|
|
54
|
+
bound is below 0.
|
|
55
|
+
|
|
56
|
+
## Effort and contrasts
|
|
57
|
+
|
|
58
|
+
- From four tries a side the result block adds an **effort** table: each arm's median tool
|
|
59
|
+
calls, agent time and, when both sides reported usage, tokens, which often shows a skill's
|
|
60
|
+
effect before pass@1 does:
|
|
61
|
+
`effort tools time tokens` / `plain 54 1m 39s 2,890k` / `with-house-greeting 3 6.6s 57k`.
|
|
62
|
+
A cell without usage is left out of the token sample only; if either side then
|
|
63
|
+
has none, the token metric is omitted, not `insufficient`. Effort's rank test needs
|
|
64
|
+
`runs: 4` a cell, as lift does (below it lift shows only a direction and effort can't be tested; the plan hints so, at `runs: 1` too); `reeval query effort` has
|
|
65
|
+
its p-values, verdicts and whether the ranges are `separated`.
|
|
66
|
+
- **One comparison per variant** against its baseline covers every measure: pass rate, each
|
|
67
|
+
check's pass rate, each judge's scores, each compare question's votes, tool calls, agent time
|
|
68
|
+
and tokens. Each is `better`, `worse`, `same` or `leans` (a difference its own test doesn't hold
|
|
69
|
+
yet, at p below 0.1 and of at least 10 points, 0.05 of a judge's score, 60/40 of 4 or more
|
|
70
|
+
decisive pairs for a question at p below 0.2, or 10% of an effort median, with the runs that would hold it;
|
|
71
|
+
weaker is `same`, unless the sample is too small for its test to show anything: "12× faster,
|
|
72
|
+
but 3 runs a side can't confirm it; 4 is the least that can"). A question counts votes, both
|
|
73
|
+
orders of each pair, as its standings do ("in 8 of 8 votes (4 of 4 decisive pairs)"). Each measure keeps its own 0.05, so with identical arms some measure reads
|
|
74
|
+
better or worse by chance in about 1 run in 14 at 4 a side, and 1 in 7 at 8 or more. The status list prints the ones that hold under the
|
|
75
|
+
variant (`Same pass rate · 5× faster.`), `reeval compare -v` the leans too, and
|
|
76
|
+
`reeval query measure_effects` has a row per measure. When both arms pass every run, a measure
|
|
77
|
+
that holds is the result (no remark), one that leans raises `too-few-tries` with the runs, and
|
|
78
|
+
all the same raises `cant-tell-apart`.
|
|
79
|
+
- With two variants or more, `reeval query contrasts` sets every pair of arms side by side,
|
|
80
|
+
variant against variant too.
|
|
81
|
+
- `reeval query skill_files` says which declared files (`SKILL.md`, references, rules) each
|
|
82
|
+
attempt opened, and `context_use` counts its tool calls as `context`, `dependency`
|
|
83
|
+
(`node_modules`, `--help`), `web`, `verify` or `workspace`: a heuristic, read it as counts.
|
|
84
|
+
|
|
85
|
+
## Labeled checks that discriminate
|
|
86
|
+
|
|
87
|
+
- **Label each requirement** with `expect`'s second argument:
|
|
88
|
+
`await expect(c, 'exports greet').toHaveFile('greet.js', /export/)`. Every matcher call
|
|
89
|
+
is recorded (`evals[].attempts[].assertions`; unlabeled ones get `<matcher> <first arg>`).
|
|
90
|
+
In a compare, keep independent requirements as `expect`, not `expect.require`: a failed
|
|
91
|
+
`expect.require` ends the attempt, and the checks after it count as not run: a side that never reached a check reads
|
|
92
|
+
`not run` and the check gets no verdict, so it can't say which arm met it. The result block's
|
|
93
|
+
check table puts a row per check under each arm (`house greeting 0 of 2 2 of 2`) with the
|
|
94
|
+
first failure's message, and folds the checks that passed everywhere into `+2 more passed`.
|
|
95
|
+
After a run, `reeval query "SELECT eval_key, label, verdict FROM check_discrimination WHERE verdict
|
|
96
|
+
LIKE 'both-%'"` and rework or drop those checks: they measure nothing about the variant.
|
|
97
|
+
Changing a label changes the eval's fingerprint. Label judges too
|
|
98
|
+
(`expect(await c.judge(q, { diff: true }), 'Social OAuth').toPass()`): the label replaces the clipped
|
|
99
|
+
question in judge rates, `reeval show`, `summary.md` and the digest.
|
|
100
|
+
|
|
101
|
+
## `questions`
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
evaluate('greets in the house style', {
|
|
105
|
+
compare: { control: plain, variants: [houseStyle] },
|
|
106
|
+
questions: [
|
|
107
|
+
// asked of each attempt as it ends: free text, or a pick from choices
|
|
108
|
+
{ question: 'How did the agent pick its greeting?', transcript: true },
|
|
109
|
+
{
|
|
110
|
+
question: 'Does greet.js handle a missing name?',
|
|
111
|
+
diff: true,
|
|
112
|
+
choices: ['yes', 'no'],
|
|
113
|
+
},
|
|
114
|
+
// pairwise: which side did better, after the run
|
|
115
|
+
{
|
|
116
|
+
question: 'Which greet.js reads more like our house style?',
|
|
117
|
+
across: 'environment',
|
|
118
|
+
diff: true,
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
question: 'Which sign-in page matches the mock?',
|
|
122
|
+
across: 'environment',
|
|
123
|
+
artifacts: ['sign-in'],
|
|
124
|
+
},
|
|
125
|
+
],
|
|
126
|
+
}, async (c) => { … })
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
- `questions` sits beside `compare`, on an eval or a group (never the config or a
|
|
130
|
+
profile), and works without a `compare` too. It's always a list of objects:
|
|
131
|
+
`{ question, across?, choices?, model?, turns?, …inputs }`, with at least one input
|
|
132
|
+
turned on (`task`, `output`, `files`, `diff`, `transcript`, `checks`, `artifacts`;
|
|
133
|
+
no `contents`, the workspaces are gone). There are no default inputs: an entry with
|
|
134
|
+
none exits 2 at plan time. An eval's list replaces its group's.
|
|
135
|
+
- **No `across`: asked of each attempt** as it ends with a pass or a fail (infra
|
|
136
|
+
errors and skipped attempts aren't asked), while the run goes on. `choices`
|
|
137
|
+
(two or more) makes the model pick one, with a reason; the result block prints
|
|
138
|
+
the tally (`yes 8 · no 4`), else the first few answers. Answers land on
|
|
139
|
+
`evals[].attempts[].answers` and in `reeval query attempt_answers` (with the
|
|
140
|
+
attempt's outcome, `harness`, `environment` and `role`, so you can split
|
|
141
|
+
answers by harness, arm or pass/fail).
|
|
142
|
+
Ask these where a control is expected to fail: every attempt is asked.
|
|
143
|
+
- **`across`: pairwise.** `'environment'` pairs two environments on one harness;
|
|
144
|
+
`'harness'` two harnesses on one environment; a list asks both pair kinds.
|
|
145
|
+
The factor must vary in the eval (else exit 2); a profile that narrows it away
|
|
146
|
+
drops the question (`1 pairwise question off (profile …)`). `choices` with
|
|
147
|
+
`across` exits 2. `--dry-run` shows the pair count. The result block prints
|
|
148
|
+
each side's wins (`with-house-greeting won 4 of 4`). Each pair is voted in
|
|
149
|
+
both orders; one whose orders disagree is a split (the position decided). With
|
|
150
|
+
three arms or more, read `reeval query question_pairs` (head to head, with
|
|
151
|
+
splits), not the pooled wins, which count a split as a win for each side. A
|
|
152
|
+
vote may carry a short `reason` (capped); do not gate on its wording.
|
|
153
|
+
- Questions annotate and never gate: pass, fail and the exit code stay the
|
|
154
|
+
checks'. `reeval analyze` puts each answer in the digest as annotation, not a
|
|
155
|
+
results line. Their model is the entry's `model`, else the cascaded
|
|
156
|
+
`question.model`, else `analysis.model`, else the run's judge (a profile's
|
|
157
|
+
included) when it's a language model (not `typesafe-ai/*`); with none, `reeval run` exits 2 at plan
|
|
158
|
+
time while `--dry-run` still lists them. Keep deterministic checks for pass or
|
|
159
|
+
fail.
|
|
160
|
+
- Each side shows only the entry's inputs (`diff: ['src/**']`,
|
|
161
|
+
`transcript: 'tools', checks: ['npm test']`); `turns` defaults to `'all'`. Keep
|
|
162
|
+
`output` off for pairwise questions, since the more confident summary wins on
|
|
163
|
+
wording. For UI evals, `artifacts: ['sign-in']` (or `true`) sends each attempt's
|
|
164
|
+
own capture of that name as an image. The `question` option takes only `model`
|
|
165
|
+
(config → profile → group → eval → entry). A pairwise question showing
|
|
166
|
+
`transcript` across harnesses warns (tool names unblind them): ask it with
|
|
167
|
+
`across: 'environment'`.
|
|
168
|
+
- Pairwise questions are answered on the host once all of an eval's attempts
|
|
169
|
+
end. Attempts that failed their checks (a judge failure doesn't count) are
|
|
170
|
+
left out of the pairs: a pairwise question on a compare whose baseline is
|
|
171
|
+
expected to fail gets no pairs, so ask it of each attempt instead. Each pair
|
|
172
|
+
goes to the model twice, order swapped, labeled A and B; the record's
|
|
173
|
+
`evals[].answers` has every vote and each side's `wins`/`n` over decided
|
|
174
|
+
votes, per pair kind. A model error is stored on the answer as `error` and
|
|
175
|
+
warned about; it never changes an outcome. To see what a question was shown,
|
|
176
|
+
open its `input`: `<eval-slug>/questions/<n>.json` for a pairwise one (each
|
|
177
|
+
pair and order's masked prompt, which side was A, screenshot refs), or
|
|
178
|
+
`questions/<n>.json` in the attempt's directory. Test with
|
|
179
|
+
`fakeAnalysis({ vote: 'A' | 'B' | 'tie' | (pair) => … | { error } })` and
|
|
180
|
+
`fakeAnalysis({ answer: 'text' | { choice, text } | (q) => … | { error } })`.
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# reeval.config.ts, harnesses and options
|
|
2
|
+
|
|
3
|
+
## Contents
|
|
4
|
+
|
|
5
|
+
- reeval.yaml
|
|
6
|
+
- Path rule and where the sandbox comes from
|
|
7
|
+
- Harnesses, keys and effort
|
|
8
|
+
- Judge models and imports
|
|
9
|
+
- Options and the cascade
|
|
10
|
+
|
|
11
|
+
## reeval.yaml
|
|
12
|
+
|
|
13
|
+
A project without TypeScript can write `reeval.yaml` (or `.yml`) instead, with no callbacks:
|
|
14
|
+
|
|
15
|
+
```yaml
|
|
16
|
+
harnesses: { kind: cursor, model: composer-2.5 }
|
|
17
|
+
judge: { model: typesafe-ai/jev-1.14.0 } # or { kind: fake-judge, score: 0.9 }
|
|
18
|
+
env:
|
|
19
|
+
NPM_TOKEN: { kind: secret, hosts: [registry.npmjs.org] }
|
|
20
|
+
environments:
|
|
21
|
+
kind: environment
|
|
22
|
+
name: base
|
|
23
|
+
layers: [{ name: base, spec: { setup: [npm ci] } }]
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
A `reeval.yaml` beside a `reeval.config.*` is an error.
|
|
27
|
+
|
|
28
|
+
## Path rule and where the sandbox comes from
|
|
29
|
+
|
|
30
|
+
One path rule for both: a relative host path resolves against the directory of the file it's
|
|
31
|
+
written in (the config's, an eval file's, or the module calling `defineEnvironment` or a helper),
|
|
32
|
+
whether it comes from a helper (`file()`, `skill.local()`, `skill.npm()`, `git()`,
|
|
33
|
+
`docker({ dockerfile })`, `fakeSandbox({ dir })`) or its data form (`{ kind: file, path }`,
|
|
34
|
+
`{ kind: fake, dir }`, …). Absolute paths stay as written.
|
|
35
|
+
|
|
36
|
+
| Where the sandbox comes from | When |
|
|
37
|
+
|---|---|
|
|
38
|
+
| The environment's `sandbox` | It sets one; `extendEnvironment` keeps the parent's when the child omits it |
|
|
39
|
+
| The cascaded `sandbox` option (`defineConfig({ sandbox })`) | The resolved environment omits it; it's the default machine, not an environment |
|
|
40
|
+
| Implicit Docker (reeval's image: Node 26, git, pnpm, the agent CLI) | Neither is set |
|
|
41
|
+
|
|
42
|
+
A named env is a total override: no skills, files or sandbox inherit from the config
|
|
43
|
+
environment. A fake profile or config puts `sandbox: fakeSandbox()` on `defaults` or
|
|
44
|
+
`defineConfig` so `plain` and skill variants run without Docker; that's for trying configs, not
|
|
45
|
+
for validating checks (an environment's own `docker()` still wins). Don't put `fakeSandbox()`
|
|
46
|
+
on `plain` itself: real Cursor runs need Docker. An oracle or canary profile sets only
|
|
47
|
+
`defaults.harnesses`, so each eval keeps its sandbox: see
|
|
48
|
+
[evals.md](evals.md#trustworthy-evals).
|
|
49
|
+
|
|
50
|
+
## Harnesses, keys and effort
|
|
51
|
+
|
|
52
|
+
Harnesses (`reeval/harness`), each with a required `model`; a list runs each as its own cell:
|
|
53
|
+
|
|
54
|
+
| Factory | Key | Notes |
|
|
55
|
+
|---|---|---|
|
|
56
|
+
| `cursor({ model, params?, effort? })` | `CURSOR_API_KEY` | Cursor's agent through the Cursor SDK. `params` such as `{ fast: 'true' }` pick the variant; `effort` maps onto the parameter `Cursor.models.list()` names for that model |
|
|
57
|
+
| `claudeCode({ model, effort? })` | `ANTHROPIC_API_KEY`, else a subscription's `CLAUDE_CODE_OAUTH_TOKEN` (`claude setup-token`) | `effort` is the Agent SDK's `/effort` |
|
|
58
|
+
| `antigravity({ model, effort? })` | `GEMINI_API_KEY` (a Gemini API key) | e.g. `gemini-3.8-flash-medium`; `effort` is `agy --effort` |
|
|
59
|
+
| `opencode({ model, effort? })` | That provider's key, e.g. `ANTHROPIC_API_KEY` | OpenCode 2; the model is `<provider>/<model>`, e.g. `google/gemini-3.8-flash`. `amazon-bedrock/…` takes `AWS_BEARER_TOKEN_BEDROCK` and an optional `AWS_REGION`; Vertex (on `direct()`; it runs through `vertex()`), Azure, Copilot and local providers are refused at plan time. Web search via Exa, whose host it allows; it also reads `.claude/skills` and `.agents/skills`. `effort` is refused where the model's API answers it with a 400 (any value on `anthropic/claude-haiku-4-5`, `openai/gpt-4.1` or `google/gemini-2.5-flash`, any but `high` on `openai/gpt-5-pro`), and such a model runs with none by default; on `openrouter()` nothing is refused |
|
|
60
|
+
| `goose({ model, effort? })` | The provider's key: `GOOGLE_GENERATIVE_AI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY` or `XAI_API_KEY` | `<provider>/<model>` on Goose's `google`, `anthropic`, `openai` or `xai` provider, e.g. `google/gemini-3.8-flash` (any gateway id on `vercel()` or `openrouter()`); `effort` is `GOOSE_THINKING_EFFORT` |
|
|
61
|
+
| `pi({ model, effort? })` | The provider's key, e.g. `GOOGLE_GENERATIVE_AI_API_KEY`, `ANTHROPIC_API_KEY`, `XAI_API_KEY`, `MISTRAL_API_KEY` | `<provider>/<model>` on any of pi's providers that sends its key in a header on a fixed host, e.g. `google/gemini-3.8-flash` (any gateway id on `vercel()` or `openrouter()`); `effort` is `--thinking` |
|
|
62
|
+
| `codex({ model, effort? })` | `OPENAI_API_KEY` | An OpenAI model id, e.g. `gpt-5.6-luna`; `effort` is `model_reasoning_effort` |
|
|
63
|
+
| `fx({ model, effort? })` | `OPENAI_API_KEY` for a bare id, `AI_GATEWAY_API_KEY` on `vercel()`, `OPENROUTER_API_KEY` on `openrouter()`, or none on `vertex()` | Vercel's fx over ACP: a gateway id (`anthropic/claude-sonnet-5`) runs only on `vercel()` or `openrouter()` in `gateway`, a bare OpenAI id (`gpt-5.5`) on OpenAI, which must accept function tools without `reasoning_effort` (not `gpt-5.6` or later), a Gemini id (`google/gemini-3.8-flash`) on `vertex()`. `effort` (`reasoning_effort`) only on `vertex()`. Its own web tools bypass the egress gateway, so `deny-all` blocks them |
|
|
64
|
+
| `grokBuild({ model, effort? })` | `XAI_API_KEY` | xAI's Grok Build CLI; an xAI model id, e.g. `grok-4.6`; `effort` is `default_reasoning_effort`. It finds MCP tools with its `search_tool` before calling them |
|
|
65
|
+
| `fakeHarness(…)` | None | See the fake trio in SKILL.md |
|
|
66
|
+
|
|
67
|
+
OpenCode, Goose, pi and Codex refuse `mcp.sse`. Cursor's `fast` defaults to `'false'` (standard
|
|
68
|
+
price; `{ fast: 'true' }` costs several times more). Other unset Cursor params take Cursor's
|
|
69
|
+
default, which the record shows (`composer-2.5[fast=false]`), so pin them for reproducible evals.
|
|
70
|
+
|
|
71
|
+
`effort` is the same field on every factory (`fx()` takes it only on `vertex()`), with whatever values that CLI takes (no shared
|
|
72
|
+
enum); Fast is not effort. The default harness id includes `[effort=<value>]` when it's set, and
|
|
73
|
+
the record stores the requested effort as `effort` and the served one as `servedEffort`.
|
|
74
|
+
|
|
75
|
+
In `reeval query` those are `reasoning_effort` (`cells`, `attempts`) and
|
|
76
|
+
`served_reasoning_effort` (`attempts`). **effort** on its own (`reeval query effort`, the
|
|
77
|
+
result block's effort table) is always the measured tool-call, time and token metric.
|
|
78
|
+
|
|
79
|
+
Keys never enter the sandbox: the agent sees a placeholder that reeval's gateway swaps on the
|
|
80
|
+
harness's own endpoints. Every harness reports tokens, which reeval records per turn,
|
|
81
|
+
attempt and run with their cost (reported, else estimated at list prices).
|
|
82
|
+
|
|
83
|
+
`gateway` (`reeval/gateway`) lists the routes model calls may take, in order: each harness, the
|
|
84
|
+
judge and analysis take the first that can serve them. Every list ends with `direct()` (each
|
|
85
|
+
provider's own key), so don't write it at the end: list it only to move it (`[direct(), vercel()]`
|
|
86
|
+
tries own keys first), or write `direct({ enabled: false })` for strict routing, where a cell no
|
|
87
|
+
entry serves stops the plan (never both). `gateway: [vercel()]` runs Claude
|
|
88
|
+
Code, Codex, Goose, pi, OpenCode, fx and Grok Build on Vercel AI Gateway with one `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`) (`effort`
|
|
89
|
+
on Goose, or on OpenCode with a model other than Gemini, falls back to `direct()`), and
|
|
90
|
+
everything else on its own key; the plan's hint says which cells fell back. Pin one with
|
|
91
|
+
the factory's or the judge's `gateway: 'vercel'`; a pin never falls back. `vertex({ project })`
|
|
92
|
+
(else `GOOGLE_CLOUD_PROJECT`) runs Claude Code, Antigravity, OpenCode, Goose, pi, Grok Build and fx (Gemini only) on Vertex AI
|
|
93
|
+
with no key: reeval mints a token from Application Default Credentials on the host and the egress
|
|
94
|
+
gateway adds it (OpenCode's Gemini model runs as `google-vertex/<id>`, Grok Build's xAI model as `xai/<id>` on Vertex's Chat Completions API).
|
|
95
|
+
There Claude Code sends `effort` only to the models its own catalog lists, so `effort` on one it
|
|
96
|
+
doesn't, such as `claude-haiku-5-5`, falls through, and with none set that model runs with no effort. The `sonnet`
|
|
97
|
+
and `opusplan` aliases fall through too: Claude Code resolves them on Vertex AI to Sonnet 4.5, which Anthropic retires on 2026-11-30, so name
|
|
98
|
+
`claude-sonnet-5-5` instead. Claude ids work in
|
|
99
|
+
either form (`claude-opus-4-5-20251101` is sent as Vertex's `claude-opus-4-5@20251101`), so one
|
|
100
|
+
string works through `[vertex()]` and its `direct()` fallback; `google/*` and `anthropic/*` judge and analysis models run there
|
|
101
|
+
too. `reeval doctor --online` checks the token, the project's Vertex AI API and that Vertex serves each model the config sends there (Claude needs enabling in Model Garden). Default `[direct()]`:
|
|
102
|
+
no gateway, so the judge, analysis and captures take only their provider's own key.
|
|
103
|
+
`gateway: [openrouter()]` runs Claude Code, Codex, Goose, OpenCode, fx, Grok Build and pi on
|
|
104
|
+
OpenRouter with one `OPENROUTER_API_KEY` (Cursor, Antigravity, and `effort` on fx, on Claude Code or
|
|
105
|
+
on Goose or OpenCode with a model other than Gemini, fall through). Every OpenRouter call records its upstream provider and cost
|
|
106
|
+
(`attempts[].upstreams`, `reportedCostUsd`; a judgment's `upstream`, `costUsd`).
|
|
107
|
+
`openrouter({ fallbacks: true })` lets reeval's own calls (judge, analysis, captures) fail over
|
|
108
|
+
between upstreams; the agents' calls take OpenRouter's routing whatever it says. Write OpenCode
|
|
109
|
+
ids as `anthropic/…`, never `openrouter/…` or `vercel/…`.
|
|
110
|
+
|
|
111
|
+
## Judge models and imports
|
|
112
|
+
|
|
113
|
+
- Judge models: `'typesafe-ai/jev-1.14.0'` (`TYPESAFE_AI_API_KEY`, recommended),
|
|
114
|
+
`'google/gemini-3.8-flash'` (`GOOGLE_GENERATIVE_AI_API_KEY`), `anthropic/*` (`ANTHROPIC_API_KEY`)
|
|
115
|
+
and `'openai/gpt-6-luna'` (`OPENAI_API_KEY`; OpenAI's Decisions API takes only its decision
|
|
116
|
+
model, so not `openai/gpt-5.5`), each on its provider's own key. reeval uses no gateway by
|
|
117
|
+
default: `AI_GATEWAY_API_KEY` is read only with `gateway: [vercel()]` in the config (or a judge
|
|
118
|
+
`gateway: 'vercel'` pin, listed there), `OPENROUTER_API_KEY` only with `openrouter()`. As a
|
|
119
|
+
judge, `vercel()` serves decision models only (Jev, `openai/gpt-6-luna-decisions`), and
|
|
120
|
+
`openrouter()` only those on its Decisions API (`typesafe/jev-1.14`), never a language model:
|
|
121
|
+
pin a language-model judge to `gateway: 'direct'`. Always a provider prefix.
|
|
122
|
+
With a `gateway` list set, a judge model takes the first listed gateway that serves it, and only
|
|
123
|
+
that gateway's key.
|
|
124
|
+
- Imports: `reeval` (`defineConfig`, `defineEnvironment`, `group`, `evaluate`,
|
|
125
|
+
`expect`, `DEFAULT_EGRESS_ALLOW`, `DEFAULT_SKILL_EXCLUDE`, types), `/harness`, `/gateway`
|
|
126
|
+
(`direct`, `vercel`), `/sandbox`
|
|
127
|
+
(`docker`, `fakeSandbox`), `/judge` (`fakeJudge`, `RECOMMENDED_JUDGE_MODEL`,
|
|
128
|
+
`DEFAULT_THRESHOLD`). There is no `test`, `describe`, `baseline`, `secret` or `skill` export.
|
|
129
|
+
|
|
130
|
+
## Options and the cascade
|
|
131
|
+
|
|
132
|
+
`defineConfig`, `group` and `evaluate` take the same options, and inner levels win:
|
|
133
|
+
|
|
134
|
+
| Option | Cascades as |
|
|
135
|
+
|---|---|
|
|
136
|
+
| `harness` | One or a list, or an edit of the inherited list: `{ add: h }`, `{ only: 'cursor:composer-2.5' }` or a function |
|
|
137
|
+
| `sandbox` | The default machine when the resolved env omits one |
|
|
138
|
+
| `environment` | One; a named env is a total override of the config environment |
|
|
139
|
+
| `env` | Merged by name |
|
|
140
|
+
| `judge` | Merged by field |
|
|
141
|
+
| `capture` | Shallow-merged camera defaults (`viewport`, `timeout`, `browser`, `model`, the vision model); `viewport` replaces the value, and the port is not an option |
|
|
142
|
+
| `runs`, `gate`, `egress`, `timeout` | Replaced (`runs` unset everywhere: 4 for a compare with variants, else 1) |
|
|
143
|
+
| `tags`, `pin` | Unioned |
|
|
144
|
+
| `compare` | Groups and evals only |
|
|
145
|
+
|
|
146
|
+
Options are plain objects, never functions. Unknown keys are errors with a hint (`repeat` → "Did
|
|
147
|
+
you mean runs?").
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# Environments, sandboxes, secrets and egress
|
|
2
|
+
|
|
3
|
+
## Contents
|
|
4
|
+
|
|
5
|
+
- Environments: skills, files, rules, MCP, setup
|
|
6
|
+
- Fixture directories and `git()` workspaces
|
|
7
|
+
- Skills and composition
|
|
8
|
+
- Sandboxes
|
|
9
|
+
- Secrets and egress
|
|
10
|
+
|
|
11
|
+
## Environments: skills, files, rules, MCP, setup
|
|
12
|
+
|
|
13
|
+
What the agent runs *in*, per attempt: the code the workspace starts as, skills, MCP servers,
|
|
14
|
+
files, rules, env vars, secrets and setup steps. Pass a spec, or a synchronous callback that
|
|
15
|
+
destructures the helpers it needs:
|
|
16
|
+
|
|
17
|
+
```ts
|
|
18
|
+
// evals/envs.ts
|
|
19
|
+
import { defineEnvironment } from 'reeval'
|
|
20
|
+
|
|
21
|
+
export const houseStyle = defineEnvironment('house-style', ({ skill }) => ({
|
|
22
|
+
// a dir with SKILL.md, relative to this file
|
|
23
|
+
skills: [skill.local('../skills/house-style')],
|
|
24
|
+
}))
|
|
25
|
+
|
|
26
|
+
export const withDocs = houseStyle.extendEnvironment(
|
|
27
|
+
'with-docs',
|
|
28
|
+
({ file, mcp }) => ({
|
|
29
|
+
// written into the workspace
|
|
30
|
+
files: { 'docs/api.md': file('../docs/api.md') },
|
|
31
|
+
// AGENTS.md (CLAUDE.md for Claude Code)
|
|
32
|
+
rules: file('../rules/house.md'),
|
|
33
|
+
// or mcp.http(url) / mcp.sse(url) (not on OpenCode, Goose, pi or Codex)
|
|
34
|
+
mcp: { docs: mcp.stdio('npx', ['-y', '@acme/docs-mcp@1.0.0']) },
|
|
35
|
+
// every attempt, before the agent
|
|
36
|
+
setup: [{ run: 'npm ci', timeout: '5m' }],
|
|
37
|
+
}),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
// Start from existing code: a repo, or one directory of it, at a pinned ref
|
|
41
|
+
export const app = defineEnvironment('acme-app', ({ git }) => ({
|
|
42
|
+
workspace: git('https://github.com/acme/app', {
|
|
43
|
+
ref: 'v2.3.0',
|
|
44
|
+
path: 'examples/basic',
|
|
45
|
+
}),
|
|
46
|
+
setup: ['npm ci'],
|
|
47
|
+
}))
|
|
48
|
+
|
|
49
|
+
// Or seed a fixture directory: a plain object of file() entries, so spread
|
|
50
|
+
// and override it
|
|
51
|
+
export const notes = defineEnvironment('acme-notes', ({ directory }) => ({
|
|
52
|
+
files: { ...directory('../fixture'), 'README.md': 'something' },
|
|
53
|
+
setup: [{ run: 'npm ci --no-audit --no-fund', timeout: '5m' }],
|
|
54
|
+
}))
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Fixture directories and `git()` workspaces
|
|
58
|
+
|
|
59
|
+
Use `directory(path, { into?, glob?, defaultIgnores? })` for a fixture directory instead of
|
|
60
|
+
mapping a file list through `file()`. It skips `.git`, `node_modules/`, `dist/`, top-level
|
|
61
|
+
`build/`, caches and editor folders (`DEFAULT_DIRECTORY_IGNORES`; `defaultIgnores: false` keeps
|
|
62
|
+
them, except `.git`). `glob` is ordered gitignore-style patterns where the last match wins and
|
|
63
|
+
`!` excludes: `glob: ['src/**', 'package.json', '!**/*.test.ts']`.
|
|
64
|
+
|
|
65
|
+
Executable files stay executable; empty directories aren't seeded.
|
|
66
|
+
|
|
67
|
+
`workspace: git(repo, { ref, path })` takes a URL, `github:owner/repo`, a GitHub `/tree/` URL or
|
|
68
|
+
a local path (relative to the file). reeval fetches it on the host before any attempt, with the
|
|
69
|
+
host's git credentials or `token: 'GITHUB_TOKEN'` (a host variable's name; never a token in the
|
|
70
|
+
URL or the eval). It copies the tree in before `files` (which win) and `setup`, and records the
|
|
71
|
+
commit and tree hash.
|
|
72
|
+
|
|
73
|
+
`init: true` adds a `.git` with one commit of the tree (no remote), for agents that expect a
|
|
74
|
+
repo. Pin `ref` to a tag or full SHA; a branch warns. The agent's diff starts after setup, so the
|
|
75
|
+
repo never counts as its change.
|
|
76
|
+
|
|
77
|
+
A bad ref, path, URL or token fails only the attempts that use it (infra error, exit 3).
|
|
78
|
+
Submodules and LFS files aren't fetched.
|
|
79
|
+
|
|
80
|
+
## Skills and composition
|
|
81
|
+
|
|
82
|
+
Skills: `skill.local | git | npm | registry | inline` (pin git refs; `placement: 'workspace'`
|
|
83
|
+
writes them into the repo's skill directory). A git, npm or plugin source URL with a password or
|
|
84
|
+
token in it is refused: reeval fetches on the host with its own git and npm credentials.
|
|
85
|
+
`extendEnvironment` is the only composition; the child wins on shared keys and **replaces**
|
|
86
|
+
`sandbox` entirely when it sets one (install is not merged onto the parent image).
|
|
87
|
+
|
|
88
|
+
A skill is a directory whose `SKILL.md` has YAML frontmatter
|
|
89
|
+
(`name`, `description`; quote values containing `: `). Baking (`from`, `install`, `dockerfile`)
|
|
90
|
+
is `sandbox: docker({ … })` on the environment.
|
|
91
|
+
|
|
92
|
+
Plugins: `plugins: [plugin.local('../plugins/review', { only: ['skills', 'mcp'] })]` (also
|
|
93
|
+
`plugin.git(repo, { ref, path })`, `plugin.npm(spec)` and
|
|
94
|
+
`plugin.marketplace('name@github:owner/repo', { ref })` for an entry in a Claude Code
|
|
95
|
+
marketplace, or `'name@npm:<spec>'`; only a marketplace in a local directory may name `file://`
|
|
96
|
+
entries). A plugin is Claude Code's layout:
|
|
97
|
+
`.claude-plugin/plugin.json` plus `skills/`, `agents/`, `commands/`, `hooks/hooks.json`,
|
|
98
|
+
`.mcp.json`, `rules/`. Its MCP servers join `mcp` as `<plugin>:<server>` (call them from the
|
|
99
|
+
fake agent as `{ mcp: 'review:docs/<tool>' }`), and it lands in `~/.reeval/plugins/<name>/`. Claude
|
|
100
|
+
Code and Grok Build load every part but `rules` (narrow with `only`), namespaced
|
|
101
|
+
(`review:reviewer`), and start the plugin's MCP servers themselves, so a plugin server reading a
|
|
102
|
+
host variable is refused there: declare it in `mcp`. Antigravity loads every part with
|
|
103
|
+
`agy plugin install`, and refuses such a server too; Codex loads `skills` and `mcp` the same way
|
|
104
|
+
(and refuses a plugin with ChatGPT apps). Cursor loads every part flat: rules, subagents, commands
|
|
105
|
+
and hooks unpacked into the workspace's `.cursor/` (left out of the diff), skills into
|
|
106
|
+
`~/.agents/skills`, so two items with one name clash; Goose takes `skills` and `mcp` beside the
|
|
107
|
+
eval's own, so a clashing name is refused too. OpenCode loads skills, MCP servers, and subagents
|
|
108
|
+
and commands (unpacked into `.opencode/`, frontmatter cut to `description`); pi skills, MCP servers
|
|
109
|
+
(as `<plugin>_<server>`) and commands (as prompt templates in `.pi/prompts/`); fx skills and MCP
|
|
110
|
+
servers only. A part a harness doesn't load, or a hook event it doesn't run, is refused at plan
|
|
111
|
+
time (`--dry-run` exits 2 naming the harness): narrow with `only`. `build: 'npm ci && npm run build'` (or `{ run, cwd: '.' }` to run
|
|
112
|
+
from the source's root) builds the plugin in a cached Docker image layer, as root with no
|
|
113
|
+
secrets, without the source's `.git` (pass a version in its commands instead of `git describe`);
|
|
114
|
+
only `path` is placed, so the output must land inside it. Docker only. Its content is fixed when the
|
|
115
|
+
image is prepared: an npm spec re-resolving or a local plugin edited mid-run waits for the next run. A server run with
|
|
116
|
+
`npx`/`uvx` needs its registry in `egress`, or a build that installs it.
|
|
117
|
+
|
|
118
|
+
A named env is a total override: no skills, files or sandbox inherit from the config
|
|
119
|
+
environment.
|
|
120
|
+
|
|
121
|
+
## Sandboxes
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
// reeval's Node image with the agent CLI
|
|
125
|
+
docker()
|
|
126
|
+
// baked once as a cached layer; no secrets
|
|
127
|
+
docker({ install: ['npm i -g firebase-tools@14'] })
|
|
128
|
+
// a Debian or Ubuntu base; reeval adds Node and the agent
|
|
129
|
+
docker({ from: 'python:3.12-slim' })
|
|
130
|
+
// ARG REEVAL_BASE and FROM ${REEVAL_BASE} to layer on reeval's image
|
|
131
|
+
docker({ dockerfile: './sandbox/Dockerfile' })
|
|
132
|
+
// limits; defaults 4g, 2 and 1024
|
|
133
|
+
docker({ memory: '8g', cpus: 4, pids: 4096 })
|
|
134
|
+
// swap on top; default the same as memory, 0 for none
|
|
135
|
+
docker({ memory: '4g', swap: 0 })
|
|
136
|
+
// opt in to apt-get at run time (default: user reeval)
|
|
137
|
+
docker({ root: true })
|
|
138
|
+
// the same sandbox and options on rootless Podman 4+ (cell id `podman`)
|
|
139
|
+
podman({ from: 'python:3.12-slim' })
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Use `podman()` only where the user runs Podman instead of Docker: it takes every `docker()`
|
|
143
|
+
option and confines the container the same way (bridge network, Docker's seccomp profile).
|
|
144
|
+
|
|
145
|
+
`tart({ image })` (an OCI macOS image or a local Tart VM) runs each attempt in a cloned macOS VM
|
|
146
|
+
on an Apple Silicon Mac, at most two at once. It needs `egress: 'allow-all'`: under the default
|
|
147
|
+
`proxy` or a restricted policy the plan exits 2. Of the real harnesses only `cursor()` runs in it
|
|
148
|
+
so far (any other exits 2 at plan time), and it needs Node 22.21 or later in the image; the fake
|
|
149
|
+
harness runs too.
|
|
150
|
+
`reeval run` never pulls the image: one `tart list` doesn't show exits 2, naming `tart pull
|
|
151
|
+
<image>`. Run `npx reeval image build` first, which pulls every `tart()` image the plan needs (and
|
|
152
|
+
refuses, exit 3, when the disk can't hold them even once Tart prunes its cache). Pin a digest
|
|
153
|
+
(`ghcr.io/cirruslabs/macos-sequoia-base@sha256:…`): Cirrus Labs publishes only `latest`, so a
|
|
154
|
+
version tag isn't there to pin. `:latest` or no tag warns, and each attempt records the digest it
|
|
155
|
+
cloned in `vmImage`.
|
|
156
|
+
|
|
157
|
+
```ts
|
|
158
|
+
import { boat, createBoat } from 'reeval/sandbox/boat'
|
|
159
|
+
// one boat.dev VM per attempt, over SSH (cell id `boat`, or `boat:large`)
|
|
160
|
+
defineEnvironment('vm', { sandbox: boat({ type: 'large' }), egress: 'allow-all' })
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
`boat()` needs `npm i @boatdev/sdk`, `BOAT_API_KEY` (or `createBoat({ apiKey, prefix })` for
|
|
164
|
+
another account) and an OpenSSH client, and plans only under `egress: 'allow-all'` and on the
|
|
165
|
+
Cursor harness so far; reeval refuses the others at plan time. It takes no image options: put the
|
|
166
|
+
toolchain's installs in the environment's `setup`.
|
|
167
|
+
|
|
168
|
+
The agent, setup and `c.exec` run as `reeval` (home `/home/reeval`) with no capabilities; builds
|
|
169
|
+
run as root. `root: true` or `capabilities` print a warning at the start of every run, so prefer
|
|
170
|
+
`install`. A process killed at the memory limit makes the attempt an infra error
|
|
171
|
+
(`error.resource.cause: 'oom'`), not a failed check: raise `memory` rather than loosening it.
|
|
172
|
+
|
|
173
|
+
Put a specific machine on `defineEnvironment`, or `sandbox` on `defineConfig`, a profile, group
|
|
174
|
+
or eval as the default when the resolved env omits one. A list is an error. `npx reeval image
|
|
175
|
+
build` builds every image the plan needs.
|
|
176
|
+
|
|
177
|
+
The cell key includes the image (`from`, `dockerfile`, `install`), so Node 22 and 24 can't share
|
|
178
|
+
a cell. Environments that share an image still share one preflight build.
|
|
179
|
+
|
|
180
|
+
## Secrets and egress
|
|
181
|
+
|
|
182
|
+
Secrets come from a `defineConfig` or `defineEnvironment` callback, never literals:
|
|
183
|
+
|
|
184
|
+
```ts
|
|
185
|
+
defineEnvironment('gemini', ({ secret }) => ({
|
|
186
|
+
env: {
|
|
187
|
+
GEMINI_API_KEY: secret.fromEnv('GEMINI_API_KEY', {
|
|
188
|
+
hosts: secret.hosts.gemini,
|
|
189
|
+
}),
|
|
190
|
+
},
|
|
191
|
+
}))
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
With `hosts`, the sandbox only sees a placeholder and reeval's gateway swaps in the real value on
|
|
195
|
+
HTTPS requests to those hosts. `{ to: 'checks' }` keeps it out of the agent's own environment
|
|
196
|
+
(only `c.exec` and setup steps get it). Without `hosts`, a process a setup step leaves running (a
|
|
197
|
+
server the checks call) holds the real value while the agent works, where the agent, as the same
|
|
198
|
+
user, can read it: broker it with `hosts`, or start the process from `c.exec` after the turn. The
|
|
199
|
+
`setup-process-secret` remark flags it.
|
|
200
|
+
|
|
201
|
+
`egress: 'proxy' | 'allow-all' | 'deny-all' | { allow: [hosts] }` limits where the sandbox may
|
|
202
|
+
connect (the harness's API, brokered hosts and MCP hosts are always allowed; `DEFAULT_EGRESS_ALLOW`
|
|
203
|
+
is npm, Yarn, PyPI and GitHub). A blocked request doesn't fail the attempt; it's recorded, and your
|
|
204
|
+
checks decide. The record's `egress.blocked` is what the policy refused; a name that didn't resolve
|
|
205
|
+
(`metadata.google.internal` outside Google Cloud) is in `egress.unresolved` instead.
|
|
206
|
+
|
|
207
|
+
The default, `'proxy'`, reaches any host over HTTP(S) through the egress gateway and nothing
|
|
208
|
+
else: SSH, a database driver or a raw TCP/UDP client gets `Network is unreachable` (and the run's
|
|
209
|
+
`proxy-only-egress` remark). Set `egress: 'allow-all'` on just the evals that need that; it keeps
|
|
210
|
+
the container's network, so such traffic goes out unrecorded (the record's `egress.partial`).
|
|
211
|
+
|
|
212
|
+
`egress: { setup, agent }` gives each phase its own policy, in `docker()` and `podman()` only:
|
|
213
|
+
`setup` covers the files and setup steps, `agent` the harness's start, every turn and every check.
|
|
214
|
+
`{ setup: 'allow-all', agent: 'deny-all' }` lets setup install dependencies while the agent
|
|
215
|
+
reaches only its API and the automatic hosts. A phase left out keeps the policy it inherits, and
|
|
216
|
+
the record tags each traffic entry with its `phase`. A connection setup leaves open (a background
|
|
217
|
+
process's tunnel) to a host the agent's policy refuses is closed as the agent starts, and the
|
|
218
|
+
`setup-tunnel-closed` remark flags it.
|