reeval 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +841 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
- package/dist/adapter-Di9H2etg.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
- package/dist/agent-B-6GFDqA.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
- package/dist/agents-KBTYhSSh.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
- package/dist/app-files-Cmh9FGr5.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
- package/dist/assemble-DNVBTBp_.js.map +1 -0
- package/dist/attach-EbDGKGTO.js +152 -0
- package/dist/attach-EbDGKGTO.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
- package/dist/attempt-host-DQvynDHk.js.map +1 -0
- package/dist/boat-Bzz8L-KQ.js +701 -0
- package/dist/boat-Bzz8L-KQ.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
- package/dist/boat-Dek8ftIt.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/bundle-DZstHDeW.js +718 -0
- package/dist/bundle-DZstHDeW.js.map +1 -0
- package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
- package/dist/capture-BkbPsfej.js.map +1 -0
- package/dist/capture-CUAvk9KM.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
- package/dist/cli-BgBBuxxJ.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
- package/dist/cli-spec-B5L9n-DV.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-DMEYR3MA.js +807 -0
- package/dist/client-DMEYR3MA.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
- package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
- package/dist/command-D_w_kDpI.js.map +1 -0
- package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
- package/dist/command-Dwlh0Vhu.js.map +1 -0
- package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
- package/dist/commands-Cuf57OYb.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
- package/dist/config-C0RKt00M.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
- package/dist/db-3o8s2EyD.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
- package/dist/db-ErnJ-gMD.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
- package/dist/docker-DrMnpL84.js +1 -0
- package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
- package/dist/doctor-B7utCSdh.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
- package/dist/environment-C0Gg8kyN.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
- package/dist/eval-source-CsW5VIXS.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
- package/dist/evaluate-BHQyAszQ.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
- package/dist/fake-bootstrap-VpisF1WE.js +15 -0
- package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
- package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
- package/dist/files-CAzVG3Rr.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
- package/dist/files-D-Lir4aE.js +1 -0
- package/dist/files-DRlJ0Xr2.js +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
- package/dist/files-qjeqXnfE.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/harness-install-Dxho1yaU.js +53 -0
- package/dist/harness-install-Dxho1yaU.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
- package/dist/images-Ch1y1t6_.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
- package/dist/init-C2HqCwnt.js.map +1 -0
- package/dist/init-CM9xwyre.js +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
- package/dist/ink-list-BbKXElcQ.js.map +1 -0
- package/dist/inputs-5UIDpjz-.js +190 -0
- package/dist/inputs-5UIDpjz-.js.map +1 -0
- package/dist/inputs-load-C82HOKnb.js +1 -0
- package/dist/inputs-load-CGPyigMC.js +95 -0
- package/dist/inputs-load-CGPyigMC.js.map +1 -0
- package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
- package/dist/jiti-radm1Soi.js +52 -0
- package/dist/jiti-radm1Soi.js.map +1 -0
- package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
- package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
- package/dist/layout-Czh5wOpz.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
- package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
- package/dist/live-C9Et_BVT.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
- package/dist/live-DGgIrVw0.js.map +1 -0
- package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
- package/dist/machine-service-2lIv5UWF.js +1 -0
- package/dist/machine-service-Ct_1XMMi.js +520 -0
- package/dist/machine-service-Ct_1XMMi.js.map +1 -0
- package/dist/machines-BnquHcBZ.js +120 -0
- package/dist/machines-BnquHcBZ.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
- package/dist/model-Bcy4Qjmm.js +1 -0
- package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
- package/dist/model-DRa7yv6K.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
- package/dist/model-E5X8YBJM.js.map +1 -0
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
- package/dist/node-BjuhqQwa.js.map +1 -0
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
- package/dist/options-C989kuVE.js +1 -0
- package/dist/own-reads-CYmBlrHK.js +15 -0
- package/dist/own-reads-CYmBlrHK.js.map +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
- package/dist/plan-BcT4G5jj.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
- package/dist/plugins-cC0sqoDA.js.map +1 -0
- package/dist/preflight-BMVDBcux.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
- package/dist/preflight-DySHXeFJ.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
- package/dist/prepare-nbX1CXXg.js +409 -0
- package/dist/prepare-nbX1CXXg.js.map +1 -0
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/providers-BkLGzaSY.js +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
- package/dist/providers-bNgfKa6m.js.map +1 -0
- package/dist/query-B2SQddoV.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
- package/dist/record--vHreTLk.js.map +1 -0
- package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
- package/dist/redact-t-L3VGy8.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
- package/dist/remarks-CYFOsnz5.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
- package/dist/resolve-Bm9ZyFR-.js +1 -0
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
- package/dist/resolve-DorSHDBs.js.map +1 -0
- package/dist/resources-DqdwihhT.js +62 -0
- package/dist/resources-DqdwihhT.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
- package/dist/results-DhMgRkR0.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
- package/dist/resume-C8Ihfy-C.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/routes-Cp4gAjBZ.js +427 -0
- package/dist/routes-Cp4gAjBZ.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
- package/dist/runtime-CPUH0Vk5.js.map +1 -0
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
- package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
- package/dist/server-ZaMzUNmV.js +1 -0
- package/dist/services-BG8xuU82.js +39 -0
- package/dist/services-BG8xuU82.js.map +1 -0
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
- package/dist/source-BcBDj-R0.js +1 -0
- package/dist/src-B2eHTCnp.js +3 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
- package/dist/src-Bb3k9Kn_.js.map +1 -0
- package/dist/src-BfOlH_Bu.js +2 -0
- package/dist/src-BuGc40IF.js +2 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C2K5_afU.js +2 -0
- package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
- package/dist/src-C3OhA5pw.js.map +1 -0
- package/dist/src-CSVJjghq.js +4 -0
- package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
- package/dist/src-CWfy8ct8.js.map +1 -0
- package/dist/src-CcrQQB0_.js +2 -0
- package/dist/src-CotUxxQV.js +2 -0
- package/dist/src-D-PwBaBB.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
- package/dist/src-DNWiU__5.js.map +1 -0
- package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
- package/dist/src-DwcTn4JY.js.map +1 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
- package/dist/src-c3ZQu7Uu.js.map +1 -0
- package/dist/src-jYqNkT-F.js +1 -0
- package/dist/start-Db5VRcRZ.js +660 -0
- package/dist/start-Db5VRcRZ.js.map +1 -0
- package/dist/state-CKIjZZRK.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/suite-CH1aaAbs.js +2 -0
- package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
- package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
- package/dist/summary-BK5Sa_N0.js.map +1 -0
- package/dist/sync-DUi_P2UJ.js +559 -0
- package/dist/sync-DUi_P2UJ.js.map +1 -0
- package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
- package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
- package/dist/tart-BQ_Ap0l4.js +508 -0
- package/dist/tart-BQ_Ap0l4.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-D1RttYfM.js +148 -0
- package/dist/tart-bake-D1RttYfM.js.map +1 -0
- package/dist/tart-bake-DjFcsG3N.js +1 -0
- package/dist/tart-cli-BOcluha1.js +1 -0
- package/dist/tart-cli-gA9xEaQL.js +89 -0
- package/dist/tart-cli-gA9xEaQL.js.map +1 -0
- package/dist/tart-hosts-CHNrfV6u.js +313 -0
- package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
- package/dist/tart-hosts-CWse-ASo.js +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
- package/dist/tart-images-CLBjfF9u.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-krsmEo44.js +433 -0
- package/dist/tool-class-krsmEo44.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
- package/dist/ui-DKYjTVpn.js +1455 -0
- package/dist/ui-DKYjTVpn.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/vercel-models-Ci33llXz.js +21 -0
- package/dist/vercel-models-Ci33llXz.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +32 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
- package/dist/views-DCjtw1jF.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
- package/dist/worker-entry-DrAYYt0M.js.map +1 -0
- package/dist/workspace-CnRbV8cM.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
- package/docker/Dockerfile +1 -1
- package/package.json +15 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-BjLqFJIZ.js.map +0 -1
- package/dist/boat-DSNgZt-t.js +0 -322
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/capture-Ccf0Hmyx.js.map +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/command-Ff0T-1Hi.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker--mukta3L.js +0 -1
- package/dist/docker-DG_uFgn2.js +0 -246
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/ink-list-BpfidQ1h.js.map +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CgYbO4Te.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/model-cgt5uvu8.js.map +0 -1
- package/dist/node-D7pAU--0.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prepare-DQEDXoDp.js +0 -268
- package/dist/prepare-DQEDXoDp.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/record-6IUzwP9K.js.map +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/runtime-Xm9oeC7o.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/summary-DDy9DfyJ.js.map +0 -1
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js +0 -586
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js +0 -344
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# Environments, sandboxes, secrets and egress
|
|
2
|
+
|
|
3
|
+
## Contents
|
|
4
|
+
|
|
5
|
+
- Environments: skills, files, rules, MCP, setup
|
|
6
|
+
- Fixture directories and `git()` workspaces
|
|
7
|
+
- Skills and composition
|
|
8
|
+
- Sandboxes
|
|
9
|
+
- Secrets and egress
|
|
10
|
+
|
|
11
|
+
## Environments: skills, files, rules, MCP, setup
|
|
12
|
+
|
|
13
|
+
What the agent runs *in*, per attempt: the code the workspace starts as, skills, MCP servers,
|
|
14
|
+
files, rules, env vars, secrets and setup steps. Pass a spec, or a synchronous callback that
|
|
15
|
+
destructures the helpers it needs:
|
|
16
|
+
|
|
17
|
+
```ts
|
|
18
|
+
// evals/envs.ts
|
|
19
|
+
import { defineEnvironment } from 'reeval'
|
|
20
|
+
|
|
21
|
+
export const houseStyle = defineEnvironment('house-style', ({ skill }) => ({
|
|
22
|
+
// a dir with SKILL.md, relative to this file
|
|
23
|
+
skills: [skill.local('../skills/house-style')],
|
|
24
|
+
}))
|
|
25
|
+
|
|
26
|
+
export const withDocs = houseStyle.extendEnvironment(
|
|
27
|
+
'with-docs',
|
|
28
|
+
({ file, mcp }) => ({
|
|
29
|
+
// written into the workspace
|
|
30
|
+
files: { 'docs/api.md': file('../docs/api.md') },
|
|
31
|
+
// AGENTS.md (CLAUDE.md for Claude Code)
|
|
32
|
+
rules: file('../rules/house.md'),
|
|
33
|
+
// or mcp.http(url) / mcp.sse(url) (not on OpenCode, Goose, pi or Codex)
|
|
34
|
+
mcp: { docs: mcp.stdio('npx', ['-y', '@acme/docs-mcp@1.0.0']) },
|
|
35
|
+
// every attempt, before the agent
|
|
36
|
+
setup: [{ run: 'npm ci', timeout: '5m' }],
|
|
37
|
+
}),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
// Start from existing code: a repo, or one directory of it, at a pinned ref
|
|
41
|
+
export const app = defineEnvironment('acme-app', ({ git }) => ({
|
|
42
|
+
workspace: git('https://github.com/acme/app', {
|
|
43
|
+
ref: 'v2.3.0',
|
|
44
|
+
path: 'examples/basic',
|
|
45
|
+
}),
|
|
46
|
+
setup: ['npm ci'],
|
|
47
|
+
}))
|
|
48
|
+
|
|
49
|
+
// Or seed a fixture directory: a plain object of file() entries, so spread
|
|
50
|
+
// and override it
|
|
51
|
+
export const notes = defineEnvironment('acme-notes', ({ directory }) => ({
|
|
52
|
+
files: { ...directory('../fixture'), 'README.md': 'something' },
|
|
53
|
+
setup: [{ run: 'npm ci --no-audit --no-fund', timeout: '5m' }],
|
|
54
|
+
}))
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Fixture directories and `git()` workspaces
|
|
58
|
+
|
|
59
|
+
Use `directory(path, { into?, glob?, defaultIgnores? })` for a fixture directory instead of
|
|
60
|
+
mapping a file list through `file()`. It skips `.git`, `node_modules/`, `dist/`, top-level
|
|
61
|
+
`build/`, caches and editor folders (`DEFAULT_DIRECTORY_IGNORES`; `defaultIgnores: false` keeps
|
|
62
|
+
them, except `.git`). `glob` is ordered gitignore-style patterns where the last match wins and
|
|
63
|
+
`!` excludes: `glob: ['src/**', 'package.json', '!**/*.test.ts']`.
|
|
64
|
+
|
|
65
|
+
Executable files stay executable; empty directories aren't seeded.
|
|
66
|
+
|
|
67
|
+
`workspace: git(repo, { ref, path })` takes a URL, `github:owner/repo`, a GitHub `/tree/` URL or
|
|
68
|
+
a local path (relative to the file). reeval fetches it on the host before any attempt, with the
|
|
69
|
+
host's git credentials or `token: 'GITHUB_TOKEN'` (a host variable's name; never a token in the
|
|
70
|
+
URL or the eval). It copies the tree in before `files` (which win) and `setup`, and records the
|
|
71
|
+
commit and tree hash.
|
|
72
|
+
|
|
73
|
+
`init: true` adds a `.git` with one commit of the tree (no remote), for agents that expect a
|
|
74
|
+
repo. Pin `ref` to a tag or full SHA; a branch warns. The agent's diff starts after setup, so the
|
|
75
|
+
repo never counts as its change.
|
|
76
|
+
|
|
77
|
+
A bad ref, path, URL or token fails only the attempts that use it (infra error, exit 3).
|
|
78
|
+
Submodules and LFS files aren't fetched.
|
|
79
|
+
|
|
80
|
+
## Skills and composition
|
|
81
|
+
|
|
82
|
+
Skills: `skill.local | git | npm | registry | inline` (pin git refs; `placement: 'workspace'`
|
|
83
|
+
writes them into the repo's skill directory). A git, npm or plugin source URL with a password or
|
|
84
|
+
token in it is refused: reeval fetches on the host with its own git and npm credentials.
|
|
85
|
+
`extendEnvironment` is the only composition; the child wins on shared keys and **replaces**
|
|
86
|
+
`sandbox` entirely when it sets one (install is not merged onto the parent image).
|
|
87
|
+
|
|
88
|
+
A skill is a directory whose `SKILL.md` has YAML frontmatter
|
|
89
|
+
(`name`, `description`; quote values containing `: `). Baking (`from`, `install`, `dockerfile`)
|
|
90
|
+
is `sandbox: docker({ … })` on the environment.
|
|
91
|
+
|
|
92
|
+
Plugins: `plugins: [plugin.local('../plugins/review', { only: ['skills', 'mcp'] })]` (also
|
|
93
|
+
`plugin.git(repo, { ref, path })`, `plugin.npm(spec)` and
|
|
94
|
+
`plugin.marketplace('name@github:owner/repo', { ref })` for an entry in a Claude Code
|
|
95
|
+
marketplace, or `'name@npm:<spec>'`; only a marketplace in a local directory may name `file://`
|
|
96
|
+
entries). A plugin is Claude Code's layout:
|
|
97
|
+
`.claude-plugin/plugin.json` plus `skills/`, `agents/`, `commands/`, `hooks/hooks.json`,
|
|
98
|
+
`.mcp.json`, `rules/`. Its MCP servers join `mcp` as `<plugin>:<server>` (call them from the
|
|
99
|
+
fake agent as `{ mcp: 'review:docs/<tool>' }`), and it lands in `~/.reeval/plugins/<name>/`. Claude
|
|
100
|
+
Code and Grok Build load every part but `rules` (narrow with `only`), namespaced
|
|
101
|
+
(`review:reviewer`), and start the plugin's MCP servers themselves, so a plugin server reading a
|
|
102
|
+
host variable is refused there: declare it in `mcp`. Antigravity loads every part with
|
|
103
|
+
`agy plugin install`, and refuses such a server too; Codex loads `skills` and `mcp` the same way
|
|
104
|
+
(and refuses a plugin with ChatGPT apps). Cursor loads every part flat: rules, subagents, commands
|
|
105
|
+
and hooks unpacked into the workspace's `.cursor/` (left out of the diff), skills into
|
|
106
|
+
`~/.agents/skills`, so two items with one name clash; Goose takes `skills` and `mcp` beside the
|
|
107
|
+
eval's own, so a clashing name is refused too. OpenCode loads skills, MCP servers, and subagents
|
|
108
|
+
and commands (unpacked into `.opencode/`, frontmatter cut to `description`); pi skills, MCP servers
|
|
109
|
+
(as `<plugin>_<server>`) and commands (as prompt templates in `.pi/prompts/`); fx skills and MCP
|
|
110
|
+
servers only. A part a harness doesn't load, or a hook event it doesn't run, is refused at plan
|
|
111
|
+
time (`--dry-run` exits 2 naming the harness): narrow with `only`. `build: 'npm ci && npm run build'` (or `{ run, cwd: '.' }` to run
|
|
112
|
+
from the source's root) builds the plugin in a cached Docker image layer, as root with no
|
|
113
|
+
secrets, without the source's `.git` (pass a version in its commands instead of `git describe`);
|
|
114
|
+
only `path` is placed, so the output must land inside it. Docker only. Its content is fixed when the
|
|
115
|
+
image is prepared: an npm spec re-resolving or a local plugin edited mid-run waits for the next run. A server run with
|
|
116
|
+
`npx`/`uvx` needs its registry in `egress`, or a build that installs it.
|
|
117
|
+
|
|
118
|
+
A named env is a total override: no skills, files or sandbox inherit from the config
|
|
119
|
+
environment.
|
|
120
|
+
|
|
121
|
+
## Sandboxes
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
// reeval's Node image with the agent CLI
|
|
125
|
+
docker()
|
|
126
|
+
// baked once as a cached layer; no secrets
|
|
127
|
+
docker({ install: ['npm i -g firebase-tools@14'] })
|
|
128
|
+
// a Debian or Ubuntu base; reeval adds Node and the agent
|
|
129
|
+
docker({ from: 'python:3.12-slim' })
|
|
130
|
+
// ARG REEVAL_BASE and FROM ${REEVAL_BASE} to layer on reeval's image
|
|
131
|
+
docker({ dockerfile: './sandbox/Dockerfile' })
|
|
132
|
+
// limits; defaults 4g, 2 and 1024
|
|
133
|
+
docker({ memory: '8g', cpus: 4, pids: 4096 })
|
|
134
|
+
// swap on top; default the same as memory, 0 for none
|
|
135
|
+
docker({ memory: '4g', swap: 0 })
|
|
136
|
+
// opt in to apt-get at run time (default: user reeval)
|
|
137
|
+
docker({ root: true })
|
|
138
|
+
// the same sandbox and options on rootless Podman 4+ (cell id `podman`)
|
|
139
|
+
podman({ from: 'python:3.12-slim' })
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Use `podman()` only where the user runs Podman instead of Docker: it takes every `docker()`
|
|
143
|
+
option and confines the container the same way (bridge network, Docker's seccomp profile).
|
|
144
|
+
|
|
145
|
+
`tart({ image })` (an OCI macOS image or a local Tart VM) runs each attempt in a cloned macOS VM
|
|
146
|
+
on an Apple Silicon Mac, at most two at once. It needs `egress: 'allow-all'`: under the default
|
|
147
|
+
`proxy` or a restricted policy the plan exits 2. Of the real harnesses only `cursor()` runs in it
|
|
148
|
+
so far (any other exits 2 at plan time), and it needs Node 22.21 or later in the image; the fake
|
|
149
|
+
harness runs too.
|
|
150
|
+
`reeval run` never pulls the image: one `tart list` doesn't show exits 2, naming `tart pull
|
|
151
|
+
<image>`. Run `npx reeval image build` first, which pulls every `tart()` image the plan needs (and
|
|
152
|
+
refuses, exit 3, when the disk can't hold them even once Tart prunes its cache). Pin a digest
|
|
153
|
+
(`ghcr.io/cirruslabs/macos-sequoia-base@sha256:…`): Cirrus Labs publishes only `latest`, so a
|
|
154
|
+
version tag isn't there to pin. `:latest` or no tag warns, and each attempt records the digest it
|
|
155
|
+
cloned in `vmImage`.
|
|
156
|
+
|
|
157
|
+
```ts
|
|
158
|
+
import { boat, createBoat } from 'reeval/sandbox/boat'
|
|
159
|
+
// one boat.dev VM per attempt, over SSH (cell id `boat`, or `boat:large`)
|
|
160
|
+
defineEnvironment('vm', { sandbox: boat({ type: 'large' }), egress: 'allow-all' })
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
`boat()` needs `npm i @boatdev/sdk`, `BOAT_API_KEY` (or `createBoat({ apiKey, prefix })` for
|
|
164
|
+
another account) and an OpenSSH client, and plans only under `egress: 'allow-all'`; reeval
|
|
165
|
+
refuses the other policies at plan time. It takes no image options: put the
|
|
166
|
+
toolchain's installs in the environment's `setup`.
|
|
167
|
+
|
|
168
|
+
The agent, setup and `c.exec` run as `reeval` (home `/home/reeval`) with no capabilities; builds
|
|
169
|
+
run as root. `root: true` or `capabilities` print a warning at the start of every run, so prefer
|
|
170
|
+
`install`. A process killed at the memory limit makes the attempt an infra error
|
|
171
|
+
(`error.resource.cause: 'oom'`), not a failed check: raise `memory` rather than loosening it.
|
|
172
|
+
|
|
173
|
+
Put a specific machine on `defineEnvironment`, or `sandbox` on `defineConfig`, a profile, group
|
|
174
|
+
or eval as the default when the resolved env omits one. A list is an error. `npx reeval image
|
|
175
|
+
build` builds every image the plan needs.
|
|
176
|
+
|
|
177
|
+
The cell key includes the image (`from`, `dockerfile`, `install`), so Node 22 and 24 can't share
|
|
178
|
+
a cell. Environments that share an image still share one preflight build.
|
|
179
|
+
|
|
180
|
+
## Secrets and egress
|
|
181
|
+
|
|
182
|
+
Secrets come from a `defineConfig` or `defineEnvironment` callback, never literals:
|
|
183
|
+
|
|
184
|
+
```ts
|
|
185
|
+
defineEnvironment('gemini', ({ secret }) => ({
|
|
186
|
+
env: {
|
|
187
|
+
GEMINI_API_KEY: secret.fromEnv('GEMINI_API_KEY', {
|
|
188
|
+
hosts: secret.hosts.gemini,
|
|
189
|
+
}),
|
|
190
|
+
},
|
|
191
|
+
}))
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
With `hosts`, the sandbox only sees a placeholder and reeval's gateway swaps in the real value on
|
|
195
|
+
HTTPS requests to those hosts. `{ to: 'checks' }` keeps it out of the agent's own environment
|
|
196
|
+
(only `c.exec` and setup steps get it). Without `hosts`, a process a setup step leaves running (a
|
|
197
|
+
server the checks call) holds the real value while the agent works, where the agent, as the same
|
|
198
|
+
user, can read it: broker it with `hosts`, or start the process from `c.exec` after the turn. The
|
|
199
|
+
`setup-process-secret` remark flags it.
|
|
200
|
+
|
|
201
|
+
`egress: 'proxy' | 'allow-all' | 'deny-all' | { allow: [hosts] }` limits where the sandbox may
|
|
202
|
+
connect (the harness's API, brokered hosts and MCP hosts are always allowed; `DEFAULT_EGRESS_ALLOW`
|
|
203
|
+
is npm, Yarn, PyPI and GitHub). A blocked request doesn't fail the attempt; it's recorded, and your
|
|
204
|
+
checks decide. The record's `egress.blocked` is what the policy refused; a name that didn't resolve
|
|
205
|
+
(`metadata.google.internal` outside Google Cloud) is in `egress.unresolved` instead.
|
|
206
|
+
|
|
207
|
+
The default, `'proxy'`, reaches any host over HTTP(S) through the egress gateway and nothing
|
|
208
|
+
else: SSH, a database driver or a raw TCP/UDP client gets `Network is unreachable` (and the run's
|
|
209
|
+
`proxy-only-egress` remark). Set `egress: 'allow-all'` on just the evals that need that; it keeps
|
|
210
|
+
the container's network, so such traffic goes out unrecorded (the record's `egress.partial`).
|
|
211
|
+
|
|
212
|
+
`egress: { setup, agent }` gives each phase its own policy, in `docker()` and `podman()` only:
|
|
213
|
+
`setup` covers the files and setup steps, `agent` the harness's start, every turn and every check.
|
|
214
|
+
`{ setup: 'allow-all', agent: 'deny-all' }` lets setup install dependencies while the agent
|
|
215
|
+
reaches only its API and the automatic hosts. A phase left out keeps the policy it inherits, and
|
|
216
|
+
the record tags each traffic entry with its `phase`. A connection setup leaves open (a background
|
|
217
|
+
process's tunnel) to a host the agent's policy refuses is closed as the agent starts, and the
|
|
218
|
+
`setup-tunnel-closed` remark flags it.
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# Evals: the attempt API, judges and matchers
|
|
2
|
+
|
|
3
|
+
## Contents
|
|
4
|
+
|
|
5
|
+
- Cases: `cases`, `caseKey`, `where`, and `gate: 'observe'`
|
|
6
|
+
- The attempt: `c.send`, `c.exec`, `c.serve`, files
|
|
7
|
+
- The judge: inputs, input size, errors
|
|
8
|
+
- Trustworthy evals
|
|
9
|
+
- Matchers
|
|
10
|
+
|
|
11
|
+
## Cases: `cases`, `caseKey`, `where`, and `gate: 'observe'`
|
|
12
|
+
|
|
13
|
+
- `evaluate(name, [options,] fn)` and `group(name, [options,] fn)`; also `.skip`, `.only`
|
|
14
|
+
(an error in CI), `.todo`. The eval id is `file › group › name`; renaming an eval restarts its history.
|
|
15
|
+
- `evaluate(name, { cases: [...] }, async (c, item) => …)` runs one eval over a list: one pass
|
|
16
|
+
rate over the cases and, with `compare`, one lift paired by case, which `minLift` gates (not
|
|
17
|
+
each case's). Prefer it to separate evals when the cases are one population: six cases at
|
|
18
|
+
`runs: 1` can reach a lift verdict where each case alone is within noise. Fewer than six
|
|
19
|
+
discordant cases shows only a direction (`insufficient` in `reeval query case_lift`), so
|
|
20
|
+
aim for six cases or more; the plan hints when there are fewer. No `runs: { bestOf }` beside it.
|
|
21
|
+
`gate: { passRate }` gates each cell's rate over the cases (the mean of each case's pass@1), so
|
|
22
|
+
one failing case of three passes `passRate: 0.6`; under `'all'` every case must pass, so a
|
|
23
|
+
failing case fails the eval. It is one eval: one row in `--dry-run` and the run output (a
|
|
24
|
+
case table under it when it fails, or with `-v`), one entry in the record with its cases
|
|
25
|
+
inside, counted once in the tally. The failing case still records `fail`.
|
|
26
|
+
- For evals that pass or fail on their own, don't use `cases`: write a `for` loop that calls
|
|
27
|
+
`evaluate` once per item, with the item in the name.
|
|
28
|
+
- Each case has a key, which names its row: a string or number is its own key, an object its
|
|
29
|
+
`id`, else its `name`; otherwise pass `caseKey: (item) => item.slug`. Keys are unique, at most
|
|
30
|
+
80 characters, with no `#` or `›`. `cases`, `caseKey` and `where` go on `evaluate`, never a
|
|
31
|
+
group or the config.
|
|
32
|
+
- Porting `evaluate.each(items)('name: %s', opts, fn)`: write `evaluate('name', { cases: items,
|
|
33
|
+
...opts }, fn)` with the old option names made plural (`harnesses`, `environments`,
|
|
34
|
+
`compare.control`). The old runs then read as this eval, keyed by what `%s` or `$id` held, so
|
|
35
|
+
`reeval compare` lines them up; a `caseKey` that changes a key starts that case's history again.
|
|
36
|
+
- `where: ({ case, environment, harness }) => boolean` (TypeScript only) picks which case runs
|
|
37
|
+
on which environment, at plan time, from each environment's `traits` (`defineEnvironment('next',
|
|
38
|
+
{ traits: { stack: 'next' } })`; never shown to the agent). Reading a trait no environment
|
|
39
|
+
defines is an error; with `compare` it sees the control, and its answer holds for the variants.
|
|
40
|
+
`--dry-run` notes `excluded on vite-app: q2`, the run's case table shows `–` there, and each case's record keeps its own in
|
|
41
|
+
`excluded`. A cell left with no cases is an error: drop that environment instead.
|
|
42
|
+
- `where` says which pairs fit, the same every run; to run only some environments today, use
|
|
43
|
+
`reeval run --trait stack=next` (repeat to AND) or a profile's
|
|
44
|
+
`limit: { traits: { stack: ['next', 'vite'] } }`. Both narrow after `where`, by the control's
|
|
45
|
+
traits under `compare`, and `--dry-run` notes `--trait stack=next leaves out vite-app`. Never
|
|
46
|
+
read a flag or a variable in `where` to do this.
|
|
47
|
+
- `evaluate(name, { gate: 'observe' }, fn)` records what agents do (with `c.observe` and
|
|
48
|
+
checks) and fails only on run or infra errors. Its block reads `ran`. `compare.minLift` and
|
|
49
|
+
`runs: { bestOf }` are errors beside it; `compare: { gate: 'observe' }` ungates a compare's cells.
|
|
50
|
+
Any gate but `'all'`, `{ passRate }` or `'observe'` is an error.
|
|
51
|
+
A comment on an eval describes the current check (why a canary fails, which path it reads),
|
|
52
|
+
not what used to be true or which product a prompt was ported from.
|
|
53
|
+
|
|
54
|
+
## The attempt: `c.send`, `c.exec`, `c.serve`, files
|
|
55
|
+
|
|
56
|
+
- `await c.send(prompt)` runs one agent turn and returns a `Turn` (`text`, `status`, `diff`,
|
|
57
|
+
`transcript`). Call it again for multi-turn;
|
|
58
|
+
`c.judge(q, { on: firstTurn, task: true, output: true })` grades an earlier one.
|
|
59
|
+
- `await c.exec(cmd)` returns `{ exitCode, stdout, stderr }`; `c.exec(cmd, { background: true })`
|
|
60
|
+
starts it without waiting and returns `{ kill() }`. `{ env: { NAME: 'value' } }` sets variables
|
|
61
|
+
for that command only, over the environment's (in the background too); a value named like a
|
|
62
|
+
secret (`*_TOKEN`, `*_KEY`, …) is masked in everything reeval writes, and a brokered secret's
|
|
63
|
+
host value goes in as its placeholder. `c.exec(cmd, { retries: 2, interval: 2000 })`
|
|
64
|
+
runs a command that exits non-zero again, up to 2 more times 2 s apart, and resolves with the
|
|
65
|
+
last try: for a check that can fail on a transient, such as a test calling a model. Any non-zero
|
|
66
|
+
exit retries (reeval doesn't read the output), and a last failed try is a failed check. `await c.serve(cmd, port?)` is that plus a
|
|
67
|
+
port (`PORT` / `$PORT` on a bash string), and returns once `cmd` listens, on `PORT` or the one
|
|
68
|
+
port it picked itself. A `port` something already listens on (the agent's dev server) throws:
|
|
69
|
+
pass none and serve on `PORT`. `c.capture()` screenshots it: see
|
|
70
|
+
[capture.md](capture.md).
|
|
71
|
+
- `await c.readFile(path)` the text or `null` (`~/` for home). `await c.writeFile(path, text)`
|
|
72
|
+
places a file after a turn (a hidden test the agent never saw) instead of seeding it in
|
|
73
|
+
`files`; later turns' `diff` leaves it out but shows any change the agent makes to it, even
|
|
74
|
+
under `.gitignore`. Don't place grading files with a `c.exec` heredoc: the next turn's diff
|
|
75
|
+
(and the judge) would count them as the agent's. The sandbox and session start on the first
|
|
76
|
+
call that needs them (`c.send`, `c.exec`, `c.readFile`, `c.writeFile`, `c.serve`,
|
|
77
|
+
`c.capture`), so a check before any turn sees what the environment set up.
|
|
78
|
+
`c.attempt.cell.role` is
|
|
79
|
+
`'control'` or `'variant'`.
|
|
80
|
+
- `c.observe(name, value)` records a JSON value about the attempt (which ORM it installed, how many
|
|
81
|
+
files it touched) that never affects pass or fail; the run output tallies it per cell and
|
|
82
|
+
`reeval query attempt_values` lists it. `c.judge()` and questions see it only with
|
|
83
|
+
`observations: true` (or a list of names). Use it for what you want to count, `expect` for what
|
|
84
|
+
must hold.
|
|
85
|
+
|
|
86
|
+
## The judge: inputs, input size, errors
|
|
87
|
+
|
|
88
|
+
- `await c.judge(question, options)`: the judge sees the question and only the inputs the call
|
|
89
|
+
turns on. Every input is off until the call sets it, and a call needs at least one (`true`, a
|
|
90
|
+
mode or a non-empty list); there are no defaults, and the config, profile, group and eval
|
|
91
|
+
`judge` take only `model`, `threshold`, `providerOptions`, `maxConcurrent` and `gateway`.
|
|
92
|
+
With none, `c.judge()` throws before any model call, and TypeScript refuses it.
|
|
93
|
+
|
|
94
|
+
| Input | Shows |
|
|
95
|
+
|---|---|
|
|
96
|
+
| `task: true` | the graded turn's prompt |
|
|
97
|
+
| `output: true` | the graded turn's final message, or `on`'s value |
|
|
98
|
+
| `files: true` | `status path` lines for the changed files, under `diff`'s patterns |
|
|
99
|
+
| `diff: true` / `['src/**', '!**/*.test.ts']` | the diff since the eval started (gitignore-style, last match wins) |
|
|
100
|
+
| `contents: ['README.md', 'src/**']` | files whole, changed or not; they leave the diff |
|
|
101
|
+
| `transcript: true` / `'tools'` / `'full'` | the turn's transcript: clipped, tool calls only, everything |
|
|
102
|
+
| `checks: true` / `['npm test']` | `c.exec` results, or those commands' |
|
|
103
|
+
| `artifacts: true` / `[shot, 'home']` | captures, sent as file parts |
|
|
104
|
+
|
|
105
|
+
Pick the inputs the question needs, usually one or two: a question about code reads `diff` or
|
|
106
|
+
`contents`, one about the agent's answer `output`, one about how it worked `transcript`, one
|
|
107
|
+
about a screenshot `artifacts`. Spell out `{ task: true, output: true, diff: true }` only when
|
|
108
|
+
the question needs all three. The diff leaves out dependencies, build output (`dist/`,
|
|
109
|
+
`.next/`, top-level `build/` and `out/`), lockfiles, `*.map`, `*.tsbuildinfo` and minified
|
|
110
|
+
files.
|
|
111
|
+
- `on` and `turns` select what `task`, `output`, `transcript` and `diff` read and show nothing alone:
|
|
112
|
+
`on: turn` grades an earlier turn, `on: value` grades any value as the output (with
|
|
113
|
+
`output: true`), and `turns: 'all'` adds every turn before the graded one. `judge-input.json`
|
|
114
|
+
records each call's `stateOptions` (every input, `false` when off) and the `state` it sent.
|
|
115
|
+
- Ask about facts in a captured picture, not whether it "looks great". Jev
|
|
116
|
+
(`typesafe-ai/jev-1.14.0`) takes PNG captures; a provider that cannot take files fails that
|
|
117
|
+
call. Prefer deterministic checks for what can be checked; use the judge for quality, with
|
|
118
|
+
plain yes/no questions that don't name one specific way of doing it.
|
|
119
|
+
- Give the environment a `.gitignore` in `files` (the project's isn't copied in): an agent that
|
|
120
|
+
runs a build otherwise fills the judge's diff with its output. reeval sends every input the
|
|
121
|
+
call turns on, uncut, and knows no model's limit: a provider error refusing it keeps the
|
|
122
|
+
provider's wording and adds one line with the input's estimated size, biggest files and the
|
|
123
|
+
inputs that were on, largest first; narrow one, e.g. `{ diff: ['src/'] }`.
|
|
124
|
+
- A failed judge call doesn't throw: `c.judge()` returns `{ verdict: 'error', score: null, error }`,
|
|
125
|
+
the later judges and checks still run, and the attempt fails as a judge error at the end, or
|
|
126
|
+
as an infra error (which `--retry-errors` reruns) when the provider never answered and nothing
|
|
127
|
+
else failed.
|
|
128
|
+
`expect(await c.judge(q, { diff: true }), name).toPass()` records each judge's result and
|
|
129
|
+
carries on.
|
|
130
|
+
|
|
131
|
+
## Trustworthy evals
|
|
132
|
+
|
|
133
|
+
Before a real run, prove each check can pass and fail with the fake harness in the eval's real
|
|
134
|
+
sandbox: an oracle `fakeHarness` that does the task right (must pass) and a canary that cheats
|
|
135
|
+
or gets it wrong (must fail). A canary that passes means a check isn't checking. Give each check
|
|
136
|
+
you rely on a canary that only it catches.
|
|
137
|
+
|
|
138
|
+
```ts
|
|
139
|
+
// reeval.config.ts: the real harness, plus two profiles that swap only the harness and judge
|
|
140
|
+
export default defineConfig({
|
|
141
|
+
harnesses: cursor({ model: 'composer-2.5' }),
|
|
142
|
+
profiles: {
|
|
143
|
+
oracle: {
|
|
144
|
+
defaults: {
|
|
145
|
+
harnesses: fakeHarness({ files: { 'greet.js': 'export const greet = …' } }),
|
|
146
|
+
judge: { model: fakeJudge({ score: 0.95 }) },
|
|
147
|
+
},
|
|
148
|
+
},
|
|
149
|
+
canary: {
|
|
150
|
+
defaults: {
|
|
151
|
+
harnesses: fakeHarness({ files: { 'greet.test.js': "test('greets', () => {})\n" } }),
|
|
152
|
+
judge: { model: fakeJudge({ score: 0.1 }) },
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
})
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
npx reeval image build --profile oracle # once: the eval's images
|
|
161
|
+
npx reeval run --profile oracle # every check passes, exit 0
|
|
162
|
+
npx reeval run --profile canary # the checks you meant fail, exit 1
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
- Set `defaults.harnesses`, never `defaults.sandbox`: the environment's `docker()` (image,
|
|
166
|
+
`install`, egress, brokered secrets) stays, so the checks run where they will for real. A
|
|
167
|
+
profile's `sandbox` only fills in for an environment that omits one, and the fake harness
|
|
168
|
+
without `layout` uses Cursor's image; `layout: 'claude-code'` (or another agent id) uses that
|
|
169
|
+
agent's image and reads rules and skills where it does.
|
|
170
|
+
- `script` steps (`{ exec }`, `{ write }`, `{ mcp }`) run inside the container as the agent, so an
|
|
171
|
+
oracle can run the real fix (`{ exec: 'node scripts/solve.mjs' }`) or call a brokered API.
|
|
172
|
+
- `fakeJudge` checks the wiring (a low score fails the check); keep the real `judge.model` on the
|
|
173
|
+
oracle, with its key, to check that a question passes good work.
|
|
174
|
+
- `fakeSandbox()` isn't the oracle's sandbox: it runs on the host, brokers nothing and enforces no
|
|
175
|
+
egress. It's for trying reeval and testing configs without Docker.
|
|
176
|
+
- An eval or group that sets its own `harnesses` keeps it (the plan notes `harnesses set by the
|
|
177
|
+
eval`), and an ungated cell's (`compare: { gate: 'observe' }`, a control's) failed checks don't change the
|
|
178
|
+
exit code: its row in the result block shows its count (`plain 0 of 4`).
|
|
179
|
+
|
|
180
|
+
Keep grading out of the
|
|
181
|
+
agent's reach: place hidden tests after the turn with `c.writeFile`, so the agent never sees
|
|
182
|
+
them, seed tests in `files` only when the agent should see them, and guard seeded files
|
|
183
|
+
with `expect(await c.readFile(path)).toBe(original)` or `expect(c).not.toHaveChanged(path)`.
|
|
184
|
+
When the behavior is the
|
|
185
|
+
outcome, judge the program's output, not the diff:
|
|
186
|
+
`c.judge(q, { on: (await c.exec('node app.js')).stdout, output: true })`. Checks run in the agent's
|
|
187
|
+
container as the same user, which suits AX evals but isn't a sealed verifier.
|
|
188
|
+
|
|
189
|
+
## Matchers
|
|
190
|
+
|
|
191
|
+
- `await expect(turn).toHaveFile(path, pattern?)`,
|
|
192
|
+
`await expect(c.exec(cmd)).toSucceed()`, `expect(verdict).toPass()`,
|
|
193
|
+
`expect(turn).toHaveSearched(query?)` / `toHaveFetched(url?)` (sync; they throw
|
|
194
|
+
`HarnessCapabilityError` when `turn.unreported` lists web events; a refused
|
|
195
|
+
search is still a `web-search` with `ok: false`; a call whose result never came has
|
|
196
|
+
`ok: null`, not `false`), plus Vitest's.
|
|
197
|
+
- Transcript matchers (sync, no `await`): `expect(turn | c).toHaveCalledSkill(name)`,
|
|
198
|
+
`toHaveCalledMcp(server, tool, input?)`, `toHaveCalledTool(name | RegExp, input?)`,
|
|
199
|
+
`toHaveRun(command?)`, `toHaveEdited(path?)`, `toHaveSearched(query?)`,
|
|
200
|
+
`toHaveFetched(url?)`. `.not` is Vitest's. `toHaveRun(string | RegExp)` matches the whole
|
|
201
|
+
command line, so `/genkit/` passes on `cat genkit.config.ts`; `toHaveRun({ program, args? })`
|
|
202
|
+
matches the program a segment (split on line breaks, `&&`, `||`, `;`, `|`, `&`) ran, by its
|
|
203
|
+
name without a directory, after `VAR=x`, `env`, `timeout`, `time`, `exec`, `sudo`, `nohup`,
|
|
204
|
+
`nice`, `command`, `if`/`then`/`do`, `bash -c`, `npx`, `bunx`, `npm exec`, `pnpm exec`/`dlx`
|
|
205
|
+
and `yarn dlx`; `args` (joined by spaces) is a substring or RegExp, e.g. `/^add\b/`.
|
|
206
|
+
`pnpm <bin>` and `npm run` aren't unwrapped, and a run counts even if its call failed.
|
|
207
|
+
`shellCommands(c | turn | c.turns)` returns those programs to count or filter. `toHaveCalledSkill`
|
|
208
|
+
is opened or loaded (a `skill` / `Skill` / `load_skill` tool, or a Read/`sed` of
|
|
209
|
+
`skills/<name>/`, on every harness), not "followed". MCP is `toHaveCalledMcp(server, tool)`
|
|
210
|
+
(the names in `mcp.<server>`); `toHaveCalledTool('mcp__<server>__<tool>')` is
|
|
211
|
+
the raw transcript name. After adapters and the normalizer, that string is
|
|
212
|
+
the same on every harness (stdio; Goose, OpenCode, pi and Codex refuse
|
|
213
|
+
`mcp.sse`). Use `toHaveEdited` / `toHaveSearched`, not
|
|
214
|
+
`toHaveCalledTool('edit')` or `'webSearch'`.
|
|
215
|
+
- Budgets: `expect(turn | c).toStayWithin({ tokens: 20_000, toolCalls: 30, duration: '5m' })`
|
|
216
|
+
(sync) caps the turns' summed tokens, tool calls (a number, or `{ bash: 5 }` per tool) and time.
|
|
217
|
+
Tokens include subagents. With `tokens`, a harness that reported no usage throws
|
|
218
|
+
`HarnessCapabilityError` (an error, not a pass). `.not`, an unknown key or a bad limit throws a
|
|
219
|
+
`TypeError` that ends the attempt as its error, not a failed check. Python:
|
|
220
|
+
`to_stay_within(tokens=, tool_calls=, duration=)`.
|
|
221
|
+
- Guard files the agent must leave alone with `expect(c).not.toHaveChanged(['test/**',
|
|
222
|
+
'package.json'])` (sync): it reads the workspace diff, so a shell or subprocess write counts
|
|
223
|
+
and a read doesn't. Strings are globs (a directory matches everything under it), RegExps work,
|
|
224
|
+
and a turn means up to that turn. `toHaveEdited` counts the files that changed while the agent
|
|
225
|
+
ran the turn (a turn only its own), so a shell write counts, but its string matches any path
|
|
226
|
+
containing it; guard with `toHaveChanged`'s globs. `toHaveEdited()` counts files that appear
|
|
227
|
+
in the workspace diff. Gitignored files don't appear there, so use `toHaveFile()` to check them.
|
|
228
|
+
- Checks don't stop the eval: a failed `expect` is recorded and the body carries on, so every
|
|
229
|
+
later check and judge still runs; the attempt fails at its end. Use `expect.require` for gates:
|
|
230
|
+
its failure ends the body there (`await expect.require(c.exec('npm run build'), 'builds')
|
|
231
|
+
.toSucceed()`, `expect.require.poll(…)` for one that waits). One the eval catches still fails
|
|
232
|
+
the attempt, and its error says the eval caught it. Only matcher failures carry on: an error
|
|
233
|
+
the body throws, a dead sandbox or a timeout ends the attempt. Wait or retry with `expect.poll`
|
|
234
|
+
(`await expect.poll(() => c.exec(cmd), { timeout: 10_000 }).toSucceed()`), which records one
|
|
235
|
+
check with its final outcome, not with `expect` in a `try` loop.
|
|
236
|
+
- **Label each requirement** with `expect`'s second argument:
|
|
237
|
+
`await expect(c, 'exports greet').toHaveFile('greet.js', /export/)`. Every matcher call
|
|
238
|
+
is recorded (`evals[].attempts[].assertions`; unlabeled ones get `<matcher> <first arg>`).
|
|
239
|
+
Changing a label changes the eval's fingerprint. Label judges too
|
|
240
|
+
(`expect(await c.judge(q, { diff: true }), 'Social OAuth').toPass()`): the label replaces the clipped
|
|
241
|
+
question in judge rates, `reeval show`, `summary.md` and the digest.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Common mistakes (and the fix)
|
|
2
|
+
|
|
3
|
+
Each row is a mistake, what reeval says and the fix.
|
|
4
|
+
|
|
5
|
+
| Mistake | reeval says | Fix |
|
|
6
|
+
|---|---|---|
|
|
7
|
+
| No `reeval.config.*` | `No reeval.config.ts … here or above. Run reeval init.` | `reeval init`, or pass `--config` |
|
|
8
|
+
| `compare` in `reeval.config.ts` | `compare can't be set in reeval.config.ts` | Put it on the group or eval |
|
|
9
|
+
| `compare: [b]` | `compare … must be an object, not a list` | `compare: { control: a, variants: [b] }` |
|
|
10
|
+
| `compare: { variants: [b] }` | `compare … has no control` | `compare: { control: a, variants: [b] }` |
|
|
11
|
+
| `compare: { baseline: a, … }` | `Unknown compare option "baseline" … Did you mean "control"?` | `compare: { control: a, … }` |
|
|
12
|
+
| `compare: { strategy: 'peer' }` | `Unknown compare option "strategy"` | Drop it. Variants already run on their own |
|
|
13
|
+
| `environments` beside `compare` | `… sets both environments and compare` | Move the environment into `compare.control` |
|
|
14
|
+
| `evaluate.each(cases)(name, fn)` | `evaluate.each is not a function` | `evaluate(name, { cases }, async (c, item) => …)` |
|
|
15
|
+
| `cases` for items that should each pass or fail alone | One failing case fails the whole eval | A loop: `for (const item of items) evaluate(nameOf(item), fn)` |
|
|
16
|
+
| `where` without `cases`, or on a group | `… sets where without cases` / `Unknown option "where" in group(…)` | Put `where` beside `cases` on the eval |
|
|
17
|
+
| `survey(…)`, or `gate: 'none'` | `survey is not a function` / `gate in … is 'all' …, { passRate } … or 'observe'` | `evaluate(name, { gate: 'observe' }, fn)` |
|
|
18
|
+
| `compare: false` on an eval or group | `compare: false can't be set` | Use a profile's `limit.compare: false` |
|
|
19
|
+
| A question and no language model | `questions "…" (…) needs a language model` (exit 2) | Set `analysis.model`, give the question its own `model`, or set `question: { model }` |
|
|
20
|
+
| `import { baseline }` | `reeval has no baseline` | Name the control in `compare: { control }`; it isn't an export |
|
|
21
|
+
| `repeat: 3` / `times` | `Unknown option "repeat" … Did you mean "runs"?` | `runs: 3` |
|
|
22
|
+
| `runs: { bestOf: 4 }` | `runs.bestOf … must be an odd whole number of 3 or more` | `runs: { bestOf: 3 }` or `5` |
|
|
23
|
+
| `runs: { bestOf: 3 }` with `compare` | `… compares environments, so it can't take runs: { bestOf: 3 }` | a fixed count, `runs: 4` |
|
|
24
|
+
| Judge model `'gemini-3.8-flash'` | needs a provider prefix | `'google/gemini-3.8-flash'` |
|
|
25
|
+
| `opencode({ model: 'gemini-3.8-flash' })` | `an OpenCode model id names its provider` (exit 2) | `'google/gemini-3.8-flash'` |
|
|
26
|
+
| No `judge.model` anywhere | `c.judge() has no model` | Set one, or `fakeJudge()` |
|
|
27
|
+
| The judge's provider refuses a big input | The provider's message, then `The judge's input was ~…k tokens (estimated): diff …, of which …` | Add a `.gitignore` to the environment, or narrow the call: `{ diff: ['src/'] }` |
|
|
28
|
+
| `c.judge(q)` with no inputs, expecting the judge to see the task, answer and diff by default | `c.judge() shows the judge nothing: turn on at least one input, …` (TypeScript refuses it too) | There are no default inputs: turn on what the question needs, `{ diff: true }`, `{ contents: ['README.md'] }`, `{ output: true }`, `{ transcript: 'tools' }` |
|
|
29
|
+
| `c.judge(q, { on: value })` alone | `c.judge() shows the judge nothing: …`, or with another input `… on gives a value to grade as the output, but output is off. Pass output: true.` | `{ on: value, output: true }`; `on` and `turns` only select |
|
|
30
|
+
| An input (`diff`, `contents`, …) on a config, profile, group or eval `judge` | `Unknown judge option "diff" in …. diff is per call: c.judge(question, { diff }).` | Put it on each `c.judge()` call |
|
|
31
|
+
| A question as a string, or with no input | `questions … takes { question, model?, across?, choices?, turns? } with at least one of task, output, …` / `… shows the model nothing: …` (exit 2) | `{ question: 'Which is cleaner?', across: 'environment', diff: true }` |
|
|
32
|
+
| `cursor()` with `fakeSandbox()` | real harnesses need `docker()` | Use `fakeHarness()` or `docker()` |
|
|
33
|
+
| Forgetting `await` on `toHaveFile`, or on `toSucceed` given `c.exec()`'s promise | ``expect(…).toHaveFile() at <file>:<line> wasn't awaited. Its check is async: write `await …` `` (the eval fails) | `await expect(...)`; `expect(await c.exec(cmd)).toSucceed()` needs no `await` |
|
|
34
|
+
| `toPass()` on an exec result, `toSucceed()` on a verdict | a type error, ``Type 'WrongSubject<"toPass() takes the verdict from await c.judge()">' has no call signatures`` | `toSucceed()` for `c.exec()`, `toPass()` for `await c.judge()` |
|
|
35
|
+
| Secret as a string literal | refused | `secret.fromEnv('NAME')` in a callback |
|
|
36
|
+
| `workspace: 'https://…'` | `must come from the git helper` | `({ git }) => ({ workspace: git('https://…', { ref }) })` |
|
|
37
|
+
| `async` environment callback | returned a Promise | Return synchronously; fetch in a `setup` step |
|
|
38
|
+
| Test file named `*.test.ts` | `No eval files in … match include: …` (exit 2) | `*.eval.ts` under `evals/` |
|
|
39
|
+
| `.env.local` outside the config's directory | `Missing credentials: …` | Put it next to `reeval.config.ts` |
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Vocabulary
|
|
2
|
+
|
|
3
|
+
reeval's terms, and what other eval frameworks (Inspect, Harbor, SWE-bench) call them. Use
|
|
4
|
+
reeval's word in code and messages.
|
|
5
|
+
|
|
6
|
+
| reeval | Elsewhere called |
|
|
7
|
+
|---|---|
|
|
8
|
+
| eval (`evaluate(name, fn)`) | task, test case |
|
|
9
|
+
| case (an item of `cases`) | sample, example, instance |
|
|
10
|
+
| attempt (one run of a cell; `runs: n`) | trial (Anthropic, Harbor), epoch (Inspect), repetition |
|
|
11
|
+
| harness (`cursor()`, …) | agent harness, scaffold, solver (reeval itself is the *evaluation* harness) |
|
|
12
|
+
| cell (harness × environment × sandbox) | a factorial-design cell |
|
|
13
|
+
| control, variant | baseline, treatment |
|
|
14
|
+
| lift (variant pass@1 − control pass@1, 95% interval) | treatment effect, delta, uplift |
|
|
15
|
+
| gate (how a cell's attempts decide its outcome) | `'all'` is pass^k, `{ passRate }` a pass@1 threshold; epoch reducer (Inspect), quality gate |
|
|
16
|
+
| pass@1 (passes over passes and fails; `pass_at_1` in `reeval query`) | success rate, resolve rate (SWE-bench) |
|
|
17
|
+
| checks, judge (`expect`, `c.exec`, `c.judge`) | grader, verifier, scorer, evaluator |
|
|
18
|
+
| transcript (`transcript.jsonl`) | trajectory, trace |
|
|
19
|
+
| effort (measured tool calls, agent time, tokens, cost) | efficiency |
|