reeval 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (481) hide show
  1. package/CHANGELOG.md +841 -0
  2. package/README.md +15 -15
  3. package/assets/gateway-models.mjs +65 -31
  4. package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
  5. package/dist/adapter-Di9H2etg.js.map +1 -0
  6. package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
  7. package/dist/agent-B-6GFDqA.js.map +1 -0
  8. package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
  9. package/dist/agents-KBTYhSSh.js.map +1 -0
  10. package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
  11. package/dist/app-files-Cmh9FGr5.js.map +1 -0
  12. package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
  13. package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
  14. package/dist/assemble-DNVBTBp_.js.map +1 -0
  15. package/dist/attach-EbDGKGTO.js +152 -0
  16. package/dist/attach-EbDGKGTO.js.map +1 -0
  17. package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
  18. package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
  19. package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
  20. package/dist/attempt-host-DQvynDHk.js.map +1 -0
  21. package/dist/boat-Bzz8L-KQ.js +701 -0
  22. package/dist/boat-Bzz8L-KQ.js.map +1 -0
  23. package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
  24. package/dist/boat-Dek8ftIt.js.map +1 -0
  25. package/dist/bridge/index.mjs +6 -1
  26. package/dist/bundle-DZstHDeW.js +718 -0
  27. package/dist/bundle-DZstHDeW.js.map +1 -0
  28. package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
  29. package/dist/capture-BkbPsfej.js.map +1 -0
  30. package/dist/capture-CUAvk9KM.js +1 -0
  31. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  32. package/dist/cases-CM-MQRQp.js.map +1 -0
  33. package/dist/cases-FvhbM8M5.js +1 -0
  34. package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
  35. package/dist/cli-BgBBuxxJ.js.map +1 -0
  36. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
  37. package/dist/cli-spec-B5L9n-DV.js.map +1 -0
  38. package/dist/cli.js +2 -2
  39. package/dist/client-DMEYR3MA.js +807 -0
  40. package/dist/client-DMEYR3MA.js.map +1 -0
  41. package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
  42. package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
  43. package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
  44. package/dist/command-D_w_kDpI.js.map +1 -0
  45. package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
  46. package/dist/command-Dwlh0Vhu.js.map +1 -0
  47. package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
  48. package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
  49. package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
  50. package/dist/commands-Cuf57OYb.js.map +1 -0
  51. package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
  52. package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
  53. package/dist/config-C0RKt00M.js.map +1 -0
  54. package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
  55. package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
  56. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  57. package/dist/cost-C21gGrju.js.map +1 -0
  58. package/dist/cost-IM80LyGD.js +1 -0
  59. package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
  60. package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
  61. package/dist/db-3o8s2EyD.js +2 -0
  62. package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
  63. package/dist/db-ErnJ-gMD.js.map +1 -0
  64. package/dist/debug-CIXzB4wK.js.map +1 -1
  65. package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
  66. package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
  67. package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
  68. package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
  69. package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
  70. package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
  71. package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
  72. package/dist/docker-DrMnpL84.js +1 -0
  73. package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
  74. package/dist/doctor-B7utCSdh.js.map +1 -0
  75. package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
  76. package/dist/environment-C0Gg8kyN.js.map +1 -0
  77. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  78. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  79. package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
  80. package/dist/eval-source-CsW5VIXS.js.map +1 -0
  81. package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
  82. package/dist/evaluate-BHQyAszQ.js.map +1 -0
  83. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
  84. package/dist/fake-bootstrap-VpisF1WE.js +15 -0
  85. package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
  86. package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
  87. package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
  88. package/dist/files-BjOJw4sn.js +1 -0
  89. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  90. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  91. package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
  92. package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
  93. package/dist/files-CAzVG3Rr.js +1 -0
  94. package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
  95. package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
  96. package/dist/files-D-Lir4aE.js +1 -0
  97. package/dist/files-DRlJ0Xr2.js +1 -0
  98. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  99. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  100. package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
  101. package/dist/files-qjeqXnfE.js.map +1 -0
  102. package/dist/gateway.d.ts +1 -1
  103. package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
  104. package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
  105. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  106. package/dist/harness-FcTyyPco.js.map +1 -0
  107. package/dist/harness-install-Dxho1yaU.js +53 -0
  108. package/dist/harness-install-Dxho1yaU.js.map +1 -0
  109. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  110. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  111. package/dist/harness.d.ts +11 -8
  112. package/dist/harness.js +1 -1
  113. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  114. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  115. package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
  116. package/dist/images-Ch1y1t6_.js.map +1 -0
  117. package/dist/index.d.ts +46 -7
  118. package/dist/index.js +6 -6
  119. package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
  120. package/dist/init-C2HqCwnt.js.map +1 -0
  121. package/dist/init-CM9xwyre.js +1 -0
  122. package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
  123. package/dist/ink-list-BbKXElcQ.js.map +1 -0
  124. package/dist/inputs-5UIDpjz-.js +190 -0
  125. package/dist/inputs-5UIDpjz-.js.map +1 -0
  126. package/dist/inputs-load-C82HOKnb.js +1 -0
  127. package/dist/inputs-load-CGPyigMC.js +95 -0
  128. package/dist/inputs-load-CGPyigMC.js.map +1 -0
  129. package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
  130. package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
  131. package/dist/jiti-radm1Soi.js +52 -0
  132. package/dist/jiti-radm1Soi.js.map +1 -0
  133. package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
  134. package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
  135. package/dist/judge.d.ts +3 -1
  136. package/dist/judge.js +1 -1
  137. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  138. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  139. package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
  140. package/dist/layout-Czh5wOpz.js.map +1 -0
  141. package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
  142. package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
  143. package/dist/live-C9Et_BVT.js +1 -0
  144. package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
  145. package/dist/live-DGgIrVw0.js.map +1 -0
  146. package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
  147. package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
  148. package/dist/machine-service-2lIv5UWF.js +1 -0
  149. package/dist/machine-service-Ct_1XMMi.js +520 -0
  150. package/dist/machine-service-Ct_1XMMi.js.map +1 -0
  151. package/dist/machines-BnquHcBZ.js +120 -0
  152. package/dist/machines-BnquHcBZ.js.map +1 -0
  153. package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
  154. package/dist/model-Bcy4Qjmm.js +1 -0
  155. package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
  156. package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
  157. package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
  158. package/dist/model-DRa7yv6K.js.map +1 -0
  159. package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
  160. package/dist/model-E5X8YBJM.js.map +1 -0
  161. package/dist/model-calls-B0BoQfJ4.js +95 -0
  162. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  163. package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
  164. package/dist/node-BjuhqQwa.js.map +1 -0
  165. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  166. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  167. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
  168. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
  169. package/dist/options-C989kuVE.js +1 -0
  170. package/dist/own-reads-CYmBlrHK.js +15 -0
  171. package/dist/own-reads-CYmBlrHK.js.map +1 -0
  172. package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
  173. package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
  174. package/dist/pi-providers-HygmL1y-.js +552 -0
  175. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  176. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  177. package/dist/plain-Ctn8oUdp.js.map +1 -0
  178. package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
  179. package/dist/plan-BcT4G5jj.js.map +1 -0
  180. package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
  181. package/dist/plugins-cC0sqoDA.js.map +1 -0
  182. package/dist/preflight-BMVDBcux.js +1 -0
  183. package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
  184. package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
  185. package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
  186. package/dist/preflight-DySHXeFJ.js.map +1 -0
  187. package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
  188. package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
  189. package/dist/prepare-nbX1CXXg.js +409 -0
  190. package/dist/prepare-nbX1CXXg.js.map +1 -0
  191. package/dist/prices-B1vm_lPk.js +1 -0
  192. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  193. package/dist/prices-BrY_ihLs.js.map +1 -0
  194. package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
  195. package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
  196. package/dist/provider-DHT20Gl8.js +1 -0
  197. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  198. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  199. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  200. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  201. package/dist/providers-BkLGzaSY.js +1 -0
  202. package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
  203. package/dist/providers-bNgfKa6m.js.map +1 -0
  204. package/dist/query-B2SQddoV.js +1 -0
  205. package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
  206. package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
  207. package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
  208. package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
  209. package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
  210. package/dist/record--vHreTLk.js.map +1 -0
  211. package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
  212. package/dist/redact-t-L3VGy8.js.map +1 -0
  213. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  214. package/dist/remarks-BpdM_2RC.js.map +1 -0
  215. package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
  216. package/dist/remarks-CYFOsnz5.js.map +1 -0
  217. package/dist/remarks-CleoDlYw.js +1 -0
  218. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  219. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  220. package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
  221. package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
  222. package/dist/resolve-Bm9ZyFR-.js +1 -0
  223. package/dist/resolve-Bul8lHaa.js +205 -0
  224. package/dist/resolve-Bul8lHaa.js.map +1 -0
  225. package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
  226. package/dist/resolve-DorSHDBs.js.map +1 -0
  227. package/dist/resources-DqdwihhT.js +62 -0
  228. package/dist/resources-DqdwihhT.js.map +1 -0
  229. package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
  230. package/dist/results-DhMgRkR0.js.map +1 -0
  231. package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
  232. package/dist/resume-C8Ihfy-C.js.map +1 -0
  233. package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
  234. package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
  235. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  236. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  237. package/dist/routes-Cp4gAjBZ.js +427 -0
  238. package/dist/routes-Cp4gAjBZ.js.map +1 -0
  239. package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
  240. package/dist/runtime-CPUH0Vk5.js.map +1 -0
  241. package/dist/sandbox/boat.d.ts +1 -1
  242. package/dist/sandbox/boat.js +1 -1
  243. package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
  244. package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
  245. package/dist/sandbox.d.ts +1 -1
  246. package/dist/sandbox.js +1 -1
  247. package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
  248. package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
  249. package/dist/server-ZaMzUNmV.js +1 -0
  250. package/dist/services-BG8xuU82.js +39 -0
  251. package/dist/services-BG8xuU82.js.map +1 -0
  252. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  253. package/dist/shared-DtZwIAfy.js.map +1 -0
  254. package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
  255. package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
  256. package/dist/source-BcBDj-R0.js +1 -0
  257. package/dist/src-B2eHTCnp.js +3 -0
  258. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  259. package/dist/src-B5r1d23Y.js.map +1 -0
  260. package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
  261. package/dist/src-Bb3k9Kn_.js.map +1 -0
  262. package/dist/src-BfOlH_Bu.js +2 -0
  263. package/dist/src-BuGc40IF.js +2 -0
  264. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  265. package/dist/src-C-ChdnG6.js.map +1 -0
  266. package/dist/src-C2K5_afU.js +2 -0
  267. package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
  268. package/dist/src-C3OhA5pw.js.map +1 -0
  269. package/dist/src-CSVJjghq.js +4 -0
  270. package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
  271. package/dist/src-CWfy8ct8.js.map +1 -0
  272. package/dist/src-CcrQQB0_.js +2 -0
  273. package/dist/src-CotUxxQV.js +2 -0
  274. package/dist/src-D-PwBaBB.js +2 -0
  275. package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
  276. package/dist/src-DNWiU__5.js.map +1 -0
  277. package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
  278. package/dist/src-DwcTn4JY.js.map +1 -0
  279. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  280. package/dist/src-boB6D-5T.js.map +1 -0
  281. package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
  282. package/dist/src-c3ZQu7Uu.js.map +1 -0
  283. package/dist/src-jYqNkT-F.js +1 -0
  284. package/dist/start-Db5VRcRZ.js +660 -0
  285. package/dist/start-Db5VRcRZ.js.map +1 -0
  286. package/dist/state-CKIjZZRK.js +1 -0
  287. package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
  288. package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
  289. package/dist/store/local.d.ts +8 -0
  290. package/dist/store/local.js +16 -0
  291. package/dist/store/local.js.map +1 -0
  292. package/dist/store/s3.d.ts +10 -0
  293. package/dist/store/s3.js +18 -0
  294. package/dist/store/s3.js.map +1 -0
  295. package/dist/suite-CH1aaAbs.js +2 -0
  296. package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
  297. package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
  298. package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
  299. package/dist/summary-BK5Sa_N0.js.map +1 -0
  300. package/dist/sync-DUi_P2UJ.js +559 -0
  301. package/dist/sync-DUi_P2UJ.js.map +1 -0
  302. package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
  303. package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
  304. package/dist/tart-BQ_Ap0l4.js +508 -0
  305. package/dist/tart-BQ_Ap0l4.js.map +1 -0
  306. package/dist/tart-CzViW3WZ.js +186 -0
  307. package/dist/tart-CzViW3WZ.js.map +1 -0
  308. package/dist/tart-DmVuVsmX.js +1 -0
  309. package/dist/tart-bake-D1RttYfM.js +148 -0
  310. package/dist/tart-bake-D1RttYfM.js.map +1 -0
  311. package/dist/tart-bake-DjFcsG3N.js +1 -0
  312. package/dist/tart-cli-BOcluha1.js +1 -0
  313. package/dist/tart-cli-gA9xEaQL.js +89 -0
  314. package/dist/tart-cli-gA9xEaQL.js.map +1 -0
  315. package/dist/tart-hosts-CHNrfV6u.js +313 -0
  316. package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
  317. package/dist/tart-hosts-CWse-ASo.js +1 -0
  318. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
  319. package/dist/tart-images-CLBjfF9u.js.map +1 -0
  320. package/dist/testing.d.ts +36 -2
  321. package/dist/testing.js +4 -4
  322. package/dist/tool-class-krsmEo44.js +433 -0
  323. package/dist/tool-class-krsmEo44.js.map +1 -0
  324. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  325. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  326. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  327. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  328. package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
  329. package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
  330. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  331. package/dist/types-CbcnJxCy.js.map +1 -0
  332. package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
  333. package/dist/ui-DKYjTVpn.js +1455 -0
  334. package/dist/ui-DKYjTVpn.js.map +1 -0
  335. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  336. package/dist/usage-CwIRPPZ9.js.map +1 -0
  337. package/dist/vercel-models-Ci33llXz.js +21 -0
  338. package/dist/vercel-models-Ci33llXz.js.map +1 -0
  339. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  340. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  341. package/dist/vertex-DXfcXYKn.js +1 -0
  342. package/dist/view/index.d.ts +32 -3
  343. package/dist/view/index.js +8 -8
  344. package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
  345. package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
  346. package/dist/views-DCjtw1jF.js +2 -0
  347. package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
  348. package/dist/worker-entry-DrAYYt0M.js.map +1 -0
  349. package/dist/workspace-CnRbV8cM.js +1 -0
  350. package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
  351. package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
  352. package/docker/Dockerfile +1 -1
  353. package/package.json +15 -1
  354. package/skills/eval-rubric/SKILL.md +10 -0
  355. package/skills/reeval/SKILL.md +370 -0
  356. package/skills/reeval/references/capture.md +48 -0
  357. package/skills/reeval/references/cli.md +162 -0
  358. package/skills/reeval/references/compare.md +180 -0
  359. package/skills/reeval/references/config.md +147 -0
  360. package/skills/reeval/references/environments.md +218 -0
  361. package/skills/reeval/references/evals.md +241 -0
  362. package/skills/reeval/references/mistakes.md +39 -0
  363. package/skills/reeval/references/vocabulary.md +19 -0
  364. package/skills/setup/SKILL.md +219 -0
  365. package/skills/setup/references/templates.md +258 -0
  366. package/dist/adapter-C60z3wLq.js.map +0 -1
  367. package/dist/agent-BmsMXZXx.js.map +0 -1
  368. package/dist/agents-DXuFxtGC.js.map +0 -1
  369. package/dist/app-files-DT5mJyV4.js.map +0 -1
  370. package/dist/assemble-DM2PNU4R.js.map +0 -1
  371. package/dist/attempt-host-7VviAJrl.js.map +0 -1
  372. package/dist/boat-BjLqFJIZ.js.map +0 -1
  373. package/dist/boat-DSNgZt-t.js +0 -322
  374. package/dist/boat-DSNgZt-t.js.map +0 -1
  375. package/dist/capture-BzotD2hs.js +0 -1
  376. package/dist/capture-Ccf0Hmyx.js.map +0 -1
  377. package/dist/cases-Dy0mifal.js +0 -1
  378. package/dist/cases-z9E6DCd5.js.map +0 -1
  379. package/dist/cli-DTuLSlmW.js.map +0 -1
  380. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  381. package/dist/client-KM4C5r5_.js +0 -387
  382. package/dist/client-KM4C5r5_.js.map +0 -1
  383. package/dist/command-DSL7I-SI.js.map +0 -1
  384. package/dist/command-Ff0T-1Hi.js.map +0 -1
  385. package/dist/commands-xAOi_VEk.js.map +0 -1
  386. package/dist/config-SvK8pjEK.js.map +0 -1
  387. package/dist/cost-BdRgySQl.js.map +0 -1
  388. package/dist/cost-Cr26Alhq.js +0 -1
  389. package/dist/db-BzOJ4Js4.js.map +0 -1
  390. package/dist/db-CQRZ3URX.js +0 -2
  391. package/dist/docker--mukta3L.js +0 -1
  392. package/dist/docker-DG_uFgn2.js +0 -246
  393. package/dist/docker-DG_uFgn2.js.map +0 -1
  394. package/dist/doctor-DWK4ttex.js.map +0 -1
  395. package/dist/environment-cPMCOk-h.js.map +0 -1
  396. package/dist/eval-source-C9b1h8L6.js.map +0 -1
  397. package/dist/evaluate-B12Tosvq.js.map +0 -1
  398. package/dist/files-C1Rk9FYA.js +0 -1
  399. package/dist/files-DMQ9-Vv5.js +0 -1
  400. package/dist/files-Di0xDbCc.js +0 -1
  401. package/dist/files-G5jf8wBp.js +0 -1
  402. package/dist/files-s28a6czq.js.map +0 -1
  403. package/dist/harness-DXv2gnaT.js.map +0 -1
  404. package/dist/images-D2jFze2v.js.map +0 -1
  405. package/dist/init-Bi1Q3lRi.js.map +0 -1
  406. package/dist/init-CUw65o36.js +0 -1
  407. package/dist/ink-list-BpfidQ1h.js.map +0 -1
  408. package/dist/kinds-Bh2I35id.js +0 -698
  409. package/dist/kinds-Bh2I35id.js.map +0 -1
  410. package/dist/layout-Cm85tQYk.js.map +0 -1
  411. package/dist/live-CgYbO4Te.js.map +0 -1
  412. package/dist/live-DZLmj0iG.js +0 -1
  413. package/dist/matchers-CXLn-1jm.js +0 -148
  414. package/dist/matchers-CXLn-1jm.js.map +0 -1
  415. package/dist/model-Cv4GuTsZ.js +0 -1
  416. package/dist/model-CwwkrCb7.js.map +0 -1
  417. package/dist/model-cgt5uvu8.js.map +0 -1
  418. package/dist/node-D7pAU--0.js.map +0 -1
  419. package/dist/options-DPUmk7v_.js +0 -1
  420. package/dist/plain-BYUf_fUV.js.map +0 -1
  421. package/dist/plan-CsPilzpk.js.map +0 -1
  422. package/dist/plugins-BfnhazOv.js.map +0 -1
  423. package/dist/preflight-Bn3ianye.js +0 -1
  424. package/dist/preflight-GkpgrBxP.js.map +0 -1
  425. package/dist/prepare-DQEDXoDp.js +0 -268
  426. package/dist/prepare-DQEDXoDp.js.map +0 -1
  427. package/dist/prices-5m3SzMZ3.js.map +0 -1
  428. package/dist/provider-BnUoz4KB.js +0 -1
  429. package/dist/providers-1p4HKYIp.js.map +0 -1
  430. package/dist/providers-B-ijCktf.js +0 -1
  431. package/dist/query-BYhyf0LR.js +0 -1
  432. package/dist/record-6IUzwP9K.js.map +0 -1
  433. package/dist/redact-BhjtnjrU.js.map +0 -1
  434. package/dist/remarks-BNABzemt.js.map +0 -1
  435. package/dist/remarks-CAvTSq16.js +0 -1
  436. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  437. package/dist/resolve-C14Pmtej.js +0 -1
  438. package/dist/resolve-Cx3a9UQ2.js.map +0 -1
  439. package/dist/results-CQKXmHym.js.map +0 -1
  440. package/dist/resume-POpZa1ge.js.map +0 -1
  441. package/dist/runtime-Xm9oeC7o.js.map +0 -1
  442. package/dist/server-C1UJcpUh.js +0 -1
  443. package/dist/shared-Cfi45Bvy.js.map +0 -1
  444. package/dist/source-DJHmQv_j.js +0 -1
  445. package/dist/src-1dBuXzs4.js +0 -3
  446. package/dist/src-BKpioayC.js +0 -2
  447. package/dist/src-BNQznj4z.js.map +0 -1
  448. package/dist/src-BY1cCaoJ.js +0 -2
  449. package/dist/src-BbZYaQ8b.js +0 -2
  450. package/dist/src-CDFH2vVN.js +0 -2
  451. package/dist/src-CMW5R9Q0.js +0 -1
  452. package/dist/src-CV_7DkPa.js.map +0 -1
  453. package/dist/src-Cb2MWozp.js.map +0 -1
  454. package/dist/src-CrV2WA2i.js +0 -2
  455. package/dist/src-CsFaawcZ.js +0 -2
  456. package/dist/src-DAzaCTcN.js.map +0 -1
  457. package/dist/src-DvGns8cz.js.map +0 -1
  458. package/dist/src-Dw79VLa3.js.map +0 -1
  459. package/dist/src-KxS91P7L.js.map +0 -1
  460. package/dist/src-YvblOrBU.js.map +0 -1
  461. package/dist/src-wKwznPgt.js +0 -2
  462. package/dist/src-y0nOeRRE.js.map +0 -1
  463. package/dist/state-Ca55Gk_O.js +0 -1
  464. package/dist/suite-CKBa6MKt.js +0 -2
  465. package/dist/summary-DDy9DfyJ.js.map +0 -1
  466. package/dist/tart-cli-C-GsbVsQ.js +0 -1
  467. package/dist/tart-cli-CU0nFS-L.js +0 -55
  468. package/dist/tart-cli-CU0nFS-L.js.map +0 -1
  469. package/dist/tart-dWoJvWHc.js +0 -586
  470. package/dist/tart-dWoJvWHc.js.map +0 -1
  471. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  472. package/dist/tool-target-DS5d72eH.js +0 -40
  473. package/dist/tool-target-DS5d72eH.js.map +0 -1
  474. package/dist/types-C2mLpph3.js.map +0 -1
  475. package/dist/usage-DJiDnh2D.js.map +0 -1
  476. package/dist/vercel-CGlSBv63.js +0 -344
  477. package/dist/vercel-CGlSBv63.js.map +0 -1
  478. package/dist/vertex-O88TCwgY.js +0 -1
  479. package/dist/views-B0Yt_Q1S.js +0 -2
  480. package/dist/worker-entry-Smb4bNDc.js.map +0 -1
  481. package/dist/workspace-C0dDP75l.js +0 -1
@@ -0,0 +1,218 @@
1
+ # Environments, sandboxes, secrets and egress
2
+
3
+ ## Contents
4
+
5
+ - Environments: skills, files, rules, MCP, setup
6
+ - Fixture directories and `git()` workspaces
7
+ - Skills and composition
8
+ - Sandboxes
9
+ - Secrets and egress
10
+
11
+ ## Environments: skills, files, rules, MCP, setup
12
+
13
+ What the agent runs *in*, per attempt: the code the workspace starts as, skills, MCP servers,
14
+ files, rules, env vars, secrets and setup steps. Pass a spec, or a synchronous callback that
15
+ destructures the helpers it needs:
16
+
17
+ ```ts
18
+ // evals/envs.ts
19
+ import { defineEnvironment } from 'reeval'
20
+
21
+ export const houseStyle = defineEnvironment('house-style', ({ skill }) => ({
22
+ // a dir with SKILL.md, relative to this file
23
+ skills: [skill.local('../skills/house-style')],
24
+ }))
25
+
26
+ export const withDocs = houseStyle.extendEnvironment(
27
+ 'with-docs',
28
+ ({ file, mcp }) => ({
29
+ // written into the workspace
30
+ files: { 'docs/api.md': file('../docs/api.md') },
31
+ // AGENTS.md (CLAUDE.md for Claude Code)
32
+ rules: file('../rules/house.md'),
33
+ // or mcp.http(url) / mcp.sse(url) (not on OpenCode, Goose, pi or Codex)
34
+ mcp: { docs: mcp.stdio('npx', ['-y', '@acme/docs-mcp@1.0.0']) },
35
+ // every attempt, before the agent
36
+ setup: [{ run: 'npm ci', timeout: '5m' }],
37
+ }),
38
+ )
39
+
40
+ // Start from existing code: a repo, or one directory of it, at a pinned ref
41
+ export const app = defineEnvironment('acme-app', ({ git }) => ({
42
+ workspace: git('https://github.com/acme/app', {
43
+ ref: 'v2.3.0',
44
+ path: 'examples/basic',
45
+ }),
46
+ setup: ['npm ci'],
47
+ }))
48
+
49
+ // Or seed a fixture directory: a plain object of file() entries, so spread
50
+ // and override it
51
+ export const notes = defineEnvironment('acme-notes', ({ directory }) => ({
52
+ files: { ...directory('../fixture'), 'README.md': 'something' },
53
+ setup: [{ run: 'npm ci --no-audit --no-fund', timeout: '5m' }],
54
+ }))
55
+ ```
56
+
57
+ ## Fixture directories and `git()` workspaces
58
+
59
+ Use `directory(path, { into?, glob?, defaultIgnores? })` for a fixture directory instead of
60
+ mapping a file list through `file()`. It skips `.git`, `node_modules/`, `dist/`, top-level
61
+ `build/`, caches and editor folders (`DEFAULT_DIRECTORY_IGNORES`; `defaultIgnores: false` keeps
62
+ them, except `.git`). `glob` is ordered gitignore-style patterns where the last match wins and
63
+ `!` excludes: `glob: ['src/**', 'package.json', '!**/*.test.ts']`.
64
+
65
+ Executable files stay executable; empty directories aren't seeded.
66
+
67
+ `workspace: git(repo, { ref, path })` takes a URL, `github:owner/repo`, a GitHub `/tree/` URL or
68
+ a local path (relative to the file). reeval fetches it on the host before any attempt, with the
69
+ host's git credentials or `token: 'GITHUB_TOKEN'` (a host variable's name; never a token in the
70
+ URL or the eval). It copies the tree in before `files` (which win) and `setup`, and records the
71
+ commit and tree hash.
72
+
73
+ `init: true` adds a `.git` with one commit of the tree (no remote), for agents that expect a
74
+ repo. Pin `ref` to a tag or full SHA; a branch warns. The agent's diff starts after setup, so the
75
+ repo never counts as its change.
76
+
77
+ A bad ref, path, URL or token fails only the attempts that use it (infra error, exit 3).
78
+ Submodules and LFS files aren't fetched.
79
+
80
+ ## Skills and composition
81
+
82
+ Skills: `skill.local | git | npm | registry | inline` (pin git refs; `placement: 'workspace'`
83
+ writes them into the repo's skill directory). A git, npm or plugin source URL with a password or
84
+ token in it is refused: reeval fetches on the host with its own git and npm credentials.
85
+ `extendEnvironment` is the only composition; the child wins on shared keys and **replaces**
86
+ `sandbox` entirely when it sets one (install is not merged onto the parent image).
87
+
88
+ A skill is a directory whose `SKILL.md` has YAML frontmatter
89
+ (`name`, `description`; quote values containing `: `). Baking (`from`, `install`, `dockerfile`)
90
+ is `sandbox: docker({ … })` on the environment.
91
+
92
+ Plugins: `plugins: [plugin.local('../plugins/review', { only: ['skills', 'mcp'] })]` (also
93
+ `plugin.git(repo, { ref, path })`, `plugin.npm(spec)` and
94
+ `plugin.marketplace('name@github:owner/repo', { ref })` for an entry in a Claude Code
95
+ marketplace, or `'name@npm:<spec>'`; only a marketplace in a local directory may name `file://`
96
+ entries). A plugin is Claude Code's layout:
97
+ `.claude-plugin/plugin.json` plus `skills/`, `agents/`, `commands/`, `hooks/hooks.json`,
98
+ `.mcp.json`, `rules/`. Its MCP servers join `mcp` as `<plugin>:<server>` (call them from the
99
+ fake agent as `{ mcp: 'review:docs/<tool>' }`), and it lands in `~/.reeval/plugins/<name>/`. Claude
100
+ Code and Grok Build load every part but `rules` (narrow with `only`), namespaced
101
+ (`review:reviewer`), and start the plugin's MCP servers themselves, so a plugin server reading a
102
+ host variable is refused there: declare it in `mcp`. Antigravity loads every part with
103
+ `agy plugin install`, and refuses such a server too; Codex loads `skills` and `mcp` the same way
104
+ (and refuses a plugin with ChatGPT apps). Cursor loads every part flat: rules, subagents, commands
105
+ and hooks unpacked into the workspace's `.cursor/` (left out of the diff), skills into
106
+ `~/.agents/skills`, so two items with one name clash; Goose takes `skills` and `mcp` beside the
107
+ eval's own, so a clashing name is refused too. OpenCode loads skills, MCP servers, and subagents
108
+ and commands (unpacked into `.opencode/`, frontmatter cut to `description`); pi skills, MCP servers
109
+ (as `<plugin>_<server>`) and commands (as prompt templates in `.pi/prompts/`); fx skills and MCP
110
+ servers only. A part a harness doesn't load, or a hook event it doesn't run, is refused at plan
111
+ time (`--dry-run` exits 2 naming the harness): narrow with `only`. `build: 'npm ci && npm run build'` (or `{ run, cwd: '.' }` to run
112
+ from the source's root) builds the plugin in a cached Docker image layer, as root with no
113
+ secrets, without the source's `.git` (pass a version in its commands instead of `git describe`);
114
+ only `path` is placed, so the output must land inside it. Docker only. Its content is fixed when the
115
+ image is prepared: an npm spec re-resolving or a local plugin edited mid-run waits for the next run. A server run with
116
+ `npx`/`uvx` needs its registry in `egress`, or a build that installs it.
117
+
118
+ A named env is a total override: no skills, files or sandbox inherit from the config
119
+ environment.
120
+
121
+ ## Sandboxes
122
+
123
+ ```ts
124
+ // reeval's Node image with the agent CLI
125
+ docker()
126
+ // baked once as a cached layer; no secrets
127
+ docker({ install: ['npm i -g firebase-tools@14'] })
128
+ // a Debian or Ubuntu base; reeval adds Node and the agent
129
+ docker({ from: 'python:3.12-slim' })
130
+ // ARG REEVAL_BASE and FROM ${REEVAL_BASE} to layer on reeval's image
131
+ docker({ dockerfile: './sandbox/Dockerfile' })
132
+ // limits; defaults 4g, 2 and 1024
133
+ docker({ memory: '8g', cpus: 4, pids: 4096 })
134
+ // swap on top; default the same as memory, 0 for none
135
+ docker({ memory: '4g', swap: 0 })
136
+ // opt in to apt-get at run time (default: user reeval)
137
+ docker({ root: true })
138
+ // the same sandbox and options on rootless Podman 4+ (cell id `podman`)
139
+ podman({ from: 'python:3.12-slim' })
140
+ ```
141
+
142
+ Use `podman()` only where the user runs Podman instead of Docker: it takes every `docker()`
143
+ option and confines the container the same way (bridge network, Docker's seccomp profile).
144
+
145
+ `tart({ image })` (an OCI macOS image or a local Tart VM) runs each attempt in a cloned macOS VM
146
+ on an Apple Silicon Mac, at most two at once. It needs `egress: 'allow-all'`: under the default
147
+ `proxy` or a restricted policy the plan exits 2. Of the real harnesses only `cursor()` runs in it
148
+ so far (any other exits 2 at plan time), and it needs Node 22.21 or later in the image; the fake
149
+ harness runs too.
150
+ `reeval run` never pulls the image: one `tart list` doesn't show exits 2, naming `tart pull
151
+ <image>`. Run `npx reeval image build` first, which pulls every `tart()` image the plan needs (and
152
+ refuses, exit 3, when the disk can't hold them even once Tart prunes its cache). Pin a digest
153
+ (`ghcr.io/cirruslabs/macos-sequoia-base@sha256:…`): Cirrus Labs publishes only `latest`, so a
154
+ version tag isn't there to pin. `:latest` or no tag warns, and each attempt records the digest it
155
+ cloned in `vmImage`.
156
+
157
+ ```ts
158
+ import { boat, createBoat } from 'reeval/sandbox/boat'
159
+ // one boat.dev VM per attempt, over SSH (cell id `boat`, or `boat:large`)
160
+ defineEnvironment('vm', { sandbox: boat({ type: 'large' }), egress: 'allow-all' })
161
+ ```
162
+
163
+ `boat()` needs `npm i @boatdev/sdk`, `BOAT_API_KEY` (or `createBoat({ apiKey, prefix })` for
164
+ another account) and an OpenSSH client, and plans only under `egress: 'allow-all'`; reeval
165
+ refuses the other policies at plan time. It takes no image options: put the
166
+ toolchain's installs in the environment's `setup`.
167
+
168
+ The agent, setup and `c.exec` run as `reeval` (home `/home/reeval`) with no capabilities; builds
169
+ run as root. `root: true` or `capabilities` print a warning at the start of every run, so prefer
170
+ `install`. A process killed at the memory limit makes the attempt an infra error
171
+ (`error.resource.cause: 'oom'`), not a failed check: raise `memory` rather than loosening it.
172
+
173
+ Put a specific machine on `defineEnvironment`, or `sandbox` on `defineConfig`, a profile, group
174
+ or eval as the default when the resolved env omits one. A list is an error. `npx reeval image
175
+ build` builds every image the plan needs.
176
+
177
+ The cell key includes the image (`from`, `dockerfile`, `install`), so Node 22 and 24 can't share
178
+ a cell. Environments that share an image still share one preflight build.
179
+
180
+ ## Secrets and egress
181
+
182
+ Secrets come from a `defineConfig` or `defineEnvironment` callback, never literals:
183
+
184
+ ```ts
185
+ defineEnvironment('gemini', ({ secret }) => ({
186
+ env: {
187
+ GEMINI_API_KEY: secret.fromEnv('GEMINI_API_KEY', {
188
+ hosts: secret.hosts.gemini,
189
+ }),
190
+ },
191
+ }))
192
+ ```
193
+
194
+ With `hosts`, the sandbox only sees a placeholder and reeval's gateway swaps in the real value on
195
+ HTTPS requests to those hosts. `{ to: 'checks' }` keeps it out of the agent's own environment
196
+ (only `c.exec` and setup steps get it). Without `hosts`, a process a setup step leaves running (a
197
+ server the checks call) holds the real value while the agent works, where the agent, as the same
198
+ user, can read it: broker it with `hosts`, or start the process from `c.exec` after the turn. The
199
+ `setup-process-secret` remark flags it.
200
+
201
+ `egress: 'proxy' | 'allow-all' | 'deny-all' | { allow: [hosts] }` limits where the sandbox may
202
+ connect (the harness's API, brokered hosts and MCP hosts are always allowed; `DEFAULT_EGRESS_ALLOW`
203
+ is npm, Yarn, PyPI and GitHub). A blocked request doesn't fail the attempt; it's recorded, and your
204
+ checks decide. The record's `egress.blocked` is what the policy refused; a name that didn't resolve
205
+ (`metadata.google.internal` outside Google Cloud) is in `egress.unresolved` instead.
206
+
207
+ The default, `'proxy'`, reaches any host over HTTP(S) through the egress gateway and nothing
208
+ else: SSH, a database driver or a raw TCP/UDP client gets `Network is unreachable` (and the run's
209
+ `proxy-only-egress` remark). Set `egress: 'allow-all'` on just the evals that need that; it keeps
210
+ the container's network, so such traffic goes out unrecorded (the record's `egress.partial`).
211
+
212
+ `egress: { setup, agent }` gives each phase its own policy, in `docker()` and `podman()` only:
213
+ `setup` covers the files and setup steps, `agent` the harness's start, every turn and every check.
214
+ `{ setup: 'allow-all', agent: 'deny-all' }` lets setup install dependencies while the agent
215
+ reaches only its API and the automatic hosts. A phase left out keeps the policy it inherits, and
216
+ the record tags each traffic entry with its `phase`. A connection setup leaves open (a background
217
+ process's tunnel) to a host the agent's policy refuses is closed as the agent starts, and the
218
+ `setup-tunnel-closed` remark flags it.
@@ -0,0 +1,241 @@
1
+ # Evals: the attempt API, judges and matchers
2
+
3
+ ## Contents
4
+
5
+ - Cases: `cases`, `caseKey`, `where`, and `gate: 'observe'`
6
+ - The attempt: `c.send`, `c.exec`, `c.serve`, files
7
+ - The judge: inputs, input size, errors
8
+ - Trustworthy evals
9
+ - Matchers
10
+
11
+ ## Cases: `cases`, `caseKey`, `where`, and `gate: 'observe'`
12
+
13
+ - `evaluate(name, [options,] fn)` and `group(name, [options,] fn)`; also `.skip`, `.only`
14
+ (an error in CI), `.todo`. The eval id is `file › group › name`; renaming an eval restarts its history.
15
+ - `evaluate(name, { cases: [...] }, async (c, item) => …)` runs one eval over a list: one pass
16
+ rate over the cases and, with `compare`, one lift paired by case, which `minLift` gates (not
17
+ each case's). Prefer it to separate evals when the cases are one population: six cases at
18
+ `runs: 1` can reach a lift verdict where each case alone is within noise. Fewer than six
19
+ discordant cases shows only a direction (`insufficient` in `reeval query case_lift`), so
20
+ aim for six cases or more; the plan hints when there are fewer. No `runs: { bestOf }` beside it.
21
+ `gate: { passRate }` gates each cell's rate over the cases (the mean of each case's pass@1), so
22
+ one failing case of three passes `passRate: 0.6`; under `'all'` every case must pass, so a
23
+ failing case fails the eval. It is one eval: one row in `--dry-run` and the run output (a
24
+ case table under it when it fails, or with `-v`), one entry in the record with its cases
25
+ inside, counted once in the tally. The failing case still records `fail`.
26
+ - For evals that pass or fail on their own, don't use `cases`: write a `for` loop that calls
27
+ `evaluate` once per item, with the item in the name.
28
+ - Each case has a key, which names its row: a string or number is its own key, an object its
29
+ `id`, else its `name`; otherwise pass `caseKey: (item) => item.slug`. Keys are unique, at most
30
+ 80 characters, with no `#` or `›`. `cases`, `caseKey` and `where` go on `evaluate`, never a
31
+ group or the config.
32
+ - Porting `evaluate.each(items)('name: %s', opts, fn)`: write `evaluate('name', { cases: items,
33
+ ...opts }, fn)` with the old option names made plural (`harnesses`, `environments`,
34
+ `compare.control`). The old runs then read as this eval, keyed by what `%s` or `$id` held, so
35
+ `reeval compare` lines them up; a `caseKey` that changes a key starts that case's history again.
36
+ - `where: ({ case, environment, harness }) => boolean` (TypeScript only) picks which case runs
37
+ on which environment, at plan time, from each environment's `traits` (`defineEnvironment('next',
38
+ { traits: { stack: 'next' } })`; never shown to the agent). Reading a trait no environment
39
+ defines is an error; with `compare` it sees the control, and its answer holds for the variants.
40
+ `--dry-run` notes `excluded on vite-app: q2`, the run's case table shows `–` there, and each case's record keeps its own in
41
+ `excluded`. A cell left with no cases is an error: drop that environment instead.
42
+ - `where` says which pairs fit, the same every run; to run only some environments today, use
43
+ `reeval run --trait stack=next` (repeat to AND) or a profile's
44
+ `limit: { traits: { stack: ['next', 'vite'] } }`. Both narrow after `where`, by the control's
45
+ traits under `compare`, and `--dry-run` notes `--trait stack=next leaves out vite-app`. Never
46
+ read a flag or a variable in `where` to do this.
47
+ - `evaluate(name, { gate: 'observe' }, fn)` records what agents do (with `c.observe` and
48
+ checks) and fails only on run or infra errors. Its block reads `ran`. `compare.minLift` and
49
+ `runs: { bestOf }` are errors beside it; `compare: { gate: 'observe' }` ungates a compare's cells.
50
+ Any gate but `'all'`, `{ passRate }` or `'observe'` is an error.
51
+ A comment on an eval describes the current check (why a canary fails, which path it reads),
52
+ not what used to be true or which product a prompt was ported from.
53
+
54
+ ## The attempt: `c.send`, `c.exec`, `c.serve`, files
55
+
56
+ - `await c.send(prompt)` runs one agent turn and returns a `Turn` (`text`, `status`, `diff`,
57
+ `transcript`). Call it again for multi-turn;
58
+ `c.judge(q, { on: firstTurn, task: true, output: true })` grades an earlier one.
59
+ - `await c.exec(cmd)` returns `{ exitCode, stdout, stderr }`; `c.exec(cmd, { background: true })`
60
+ starts it without waiting and returns `{ kill() }`. `{ env: { NAME: 'value' } }` sets variables
61
+ for that command only, over the environment's (in the background too); a value named like a
62
+ secret (`*_TOKEN`, `*_KEY`, …) is masked in everything reeval writes, and a brokered secret's
63
+ host value goes in as its placeholder. `c.exec(cmd, { retries: 2, interval: 2000 })`
64
+ runs a command that exits non-zero again, up to 2 more times 2 s apart, and resolves with the
65
+ last try: for a check that can fail on a transient, such as a test calling a model. Any non-zero
66
+ exit retries (reeval doesn't read the output), and a last failed try is a failed check. `await c.serve(cmd, port?)` is that plus a
67
+ port (`PORT` / `$PORT` on a bash string), and returns once `cmd` listens, on `PORT` or the one
68
+ port it picked itself. A `port` something already listens on (the agent's dev server) throws:
69
+ pass none and serve on `PORT`. `c.capture()` screenshots it: see
70
+ [capture.md](capture.md).
71
+ - `await c.readFile(path)` the text or `null` (`~/` for home). `await c.writeFile(path, text)`
72
+ places a file after a turn (a hidden test the agent never saw) instead of seeding it in
73
+ `files`; later turns' `diff` leaves it out but shows any change the agent makes to it, even
74
+ under `.gitignore`. Don't place grading files with a `c.exec` heredoc: the next turn's diff
75
+ (and the judge) would count them as the agent's. The sandbox and session start on the first
76
+ call that needs them (`c.send`, `c.exec`, `c.readFile`, `c.writeFile`, `c.serve`,
77
+ `c.capture`), so a check before any turn sees what the environment set up.
78
+ `c.attempt.cell.role` is
79
+ `'control'` or `'variant'`.
80
+ - `c.observe(name, value)` records a JSON value about the attempt (which ORM it installed, how many
81
+ files it touched) that never affects pass or fail; the run output tallies it per cell and
82
+ `reeval query attempt_values` lists it. `c.judge()` and questions see it only with
83
+ `observations: true` (or a list of names). Use it for what you want to count, `expect` for what
84
+ must hold.
85
+
86
+ ## The judge: inputs, input size, errors
87
+
88
+ - `await c.judge(question, options)`: the judge sees the question and only the inputs the call
89
+ turns on. Every input is off until the call sets it, and a call needs at least one (`true`, a
90
+ mode or a non-empty list); there are no defaults, and the config, profile, group and eval
91
+ `judge` take only `model`, `threshold`, `providerOptions`, `maxConcurrent` and `gateway`.
92
+ With none, `c.judge()` throws before any model call, and TypeScript refuses it.
93
+
94
+ | Input | Shows |
95
+ |---|---|
96
+ | `task: true` | the graded turn's prompt |
97
+ | `output: true` | the graded turn's final message, or `on`'s value |
98
+ | `files: true` | `status path` lines for the changed files, under `diff`'s patterns |
99
+ | `diff: true` / `['src/**', '!**/*.test.ts']` | the diff since the eval started (gitignore-style, last match wins) |
100
+ | `contents: ['README.md', 'src/**']` | files whole, changed or not; they leave the diff |
101
+ | `transcript: true` / `'tools'` / `'full'` | the turn's transcript: clipped, tool calls only, everything |
102
+ | `checks: true` / `['npm test']` | `c.exec` results, or those commands' |
103
+ | `artifacts: true` / `[shot, 'home']` | captures, sent as file parts |
104
+
105
+ Pick the inputs the question needs, usually one or two: a question about code reads `diff` or
106
+ `contents`, one about the agent's answer `output`, one about how it worked `transcript`, one
107
+ about a screenshot `artifacts`. Spell out `{ task: true, output: true, diff: true }` only when
108
+ the question needs all three. The diff leaves out dependencies, build output (`dist/`,
109
+ `.next/`, top-level `build/` and `out/`), lockfiles, `*.map`, `*.tsbuildinfo` and minified
110
+ files.
111
+ - `on` and `turns` select what `task`, `output`, `transcript` and `diff` read and show nothing alone:
112
+ `on: turn` grades an earlier turn, `on: value` grades any value as the output (with
113
+ `output: true`), and `turns: 'all'` adds every turn before the graded one. `judge-input.json`
114
+ records each call's `stateOptions` (every input, `false` when off) and the `state` it sent.
115
+ - Ask about facts in a captured picture, not whether it "looks great". Jev
116
+ (`typesafe-ai/jev-1.14.0`) takes PNG captures; a provider that cannot take files fails that
117
+ call. Prefer deterministic checks for what can be checked; use the judge for quality, with
118
+ plain yes/no questions that don't name one specific way of doing it.
119
+ - Give the environment a `.gitignore` in `files` (the project's isn't copied in): an agent that
120
+ runs a build otherwise fills the judge's diff with its output. reeval sends every input the
121
+ call turns on, uncut, and knows no model's limit: a provider error refusing it keeps the
122
+ provider's wording and adds one line with the input's estimated size, biggest files and the
123
+ inputs that were on, largest first; narrow one, e.g. `{ diff: ['src/'] }`.
124
+ - A failed judge call doesn't throw: `c.judge()` returns `{ verdict: 'error', score: null, error }`,
125
+ the later judges and checks still run, and the attempt fails as a judge error at the end, or
126
+ as an infra error (which `--retry-errors` reruns) when the provider never answered and nothing
127
+ else failed.
128
+ `expect(await c.judge(q, { diff: true }), name).toPass()` records each judge's result and
129
+ carries on.
130
+
131
+ ## Trustworthy evals
132
+
133
+ Before a real run, prove each check can pass and fail with the fake harness in the eval's real
134
+ sandbox: an oracle `fakeHarness` that does the task right (must pass) and a canary that cheats
135
+ or gets it wrong (must fail). A canary that passes means a check isn't checking. Give each check
136
+ you rely on a canary that only it catches.
137
+
138
+ ```ts
139
+ // reeval.config.ts: the real harness, plus two profiles that swap only the harness and judge
140
+ export default defineConfig({
141
+ harnesses: cursor({ model: 'composer-2.5' }),
142
+ profiles: {
143
+ oracle: {
144
+ defaults: {
145
+ harnesses: fakeHarness({ files: { 'greet.js': 'export const greet = …' } }),
146
+ judge: { model: fakeJudge({ score: 0.95 }) },
147
+ },
148
+ },
149
+ canary: {
150
+ defaults: {
151
+ harnesses: fakeHarness({ files: { 'greet.test.js': "test('greets', () => {})\n" } }),
152
+ judge: { model: fakeJudge({ score: 0.1 }) },
153
+ },
154
+ },
155
+ },
156
+ })
157
+ ```
158
+
159
+ ```bash
160
+ npx reeval image build --profile oracle # once: the eval's images
161
+ npx reeval run --profile oracle # every check passes, exit 0
162
+ npx reeval run --profile canary # the checks you meant fail, exit 1
163
+ ```
164
+
165
+ - Set `defaults.harnesses`, never `defaults.sandbox`: the environment's `docker()` (image,
166
+ `install`, egress, brokered secrets) stays, so the checks run where they will for real. A
167
+ profile's `sandbox` only fills in for an environment that omits one, and the fake harness
168
+ without `layout` uses Cursor's image; `layout: 'claude-code'` (or another agent id) uses that
169
+ agent's image and reads rules and skills where it does.
170
+ - `script` steps (`{ exec }`, `{ write }`, `{ mcp }`) run inside the container as the agent, so an
171
+ oracle can run the real fix (`{ exec: 'node scripts/solve.mjs' }`) or call a brokered API.
172
+ - `fakeJudge` checks the wiring (a low score fails the check); keep the real `judge.model` on the
173
+ oracle, with its key, to check that a question passes good work.
174
+ - `fakeSandbox()` isn't the oracle's sandbox: it runs on the host, brokers nothing and enforces no
175
+ egress. It's for trying reeval and testing configs without Docker.
176
+ - An eval or group that sets its own `harnesses` keeps it (the plan notes `harnesses set by the
177
+ eval`), and an ungated cell's (`compare: { gate: 'observe' }`, a control's) failed checks don't change the
178
+ exit code: its row in the result block shows its count (`plain 0 of 4`).
179
+
180
+ Keep grading out of the
181
+ agent's reach: place hidden tests after the turn with `c.writeFile`, so the agent never sees
182
+ them, seed tests in `files` only when the agent should see them, and guard seeded files
183
+ with `expect(await c.readFile(path)).toBe(original)` or `expect(c).not.toHaveChanged(path)`.
184
+ When the behavior is the
185
+ outcome, judge the program's output, not the diff:
186
+ `c.judge(q, { on: (await c.exec('node app.js')).stdout, output: true })`. Checks run in the agent's
187
+ container as the same user, which suits AX evals but isn't a sealed verifier.
188
+
189
+ ## Matchers
190
+
191
+ - `await expect(turn).toHaveFile(path, pattern?)`,
192
+ `await expect(c.exec(cmd)).toSucceed()`, `expect(verdict).toPass()`,
193
+ `expect(turn).toHaveSearched(query?)` / `toHaveFetched(url?)` (sync; they throw
194
+ `HarnessCapabilityError` when `turn.unreported` lists web events; a refused
195
+ search is still a `web-search` with `ok: false`; a call whose result never came has
196
+ `ok: null`, not `false`), plus Vitest's.
197
+ - Transcript matchers (sync, no `await`): `expect(turn | c).toHaveCalledSkill(name)`,
198
+ `toHaveCalledMcp(server, tool, input?)`, `toHaveCalledTool(name | RegExp, input?)`,
199
+ `toHaveRun(command?)`, `toHaveEdited(path?)`, `toHaveSearched(query?)`,
200
+ `toHaveFetched(url?)`. `.not` is Vitest's. `toHaveRun(string | RegExp)` matches the whole
201
+ command line, so `/genkit/` passes on `cat genkit.config.ts`; `toHaveRun({ program, args? })`
202
+ matches the program a segment (split on line breaks, `&&`, `||`, `;`, `|`, `&`) ran, by its
203
+ name without a directory, after `VAR=x`, `env`, `timeout`, `time`, `exec`, `sudo`, `nohup`,
204
+ `nice`, `command`, `if`/`then`/`do`, `bash -c`, `npx`, `bunx`, `npm exec`, `pnpm exec`/`dlx`
205
+ and `yarn dlx`; `args` (joined by spaces) is a substring or RegExp, e.g. `/^add\b/`.
206
+ `pnpm <bin>` and `npm run` aren't unwrapped, and a run counts even if its call failed.
207
+ `shellCommands(c | turn | c.turns)` returns those programs to count or filter. `toHaveCalledSkill`
208
+ is opened or loaded (a `skill` / `Skill` / `load_skill` tool, or a Read/`sed` of
209
+ `skills/<name>/`, on every harness), not "followed". MCP is `toHaveCalledMcp(server, tool)`
210
+ (the names in `mcp.<server>`); `toHaveCalledTool('mcp__<server>__<tool>')` is
211
+ the raw transcript name. After adapters and the normalizer, that string is
212
+ the same on every harness (stdio; Goose, OpenCode, pi and Codex refuse
213
+ `mcp.sse`). Use `toHaveEdited` / `toHaveSearched`, not
214
+ `toHaveCalledTool('edit')` or `'webSearch'`.
215
+ - Budgets: `expect(turn | c).toStayWithin({ tokens: 20_000, toolCalls: 30, duration: '5m' })`
216
+ (sync) caps the turns' summed tokens, tool calls (a number, or `{ bash: 5 }` per tool) and time.
217
+ Tokens include subagents. With `tokens`, a harness that reported no usage throws
218
+ `HarnessCapabilityError` (an error, not a pass). `.not`, an unknown key or a bad limit throws a
219
+ `TypeError` that ends the attempt as its error, not a failed check. Python:
220
+ `to_stay_within(tokens=, tool_calls=, duration=)`.
221
+ - Guard files the agent must leave alone with `expect(c).not.toHaveChanged(['test/**',
222
+ 'package.json'])` (sync): it reads the workspace diff, so a shell or subprocess write counts
223
+ and a read doesn't. Strings are globs (a directory matches everything under it), RegExps work,
224
+ and a turn means up to that turn. `toHaveEdited` counts the files that changed while the agent
225
+ ran the turn (a turn only its own), so a shell write counts, but its string matches any path
226
+ containing it; guard with `toHaveChanged`'s globs. `toHaveEdited()` counts files that appear
227
+ in the workspace diff. Gitignored files don't appear there, so use `toHaveFile()` to check them.
228
+ - Checks don't stop the eval: a failed `expect` is recorded and the body carries on, so every
229
+ later check and judge still runs; the attempt fails at its end. Use `expect.require` for gates:
230
+ its failure ends the body there (`await expect.require(c.exec('npm run build'), 'builds')
231
+ .toSucceed()`, `expect.require.poll(…)` for one that waits). One the eval catches still fails
232
+ the attempt, and its error says the eval caught it. Only matcher failures carry on: an error
233
+ the body throws, a dead sandbox or a timeout ends the attempt. Wait or retry with `expect.poll`
234
+ (`await expect.poll(() => c.exec(cmd), { timeout: 10_000 }).toSucceed()`), which records one
235
+ check with its final outcome, not with `expect` in a `try` loop.
236
+ - **Label each requirement** with `expect`'s second argument:
237
+ `await expect(c, 'exports greet').toHaveFile('greet.js', /export/)`. Every matcher call
238
+ is recorded (`evals[].attempts[].assertions`; unlabeled ones get `<matcher> <first arg>`).
239
+ Changing a label changes the eval's fingerprint. Label judges too
240
+ (`expect(await c.judge(q, { diff: true }), 'Social OAuth').toPass()`): the label replaces the clipped
241
+ question in judge rates, `reeval show`, `summary.md` and the digest.
@@ -0,0 +1,39 @@
1
+ # Common mistakes (and the fix)
2
+
3
+ Each row is a mistake, what reeval says and the fix.
4
+
5
+ | Mistake | reeval says | Fix |
6
+ |---|---|---|
7
+ | No `reeval.config.*` | `No reeval.config.ts … here or above. Run reeval init.` | `reeval init`, or pass `--config` |
8
+ | `compare` in `reeval.config.ts` | `compare can't be set in reeval.config.ts` | Put it on the group or eval |
9
+ | `compare: [b]` | `compare … must be an object, not a list` | `compare: { control: a, variants: [b] }` |
10
+ | `compare: { variants: [b] }` | `compare … has no control` | `compare: { control: a, variants: [b] }` |
11
+ | `compare: { baseline: a, … }` | `Unknown compare option "baseline" … Did you mean "control"?` | `compare: { control: a, … }` |
12
+ | `compare: { strategy: 'peer' }` | `Unknown compare option "strategy"` | Drop it. Variants already run on their own |
13
+ | `environments` beside `compare` | `… sets both environments and compare` | Move the environment into `compare.control` |
14
+ | `evaluate.each(cases)(name, fn)` | `evaluate.each is not a function` | `evaluate(name, { cases }, async (c, item) => …)` |
15
+ | `cases` for items that should each pass or fail alone | One failing case fails the whole eval | A loop: `for (const item of items) evaluate(nameOf(item), fn)` |
16
+ | `where` without `cases`, or on a group | `… sets where without cases` / `Unknown option "where" in group(…)` | Put `where` beside `cases` on the eval |
17
+ | `survey(…)`, or `gate: 'none'` | `survey is not a function` / `gate in … is 'all' …, { passRate } … or 'observe'` | `evaluate(name, { gate: 'observe' }, fn)` |
18
+ | `compare: false` on an eval or group | `compare: false can't be set` | Use a profile's `limit.compare: false` |
19
+ | A question and no language model | `questions "…" (…) needs a language model` (exit 2) | Set `analysis.model`, give the question its own `model`, or set `question: { model }` |
20
+ | `import { baseline }` | `reeval has no baseline` | Name the control in `compare: { control }`; it isn't an export |
21
+ | `repeat: 3` / `times` | `Unknown option "repeat" … Did you mean "runs"?` | `runs: 3` |
22
+ | `runs: { bestOf: 4 }` | `runs.bestOf … must be an odd whole number of 3 or more` | `runs: { bestOf: 3 }` or `5` |
23
+ | `runs: { bestOf: 3 }` with `compare` | `… compares environments, so it can't take runs: { bestOf: 3 }` | a fixed count, `runs: 4` |
24
+ | Judge model `'gemini-3.8-flash'` | needs a provider prefix | `'google/gemini-3.8-flash'` |
25
+ | `opencode({ model: 'gemini-3.8-flash' })` | `an OpenCode model id names its provider` (exit 2) | `'google/gemini-3.8-flash'` |
26
+ | No `judge.model` anywhere | `c.judge() has no model` | Set one, or `fakeJudge()` |
27
+ | The judge's provider refuses a big input | The provider's message, then `The judge's input was ~…k tokens (estimated): diff …, of which …` | Add a `.gitignore` to the environment, or narrow the call: `{ diff: ['src/'] }` |
28
+ | `c.judge(q)` with no inputs, expecting the judge to see the task, answer and diff by default | `c.judge() shows the judge nothing: turn on at least one input, …` (TypeScript refuses it too) | There are no default inputs: turn on what the question needs, `{ diff: true }`, `{ contents: ['README.md'] }`, `{ output: true }`, `{ transcript: 'tools' }` |
29
+ | `c.judge(q, { on: value })` alone | `c.judge() shows the judge nothing: …`, or with another input `… on gives a value to grade as the output, but output is off. Pass output: true.` | `{ on: value, output: true }`; `on` and `turns` only select |
30
+ | An input (`diff`, `contents`, …) on a config, profile, group or eval `judge` | `Unknown judge option "diff" in …. diff is per call: c.judge(question, { diff }).` | Put it on each `c.judge()` call |
31
+ | A question as a string, or with no input | `questions … takes { question, model?, across?, choices?, turns? } with at least one of task, output, …` / `… shows the model nothing: …` (exit 2) | `{ question: 'Which is cleaner?', across: 'environment', diff: true }` |
32
+ | `cursor()` with `fakeSandbox()` | real harnesses need `docker()` | Use `fakeHarness()` or `docker()` |
33
+ | Forgetting `await` on `toHaveFile`, or on `toSucceed` given `c.exec()`'s promise | ``expect(…).toHaveFile() at <file>:<line> wasn't awaited. Its check is async: write `await …` `` (the eval fails) | `await expect(...)`; `expect(await c.exec(cmd)).toSucceed()` needs no `await` |
34
+ | `toPass()` on an exec result, `toSucceed()` on a verdict | a type error, ``Type 'WrongSubject<"toPass() takes the verdict from await c.judge()">' has no call signatures`` | `toSucceed()` for `c.exec()`, `toPass()` for `await c.judge()` |
35
+ | Secret as a string literal | refused | `secret.fromEnv('NAME')` in a callback |
36
+ | `workspace: 'https://…'` | `must come from the git helper` | `({ git }) => ({ workspace: git('https://…', { ref }) })` |
37
+ | `async` environment callback | returned a Promise | Return synchronously; fetch in a `setup` step |
38
+ | Test file named `*.test.ts` | `No eval files in … match include: …` (exit 2) | `*.eval.ts` under `evals/` |
39
+ | `.env.local` outside the config's directory | `Missing credentials: …` | Put it next to `reeval.config.ts` |
@@ -0,0 +1,19 @@
1
+ # Vocabulary
2
+
3
+ reeval's terms, and what other eval frameworks (Inspect, Harbor, SWE-bench) call them. Use
4
+ reeval's word in code and messages.
5
+
6
+ | reeval | Elsewhere called |
7
+ |---|---|
8
+ | eval (`evaluate(name, fn)`) | task, test case |
9
+ | case (an item of `cases`) | sample, example, instance |
10
+ | attempt (one run of a cell; `runs: n`) | trial (Anthropic, Harbor), epoch (Inspect), repetition |
11
+ | harness (`cursor()`, …) | agent harness, scaffold, solver (reeval itself is the *evaluation* harness) |
12
+ | cell (harness × environment × sandbox) | a factorial-design cell |
13
+ | control, variant | baseline, treatment |
14
+ | lift (variant pass@1 − control pass@1, 95% interval) | treatment effect, delta, uplift |
15
+ | gate (how a cell's attempts decide its outcome) | `'all'` is pass^k, `{ passRate }` a pass@1 threshold; epoch reducer (Inspect), quality gate |
16
+ | pass@1 (passes over passes and fails; `pass_at_1` in `reeval query`) | success rate, resolve rate (SWE-bench) |
17
+ | checks, judge (`expect`, `c.exec`, `c.judge`) | grader, verifier, scorer, evaluator |
18
+ | transcript (`transcript.jsonl`) | trajectory, trace |
19
+ | effort (measured tool calls, agent time, tokens, cost) | efficiency |