reeval 0.19.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (436) hide show
  1. package/CHANGELOG.md +803 -0
  2. package/README.md +15 -15
  3. package/THIRD_PARTY_NOTICES.md +5 -5
  4. package/assets/gateway-models.mjs +65 -31
  5. package/dist/{adapter-C8xPuXYq.js → adapter-BdzhdRYo.js} +8 -6
  6. package/dist/adapter-BdzhdRYo.js.map +1 -0
  7. package/dist/{agent-CBoLndM7.js → agent-Dx-bDmlV.js} +179 -31
  8. package/dist/agent-Dx-bDmlV.js.map +1 -0
  9. package/dist/{agents-C6zhkxp3.js → agents-l0-k7vw3.js} +362 -102
  10. package/dist/agents-l0-k7vw3.js.map +1 -0
  11. package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
  12. package/dist/app-files-B-CQYB0c.js.map +1 -0
  13. package/dist/{assemble-UPCRrtOz.js → assemble-CYMpN8LC.js} +1 -1
  14. package/dist/{assemble-t6h018An.js → assemble-CsZIcJo8.js} +171 -18
  15. package/dist/assemble-CsZIcJo8.js.map +1 -0
  16. package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
  17. package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
  18. package/dist/{attempt-host-CIm1-11m.js → attempt-host-BwSmNYkX.js} +17 -8
  19. package/dist/attempt-host-BwSmNYkX.js.map +1 -0
  20. package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
  21. package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
  22. package/dist/{boat-CJCtzfB1.js → boat-su1Y7LPd.js} +17 -14
  23. package/dist/boat-su1Y7LPd.js.map +1 -0
  24. package/dist/bridge/index.mjs +113 -8
  25. package/dist/{capture-iBzFiC0j.js → capture-BqIAeFyC.js} +55 -29
  26. package/dist/{capture-iBzFiC0j.js.map → capture-BqIAeFyC.js.map} +1 -1
  27. package/dist/capture-D0Ln5aCP.js +1 -0
  28. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  29. package/dist/cases-CM-MQRQp.js.map +1 -0
  30. package/dist/cases-FvhbM8M5.js +1 -0
  31. package/dist/{cli-20Fpvs5H.js → cli-s_3opn_I.js} +219 -109
  32. package/dist/cli-s_3opn_I.js.map +1 -0
  33. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
  34. package/dist/cli-spec-DxzIjmq_.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/client-C5fIzHQa.js +803 -0
  37. package/dist/client-C5fIzHQa.js.map +1 -0
  38. package/dist/{command-CRh-cb6Z.js → command-B5OAIjp7.js} +25 -21
  39. package/dist/command-B5OAIjp7.js.map +1 -0
  40. package/dist/{command-BLNVDiIg.js → command-BJ7KqczP.js} +10 -10
  41. package/dist/{command-BLNVDiIg.js.map → command-BJ7KqczP.js.map} +1 -1
  42. package/dist/{command-aUJvENwy.js → command-BXLZsEwQ.js} +16 -16
  43. package/dist/{command-aUJvENwy.js.map → command-BXLZsEwQ.js.map} +1 -1
  44. package/dist/{command-B5OyHpOu.js → command-DA_SHQUQ.js} +16 -16
  45. package/dist/{command-B5OyHpOu.js.map → command-DA_SHQUQ.js.map} +1 -1
  46. package/dist/{commands-CWy-WndG.js → commands-BKBamOcX.js} +174 -40
  47. package/dist/commands-BKBamOcX.js.map +1 -0
  48. package/dist/{commands-CfdBX-mc.js → commands-BPvAOEmb.js} +1 -1
  49. package/dist/{config-B9mB1kF5.js → config-C0URar0t.js} +255 -89
  50. package/dist/config-C0URar0t.js.map +1 -0
  51. package/dist/{context-Bkv3ndjq.js → context-Sj58aJEu.js} +2 -2
  52. package/dist/{context-Bkv3ndjq.js.map → context-Sj58aJEu.js.map} +1 -1
  53. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  54. package/dist/cost-C21gGrju.js.map +1 -0
  55. package/dist/cost-IM80LyGD.js +1 -0
  56. package/dist/{cursor-catalog-BSn9xjmo.js → cursor-catalog-FWgSo-T1.js} +4 -4
  57. package/dist/{cursor-catalog-BSn9xjmo.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
  58. package/dist/db-Bc4L-Wcr.js +2 -0
  59. package/dist/{db-zgC1tvKB.js → db-BjAWxVAG.js} +142 -248
  60. package/dist/db-BjAWxVAG.js.map +1 -0
  61. package/dist/debug-CIXzB4wK.js.map +1 -1
  62. package/dist/{default-effort-BAy8ixd1.js → default-effort-CWlhZ6U1.js} +1 -1
  63. package/dist/{default-effort-DGhQmu95.js → default-effort-Ck2GYIxX.js} +4 -4
  64. package/dist/{default-effort-DGhQmu95.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
  65. package/dist/{dist-CAFD1yfv.js → dist-DeqeCUZe.js} +15 -8
  66. package/dist/dist-DeqeCUZe.js.map +1 -0
  67. package/dist/{docker-DAMnm3L9.js → docker-CDdK6Ity.js} +49 -22
  68. package/dist/docker-CDdK6Ity.js.map +1 -0
  69. package/dist/{docker-Cqw-CyWz.js → docker-D74l67k8.js} +4 -4
  70. package/dist/{docker-Cqw-CyWz.js.map → docker-D74l67k8.js.map} +1 -1
  71. package/dist/{docker-Dr4XeDc8.js → docker-DMug3l-a.js} +1 -1
  72. package/dist/{doctor-B2PohneB.js → doctor-eFxyn6at.js} +85 -46
  73. package/dist/doctor-eFxyn6at.js.map +1 -0
  74. package/dist/{environment-DB8o3hGi.js → environment-CJIKz-Ud.js} +49 -11
  75. package/dist/environment-CJIKz-Ud.js.map +1 -0
  76. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  77. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  78. package/dist/{eval-source-D3sbAzt-.js → eval-source-CXIRJ91v.js} +24 -12
  79. package/dist/eval-source-CXIRJ91v.js.map +1 -0
  80. package/dist/{evaluate-CwKeN_fo.js → evaluate-DA4QsrHO.js} +792 -281
  81. package/dist/evaluate-DA4QsrHO.js.map +1 -0
  82. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
  83. package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
  84. package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
  85. package/dist/{files-BiFU16BW.js → files-BVVWAS0A.js} +6 -6
  86. package/dist/{files-BiFU16BW.js.map → files-BVVWAS0A.js.map} +1 -1
  87. package/dist/files-BaqXvB88.js +1 -0
  88. package/dist/{files-BEwfjD8J.js → files-Bey1VAex.js} +3 -3
  89. package/dist/{files-BEwfjD8J.js.map → files-Bey1VAex.js.map} +1 -1
  90. package/dist/files-BjOJw4sn.js +1 -0
  91. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  92. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  93. package/dist/files-CNHhSyb3.js +1 -0
  94. package/dist/files-CW1lChNh.js +1 -0
  95. package/dist/{files-BRzV7aPh.js → files-D6j-XpaK.js} +5 -5
  96. package/dist/{files-BRzV7aPh.js.map → files-D6j-XpaK.js.map} +1 -1
  97. package/dist/{files-Cl7BQ5gE.js → files-F44NRCN0.js} +143 -20
  98. package/dist/files-F44NRCN0.js.map +1 -0
  99. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  100. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  101. package/dist/gateway.d.ts +1 -1
  102. package/dist/{git-fDiJ8UIV.js → git-Jqkf9SBa.js} +3 -3
  103. package/dist/{git-fDiJ8UIV.js.map → git-Jqkf9SBa.js.map} +1 -1
  104. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  105. package/dist/harness-FcTyyPco.js.map +1 -0
  106. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  107. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  108. package/dist/harness.d.ts +11 -8
  109. package/dist/harness.js +1 -1
  110. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  111. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  112. package/dist/{images-BNgcR77K.js → images-Dsc4Wi-F.js} +65 -30
  113. package/dist/images-Dsc4Wi-F.js.map +1 -0
  114. package/dist/index.d.ts +46 -7
  115. package/dist/index.js +6 -6
  116. package/dist/init-C0oUG3jh.js +1 -0
  117. package/dist/{init-DbuZVRrW.js → init-QB5ecPDA.js} +33 -14
  118. package/dist/init-QB5ecPDA.js.map +1 -0
  119. package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
  120. package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
  121. package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
  122. package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
  123. package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
  124. package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
  125. package/dist/judge.d.ts +3 -1
  126. package/dist/judge.js +1 -1
  127. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  128. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  129. package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
  130. package/dist/layout-DfQa4mJs.js.map +1 -0
  131. package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
  132. package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
  133. package/dist/live-DwjWJTdn.js +1 -0
  134. package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
  135. package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
  136. package/dist/{live-CnGrRM6j.js → live-y1p3oNXH.js} +9 -9
  137. package/dist/{live-CnGrRM6j.js.map → live-y1p3oNXH.js.map} +1 -1
  138. package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
  139. package/dist/model-3pZZzloU.js.map +1 -0
  140. package/dist/{model-047Dcd-a.js → model-DEjwetkJ.js} +1 -1
  141. package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
  142. package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
  143. package/dist/model-calls-B0BoQfJ4.js +95 -0
  144. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  145. package/dist/{model-DJjLw-0r.js → model-jRKSQBg6.js} +5 -5
  146. package/dist/{model-DJjLw-0r.js.map → model-jRKSQBg6.js.map} +1 -1
  147. package/dist/model-nfewGTzB.js +1 -0
  148. package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
  149. package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
  150. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  151. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  152. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
  153. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
  154. package/dist/options-DpsJ_5CU.js +1 -0
  155. package/dist/{patch-DH4e_Gkd.js → patch-Bx-qp8Li.js} +2 -2
  156. package/dist/{patch-DH4e_Gkd.js.map → patch-Bx-qp8Li.js.map} +1 -1
  157. package/dist/pi-providers-HygmL1y-.js +552 -0
  158. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  159. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  160. package/dist/plain-Ctn8oUdp.js.map +1 -0
  161. package/dist/{plan-Cgq9NCEX.js → plan-ChO0A-xn.js} +63 -12
  162. package/dist/plan-ChO0A-xn.js.map +1 -0
  163. package/dist/{plugins-BkmPxDiY.js → plugins-BioMWncI.js} +636 -9
  164. package/dist/plugins-BioMWncI.js.map +1 -0
  165. package/dist/{preflight-R-InZCQb.js → preflight-BUjk_6wT.js} +1 -1
  166. package/dist/{preflight-CtlloeOQ.js → preflight-CJSGzMzc.js} +8 -9
  167. package/dist/preflight-CJSGzMzc.js.map +1 -0
  168. package/dist/preflight-DD3fI4Hz.js +1 -0
  169. package/dist/{preflight-BElE_q0u.js → preflight-qFCXmxha.js} +4 -4
  170. package/dist/{preflight-BElE_q0u.js.map → preflight-qFCXmxha.js.map} +1 -1
  171. package/dist/{prepare-C7f5mPl5.js → prepare-D7oSSFNm.js} +9 -9
  172. package/dist/{prepare-C7f5mPl5.js.map → prepare-D7oSSFNm.js.map} +1 -1
  173. package/dist/{prepare-CzkAdbXm.js → prepare-xHU2BpN-.js} +2 -2
  174. package/dist/prices-B1vm_lPk.js +1 -0
  175. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  176. package/dist/prices-BrY_ihLs.js.map +1 -0
  177. package/dist/{provenance-zsc4--dP.js → provenance-B-flBAzR.js} +5 -5
  178. package/dist/{provenance-zsc4--dP.js.map → provenance-B-flBAzR.js.map} +1 -1
  179. package/dist/provider-DHT20Gl8.js +1 -0
  180. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  181. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  182. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  183. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  184. package/dist/{providers-B8xYMeah.js → providers-B6StVyfG.js} +215 -48
  185. package/dist/providers-B6StVyfG.js.map +1 -0
  186. package/dist/providers-Dljf6al6.js +1 -0
  187. package/dist/query-BAc2tQZj.js +1 -0
  188. package/dist/{query-Bb3SYICk.js → query-CkLmJv7R.js} +2 -2
  189. package/dist/{query-Bb3SYICk.js.map → query-CkLmJv7R.js.map} +1 -1
  190. package/dist/{questions-CWlR9ffu.js → questions-CJIh2FSX.js} +7 -7
  191. package/dist/{questions-CWlR9ffu.js.map → questions-CJIh2FSX.js.map} +1 -1
  192. package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
  193. package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
  194. package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
  195. package/dist/redact-DL2BTykw.js.map +1 -0
  196. package/dist/{remarks-vtkPmQb9.js → remarks-B3wjKscW.js} +46 -11
  197. package/dist/remarks-B3wjKscW.js.map +1 -0
  198. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  199. package/dist/remarks-BpdM_2RC.js.map +1 -0
  200. package/dist/remarks-CleoDlYw.js +1 -0
  201. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  202. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  203. package/dist/{remote-BgYWqogI.js → remote-C88amFRN.js} +6 -6
  204. package/dist/{remote-BgYWqogI.js.map → remote-C88amFRN.js.map} +1 -1
  205. package/dist/resolve-Bul8lHaa.js +205 -0
  206. package/dist/resolve-Bul8lHaa.js.map +1 -0
  207. package/dist/resolve-DW92wpIZ.js +1 -0
  208. package/dist/{resolve-Do0ZaYH6.js → resolve-G576PiFJ.js} +14 -11
  209. package/dist/resolve-G576PiFJ.js.map +1 -0
  210. package/dist/{results-JyuM64gy.js → results-zVHIW6xw.js} +9 -22
  211. package/dist/results-zVHIW6xw.js.map +1 -0
  212. package/dist/{resume-AW5oXzUc.js → resume-D_PcUXfG.js} +94 -16
  213. package/dist/resume-D_PcUXfG.js.map +1 -0
  214. package/dist/{retry-ChzISxmy.js → retry-BvTzCxdc.js} +4 -4
  215. package/dist/{retry-ChzISxmy.js.map → retry-BvTzCxdc.js.map} +1 -1
  216. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  217. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  218. package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
  219. package/dist/routes-B0F2R-K7.js.map +1 -0
  220. package/dist/{runtime-BpW1f1Mx.js → runtime-B3r_pAq6.js} +4 -4
  221. package/dist/{runtime-BpW1f1Mx.js.map → runtime-B3r_pAq6.js.map} +1 -1
  222. package/dist/sandbox/boat.d.ts +1 -1
  223. package/dist/sandbox/boat.js +1 -1
  224. package/dist/{sandbox-kSgr7vo5.js → sandbox-Duh3z1TV.js} +3 -3
  225. package/dist/{sandbox-kSgr7vo5.js.map → sandbox-Duh3z1TV.js.map} +1 -1
  226. package/dist/sandbox.d.ts +1 -1
  227. package/dist/sandbox.js +1 -1
  228. package/dist/server-CGEsvVRG.js +1 -0
  229. package/dist/{server-Cn8VqO9S.js → server-DUWMRKik.js} +8 -8
  230. package/dist/{server-Cn8VqO9S.js.map → server-DUWMRKik.js.map} +1 -1
  231. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  232. package/dist/shared-DtZwIAfy.js.map +1 -0
  233. package/dist/{source-CYOhR98B.js → source-CHuN4Nfg.js} +2 -2
  234. package/dist/{source-CYOhR98B.js.map → source-CHuN4Nfg.js.map} +1 -1
  235. package/dist/source-Wz1Nrfue.js +1 -0
  236. package/dist/src--yl1OypK.js +2 -0
  237. package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
  238. package/dist/src-B-HuDqpH.js.map +1 -0
  239. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  240. package/dist/src-B5r1d23Y.js.map +1 -0
  241. package/dist/{src-Ck2r_EMT.js → src-BHJ4MWvY.js} +504 -119
  242. package/dist/src-BHJ4MWvY.js.map +1 -0
  243. package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
  244. package/dist/src-BI0v8qlZ.js.map +1 -0
  245. package/dist/src-BL-_0QoE.js +1 -0
  246. package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
  247. package/dist/src-BqNzHHWn.js.map +1 -0
  248. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  249. package/dist/src-C-ChdnG6.js.map +1 -0
  250. package/dist/src-C-mOSczH.js +2 -0
  251. package/dist/src-CZaYu6ER.js +3 -0
  252. package/dist/src-CeBhwD8k.js +2 -0
  253. package/dist/src-CkEiiyON.js +2 -0
  254. package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
  255. package/dist/src-CmA5Yev6.js.map +1 -0
  256. package/dist/src-CwH_nEaw.js +2 -0
  257. package/dist/src-DA95rM6v.js +4 -0
  258. package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
  259. package/dist/src-DL-EE4DU.js.map +1 -0
  260. package/dist/src-De0KY5S_.js +2 -0
  261. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  262. package/dist/src-boB6D-5T.js.map +1 -0
  263. package/dist/state-B3jex5KU.js +1 -0
  264. package/dist/{state-BOLt8pJg.js → state-DUBRDyMk.js} +22 -22
  265. package/dist/{state-BOLt8pJg.js.map → state-DUBRDyMk.js.map} +1 -1
  266. package/dist/store/local.d.ts +8 -0
  267. package/dist/store/local.js +16 -0
  268. package/dist/store/local.js.map +1 -0
  269. package/dist/store/s3.d.ts +10 -0
  270. package/dist/store/s3.js +18 -0
  271. package/dist/store/s3.js.map +1 -0
  272. package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
  273. package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
  274. package/dist/suite-Bhs9-Z7O.js +2 -0
  275. package/dist/{summary-DU5zaibx.js → summary-CSp4cfnr.js} +25 -7
  276. package/dist/{summary-DU5zaibx.js.map → summary-CSp4cfnr.js.map} +1 -1
  277. package/dist/sync-CxAY6Dki.js +559 -0
  278. package/dist/sync-CxAY6Dki.js.map +1 -0
  279. package/dist/tart-CcrrtSrp.js +704 -0
  280. package/dist/tart-CcrrtSrp.js.map +1 -0
  281. package/dist/tart-CzViW3WZ.js +186 -0
  282. package/dist/tart-CzViW3WZ.js.map +1 -0
  283. package/dist/tart-DmVuVsmX.js +1 -0
  284. package/dist/tart-bake-F-6seRC9.js +192 -0
  285. package/dist/tart-bake-F-6seRC9.js.map +1 -0
  286. package/dist/tart-bake-RK7RaT9E.js +1 -0
  287. package/dist/tart-cli-B86Jqo1t.js +89 -0
  288. package/dist/tart-cli-B86Jqo1t.js.map +1 -0
  289. package/dist/tart-cli-ClvRniLQ.js +1 -0
  290. package/dist/tart-hosts-Brii7zHM.js +1 -0
  291. package/dist/tart-hosts-DPyZsg-1.js +313 -0
  292. package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
  293. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
  294. package/dist/tart-images-CWSf03w8.js.map +1 -0
  295. package/dist/testing.d.ts +36 -2
  296. package/dist/testing.js +4 -4
  297. package/dist/tool-class-CVDdJy0H.js +433 -0
  298. package/dist/tool-class-CVDdJy0H.js.map +1 -0
  299. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  300. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  301. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  302. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  303. package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
  304. package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
  305. package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
  306. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  307. package/dist/types-CbcnJxCy.js.map +1 -0
  308. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  309. package/dist/usage-CwIRPPZ9.js.map +1 -0
  310. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  311. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  312. package/dist/vertex-DXfcXYKn.js +1 -0
  313. package/dist/view/index.d.ts +27 -3
  314. package/dist/view/index.js +8 -8
  315. package/dist/{views-DAIiLiV3.js → views-DRxXCyIH.js} +6 -6
  316. package/dist/{views-DAIiLiV3.js.map → views-DRxXCyIH.js.map} +1 -1
  317. package/dist/views-i67V1ly3.js +2 -0
  318. package/dist/{worker-entry-BIROTPX1.js → worker-entry-C3Q4U63F.js} +12 -9
  319. package/dist/worker-entry-C3Q4U63F.js.map +1 -0
  320. package/dist/workspace-DhUOW8if.js +1 -0
  321. package/dist/{workspace-uuE51x35.js → workspace-P72SXLXr.js} +7 -7
  322. package/dist/{workspace-uuE51x35.js.map → workspace-P72SXLXr.js.map} +1 -1
  323. package/package.json +21 -10
  324. package/skills/eval-rubric/SKILL.md +10 -0
  325. package/skills/reeval/SKILL.md +370 -0
  326. package/skills/reeval/references/capture.md +48 -0
  327. package/skills/reeval/references/cli.md +162 -0
  328. package/skills/reeval/references/compare.md +180 -0
  329. package/skills/reeval/references/config.md +147 -0
  330. package/skills/reeval/references/environments.md +218 -0
  331. package/skills/reeval/references/evals.md +241 -0
  332. package/skills/reeval/references/mistakes.md +39 -0
  333. package/skills/reeval/references/vocabulary.md +19 -0
  334. package/skills/setup/SKILL.md +219 -0
  335. package/skills/setup/references/templates.md +258 -0
  336. package/dist/adapter-C8xPuXYq.js.map +0 -1
  337. package/dist/agent-CBoLndM7.js.map +0 -1
  338. package/dist/agents-C6zhkxp3.js.map +0 -1
  339. package/dist/app-files-DT5mJyV4.js.map +0 -1
  340. package/dist/assemble-t6h018An.js.map +0 -1
  341. package/dist/attempt-host-CIm1-11m.js.map +0 -1
  342. package/dist/boat-CJCtzfB1.js.map +0 -1
  343. package/dist/capture-DoxFUN3J.js +0 -1
  344. package/dist/cases-Dy0mifal.js +0 -1
  345. package/dist/cases-z9E6DCd5.js.map +0 -1
  346. package/dist/cli-20Fpvs5H.js.map +0 -1
  347. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  348. package/dist/client-KM4C5r5_.js +0 -387
  349. package/dist/client-KM4C5r5_.js.map +0 -1
  350. package/dist/command-CRh-cb6Z.js.map +0 -1
  351. package/dist/commands-CWy-WndG.js.map +0 -1
  352. package/dist/config-B9mB1kF5.js.map +0 -1
  353. package/dist/cost-BdRgySQl.js.map +0 -1
  354. package/dist/cost-Cr26Alhq.js +0 -1
  355. package/dist/db-P2fXw4nx.js +0 -2
  356. package/dist/db-zgC1tvKB.js.map +0 -1
  357. package/dist/dist-CAFD1yfv.js.map +0 -1
  358. package/dist/docker-DAMnm3L9.js.map +0 -1
  359. package/dist/doctor-B2PohneB.js.map +0 -1
  360. package/dist/environment-DB8o3hGi.js.map +0 -1
  361. package/dist/eval-source-D3sbAzt-.js.map +0 -1
  362. package/dist/evaluate-CwKeN_fo.js.map +0 -1
  363. package/dist/files-B4KQUiM7.js +0 -1
  364. package/dist/files-C1Rk9FYA.js +0 -1
  365. package/dist/files-C3mCC8Mn.js +0 -1
  366. package/dist/files-Cl7BQ5gE.js.map +0 -1
  367. package/dist/files-DnlA1A7N.js +0 -1
  368. package/dist/harness-DXv2gnaT.js.map +0 -1
  369. package/dist/images-BNgcR77K.js.map +0 -1
  370. package/dist/init-4RPeD-K-.js +0 -1
  371. package/dist/init-DbuZVRrW.js.map +0 -1
  372. package/dist/kinds-C1oY3qyC.js +0 -697
  373. package/dist/kinds-C1oY3qyC.js.map +0 -1
  374. package/dist/layout-Cm85tQYk.js.map +0 -1
  375. package/dist/live-CJCK11nh.js +0 -1
  376. package/dist/matchers-CXLn-1jm.js +0 -148
  377. package/dist/matchers-CXLn-1jm.js.map +0 -1
  378. package/dist/model-Cv4GuTsZ.js +0 -1
  379. package/dist/model-CwwkrCb7.js.map +0 -1
  380. package/dist/options-CHubslQf.js +0 -1
  381. package/dist/plain-BYUf_fUV.js.map +0 -1
  382. package/dist/plan-Cgq9NCEX.js.map +0 -1
  383. package/dist/plugins-BkmPxDiY.js.map +0 -1
  384. package/dist/preflight-4gWSy0Pe.js +0 -1
  385. package/dist/preflight-CtlloeOQ.js.map +0 -1
  386. package/dist/prices-5m3SzMZ3.js.map +0 -1
  387. package/dist/provider-BnUoz4KB.js +0 -1
  388. package/dist/providers-B8xYMeah.js.map +0 -1
  389. package/dist/providers-CVu8e0xS.js +0 -1
  390. package/dist/query-DIfkUcPh.js +0 -1
  391. package/dist/redact-BhjtnjrU.js.map +0 -1
  392. package/dist/remarks-CAvTSq16.js +0 -1
  393. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  394. package/dist/remarks-vtkPmQb9.js.map +0 -1
  395. package/dist/resolve-Do0ZaYH6.js.map +0 -1
  396. package/dist/resolve-amw_2Odz.js +0 -1
  397. package/dist/results-JyuM64gy.js.map +0 -1
  398. package/dist/resume-AW5oXzUc.js.map +0 -1
  399. package/dist/server-2_gUnVvf.js +0 -1
  400. package/dist/shared-Cfi45Bvy.js.map +0 -1
  401. package/dist/source-CY3etKzw.js +0 -1
  402. package/dist/src-6Zzyg4Qd.js +0 -2
  403. package/dist/src-BDhcPtnI.js +0 -2
  404. package/dist/src-BNQznj4z.js.map +0 -1
  405. package/dist/src-BRMsk9nh.js +0 -2
  406. package/dist/src-CV_7DkPa.js.map +0 -1
  407. package/dist/src-Cb2MWozp.js.map +0 -1
  408. package/dist/src-Ck2r_EMT.js.map +0 -1
  409. package/dist/src-DAzaCTcN.js.map +0 -1
  410. package/dist/src-DB7N8TYJ.js +0 -2
  411. package/dist/src-DPDaAYwh.js +0 -2
  412. package/dist/src-Dbk19BiP.js +0 -2
  413. package/dist/src-DtodMT59.js +0 -1
  414. package/dist/src-DvGns8cz.js.map +0 -1
  415. package/dist/src-Dw79VLa3.js.map +0 -1
  416. package/dist/src-KxS91P7L.js.map +0 -1
  417. package/dist/src-T-OeuJLr.js +0 -2
  418. package/dist/src-YvblOrBU.js.map +0 -1
  419. package/dist/src-zntEDlnx.js +0 -3
  420. package/dist/state-kimtIhBP.js +0 -1
  421. package/dist/suite-CKBa6MKt.js +0 -2
  422. package/dist/tart-BTVXp8rz.js +0 -373
  423. package/dist/tart-BTVXp8rz.js.map +0 -1
  424. package/dist/tart-cli-BIC9HqPg.js +0 -37
  425. package/dist/tart-cli-BIC9HqPg.js.map +0 -1
  426. package/dist/tart-cli-BMrkmBjI.js +0 -1
  427. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  428. package/dist/tool-target-DS5d72eH.js +0 -40
  429. package/dist/tool-target-DS5d72eH.js.map +0 -1
  430. package/dist/types-C2mLpph3.js.map +0 -1
  431. package/dist/usage-DJiDnh2D.js.map +0 -1
  432. package/dist/vercel-CGlSBv63.js.map +0 -1
  433. package/dist/vertex-O88TCwgY.js +0 -1
  434. package/dist/views-DtRhj2Tn.js +0 -2
  435. package/dist/worker-entry-BIROTPX1.js.map +0 -1
  436. package/dist/workspace-CcQRT31C.js +0 -1
@@ -0,0 +1,180 @@
1
+ # compare, cells, lift and questions
2
+
3
+ ## Contents
4
+
5
+ - What `compare` runs
6
+ - Gates and lift
7
+ - Effort and contrasts
8
+ - Labeled checks that discriminate
9
+ - `questions`
10
+
11
+ ## What `compare` runs
12
+
13
+ `compare` answers "does this environment help?". `compare: { control, variants }` names
14
+ both sides: the control is what each variant is measured against, and `control` is required.
15
+
16
+ Each variant runs as that environment on its own, with the same prompt and checks. `compare`
17
+ does not layer, inherit or extend. Share setup first (the example is in SKILL.md).
18
+
19
+ - Attempts = evals × harnesses × (1 control + variants) × runs. The example is 1 eval, 2 cells
20
+ (`<harness>/plain/<sandbox>` control, `<harness>/house-style/<sandbox>` variant), 8 attempts.
21
+ - With `runs` unset at every level, a compare with variants runs 4 a cell, the fewest tries a
22
+ side a lift verdict needs (the plan notes `runs 4 (compare default)`); anything else runs once,
23
+ including a compare with two or more `cases`: its lift pairs cases, so add cases rather than
24
+ runs (the plan notes `runs 1 a case`).
25
+ Any level that sets `runs`, `runs: 1` and `--runs` included, replaces it, and a profile's
26
+ `limit.runs` caps it. A compare with `cases` runs each case once: its lift pools over the cases.
27
+ Don't set `runs: 1` in the config unless you mean every compare to run once.
28
+ - `parent.extendEnvironment(name, spec)` appends the child's spec as a new layer. The child wins
29
+ on shared keys; `setup` concatenates. Call it before `compare`. Two stacks that should not
30
+ share are two `defineEnvironment` calls, then `{ control, variants }`.
31
+ - `environments` beside `compare` (on the same object) is exit 2: move it to `compare.control`.
32
+ An inherited `environments` is not applied as files, skills or variables. A side that omits
33
+ `sandbox` uses the cascaded `sandbox` option, else implicit Docker; `.extend()` keeps the
34
+ parent env's sandbox when the child omits it.
35
+ - There is no `strategy` field. A profile's `limit.compare: false` strips variants and runs
36
+ the control, which then gates like an eval without compare (as does `maxVariants: 0`), whatever
37
+ `compare.gate` says. `compare: false` on `evaluate` or `group` is an error.
38
+
39
+ ## Gates and lift
40
+
41
+ - Only variant cells gate by default (`gate?: 'variants' | 'all' | 'observe'`). A control's misses
42
+ show in its arm's count (`plain 0 of 2`) and never fail the run. `compare.harnesses: [cursor({ model: 'composer-2.5' })]` compares on some harnesses only.
43
+ - With `runs > 1`, a cell passes only if every counted attempt passes (pass^k), unless
44
+ `gate: { passRate: 0.8 }` (a pass@1 threshold; pass@1 is passes over passes and fails). Infra errors don't count; a cell is `infra_error`
45
+ only when nothing counted.
46
+ - **Lift** is variant pass@1 minus control pass@1, with a 95% interval. The result block
47
+ prints it as an effect word after the variant's count: `helped`, `made it worse`, or `can't
48
+ tell yet` below four tries a side or when the interval crosses 0; `-v` adds the difference
49
+ and interval. `reeval query factor_effects` and `reeval analyze` use the same four: below
50
+ it the lift is `insufficient` (not measured), never a results line. When the eval's sources changed across the compared runs, that pass@1
51
+ difference is `not comparable: the eval changed`, not an effect, and mixed counts such as
52
+ `6/10 vs 8/9` are not a results line.
53
+ - `compare: { control, variants: [x], minLift: 0 }` fails a variant whose interval's lower
54
+ bound is below 0.
55
+
56
+ ## Effort and contrasts
57
+
58
+ - From four tries a side the result block adds an **effort** table: each arm's median tool
59
+ calls, agent time and, when both sides reported usage, tokens, which often shows a skill's
60
+ effect before pass@1 does:
61
+ `effort tools time tokens` / `plain 54 1m 39s 2,890k` / `with-house-greeting 3 6.6s 57k`.
62
+ A cell without usage is left out of the token sample only; if either side then
63
+ has none, the token metric is omitted, not `insufficient`. Effort's rank test needs
64
+ `runs: 4` a cell, as lift does (below it lift shows only a direction and effort can't be tested; the plan hints so, at `runs: 1` too); `reeval query effort` has
65
+ its p-values, verdicts and whether the ranges are `separated`.
66
+ - **One comparison per variant** against its baseline covers every measure: pass rate, each
67
+ check's pass rate, each judge's scores, each compare question's votes, tool calls, agent time
68
+ and tokens. Each is `better`, `worse`, `same` or `leans` (a difference its own test doesn't hold
69
+ yet, at p below 0.1 and of at least 10 points, 0.05 of a judge's score, 60/40 of 4 or more
70
+ decisive pairs for a question at p below 0.2, or 10% of an effort median, with the runs that would hold it;
71
+ weaker is `same`, unless the sample is too small for its test to show anything: "12× faster,
72
+ but 3 runs a side can't confirm it; 4 is the least that can"). A question counts votes, both
73
+ orders of each pair, as its standings do ("in 8 of 8 votes (4 of 4 decisive pairs)"). Each measure keeps its own 0.05, so with identical arms some measure reads
74
+ better or worse by chance in about 1 run in 14 at 4 a side, and 1 in 7 at 8 or more. The status list prints the ones that hold under the
75
+ variant (`Same pass rate · 5× faster.`), `reeval compare -v` the leans too, and
76
+ `reeval query measure_effects` has a row per measure. When both arms pass every run, a measure
77
+ that holds is the result (no remark), one that leans raises `too-few-tries` with the runs, and
78
+ all the same raises `cant-tell-apart`.
79
+ - With two variants or more, `reeval query contrasts` sets every pair of arms side by side,
80
+ variant against variant too.
81
+ - `reeval query skill_files` says which declared files (`SKILL.md`, references, rules) each
82
+ attempt opened, and `context_use` counts its tool calls as `context`, `dependency`
83
+ (`node_modules`, `--help`), `web`, `verify` or `workspace`: a heuristic, read it as counts.
84
+
85
+ ## Labeled checks that discriminate
86
+
87
+ - **Label each requirement** with `expect`'s second argument:
88
+ `await expect(c, 'exports greet').toHaveFile('greet.js', /export/)`. Every matcher call
89
+ is recorded (`evals[].attempts[].assertions`; unlabeled ones get `<matcher> <first arg>`).
90
+ In a compare, keep independent requirements as `expect`, not `expect.require`: a failed
91
+ `expect.require` ends the attempt, and the checks after it count as not run: a side that never reached a check reads
92
+ `not run` and the check gets no verdict, so it can't say which arm met it. The result block's
93
+ check table puts a row per check under each arm (`house greeting 0 of 2 2 of 2`) with the
94
+ first failure's message, and folds the checks that passed everywhere into `+2 more passed`.
95
+ After a run, `reeval query "SELECT eval_key, label, verdict FROM check_discrimination WHERE verdict
96
+ LIKE 'both-%'"` and rework or drop those checks: they measure nothing about the variant.
97
+ Changing a label changes the eval's fingerprint. Label judges too
98
+ (`expect(await c.judge(q, { diff: true }), 'Social OAuth').toPass()`): the label replaces the clipped
99
+ question in judge rates, `reeval show`, `summary.md` and the digest.
100
+
101
+ ## `questions`
102
+
103
+ ```ts
104
+ evaluate('greets in the house style', {
105
+ compare: { control: plain, variants: [houseStyle] },
106
+ questions: [
107
+ // asked of each attempt as it ends: free text, or a pick from choices
108
+ { question: 'How did the agent pick its greeting?', transcript: true },
109
+ {
110
+ question: 'Does greet.js handle a missing name?',
111
+ diff: true,
112
+ choices: ['yes', 'no'],
113
+ },
114
+ // pairwise: which side did better, after the run
115
+ {
116
+ question: 'Which greet.js reads more like our house style?',
117
+ across: 'environment',
118
+ diff: true,
119
+ },
120
+ {
121
+ question: 'Which sign-in page matches the mock?',
122
+ across: 'environment',
123
+ artifacts: ['sign-in'],
124
+ },
125
+ ],
126
+ }, async (c) => { … })
127
+ ```
128
+
129
+ - `questions` sits beside `compare`, on an eval or a group (never the config or a
130
+ profile), and works without a `compare` too. It's always a list of objects:
131
+ `{ question, across?, choices?, model?, turns?, …inputs }`, with at least one input
132
+ turned on (`task`, `output`, `files`, `diff`, `transcript`, `checks`, `artifacts`;
133
+ no `contents`, the workspaces are gone). There are no default inputs: an entry with
134
+ none exits 2 at plan time. An eval's list replaces its group's.
135
+ - **No `across`: asked of each attempt** as it ends with a pass or a fail (infra
136
+ errors and skipped attempts aren't asked), while the run goes on. `choices`
137
+ (two or more) makes the model pick one, with a reason; the result block prints
138
+ the tally (`yes 8 · no 4`), else the first few answers. Answers land on
139
+ `evals[].attempts[].answers` and in `reeval query attempt_answers` (with the
140
+ attempt's outcome, `harness`, `environment` and `role`, so you can split
141
+ answers by harness, arm or pass/fail).
142
+ Ask these where a control is expected to fail: every attempt is asked.
143
+ - **`across`: pairwise.** `'environment'` pairs two environments on one harness;
144
+ `'harness'` two harnesses on one environment; a list asks both pair kinds.
145
+ The factor must vary in the eval (else exit 2); a profile that narrows it away
146
+ drops the question (`1 pairwise question off (profile …)`). `choices` with
147
+ `across` exits 2. `--dry-run` shows the pair count. The result block prints
148
+ each side's wins (`with-house-greeting won 4 of 4`). Each pair is voted in
149
+ both orders; one whose orders disagree is a split (the position decided). With
150
+ three arms or more, read `reeval query question_pairs` (head to head, with
151
+ splits), not the pooled wins, which count a split as a win for each side. A
152
+ vote may carry a short `reason` (capped); do not gate on its wording.
153
+ - Questions annotate and never gate: pass, fail and the exit code stay the
154
+ checks'. `reeval analyze` puts each answer in the digest as annotation, not a
155
+ results line. Their model is the entry's `model`, else the cascaded
156
+ `question.model`, else `analysis.model`, else the run's judge (a profile's
157
+ included) when it's a language model (not `typesafe-ai/*`); with none, `reeval run` exits 2 at plan
158
+ time while `--dry-run` still lists them. Keep deterministic checks for pass or
159
+ fail.
160
+ - Each side shows only the entry's inputs (`diff: ['src/**']`,
161
+ `transcript: 'tools', checks: ['npm test']`); `turns` defaults to `'all'`. Keep
162
+ `output` off for pairwise questions, since the more confident summary wins on
163
+ wording. For UI evals, `artifacts: ['sign-in']` (or `true`) sends each attempt's
164
+ own capture of that name as an image. The `question` option takes only `model`
165
+ (config → profile → group → eval → entry). A pairwise question showing
166
+ `transcript` across harnesses warns (tool names unblind them): ask it with
167
+ `across: 'environment'`.
168
+ - Pairwise questions are answered on the host once all of an eval's attempts
169
+ end. Attempts that failed their checks (a judge failure doesn't count) are
170
+ left out of the pairs: a pairwise question on a compare whose baseline is
171
+ expected to fail gets no pairs, so ask it of each attempt instead. Each pair
172
+ goes to the model twice, order swapped, labeled A and B; the record's
173
+ `evals[].answers` has every vote and each side's `wins`/`n` over decided
174
+ votes, per pair kind. A model error is stored on the answer as `error` and
175
+ warned about; it never changes an outcome. To see what a question was shown,
176
+ open its `input`: `<eval-slug>/questions/<n>.json` for a pairwise one (each
177
+ pair and order's masked prompt, which side was A, screenshot refs), or
178
+ `questions/<n>.json` in the attempt's directory. Test with
179
+ `fakeAnalysis({ vote: 'A' | 'B' | 'tie' | (pair) => … | { error } })` and
180
+ `fakeAnalysis({ answer: 'text' | { choice, text } | (q) => … | { error } })`.
@@ -0,0 +1,147 @@
1
+ # reeval.config.ts, harnesses and options
2
+
3
+ ## Contents
4
+
5
+ - reeval.yaml
6
+ - Path rule and where the sandbox comes from
7
+ - Harnesses, keys and effort
8
+ - Judge models and imports
9
+ - Options and the cascade
10
+
11
+ ## reeval.yaml
12
+
13
+ A project without TypeScript can write `reeval.yaml` (or `.yml`) instead, with no callbacks:
14
+
15
+ ```yaml
16
+ harnesses: { kind: cursor, model: composer-2.5 }
17
+ judge: { model: typesafe-ai/jev-1.14.0 } # or { kind: fake-judge, score: 0.9 }
18
+ env:
19
+ NPM_TOKEN: { kind: secret, hosts: [registry.npmjs.org] }
20
+ environments:
21
+ kind: environment
22
+ name: base
23
+ layers: [{ name: base, spec: { setup: [npm ci] } }]
24
+ ```
25
+
26
+ A `reeval.yaml` beside a `reeval.config.*` is an error.
27
+
28
+ ## Path rule and where the sandbox comes from
29
+
30
+ One path rule for both: a relative host path resolves against the directory of the file it's
31
+ written in (the config's, an eval file's, or the module calling `defineEnvironment` or a helper),
32
+ whether it comes from a helper (`file()`, `skill.local()`, `skill.npm()`, `git()`,
33
+ `docker({ dockerfile })`, `fakeSandbox({ dir })`) or its data form (`{ kind: file, path }`,
34
+ `{ kind: fake, dir }`, …). Absolute paths stay as written.
35
+
36
+ | Where the sandbox comes from | When |
37
+ |---|---|
38
+ | The environment's `sandbox` | It sets one; `extendEnvironment` keeps the parent's when the child omits it |
39
+ | The cascaded `sandbox` option (`defineConfig({ sandbox })`) | The resolved environment omits it; it's the default machine, not an environment |
40
+ | Implicit Docker (reeval's image: Node 26, git, pnpm, the agent CLI) | Neither is set |
41
+
42
+ A named env is a total override: no skills, files or sandbox inherit from the config
43
+ environment. A fake profile or config puts `sandbox: fakeSandbox()` on `defaults` or
44
+ `defineConfig` so `plain` and skill variants run without Docker; that's for trying configs, not
45
+ for validating checks (an environment's own `docker()` still wins). Don't put `fakeSandbox()`
46
+ on `plain` itself: real Cursor runs need Docker. An oracle or canary profile sets only
47
+ `defaults.harnesses`, so each eval keeps its sandbox: see
48
+ [evals.md](evals.md#trustworthy-evals).
49
+
50
+ ## Harnesses, keys and effort
51
+
52
+ Harnesses (`reeval/harness`), each with a required `model`; a list runs each as its own cell:
53
+
54
+ | Factory | Key | Notes |
55
+ |---|---|---|
56
+ | `cursor({ model, params?, effort? })` | `CURSOR_API_KEY` | Cursor's agent through the Cursor SDK. `params` such as `{ fast: 'true' }` pick the variant; `effort` maps onto the parameter `Cursor.models.list()` names for that model |
57
+ | `claudeCode({ model, effort? })` | `ANTHROPIC_API_KEY`, else a subscription's `CLAUDE_CODE_OAUTH_TOKEN` (`claude setup-token`) | `effort` is the Agent SDK's `/effort` |
58
+ | `antigravity({ model, effort? })` | `GEMINI_API_KEY` (a Gemini API key) | e.g. `gemini-3.8-flash-medium`; `effort` is `agy --effort` |
59
+ | `opencode({ model, effort? })` | That provider's key, e.g. `ANTHROPIC_API_KEY` | OpenCode 2; the model is `<provider>/<model>`, e.g. `google/gemini-3.8-flash`. `amazon-bedrock/…` takes `AWS_BEARER_TOKEN_BEDROCK` and an optional `AWS_REGION`; Vertex (on `direct()`; it runs through `vertex()`), Azure, Copilot and local providers are refused at plan time. Web search via Exa, whose host it allows; it also reads `.claude/skills` and `.agents/skills`. `effort` is refused where the model's API answers it with a 400 (any value on `anthropic/claude-haiku-4-5`, `openai/gpt-4.1` or `google/gemini-2.5-flash`, any but `high` on `openai/gpt-5-pro`), and such a model runs with none by default; on `openrouter()` nothing is refused |
60
+ | `goose({ model, effort? })` | The provider's key: `GOOGLE_GENERATIVE_AI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY` or `XAI_API_KEY` | `<provider>/<model>` on Goose's `google`, `anthropic`, `openai` or `xai` provider, e.g. `google/gemini-3.8-flash` (any gateway id on `vercel()` or `openrouter()`); `effort` is `GOOSE_THINKING_EFFORT` |
61
+ | `pi({ model, effort? })` | The provider's key, e.g. `GOOGLE_GENERATIVE_AI_API_KEY`, `ANTHROPIC_API_KEY`, `XAI_API_KEY`, `MISTRAL_API_KEY` | `<provider>/<model>` on any of pi's providers that sends its key in a header on a fixed host, e.g. `google/gemini-3.8-flash` (any gateway id on `vercel()` or `openrouter()`); `effort` is `--thinking` |
62
+ | `codex({ model, effort? })` | `OPENAI_API_KEY` | An OpenAI model id, e.g. `gpt-5.6-luna`; `effort` is `model_reasoning_effort` |
63
+ | `fx({ model, effort? })` | `OPENAI_API_KEY` for a bare id, `AI_GATEWAY_API_KEY` on `vercel()`, `OPENROUTER_API_KEY` on `openrouter()`, or none on `vertex()` | Vercel's fx over ACP: a gateway id (`anthropic/claude-sonnet-5`) runs only on `vercel()` or `openrouter()` in `gateway`, a bare OpenAI id (`gpt-5.5`) on OpenAI, which must accept function tools without `reasoning_effort` (not `gpt-5.6` or later), a Gemini id (`google/gemini-3.8-flash`) on `vertex()`. `effort` (`reasoning_effort`) only on `vertex()`. Its own web tools bypass the egress gateway, so `deny-all` blocks them |
64
+ | `grokBuild({ model, effort? })` | `XAI_API_KEY` | xAI's Grok Build CLI; an xAI model id, e.g. `grok-4.6`; `effort` is `default_reasoning_effort`. It finds MCP tools with its `search_tool` before calling them |
65
+ | `fakeHarness(…)` | None | See the fake trio in SKILL.md |
66
+
67
+ OpenCode, Goose, pi and Codex refuse `mcp.sse`. Cursor's `fast` defaults to `'false'` (standard
68
+ price; `{ fast: 'true' }` costs several times more). Other unset Cursor params take Cursor's
69
+ default, which the record shows (`composer-2.5[fast=false]`), so pin them for reproducible evals.
70
+
71
+ `effort` is the same field on every factory (`fx()` takes it only on `vertex()`), with whatever values that CLI takes (no shared
72
+ enum); Fast is not effort. The default harness id includes `[effort=<value>]` when it's set, and
73
+ the record stores the requested effort as `effort` and the served one as `servedEffort`.
74
+
75
+ In `reeval query` those are `reasoning_effort` (`cells`, `attempts`) and
76
+ `served_reasoning_effort` (`attempts`). **effort** on its own (`reeval query effort`, the
77
+ result block's effort table) is always the measured tool-call, time and token metric.
78
+
79
+ Keys never enter the sandbox: the agent sees a placeholder that reeval's gateway swaps on the
80
+ harness's own endpoints. Every harness reports tokens, which reeval records per turn,
81
+ attempt and run with their cost (reported, else estimated at list prices).
82
+
83
+ `gateway` (`reeval/gateway`) lists the routes model calls may take, in order: each harness, the
84
+ judge and analysis take the first that can serve them. Every list ends with `direct()` (each
85
+ provider's own key), so don't write it at the end: list it only to move it (`[direct(), vercel()]`
86
+ tries own keys first), or write `direct({ enabled: false })` for strict routing, where a cell no
87
+ entry serves stops the plan (never both). `gateway: [vercel()]` runs Claude
88
+ Code, Codex, Goose, pi, OpenCode, fx and Grok Build on Vercel AI Gateway with one `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`) (`effort`
89
+ on Goose, or on OpenCode with a model other than Gemini, falls back to `direct()`), and
90
+ everything else on its own key; the plan's hint says which cells fell back. Pin one with
91
+ the factory's or the judge's `gateway: 'vercel'`; a pin never falls back. `vertex({ project })`
92
+ (else `GOOGLE_CLOUD_PROJECT`) runs Claude Code, Antigravity, OpenCode, Goose, pi, Grok Build and fx (Gemini only) on Vertex AI
93
+ with no key: reeval mints a token from Application Default Credentials on the host and the egress
94
+ gateway adds it (OpenCode's Gemini model runs as `google-vertex/<id>`, Grok Build's xAI model as `xai/<id>` on Vertex's Chat Completions API).
95
+ There Claude Code sends `effort` only to the models its own catalog lists, so `effort` on one it
96
+ doesn't, such as `claude-haiku-5-5`, falls through, and with none set that model runs with no effort. The `sonnet`
97
+ and `opusplan` aliases fall through too: Claude Code resolves them on Vertex AI to Sonnet 4.5, which Anthropic retires on 2026-11-30, so name
98
+ `claude-sonnet-5-5` instead. Claude ids work in
99
+ either form (`claude-opus-4-5-20251101` is sent as Vertex's `claude-opus-4-5@20251101`), so one
100
+ string works through `[vertex()]` and its `direct()` fallback; `google/*` and `anthropic/*` judge and analysis models run there
101
+ too. `reeval doctor --online` checks the token, the project's Vertex AI API and that Vertex serves each model the config sends there (Claude needs enabling in Model Garden). Default `[direct()]`:
102
+ no gateway, so the judge, analysis and captures take only their provider's own key.
103
+ `gateway: [openrouter()]` runs Claude Code, Codex, Goose, OpenCode, fx, Grok Build and pi on
104
+ OpenRouter with one `OPENROUTER_API_KEY` (Cursor, Antigravity, and `effort` on fx, on Claude Code or
105
+ on Goose or OpenCode with a model other than Gemini, fall through). Every OpenRouter call records its upstream provider and cost
106
+ (`attempts[].upstreams`, `reportedCostUsd`; a judgment's `upstream`, `costUsd`).
107
+ `openrouter({ fallbacks: true })` lets reeval's own calls (judge, analysis, captures) fail over
108
+ between upstreams; the agents' calls take OpenRouter's routing whatever it says. Write OpenCode
109
+ ids as `anthropic/…`, never `openrouter/…` or `vercel/…`.
110
+
111
+ ## Judge models and imports
112
+
113
+ - Judge models: `'typesafe-ai/jev-1.14.0'` (`TYPESAFE_AI_API_KEY`, recommended),
114
+ `'google/gemini-3.8-flash'` (`GOOGLE_GENERATIVE_AI_API_KEY`), `anthropic/*` (`ANTHROPIC_API_KEY`)
115
+ and `'openai/gpt-6-luna'` (`OPENAI_API_KEY`; OpenAI's Decisions API takes only its decision
116
+ model, so not `openai/gpt-5.5`), each on its provider's own key. reeval uses no gateway by
117
+ default: `AI_GATEWAY_API_KEY` is read only with `gateway: [vercel()]` in the config (or a judge
118
+ `gateway: 'vercel'` pin, listed there), `OPENROUTER_API_KEY` only with `openrouter()`. As a
119
+ judge, `vercel()` serves decision models only (Jev, `openai/gpt-6-luna-decisions`), and
120
+ `openrouter()` only those on its Decisions API (`typesafe/jev-1.14`), never a language model:
121
+ pin a language-model judge to `gateway: 'direct'`. Always a provider prefix.
122
+ With a `gateway` list set, a judge model takes the first listed gateway that serves it, and only
123
+ that gateway's key.
124
+ - Imports: `reeval` (`defineConfig`, `defineEnvironment`, `group`, `evaluate`,
125
+ `expect`, `DEFAULT_EGRESS_ALLOW`, `DEFAULT_SKILL_EXCLUDE`, types), `/harness`, `/gateway`
126
+ (`direct`, `vercel`), `/sandbox`
127
+ (`docker`, `fakeSandbox`), `/judge` (`fakeJudge`, `RECOMMENDED_JUDGE_MODEL`,
128
+ `DEFAULT_THRESHOLD`). There is no `test`, `describe`, `baseline`, `secret` or `skill` export.
129
+
130
+ ## Options and the cascade
131
+
132
+ `defineConfig`, `group` and `evaluate` take the same options, and inner levels win:
133
+
134
+ | Option | Cascades as |
135
+ |---|---|
136
+ | `harness` | One or a list, or an edit of the inherited list: `{ add: h }`, `{ only: 'cursor:composer-2.5' }` or a function |
137
+ | `sandbox` | The default machine when the resolved env omits one |
138
+ | `environment` | One; a named env is a total override of the config environment |
139
+ | `env` | Merged by name |
140
+ | `judge` | Merged by field |
141
+ | `capture` | Shallow-merged camera defaults (`viewport`, `timeout`, `browser`, `model`, the vision model); `viewport` replaces the value, and the port is not an option |
142
+ | `runs`, `gate`, `egress`, `timeout` | Replaced (`runs` unset everywhere: 4 for a compare with variants, else 1) |
143
+ | `tags`, `pin` | Unioned |
144
+ | `compare` | Groups and evals only |
145
+
146
+ Options are plain objects, never functions. Unknown keys are errors with a hint (`repeat` → "Did
147
+ you mean runs?").
@@ -0,0 +1,218 @@
1
+ # Environments, sandboxes, secrets and egress
2
+
3
+ ## Contents
4
+
5
+ - Environments: skills, files, rules, MCP, setup
6
+ - Fixture directories and `git()` workspaces
7
+ - Skills and composition
8
+ - Sandboxes
9
+ - Secrets and egress
10
+
11
+ ## Environments: skills, files, rules, MCP, setup
12
+
13
+ What the agent runs *in*, per attempt: the code the workspace starts as, skills, MCP servers,
14
+ files, rules, env vars, secrets and setup steps. Pass a spec, or a synchronous callback that
15
+ destructures the helpers it needs:
16
+
17
+ ```ts
18
+ // evals/envs.ts
19
+ import { defineEnvironment } from 'reeval'
20
+
21
+ export const houseStyle = defineEnvironment('house-style', ({ skill }) => ({
22
+ // a dir with SKILL.md, relative to this file
23
+ skills: [skill.local('../skills/house-style')],
24
+ }))
25
+
26
+ export const withDocs = houseStyle.extendEnvironment(
27
+ 'with-docs',
28
+ ({ file, mcp }) => ({
29
+ // written into the workspace
30
+ files: { 'docs/api.md': file('../docs/api.md') },
31
+ // AGENTS.md (CLAUDE.md for Claude Code)
32
+ rules: file('../rules/house.md'),
33
+ // or mcp.http(url) / mcp.sse(url) (not on OpenCode, Goose, pi or Codex)
34
+ mcp: { docs: mcp.stdio('npx', ['-y', '@acme/docs-mcp@1.0.0']) },
35
+ // every attempt, before the agent
36
+ setup: [{ run: 'npm ci', timeout: '5m' }],
37
+ }),
38
+ )
39
+
40
+ // Start from existing code: a repo, or one directory of it, at a pinned ref
41
+ export const app = defineEnvironment('acme-app', ({ git }) => ({
42
+ workspace: git('https://github.com/acme/app', {
43
+ ref: 'v2.3.0',
44
+ path: 'examples/basic',
45
+ }),
46
+ setup: ['npm ci'],
47
+ }))
48
+
49
+ // Or seed a fixture directory: a plain object of file() entries, so spread
50
+ // and override it
51
+ export const notes = defineEnvironment('acme-notes', ({ directory }) => ({
52
+ files: { ...directory('../fixture'), 'README.md': 'something' },
53
+ setup: [{ run: 'npm ci --no-audit --no-fund', timeout: '5m' }],
54
+ }))
55
+ ```
56
+
57
+ ## Fixture directories and `git()` workspaces
58
+
59
+ Use `directory(path, { into?, glob?, defaultIgnores? })` for a fixture directory instead of
60
+ mapping a file list through `file()`. It skips `.git`, `node_modules/`, `dist/`, top-level
61
+ `build/`, caches and editor folders (`DEFAULT_DIRECTORY_IGNORES`; `defaultIgnores: false` keeps
62
+ them, except `.git`). `glob` is ordered gitignore-style patterns where the last match wins and
63
+ `!` excludes: `glob: ['src/**', 'package.json', '!**/*.test.ts']`.
64
+
65
+ Executable files stay executable; empty directories aren't seeded.
66
+
67
+ `workspace: git(repo, { ref, path })` takes a URL, `github:owner/repo`, a GitHub `/tree/` URL or
68
+ a local path (relative to the file). reeval fetches it on the host before any attempt, with the
69
+ host's git credentials or `token: 'GITHUB_TOKEN'` (a host variable's name; never a token in the
70
+ URL or the eval). It copies the tree in before `files` (which win) and `setup`, and records the
71
+ commit and tree hash.
72
+
73
+ `init: true` adds a `.git` with one commit of the tree (no remote), for agents that expect a
74
+ repo. Pin `ref` to a tag or full SHA; a branch warns. The agent's diff starts after setup, so the
75
+ repo never counts as its change.
76
+
77
+ A bad ref, path, URL or token fails only the attempts that use it (infra error, exit 3).
78
+ Submodules and LFS files aren't fetched.
79
+
80
+ ## Skills and composition
81
+
82
+ Skills: `skill.local | git | npm | registry | inline` (pin git refs; `placement: 'workspace'`
83
+ writes them into the repo's skill directory). A git, npm or plugin source URL with a password or
84
+ token in it is refused: reeval fetches on the host with its own git and npm credentials.
85
+ `extendEnvironment` is the only composition; the child wins on shared keys and **replaces**
86
+ `sandbox` entirely when it sets one (install is not merged onto the parent image).
87
+
88
+ A skill is a directory whose `SKILL.md` has YAML frontmatter
89
+ (`name`, `description`; quote values containing `: `). Baking (`from`, `install`, `dockerfile`)
90
+ is `sandbox: docker({ … })` on the environment.
91
+
92
+ Plugins: `plugins: [plugin.local('../plugins/review', { only: ['skills', 'mcp'] })]` (also
93
+ `plugin.git(repo, { ref, path })`, `plugin.npm(spec)` and
94
+ `plugin.marketplace('name@github:owner/repo', { ref })` for an entry in a Claude Code
95
+ marketplace, or `'name@npm:<spec>'`; only a marketplace in a local directory may name `file://`
96
+ entries). A plugin is Claude Code's layout:
97
+ `.claude-plugin/plugin.json` plus `skills/`, `agents/`, `commands/`, `hooks/hooks.json`,
98
+ `.mcp.json`, `rules/`. Its MCP servers join `mcp` as `<plugin>:<server>` (call them from the
99
+ fake agent as `{ mcp: 'review:docs/<tool>' }`), and it lands in `~/.reeval/plugins/<name>/`. Claude
100
+ Code and Grok Build load every part but `rules` (narrow with `only`), namespaced
101
+ (`review:reviewer`), and start the plugin's MCP servers themselves, so a plugin server reading a
102
+ host variable is refused there: declare it in `mcp`. Antigravity loads every part with
103
+ `agy plugin install`, and refuses such a server too; Codex loads `skills` and `mcp` the same way
104
+ (and refuses a plugin with ChatGPT apps). Cursor loads every part flat: rules, subagents, commands
105
+ and hooks unpacked into the workspace's `.cursor/` (left out of the diff), skills into
106
+ `~/.agents/skills`, so two items with one name clash; Goose takes `skills` and `mcp` beside the
107
+ eval's own, so a clashing name is refused too. OpenCode loads skills, MCP servers, and subagents
108
+ and commands (unpacked into `.opencode/`, frontmatter cut to `description`); pi skills, MCP servers
109
+ (as `<plugin>_<server>`) and commands (as prompt templates in `.pi/prompts/`); fx skills and MCP
110
+ servers only. A part a harness doesn't load, or a hook event it doesn't run, is refused at plan
111
+ time (`--dry-run` exits 2 naming the harness): narrow with `only`. `build: 'npm ci && npm run build'` (or `{ run, cwd: '.' }` to run
112
+ from the source's root) builds the plugin in a cached Docker image layer, as root with no
113
+ secrets, without the source's `.git` (pass a version in its commands instead of `git describe`);
114
+ only `path` is placed, so the output must land inside it. Docker only. Its content is fixed when the
115
+ image is prepared: an npm spec re-resolving or a local plugin edited mid-run waits for the next run. A server run with
116
+ `npx`/`uvx` needs its registry in `egress`, or a build that installs it.
117
+
118
+ A named env is a total override: no skills, files or sandbox inherit from the config
119
+ environment.
120
+
121
+ ## Sandboxes
122
+
123
+ ```ts
124
+ // reeval's Node image with the agent CLI
125
+ docker()
126
+ // baked once as a cached layer; no secrets
127
+ docker({ install: ['npm i -g firebase-tools@14'] })
128
+ // a Debian or Ubuntu base; reeval adds Node and the agent
129
+ docker({ from: 'python:3.12-slim' })
130
+ // ARG REEVAL_BASE and FROM ${REEVAL_BASE} to layer on reeval's image
131
+ docker({ dockerfile: './sandbox/Dockerfile' })
132
+ // limits; defaults 4g, 2 and 1024
133
+ docker({ memory: '8g', cpus: 4, pids: 4096 })
134
+ // swap on top; default the same as memory, 0 for none
135
+ docker({ memory: '4g', swap: 0 })
136
+ // opt in to apt-get at run time (default: user reeval)
137
+ docker({ root: true })
138
+ // the same sandbox and options on rootless Podman 4+ (cell id `podman`)
139
+ podman({ from: 'python:3.12-slim' })
140
+ ```
141
+
142
+ Use `podman()` only where the user runs Podman instead of Docker: it takes every `docker()`
143
+ option and confines the container the same way (bridge network, Docker's seccomp profile).
144
+
145
+ `tart({ image })` (an OCI macOS image or a local Tart VM) runs each attempt in a cloned macOS VM
146
+ on an Apple Silicon Mac, at most two at once. It needs `egress: 'allow-all'`: under the default
147
+ `proxy` or a restricted policy the plan exits 2. Of the real harnesses only `cursor()` runs in it
148
+ so far (any other exits 2 at plan time), and it needs Node 22.21 or later in the image; the fake
149
+ harness runs too.
150
+ `reeval run` never pulls the image: one `tart list` doesn't show exits 2, naming `tart pull
151
+ <image>`. Run `npx reeval image build` first, which pulls every `tart()` image the plan needs (and
152
+ refuses, exit 3, when the disk can't hold them even once Tart prunes its cache). Pin a digest
153
+ (`ghcr.io/cirruslabs/macos-sequoia-base@sha256:…`): Cirrus Labs publishes only `latest`, so a
154
+ version tag isn't there to pin. `:latest` or no tag warns, and each attempt records the digest it
155
+ cloned in `vmImage`.
156
+
157
+ ```ts
158
+ import { boat, createBoat } from 'reeval/sandbox/boat'
159
+ // one boat.dev VM per attempt, over SSH (cell id `boat`, or `boat:large`)
160
+ defineEnvironment('vm', { sandbox: boat({ type: 'large' }), egress: 'allow-all' })
161
+ ```
162
+
163
+ `boat()` needs `npm i @boatdev/sdk`, `BOAT_API_KEY` (or `createBoat({ apiKey, prefix })` for
164
+ another account) and an OpenSSH client, and plans only under `egress: 'allow-all'` and on the
165
+ Cursor harness so far; reeval refuses the others at plan time. It takes no image options: put the
166
+ toolchain's installs in the environment's `setup`.
167
+
168
+ The agent, setup and `c.exec` run as `reeval` (home `/home/reeval`) with no capabilities; builds
169
+ run as root. `root: true` or `capabilities` print a warning at the start of every run, so prefer
170
+ `install`. A process killed at the memory limit makes the attempt an infra error
171
+ (`error.resource.cause: 'oom'`), not a failed check: raise `memory` rather than loosening it.
172
+
173
+ Put a specific machine on `defineEnvironment`, or `sandbox` on `defineConfig`, a profile, group
174
+ or eval as the default when the resolved env omits one. A list is an error. `npx reeval image
175
+ build` builds every image the plan needs.
176
+
177
+ The cell key includes the image (`from`, `dockerfile`, `install`), so Node 22 and 24 can't share
178
+ a cell. Environments that share an image still share one preflight build.
179
+
180
+ ## Secrets and egress
181
+
182
+ Secrets come from a `defineConfig` or `defineEnvironment` callback, never literals:
183
+
184
+ ```ts
185
+ defineEnvironment('gemini', ({ secret }) => ({
186
+ env: {
187
+ GEMINI_API_KEY: secret.fromEnv('GEMINI_API_KEY', {
188
+ hosts: secret.hosts.gemini,
189
+ }),
190
+ },
191
+ }))
192
+ ```
193
+
194
+ With `hosts`, the sandbox only sees a placeholder and reeval's gateway swaps in the real value on
195
+ HTTPS requests to those hosts. `{ to: 'checks' }` keeps it out of the agent's own environment
196
+ (only `c.exec` and setup steps get it). Without `hosts`, a process a setup step leaves running (a
197
+ server the checks call) holds the real value while the agent works, where the agent, as the same
198
+ user, can read it: broker it with `hosts`, or start the process from `c.exec` after the turn. The
199
+ `setup-process-secret` remark flags it.
200
+
201
+ `egress: 'proxy' | 'allow-all' | 'deny-all' | { allow: [hosts] }` limits where the sandbox may
202
+ connect (the harness's API, brokered hosts and MCP hosts are always allowed; `DEFAULT_EGRESS_ALLOW`
203
+ is npm, Yarn, PyPI and GitHub). A blocked request doesn't fail the attempt; it's recorded, and your
204
+ checks decide. The record's `egress.blocked` is what the policy refused; a name that didn't resolve
205
+ (`metadata.google.internal` outside Google Cloud) is in `egress.unresolved` instead.
206
+
207
+ The default, `'proxy'`, reaches any host over HTTP(S) through the egress gateway and nothing
208
+ else: SSH, a database driver or a raw TCP/UDP client gets `Network is unreachable` (and the run's
209
+ `proxy-only-egress` remark). Set `egress: 'allow-all'` on just the evals that need that; it keeps
210
+ the container's network, so such traffic goes out unrecorded (the record's `egress.partial`).
211
+
212
+ `egress: { setup, agent }` gives each phase its own policy, in `docker()` and `podman()` only:
213
+ `setup` covers the files and setup steps, `agent` the harness's start, every turn and every check.
214
+ `{ setup: 'allow-all', agent: 'deny-all' }` lets setup install dependencies while the agent
215
+ reaches only its API and the automatic hosts. A phase left out keeps the policy it inherits, and
216
+ the record tags each traffic entry with its `phase`. A connection setup leaves open (a background
217
+ process's tunnel) to a host the agent's policy refuses is closed as the agent starts, and the
218
+ `setup-tunnel-closed` remark flags it.