reeval 0.19.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (436) hide show
  1. package/CHANGELOG.md +803 -0
  2. package/README.md +15 -15
  3. package/THIRD_PARTY_NOTICES.md +5 -5
  4. package/assets/gateway-models.mjs +65 -31
  5. package/dist/{adapter-C8xPuXYq.js → adapter-BdzhdRYo.js} +8 -6
  6. package/dist/adapter-BdzhdRYo.js.map +1 -0
  7. package/dist/{agent-CBoLndM7.js → agent-Dx-bDmlV.js} +179 -31
  8. package/dist/agent-Dx-bDmlV.js.map +1 -0
  9. package/dist/{agents-C6zhkxp3.js → agents-l0-k7vw3.js} +362 -102
  10. package/dist/agents-l0-k7vw3.js.map +1 -0
  11. package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
  12. package/dist/app-files-B-CQYB0c.js.map +1 -0
  13. package/dist/{assemble-UPCRrtOz.js → assemble-CYMpN8LC.js} +1 -1
  14. package/dist/{assemble-t6h018An.js → assemble-CsZIcJo8.js} +171 -18
  15. package/dist/assemble-CsZIcJo8.js.map +1 -0
  16. package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
  17. package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
  18. package/dist/{attempt-host-CIm1-11m.js → attempt-host-BwSmNYkX.js} +17 -8
  19. package/dist/attempt-host-BwSmNYkX.js.map +1 -0
  20. package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
  21. package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
  22. package/dist/{boat-CJCtzfB1.js → boat-su1Y7LPd.js} +17 -14
  23. package/dist/boat-su1Y7LPd.js.map +1 -0
  24. package/dist/bridge/index.mjs +113 -8
  25. package/dist/{capture-iBzFiC0j.js → capture-BqIAeFyC.js} +55 -29
  26. package/dist/{capture-iBzFiC0j.js.map → capture-BqIAeFyC.js.map} +1 -1
  27. package/dist/capture-D0Ln5aCP.js +1 -0
  28. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  29. package/dist/cases-CM-MQRQp.js.map +1 -0
  30. package/dist/cases-FvhbM8M5.js +1 -0
  31. package/dist/{cli-20Fpvs5H.js → cli-s_3opn_I.js} +219 -109
  32. package/dist/cli-s_3opn_I.js.map +1 -0
  33. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
  34. package/dist/cli-spec-DxzIjmq_.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/client-C5fIzHQa.js +803 -0
  37. package/dist/client-C5fIzHQa.js.map +1 -0
  38. package/dist/{command-CRh-cb6Z.js → command-B5OAIjp7.js} +25 -21
  39. package/dist/command-B5OAIjp7.js.map +1 -0
  40. package/dist/{command-BLNVDiIg.js → command-BJ7KqczP.js} +10 -10
  41. package/dist/{command-BLNVDiIg.js.map → command-BJ7KqczP.js.map} +1 -1
  42. package/dist/{command-aUJvENwy.js → command-BXLZsEwQ.js} +16 -16
  43. package/dist/{command-aUJvENwy.js.map → command-BXLZsEwQ.js.map} +1 -1
  44. package/dist/{command-B5OyHpOu.js → command-DA_SHQUQ.js} +16 -16
  45. package/dist/{command-B5OyHpOu.js.map → command-DA_SHQUQ.js.map} +1 -1
  46. package/dist/{commands-CWy-WndG.js → commands-BKBamOcX.js} +174 -40
  47. package/dist/commands-BKBamOcX.js.map +1 -0
  48. package/dist/{commands-CfdBX-mc.js → commands-BPvAOEmb.js} +1 -1
  49. package/dist/{config-B9mB1kF5.js → config-C0URar0t.js} +255 -89
  50. package/dist/config-C0URar0t.js.map +1 -0
  51. package/dist/{context-Bkv3ndjq.js → context-Sj58aJEu.js} +2 -2
  52. package/dist/{context-Bkv3ndjq.js.map → context-Sj58aJEu.js.map} +1 -1
  53. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  54. package/dist/cost-C21gGrju.js.map +1 -0
  55. package/dist/cost-IM80LyGD.js +1 -0
  56. package/dist/{cursor-catalog-BSn9xjmo.js → cursor-catalog-FWgSo-T1.js} +4 -4
  57. package/dist/{cursor-catalog-BSn9xjmo.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
  58. package/dist/db-Bc4L-Wcr.js +2 -0
  59. package/dist/{db-zgC1tvKB.js → db-BjAWxVAG.js} +142 -248
  60. package/dist/db-BjAWxVAG.js.map +1 -0
  61. package/dist/debug-CIXzB4wK.js.map +1 -1
  62. package/dist/{default-effort-BAy8ixd1.js → default-effort-CWlhZ6U1.js} +1 -1
  63. package/dist/{default-effort-DGhQmu95.js → default-effort-Ck2GYIxX.js} +4 -4
  64. package/dist/{default-effort-DGhQmu95.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
  65. package/dist/{dist-CAFD1yfv.js → dist-DeqeCUZe.js} +15 -8
  66. package/dist/dist-DeqeCUZe.js.map +1 -0
  67. package/dist/{docker-DAMnm3L9.js → docker-CDdK6Ity.js} +49 -22
  68. package/dist/docker-CDdK6Ity.js.map +1 -0
  69. package/dist/{docker-Cqw-CyWz.js → docker-D74l67k8.js} +4 -4
  70. package/dist/{docker-Cqw-CyWz.js.map → docker-D74l67k8.js.map} +1 -1
  71. package/dist/{docker-Dr4XeDc8.js → docker-DMug3l-a.js} +1 -1
  72. package/dist/{doctor-B2PohneB.js → doctor-eFxyn6at.js} +85 -46
  73. package/dist/doctor-eFxyn6at.js.map +1 -0
  74. package/dist/{environment-DB8o3hGi.js → environment-CJIKz-Ud.js} +49 -11
  75. package/dist/environment-CJIKz-Ud.js.map +1 -0
  76. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  77. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  78. package/dist/{eval-source-D3sbAzt-.js → eval-source-CXIRJ91v.js} +24 -12
  79. package/dist/eval-source-CXIRJ91v.js.map +1 -0
  80. package/dist/{evaluate-CwKeN_fo.js → evaluate-DA4QsrHO.js} +792 -281
  81. package/dist/evaluate-DA4QsrHO.js.map +1 -0
  82. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
  83. package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
  84. package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
  85. package/dist/{files-BiFU16BW.js → files-BVVWAS0A.js} +6 -6
  86. package/dist/{files-BiFU16BW.js.map → files-BVVWAS0A.js.map} +1 -1
  87. package/dist/files-BaqXvB88.js +1 -0
  88. package/dist/{files-BEwfjD8J.js → files-Bey1VAex.js} +3 -3
  89. package/dist/{files-BEwfjD8J.js.map → files-Bey1VAex.js.map} +1 -1
  90. package/dist/files-BjOJw4sn.js +1 -0
  91. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  92. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  93. package/dist/files-CNHhSyb3.js +1 -0
  94. package/dist/files-CW1lChNh.js +1 -0
  95. package/dist/{files-BRzV7aPh.js → files-D6j-XpaK.js} +5 -5
  96. package/dist/{files-BRzV7aPh.js.map → files-D6j-XpaK.js.map} +1 -1
  97. package/dist/{files-Cl7BQ5gE.js → files-F44NRCN0.js} +143 -20
  98. package/dist/files-F44NRCN0.js.map +1 -0
  99. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  100. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  101. package/dist/gateway.d.ts +1 -1
  102. package/dist/{git-fDiJ8UIV.js → git-Jqkf9SBa.js} +3 -3
  103. package/dist/{git-fDiJ8UIV.js.map → git-Jqkf9SBa.js.map} +1 -1
  104. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  105. package/dist/harness-FcTyyPco.js.map +1 -0
  106. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  107. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  108. package/dist/harness.d.ts +11 -8
  109. package/dist/harness.js +1 -1
  110. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  111. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  112. package/dist/{images-BNgcR77K.js → images-Dsc4Wi-F.js} +65 -30
  113. package/dist/images-Dsc4Wi-F.js.map +1 -0
  114. package/dist/index.d.ts +46 -7
  115. package/dist/index.js +6 -6
  116. package/dist/init-C0oUG3jh.js +1 -0
  117. package/dist/{init-DbuZVRrW.js → init-QB5ecPDA.js} +33 -14
  118. package/dist/init-QB5ecPDA.js.map +1 -0
  119. package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
  120. package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
  121. package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
  122. package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
  123. package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
  124. package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
  125. package/dist/judge.d.ts +3 -1
  126. package/dist/judge.js +1 -1
  127. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  128. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  129. package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
  130. package/dist/layout-DfQa4mJs.js.map +1 -0
  131. package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
  132. package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
  133. package/dist/live-DwjWJTdn.js +1 -0
  134. package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
  135. package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
  136. package/dist/{live-CnGrRM6j.js → live-y1p3oNXH.js} +9 -9
  137. package/dist/{live-CnGrRM6j.js.map → live-y1p3oNXH.js.map} +1 -1
  138. package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
  139. package/dist/model-3pZZzloU.js.map +1 -0
  140. package/dist/{model-047Dcd-a.js → model-DEjwetkJ.js} +1 -1
  141. package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
  142. package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
  143. package/dist/model-calls-B0BoQfJ4.js +95 -0
  144. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  145. package/dist/{model-DJjLw-0r.js → model-jRKSQBg6.js} +5 -5
  146. package/dist/{model-DJjLw-0r.js.map → model-jRKSQBg6.js.map} +1 -1
  147. package/dist/model-nfewGTzB.js +1 -0
  148. package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
  149. package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
  150. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  151. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  152. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
  153. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
  154. package/dist/options-DpsJ_5CU.js +1 -0
  155. package/dist/{patch-DH4e_Gkd.js → patch-Bx-qp8Li.js} +2 -2
  156. package/dist/{patch-DH4e_Gkd.js.map → patch-Bx-qp8Li.js.map} +1 -1
  157. package/dist/pi-providers-HygmL1y-.js +552 -0
  158. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  159. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  160. package/dist/plain-Ctn8oUdp.js.map +1 -0
  161. package/dist/{plan-Cgq9NCEX.js → plan-ChO0A-xn.js} +63 -12
  162. package/dist/plan-ChO0A-xn.js.map +1 -0
  163. package/dist/{plugins-BkmPxDiY.js → plugins-BioMWncI.js} +636 -9
  164. package/dist/plugins-BioMWncI.js.map +1 -0
  165. package/dist/{preflight-R-InZCQb.js → preflight-BUjk_6wT.js} +1 -1
  166. package/dist/{preflight-CtlloeOQ.js → preflight-CJSGzMzc.js} +8 -9
  167. package/dist/preflight-CJSGzMzc.js.map +1 -0
  168. package/dist/preflight-DD3fI4Hz.js +1 -0
  169. package/dist/{preflight-BElE_q0u.js → preflight-qFCXmxha.js} +4 -4
  170. package/dist/{preflight-BElE_q0u.js.map → preflight-qFCXmxha.js.map} +1 -1
  171. package/dist/{prepare-C7f5mPl5.js → prepare-D7oSSFNm.js} +9 -9
  172. package/dist/{prepare-C7f5mPl5.js.map → prepare-D7oSSFNm.js.map} +1 -1
  173. package/dist/{prepare-CzkAdbXm.js → prepare-xHU2BpN-.js} +2 -2
  174. package/dist/prices-B1vm_lPk.js +1 -0
  175. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  176. package/dist/prices-BrY_ihLs.js.map +1 -0
  177. package/dist/{provenance-zsc4--dP.js → provenance-B-flBAzR.js} +5 -5
  178. package/dist/{provenance-zsc4--dP.js.map → provenance-B-flBAzR.js.map} +1 -1
  179. package/dist/provider-DHT20Gl8.js +1 -0
  180. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  181. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  182. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  183. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  184. package/dist/{providers-B8xYMeah.js → providers-B6StVyfG.js} +215 -48
  185. package/dist/providers-B6StVyfG.js.map +1 -0
  186. package/dist/providers-Dljf6al6.js +1 -0
  187. package/dist/query-BAc2tQZj.js +1 -0
  188. package/dist/{query-Bb3SYICk.js → query-CkLmJv7R.js} +2 -2
  189. package/dist/{query-Bb3SYICk.js.map → query-CkLmJv7R.js.map} +1 -1
  190. package/dist/{questions-CWlR9ffu.js → questions-CJIh2FSX.js} +7 -7
  191. package/dist/{questions-CWlR9ffu.js.map → questions-CJIh2FSX.js.map} +1 -1
  192. package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
  193. package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
  194. package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
  195. package/dist/redact-DL2BTykw.js.map +1 -0
  196. package/dist/{remarks-vtkPmQb9.js → remarks-B3wjKscW.js} +46 -11
  197. package/dist/remarks-B3wjKscW.js.map +1 -0
  198. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  199. package/dist/remarks-BpdM_2RC.js.map +1 -0
  200. package/dist/remarks-CleoDlYw.js +1 -0
  201. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  202. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  203. package/dist/{remote-BgYWqogI.js → remote-C88amFRN.js} +6 -6
  204. package/dist/{remote-BgYWqogI.js.map → remote-C88amFRN.js.map} +1 -1
  205. package/dist/resolve-Bul8lHaa.js +205 -0
  206. package/dist/resolve-Bul8lHaa.js.map +1 -0
  207. package/dist/resolve-DW92wpIZ.js +1 -0
  208. package/dist/{resolve-Do0ZaYH6.js → resolve-G576PiFJ.js} +14 -11
  209. package/dist/resolve-G576PiFJ.js.map +1 -0
  210. package/dist/{results-JyuM64gy.js → results-zVHIW6xw.js} +9 -22
  211. package/dist/results-zVHIW6xw.js.map +1 -0
  212. package/dist/{resume-AW5oXzUc.js → resume-D_PcUXfG.js} +94 -16
  213. package/dist/resume-D_PcUXfG.js.map +1 -0
  214. package/dist/{retry-ChzISxmy.js → retry-BvTzCxdc.js} +4 -4
  215. package/dist/{retry-ChzISxmy.js.map → retry-BvTzCxdc.js.map} +1 -1
  216. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  217. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  218. package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
  219. package/dist/routes-B0F2R-K7.js.map +1 -0
  220. package/dist/{runtime-BpW1f1Mx.js → runtime-B3r_pAq6.js} +4 -4
  221. package/dist/{runtime-BpW1f1Mx.js.map → runtime-B3r_pAq6.js.map} +1 -1
  222. package/dist/sandbox/boat.d.ts +1 -1
  223. package/dist/sandbox/boat.js +1 -1
  224. package/dist/{sandbox-kSgr7vo5.js → sandbox-Duh3z1TV.js} +3 -3
  225. package/dist/{sandbox-kSgr7vo5.js.map → sandbox-Duh3z1TV.js.map} +1 -1
  226. package/dist/sandbox.d.ts +1 -1
  227. package/dist/sandbox.js +1 -1
  228. package/dist/server-CGEsvVRG.js +1 -0
  229. package/dist/{server-Cn8VqO9S.js → server-DUWMRKik.js} +8 -8
  230. package/dist/{server-Cn8VqO9S.js.map → server-DUWMRKik.js.map} +1 -1
  231. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  232. package/dist/shared-DtZwIAfy.js.map +1 -0
  233. package/dist/{source-CYOhR98B.js → source-CHuN4Nfg.js} +2 -2
  234. package/dist/{source-CYOhR98B.js.map → source-CHuN4Nfg.js.map} +1 -1
  235. package/dist/source-Wz1Nrfue.js +1 -0
  236. package/dist/src--yl1OypK.js +2 -0
  237. package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
  238. package/dist/src-B-HuDqpH.js.map +1 -0
  239. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  240. package/dist/src-B5r1d23Y.js.map +1 -0
  241. package/dist/{src-Ck2r_EMT.js → src-BHJ4MWvY.js} +504 -119
  242. package/dist/src-BHJ4MWvY.js.map +1 -0
  243. package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
  244. package/dist/src-BI0v8qlZ.js.map +1 -0
  245. package/dist/src-BL-_0QoE.js +1 -0
  246. package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
  247. package/dist/src-BqNzHHWn.js.map +1 -0
  248. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  249. package/dist/src-C-ChdnG6.js.map +1 -0
  250. package/dist/src-C-mOSczH.js +2 -0
  251. package/dist/src-CZaYu6ER.js +3 -0
  252. package/dist/src-CeBhwD8k.js +2 -0
  253. package/dist/src-CkEiiyON.js +2 -0
  254. package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
  255. package/dist/src-CmA5Yev6.js.map +1 -0
  256. package/dist/src-CwH_nEaw.js +2 -0
  257. package/dist/src-DA95rM6v.js +4 -0
  258. package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
  259. package/dist/src-DL-EE4DU.js.map +1 -0
  260. package/dist/src-De0KY5S_.js +2 -0
  261. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  262. package/dist/src-boB6D-5T.js.map +1 -0
  263. package/dist/state-B3jex5KU.js +1 -0
  264. package/dist/{state-BOLt8pJg.js → state-DUBRDyMk.js} +22 -22
  265. package/dist/{state-BOLt8pJg.js.map → state-DUBRDyMk.js.map} +1 -1
  266. package/dist/store/local.d.ts +8 -0
  267. package/dist/store/local.js +16 -0
  268. package/dist/store/local.js.map +1 -0
  269. package/dist/store/s3.d.ts +10 -0
  270. package/dist/store/s3.js +18 -0
  271. package/dist/store/s3.js.map +1 -0
  272. package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
  273. package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
  274. package/dist/suite-Bhs9-Z7O.js +2 -0
  275. package/dist/{summary-DU5zaibx.js → summary-CSp4cfnr.js} +25 -7
  276. package/dist/{summary-DU5zaibx.js.map → summary-CSp4cfnr.js.map} +1 -1
  277. package/dist/sync-CxAY6Dki.js +559 -0
  278. package/dist/sync-CxAY6Dki.js.map +1 -0
  279. package/dist/tart-CcrrtSrp.js +704 -0
  280. package/dist/tart-CcrrtSrp.js.map +1 -0
  281. package/dist/tart-CzViW3WZ.js +186 -0
  282. package/dist/tart-CzViW3WZ.js.map +1 -0
  283. package/dist/tart-DmVuVsmX.js +1 -0
  284. package/dist/tart-bake-F-6seRC9.js +192 -0
  285. package/dist/tart-bake-F-6seRC9.js.map +1 -0
  286. package/dist/tart-bake-RK7RaT9E.js +1 -0
  287. package/dist/tart-cli-B86Jqo1t.js +89 -0
  288. package/dist/tart-cli-B86Jqo1t.js.map +1 -0
  289. package/dist/tart-cli-ClvRniLQ.js +1 -0
  290. package/dist/tart-hosts-Brii7zHM.js +1 -0
  291. package/dist/tart-hosts-DPyZsg-1.js +313 -0
  292. package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
  293. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
  294. package/dist/tart-images-CWSf03w8.js.map +1 -0
  295. package/dist/testing.d.ts +36 -2
  296. package/dist/testing.js +4 -4
  297. package/dist/tool-class-CVDdJy0H.js +433 -0
  298. package/dist/tool-class-CVDdJy0H.js.map +1 -0
  299. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  300. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  301. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  302. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  303. package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
  304. package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
  305. package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
  306. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  307. package/dist/types-CbcnJxCy.js.map +1 -0
  308. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  309. package/dist/usage-CwIRPPZ9.js.map +1 -0
  310. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  311. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  312. package/dist/vertex-DXfcXYKn.js +1 -0
  313. package/dist/view/index.d.ts +27 -3
  314. package/dist/view/index.js +8 -8
  315. package/dist/{views-DAIiLiV3.js → views-DRxXCyIH.js} +6 -6
  316. package/dist/{views-DAIiLiV3.js.map → views-DRxXCyIH.js.map} +1 -1
  317. package/dist/views-i67V1ly3.js +2 -0
  318. package/dist/{worker-entry-BIROTPX1.js → worker-entry-C3Q4U63F.js} +12 -9
  319. package/dist/worker-entry-C3Q4U63F.js.map +1 -0
  320. package/dist/workspace-DhUOW8if.js +1 -0
  321. package/dist/{workspace-uuE51x35.js → workspace-P72SXLXr.js} +7 -7
  322. package/dist/{workspace-uuE51x35.js.map → workspace-P72SXLXr.js.map} +1 -1
  323. package/package.json +21 -10
  324. package/skills/eval-rubric/SKILL.md +10 -0
  325. package/skills/reeval/SKILL.md +370 -0
  326. package/skills/reeval/references/capture.md +48 -0
  327. package/skills/reeval/references/cli.md +162 -0
  328. package/skills/reeval/references/compare.md +180 -0
  329. package/skills/reeval/references/config.md +147 -0
  330. package/skills/reeval/references/environments.md +218 -0
  331. package/skills/reeval/references/evals.md +241 -0
  332. package/skills/reeval/references/mistakes.md +39 -0
  333. package/skills/reeval/references/vocabulary.md +19 -0
  334. package/skills/setup/SKILL.md +219 -0
  335. package/skills/setup/references/templates.md +258 -0
  336. package/dist/adapter-C8xPuXYq.js.map +0 -1
  337. package/dist/agent-CBoLndM7.js.map +0 -1
  338. package/dist/agents-C6zhkxp3.js.map +0 -1
  339. package/dist/app-files-DT5mJyV4.js.map +0 -1
  340. package/dist/assemble-t6h018An.js.map +0 -1
  341. package/dist/attempt-host-CIm1-11m.js.map +0 -1
  342. package/dist/boat-CJCtzfB1.js.map +0 -1
  343. package/dist/capture-DoxFUN3J.js +0 -1
  344. package/dist/cases-Dy0mifal.js +0 -1
  345. package/dist/cases-z9E6DCd5.js.map +0 -1
  346. package/dist/cli-20Fpvs5H.js.map +0 -1
  347. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  348. package/dist/client-KM4C5r5_.js +0 -387
  349. package/dist/client-KM4C5r5_.js.map +0 -1
  350. package/dist/command-CRh-cb6Z.js.map +0 -1
  351. package/dist/commands-CWy-WndG.js.map +0 -1
  352. package/dist/config-B9mB1kF5.js.map +0 -1
  353. package/dist/cost-BdRgySQl.js.map +0 -1
  354. package/dist/cost-Cr26Alhq.js +0 -1
  355. package/dist/db-P2fXw4nx.js +0 -2
  356. package/dist/db-zgC1tvKB.js.map +0 -1
  357. package/dist/dist-CAFD1yfv.js.map +0 -1
  358. package/dist/docker-DAMnm3L9.js.map +0 -1
  359. package/dist/doctor-B2PohneB.js.map +0 -1
  360. package/dist/environment-DB8o3hGi.js.map +0 -1
  361. package/dist/eval-source-D3sbAzt-.js.map +0 -1
  362. package/dist/evaluate-CwKeN_fo.js.map +0 -1
  363. package/dist/files-B4KQUiM7.js +0 -1
  364. package/dist/files-C1Rk9FYA.js +0 -1
  365. package/dist/files-C3mCC8Mn.js +0 -1
  366. package/dist/files-Cl7BQ5gE.js.map +0 -1
  367. package/dist/files-DnlA1A7N.js +0 -1
  368. package/dist/harness-DXv2gnaT.js.map +0 -1
  369. package/dist/images-BNgcR77K.js.map +0 -1
  370. package/dist/init-4RPeD-K-.js +0 -1
  371. package/dist/init-DbuZVRrW.js.map +0 -1
  372. package/dist/kinds-C1oY3qyC.js +0 -697
  373. package/dist/kinds-C1oY3qyC.js.map +0 -1
  374. package/dist/layout-Cm85tQYk.js.map +0 -1
  375. package/dist/live-CJCK11nh.js +0 -1
  376. package/dist/matchers-CXLn-1jm.js +0 -148
  377. package/dist/matchers-CXLn-1jm.js.map +0 -1
  378. package/dist/model-Cv4GuTsZ.js +0 -1
  379. package/dist/model-CwwkrCb7.js.map +0 -1
  380. package/dist/options-CHubslQf.js +0 -1
  381. package/dist/plain-BYUf_fUV.js.map +0 -1
  382. package/dist/plan-Cgq9NCEX.js.map +0 -1
  383. package/dist/plugins-BkmPxDiY.js.map +0 -1
  384. package/dist/preflight-4gWSy0Pe.js +0 -1
  385. package/dist/preflight-CtlloeOQ.js.map +0 -1
  386. package/dist/prices-5m3SzMZ3.js.map +0 -1
  387. package/dist/provider-BnUoz4KB.js +0 -1
  388. package/dist/providers-B8xYMeah.js.map +0 -1
  389. package/dist/providers-CVu8e0xS.js +0 -1
  390. package/dist/query-DIfkUcPh.js +0 -1
  391. package/dist/redact-BhjtnjrU.js.map +0 -1
  392. package/dist/remarks-CAvTSq16.js +0 -1
  393. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  394. package/dist/remarks-vtkPmQb9.js.map +0 -1
  395. package/dist/resolve-Do0ZaYH6.js.map +0 -1
  396. package/dist/resolve-amw_2Odz.js +0 -1
  397. package/dist/results-JyuM64gy.js.map +0 -1
  398. package/dist/resume-AW5oXzUc.js.map +0 -1
  399. package/dist/server-2_gUnVvf.js +0 -1
  400. package/dist/shared-Cfi45Bvy.js.map +0 -1
  401. package/dist/source-CY3etKzw.js +0 -1
  402. package/dist/src-6Zzyg4Qd.js +0 -2
  403. package/dist/src-BDhcPtnI.js +0 -2
  404. package/dist/src-BNQznj4z.js.map +0 -1
  405. package/dist/src-BRMsk9nh.js +0 -2
  406. package/dist/src-CV_7DkPa.js.map +0 -1
  407. package/dist/src-Cb2MWozp.js.map +0 -1
  408. package/dist/src-Ck2r_EMT.js.map +0 -1
  409. package/dist/src-DAzaCTcN.js.map +0 -1
  410. package/dist/src-DB7N8TYJ.js +0 -2
  411. package/dist/src-DPDaAYwh.js +0 -2
  412. package/dist/src-Dbk19BiP.js +0 -2
  413. package/dist/src-DtodMT59.js +0 -1
  414. package/dist/src-DvGns8cz.js.map +0 -1
  415. package/dist/src-Dw79VLa3.js.map +0 -1
  416. package/dist/src-KxS91P7L.js.map +0 -1
  417. package/dist/src-T-OeuJLr.js +0 -2
  418. package/dist/src-YvblOrBU.js.map +0 -1
  419. package/dist/src-zntEDlnx.js +0 -3
  420. package/dist/state-kimtIhBP.js +0 -1
  421. package/dist/suite-CKBa6MKt.js +0 -2
  422. package/dist/tart-BTVXp8rz.js +0 -373
  423. package/dist/tart-BTVXp8rz.js.map +0 -1
  424. package/dist/tart-cli-BIC9HqPg.js +0 -37
  425. package/dist/tart-cli-BIC9HqPg.js.map +0 -1
  426. package/dist/tart-cli-BMrkmBjI.js +0 -1
  427. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  428. package/dist/tool-target-DS5d72eH.js +0 -40
  429. package/dist/tool-target-DS5d72eH.js.map +0 -1
  430. package/dist/types-C2mLpph3.js.map +0 -1
  431. package/dist/usage-DJiDnh2D.js.map +0 -1
  432. package/dist/vercel-CGlSBv63.js.map +0 -1
  433. package/dist/vertex-O88TCwgY.js +0 -1
  434. package/dist/views-DtRhj2Tn.js +0 -2
  435. package/dist/worker-entry-BIROTPX1.js.map +0 -1
  436. package/dist/workspace-CcQRT31C.js +0 -1
@@ -0,0 +1,241 @@
1
+ # Evals: the attempt API, judges and matchers
2
+
3
+ ## Contents
4
+
5
+ - Cases: `cases`, `caseKey`, `where`, and `gate: 'observe'`
6
+ - The attempt: `c.send`, `c.exec`, `c.serve`, files
7
+ - The judge: inputs, input size, errors
8
+ - Trustworthy evals
9
+ - Matchers
10
+
11
+ ## Cases: `cases`, `caseKey`, `where`, and `gate: 'observe'`
12
+
13
+ - `evaluate(name, [options,] fn)` and `group(name, [options,] fn)`; also `.skip`, `.only`
14
+ (an error in CI), `.todo`. The eval id is `file › group › name`; renaming an eval restarts its history.
15
+ - `evaluate(name, { cases: [...] }, async (c, item) => …)` runs one eval over a list: one pass
16
+ rate over the cases and, with `compare`, one lift paired by case, which `minLift` gates (not
17
+ each case's). Prefer it to separate evals when the cases are one population: six cases at
18
+ `runs: 1` can reach a lift verdict where each case alone is within noise. Fewer than six
19
+ discordant cases shows only a direction (`insufficient` in `reeval query case_lift`), so
20
+ aim for six cases or more; the plan hints when there are fewer. No `runs: { bestOf }` beside it.
21
+ `gate: { passRate }` gates each cell's rate over the cases (the mean of each case's pass@1), so
22
+ one failing case of three passes `passRate: 0.6`; under `'all'` every case must pass, so a
23
+ failing case fails the eval. It is one eval: one row in `--dry-run` and the run output (a
24
+ case table under it when it fails, or with `-v`), one entry in the record with its cases
25
+ inside, counted once in the tally. The failing case still records `fail`.
26
+ - For evals that pass or fail on their own, don't use `cases`: write a `for` loop that calls
27
+ `evaluate` once per item, with the item in the name.
28
+ - Each case has a key, which names its row: a string or number is its own key, an object its
29
+ `id`, else its `name`; otherwise pass `caseKey: (item) => item.slug`. Keys are unique, at most
30
+ 80 characters, with no `#` or `›`. `cases`, `caseKey` and `where` go on `evaluate`, never a
31
+ group or the config.
32
+ - Porting `evaluate.each(items)('name: %s', opts, fn)`: write `evaluate('name', { cases: items,
33
+ ...opts }, fn)` with the old option names made plural (`harnesses`, `environments`,
34
+ `compare.control`). The old runs then read as this eval, keyed by what `%s` or `$id` held, so
35
+ `reeval compare` lines them up; a `caseKey` that changes a key starts that case's history again.
36
+ - `where: ({ case, environment, harness }) => boolean` (TypeScript only) picks which case runs
37
+ on which environment, at plan time, from each environment's `traits` (`defineEnvironment('next',
38
+ { traits: { stack: 'next' } })`; never shown to the agent). Reading a trait no environment
39
+ defines is an error; with `compare` it sees the control, and its answer holds for the variants.
40
+ `--dry-run` notes `excluded on vite-app: q2`, the run's case table shows `–` there, and each case's record keeps its own in
41
+ `excluded`. A cell left with no cases is an error: drop that environment instead.
42
+ - `where` says which pairs fit, the same every run; to run only some environments today, use
43
+ `reeval run --trait stack=next` (repeat to AND) or a profile's
44
+ `limit: { traits: { stack: ['next', 'vite'] } }`. Both narrow after `where`, by the control's
45
+ traits under `compare`, and `--dry-run` notes `--trait stack=next leaves out vite-app`. Never
46
+ read a flag or a variable in `where` to do this.
47
+ - `evaluate(name, { gate: 'observe' }, fn)` records what agents do (with `c.observe` and
48
+ checks) and fails only on run or infra errors. Its block reads `ran`. `compare.minLift` and
49
+ `runs: { bestOf }` are errors beside it; `compare: { gate: 'observe' }` ungates a compare's cells.
50
+ Any gate but `'all'`, `{ passRate }` or `'observe'` is an error.
51
+ A comment on an eval describes the current check (why a canary fails, which path it reads),
52
+ not what used to be true or which product a prompt was ported from.
53
+
54
+ ## The attempt: `c.send`, `c.exec`, `c.serve`, files
55
+
56
+ - `await c.send(prompt)` runs one agent turn and returns a `Turn` (`text`, `status`, `diff`,
57
+ `transcript`). Call it again for multi-turn;
58
+ `c.judge(q, { on: firstTurn, task: true, output: true })` grades an earlier one.
59
+ - `await c.exec(cmd)` returns `{ exitCode, stdout, stderr }`; `c.exec(cmd, { background: true })`
60
+ starts it without waiting and returns `{ kill() }`. `{ env: { NAME: 'value' } }` sets variables
61
+ for that command only, over the environment's (in the background too); a value named like a
62
+ secret (`*_TOKEN`, `*_KEY`, …) is masked in everything reeval writes, and a brokered secret's
63
+ host value goes in as its placeholder. `c.exec(cmd, { retries: 2, interval: 2000 })`
64
+ runs a command that exits non-zero again, up to 2 more times 2 s apart, and resolves with the
65
+ last try: for a check that can fail on a transient, such as a test calling a model. Any non-zero
66
+ exit retries (reeval doesn't read the output), and a last failed try is a failed check. `await c.serve(cmd, port?)` is that plus a
67
+ port (`PORT` / `$PORT` on a bash string), and returns once `cmd` listens, on `PORT` or the one
68
+ port it picked itself. A `port` something already listens on (the agent's dev server) throws:
69
+ pass none and serve on `PORT`. `c.capture()` screenshots it: see
70
+ [capture.md](capture.md).
71
+ - `await c.readFile(path)` the text or `null` (`~/` for home). `await c.writeFile(path, text)`
72
+ places a file after a turn (a hidden test the agent never saw) instead of seeding it in
73
+ `files`; later turns' `diff` leaves it out but shows any change the agent makes to it, even
74
+ under `.gitignore`. Don't place grading files with a `c.exec` heredoc: the next turn's diff
75
+ (and the judge) would count them as the agent's. The sandbox and session start on the first
76
+ call that needs them (`c.send`, `c.exec`, `c.readFile`, `c.writeFile`, `c.serve`,
77
+ `c.capture`), so a check before any turn sees what the environment set up.
78
+ `c.attempt.cell.role` is
79
+ `'control'` or `'variant'`.
80
+ - `c.observe(name, value)` records a JSON value about the attempt (which ORM it installed, how many
81
+ files it touched) that never affects pass or fail; the run output tallies it per cell and
82
+ `reeval query attempt_values` lists it. `c.judge()` and questions see it only with
83
+ `observations: true` (or a list of names). Use it for what you want to count, `expect` for what
84
+ must hold.
85
+
86
+ ## The judge: inputs, input size, errors
87
+
88
+ - `await c.judge(question, options)`: the judge sees the question and only the inputs the call
89
+ turns on. Every input is off until the call sets it, and a call needs at least one (`true`, a
90
+ mode or a non-empty list); there are no defaults, and the config, profile, group and eval
91
+ `judge` take only `model`, `threshold`, `providerOptions`, `maxConcurrent` and `gateway`.
92
+ With none, `c.judge()` throws before any model call, and TypeScript refuses it.
93
+
94
+ | Input | Shows |
95
+ |---|---|
96
+ | `task: true` | the graded turn's prompt |
97
+ | `output: true` | the graded turn's final message, or `on`'s value |
98
+ | `files: true` | `status path` lines for the changed files, under `diff`'s patterns |
99
+ | `diff: true` / `['src/**', '!**/*.test.ts']` | the diff since the eval started (gitignore-style, last match wins) |
100
+ | `contents: ['README.md', 'src/**']` | files whole, changed or not; they leave the diff |
101
+ | `transcript: true` / `'tools'` / `'full'` | the turn's transcript: clipped, tool calls only, everything |
102
+ | `checks: true` / `['npm test']` | `c.exec` results, or those commands' |
103
+ | `artifacts: true` / `[shot, 'home']` | captures, sent as file parts |
104
+
105
+ Pick the inputs the question needs, usually one or two: a question about code reads `diff` or
106
+ `contents`, one about the agent's answer `output`, one about how it worked `transcript`, one
107
+ about a screenshot `artifacts`. Spell out `{ task: true, output: true, diff: true }` only when
108
+ the question needs all three. The diff leaves out dependencies, build output (`dist/`,
109
+ `.next/`, top-level `build/` and `out/`), lockfiles, `*.map`, `*.tsbuildinfo` and minified
110
+ files.
111
+ - `on` and `turns` select what `task`, `output`, `transcript` and `diff` read and show nothing alone:
112
+ `on: turn` grades an earlier turn, `on: value` grades any value as the output (with
113
+ `output: true`), and `turns: 'all'` adds every turn before the graded one. `judge-input.json`
114
+ records each call's `stateOptions` (every input, `false` when off) and the `state` it sent.
115
+ - Ask about facts in a captured picture, not whether it "looks great". Jev
116
+ (`typesafe-ai/jev-1.14.0`) takes PNG captures; a provider that cannot take files fails that
117
+ call. Prefer deterministic checks for what can be checked; use the judge for quality, with
118
+ plain yes/no questions that don't name one specific way of doing it.
119
+ - Give the environment a `.gitignore` in `files` (the project's isn't copied in): an agent that
120
+ runs a build otherwise fills the judge's diff with its output. reeval sends every input the
121
+ call turns on, uncut, and knows no model's limit: a provider error refusing it keeps the
122
+ provider's wording and adds one line with the input's estimated size, biggest files and the
123
+ inputs that were on, largest first; narrow one, e.g. `{ diff: ['src/'] }`.
124
+ - A failed judge call doesn't throw: `c.judge()` returns `{ verdict: 'error', score: null, error }`,
125
+ the later judges and checks still run, and the attempt fails as a judge error at the end, or
126
+ as an infra error (which `--retry-errors` reruns) when the provider never answered and nothing
127
+ else failed.
128
+ `expect(await c.judge(q, { diff: true }), name).toPass()` records each judge's result and
129
+ carries on.
130
+
131
+ ## Trustworthy evals
132
+
133
+ Before a real run, prove each check can pass and fail with the fake harness in the eval's real
134
+ sandbox: an oracle `fakeHarness` that does the task right (must pass) and a canary that cheats
135
+ or gets it wrong (must fail). A canary that passes means a check isn't checking. Give each check
136
+ you rely on a canary that only it catches.
137
+
138
+ ```ts
139
+ // reeval.config.ts: the real harness, plus two profiles that swap only the harness and judge
140
+ export default defineConfig({
141
+ harnesses: cursor({ model: 'composer-2.5' }),
142
+ profiles: {
143
+ oracle: {
144
+ defaults: {
145
+ harnesses: fakeHarness({ files: { 'greet.js': 'export const greet = …' } }),
146
+ judge: { model: fakeJudge({ score: 0.95 }) },
147
+ },
148
+ },
149
+ canary: {
150
+ defaults: {
151
+ harnesses: fakeHarness({ files: { 'greet.test.js': "test('greets', () => {})\n" } }),
152
+ judge: { model: fakeJudge({ score: 0.1 }) },
153
+ },
154
+ },
155
+ },
156
+ })
157
+ ```
158
+
159
+ ```bash
160
+ npx reeval image build --profile oracle # once: the eval's images
161
+ npx reeval run --profile oracle # every check passes, exit 0
162
+ npx reeval run --profile canary # the checks you meant fail, exit 1
163
+ ```
164
+
165
+ - Set `defaults.harnesses`, never `defaults.sandbox`: the environment's `docker()` (image,
166
+ `install`, egress, brokered secrets) stays, so the checks run where they will for real. A
167
+ profile's `sandbox` only fills in for an environment that omits one, and the fake harness
168
+ without `layout` uses Cursor's image; `layout: 'claude-code'` (or another agent id) uses that
169
+ agent's image and reads rules and skills where it does.
170
+ - `script` steps (`{ exec }`, `{ write }`, `{ mcp }`) run inside the container as the agent, so an
171
+ oracle can run the real fix (`{ exec: 'node scripts/solve.mjs' }`) or call a brokered API.
172
+ - `fakeJudge` checks the wiring (a low score fails the check); keep the real `judge.model` on the
173
+ oracle, with its key, to check that a question passes good work.
174
+ - `fakeSandbox()` isn't the oracle's sandbox: it runs on the host, brokers nothing and enforces no
175
+ egress. It's for trying reeval and testing configs without Docker.
176
+ - An eval or group that sets its own `harnesses` keeps it (the plan notes `harnesses set by the
177
+ eval`), and an ungated cell's (`compare: { gate: 'observe' }`, a control's) failed checks don't change the
178
+ exit code: its row in the result block shows its count (`plain 0 of 4`).
179
+
180
+ Keep grading out of the
181
+ agent's reach: place hidden tests after the turn with `c.writeFile`, so the agent never sees
182
+ them, seed tests in `files` only when the agent should see them, and guard seeded files
183
+ with `expect(await c.readFile(path)).toBe(original)` or `expect(c).not.toHaveChanged(path)`.
184
+ When the behavior is the
185
+ outcome, judge the program's output, not the diff:
186
+ `c.judge(q, { on: (await c.exec('node app.js')).stdout, output: true })`. Checks run in the agent's
187
+ container as the same user, which suits AX evals but isn't a sealed verifier.
188
+
189
+ ## Matchers
190
+
191
+ - `await expect(turn).toHaveFile(path, pattern?)`,
192
+ `await expect(c.exec(cmd)).toSucceed()`, `expect(verdict).toPass()`,
193
+ `expect(turn).toHaveSearched(query?)` / `toHaveFetched(url?)` (sync; they throw
194
+ `HarnessCapabilityError` when `turn.unreported` lists web events; a refused
195
+ search is still a `web-search` with `ok: false`; a call whose result never came has
196
+ `ok: null`, not `false`), plus Vitest's.
197
+ - Transcript matchers (sync, no `await`): `expect(turn | c).toHaveCalledSkill(name)`,
198
+ `toHaveCalledMcp(server, tool, input?)`, `toHaveCalledTool(name | RegExp, input?)`,
199
+ `toHaveRun(command?)`, `toHaveEdited(path?)`, `toHaveSearched(query?)`,
200
+ `toHaveFetched(url?)`. `.not` is Vitest's. `toHaveRun(string | RegExp)` matches the whole
201
+ command line, so `/genkit/` passes on `cat genkit.config.ts`; `toHaveRun({ program, args? })`
202
+ matches the program a segment (split on line breaks, `&&`, `||`, `;`, `|`, `&`) ran, by its
203
+ name without a directory, after `VAR=x`, `env`, `timeout`, `time`, `exec`, `sudo`, `nohup`,
204
+ `nice`, `command`, `if`/`then`/`do`, `bash -c`, `npx`, `bunx`, `npm exec`, `pnpm exec`/`dlx`
205
+ and `yarn dlx`; `args` (joined by spaces) is a substring or RegExp, e.g. `/^add\b/`.
206
+ `pnpm <bin>` and `npm run` aren't unwrapped, and a run counts even if its call failed.
207
+ `shellCommands(c | turn | c.turns)` returns those programs to count or filter. `toHaveCalledSkill`
208
+ is opened or loaded (a `skill` / `Skill` / `load_skill` tool, or a Read/`sed` of
209
+ `skills/<name>/`, on every harness), not "followed". MCP is `toHaveCalledMcp(server, tool)`
210
+ (the names in `mcp.<server>`); `toHaveCalledTool('mcp__<server>__<tool>')` is
211
+ the raw transcript name. After adapters and the normalizer, that string is
212
+ the same on every harness (stdio; Goose, OpenCode, pi and Codex refuse
213
+ `mcp.sse`). Use `toHaveEdited` / `toHaveSearched`, not
214
+ `toHaveCalledTool('edit')` or `'webSearch'`.
215
+ - Budgets: `expect(turn | c).toStayWithin({ tokens: 20_000, toolCalls: 30, duration: '5m' })`
216
+ (sync) caps the turns' summed tokens, tool calls (a number, or `{ bash: 5 }` per tool) and time.
217
+ Tokens include subagents. With `tokens`, a harness that reported no usage throws
218
+ `HarnessCapabilityError` (an error, not a pass). `.not`, an unknown key or a bad limit throws a
219
+ `TypeError` that ends the attempt as its error, not a failed check. Python:
220
+ `to_stay_within(tokens=, tool_calls=, duration=)`.
221
+ - Guard files the agent must leave alone with `expect(c).not.toHaveChanged(['test/**',
222
+ 'package.json'])` (sync): it reads the workspace diff, so a shell or subprocess write counts
223
+ and a read doesn't. Strings are globs (a directory matches everything under it), RegExps work,
224
+ and a turn means up to that turn. `toHaveEdited` counts the files that changed while the agent
225
+ ran the turn (a turn only its own), so a shell write counts, but its string matches any path
226
+ containing it; guard with `toHaveChanged`'s globs. `toHaveEdited()` counts files that appear
227
+ in the workspace diff. Gitignored files don't appear there, so use `toHaveFile()` to check them.
228
+ - Checks don't stop the eval: a failed `expect` is recorded and the body carries on, so every
229
+ later check and judge still runs; the attempt fails at its end. Use `expect.require` for gates:
230
+ its failure ends the body there (`await expect.require(c.exec('npm run build'), 'builds')
231
+ .toSucceed()`, `expect.require.poll(…)` for one that waits). One the eval catches still fails
232
+ the attempt, and its error says the eval caught it. Only matcher failures carry on: an error
233
+ the body throws, a dead sandbox or a timeout ends the attempt. Wait or retry with `expect.poll`
234
+ (`await expect.poll(() => c.exec(cmd), { timeout: 10_000 }).toSucceed()`), which records one
235
+ check with its final outcome, not with `expect` in a `try` loop.
236
+ - **Label each requirement** with `expect`'s second argument:
237
+ `await expect(c, 'exports greet').toHaveFile('greet.js', /export/)`. Every matcher call
238
+ is recorded (`evals[].attempts[].assertions`; unlabeled ones get `<matcher> <first arg>`).
239
+ Changing a label changes the eval's fingerprint. Label judges too
240
+ (`expect(await c.judge(q, { diff: true }), 'Social OAuth').toPass()`): the label replaces the clipped
241
+ question in judge rates, `reeval show`, `summary.md` and the digest.
@@ -0,0 +1,39 @@
1
+ # Common mistakes (and the fix)
2
+
3
+ Each row is a mistake, what reeval says and the fix.
4
+
5
+ | Mistake | reeval says | Fix |
6
+ |---|---|---|
7
+ | No `reeval.config.*` | `No reeval.config.ts … here or above. Run reeval init.` | `reeval init`, or pass `--config` |
8
+ | `compare` in `reeval.config.ts` | `compare can't be set in reeval.config.ts` | Put it on the group or eval |
9
+ | `compare: [b]` | `compare … must be an object, not a list` | `compare: { control: a, variants: [b] }` |
10
+ | `compare: { variants: [b] }` | `compare … has no control` | `compare: { control: a, variants: [b] }` |
11
+ | `compare: { baseline: a, … }` | `Unknown compare option "baseline" … Did you mean "control"?` | `compare: { control: a, … }` |
12
+ | `compare: { strategy: 'peer' }` | `Unknown compare option "strategy"` | Drop it. Variants already run on their own |
13
+ | `environments` beside `compare` | `… sets both environments and compare` | Move the environment into `compare.control` |
14
+ | `evaluate.each(cases)(name, fn)` | `evaluate.each is not a function` | `evaluate(name, { cases }, async (c, item) => …)` |
15
+ | `cases` for items that should each pass or fail alone | One failing case fails the whole eval | A loop: `for (const item of items) evaluate(nameOf(item), fn)` |
16
+ | `where` without `cases`, or on a group | `… sets where without cases` / `Unknown option "where" in group(…)` | Put `where` beside `cases` on the eval |
17
+ | `survey(…)`, or `gate: 'none'` | `survey is not a function` / `gate in … is 'all' …, { passRate } … or 'observe'` | `evaluate(name, { gate: 'observe' }, fn)` |
18
+ | `compare: false` on an eval or group | `compare: false can't be set` | Use a profile's `limit.compare: false` |
19
+ | A question and no language model | `questions "…" (…) needs a language model` (exit 2) | Set `analysis.model`, give the question its own `model`, or set `question: { model }` |
20
+ | `import { baseline }` | `reeval has no baseline` | Name the control in `compare: { control }`; it isn't an export |
21
+ | `repeat: 3` / `times` | `Unknown option "repeat" … Did you mean "runs"?` | `runs: 3` |
22
+ | `runs: { bestOf: 4 }` | `runs.bestOf … must be an odd whole number of 3 or more` | `runs: { bestOf: 3 }` or `5` |
23
+ | `runs: { bestOf: 3 }` with `compare` | `… compares environments, so it can't take runs: { bestOf: 3 }` | a fixed count, `runs: 4` |
24
+ | Judge model `'gemini-3.8-flash'` | needs a provider prefix | `'google/gemini-3.8-flash'` |
25
+ | `opencode({ model: 'gemini-3.8-flash' })` | `an OpenCode model id names its provider` (exit 2) | `'google/gemini-3.8-flash'` |
26
+ | No `judge.model` anywhere | `c.judge() has no model` | Set one, or `fakeJudge()` |
27
+ | The judge's provider refuses a big input | The provider's message, then `The judge's input was ~…k tokens (estimated): diff …, of which …` | Add a `.gitignore` to the environment, or narrow the call: `{ diff: ['src/'] }` |
28
+ | `c.judge(q)` with no inputs, expecting the judge to see the task, answer and diff by default | `c.judge() shows the judge nothing: turn on at least one input, …` (TypeScript refuses it too) | There are no default inputs: turn on what the question needs, `{ diff: true }`, `{ contents: ['README.md'] }`, `{ output: true }`, `{ transcript: 'tools' }` |
29
+ | `c.judge(q, { on: value })` alone | `c.judge() shows the judge nothing: …`, or with another input `… on gives a value to grade as the output, but output is off. Pass output: true.` | `{ on: value, output: true }`; `on` and `turns` only select |
30
+ | An input (`diff`, `contents`, …) on a config, profile, group or eval `judge` | `Unknown judge option "diff" in …. diff is per call: c.judge(question, { diff }).` | Put it on each `c.judge()` call |
31
+ | A question as a string, or with no input | `questions … takes { question, model?, across?, choices?, turns? } with at least one of task, output, …` / `… shows the model nothing: …` (exit 2) | `{ question: 'Which is cleaner?', across: 'environment', diff: true }` |
32
+ | `cursor()` with `fakeSandbox()` | real harnesses need `docker()` | Use `fakeHarness()` or `docker()` |
33
+ | Forgetting `await` on `toHaveFile`, or on `toSucceed` given `c.exec()`'s promise | ``expect(…).toHaveFile() at <file>:<line> wasn't awaited. Its check is async: write `await …` `` (the eval fails) | `await expect(...)`; `expect(await c.exec(cmd)).toSucceed()` needs no `await` |
34
+ | `toPass()` on an exec result, `toSucceed()` on a verdict | a type error, ``Type 'WrongSubject<"toPass() takes the verdict from await c.judge()">' has no call signatures`` | `toSucceed()` for `c.exec()`, `toPass()` for `await c.judge()` |
35
+ | Secret as a string literal | refused | `secret.fromEnv('NAME')` in a callback |
36
+ | `workspace: 'https://…'` | `must come from the git helper` | `({ git }) => ({ workspace: git('https://…', { ref }) })` |
37
+ | `async` environment callback | returned a Promise | Return synchronously; fetch in a `setup` step |
38
+ | Test file named `*.test.ts` | `No eval files in … match include: …` (exit 2) | `*.eval.ts` under `evals/` |
39
+ | `.env.local` outside the config's directory | `Missing credentials: …` | Put it next to `reeval.config.ts` |
@@ -0,0 +1,19 @@
1
+ # Vocabulary
2
+
3
+ reeval's terms, and what other eval frameworks (Inspect, Harbor, SWE-bench) call them. Use
4
+ reeval's word in code and messages.
5
+
6
+ | reeval | Elsewhere called |
7
+ |---|---|
8
+ | eval (`evaluate(name, fn)`) | task, test case |
9
+ | case (an item of `cases`) | sample, example, instance |
10
+ | attempt (one run of a cell; `runs: n`) | trial (Anthropic, Harbor), epoch (Inspect), repetition |
11
+ | harness (`cursor()`, …) | agent harness, scaffold, solver (reeval itself is the *evaluation* harness) |
12
+ | cell (harness × environment × sandbox) | a factorial-design cell |
13
+ | control, variant | baseline, treatment |
14
+ | lift (variant pass@1 − control pass@1, 95% interval) | treatment effect, delta, uplift |
15
+ | gate (how a cell's attempts decide its outcome) | `'all'` is pass^k, `{ passRate }` a pass@1 threshold; epoch reducer (Inspect), quality gate |
16
+ | pass@1 (passes over passes and fails; `pass_at_1` in `reeval query`) | success rate, resolve rate (SWE-bench) |
17
+ | checks, judge (`expect`, `c.exec`, `c.judge`) | grader, verifier, scorer, evaluator |
18
+ | transcript (`transcript.jsonl`) | trajectory, trace |
19
+ | effort (measured tool calls, agent time, tokens, cost) | efficiency |
@@ -0,0 +1,219 @@
1
+ ---
2
+ name: setup
3
+ description: >
4
+ Read this first, before the reeval skill, whenever you're asked to set up, install, add,
5
+ try or get started with reeval (the eval runner for coding agents), in a project or an empty
6
+ directory, including "set up reeval here" and "add evals". It detects what's set up, asks what
7
+ to evaluate, installs reeval, writes one to three small evals that show its value (a skill with
8
+ and without, when the project has skills), runs `reeval doctor` for keys and Docker, and
9
+ explains the results.
10
+ ---
11
+
12
+ # Set up reeval
13
+
14
+ Your job is the user's first good run: a few small evals that pass `--dry-run`, the keys and
15
+ Docker sorted, and a short explanation of what the evals show. Small and readable beats complete.
16
+ For the authoring API in depth, the reeval skill (`skills/reeval/SKILL.md` in
17
+ github.com/invertase/reeval) has it; this file has everything a first setup needs.
18
+
19
+ Rules that hold throughout:
20
+
21
+ - **Never overwrite or re-initialize.** Don't run `reeval init` where a `reeval.config.*` or
22
+ `reeval.yaml` exists, never pass `--force`, never edit or delete an existing eval, and add new
23
+ files beside the user's.
24
+ - **Never write, print or ask for a key's value.** Name the variable and the file it goes in.
25
+ - **Ask before you build,** then stop and wait (step 2): always, even when reeval is half set
26
+ up or the right eval looks obvious. Only a message that already says what to evaluate
27
+ ("add an eval for the postgres skill") skips the question.
28
+
29
+ ## 1. Look before you touch anything
30
+
31
+ ```bash
32
+ ls -a; ls reeval.config.* reeval.yaml evals 2>/dev/null
33
+ grep -n '"reeval"' package.json 2>/dev/null
34
+ ls pnpm-lock.yaml yarn.lock bun.lock package-lock.json 2>/dev/null # the package manager
35
+ find . -name SKILL.md -not -path '*/node_modules/*' | head -20 # the project's skills
36
+ ls AGENTS.md CLAUDE.md GEMINI.md .cursor/rules .github/copilot-instructions.md 2>/dev/null # rules
37
+ ls CONTRIBUTING.md .mcp.json .cursor/mcp.json docs 2>/dev/null
38
+ grep -o '^[A-Z_]*=' .env.local .env 2>/dev/null # key names only
39
+ ```
40
+
41
+ | You find | State | Then |
42
+ |---|---|---|
43
+ | A `reeval.config.*` or `reeval.yaml` | Set up | No init. Read the config and the evals and run `npx reeval run --dry-run`; after step 2, add evals beside them |
44
+ | `reeval` in `package.json`, no config | Installed | `npx reeval init --bare --harness <h>` |
45
+ | Neither | New | Install, then `init --bare` (step 3) |
46
+
47
+ The directory is a **project** if it has code, docs or skills (`package.json`, `src/`,
48
+ `README.md`, `skills/`, a `.git` with commits); it's **empty** if there's nothing but dotfiles.
49
+
50
+ ## 2. Agree what to evaluate
51
+
52
+ "Set up reeval" says to set reeval up, not what to evaluate, so it doesn't answer this step,
53
+ however clear the choice looks from the repo. Your first reply is the proposal and the
54
+ question, with nothing installed or written yet: end your turn there and wait for the answer.
55
+
56
+ **In a project,** say in two or three sentences what you found and what you propose, then ask
57
+ and stop. For example: "This repo has two skills. I'd set up reeval with one eval that runs Cursor
58
+ on a billing task with and without `acme-billing`, to show what the skill changes. Shall I set
59
+ up evals for this project?" With no skills, propose the rules file or a task in the code
60
+ instead.
61
+
62
+ **In an empty directory, or when the user says no,** ask what they want to evaluate (an SDK, an
63
+ API, a library, docs or a skill they're writing, a kind of task) and stop. When they answer,
64
+ research it before you write a line: read its docs, changelog and README (search the web, fetch
65
+ the pages), note the current version, and find a few things a coding agent gets wrong without
66
+ help: a renamed API, a new default, a required option, a convention. Those are your evals.
67
+
68
+ ## 3. Install and initialize
69
+
70
+ Use the project's package manager (its lockfile), and pick the harness the user has a key for:
71
+
72
+ | Harness | `--harness` | Key |
73
+ |---|---|---|
74
+ | Cursor (default) | `cursor` | `CURSOR_API_KEY` |
75
+ | Claude Code | `claude-code` | `ANTHROPIC_API_KEY` |
76
+ | Codex | `codex` | `OPENAI_API_KEY` |
77
+ | Antigravity | `antigravity` | `GEMINI_API_KEY` |
78
+ | OpenCode, Goose, pi | `opencode`, `goose`, `pi` | The model's provider's key, e.g. `GOOGLE_GENERATIVE_AI_API_KEY` for `google/…` |
79
+
80
+ ```bash
81
+ npm init -y # only if there's no package.json
82
+ npm i -D reeval # pnpm add -D reeval · yarn add -D reeval
83
+ npx reeval init --bare --harness cursor # pnpm reeval init … · yarn reeval init …
84
+ ```
85
+
86
+ `--bare` writes only `reeval.config.ts`, the `.gitignore` entries (`.reeval/`, `.env*`) and the
87
+ `eval` script; it merges into existing files and skips the rest. It exits 2 if reeval is set up
88
+ already: that's your cue to stop initializing, not to `--force`. The config sets
89
+ `analysis.model` for `reeval analyze` (needs `GOOGLE_GENERATIVE_AI_API_KEY`) and `pr` and `nightly`
90
+ profiles. Leave them.
91
+
92
+ ## 4. Write one to three evals
93
+
94
+ Pick the low-hanging fruit, in this order:
95
+
96
+ 1. **The project has skills:** a compare of one skill, with and without. The control is an empty
97
+ workspace, the variant the same plus the skill; the task is one the skill's own rules decide.
98
+ This is the eval that shows lift. Template A in
99
+ [references/templates.md](references/templates.md).
100
+ 2. **No skills, but agent rules** (`AGENTS.md`, `CLAUDE.md`, `GEMINI.md`, `.cursor/rules`): a
101
+ compare of the rules file, with and without, on a small task in the real repo. Template D.
102
+ 3. **Any other codebase (an app, a CLI, a library):** a small task in the real repo, checked by
103
+ its typecheck, its tests or a hidden test. Template B.
104
+ 4. **An empty directory:** the researched subject, a compare of a short skill or doc you write
105
+ from the research (`skills/<subject>/SKILL.md`) against none, or a single eval. Template C.
106
+
107
+ Work in the user's code, not a copy you make up: seed the repo with `directory('..')` rather
108
+ than stub files, which hand the agent the convention you meant to test. Don't write a skill or
109
+ rules from what you already know to compare against: your checks would restate what you wrote,
110
+ so the compare measures nothing. Write one only from research (step 2, Template C).
111
+
112
+ Each eval:
113
+
114
+ - **Check rules only the skill teaches.** Read the skill (or rules file) and pick two or three
115
+ of its most specific rules that one task touches: a house convention, a helper or file it
116
+ names, a version-specific API, a non-obvious default. Ask of each: would a strong agent do
117
+ this unprompted? If yes (index a foreign key, deny public access, validate input), both arms
118
+ pass it; pick another. In a real repo, the agent copies what the code around the change
119
+ already shows (the error class, the logger), so prefer rules the code can't show: a changelog
120
+ entry, where tests go, a file to register in. Grep the repo for each rule: one that
121
+ `CONTRIBUTING.md` or the docs repeat reaches the control too, so leave those files out of
122
+ the control as well, or pick another rule. Check that the task triggers each rule: a rule for
123
+ user-facing changes needs a user-facing task. One labeled check per rule, so the check
124
+ table shows which ones the skill changed.
125
+ - **One task, one prompt, that doesn't name the answer.** Ask for the outcome in the user's
126
+ words ("an invoices table with its total"), never the rule the check looks for ("store the
127
+ total in cents"), and not in the rule's terms either ("a user-visible fix" hints at the
128
+ changelog). Reread the prompt: if it restates the rule, the control passes too. Name the
129
+ file the task is about (`src/projectId.ts`) so a check can read it, but not the files a rule
130
+ decides (where the test goes): check those with a glob or the one path the rule allows.
131
+ - **Deterministic checks first:** a file's content, a command's exit code, a script that imports
132
+ the agent's code. Label each requirement with `expect(x, 'label')`; a failed one is recorded and the eval carries on. Add a judge
133
+ (`c.judge`) only for what no check can see, with the inputs its question reads
134
+ (`{ diff: ['src/**'] }`, `{ output: true }`), and then add `judge:` to the config.
135
+ - **A name that states the question:** `stores invoice totals in cents with the billing skill`,
136
+ not `billing test`. The results and `reeval analyze` read in its terms.
137
+ - **Small:** leave `runs` unset: a compare runs 4 a cell (8 attempts), the fewest for a lift
138
+ verdict, and anything else once. Seed only the files the task
139
+ needs. If it needs the repo, seed it with `directory('..', { glob: ['!.env*', '!.reeval/',
140
+ '!evals/', '!reeval.config.*'] })`: `directory()` doesn't skip `.env*`, keys must not reach
141
+ the sandbox, and the agent must not read your checks or the oracle's answer in the config.
142
+ - **Eval files go in `evals/*.eval.ts`,** environments in `evals/environments.ts`. Paths in
143
+ `skill.local()`, `file()` and `directory()` are relative to the file that calls them. If
144
+ `evals/environments.ts` exists, append your environments below what's there and leave its
145
+ lines, comments included, as they are.
146
+
147
+ When a check is a regex or a script you wrote, add two fake-agent profiles (Template A shows
148
+ both): a **canary** that does the task wrong must fail every check, so the checks check; an
149
+ **oracle** that does it right, written another way than you'd expect (another helper name,
150
+ another order, schema-qualified `public.notes`), must pass, so the checks don't demand one
151
+ spelling: a regex that wants `on notes (` fails the agent that writes `on public.notes (`. A
152
+ check that runs a build or the project's tests needs no oracle.
153
+
154
+ ## 5. Check it plans, then the machine
155
+
156
+ ```bash
157
+ npx reeval run --dry-run # exit 0; the evals, cells and attempts you meant
158
+ npx reeval doctor # Docker, image, keys; exit 1 names each fix
159
+ ```
160
+
161
+ - Rerun `--dry-run` after every edit. Exit 2 is a config or eval mistake with a precise
162
+ message; fix it, don't work around it. Check the counts: a compare is 2 cells.
163
+ - `doctor` lists each missing key as `fix: add NAME=... to .env.local`. Tell the user exactly
164
+ those lines, and that `.env.local` goes next to `reeval.config.ts` (it's gitignored). The
165
+ analysis key is a warning only: runs pass or fail the same without it.
166
+ - No Docker, or `doctor` says the image isn't built: the user installs or starts Docker, then
167
+ runs `npx reeval image build` once (minutes, cached).
168
+ - With Docker: `npx reeval run --profile canary <eval-name>` must fail every
169
+ check you wrote, and `--profile oracle` must pass them. Name the eval: a fake agent writes the
170
+ same files for every eval, so it answers only the one it was written for. Needs no keys,
171
+ seconds each. A canary that passes means a check isn't checking; an oracle that fails means a
172
+ check is too narrow. Fix the check.
173
+
174
+ Only start a real `npx reeval run` if the user asked for it or agrees: it runs a real agent on
175
+ their key. One compare (8 attempts) on Cursor is several minutes.
176
+
177
+ ## 6. Hand over
178
+
179
+ End with a short message, not a report:
180
+
181
+ 1. What you added, one line per file, and what each eval asks.
182
+ 2. Each key the config needs by name (`CURSOR_API_KEY`, the analysis key) and whether `doctor`
183
+ found it; then what's missing, as commands: the `.env.local` lines, `npx reeval image
184
+ build`, Docker.
185
+ 3. The command to run: `npx reeval run` (or `npx reeval run <eval-name>` for one).
186
+ 4. How to read the output, in a few lines. `npx reeval run` prints a line per eval and a sentence
187
+ for each way one failed; `npx reeval show` prints its arms and checks. A real Cursor run of
188
+ Template A, from `npx reeval show`:
189
+
190
+ ```text
191
+ pass stores invoice totals in cents with the billing skill
192
+ arm pass
193
+ plain 0 of 2
194
+ with-billing-skill 2 of 2 helped
195
+ check plain with-billing-skill
196
+ total in integer cents 0 of 2 2 of 2
197
+ a currency column 0 of 2 2 of 2
198
+ +1 more passed
199
+
200
+ 1 pass
201
+ ```
202
+
203
+ - Each arm's `N of M` is passing attempts. The variant's word is its effect against the control:
204
+ `helped`, `made it worse`, or `same`; on a small sample (below 4 runs a side, or a `same` at
205
+ 2 of 4 against 2 of 4) a `too-few-tries` remark says it doesn't hold yet and names the `runs`. When
206
+ both arms pass every run, or both fail every run, at 4 runs a side or more, a `cant-tell-apart`
207
+ remark says more runs won't help: make the task harder, or check the eval. When both pass but
208
+ another measure differs, the line under the variant names it (`Same pass rate · 5× faster.`),
209
+ and a measure that only leans gets a `too-few-tries` remark with the runs that would confirm it.
210
+ The check table shows which check told the arms apart.
211
+ - `Worth knowing:` under an eval in `reeval run`'s list names what about it needs work, with the line to fix. Remarks never
212
+ change a verdict.
213
+ - Both arms passing is a result, not a broken eval: the agent already does this without the
214
+ skill. Say so, and suggest a harder task or a more specific rule from the skill next.
215
+ - A check that fails every attempt on both arms is more often a broken check than a hard task.
216
+ Open one attempt's `diff.patch` before you say the agent can't do it.
217
+ - Exit 0 means every gated cell passed; only the variant is gated in a compare.
218
+ - Details: `.reeval/runs/<run>/summary.md`, each attempt's `transcript.jsonl` and `diff.patch`;
219
+ `npx reeval studio` browses them; `npx reeval query effort` compares tools, time and tokens.