reeval 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (336) hide show
  1. package/README.md +99 -64
  2. package/THIRD_PARTY_NOTICES.md +288 -112
  3. package/assets/package.json +1 -1
  4. package/assets/pnpm-lock.yaml +30 -30
  5. package/dist/adapter-Bwy3QxAK.js +279 -0
  6. package/dist/adapter-Bwy3QxAK.js.map +1 -0
  7. package/dist/{agent-6FefIhX7.js → agent-D483ru7D.js} +6 -6
  8. package/dist/{agent-6FefIhX7.js.map → agent-D483ru7D.js.map} +1 -1
  9. package/dist/{agent-WSmTaesf.js → agent-DpE80WPz.js} +46 -15
  10. package/dist/agent-DpE80WPz.js.map +1 -0
  11. package/dist/{analysis-DlGBDjTj.js → analysis-Bxub9LMT.js} +4 -4
  12. package/dist/{analysis-DlGBDjTj.js.map → analysis-Bxub9LMT.js.map} +1 -1
  13. package/dist/assemble-D5KGrwnJ.js +1 -0
  14. package/dist/{assemble-BCZzTyR7.js → assemble-D9GWr-HS.js} +69 -41
  15. package/dist/assemble-D9GWr-HS.js.map +1 -0
  16. package/dist/{client-Bbo9SPlK.js → assert-EUIoRdIt.js} +10 -113
  17. package/dist/assert-EUIoRdIt.js.map +1 -0
  18. package/dist/bridge/index.mjs +260 -60
  19. package/dist/{capture-Bp74Z6Fr.js → capture-BQlGHiMC.js} +146 -32
  20. package/dist/capture-BQlGHiMC.js.map +1 -0
  21. package/dist/capture-CYGLpWxR.js +1 -0
  22. package/dist/{cli-CVrMh_qz.js → cli-ClpPT8kn.js} +229 -537
  23. package/dist/cli-ClpPT8kn.js.map +1 -0
  24. package/dist/{cli-spec-CKEYa8V2.js → cli-spec-qmEnQ25_.js} +50 -22
  25. package/dist/cli-spec-qmEnQ25_.js.map +1 -0
  26. package/dist/cli.js +4 -2
  27. package/dist/cli.js.map +1 -1
  28. package/dist/client-fcwgwOAD.js +106 -0
  29. package/dist/client-fcwgwOAD.js.map +1 -0
  30. package/dist/{command-DUT5l13I.js → command-Bj39y-Wm.js} +329 -87
  31. package/dist/command-Bj39y-Wm.js.map +1 -0
  32. package/dist/{command-BFP0hKEb.js → command-BlM0y8jN.js} +14 -14
  33. package/dist/{command-BFP0hKEb.js.map → command-BlM0y8jN.js.map} +1 -1
  34. package/dist/command-C-cVjlny.js +1 -0
  35. package/dist/{command-CzIFp1JQ.js → command-DbAe1Rzb.js} +19 -19
  36. package/dist/command-DbAe1Rzb.js.map +1 -0
  37. package/dist/{command-C5xFPKHM.js → command-TgbzPizb.js} +378 -74
  38. package/dist/command-TgbzPizb.js.map +1 -0
  39. package/dist/commands-CthqwkAn.js +1 -0
  40. package/dist/{commands-CLmPBrTS.js → commands-YUwlyTbr.js} +14 -11
  41. package/dist/commands-YUwlyTbr.js.map +1 -0
  42. package/dist/config-DrVSEVN0.js +774 -0
  43. package/dist/config-DrVSEVN0.js.map +1 -0
  44. package/dist/{context-DbSMEVKs.js → context-C1mxXh80.js} +11 -3
  45. package/dist/context-C1mxXh80.js.map +1 -0
  46. package/dist/contrasts-5zsj32gk.js +101 -0
  47. package/dist/contrasts-5zsj32gk.js.map +1 -0
  48. package/dist/db-CKg5NtiW.js +2 -0
  49. package/dist/{db-CfQFwJtq.js → db-DfhipHEF.js} +467 -38
  50. package/dist/db-DfhipHEF.js.map +1 -0
  51. package/dist/debug-D7bEv_UV.js +1 -0
  52. package/dist/{dist-CTzThfu_.js → dist-CuY4ehEk.js} +95 -15
  53. package/dist/dist-CuY4ehEk.js.map +1 -0
  54. package/dist/{docker-CRQGQbIt.js → docker-D1HZDS6q.js} +22 -21
  55. package/dist/docker-D1HZDS6q.js.map +1 -0
  56. package/dist/docker-DNmMPOBb.js +1 -0
  57. package/dist/{docker-DV2GNgle.js → docker-DOvjji9I.js} +3 -3
  58. package/dist/docker-DOvjji9I.js.map +1 -0
  59. package/dist/{doctor-DlKycx5H.js → doctor-D2MbNO8L.js} +262 -84
  60. package/dist/doctor-D2MbNO8L.js.map +1 -0
  61. package/dist/{egress-BhzNh7Yi.js → egress-2-qzPvGi.js} +146 -15
  62. package/dist/egress-2-qzPvGi.js.map +1 -0
  63. package/dist/environment-Cca6a4wB.js +1132 -0
  64. package/dist/environment-Cca6a4wB.js.map +1 -0
  65. package/dist/{errors-yDAEwmZo.js → errors-B6BSMs03.js} +2 -2
  66. package/dist/errors-B6BSMs03.js.map +1 -0
  67. package/dist/{errors-C9w_ud7g.js → errors-r025B0Zb.js} +4 -4
  68. package/dist/errors-r025B0Zb.js.map +1 -0
  69. package/dist/evaluate-DKIl54Hb.js +2348 -0
  70. package/dist/evaluate-DKIl54Hb.js.map +1 -0
  71. package/dist/evaluate-DMyaGQEj.d.ts +92 -0
  72. package/dist/files-BAt0i00X.js +1 -0
  73. package/dist/{files-DJUFg-4e.js → files-BXHZFWmx.js} +5 -5
  74. package/dist/{files-DJUFg-4e.js.map → files-BXHZFWmx.js.map} +1 -1
  75. package/dist/files-Bk1mC3AN.js +43 -0
  76. package/dist/files-Bk1mC3AN.js.map +1 -0
  77. package/dist/{files-CSzVADHn.js → files-BpUYBwGu.js} +11 -10
  78. package/dist/{files-CSzVADHn.js.map → files-BpUYBwGu.js.map} +1 -1
  79. package/dist/files-Bw8lWEjg.js +1 -0
  80. package/dist/files-D6J0iSdx.js +1 -0
  81. package/dist/{git-DfG-YJ1u.js → git-Crl3b87o.js} +2 -2
  82. package/dist/{git-DfG-YJ1u.js.map → git-Crl3b87o.js.map} +1 -1
  83. package/dist/harness-CEuntrla.js +323 -0
  84. package/dist/harness-CEuntrla.js.map +1 -0
  85. package/dist/harness.d.ts +16 -30
  86. package/dist/harness.js +1 -2
  87. package/dist/{header-CALhmafp.js → header-DHQv_ZrF.js} +2 -2
  88. package/dist/{header-CALhmafp.js.map → header-DHQv_ZrF.js.map} +1 -1
  89. package/dist/{images-BMgdpMAX.js → images-Bi3aGxb9.js} +114 -45
  90. package/dist/images-Bi3aGxb9.js.map +1 -0
  91. package/dist/index.d.ts +9 -37
  92. package/dist/index.js +6 -2138
  93. package/dist/{init-BIdLF0F9.js → init-Cl3f3NBT.js} +26 -8
  94. package/dist/init-Cl3f3NBT.js.map +1 -0
  95. package/dist/init-fhOXED8F.js +1 -0
  96. package/dist/{invocations-CLaL8AuW.js → invocations-BYDPe3fV.js} +4 -4
  97. package/dist/{invocations-CLaL8AuW.js.map → invocations-BYDPe3fV.js.map} +1 -1
  98. package/dist/{judge-CeYPN8P_.js → judge-DW_GDOmB.js} +11 -19
  99. package/dist/{judge-CeYPN8P_.js.map → judge-DW_GDOmB.js.map} +1 -1
  100. package/dist/judge.d.ts +2 -2
  101. package/dist/judge.js +1 -1
  102. package/dist/{live-S0qODb6o.js → live-jlfszAbb.js} +9 -20
  103. package/dist/live-jlfszAbb.js.map +1 -0
  104. package/dist/matchers-Ddmc2z7H.js +118 -0
  105. package/dist/matchers-Ddmc2z7H.js.map +1 -0
  106. package/dist/model-CXhrziBe.js +1 -0
  107. package/dist/{model-B1BZoZ9b.js → model-D0IcOQao.js} +22 -20
  108. package/dist/model-D0IcOQao.js.map +1 -0
  109. package/dist/{model-DEQo3FkZ.js → model-DaVCY3rb.js} +2 -2
  110. package/dist/{model-DEQo3FkZ.js.map → model-DaVCY3rb.js.map} +1 -1
  111. package/dist/{model-BOnlAPzs.js → model-DaX4bNLZ.js} +123 -22
  112. package/dist/model-DaX4bNLZ.js.map +1 -0
  113. package/dist/model-_bxNrk2h.js +1 -0
  114. package/dist/{node-Co7EXge0.js → node-B3ydHo6t.js} +10 -47
  115. package/dist/node-B3ydHo6t.js.map +1 -0
  116. package/dist/object-CLf2-tNq.js +9 -0
  117. package/dist/object-CLf2-tNq.js.map +1 -0
  118. package/dist/{opencode-providers-ByD85Q0l.js → opencode-providers-J9colP0-.js} +96 -21
  119. package/dist/opencode-providers-J9colP0-.js.map +1 -0
  120. package/dist/options-C0d0AzeR.js +1118 -0
  121. package/dist/options-C0d0AzeR.js.map +1 -0
  122. package/dist/{pairs-DDsjgm5F.js → pairs-CnInRu-G.js} +76 -2
  123. package/dist/{pairs-DDsjgm5F.js.map → pairs-CnInRu-G.js.map} +1 -1
  124. package/dist/{patch-bDV8YFW0.js → patch-B1bnfhN-.js} +6 -6
  125. package/dist/{patch-bDV8YFW0.js.map → patch-B1bnfhN-.js.map} +1 -1
  126. package/dist/{plan-DUydFc_1.js → plan-DgQxp9yy.js} +34 -13
  127. package/dist/plan-DgQxp9yy.js.map +1 -0
  128. package/dist/{preflight-BkoSxf4C.js → preflight-BRzD30L5.js} +8 -7
  129. package/dist/preflight-BRzD30L5.js.map +1 -0
  130. package/dist/preflight-DlQvbySb.js +1 -0
  131. package/dist/{prepare-CUAWHebF.js → prepare-B_y-AFMv.js} +38 -30
  132. package/dist/prepare-B_y-AFMv.js.map +1 -0
  133. package/dist/prepare-DCKTLHN3.js +2 -0
  134. package/dist/{provenance-D7HK3vzt.js → provenance-DQ0AXn6c.js} +5 -5
  135. package/dist/{provenance-D7HK3vzt.js.map → provenance-DQ0AXn6c.js.map} +1 -1
  136. package/dist/provider-DehR4CMN.js +1 -0
  137. package/dist/providers-B9yTdIqF.js +1 -0
  138. package/dist/providers-DoTq1jHW.js +165 -0
  139. package/dist/providers-DoTq1jHW.js.map +1 -0
  140. package/dist/query-BO-ux7RB.js +139 -0
  141. package/dist/query-BO-ux7RB.js.map +1 -0
  142. package/dist/query-CRQfsw0e.js +1 -0
  143. package/dist/questions-C2dym2ce.js +524 -0
  144. package/dist/questions-C2dym2ce.js.map +1 -0
  145. package/dist/{record-BRP9vZrj.js → record-BKmMvZ6O.js} +9 -1
  146. package/dist/{record-BRP9vZrj.js.map → record-BKmMvZ6O.js.map} +1 -1
  147. package/dist/remote-CPPW86Q9.js +378 -0
  148. package/dist/remote-CPPW86Q9.js.map +1 -0
  149. package/dist/{results-Di7m8dQ3.js → results-BE2f-1XS.js} +6 -5
  150. package/dist/results-BE2f-1XS.js.map +1 -0
  151. package/dist/{resume-SUz1GskV.js → resume-Ca1c8vix.js} +10 -10
  152. package/dist/{resume-SUz1GskV.js.map → resume-Ca1c8vix.js.map} +1 -1
  153. package/dist/{root-CvdRlRZX.js → root-DICDHCtp.js} +4 -4
  154. package/dist/{root-CvdRlRZX.js.map → root-DICDHCtp.js.map} +1 -1
  155. package/dist/{runtime-DdsfnRz_.js → runtime-PX_oXiCS.js} +23 -71
  156. package/dist/runtime-PX_oXiCS.js.map +1 -0
  157. package/dist/sandbox-DLrsvu7U.js +70 -0
  158. package/dist/sandbox-DLrsvu7U.js.map +1 -0
  159. package/dist/sandbox.d.ts +19 -6
  160. package/dist/sandbox.js +1 -2
  161. package/dist/server-Dp8eOeK5.js +1 -0
  162. package/dist/{server-BNnXtmS8.js → server-TUY5xA_A.js} +11 -11
  163. package/dist/server-TUY5xA_A.js.map +1 -0
  164. package/dist/shared-CA61BuPJ.js +188 -0
  165. package/dist/shared-CA61BuPJ.js.map +1 -0
  166. package/dist/{source-Dof0NQBm.js → source-DJGS0a0u.js} +2 -2
  167. package/dist/{source-Dof0NQBm.js.map → source-DJGS0a0u.js.map} +1 -1
  168. package/dist/source-DdbLGR1M.js +1 -0
  169. package/dist/src-BxfCKA3_.js +2 -0
  170. package/dist/{src-C1E9HDWO.js → src-C7_1SqDP.js} +3 -3
  171. package/dist/{src-C1E9HDWO.js.map → src-C7_1SqDP.js.map} +1 -1
  172. package/dist/{src-DDNXsD9H.js → src-CF17Ug08.js} +5 -4
  173. package/dist/src-CF17Ug08.js.map +1 -0
  174. package/dist/{src-C_4Umgh2.js → src-CFp86GtV.js} +52 -51
  175. package/dist/src-CFp86GtV.js.map +1 -0
  176. package/dist/src-CLoOMVWI.js +3 -0
  177. package/dist/src-CcrQbpG8.js +2 -0
  178. package/dist/src-DCFtJRp0.js +2 -0
  179. package/dist/src-DIL0jST0.js +368 -0
  180. package/dist/src-DIL0jST0.js.map +1 -0
  181. package/dist/src-DSYYPEOj.js +2 -0
  182. package/dist/src-DYO3DdnQ.js +1 -0
  183. package/dist/src-DvCg0NBs.js +324 -0
  184. package/dist/src-DvCg0NBs.js.map +1 -0
  185. package/dist/{src-COgE7991.js → src-DvK0EIZx.js} +5 -5
  186. package/dist/{src-COgE7991.js.map → src-DvK0EIZx.js.map} +1 -1
  187. package/dist/{src-DVy58-zZ.js → src-FGXhkEkH.js} +5 -5
  188. package/dist/{src-DVy58-zZ.js.map → src-FGXhkEkH.js.map} +1 -1
  189. package/dist/src-WD-zgQYZ.js +2 -0
  190. package/dist/src-_oI9AePb.js +2 -0
  191. package/dist/{src-8rRXy4u9.js → src-ftu5wPSt.js} +1652 -182
  192. package/dist/src-ftu5wPSt.js.map +1 -0
  193. package/dist/src-qsKCqv7a.js +2 -0
  194. package/dist/{src-D95O-Smv.js → src-u8_ixn_F.js} +15 -11
  195. package/dist/src-u8_ixn_F.js.map +1 -0
  196. package/dist/state-CeR6oz0k.js +66 -0
  197. package/dist/state-CeR6oz0k.js.map +1 -0
  198. package/dist/{state-CiXyLqg0.js → state-DfpHox6b.js} +36 -24
  199. package/dist/state-DfpHox6b.js.map +1 -0
  200. package/dist/state-DpAxYyED.js +1 -0
  201. package/dist/studio-app/assets/index-CBkhNAv8.js +58 -0
  202. package/dist/studio-app/assets/index-CwQN3f_h.css +2 -0
  203. package/dist/studio-app/index.html +2 -2
  204. package/dist/suite-Cw_7G2RM.js +429 -0
  205. package/dist/suite-Cw_7G2RM.js.map +1 -0
  206. package/dist/suite-DamAEnNF.js +1 -0
  207. package/dist/summary-BUKfU-9l.js +611 -0
  208. package/dist/summary-BUKfU-9l.js.map +1 -0
  209. package/dist/{summary-2OAWF2N7.js → summary-Baa4rTHS.js} +7 -7
  210. package/dist/{summary-2OAWF2N7.js.map → summary-Baa4rTHS.js.map} +1 -1
  211. package/dist/testing.d.ts +3 -2
  212. package/dist/testing.js +3 -2
  213. package/dist/{tool-target-BU1JnSE9.js → tool-target-D2mcYlSQ.js} +14 -7
  214. package/dist/tool-target-D2mcYlSQ.js.map +1 -0
  215. package/dist/{tools-B10Uj7OI.js → tools-DLHgetpx.js} +6 -6
  216. package/dist/tools-DLHgetpx.js.map +1 -0
  217. package/dist/transcript-BNNANU8A.js +64 -0
  218. package/dist/transcript-BNNANU8A.js.map +1 -0
  219. package/dist/{types-D8RLfkFP.d.ts → types-BYdlwMwo.d.ts} +398 -46
  220. package/dist/{usage-DDF_6tkV.js → usage-CleNKkly.js} +4 -2
  221. package/dist/{usage-DDF_6tkV.js.map → usage-CleNKkly.js.map} +1 -1
  222. package/dist/view/index.d.ts +290 -8
  223. package/dist/view/index.js +7 -7
  224. package/dist/{views-Dqa5ojfZ.js → views-B8w2J-gD.js} +19 -24
  225. package/dist/views-B8w2J-gD.js.map +1 -0
  226. package/dist/views-DblnI3kA.js +2 -0
  227. package/dist/{worker-entry-BAikHl9F.js → worker-entry-DNnveLqx.js} +16 -229
  228. package/dist/worker-entry-DNnveLqx.js.map +1 -0
  229. package/dist/{workspace-C-TrNIUc.js → workspace-C4RA3srV.js} +7 -7
  230. package/dist/{workspace-C-TrNIUc.js.map → workspace-C4RA3srV.js.map} +1 -1
  231. package/dist/workspace-C_X0tdv6.js +1 -0
  232. package/package.json +12 -12
  233. package/dist/agent-WSmTaesf.js.map +0 -1
  234. package/dist/assemble-BCZzTyR7.js.map +0 -1
  235. package/dist/assemble-C5qq5yT2.js +0 -2
  236. package/dist/capture-Bp74Z6Fr.js.map +0 -1
  237. package/dist/capture-DS-cPyuE.js +0 -2
  238. package/dist/cli-CVrMh_qz.js.map +0 -1
  239. package/dist/cli-spec-CKEYa8V2.js.map +0 -1
  240. package/dist/client-Bbo9SPlK.js.map +0 -1
  241. package/dist/command-BrmXUC-W.js +0 -2
  242. package/dist/command-C5xFPKHM.js.map +0 -1
  243. package/dist/command-CzIFp1JQ.js.map +0 -1
  244. package/dist/command-DUT5l13I.js.map +0 -1
  245. package/dist/commands-CLmPBrTS.js.map +0 -1
  246. package/dist/commands-fI3G2Wnh.js +0 -2
  247. package/dist/config-CGvBeewH.js +0 -457
  248. package/dist/config-CGvBeewH.js.map +0 -1
  249. package/dist/context-DbSMEVKs.js.map +0 -1
  250. package/dist/db-CSSFtGcN.js +0 -3
  251. package/dist/db-CfQFwJtq.js.map +0 -1
  252. package/dist/debug-SEmW4w8f.js +0 -2
  253. package/dist/dist-CTzThfu_.js.map +0 -1
  254. package/dist/docker-CRQGQbIt.js.map +0 -1
  255. package/dist/docker-DV2GNgle.js.map +0 -1
  256. package/dist/docker-Df_YOxxB.js +0 -2
  257. package/dist/doctor-DlKycx5H.js.map +0 -1
  258. package/dist/duration-JsJrIWXq.js +0 -26
  259. package/dist/duration-JsJrIWXq.js.map +0 -1
  260. package/dist/egress-BhzNh7Yi.js.map +0 -1
  261. package/dist/env-BQqj93gu.js +0 -21
  262. package/dist/env-BQqj93gu.js.map +0 -1
  263. package/dist/environment-1jE-HhO-.js +0 -657
  264. package/dist/environment-1jE-HhO-.js.map +0 -1
  265. package/dist/errors-C9w_ud7g.js.map +0 -1
  266. package/dist/errors-yDAEwmZo.js.map +0 -1
  267. package/dist/fake-instructions-CUI33I_F.js +0 -32
  268. package/dist/fake-instructions-CUI33I_F.js.map +0 -1
  269. package/dist/family-D2b7WD10.js +0 -67
  270. package/dist/family-D2b7WD10.js.map +0 -1
  271. package/dist/files-AiEKuEZe.js +0 -2
  272. package/dist/files-BM78_nnF.js +0 -2
  273. package/dist/files-BMCvfB4X.js +0 -2
  274. package/dist/harness-CeCp2GSW.js +0 -171
  275. package/dist/harness-CeCp2GSW.js.map +0 -1
  276. package/dist/images-BMgdpMAX.js.map +0 -1
  277. package/dist/index.js.map +0 -1
  278. package/dist/init-B6vegaP_.js +0 -2
  279. package/dist/init-BIdLF0F9.js.map +0 -1
  280. package/dist/jiti-BcJHCtrb.js +0 -46
  281. package/dist/jiti-BcJHCtrb.js.map +0 -1
  282. package/dist/live-S0qODb6o.js.map +0 -1
  283. package/dist/model-B1BZoZ9b.js.map +0 -1
  284. package/dist/model-BOnlAPzs.js.map +0 -1
  285. package/dist/model-DWeg5J7z.js +0 -2
  286. package/dist/model-FXko2kl7.js +0 -2
  287. package/dist/node-Co7EXge0.js.map +0 -1
  288. package/dist/opencode-providers-ByD85Q0l.js.map +0 -1
  289. package/dist/options-CKqyPGxX.js +0 -739
  290. package/dist/options-CKqyPGxX.js.map +0 -1
  291. package/dist/plan-DUydFc_1.js.map +0 -1
  292. package/dist/preflight-BkoSxf4C.js.map +0 -1
  293. package/dist/preflight-D5LW7Oqi.js +0 -2
  294. package/dist/prepare-CUAWHebF.js.map +0 -1
  295. package/dist/prepare-oji2oO8W.js +0 -3
  296. package/dist/process-CSVO_jsc.js +0 -119
  297. package/dist/process-CSVO_jsc.js.map +0 -1
  298. package/dist/provider-BDbNOQhQ.js +0 -2
  299. package/dist/providers-B1DzMvmH.js +0 -107
  300. package/dist/providers-B1DzMvmH.js.map +0 -1
  301. package/dist/providers-CpEfNU1F.js +0 -2
  302. package/dist/query-BjBMLa-C.js +0 -91
  303. package/dist/query-BjBMLa-C.js.map +0 -1
  304. package/dist/query-oPO_XrTy.js +0 -2
  305. package/dist/questions-B8by4Bbt.js +0 -234
  306. package/dist/questions-B8by4Bbt.js.map +0 -1
  307. package/dist/results-Di7m8dQ3.js.map +0 -1
  308. package/dist/runtime-DdsfnRz_.js.map +0 -1
  309. package/dist/sandbox-D5v-of6d.js +0 -139
  310. package/dist/sandbox-D5v-of6d.js.map +0 -1
  311. package/dist/server-BNnXtmS8.js.map +0 -1
  312. package/dist/server-BNwoKzlm.js +0 -2
  313. package/dist/source-Bk_0ZaEY.js +0 -2
  314. package/dist/src-8rRXy4u9.js.map +0 -1
  315. package/dist/src-BGWuWp6I.js +0 -3
  316. package/dist/src-BRt0O02i.js +0 -3
  317. package/dist/src-Bx2x1ekD.js +0 -3
  318. package/dist/src-C_4Umgh2.js.map +0 -1
  319. package/dist/src-D95O-Smv.js.map +0 -1
  320. package/dist/src-DDNXsD9H.js.map +0 -1
  321. package/dist/src-DLa-ESW0.js +0 -4
  322. package/dist/src-cN0aB-WR.js +0 -3
  323. package/dist/src-cdIVjC5b.js +0 -2
  324. package/dist/src-sIsDHWYq.js +0 -3
  325. package/dist/state-Bo5zg4da.js +0 -2
  326. package/dist/state-CiXyLqg0.js.map +0 -1
  327. package/dist/studio-app/assets/index-B9MCTfuu.css +0 -2
  328. package/dist/studio-app/assets/index-CROH2NB3.js +0 -58
  329. package/dist/summary-BTksrIxn.js +0 -254
  330. package/dist/summary-BTksrIxn.js.map +0 -1
  331. package/dist/tool-target-BU1JnSE9.js.map +0 -1
  332. package/dist/tools-B10Uj7OI.js.map +0 -1
  333. package/dist/views-Cs2Oqayd.js +0 -3
  334. package/dist/views-Dqa5ojfZ.js.map +0 -1
  335. package/dist/worker-entry-BAikHl9F.js.map +0 -1
  336. package/dist/workspace-Do_SiWBs.js +0 -2
@@ -41,6 +41,10 @@ type EffortContrast = {
41
41
  agentMsP: number | null;
42
42
  /** The variant's and control's tool-call ranges don't overlap. */
43
43
  separated: boolean;
44
+ /** Their agent-time ranges don't overlap. */
45
+ timeSeparated: boolean;
46
+ /** Their token ranges don't overlap; null when the token test was omitted. */
47
+ tokensSeparated: boolean | null;
44
48
  /** On tool calls, the steadier measure; `timeVerdict` on agent time. */
45
49
  verdict: EffortVerdict;
46
50
  timeVerdict: EffortVerdict;
@@ -67,6 +71,21 @@ declare function effortVerdict(xs: number[], ys: number[]): {
67
71
  p: number | null;
68
72
  verdict: EffortVerdict;
69
73
  };
74
+ /**
75
+ * The fewest attempts a side for "no overlap" to say anything: at 3 against 3 one ordering in
76
+ * ten separates by chance; at 1 against 1 every pair of unequal values does.
77
+ */
78
+ declare const DIRECTIONAL_MIN = 3;
79
+ /**
80
+ * The fewest runs per cell whose effort test can reach `EFFORT_ALPHA` (4 against 4: p = 2/70).
81
+ */
82
+ declare const EFFORT_MIN_RUNS: number;
83
+ type EffortMetric = 'tools' | 'agentS' | 'tokens';
84
+ /**
85
+ * The metrics whose test is `insufficient` but whose ranges don't overlap at
86
+ * `DIRECTIONAL_MIN` a side or more: a direction, not an effect.
87
+ */
88
+ declare function directional(c: EffortContrast): EffortMetric[];
70
89
  declare function contrastEffort(variant: EffortSample[], control: EffortSample[]): EffortContrast | null;
71
90
  /** The counted attempts of one cell of an eval, as effort samples. */
72
91
  declare const cellSamples: (e: Pick<EvalEntry, 'attempts'>, cellKey: string) => EffortSample[];
@@ -76,13 +95,17 @@ declare function cellContrast(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey:
76
95
  /** `4`, `4.5`: medians of counts are whole or halves. */
77
96
  declare const formatCount: (n: number) => string;
78
97
  declare const formatRatio: (r: number | null) => string;
79
- /** Seconds for agent time: `21s`, `1m 05s`. */
98
+ /** Two agent times that both print `<0.1s` have no ratio worth printing. */
99
+ declare const bothUnderATenth: (a: number, b: number) => boolean;
100
+ /** Seconds for agent time: `<0.1s`, `0.4s`, `21s`, `1m 05s`. */
80
101
  declare function formatSeconds(ms: number): string;
81
102
  declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
82
103
  /**
83
104
  * `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
84
105
  * against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
85
106
  * when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
107
+ * A metric too small to test whose ranges don't overlap says so: `(×0.37, no overlap)`. Agent
108
+ * times both under a tenth of a second have no ratio: `agent <0.1s vs <0.1s`.
86
109
  */
87
110
  declare function effortLabel(c: EffortContrast): string;
88
111
  /** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
@@ -281,6 +304,10 @@ type AttemptRecord = {
281
304
  webSearches: number | null;
282
305
  webFetches: number | null;
283
306
  reasoningMs: number;
307
+ /** Transcript tool events with `stalled: true` (long shell or edit). */
308
+ stalledTools: number;
309
+ /** Longest tool `durationMs` in the transcript; null when none reported a duration. */
310
+ maxToolMs: number | null;
284
311
  };
285
312
  judgements: JudgementRecord[];
286
313
  /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
@@ -335,7 +362,9 @@ type EvalEntry = {
335
362
  * The compare's questions, answered after its attempts by pairwise votes, one per question;
336
363
  * unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
337
364
  * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
338
- * (rates by cut, votes, ties, a failed call, a sample of reasons).
365
+ * (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
366
+ * `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
367
+ * masked and screenshots as refs; unset where the eval's files weren't kept.
339
368
  */
340
369
  answers?: CompareAnswer[];
341
370
  /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
@@ -358,7 +387,7 @@ type FamilyInterval = {
358
387
  */
359
388
  method: 'wilson' | 'newcombe-paired' | 't-cases';
360
389
  };
361
- /** One cell's pass rate over a family's cases. */
390
+ /** One cell's pass@1 over a family's cases. */
362
391
  type FamilyRate = {
363
392
  /** The cell's key, the same in every case. */
364
393
  cell: string;
@@ -479,7 +508,7 @@ type FingerprintParts = {
479
508
  environment: string | null;
480
509
  image: string | null;
481
510
  agentCli: string | null;
482
- /** Over the judge's model, threshold and version. */
511
+ /** Over the judge's model, threshold and version, and the config's `state` when it sets one. */
483
512
  judge: string;
484
513
  /** Over the attempt's context files. */
485
514
  context: string;
@@ -531,9 +560,10 @@ type CheckTally = {
531
560
  notRun: number;
532
561
  };
533
562
  /**
534
- * One check over a cell's counted attempts. `verdict` (variant cells): `discriminates` when it
535
- * passes on every counted attempt of one side and none of the other; `both-pass` or `both-fail`
536
- * when it does the same on both sides, so it can't tell them apart; `mixed` otherwise.
563
+ * One check over a cell's counted attempts. `verdict` (variant cells), over the attempts that
564
+ * reached the check: `discriminates` when it passes on every one of one side and none of the
565
+ * other; `both-pass` or `both-fail` when it does the same on both sides, so it can't tell them
566
+ * apart; `mixed` otherwise. No verdict when a side never reached it: not run is not a fail.
537
567
  */
538
568
  type CellCheck = CheckTally & {
539
569
  label: string;
@@ -563,7 +593,7 @@ type Cost = {
563
593
  source: 'reported' | 'unreported';
564
594
  };
565
595
  type RunRecord = {
566
- kind: 'es-run';
596
+ kind: 'reeval-run';
567
597
  schemaVersion: typeof SCHEMA_VERSION;
568
598
  /** Also the file name: `<UTC start>_<short sha>_<author or ci>`. */
569
599
  runId: string;
@@ -644,7 +674,8 @@ type RunRecord = {
644
674
  };
645
675
  /** What ran and graded the agents: the machine, images, harnesses and judge. */
646
676
  type Stack = {
647
- es: string;
677
+ /** The reeval version that wrote the run. */
678
+ reeval: string;
648
679
  /** `process.versions.node`: under Bun or Deno, the Node version they're compatible with. */
649
680
  node: string;
650
681
  /**
@@ -710,6 +741,8 @@ type Stack = {
710
741
  threshold: number;
711
742
  /** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
712
743
  version: string;
744
+ /** What the judge sees by default, every key resolved; absent when the config and profile leave `state` at its defaults. */
745
+ state?: ResolvedJudgeStateOptions;
713
746
  };
714
747
  /** Union of every attempt's context files. */
715
748
  context: ContextFile[];
@@ -801,7 +834,8 @@ type FileEditEvent = {
801
834
  };
802
835
  /**
803
836
  * Normalised agent events. Each tool call is one event, merged with its result; `durationMs`
804
- * is null when the harness never reported the end.
837
+ * is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
838
+ * when the harness never reported the end. Long shell or edit calls set `stalled: true`.
805
839
  */
806
840
  type TranscriptEvent = EventTime & ({
807
841
  type: 'text' | 'reasoning';
@@ -809,6 +843,8 @@ type TranscriptEvent = EventTime & ({
809
843
  durationMs: number | null;
810
844
  } | (FileEditEvent & {
811
845
  durationMs: number | null;
846
+ /** True when a shell-like tool or edit ran at least two minutes. */
847
+ stalled?: boolean;
812
848
  }) | {
813
849
  type: 'web-search';
814
850
  toolCallId: string;
@@ -818,12 +854,16 @@ type TranscriptEvent = EventTime & ({
818
854
  /** Whether the search succeeded. Absent means it did. */
819
855
  ok?: boolean;
820
856
  durationMs: number | null;
857
+ /** True when a shell-like tool or edit ran at least two minutes. */
858
+ stalled?: boolean;
821
859
  } | {
822
860
  type: 'web-fetch';
823
861
  toolCallId: string;
824
862
  url: string;
825
863
  ok: boolean;
826
864
  durationMs: number | null;
865
+ /** True when a shell-like tool or edit ran at least two minutes. */
866
+ stalled?: boolean;
827
867
  } | {
828
868
  type: 'tool';
829
869
  toolCallId: string;
@@ -832,6 +872,8 @@ type TranscriptEvent = EventTime & ({
832
872
  output: unknown;
833
873
  ok: boolean;
834
874
  durationMs: number | null;
875
+ /** True when a shell-like tool or edit ran at least two minutes. */
876
+ stalled?: boolean;
835
877
  } | {
836
878
  type: 'error';
837
879
  message: string;
@@ -850,7 +892,9 @@ type ExecResult = {
850
892
  };
851
893
  /** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
852
894
  type CaptureWindow = {
895
+ /** In CSS pixels. */
853
896
  width: number;
897
+ /** In CSS pixels. */
854
898
  height: number;
855
899
  };
856
900
  /**
@@ -912,6 +956,7 @@ type CaptureUse = {
912
956
  * pass it as the first argument to `c.capture`.
913
957
  */
914
958
  type CaptureOptions = {
959
+ /** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
915
960
  viewport?: Viewport;
916
961
  /** How long to poll before failing. Default `'60s'`. */
917
962
  timeout?: Duration;
@@ -1017,6 +1062,7 @@ type Turn = TurnData & {
1017
1062
  exec(command: string): Promise<ExecResult>;
1018
1063
  readFile(path: string): Promise<string | null>;
1019
1064
  };
1065
+ /** Any JSON value. */
1020
1066
  type JSONValue = null | boolean | number | string | JSONValue[] | {
1021
1067
  [key: string]: JSONValue;
1022
1068
  };
@@ -1053,10 +1099,11 @@ type JudgeArtifactRef = {
1053
1099
  */
1054
1100
  type FakeJudgeModel = {
1055
1101
  kind: 'fake-judge';
1102
+ /** The score (0–1) every call gets, or a function of the state it would see. */
1056
1103
  score: number | ((state: JudgeState) => number);
1057
1104
  };
1058
1105
  /**
1059
- * `'provider/model'` (e.g. `'typesafe-ai/jev-1.13.0'`, `'google/gemini-3.8-flash'`), an AI SDK
1106
+ * `'provider/model'` (e.g. `'typesafe-ai/jev-1.14.0'`, `'google/gemini-3.8-flash'`), an AI SDK
1060
1107
  * decision model object (per group, eval or call only), or a fake judge.
1061
1108
  */
1062
1109
  type JudgeModel = string | FakeJudgeModel | {
@@ -1064,28 +1111,77 @@ type JudgeModel = string | FakeJudgeModel | {
1064
1111
  };
1065
1112
  /** Merged field by field down the cascade, so per-model settings can join as new fields. */
1066
1113
  type JudgeOptions = {
1114
+ /** The model that grades. */
1067
1115
  model?: JudgeModel;
1068
1116
  /** Minimum score (0–1) for a `pass` verdict. */
1069
1117
  threshold?: number;
1118
+ /** AI SDK provider options for the judge's calls, keyed by provider. */
1070
1119
  providerOptions?: Record<string, Record<string, JSONValue>>;
1071
1120
  /** The most judge calls to this model's provider at once, across the run. */
1072
1121
  maxConcurrent?: number;
1073
1122
  /**
1074
- * Which changed files the judge's diff shows: gitignore-style patterns over workspace paths,
1075
- * where the last matching pattern wins and a `!` pattern excludes. Default: every changed file.
1123
+ * The judge's state: what the judge is given beside the question, not the workspace's or the
1124
+ * attempt's state. Merged key by key down the cascade.
1076
1125
  */
1077
- include?: string[];
1078
- /** Workspace files the judge sees whole, as they are now, whether or not the agent changed them. */
1079
- files?: string[];
1126
+ state?: JudgeStateOptions;
1127
+ };
1128
+ /**
1129
+ * What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
1130
+ * keeps its default, which is what the judge gets with no `state` option at all.
1131
+ */
1132
+ type JudgeStateOptions = {
1133
+ /** The graded turn's prompt. Default true. */
1134
+ task?: boolean;
1135
+ /** The graded turn's final message, or `on`'s value. Default true. */
1136
+ output?: boolean;
1137
+ /** `status path` lines for the changed files, under `diff`'s patterns. Default true. */
1138
+ files?: boolean;
1139
+ /**
1140
+ * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
1141
+ * the last matching pattern wins and `!` excludes. Default true.
1142
+ */
1143
+ diff?: boolean | string[];
1144
+ /**
1145
+ * Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
1146
+ * patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
1147
+ * the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
1148
+ * These files leave the diff. Default none.
1149
+ */
1150
+ contents?: string[];
1151
+ /**
1152
+ * The graded turn's transcript as text lines: `true` is messages and tool calls with tool
1153
+ * output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
1154
+ * Default false.
1155
+ */
1156
+ transcript?: boolean | 'tools' | 'full';
1157
+ /**
1158
+ * `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
1159
+ * `transcript`) transcript. Default `'last'`.
1160
+ */
1161
+ turns?: 'last' | 'all';
1162
+ /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
1163
+ checks?: boolean | string[];
1164
+ };
1165
+ /** `JudgeStateOptions` with every key set: what one judge call saw. */
1166
+ type ResolvedJudgeStateOptions = {
1167
+ task: boolean;
1168
+ output: boolean;
1169
+ files: boolean;
1170
+ diff: boolean | string[];
1171
+ contents: string[];
1172
+ transcript: boolean | 'tools' | 'full';
1173
+ turns: 'last' | 'all';
1174
+ checks: boolean | string[];
1080
1175
  };
1081
1176
  type JudgeCallOptions = JudgeOptions & {
1082
1177
  /** What to grade instead of the latest turn: another turn, or a value used as the output. */
1083
1178
  on?: Turn | JSONValue;
1084
1179
  /**
1085
- * Captures this call sends the judge as file parts. A `c.capture()` handle or its name.
1086
- * Call-only: the bytes are on this attempt. The provider that cannot take files fails the call.
1180
+ * Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
1181
+ * `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
1182
+ * captures; a provider that cannot take files fails the call.
1087
1183
  */
1088
- artifacts?: Array<Capture | string>;
1184
+ artifacts?: true | Array<Capture | string>;
1089
1185
  };
1090
1186
  type Verdict = {
1091
1187
  verdict: 'pass' | 'fail';
@@ -1108,12 +1204,16 @@ type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
1108
1204
  score: null;
1109
1205
  error: string;
1110
1206
  };
1207
+ /**
1208
+ * What an eval's body gets. The sandbox and the agent session start on the first call that needs
1209
+ * them (`send`, `exec`, `readFile`, `writeFile`, `serve` or `capture`), once however many ask.
1210
+ */
1111
1211
  type AttemptContext = {
1112
- /** Sends the next turn of this eval's agent session, starting the sandbox on first use. */
1212
+ /** Sends the next turn of this eval's agent session. */
1113
1213
  send(prompt: string): Promise<Turn>;
1114
1214
  /**
1115
1215
  * Asks the judge a yes/no question about the latest turn (or `options.on`).
1116
- * `options.artifacts` is a list of `c.capture()` handles or names.
1216
+ * `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
1117
1217
  */
1118
1218
  judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1119
1219
  exec(command: string): Promise<ExecResult>;
@@ -1132,7 +1232,6 @@ type AttemptContext = {
1132
1232
  * Screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`.
1133
1233
  * First argument is the container port or a `ServeHandle` from `c.serve`.
1134
1234
  * `instructions` runs a vision journey on that same tab first.
1135
- * Needs a sandbox, same as `c.exec`.
1136
1235
  */
1137
1236
  capture(target: number | ServeHandle, options?: CaptureCall): Promise<Capture>;
1138
1237
  /** Workspace-relative, or `~/` for the home directory; `null` when there's no such file. */
@@ -1140,14 +1239,9 @@ type AttemptContext = {
1140
1239
  /**
1141
1240
  * Writes a file into the sandbox (workspace-relative, or `~/`), creating its directories.
1142
1241
  * Later turns' `diff` leaves it out, as it does the environment's files, but shows any change
1143
- * made to it afterwards. Needs a sandbox, same as `c.exec`.
1242
+ * made to it afterwards.
1144
1243
  */
1145
1244
  writeFile(path: string, contents: string | Uint8Array): Promise<void>;
1146
- /**
1147
- * Starts the sandbox and the agent session without a turn, for checks on what the environment
1148
- * set up. `c.send()` starts them too; calling both starts them once.
1149
- */
1150
- sandbox(): Promise<Pick<AttemptContext, 'exec' | 'readFile'>>;
1151
1245
  /** Every turn so far, oldest first. */
1152
1246
  readonly turns: readonly Turn[];
1153
1247
  /** Aborted on timeout or Ctrl-C. */
@@ -1160,12 +1254,14 @@ type AttemptContext = {
1160
1254
  };
1161
1255
  };
1162
1256
  type EvalFn = (c: AttemptContext) => Promise<void>;
1257
+ /** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
1163
1258
  type FakeHarnessDescriptor = {
1164
1259
  kind: 'fake';
1165
1260
  /** Cell id; defaults to `fake`. */
1166
1261
  id?: string;
1167
1262
  /** Files the fake agent writes into the sandbox on every turn. */
1168
1263
  files?: Record<string, string>;
1264
+ /** The fake agent's final message on every turn; default `Done.`. */
1169
1265
  text?: string;
1170
1266
  /** Simulated turn length, honours the abort signal. */
1171
1267
  delayMs?: number;
@@ -1217,10 +1313,11 @@ type FakeHarnessDescriptor = {
1217
1313
  * Behave like this agent: its image, and where it reads rules and skills (`CLAUDE.md` and
1218
1314
  * `.claude/skills` for `claude-code`, `~/.gemini/config/skills` for `antigravity`,
1219
1315
  * `~/.config/opencode/skills` for `opencode`, Goose's private home for `goose`,
1220
- * `~/.pi/agent/skills` for `pi`, `~/.agents/skills` for `codex`). Defaults to the
1221
- * fake's own (`AGENTS.md`, `.agents/skills`) in Cursor's image.
1316
+ * `~/.pi/agent/skills` for `pi`, `~/.agents/skills` for `codex`, fx's private home for
1317
+ * `fx`, Grok's private home for `grok-build`). Defaults to the fake's own (`AGENTS.md`,
1318
+ * `.agents/skills`) in Cursor's image.
1222
1319
  */
1223
- layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex';
1320
+ layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex' | 'fx' | 'grok-build';
1224
1321
  };
1225
1322
  /**
1226
1323
  * One action of a scripted fake agent. `exec` is reported as a `shell` tool call whose output is
@@ -1406,28 +1503,82 @@ type CodexHarnessDescriptor = {
1406
1503
  */
1407
1504
  startTimeout?: Duration;
1408
1505
  };
1506
+ /**
1507
+ * fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
1508
+ * OpenAI's API by the model id, and reports each turn's tokens.
1509
+ */
1510
+ type FxHarnessDescriptor = {
1511
+ kind: 'fx';
1512
+ /**
1513
+ * A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
1514
+ * `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
1515
+ * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-4.1`.
1516
+ */
1517
+ model: string;
1518
+ /** Cell id; defaults to `fx:<model>`. */
1519
+ id?: string;
1520
+ /** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
1521
+ maxConcurrent?: number;
1522
+ /**
1523
+ * How long the harness's session may take to become ready (its bootstrap and first session)
1524
+ * before the attempt is an infra error, after one retry; default 90s.
1525
+ */
1526
+ startTimeout?: Duration;
1527
+ };
1528
+ /**
1529
+ * xAI's Grok Build through `@reeval/harness-grok-build` (`grok agent stdio`), made by
1530
+ * `grokBuild()`. It runs xAI models on an xAI API key and reports each turn's tokens.
1531
+ */
1532
+ type GrokBuildHarnessDescriptor = {
1533
+ kind: 'grok-build';
1534
+ /** An xAI model id Grok Build runs on the xAI API, e.g. `grok-4.6`. */
1535
+ model: string;
1536
+ /**
1537
+ * Reasoning effort as Grok's `default_reasoning_effort`, e.g. `low`, `medium` or `high`, for a
1538
+ * model that takes one.
1539
+ */
1540
+ effort?: string;
1541
+ /** Cell id; defaults to `grok-build:<model>`, or `grok-build:<model>[effort=<value>]` when `effort` is set. */
1542
+ id?: string;
1543
+ /** The most Grok Build attempts at once, across the run, e.g. for the key's rate limits. */
1544
+ maxConcurrent?: number;
1545
+ /**
1546
+ * How long the harness's session may take to become ready (its bootstrap and first session)
1547
+ * before the attempt is an infra error, after one retry; default 90s.
1548
+ */
1549
+ startTimeout?: Duration;
1550
+ };
1409
1551
  /** A real coding agent, driven through its AI SDK harness adapter. */
1410
- type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor;
1552
+ type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
1553
+ /** The agent an attempt runs: a real harness, or the fake one. */
1411
1554
  type HarnessDescriptor = FakeHarnessDescriptor | AgentHarnessDescriptor;
1555
+ /** A temp directory on the host instead of a container, made by `fakeSandbox()`. */
1412
1556
  type FakeSandboxDescriptor = {
1413
1557
  kind: 'fake';
1558
+ /** Cell id; defaults to `fake`. */
1414
1559
  id?: string;
1415
1560
  /** Parent directory for the per-eval workspace (defaults to the OS temp dir). */
1416
1561
  dir?: string;
1417
1562
  /** The most attempts on this sandbox kind at once, across the run. */
1418
1563
  maxConcurrent?: number;
1419
1564
  };
1565
+ /** A local Docker container, made by `docker()`, or a Podman one, made by `podman()`. */
1420
1566
  type DockerSandboxDescriptor = {
1421
1567
  kind: 'docker';
1422
- /** Cell id; defaults to `docker`, or `docker:<image>` for other images, plus an install hash. */
1568
+ /** The container CLI the sandbox runs on: `podman` for `podman()`, Docker when unset. */
1569
+ engine?: 'docker' | 'podman';
1570
+ /**
1571
+ * Cell id; defaults to `docker` (`podman`), or `docker:<image>` for other images, plus an
1572
+ * install hash.
1573
+ */
1423
1574
  id?: string;
1424
1575
  /** An image that already has the harness baked in, used as is. */
1425
1576
  image?: string;
1426
1577
  /** A Debian or Ubuntu based toolchain image; reeval bakes Node and the harness on top, cached. */
1427
1578
  from?: string;
1428
1579
  /**
1429
- * An absolute path to a Dockerfile (`docker()` resolves it against the calling file), built from
1430
- * its directory with `REEVAL_BASE` set to reeval's sandbox image and cached by content hash.
1580
+ * A Dockerfile, absolute or relative to the directory of the file it's written in, built from its
1581
+ * directory with `REEVAL_BASE` set to reeval's sandbox image and cached by content hash.
1431
1582
  */
1432
1583
  dockerfile?: string;
1433
1584
  /** Baked onto the sandbox image in one build container: in order, no secrets, no workspace. */
@@ -1485,13 +1636,17 @@ type SecretOptions = {
1485
1636
  /** `secret.template('Bearer ${TOKEN}')`: host variables substituted into a string, for MCP headers. */
1486
1637
  type SecretTemplate = {
1487
1638
  kind: 'secret-template';
1639
+ /** The string, with `${NAME}` for each host variable to substitute. */
1488
1640
  template: string;
1489
1641
  };
1642
+ /** A variable's value: a literal string, or a host secret. */
1490
1643
  type EnvValue = string | Secret;
1644
+ /** Environment variables by name. */
1491
1645
  type EnvVars = Record<string, EnvValue>;
1492
1646
  /** A file on the host, read when an attempt starts. Made by the `file` helper. */
1493
1647
  type HostFile = {
1494
1648
  kind: 'file';
1649
+ /** Its path on the host, absolute or relative to the directory of the file it's written in. */
1495
1650
  path: string;
1496
1651
  };
1497
1652
  /** Options for the `directory` helper. */
@@ -1509,62 +1664,92 @@ type DirectoryOptions = {
1509
1664
  /** `false` walks into `DEFAULT_DIRECTORY_IGNORES` too. `.git` is always skipped. */
1510
1665
  defaultIgnores?: boolean;
1511
1666
  };
1667
+ /** An MCP server the agent gets, made by the `mcp` helper: a command in the sandbox, or a URL. */
1512
1668
  type McpServer = {
1513
1669
  kind: 'mcp';
1514
1670
  transport: 'stdio';
1671
+ /** The command that starts the server in the sandbox. */
1515
1672
  command: string;
1516
1673
  args: string[];
1674
+ /** Variables the server process gets. */
1517
1675
  env: EnvVars;
1518
1676
  } | {
1519
1677
  kind: 'mcp';
1678
+ /** Streamable HTTP, or the older SSE transport. */
1520
1679
  transport: 'http' | 'sse';
1521
1680
  url: string;
1681
+ /** Request headers, with secrets brokered by the egress gateway. */
1522
1682
  headers: Record<string, string | Secret | SecretTemplate>;
1523
1683
  };
1524
1684
  /** `harness` (default): the agent's own skills, via its adapter. `workspace`: its repo skill dir. */
1525
1685
  type SkillPlacement = 'harness' | 'workspace';
1686
+ /** Skills to place for the agent, made by the `skill` helper. */
1526
1687
  type SkillSource = {
1527
1688
  kind: 'skill';
1528
1689
  placement: SkillPlacement;
1529
1690
  } & ({
1530
1691
  source: 'local';
1692
+ /** A host directory with a `SKILL.md`, or a directory of them. */
1531
1693
  path: string;
1694
+ /** The skill names to keep; default every one found. */
1532
1695
  only?: string[];
1696
+ /** Paths in each skill directory to leave out, gitignore-style. */
1533
1697
  exclude?: string[];
1534
1698
  } | {
1535
1699
  source: 'git';
1700
+ /** As written: a URL, `github:owner/repo`, or a GitHub `/tree/<ref>/<path>` URL. */
1536
1701
  repo: string;
1702
+ /** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
1537
1703
  ref?: string;
1704
+ /** The directory in the repository to read skills from. */
1538
1705
  path?: string;
1706
+ /** The skill names to keep; default every one found. */
1539
1707
  only?: string[];
1708
+ /** Paths in each skill directory to leave out, gitignore-style. */
1540
1709
  exclude?: string[];
1541
1710
  } | {
1542
1711
  source: 'registry';
1712
+ /** A skills.sh id, `owner/repo@skill`. */
1543
1713
  id: string;
1714
+ /** A git ref of the skill's repository. */
1544
1715
  version?: string;
1545
1716
  } | {
1546
1717
  source: 'npm';
1718
+ /** A pinned npm package spec. */
1547
1719
  spec: string;
1720
+ /** The directory in the package to read skills from; default `skills`. */
1548
1721
  path: string;
1722
+ /** The skill names to keep; default every one found. */
1549
1723
  only?: string[];
1724
+ /** Paths in each skill directory to leave out, gitignore-style. */
1550
1725
  exclude?: string[];
1551
1726
  } | {
1552
1727
  source: 'inline';
1728
+ /** The skill's name, as in `SKILL.md`'s frontmatter. */
1553
1729
  name: string;
1730
+ /** The skill's description, as in `SKILL.md`'s frontmatter. */
1554
1731
  description: string;
1732
+ /** The body of `SKILL.md`. */
1555
1733
  content: string;
1734
+ /** Other files in the skill directory, by relative path. */
1556
1735
  files: Record<string, string>;
1557
1736
  });
1558
1737
  /** A `docker({ install })` command, baked into a cached image layer: no secrets, root, cwd `/`. */
1559
1738
  type InstallStep = string | {
1739
+ /** The shell command. */
1560
1740
  run: string;
1561
1741
  timeout?: Duration;
1562
1742
  };
1563
- /** Runs in every attempt before the agent starts, with secrets, outside the eval timeout. */
1564
- /** A shell command, or `{ run, timeout?, cwd? }`; `cwd` is relative to the workspace or absolute. */
1743
+ /**
1744
+ * Runs in every attempt before the agent starts, with secrets, outside the eval timeout: a shell
1745
+ * command, or `{ run, timeout?, cwd? }`.
1746
+ */
1565
1747
  type SetupStep = string | {
1748
+ /** The shell command. */
1566
1749
  run: string;
1750
+ /** Default 5m. */
1567
1751
  timeout?: Duration;
1752
+ /** Relative to the workspace, or absolute; default the workspace. */
1568
1753
  cwd?: string;
1569
1754
  };
1570
1755
  /**
@@ -1584,6 +1769,7 @@ type GitWorkspace = {
1584
1769
  kind: 'git';
1585
1770
  /** As written: a URL, `github:owner/repo`, or a local path. */
1586
1771
  repo: string;
1772
+ /** What reeval fetches: the clone URL, or an absolute path for a local repository. */
1587
1773
  url: string;
1588
1774
  /** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
1589
1775
  ref?: string;
@@ -1609,6 +1795,7 @@ type EnvironmentSpec = {
1609
1795
  sandbox?: SandboxDescriptor;
1610
1796
  /** The code the workspace starts as, before `files`, `rules` and `setup`; a child's replaces its parent's. */
1611
1797
  workspace?: GitWorkspace;
1798
+ /** Variables and secrets for the sandbox, merged by name over the parent's. */
1612
1799
  env?: EnvVars;
1613
1800
  /** Replaces the inherited policy; an `extend` child's replaces its parent's. */
1614
1801
  egress?: EgressPolicy;
@@ -1616,13 +1803,18 @@ type EnvironmentSpec = {
1616
1803
  files?: Record<string, string | HostFile>;
1617
1804
  /** Project rules, written where the harness's agent reads them: `AGENTS.md`, or `CLAUDE.md` for Claude Code. */
1618
1805
  rules?: string | HostFile;
1806
+ /** Shell commands run in every attempt before the agent starts; a child's run after its parent's. */
1619
1807
  setup?: SetupStep[];
1808
+ /** Skills placed for the agent. */
1620
1809
  skills?: SkillSource[];
1810
+ /** MCP servers the agent gets, by name. */
1621
1811
  mcp?: Record<string, McpServer>;
1622
1812
  };
1623
1813
  /** One environment in an `extend` chain. */
1624
1814
  type EnvironmentLayer = {
1815
+ /** The name passed to `defineEnvironment` or `extend`. */
1625
1816
  name: string;
1817
+ /** The spec as that layer wrote it, before merging. */
1626
1818
  spec: EnvironmentSpec;
1627
1819
  };
1628
1820
  /**
@@ -1631,7 +1823,9 @@ type EnvironmentLayer = {
1631
1823
  */
1632
1824
  type Environment = {
1633
1825
  readonly kind: 'environment';
1826
+ /** The name it was defined or extended under, which names its cells. */
1634
1827
  readonly name: string;
1828
+ /** Its `extend` chain, root first. */
1635
1829
  readonly layers: readonly EnvironmentLayer[];
1636
1830
  /**
1637
1831
  * A child environment layered on this one; wherever it sets a key this one has, it wins.
@@ -1767,14 +1961,71 @@ type CompareOptions<E = Environment> = {
1767
1961
  };
1768
1962
  /**
1769
1963
  * A question for pairwise judgement: its text, or the text and the `'provider/model'` that
1770
- * answers it. Without a model, `analysis.model` answers, else the judge's when it's a language model.
1964
+ * answers it. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the
1965
+ * judge's when it's a language model.
1771
1966
  * `across` is the factor it varies when pairing; without it (and without a one-factor
1772
- * `compare.across`) both pair kinds run.
1967
+ * `compare.across`) both pair kinds run. `state` and `artifacts` choose what each side shows.
1773
1968
  */
1774
1969
  type CompareQuestion = string | {
1775
1970
  question: string;
1971
+ /** `'provider/model'` that answers it, over the cascaded `question.model`. */
1776
1972
  model?: string;
1973
+ /** The factor it varies when pairing. */
1777
1974
  across?: PairFactor;
1975
+ /** What each side shows, over the cascaded `question.state`, key by key. */
1976
+ state?: QuestionStateOptions;
1977
+ /** Each side's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
1978
+ artifacts?: true | string[];
1979
+ };
1980
+ /**
1981
+ * What each side of a compare question shows: the keys of `judge.state` but `contents` (the
1982
+ * attempts' workspaces are gone when questions are asked). An unset key keeps its default, which
1983
+ * is what a question shows with no `state` at all: the task and the diff.
1984
+ */
1985
+ type QuestionStateOptions = {
1986
+ /** The prompts the attempt was sent, once for both sides when they match. Default true. */
1987
+ task?: boolean;
1988
+ /** The final message: the agent's own account of its work. Default false. */
1989
+ output?: boolean;
1990
+ /** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
1991
+ files?: boolean;
1992
+ /**
1993
+ * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
1994
+ * the last matching pattern wins and `!` excludes. Default true.
1995
+ */
1996
+ diff?: boolean | string[];
1997
+ /**
1998
+ * The transcript as text lines: `true` is messages and tool calls with tool output clipped,
1999
+ * `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
2000
+ * harness's tools, so a pair of two harnesses isn't blind. Default false.
2001
+ */
2002
+ transcript?: boolean | 'tools' | 'full';
2003
+ /**
2004
+ * `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
2005
+ * transcript; `'last'`: the last turn only. Default `'all'`.
2006
+ */
2007
+ turns?: 'last' | 'all';
2008
+ /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
2009
+ checks?: boolean | string[];
2010
+ };
2011
+ /** `QuestionStateOptions` with every key set: what each side of one question showed. */
2012
+ type ResolvedQuestionStateOptions = {
2013
+ task: boolean;
2014
+ output: boolean;
2015
+ files: boolean;
2016
+ diff: boolean | string[];
2017
+ transcript: boolean | 'tools' | 'full';
2018
+ turns: 'last' | 'all';
2019
+ checks: boolean | string[];
2020
+ };
2021
+ /** Defaults for every compare question: `model` and `artifacts` replaced, `state` merged key by key. */
2022
+ type QuestionOptions = {
2023
+ /** `'provider/model'` that answers every question below it without its own `model`. */
2024
+ model?: string;
2025
+ /** What each side shows. */
2026
+ state?: QuestionStateOptions;
2027
+ /** Each side's own captures, by name; `true` sends all of them. Default none. */
2028
+ artifacts?: true | string[];
1778
2029
  };
1779
2030
  /** A matrix factor a compare question varies when pairing; the others are held. */
1780
2031
  type PairFactor = 'environment' | 'harness';
@@ -1799,6 +2050,11 @@ type PairVote = {
1799
2050
  swapped: boolean;
1800
2051
  /** The model's short reason for this pick; omitted when it gave none. Capped. */
1801
2052
  reason?: string;
2053
+ /** The captures each named side showed, by name; set when the question sends `artifacts`. */
2054
+ artifacts?: {
2055
+ a: string[];
2056
+ b: string[];
2057
+ };
1802
2058
  };
1803
2059
  /** A compare question's answer: every vote, and each side's wins over decided votes per cut. */
1804
2060
  type CompareAnswer = {
@@ -1807,6 +2063,10 @@ type CompareAnswer = {
1807
2063
  model: string;
1808
2064
  /** The model id the provider says served the calls, when it reports one. */
1809
2065
  resolvedModel?: string;
2066
+ /** What each side showed, every key resolved; unset on answers from before questions took `state`, which showed the defaults. */
2067
+ stateOptions?: ResolvedQuestionStateOptions;
2068
+ /** The captures it sent of each side, as asked: `true` for all, or their names. */
2069
+ artifacts?: true | string[];
1810
2070
  votes: PairVote[];
1811
2071
  /** Wins over decided pairs, per named side, per cut. */
1812
2072
  rates: Array<{
@@ -1817,7 +2077,10 @@ type CompareAnswer = {
1817
2077
  }>;
1818
2078
  /** Why some or all of its votes are missing: the model call failed. Outcomes never change. */
1819
2079
  error?: string;
2080
+ /** `questions/<n>.json` under the eval's directory: what the model was sent, per pair and order. */
2081
+ input?: FileRef;
1820
2082
  };
2083
+ /** An option a profile's `limit` can narrow, and `pin` can protect. */
1821
2084
  type PinnableKey = 'harness' | 'runs' | 'compare';
1822
2085
  /**
1823
2086
  * How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
@@ -1833,7 +2096,7 @@ type CellGate = GatePolicy | 'none';
1833
2096
  type CompareSpec<E = Environment> = CompareOptions<E>;
1834
2097
  /** Options other than `environment` and `compare`, which are exclusive of each other. */
1835
2098
  type EvalOptionFields = {
1836
- /** The only list: each harness is its own cell. */
2099
+ /** Which agents run the eval; each harness is its own cell. */
1837
2100
  harness?: HarnessOption;
1838
2101
  /**
1839
2102
  * The default machine when the resolved environment omits `sandbox`. Cascades like `harness`.
@@ -1851,11 +2114,15 @@ type EvalOptionFields = {
1851
2114
  * environment, then groups and the eval.
1852
2115
  */
1853
2116
  egress?: EgressPolicy;
2117
+ /** Defaults for `c.judge()`, merged field by field down the cascade. */
1854
2118
  judge?: JudgeOptions;
1855
2119
  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
1856
2120
  capture?: CaptureOptions;
2121
+ /** Defaults for every compare question: `model`, and `state` merged key by key, down the cascade. */
2122
+ question?: QuestionOptions;
1857
2123
  /** Per attempt. */
1858
2124
  timeout?: Duration;
2125
+ /** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
1859
2126
  tags?: string[];
1860
2127
  /** Keys a profile's `limit` may not narrow for this group or eval. */
1861
2128
  pin?: PinnableKey[];
@@ -1873,20 +2140,36 @@ type EnvironmentOrCompare<E> = {
1873
2140
  } | {
1874
2141
  /** Forbidden beside `compare.baseline`, which names the control. */
1875
2142
  environment?: never;
2143
+ /** Environments to run side by side; `baseline` is the control. */
1876
2144
  compare: CompareSpec<E> & {
1877
2145
  baseline: E;
1878
2146
  };
1879
2147
  } | {
1880
2148
  /** The control: the compare has no `baseline`. */
1881
2149
  environment?: E;
2150
+ /** Environments to run side by side against `environment`, the control. */
1882
2151
  compare: CompareSpec<E> & {
1883
2152
  baseline?: undefined;
1884
2153
  };
1885
2154
  };
1886
2155
  type EvalOptions = EvalOptionFields & EnvironmentOrCompare<Environment>;
1887
2156
  type GroupOptions = EvalOptions;
2157
+ /**
2158
+ * `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
2159
+ * by name or a fake judge, and environments without `extend`. The same `environment` / compare
2160
+ * object split as `EvalOptions`.
2161
+ */
2162
+ type EvalOptionsData = Omit<EvalOptionFields, 'harness' | 'judge'> & {
2163
+ /** Which agents run the eval; each harness is its own cell. */
2164
+ harness?: HarnessData;
2165
+ /** Defaults for `c.judge()`, merged field by field down the cascade. */
2166
+ judge?: ConfigJudgeOptions;
2167
+ } & EnvironmentOrCompare<EnvironmentData>;
2168
+ /** An `Environment` as data, as JSON carries it: without `extend` / `extendEnvironment`. */
2169
+ type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend'>;
1888
2170
  /** The config file is plain data sent to workers, so its judge model can't be an object. */
1889
2171
  type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
2172
+ /** `'provider/model'`, or a fake judge. */
1890
2173
  model?: string | FakeJudgeModel;
1891
2174
  };
1892
2175
  type HistoryConfig = {
@@ -1897,22 +2180,35 @@ type HistoryConfig = {
1897
2180
  };
1898
2181
  /** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
1899
2182
  type Budget = {
2183
+ /** The most attempts the plan may have. */
1900
2184
  attempts?: number;
2185
+ /** How long the run may go on, from the first attempt's start. */
1901
2186
  duration?: Duration;
1902
2187
  };
1903
2188
  /** What evals get when they don't say otherwise: under groups and evals. */
1904
2189
  type ProfileDefaults = {
2190
+ /** The agents every eval runs on, unless a group or eval replaces them. */
1905
2191
  harness?: HarnessDescriptor | HarnessDescriptor[];
1906
2192
  /** The default machine when the resolved environment omits `sandbox`. */
1907
2193
  sandbox?: SandboxDescriptor;
2194
+ /** What every eval runs in when it names no environment. */
1908
2195
  environment?: Environment;
2196
+ /** Attempts per cell. */
1909
2197
  runs?: number;
2198
+ /** How each cell's attempts decide its outcome. */
1910
2199
  gate?: GatePolicy;
2200
+ /** Variables and secrets for the sandbox, merged by name over the config's. */
1911
2201
  env?: EnvVars;
2202
+ /** Where the sandbox may connect; replaces the config's policy. */
1912
2203
  egress?: EgressPolicy;
2204
+ /** Defaults for `c.judge()`, merged field by field over the config's. */
1913
2205
  judge?: ConfigJudgeOptions;
2206
+ /** Per attempt. */
1914
2207
  timeout?: Duration;
2208
+ /** Screenshot defaults for `c.capture()`, shallow-merged over the config's. */
1915
2209
  capture?: CaptureOptions;
2210
+ /** Defaults for every compare question, merged over the config's. */
2211
+ question?: QuestionOptions;
1916
2212
  };
1917
2213
  /** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
1918
2214
  type ProfileLimit = {
@@ -1927,10 +2223,15 @@ type ProfileLimit = {
1927
2223
  };
1928
2224
  /** How a run is sized: chosen with `--profile`, `REEVAL_PROFILE` or `defaultProfile`. */
1929
2225
  type Profile = {
2226
+ /** What evals get when they don't say otherwise: under groups and evals. */
1930
2227
  defaults?: ProfileDefaults;
2228
+ /** Caps over every eval, which only narrow; `pin` on a group or eval skips them. */
1931
2229
  limit?: ProfileLimit;
2230
+ /** Judge settings for this profile, the same as `defaults.judge`, which wins field by field. */
1932
2231
  judge?: ConfigJudgeOptions;
2232
+ /** Merged over the config's `budget`. */
1933
2233
  budget?: Budget;
2234
+ /** When this profile's runs are copied into the history folder. */
1934
2235
  record?: RecordMode;
1935
2236
  /** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
1936
2237
  tags?: string[];
@@ -1974,12 +2275,48 @@ type FakePick = FakeVote | {
1974
2275
  winner: FakeVote;
1975
2276
  reason?: string;
1976
2277
  };
1977
- /** What a compare question's model is shown: the prompt, and the two diffs in slot order. */
2278
+ /** What a compare question's model is shown: the prompt, and the two sides in slot order. */
1978
2279
  type BlindPair = {
1979
2280
  question: string;
1980
2281
  prompt: string;
1981
- a: string;
1982
- b: string;
2282
+ a: BlindSide;
2283
+ b: BlindSide;
2284
+ };
2285
+ /** A turn of one side, with the keys its question's `state` turned on. */
2286
+ type BlindTurn = {
2287
+ task?: string;
2288
+ output?: string;
2289
+ transcript?: string;
2290
+ };
2291
+ /**
2292
+ * One side of a blinded pair: only the keys its question's `state` turned on. With
2293
+ * `turns: 'all'`, `task`, `output` and `transcript` are under `turns`; a task both sides share is
2294
+ * the pair's prompt's, not either side's.
2295
+ */
2296
+ type BlindSide = BlindTurn & {
2297
+ turns?: BlindTurn[];
2298
+ files?: string[];
2299
+ /** The diff as the model sees it: dependency and binary noise dropped. */
2300
+ diff?: string;
2301
+ /** A command the other side ran and this one didn't is `{ command, notRun: true }`. */
2302
+ checks?: Array<{
2303
+ command: string;
2304
+ exitCode: number;
2305
+ stdout: string;
2306
+ stderr: string;
2307
+ } | {
2308
+ command: string;
2309
+ notRun: true;
2310
+ }>;
2311
+ /** Its captures, in the order sent, labelled by position; a name it lacks is `missing`. */
2312
+ artifacts?: Array<{
2313
+ label: string;
2314
+ mediaType: 'image/png';
2315
+ bytes: number;
2316
+ } | {
2317
+ label: string;
2318
+ missing: true;
2319
+ }>;
1983
2320
  };
1984
2321
  /** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
1985
2322
  type AnalysisConfig = {
@@ -1997,33 +2334,46 @@ type AnalysisConfig = {
1997
2334
  /** The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap. */
1998
2335
  maxOutputTokens?: number;
1999
2336
  };
2000
- type EsConfig = {
2337
+ type ReevalConfig = {
2338
+ /** Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`. */
2001
2339
  include?: string[];
2340
+ /** The agents every eval runs on, unless a group or eval replaces them; default Cursor. */
2002
2341
  harness?: HarnessDescriptor | HarnessDescriptor[];
2003
2342
  /**
2004
2343
  * The default machine when the resolved environment omits `sandbox`. Not an environment:
2005
2344
  * it does not invent skills, files or a name. Implicit `docker()` when nobody sets one.
2006
2345
  */
2007
2346
  sandbox?: SandboxDescriptor;
2347
+ /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
2008
2348
  environment?: Environment;
2349
+ /** Attempts per cell; default 1. */
2009
2350
  runs?: number;
2010
2351
  /** How each cell's attempts decide its outcome; default `'all'`. */
2011
2352
  gate?: GatePolicy;
2353
+ /** Variables and secrets for the sandbox, merged by name down the cascade. */
2012
2354
  env?: EnvVars;
2355
+ /** Where the sandbox may connect; default `'allow-all'`. */
2013
2356
  egress?: EgressPolicy;
2357
+ /** Defaults for `c.judge()`, merged field by field down the cascade. */
2014
2358
  judge?: ConfigJudgeOptions;
2359
+ /** Per attempt; default `'15m'`. */
2015
2360
  timeout?: Duration;
2016
2361
  /**
2017
2362
  * Screenshot defaults for `c.capture()`. `REEVAL_CAPTURE_BROWSER=0` refuses the host
2018
2363
  * binary when `c.capture()` runs; it does not change this object.
2019
2364
  */
2020
2365
  capture?: CaptureOptions;
2366
+ /** Defaults for every compare question: the model that answers it and what each side shows. Merged down the cascade. */
2367
+ question?: QuestionOptions;
2021
2368
  /** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
2022
2369
  concurrency?: number;
2023
2370
  /** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
2024
2371
  record?: RecordMode;
2372
+ /** The history folder, and which attempts keep their artifacts there. */
2025
2373
  history?: HistoryConfig;
2374
+ /** Caps on a run; a profile's `budget` is merged over it. */
2026
2375
  budget?: Budget;
2376
+ /** Named presets of `defaults`, `limit`, `judge`, `budget`, `record` and `tags`. */
2027
2377
  profiles?: Record<string, Profile>;
2028
2378
  /** The profile when neither `--profile` nor `REEVAL_PROFILE` names one: `ci` when `CI` is set. */
2029
2379
  defaultProfile?: {
@@ -2042,6 +2392,7 @@ type EsConfig = {
2042
2392
  keep?: number;
2043
2393
  traceBufferMb?: number;
2044
2394
  };
2395
+ /** `reeval analyze`, compare questions without a model, and `agent`'s notes during a run. */
2045
2396
  analysis?: AnalysisConfig;
2046
2397
  };
2047
2398
  /** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
@@ -2108,6 +2459,7 @@ type CellRole = 'control' | 'variant';
2108
2459
  type Cell = {
2109
2460
  key: string;
2110
2461
  harness: string;
2462
+ /** The environment's own name, also on a variant cell (`with-house-greeting`, with no prefix). */
2111
2463
  environment: string;
2112
2464
  sandbox: string;
2113
2465
  role: CellRole;
@@ -2269,5 +2621,5 @@ type SandboxWarning = {
2269
2621
  message: string;
2270
2622
  };
2271
2623
  //#endregion
2272
- export { HarnessData as $, EffortContrast as $t, DirectoryOptions as A, Turn as At, EsConfig as B, EvalEntry as Bt, CompareCut as C, Secret as Ct, ConfigJudgeOptions as D, SkillSource as Dt, ConfigHelpers as E, SetupStep as Et, EnvVars as F, Wait as Ft, FakeHarnessDescriptor as G, HistoryCell as Gt, EvalOptions as H, FamilyRate as Ht, Environment as I, WebEventType as It, FakeSandboxDescriptor as J, Lift as Jt, FakeJudgeModel as K, HistoryEntry as Kt, EnvironmentHelpers as L, ArtifactRef as Lt, Duration as M, TurnStatus as Mt, EgressPolicy as N, Verdict as Nt, CursorHarnessDescriptor as O, TokenUsage as Ot, EgressRecord as P, Viewport as Pt, GroupOptions as Q, EFFORT_ALPHA as Qt, EnvironmentInput as R, AttemptRecord as Rt, CompareAnswer as S, SandboxWarning as St, CompareQuestion as T, ServeHandle as Tt, ExecResult as U, FamilyRecord as Ut, EvalFn as V, FamilyLift as Vt, FakeAnalysisModel as W, FileRef as Wt, FakeVote as X, StackSummary as Xt, FakeScriptStep as Y, RunRecord as Yt, GooseHarnessDescriptor as Z, CellEffort as Zt, Cell as _, formatSeconds as _n, PinnableKey as _t, BackgroundHandle as a, cellContrast as an, JudgeArtifactRef as at, ClaudeCodeHarnessDescriptor as b, ProfileLimit as bt, Capture as c, contrastEffort as cn, JudgeModel as ct, CaptureOptions as d, effortOf as dn, McpServer as dt, EffortSample as en, HarnessDescriptor as et, CaptureUse as f, effortSample as fn, OpenCodeHarnessDescriptor as ft, CaptureWindow as g, formatRatio as gn, PiHarnessDescriptor as gt, CaptureWait as h, formatCount as hn, PairVote as ht, AttemptContext as i, VERDICT_TEXT as in, HostFile as it, DockerSandboxDescriptor as j, TurnData as jt, Diff as k, TranscriptEvent as kt, CaptureCall as l, effortChangeLabel as ln, JudgeOptions as lt, CaptureUseScript as m, effortVerdict as mn, PairFactor as mt, AnalysisConfig as n, Spread as nn, HarnessOption as nt, BlindPair as o, cellEffort as on, JudgeCallOptions as ot, CaptureUseAction as p, effortSummary as pn, Outcome as pt, FakeNoteInput as q, JudgementRecord as qt, AntigravityHarnessDescriptor as r, TOKENS_UNREPORTED as rn, HistoryConfig as rt, Budget as s, cellSamples as sn, JudgeFailure as st, AgentHarnessDescriptor as t, EffortVerdict as tn, HarnessEdit as tt, CaptureInstruction as u, effortLabel as un, JudgeState as ut, CellRole as v, tokenValues as vn, Profile as vt, CompareOptions as w, SecretTemplate as wt, CodexHarnessDescriptor as x, SandboxDescriptor as xt, ClaudeCodeEffort as y, ProfileDefaults as yt, EnvironmentSpec as z, CellRecord as zt };
2273
- //# sourceMappingURL=types-D8RLfkFP.d.ts.map
2624
+ export { GooseHarnessDescriptor as $, JudgementRecord as $t, DirectoryOptions as A, ServeHandle as At, ErrorKind as B, Wait as Bt, CompareCut as C, effortSample as Cn, ProfileDefaults as Ct, ConfigJudgeOptions as D, formatRatio as Dn, SandboxWarning as Dt, ConfigHelpers as E, formatCount as En, SandboxDescriptor as Et, EnvVars as F, Turn as Ft, FakeAnalysisModel as G, CheckTally as Gt, EvalOptions as H, ArtifactRef as Ht, Environment as I, TurnData as It, FakeNoteInput as J, FamilyRate as Jt, FakeHarnessDescriptor as K, EvalEntry as Kt, EnvironmentHelpers as L, TurnStatus as Lt, Duration as M, SkillSource as Mt, EgressPolicy as N, TokenUsage as Nt, CursorHarnessDescriptor as O, formatSeconds as On, Secret as Ot, EgressRecord as P, TranscriptEvent as Pt, FxHarnessDescriptor as Q, HistoryEntry as Qt, EnvironmentInput as R, Verdict as Rt, CompareAnswer as S, effortOf as Sn, Profile as St, CompareQuestion as T, effortVerdict as Tn, ReevalConfig as Tt, EvalOptionsData as U, AttemptRecord as Ut, EvalFn as V, WebEventType as Vt, ExecResult as W, CellRecord as Wt, FakeScriptStep as X, FileRef as Xt, FakeSandboxDescriptor as Y, FamilyRecord as Yt, FakeVote as Z, HistoryCell as Zt, Cell as _, cellSamples as _n, Outcome as _t, BackgroundHandle as a, EFFORT_ALPHA as an, HarnessOption as at, ClaudeCodeHarnessDescriptor as b, effortChangeLabel as bn, PiHarnessDescriptor as bt, Capture as c, EffortMetric as cn, JudgeArtifactRef as ct, CaptureOptions as d, Spread as dn, JudgeModel as dt, Lift as en, GrokBuildHarnessDescriptor as et, CaptureUse as f, TOKENS_UNREPORTED as fn, JudgeOptions as ft, CaptureWindow as g, cellEffort as gn, OpenCodeHarnessDescriptor as gt, CaptureWait as h, cellContrast as hn, McpServer as ht, AttemptContext as i, DIRECTIONAL_MIN as in, HarnessEdit as it, DockerSandboxDescriptor as j, SetupStep as jt, Diff as k, tokenValues as kn, SecretTemplate as kt, CaptureCall as l, EffortSample as ln, JudgeCallOptions as lt, CaptureUseScript as m, bothUnderATenth as mn, JudgeStateOptions as mt, AnalysisConfig as n, StackSummary as nn, HarnessData as nt, BlindPair as o, EFFORT_MIN_RUNS as on, HistoryConfig as ot, CaptureUseAction as p, VERDICT_TEXT as pn, JudgeState as pt, FakeJudgeModel as q, FamilyLift as qt, AntigravityHarnessDescriptor as r, CellEffort as rn, HarnessDescriptor as rt, Budget as s, EffortContrast as sn, HostFile as st, AgentHarnessDescriptor as t, RunRecord as tn, GroupOptions as tt, CaptureInstruction as u, EffortVerdict as un, JudgeFailure as ut, CellRole as v, contrastEffort as vn, PairFactor as vt, CompareOptions as w, effortSummary as wn, ProfileLimit as wt, CodexHarnessDescriptor as x, effortLabel as xn, PinnableKey as xt, ClaudeCodeEffort as y, directional as yn, PairVote as yt, EnvironmentSpec as z, Viewport as zt };
2625
+ //# sourceMappingURL=types-BYdlwMwo.d.ts.map