reeval 0.14.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (364) hide show
  1. package/README.md +15 -15
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
  5. package/dist/adapter-CRKYPkRt.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
  7. package/dist/agent-BgH4-kFc.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
  9. package/dist/app-files-BMVBr079.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
  11. package/dist/assemble-BVbI6NW-.js.map +1 -0
  12. package/dist/assemble-SiHa4__F.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
  16. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
  18. package/dist/capture-iQ6kYJV0.js.map +1 -0
  19. package/dist/capture-px7Ey95_.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
  21. package/dist/cli-1YQU-66F.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
  28. package/dist/command-8_Cg5v6b.js.map +1 -0
  29. package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
  30. package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
  31. package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
  32. package/dist/command-C-HeGlap.js.map +1 -0
  33. package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
  34. package/dist/command-Cr1zjRvl.js.map +1 -0
  35. package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
  36. package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
  37. package/dist/commands-BvTNZ2-b.js.map +1 -0
  38. package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
  39. package/dist/config-CneBZB84.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-BnXyamuj.js +2 -0
  46. package/dist/db-Cnh3jIv_.js +1530 -0
  47. package/dist/db-Cnh3jIv_.js.map +1 -0
  48. package/dist/debug-CAx3qcaP.js +1 -0
  49. package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
  50. package/dist/debug-CgG3T071.js.map +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
  52. package/dist/docker-BihzJwbV.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
  54. package/dist/docker-Cqq03H72.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
  57. package/dist/doctor-Dl4vMeqt.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
  61. package/dist/environment-Cf5HMjib.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
  63. package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
  64. package/dist/eval-source-CAE4y2Sr.js +146 -0
  65. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
  67. package/dist/evaluate-CUtgyr43.js.map +1 -0
  68. package/dist/evaluate-D35T51AC.d.ts +45 -0
  69. package/dist/files-BEnUJW1e.js +1 -0
  70. package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
  71. package/dist/files-BRViE7m-.js.map +1 -0
  72. package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
  73. package/dist/files-BSqA2sZE.js.map +1 -0
  74. package/dist/files-C3tQtD26.js +1 -0
  75. package/dist/files-CNiJ_fZf.js +37 -0
  76. package/dist/files-CNiJ_fZf.js.map +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/files-Dts7EY3H.js +1 -0
  80. package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
  81. package/dist/gate-BC6SdQZs.js.map +1 -0
  82. package/dist/gateway.d.ts +22 -9
  83. package/dist/gateway.js +30 -8
  84. package/dist/gateway.js.map +1 -1
  85. package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
  86. package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
  87. package/dist/harness-3NVyTSGK.js +392 -0
  88. package/dist/harness-3NVyTSGK.js.map +1 -0
  89. package/dist/harness.d.ts +12 -8
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
  92. package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
  93. package/dist/index.d.ts +99 -11
  94. package/dist/index.js +5 -5
  95. package/dist/init-okoALIah.js +1 -0
  96. package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
  97. package/dist/init-sFBgODhu.js.map +1 -0
  98. package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
  99. package/dist/invocations-C9ejlOYl.js.map +1 -0
  100. package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
  101. package/dist/judge-Bg4v4e_K.js.map +1 -0
  102. package/dist/judge.d.ts +5 -7
  103. package/dist/judge.js +4 -8
  104. package/dist/judge.js.map +1 -1
  105. package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
  106. package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
  107. package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
  108. package/dist/matchers-BKPuaXlj.js.map +1 -0
  109. package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
  110. package/dist/model-2rBZrdZA.js.map +1 -0
  111. package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
  112. package/dist/model-3MJtpho8.js.map +1 -0
  113. package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
  114. package/dist/model-Bz7-j71_.js.map +1 -0
  115. package/dist/model-DkPwypN4.js +1 -0
  116. package/dist/model-bREXVXwT.js +1 -0
  117. package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
  118. package/dist/node-fP-H2uxn.js.map +1 -0
  119. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  120. package/dist/object-BkXGPKWY.js.map +1 -0
  121. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  122. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  123. package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
  124. package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
  125. package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
  126. package/dist/patch-6T9XAipm.js.map +1 -0
  127. package/dist/patch-CquGAGX5.js +76 -0
  128. package/dist/patch-CquGAGX5.js.map +1 -0
  129. package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
  130. package/dist/plan-Dw2BjafJ.js.map +1 -0
  131. package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
  132. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  133. package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
  134. package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
  135. package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
  136. package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
  137. package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
  138. package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
  139. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  140. package/dist/prices-QaS-kjSQ.js.map +1 -0
  141. package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
  142. package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
  143. package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
  144. package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
  145. package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
  146. package/dist/query-Bt6vLJ3m.js +1 -0
  147. package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
  148. package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
  149. package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
  150. package/dist/questions-CkAwePPR.js.map +1 -0
  151. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  152. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  153. package/dist/redact-DVIaahLv.js +658 -0
  154. package/dist/redact-DVIaahLv.js.map +1 -0
  155. package/dist/remarks-Bp6AwPgb.js +1 -0
  156. package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
  157. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  158. package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
  159. package/dist/remarks-IFFe_oty.js.map +1 -0
  160. package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
  161. package/dist/remarks-eBcrIZRu.js.map +1 -0
  162. package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
  163. package/dist/remote-DfJYiP-R.js.map +1 -0
  164. package/dist/resolve-CT4XZIfV.js +1 -0
  165. package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
  166. package/dist/resolve-D_cLa4tW.js.map +1 -0
  167. package/dist/results-DrKJuIQF.js +63 -0
  168. package/dist/results-DrKJuIQF.js.map +1 -0
  169. package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
  170. package/dist/resume-BmIM52MJ.js.map +1 -0
  171. package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
  172. package/dist/runtime-CZVpkr04.js.map +1 -0
  173. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
  174. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  175. package/dist/sandbox.d.ts +1 -1
  176. package/dist/sandbox.js +1 -1
  177. package/dist/server-CKJOnIB0.js +1 -0
  178. package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
  179. package/dist/server-DS8BGhGU.js.map +1 -0
  180. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  181. package/dist/shared-B_lvGqWz.js.map +1 -0
  182. package/dist/source-CgUqpumI.js +1 -0
  183. package/dist/source-DcXv4de-.js +59 -0
  184. package/dist/source-DcXv4de-.js.map +1 -0
  185. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  186. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  187. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  188. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  189. package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
  190. package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
  191. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  192. package/dist/src-BO8AYqJ5.js.map +1 -0
  193. package/dist/src-BTqJbvj6.js +2 -0
  194. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  195. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BkiComGh.js +2 -0
  197. package/dist/src-BvgeZ4tS.js +2 -0
  198. package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
  199. package/dist/src-C0ySiej5.js.map +1 -0
  200. package/dist/src-CgJrz5p0.js +3 -0
  201. package/dist/src-Cp-2Hdib.js +2 -0
  202. package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
  203. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  204. package/dist/src-DSZyg0Uq.js.map +1 -0
  205. package/dist/src-Dk2Na0b-.js +2 -0
  206. package/dist/src-OESRtvOi.js +2 -0
  207. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  208. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  209. package/dist/src-h5uDlk3m.js +1 -0
  210. package/dist/state-C97udoh3.js +1 -0
  211. package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
  212. package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
  213. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  214. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  215. package/dist/suite-ByT1lsdD.js +2 -0
  216. package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
  217. package/dist/suite-DieLXvCT.js.map +1 -0
  218. package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
  219. package/dist/summary-CSzyBVsF.js.map +1 -0
  220. package/dist/tarball-CHcahyNK.js +258 -0
  221. package/dist/tarball-CHcahyNK.js.map +1 -0
  222. package/dist/testing.d.ts +426 -143
  223. package/dist/testing.js +12 -10
  224. package/dist/testing.js.map +1 -1
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  228. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  229. package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
  230. package/dist/transcript-BJrKfgRW.js.map +1 -0
  231. package/dist/types-C5vI3g1C.js.map +1 -1
  232. package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +548 -121
  238. package/dist/view/index.js +9 -9
  239. package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
  240. package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
  241. package/dist/views-Ca-rTixp.js +2 -0
  242. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
  243. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  244. package/dist/workspace-8uHtBSve.js +1 -0
  245. package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
  246. package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +5 -1
  249. package/skills/eval-rubric/SKILL.md +46 -104
  250. package/dist/adapter-BloGVZ_a.js.map +0 -1
  251. package/dist/agent-m-_t-IuW.js.map +0 -1
  252. package/dist/app-files-C-jwhRDt.js.map +0 -1
  253. package/dist/assemble-BDEpelpi.js.map +0 -1
  254. package/dist/assemble-C5bfncHU.js +0 -1
  255. package/dist/capture-CVRwRTnb.js.map +0 -1
  256. package/dist/capture-CherbjzB.js +0 -1
  257. package/dist/cli-C_4gaJ50.js.map +0 -1
  258. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  259. package/dist/client-D5hNjpEx.js.map +0 -1
  260. package/dist/command-BWAuwQNk.js +0 -91
  261. package/dist/command-BWAuwQNk.js.map +0 -1
  262. package/dist/command-BXLUfw4h.js.map +0 -1
  263. package/dist/command-Be6BSE2L.js +0 -335
  264. package/dist/command-Be6BSE2L.js.map +0 -1
  265. package/dist/command-Bxdy62dW.js.map +0 -1
  266. package/dist/commands-jzigbNFb.js.map +0 -1
  267. package/dist/config-BrdHA9bJ.js.map +0 -1
  268. package/dist/cost-D4elx933.js.map +0 -1
  269. package/dist/cost-DmnAod9x.js +0 -1
  270. package/dist/db-CAfksinX.js +0 -2
  271. package/dist/db-DO0p7xgP.js +0 -619
  272. package/dist/db-DO0p7xgP.js.map +0 -1
  273. package/dist/debug-D24NOQH1.js.map +0 -1
  274. package/dist/debug-D7bEv_UV.js +0 -1
  275. package/dist/docker-BnRlTpZA.js.map +0 -1
  276. package/dist/docker-D01slDF3.js.map +0 -1
  277. package/dist/doctor-DQ2p0euG.js.map +0 -1
  278. package/dist/env-BQqj93gu.js.map +0 -1
  279. package/dist/environment-CO0P0soF.js.map +0 -1
  280. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  281. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  282. package/dist/facts-DzOUTreB.js +0 -906
  283. package/dist/facts-DzOUTreB.js.map +0 -1
  284. package/dist/files-B-CJeeEh.js.map +0 -1
  285. package/dist/files-BAt0i00X.js +0 -1
  286. package/dist/files-Bp5j5BqI.js +0 -1
  287. package/dist/files-C2sjmXhk.js +0 -1
  288. package/dist/files-C_gQNVB-.js +0 -65
  289. package/dist/files-C_gQNVB-.js.map +0 -1
  290. package/dist/files-Ca0d6Uq3.js.map +0 -1
  291. package/dist/files-kUCrswI6.js.map +0 -1
  292. package/dist/generate-DZ7vp69F.js +0 -1647
  293. package/dist/generate-DZ7vp69F.js.map +0 -1
  294. package/dist/harness-COVXSwJ6.js +0 -354
  295. package/dist/harness-COVXSwJ6.js.map +0 -1
  296. package/dist/header-BLPP6aCU.js.map +0 -1
  297. package/dist/init-RgCQXZhb.js +0 -1
  298. package/dist/init-SRalM5xF.js.map +0 -1
  299. package/dist/insights-TyAvz9aV.js +0 -68
  300. package/dist/insights-TyAvz9aV.js.map +0 -1
  301. package/dist/invocations-DzyKSJBa.js.map +0 -1
  302. package/dist/judge-yLvjC5A3.js.map +0 -1
  303. package/dist/match-Bz0nSdzN.js.map +0 -1
  304. package/dist/matchers-D4eUqx4j.js.map +0 -1
  305. package/dist/model-1ZLr9Dcv.js.map +0 -1
  306. package/dist/model-B45lDUV1.js.map +0 -1
  307. package/dist/model-BTZq8HIZ.js +0 -1
  308. package/dist/model-CgnOpss4.js.map +0 -1
  309. package/dist/model-Ch8kX87u.js +0 -1
  310. package/dist/node-BSq5qHnQ.js.map +0 -1
  311. package/dist/object-CLf2-tNq.js.map +0 -1
  312. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  313. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  314. package/dist/pairs-CHfZNRT6.js.map +0 -1
  315. package/dist/patch-DWf-I_Q_.js.map +0 -1
  316. package/dist/plan-BIwUzlB8.js.map +0 -1
  317. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  318. package/dist/prices-BcLEXvdY.js.map +0 -1
  319. package/dist/query-Cvbwl804.js +0 -1
  320. package/dist/questions-bAudIc04.js.map +0 -1
  321. package/dist/redact-CRGdfV2s.js +0 -240
  322. package/dist/redact-CRGdfV2s.js.map +0 -1
  323. package/dist/remarks-CXwSnSjR.js +0 -278
  324. package/dist/remarks-CXwSnSjR.js.map +0 -1
  325. package/dist/remarks-CyAu2DiB.js +0 -1
  326. package/dist/remarks-mujJv1Ey.js.map +0 -1
  327. package/dist/remote-CKcEKja4.js.map +0 -1
  328. package/dist/resolve-B5-uqmJ6.js +0 -1
  329. package/dist/resolve-CBoKzQMP.js.map +0 -1
  330. package/dist/results-CMgZEANy.js +0 -78
  331. package/dist/results-CMgZEANy.js.map +0 -1
  332. package/dist/resume-B9SOSnK9.js.map +0 -1
  333. package/dist/rules-VhftPOOT.js.map +0 -1
  334. package/dist/run-CwIEK16q.js +0 -88
  335. package/dist/run-CwIEK16q.js.map +0 -1
  336. package/dist/runtime-BgQRyCLl.js.map +0 -1
  337. package/dist/server-CHRSgKRp.js +0 -1
  338. package/dist/server-TGW4WyAz.js.map +0 -1
  339. package/dist/shared-BDUzStad.js.map +0 -1
  340. package/dist/source-B1WvBIQt.js +0 -40
  341. package/dist/source-B1WvBIQt.js.map +0 -1
  342. package/dist/source-DAzztZwU.js +0 -1
  343. package/dist/src-1jHbFaGW.js +0 -1
  344. package/dist/src-8zoX_qrb.js.map +0 -1
  345. package/dist/src-C_9wgowC.js +0 -2
  346. package/dist/src-ChhtI6BP.js.map +0 -1
  347. package/dist/src-D005Cbqz.js +0 -2
  348. package/dist/src-DIfRbp5a.js +0 -3
  349. package/dist/src-Dg8121Xy.js +0 -2
  350. package/dist/src-ElBqEN-S.js +0 -2
  351. package/dist/src-NSdQ9zI6.js.map +0 -1
  352. package/dist/src-YHGaUrGE.js +0 -2
  353. package/dist/src-mJVxw1Jm.js +0 -2
  354. package/dist/state-vznMh2wl.js +0 -1
  355. package/dist/suite-BsWGHzlG.js +0 -2
  356. package/dist/suite-WgT7m6Vs.js.map +0 -1
  357. package/dist/summary-DIGEVr08.js.map +0 -1
  358. package/dist/text-CrVy5ioL.js.map +0 -1
  359. package/dist/transcript-BTxJpW4R.js.map +0 -1
  360. package/dist/usage-DF3_p5pE.js +0 -1
  361. package/dist/validate-BhwqrVsm.js.map +0 -1
  362. package/dist/views-BW_C5eVH.js +0 -2
  363. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  364. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,3 +1,63 @@
1
+ //#region src/declaration.d.ts
2
+ /** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
3
+ type DeclaredMode = 'run' | 'skip' | 'only';
4
+ /** A `group`: options its evals and groups inherit, and its children in declaration order. */
5
+ type DeclaredGroup = {
6
+ type: 'group';
7
+ /** The group's name, as passed to `group`. */
8
+ name: string;
9
+ mode: DeclaredMode;
10
+ /** The group's own options, before the cascade. */
11
+ options: EvalOptionsData;
12
+ children: DeclaredNode[];
13
+ };
14
+ /** An `evaluate`: one eval, planned into a test per attempt. */
15
+ type DeclaredEval = {
16
+ type: 'eval';
17
+ /** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
18
+ name: string;
19
+ mode: DeclaredMode;
20
+ /** The eval's own options, before the cascade. */
21
+ options: EvalOptionsData;
22
+ /** Names the eval's body to whoever runs it. */
23
+ body: string;
24
+ source: DeclaredSource;
25
+ /** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
26
+ family?: DeclaredFamily;
27
+ };
28
+ /**
29
+ * An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
30
+ * text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
31
+ * whitespace, non-ASCII characters as themselves).
32
+ */
33
+ type DeclaredSource = {
34
+ /** sha256 of the body function's source text. */
35
+ body: string;
36
+ /** sha256 of the case's data; set on each case of an `evaluate.each`. */
37
+ case?: string;
38
+ };
39
+ /** Where a case of a pooled `evaluate.each` sits in its list. */
40
+ type DeclaredFamily = {
41
+ /** The name template passed to `evaluate.each`. */
42
+ name: string;
43
+ /** The case's index, from 0. */
44
+ case: number;
45
+ /** How many cases the list has. */
46
+ cases: number;
47
+ };
48
+ /** `evaluate.todo` or `group.todo`. */
49
+ type DeclaredTodo = {
50
+ type: 'todo';
51
+ /** The name passed to `.todo`. */
52
+ name: string;
53
+ };
54
+ /** One node of the tree: a group, an eval or a todo. */
55
+ type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
56
+ /** An eval file's top-level groups, evals and todos, in declaration order. */
57
+ type DeclarationTree = {
58
+ children: DeclaredNode[];
59
+ };
60
+ //#endregion
1
61
  //#region src/view/effort.d.ts
2
62
  /** Below this two-sided p, an effort difference is an effect. */
3
63
  declare const EFFORT_ALPHA = 0.05;
@@ -127,7 +187,7 @@ type FileRef = {
127
187
  sha256: string;
128
188
  bytes: number;
129
189
  };
130
- /** A generic attempt file from `c.capture()` (and later video). */
190
+ /** A generic attempt file from `c.capture()`. */
131
191
  type ArtifactRef = FileRef & {
132
192
  name: string;
133
193
  mediaType: 'image/png';
@@ -157,15 +217,18 @@ type GitInfo = {
157
217
  */
158
218
  sha: string;
159
219
  branch: string | null;
160
- /** Uncommitted changes, outside `.reeval/` and the history folder, when the run started. */
220
+ /** Uncommitted changes, outside the `.reeval/` directory and the history folder, when the run started. */
161
221
  dirty: boolean;
162
222
  /** sha256 of the uncommitted changes (tracked and untracked, as a binary patch), when dirty. */
163
223
  diffHash: string | null;
164
- /** Those changes as `worktree.patch` in the artifact dir; apply on `sha` to reproduce. */
224
+ /**
225
+ * Those changes are stored as the `worktree.patch` file in the artifact directory. Applying it
226
+ * to `sha` reproduces the workspace.
227
+ */
165
228
  worktreePatch: FileRef | null;
166
229
  /**
167
- * Why a dirty run has no `worktree.patch`: too large, artifacts: 'none', or it looked like it held
168
- * a secret (then where, and which rule, never the value).
230
+ * This explains why a dirty run has no `worktree.patch` file: it was too large, `artifacts` was
231
+ * `'none'`, or it looked like it held a secret (then where, and which rule, never the value).
169
232
  */
170
233
  worktreePatchNote: string | null;
171
234
  commitTime: string;
@@ -192,7 +255,7 @@ type JudgeRetry = {
192
255
  error: string;
193
256
  delayMs: number;
194
257
  };
195
- type JudgementRecord = {
258
+ type JudgmentRecord = {
196
259
  question: string;
197
260
  verdict: 'pass' | 'fail';
198
261
  score: number;
@@ -202,9 +265,15 @@ type JudgementRecord = {
202
265
  resolvedModel?: string;
203
266
  /** The gateway from the config's `gateway` list the call went through; unset without one. */
204
267
  gateway?: string;
268
+ /** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
269
+ upstream?: string;
270
+ /** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
271
+ costUsd?: number;
205
272
  /**
206
- * `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
207
- * verdict can be followed across runs (and an edited question starts a new series).
273
+ * `<index>:<sha256(question) prefix>`: the same question at the same place in the evaluation, so a
274
+ * verdict can be followed across runs (and an edited question starts a new series). `<index>`
275
+ * is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
276
+ * whichever finishes first.
208
277
  */
209
278
  key: string;
210
279
  /** Transient provider errors retried before this verdict, with the backoff before each retry. */
@@ -213,33 +282,44 @@ type JudgementRecord = {
213
282
  artifacts?: JudgeArtifactRef[];
214
283
  };
215
284
  type AttemptFiles = {
216
- /** Each environment setup step's command, exit code, duration and output. */
285
+ /** Each environment setup step's command, exit code, duration, and output. */
217
286
  setupLog?: FileRef;
218
287
  transcript?: FileRef;
219
- /** Unnormalised harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
288
+ /** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
220
289
  rawTranscript?: FileRef;
221
290
  diff?: FileRef;
222
291
  judgeInput?: FileRef;
223
- /** PNGs from `c.capture()`, under `artifacts/<name>.png`. */
292
+ /** PNGs from `c.capture()`, under the `artifacts/<name>.png` path. */
224
293
  artifacts?: ArtifactRef[];
225
294
  };
226
295
  type ContextFile = {
227
296
  path: string;
228
297
  sha256: string;
229
298
  };
299
+ /** An upstream provider behind a gateway, and how many of the attempt's calls it served. */
300
+ type UpstreamCount = {
301
+ provider: string;
302
+ calls: number;
303
+ };
230
304
  type AttemptRecord = {
231
305
  /** `<eval key> # <cell key>#<n>`: unique in the run. */
232
306
  key: string;
233
- /** `<cell key>#<n>`: unique in its eval. */
307
+ /** `<cell key>#<n>`: unique in its evaluation. */
234
308
  id: string;
235
309
  /** The cell's key; see `EvalEntry.cells`. */
236
310
  cell: string;
237
311
  /** Run number within the cell, from 1. */
238
312
  n: number;
313
+ /**
314
+ * In a `runs: { bestOf }` cell: `added` when it ran past the majority's first attempts (the
315
+ * earlier ones split, or an infra error left them short), `skipped` when it didn't run because
316
+ * the earlier ones already had a majority. Unset on the attempts that always run.
317
+ */
318
+ bestOf?: 'added' | 'skipped';
239
319
  /** Start order across the run, from 1; null for attempts that never started. */
240
320
  seq: number | null;
241
321
  startedAt: string | null;
242
- /** The image the container started from, in `stack.images`; null for the fake sandbox. */
322
+ /** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
243
323
  imageId: string | null;
244
324
  /** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
245
325
  env: {
@@ -247,16 +327,17 @@ type AttemptRecord = {
247
327
  secrets: string[];
248
328
  brokered: string[];
249
329
  };
250
- /** Skills, rules and MCP config visible to the agent in its sandbox, by content hash. */
330
+ /** Skills, rules, and MCP config visible to the agent in its sandbox, by content hash. */
251
331
  context: ContextFile[];
252
332
  /** The egress policy and, when the sandbox records it, the hosts contacted and refused; null if it never ran. */
253
333
  egress: EgressRecord | null;
254
334
  /**
255
- * The models the harness's own API calls were served by, as the egress gateway saw them: named
256
- * in the request (e.g. `gemini-3.8-flash` for the id `gemini-3.8-flash-low`, plus any the agent
257
- * uses on the side) or, for Codex, in the response, with its effort
258
- * (`gpt-5.6-luna[effort=medium]`); null when neither names one (Cursor) or nothing
259
- * recorded them.
335
+ * The models served by the harness's API calls, as observed by the egress gateway. Values may
336
+ * come from requests or responses and include reported effort; null when no model was named or
337
+ * recorded. The gateway reads the request URL for Antigravity, Goose, pi, and OpenCode on a
338
+ * `google/` model, so a request for `<model>-low` records `<model>`, and the API response for
339
+ * Codex, Grok Build, and OpenCode on another provider. Null for Cursor and Claude Code, whose
340
+ * API requests name no model.
260
341
  */
261
342
  servedModels: string[] | null;
262
343
  /**
@@ -265,13 +346,24 @@ type AttemptRecord = {
265
346
  * gateway's, so never part of `servedModels`. Absent when there's none.
266
347
  */
267
348
  askedModels?: string[];
349
+ /**
350
+ * The upstream providers a gateway said served the harness's own API calls (OpenRouter's
351
+ * `provider`), each with how many it served, in the order first seen; more than one means the
352
+ * gateway spread or failed the calls over between them. Absent when the gateway names none.
353
+ */
354
+ upstreams?: UpstreamCount[];
355
+ /**
356
+ * What a gateway said the harness's own API calls cost, in US dollars, summed (OpenRouter's
357
+ * `usage.cost`); absent when it names none.
358
+ */
359
+ reportedCostUsd?: number;
268
360
  /**
269
361
  * The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
270
362
  * null when the factory left it unset. Fast is not this.
271
363
  */
272
364
  effort: string | null;
273
365
  /**
274
- * The reasoning effort the harness's API said it ran at, from `servedModels` /
366
+ * The reasoning effort the harness's API said it ran at, from `servedModels` or
275
367
  * each turn's `model` (`medium` from `gpt-5.6-luna[effort=medium]`); null when those
276
368
  * labels name none. Fast (`[fast=true]`) is not this.
277
369
  */
@@ -280,7 +372,8 @@ type AttemptRecord = {
280
372
  error: AttemptError | null;
281
373
  durationMs: number;
282
374
  /**
283
- * `text` is the agent's final message, cut at 8 KB; the whole of it is in `transcript.jsonl`.
375
+ * `text` is the agent's final message, cut at 8 KB; the whole of it is in the `transcript.jsonl`
376
+ * file.
284
377
  * `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
285
378
  */
286
379
  turns: Array<{
@@ -303,8 +396,8 @@ type AttemptRecord = {
303
396
  /** Tokens across the attempt's turns; null when the harness reported none. */
304
397
  usage: TokenUsage | null;
305
398
  /**
306
- * Tokens the capture vision loop used. Infrastructure only: never folded
307
- * into `usage`, effort, compare or lift.
399
+ * Tokens the capture vision loop used. Infrastructure only: never included
400
+ * in `usage`, effort, compare, or lift.
308
401
  */
309
402
  captureUse?: {
310
403
  model: string;
@@ -312,7 +405,7 @@ type AttemptRecord = {
312
405
  usage: TokenUsage | null;
313
406
  };
314
407
  timings: PhaseTimings;
315
- /** Counts from the transcript, so a run's shape is visible without reading it. `webSearches` includes refused searches. */
408
+ /** Counts from the transcript, so the attempt's activity is available without reading the transcript. `webSearches` includes refused searches. */
316
409
  activity: {
317
410
  tools: number;
318
411
  fileEdits: number;
@@ -325,9 +418,34 @@ type AttemptRecord = {
325
418
  /** Longest tool `durationMs` in the transcript; null when none reported a duration. */
326
419
  maxToolMs: number | null;
327
420
  };
328
- judgements: JudgementRecord[];
421
+ judgements: JudgmentRecord[];
329
422
  /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
330
423
  startRetries?: StartRetry[];
424
+ /**
425
+ * Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
426
+ * without `hosts` in their environment, so they hold its real value while the agent works and
427
+ * the agent, as the same user, can read it from them. Found by the secret's variable name, so a
428
+ * copy under another name or in a command's arguments isn't. At most 20, each
429
+ * `{ command, secrets }`: its command line, masked and cut at 200 characters, and those secrets'
430
+ * names. Absent when there were none.
431
+ */
432
+ setupProcesses?: SetupProcess[];
433
+ /**
434
+ * Copies of the attempt's workspace in its directory in `.reeval/runs` on the machine that ran
435
+ * it (never in the history folder), in the order taken: `turn-<n>` after each `c.send` with
436
+ * `snapshot: 'turns'`, then `final` after the checks. Each has its form, path and size, its
437
+ * file count, the directories left out by name, the files whose secret-looking values were
438
+ * masked and the binary ones left out for holding one, or why it wasn't taken. Absent with
439
+ * `snapshot: false`, and when the sandbox was gone first (a timeout or a cancel).
440
+ */
441
+ snapshots?: Snapshot[];
442
+ /**
443
+ * Each `c.serve` call that listened, in call order: `{ command, port, portEnv, detected,
444
+ * address, readyMs }`. `port` is where it listens; `detected` is true when that isn't the
445
+ * `PORT` it was given (`portEnv`), so reeval took the one port that started listening after
446
+ * the command did. Absent when the attempt served nothing.
447
+ */
448
+ serves?: ServeRecord[];
331
449
  /**
332
450
  * Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
333
451
  * `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
@@ -342,9 +460,9 @@ type AttemptRecord = {
342
460
  /** Every try, in order, when it was given `retries`; its log has each try's output. */
343
461
  tries?: ExecTry[];
344
462
  }>;
345
- /** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
463
+ /** Every `expect` and `expect.require` matcher call that settled, in call order; unset with none. */
346
464
  assertions?: AssertionRecord[];
347
- /** Files under a dependency directory (node_modules/, .venv/, …) collapse to one entry with `count`. */
465
+ /** Files under a dependency directory, such as the `node_modules/` directory or `.venv/` directory, collapse to one entry with `count`. */
348
466
  changedFiles: Array<FileChange & {
349
467
  count?: number;
350
468
  }>;
@@ -360,7 +478,7 @@ type AttemptRecord = {
360
478
  type EvalEntry = {
361
479
  /** `file › group › … › name`, as shown. */
362
480
  id: string;
363
- /** What the eval is matched on across runs; currently the same as `id`. */
481
+ /** Used to match an evaluation across runs; equal to `id`. */
364
482
  key: string;
365
483
  /** Directory name inside the artifact directory, from `key`. */
366
484
  slug: string;
@@ -369,18 +487,28 @@ type EvalEntry = {
369
487
  name: string;
370
488
  /** Results are comparable only when fingerprints match. */
371
489
  fingerprint: string;
372
- /** The eval file and every local file it imports, directly or not, by content hash. */
490
+ /**
491
+ * The evaluation file, by content hash: where the evaluation lives. Records from before `source`
492
+ * also list the local files it imported, and were fingerprinted over them.
493
+ */
373
494
  sources: Array<{
374
495
  path: string;
375
496
  sha256: string;
376
497
  }>;
498
+ /**
499
+ * Hashes of what the `sources` fingerprint part covers but `revision`: the eval's body, its
500
+ * case data and its options with its groups'. Unset on records from before it.
501
+ */
502
+ source?: EvalSource;
503
+ /** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
504
+ revision?: Revision;
377
505
  tags: string[];
378
- /** The baseline environment, when the eval uses `compare`. */
506
+ /** The baseline environment, when the evaluation uses `compare`. */
379
507
  compare?: EvalCompareMeta;
380
508
  /**
381
- * Over gated cells: `fail` if any failed, else `infra_error` if any is (a cell is `infra_error`
382
- * only when nothing counted), else `skipped` if all were, else `pass`. An ungated cell's
383
- * failures don't fail the eval.
509
+ * Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
510
+ * `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
511
+ * cell's failures don't fail the evaluation.
384
512
  */
385
513
  outcome: Outcome;
386
514
  /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
@@ -393,9 +521,9 @@ type EvalEntry = {
393
521
  * The eval's pairwise questions (those with `across`), answered after its attempts by votes,
394
522
  * one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
395
523
  * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
396
- * (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
397
- * `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
398
- * masked and screenshots as refs; unset where the eval's files weren't kept.
524
+ * (rates by cut, votes, ties, a failed call, and a sample of reasons). Each answer's `input` is
525
+ * the `<eval-slug>/questions/<n>.json` file: every call its model was sent, per pair and order,
526
+ * with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
399
527
  */
400
528
  answers?: CompareAnswer[];
401
529
  /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
@@ -412,7 +540,7 @@ type FamilyInterval = {
412
540
  hi: number;
413
541
  level: 0.95;
414
542
  /**
415
- * `wilson` / `newcombe-paired`: one counted attempt a case and side, so each case is one
543
+ * `wilson` or `newcombe-paired`: one counted attempt a case and side, so each case is one
416
544
  * Bernoulli trial (or pair). `t-cases`: several, so a t interval over the cases' pass@1, with
417
545
  * attempts within a case never counted as independent.
418
546
  */
@@ -487,17 +615,17 @@ type FamilyLift = {
487
615
  outcome: 'pass' | 'fail';
488
616
  effort?: FamilyEffort;
489
617
  };
490
- /** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an eval. */
618
+ /** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an evaluation. */
491
619
  type FamilyRecord = {
492
620
  /** `file › group › … › template`. */
493
621
  id: string;
494
622
  file: string;
495
623
  group: string[];
496
- /** The name template, e.g. `fixes $id`. */
624
+ /** The name template, for example `fixes $id`. */
497
625
  name: string;
498
626
  /** Cases in the list. */
499
627
  cases: number;
500
- /** Cases with an eval in this run; fewer when a filter or tag left some out. */
628
+ /** Cases with an evaluation in this run; fewer when a filter or tag left some out. */
501
629
  ran: number;
502
630
  /** One per cell. */
503
631
  rates: FamilyRate[];
@@ -535,26 +663,34 @@ type Lift = {
535
663
  };
536
664
  /** Each input of a cell fingerprint, hashed on its own, so a change can be named. */
537
665
  type FingerprintParts = {
538
- /** Over `EvalEntry.sources`. */
666
+ /** Over `EvalEntry.source` and `EvalEntry.revision`; on records from before them, over `EvalEntry.sources`. */
539
667
  sources: string;
540
668
  /** Over the harness descriptor, less concurrency caps. */
541
669
  harness: string;
542
670
  environment: string | null;
543
671
  image: string | null;
544
672
  agentCli: string | null;
545
- /** Over the judge's model, threshold and version, and the config's `state` when it sets one. */
673
+ /** Over the judge's model, threshold and version. */
546
674
  judge: string;
547
675
  /** Over the attempt's context files. */
548
676
  context: string;
549
677
  /** The gateway its model calls went through; unset on `direct`, each provider's own key. */
550
678
  gateway?: string;
679
+ /**
680
+ * The cell's gate (`none`, `passRate 0.5`); unset on the default `all`. Compare reads a change
681
+ * in it as a policy change, not a reason the runs can't be compared.
682
+ */
683
+ gate?: string;
551
684
  };
552
- /** One harness and environment of an eval, and how its attempts went. */
685
+ /** One harness and environment of an evaluation, and how its attempts went. */
553
686
  type CellRecord = Cell & {
554
687
  /** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
555
688
  fingerprint: string;
556
689
  fingerprintParts: FingerprintParts;
557
- /** The environment's digest: the join key into `RunRecord.environments`; null for none. */
690
+ /**
691
+ * The environment's digest; null for none. With `environment`, the key into
692
+ * `RunRecord.environments`: environments with the same contents share a digest.
693
+ */
558
694
  environmentDigest: string | null;
559
695
  harnessKind: string;
560
696
  /**
@@ -569,10 +705,15 @@ type CellRecord = Cell & {
569
705
  * Fast is not this.
570
706
  */
571
707
  effort: string | null;
572
- /** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the eval. */
708
+ /** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the evaluation. */
573
709
  gate: CellGate;
574
710
  /** Environment keys the variant replaced on the control's (or a child on its parent's). */
575
711
  overrides: string[];
712
+ /**
713
+ * Set by `runs: { bestOf }`: the cell stopped once a majority of this many attempts agreed, and
714
+ * gates on that majority. `runs` still counts every planned attempt, the skipped ones included.
715
+ */
716
+ bestOf?: number;
576
717
  runs: number;
577
718
  pass: number;
578
719
  fail: number;
@@ -588,7 +729,7 @@ type CellRecord = Cell & {
588
729
  outcome: Outcome;
589
730
  lift?: Lift;
590
731
  /**
591
- * Each check (assertion label) seen in any attempt of the eval, over this cell's counted
732
+ * Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
592
733
  * attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
593
734
  */
594
735
  checks?: CellCheck[];
@@ -601,10 +742,10 @@ type CheckTally = {
601
742
  notRun: number;
602
743
  };
603
744
  /**
604
- * One check over a cell's counted attempts. `verdict` (variant cells), over the attempts that
605
- * reached the check: `discriminates` when it passes on every one of one side and none of the
606
- * other; `both-pass` or `both-fail` when it does the same on both sides, so it can't tell them
607
- * apart; `mixed` otherwise. No verdict when a side never reached it: not run is not a fail.
745
+ * One check over a cell's counted attempts. `verdict` (variant cells) is `discriminates` when
746
+ * every answered instance passes on one side and fails on the other, `both-pass` when every
747
+ * answered instance passes on both sides, `both-fail` when every answered instance fails on both
748
+ * sides, or `mixed` otherwise. It is absent when either side has no pass or fail.
608
749
  */
609
750
  type CellCheck = CheckTally & {
610
751
  label: string;
@@ -664,7 +805,10 @@ type RunRecord = {
664
805
  harness?: string[];
665
806
  environment?: string;
666
807
  sandbox?: string;
808
+ /** With `bestOf`, the most attempts a cell runs. */
667
809
  runs?: number;
810
+ /** Set by `defaults.runs: { bestOf }`. */
811
+ bestOf?: number;
668
812
  timeout?: string | number;
669
813
  env?: string[];
670
814
  };
@@ -698,7 +842,13 @@ type RunRecord = {
698
842
  config: {
699
843
  sandbox: string;
700
844
  environment: string | null;
845
+ /**
846
+ * The config's `runs` (with `bestOf`, the most a cell runs); 1 when it sets none, though a
847
+ * compare's cells then run 4 each.
848
+ */
701
849
  runs: number;
850
+ /** Set by the config's `runs: { bestOf }`. */
851
+ bestOf?: number;
702
852
  judgeModel: string | null;
703
853
  threshold: number;
704
854
  timeoutMs: number;
@@ -717,28 +867,26 @@ type RunRecord = {
717
867
  /** Each pooled `evaluate.each` that ran; unset when none did. */
718
868
  families?: FamilyRecord[];
719
869
  /**
720
- * Errors outside any attempt, redacted: an eval file that didn't load, a rejection or throw no
721
- * attempt awaited, a worker that died. Any one fails the run. Unset when there were none.
870
+ * These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
871
+ * or throw no attempt awaited, or a worker that died. Any one fails the run. Unset when there
872
+ * were none.
722
873
  */
723
874
  errors?: string[];
724
875
  };
725
- /** What ran and graded the agents: the machine, images, harnesses and judge. */
876
+ /** These fields describe what ran and graded the agents: the machine, images, harnesses, and judge. */
726
877
  type Stack = {
727
878
  /** The reeval version that wrote the run. */
728
879
  reeval: string;
729
- /** `process.versions.node`: under Bun or Deno, the Node version they're compatible with. */
880
+ /** `process.versions.node`, or the compatible Node version reported by another runtime. */
730
881
  node: string;
731
- /**
732
- * The runtime reeval ran on, `node`, `bun`, `deno` or another, and its own version. Absent in
733
- * runs recorded before it, which ran on Node.
734
- */
882
+ /** The runtime reeval ran on, `node`, `bun`, `deno`, or another, and its version. */
735
883
  runtime?: {
736
884
  name: string;
737
885
  version: string;
738
886
  };
739
887
  /**
740
- * The language the evals are written in, `typescript` for `*.eval.ts`, and the version of its
741
- * toolchain when the runner reports one. Absent in runs recorded before it.
888
+ * The evaluation language, `typescript` for `*.eval.ts` files, and the version of its toolchain
889
+ * when the runner reports one.
742
890
  */
743
891
  language?: {
744
892
  name: string;
@@ -746,16 +894,17 @@ type Stack = {
746
894
  };
747
895
  os: string;
748
896
  arch: string;
749
- /** Every image an attempt started from. `id` is content-addressed; `repoDigests` only exist once pushed. */
897
+ /** Every image an attempt started from. `id` is content-addressed; `repoDigests` exist only after the image is pushed. */
750
898
  images: Array<AttemptImage & {
751
899
  repoDigests: string[];
752
900
  }>;
753
901
  /** Exact versions of the packages that run and grade the agent. */
754
902
  packages: Record<string, string>;
755
903
  /**
756
- * Each harness: its configured `model` and requested `effort`, `resolvedModels` (what it
757
- * reported serving, with the variant it resolved), and `resolvedEffort` (reasoning-effort
758
- * suffixes those labels named). Fast (`[fast=true]`) is not `resolvedEffort`.
904
+ * For each harness, the record includes its configured `model` and requested `effort`,
905
+ * `resolvedModels` (what it reported serving, with the variant it resolved), and
906
+ * `resolvedEffort` (reasoning-effort suffixes those labels named). Fast (`[fast=true]`) is not
907
+ * `resolvedEffort`.
759
908
  */
760
909
  harnesses: Array<{
761
910
  id: string;
@@ -791,12 +940,12 @@ type Stack = {
791
940
  threshold: number;
792
941
  /** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
793
942
  version: string;
794
- /** What the judge sees by default, every key resolved; absent when the config and profile leave `state` at its defaults. */
943
+ /** The config's default judge inputs, every key resolved. Unset when the config names none, which reeval requires: each `c.judge()` call names its own. */
795
944
  state?: ResolvedJudgeStateOptions;
796
945
  };
797
946
  /** Union of every attempt's context files. */
798
947
  context: ContextFile[];
799
- /** What eval fingerprints hash; fingerprints from different scopes aren't comparable. */
948
+ /** What evaluation fingerprints hash; fingerprints from different scopes aren't comparable. */
800
949
  fingerprintScope: string[];
801
950
  };
802
951
  /** The parts of a run's stack that `reeval compare` checks for drift. */
@@ -824,8 +973,15 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
824
973
  };
825
974
  /** Median tool calls, agent time and tokens over the counted attempts; null when none counted. */
826
975
  effort: CellEffort | null;
827
- /** Of `fail`, the attempts the run's interrupt cancelled (`error.kind: 'aborted'`); unset when none. */
828
- cancelled?: number;
976
+ /** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
977
+ canceled?: number;
978
+ };
979
+ /** A family's pooled lift as the history listing keeps it: enough to compare its gate. */
980
+ type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
981
+ lo: number;
982
+ hi: number;
983
+ /** Unset when the compare has no `minLift`, so the lift can't fail. */
984
+ minLift?: number;
829
985
  };
830
986
  /**
831
987
  * One line of the derived history listing: cached locally by the CLI, and the run header in the
@@ -860,636 +1016,24 @@ type HistoryEntry = {
860
1016
  slug: string;
861
1017
  fingerprint: string;
862
1018
  sources: EvalEntry['sources'];
1019
+ source?: EvalEntry['source'];
1020
+ revision?: EvalEntry['revision'];
863
1021
  outcome: Outcome;
864
1022
  passRate: number | null;
865
1023
  cells: HistoryCell[];
866
1024
  }>;
867
1025
  /** The run's pooled families; unset when none ran. */
868
- families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'>>;
1026
+ families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'> & {
1027
+ /** The eval ids of its cases in this run. */
1028
+ caseIds: string[];
1029
+ lifts: FamilyLiftSummary[];
1030
+ }>;
869
1031
  /** The run's errors outside any attempt; unset when there were none. */
870
1032
  errors?: string[];
871
1033
  };
872
1034
  //#endregion
873
- //#region src/view/suite.d.ts
874
- /**
875
- * A variant against its baseline: `better` or `worse` when the stored 95% interval on the
876
- * difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
877
- * including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
878
- */
879
- type Effect = 'better' | 'worse' | 'same' | 'unclear';
880
- /**
881
- * What an eval's cells compare: `change` when it has variants (or several environments),
882
- * `harness` when it has several harnesses, `both`, or `none`.
883
- */
884
- type Across = 'none' | 'change' | 'harness' | 'both';
885
- /**
886
- * One finished run's facts. Every node has a `path`, a stable id to cite it by, and `paths`
887
- * lists them all in the order the nodes appear.
888
- *
889
- * A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
890
- *
891
- * - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
892
- * (`evals/a.eval.ts › fixes $id`);
893
- * - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
894
- * - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
895
- * - `<cell>/checks/<label>`: a check, by its assertion label;
896
- * - `<cell>/effort`: the cell's effort, only when it was tested;
897
- * - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
898
- *
899
- * A case segment never has a raw `+` and a cell segment always has two, so the two can't be
900
- * confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
901
- * fact in every run that has it.
902
- */
903
- type SuiteResult = {
904
- runId: string;
905
- status: RunRecord['status'];
906
- /**
907
- * The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
908
- * an attempt, and `skipped` on an interrupted run.
909
- */
910
- outcome: Outcome;
911
- /**
912
- * Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
913
- * awaited, a worker that died. Any one fails the run.
914
- */
915
- errors: string[];
916
- /**
917
- * Outcomes counted per block: each family case is one, each eval with no cases is one, and a
918
- * family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
919
- * `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
920
- */
921
- tally: SuiteTally;
922
- /** Sorted by path. */
923
- files: ResultFile[];
924
- /** Every node's `path`, in the order they appear. */
925
- paths: string[];
926
- };
927
- type SuiteTally = {
928
- errors: number;
929
- pass: number;
930
- notGated: number;
931
- fail: number;
932
- infraError: number;
933
- cancelled: number;
934
- skipped: number;
935
- };
936
- /**
937
- * An eval's or a family case's outcome: the record's, except `cancelled` where its only failed
938
- * gated attempts are ones the run's interrupt cancelled, since the agent didn't fail those.
939
- */
940
- type ResultOutcome = Outcome | 'cancelled';
941
- type ResultFile = {
942
- path: string;
943
- evals: EvalResult[];
944
- };
945
- type EvalResult = {
946
- path: string;
947
- /** The eval's id, or the family's. */
948
- id: string;
949
- group: string[];
950
- /** The eval's name; on a family, its template. */
951
- name: string;
952
- /** A family's name template (`fixes $id`); null on a single eval. */
953
- template: string | null;
954
- /** Source order in the file. */
955
- order: number;
956
- across: Across;
957
- /**
958
- * A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
959
- * unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
960
- */
961
- family: boolean;
962
- /** A placeholder (`evaluate.todo`) that never ran. */
963
- todo: boolean;
964
- /** On a family: `fail` when a case failed or its pooled gate did. */
965
- outcome: ResultOutcome;
966
- cells: CellResult[] | null;
967
- cases: CaseResult[] | null;
968
- /** A family's variants against their baseline, pooled over the cases; null on a single eval. */
969
- pooled: PooledArm[] | null;
970
- /**
971
- * Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
972
- * each question asked of any case, in first-asked order, summed over the cases that answered it.
973
- */
974
- questions: Array<QuestionResult | AttemptQuestionResult>;
975
- };
976
- /** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
977
- type QuestionResult = {
978
- kind: 'pairs';
979
- question: string;
980
- /** Pairs voted, each counted once per order it was asked in. */
981
- votes: number;
982
- rates: Array<{
983
- cut: 'within' | 'across';
984
- name: string;
985
- wins: number;
986
- n: number;
987
- }>;
988
- /** Why some or all of its votes are missing: the model call failed. */
989
- error: string | null;
990
- };
991
- /** A question asked of each attempt on its own: what each one answered. Never gates. */
992
- type AttemptQuestionResult = {
993
- kind: 'attempts';
994
- question: string;
995
- /** Attempts it was asked of, failed calls included. */
996
- asked: number;
997
- /** How many attempts picked each choice, in the question's order; null on a free-text question. */
998
- choices: Array<{
999
- choice: string;
1000
- n: number;
1001
- }> | null;
1002
- /** Each attempt's answer, in attempt order; failed calls left out. */
1003
- answers: Array<{
1004
- attempt: string;
1005
- choice: string | null;
1006
- text: string;
1007
- }>;
1008
- /** Calls that failed, and the first one's error. */
1009
- failed: number;
1010
- error: string | null;
1011
- };
1012
- type CaseResult = {
1013
- path: string;
1014
- /** The case's own eval id. */
1015
- id: string;
1016
- /** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
1017
- label: string;
1018
- name: string;
1019
- /** Its place in the case list. */
1020
- order: number;
1021
- outcome: ResultOutcome;
1022
- cells: CellResult[];
1023
- };
1024
- type CellResult = {
1025
- path: string;
1026
- key: string;
1027
- harness: string;
1028
- /** `-` when no environment is set. */
1029
- environment: string;
1030
- sandbox: string;
1031
- /** The variant's name; null on a control. */
1032
- variant: string | null;
1033
- /** The control of an eval that has variants: the arm the variants are measured against. */
1034
- baseline: boolean;
1035
- /** On a variant, its baseline cell's key; else null. */
1036
- vs: string | null;
1037
- /** The cell's place in the eval's cells. */
1038
- order: number;
1039
- /** Its failures fail the eval (its gate isn't `'none'`). */
1040
- gated: boolean;
1041
- outcome: Outcome;
1042
- /** Every attempt: `passed + failed + infraError + skipped`. */
1043
- attempts: number;
1044
- passed: number;
1045
- failed: number;
1046
- /** Infrastructure errors, which are neither passes nor fails. */
1047
- infraError: number;
1048
- skipped: number;
1049
- fingerprint: string;
1050
- /** The skill sources and files the cell's environment placed, as written. */
1051
- context: string[];
1052
- /**
1053
- * Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
1054
- * attempt (every one an infra error or skipped) lists each as `not run`.
1055
- */
1056
- checks: CheckResult[];
1057
- /** On a variant, against its baseline; null on a control. */
1058
- effect: Effect | null;
1059
- /** The stored 95% interval on the variant's difference from its baseline. Not for display. */
1060
- interval: Interval | null;
1061
- /** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
1062
- minLift: number | null;
1063
- /** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
1064
- effort: EffortCounters | null;
1065
- /** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
1066
- effortDiffers: boolean;
1067
- /** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
1068
- errors: CellError[];
1069
- /** Web searches and fetches per counted attempt; null where the harness can't report them. */
1070
- searches: Array<number | null>;
1071
- fetches: Array<number | null>;
1072
- };
1073
- /** A cell's attempts the run's interrupt cancelled: recorded as fails, but not the agent's. */
1074
- declare const cancelledOf: (cell: CellResult) => number;
1075
- /** A cell's counted tries: its passes and fails, less the ones the interrupt cancelled. */
1076
- declare const countedOf: (cell: CellResult) => number;
1077
- type CheckResult = {
1078
- path: string;
1079
- label: string;
1080
- /** First-seen order across the eval's attempts. */
1081
- order: number;
1082
- outcome: CheckOutcome;
1083
- /**
1084
- * On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
1085
- * null on a control or without one.
1086
- */
1087
- effect: Effect | null;
1088
- /** Set when the check is a judge verdict. */
1089
- judge: JudgeResult | null;
1090
- /**
1091
- * A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
1092
- * it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
1093
- * when none did.
1094
- */
1095
- retried?: string;
1096
- };
1097
- /**
1098
- * `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
1099
- * none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
1100
- * reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
1101
- * first failure's message.
1102
- */
1103
- type CheckOutcome = {
1104
- status: 'not run';
1105
- } | {
1106
- status: 'ran';
1107
- passed: number;
1108
- failed: number;
1109
- errored: number;
1110
- notRun: number;
1111
- assertion: string | null;
1112
- };
1113
- /**
1114
- * One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
1115
- * an error), its message's first line, and how often.
1116
- */
1117
- type CellError = {
1118
- kind: ErrorKind | 'turn';
1119
- message: string;
1120
- attempts: number;
1121
- };
1122
- type JudgeResult = {
1123
- question: string;
1124
- threshold: number;
1125
- /** One per counted attempt that judged it. */
1126
- scores: number[];
1127
- };
1128
- type Interval = {
1129
- low: number;
1130
- high: number;
1131
- };
1132
- /** A null counter is one the harness didn't record; a recorded zero stays zero. */
1133
- type EffortCounters = {
1134
- tools: number | null;
1135
- agentMs: number | null;
1136
- tokens: number | null;
1137
- };
1138
- type PooledArm = {
1139
- path: string;
1140
- /** The variant cell's key, the same in every case. */
1141
- key: string;
1142
- vs: string;
1143
- effect: Effect;
1144
- /** Cases with counted attempts on both sides. */
1145
- pairs: number;
1146
- interval: Interval;
1147
- /** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
1148
- minLift: number | null;
1149
- outcome: 'pass' | 'fail';
1150
- };
1151
- type Side = {
1152
- pass: number;
1153
- counted: number;
1154
- };
1155
- /**
1156
- * The effect of `variant` against `control`, from the interval on their difference. A side
1157
- * with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
1158
- */
1159
- declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
1160
- /** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
1161
- declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
1162
- /** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
1163
- declare const factSegment: (s: string) => string;
1164
- declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
1165
- /**
1166
- * An error's one line: its first line with any letter or digit in it, joined with the next one
1167
- * when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
1168
- */
1169
- declare function messageLine(text: string): string;
1170
- /** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
1171
- declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
1172
- /**
1173
- * A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
1174
- * gating it, so it reads `not gated`, never `pass`.
1175
- */
1176
- declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
1177
- /** The facts of a run, from its record. Pure: the same record always gives the same result. */
1178
- declare function suiteResult(record: RunRecord): SuiteResult;
1179
- //#endregion
1180
- //#region src/view/insights.d.ts
1181
- /** `<history>/<run>/insights.json`, beside the record. */
1182
- declare const INSIGHTS_FILE = "insights.json";
1183
- /** `RunInsights.model` when no analysis model wrote the file: it holds deterministic remarks only. */
1184
- declare const NO_MODEL = "none";
1185
- /** Whether an analysis model wrote these insights, rather than reeval alone for its remarks. */
1186
- declare const fromModel: (insights: RunInsights) => boolean;
1187
- /** What one attempt tests. Only an analysis model can tell; without one every eval is `other`. */
1188
- type Job = 'choose' | 'finish' | 'other';
1189
- /**
1190
- * A run's insights, `<history>/<run>/insights.json`. Written create-only; the record is untouched.
1191
- * They're for the facts they were written from: a `factsHash` that isn't the run's marks them stale.
1192
- */
1193
- type RunInsights = {
1194
- kind: 'reeval-insights';
1195
- schemaVersion: 1;
1196
- runId: string;
1197
- /**
1198
- * The analysis model, as configured (`google/gemini-3.8-flash`, or `fake`); `NO_MODEL` when
1199
- * reeval wrote the file without one, to hold the run's deterministic remarks only.
1200
- */
1201
- model: string;
1202
- /** The prompts' version: insights from another version were asked differently. */
1203
- promptVersion: string;
1204
- /** sha256 of the run's `SuiteResult` as canonical JSON: the facts the model saw. */
1205
- factsHash: string;
1206
- createdAt: string;
1207
- /**
1208
- * Over every call; `outputTokens` includes `reasoningTokens`, the model's thinking, when it
1209
- * reports them. `costUsd` is null when reeval doesn't know the model's price.
1210
- */
1211
- usage: {
1212
- inputTokens: number;
1213
- outputTokens: number;
1214
- reasoningTokens?: number;
1215
- costUsd: number | null;
1216
- };
1217
- /**
1218
- * `analysis.maxTokens`, and the evals (`SuiteResult` paths) it ran out before: they print
1219
- * without insights. Absent when no budget applied.
1220
- */
1221
- budget?: {
1222
- maxTokens: number;
1223
- skipped: string[];
1224
- };
1225
- /**
1226
- * Evals whose calls failed twice (an error, a reply cut off at the output cap, or a call past its
1227
- * timeout), with the last error: they print without insights. Absent when none failed.
1228
- */
1229
- failed?: Array<{
1230
- evalId: string;
1231
- error: string;
1232
- }>;
1233
- /**
1234
- * Set when the run was stopped (Ctrl-C, or SIGTERM) before every eval's insights had finished:
1235
- * the evals that hadn't, which print without insights. Absent otherwise, even when the stop came
1236
- * during the headline call, which leaves `headline` null.
1237
- */
1238
- stopped?: {
1239
- signal: 'SIGINT' | 'SIGTERM';
1240
- skipped: string[];
1241
- };
1242
- /** One sentence for the whole run, citing facts; null when the model's was dropped. */
1243
- headline: Sentence | null;
1244
- /** One per eval (a family is one), in the order the output prints them. */
1245
- evals: EvalInsight[];
1246
- /**
1247
- * Remarks about the evals' own quality, ranked: every one this run raised, including those an
1248
- * earlier run printed first (`firstSeenRun`), which don't print again.
1249
- */
1250
- remarks: Remark[];
1251
- /**
1252
- * What the checks dropped, and how many sentences were asked for again, so the output can say
1253
- * so. `drops` is each dropped sentence: its slot (`<eval> › answer`, `› row <path>`, `headline`),
1254
- * its text and why.
1255
- */
1256
- validation: {
1257
- dropped: number;
1258
- regenerated: number;
1259
- drops?: Array<{
1260
- slot: string;
1261
- text: string;
1262
- reason: string;
1263
- }>;
1264
- };
1265
- /** Set when insights failed; the facts still print. */
1266
- error?: string;
1267
- };
1268
- type EvalInsight = {
1269
- /** The eval's `SuiteResult` path; for a family, the family's. */
1270
- evalId: string;
1271
- intent: {
1272
- /** What one attempt is testing. Can't be read from the cell shape. */
1273
- job: Job;
1274
- /** What the cells compare. Deterministic, copied from the facts. */
1275
- across: Across;
1276
- /** The eval's question in plain words: "Do agents pick Genkit for a production AI feature?" */
1277
- question: string;
1278
- /** The tool a choose eval asks about, and the change a measure eval makes. */
1279
- subject: {
1280
- tool?: string;
1281
- change?: string;
1282
- };
1283
- /** How the eval name relates to what the eval checks. */
1284
- name: 'states-intent' | 'no-intent' | 'contradicts';
1285
- /**
1286
- * sha256 of the eval's id, sources, first attempt's prompts, judge questions, check labels,
1287
- * the model and the prompt version: the classification's cache key.
1288
- */
1289
- key: string;
1290
- };
1291
- /** The answer to `question`, at most two sentences; null when it was dropped. */
1292
- answer: Sentence | null;
1293
- /** Up to three supporting points: what moved, what broke, what the agent did. */
1294
- findings: Sentence[];
1295
- /**
1296
- * At most one sentence per row (a case, an arm or a harness), keyed by the row's fact path.
1297
- * Only rows the eval's layout opens a slot on are kept.
1298
- */
1299
- rows?: Array<Sentence & {
1300
- row: string;
1301
- }>;
1302
- /**
1303
- * Per attempt (`<cell path>#<n>`), the tools the reply recommended or used, each found
1304
- * verbatim in that attempt's final message. Only on `choose` evals.
1305
- */
1306
- namedTools?: Array<{
1307
- attempt: string;
1308
- tools: string[];
1309
- }>;
1310
- };
1311
- /**
1312
- * A plain sentence citing the facts it states: `SuiteResult` paths
1313
- * (`evals%2Fbilling.eval.ts › bills in pence/fake+-+fake/checks/amount`) or a transcript line
1314
- * (`transcript:<cell path>#<n>#L<line>`). A sentence citing nothing is dropped.
1315
- */
1316
- type Sentence = {
1317
- text: string;
1318
- cites: string[];
1319
- };
1320
- /**
1321
- * Each remark kind's origin and severity: an analysis model raises the `model` kinds, through the
1322
- * gates (a known kind, a quote verbatim on the cited line); reeval's rules raise the others. The
1323
- * severity is the kind's, never the model's.
1324
- */
1325
- declare const REMARK_KINDS: Record<Remark['kind'], {
1326
- origin: Remark['origin'];
1327
- severity: 'concern' | 'suggestion';
1328
- }>;
1329
- /** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
1330
- declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "proxy-only-egress" | "slow-mcp-start" | "too-few-tries")[];
1331
- /** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
1332
- type Remark = {
1333
- /**
1334
- * sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
1335
- * per kind (and server) per eval version.
1336
- */
1337
- id: string;
1338
- evalId: string;
1339
- /** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
1340
- kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start' | 'proxy-only-egress';
1341
- /** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
1342
- severity: 'concern' | 'suggestion';
1343
- /** One plain sentence: what's wrong. */
1344
- text: string;
1345
- /**
1346
- * Where the problem is: `source:<file>#L<n>` with `quote`, found verbatim on that line of the
1347
- * file at the run's source hash. Every model remark has one.
1348
- */
1349
- source?: {
1350
- at: string;
1351
- quote: string;
1352
- };
1353
- /** Facts from this run (or history facts) showing it mattered now; a model's, as it cited them. */
1354
- evidence: string[];
1355
- /**
1356
- * An edit at a source location (`source:<file>#L<n>`) in the eval. A model remark's `edit` is
1357
- * empty when the model gave none, and its `at` is the quoted line when it named no line.
1358
- */
1359
- fix: {
1360
- at: string;
1361
- edit: string;
1362
- };
1363
- origin: 'model' | 'deterministic';
1364
- /**
1365
- * The first run that printed this `id`, or this run when none has: it prints only in that run,
1366
- * except `slow-mcp-start`, which is about the environment, not the eval's source, and prints in
1367
- * every run it happens.
1368
- */
1369
- firstSeenRun: string;
1370
- /**
1371
- * Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
1372
- * when the cap or a run without `-v` left it out, so a later run prints it. Missing: printed in
1373
- * `firstSeenRun` only.
1374
- */
1375
- printed?: boolean;
1376
- /** Set on later runs once the cited span changed: the remark was acted on. */
1377
- actedOn?: {
1378
- run: string;
1379
- };
1380
- };
1381
- //#endregion
1382
- //#region src/insights/input.d.ts
1383
- /** One fact the model may cite, by its path. */
1384
- type InsightFact = {
1385
- path: string;
1386
- kind: string;
1387
- } & Record<string, unknown>;
1388
- type InsightAttempt = {
1389
- /** `<cell path>#<n>`. */
1390
- id: string;
1391
- cell: string;
1392
- outcome: string;
1393
- /** The agent's final message, clipped. */
1394
- lastText: string;
1395
- /** A failed attempt's transcript lines, `L<n> …`, each citable as `transcript:<id>#L<n>`. */
1396
- lines: string[];
1397
- };
1398
- /** Where an insight sentence can go, as the eval's layout opens them. */
1399
- type InsightSlots = {
1400
- /** Row paths that take one sentence each. */
1401
- rows: string[];
1402
- findings: boolean;
1403
- namedTools: boolean;
1404
- };
1405
- /** One eval as the classifier and the insight model see it. */
1406
- type InsightInput = {
1407
- call: 'classify' | 'insights';
1408
- eval: {
1409
- path: string;
1410
- /** The eval's name, or a family's template: the author's statement of what it asks. */
1411
- name: string;
1412
- group: string[];
1413
- file: string;
1414
- across: Across;
1415
- family: boolean;
1416
- verdict: string;
1417
- };
1418
- /** The first attempt's prompts, in order. */
1419
- prompts: string[];
1420
- judgeQuestions: string[];
1421
- checks: string[];
1422
- /** Set on the insights call: the classification. */
1423
- intent?: {
1424
- job: Job;
1425
- question: string;
1426
- subject: {
1427
- tool?: string;
1428
- change?: string;
1429
- };
1430
- };
1431
- /** Set on the insights call. */
1432
- slots?: InsightSlots;
1433
- facts: InsightFact[];
1434
- attempts: InsightAttempt[];
1435
- /**
1436
- * Set on the insights call: the eval's declaration in its file, each line `L<n> …` as the file
1437
- * numbers it, which a remark's `source` and `fix` cite.
1438
- */
1439
- source?: {
1440
- file: string;
1441
- lines: string[];
1442
- };
1443
- /** What was left out to fit the eval's size cap (`trim.ts`), when anything was. */
1444
- trimmed?: string[];
1445
- /** On a regeneration: why the earlier sentences were refused. */
1446
- retry?: string[];
1447
- };
1448
- /** The headline call's input: each eval's verdict and answer, and the run's tally with its total. */
1449
- type HeadlineInput = {
1450
- call: 'headline';
1451
- /**
1452
- * Counts results, not evals: each case of a family counts one, as the tally line does, and
1453
- * `total` is their sum, so no count is the model's arithmetic; `errors` (outside any eval) aren't
1454
- * results and stay out of it. The eval count is `evals.length`.
1455
- */
1456
- tally: SuiteResult['tally'] & {
1457
- total: number;
1458
- unit: 'results';
1459
- };
1460
- evals: Array<{
1461
- path: string;
1462
- name: string;
1463
- verdict: string;
1464
- answer: string | null;
1465
- }>;
1466
- retry?: string[];
1467
- };
1468
- /** A fact as the model sees it: its short id (`F1`), which is what it cites. */
1469
- type ModelFact = InsightFact & {
1470
- id: string;
1471
- };
1472
- /** An attempt as the model sees it: `A1`, its cell's fact id, and its lines citable as `A1#L<n>`. */
1473
- type ModelAttempt = Omit<InsightAttempt, 'id' | 'cell'> & {
1474
- id: string;
1475
- cell: string;
1476
- };
1477
- /** One eval's call as the model sees it: facts, attempts and row slots by short id. */
1478
- type ModelInsightInput = Omit<InsightInput, 'facts' | 'attempts' | 'slots'> & {
1479
- facts: ModelFact[];
1480
- attempts: ModelAttempt[];
1481
- slots?: Omit<InsightSlots, 'rows'> & {
1482
- rows: string[];
1483
- };
1484
- };
1485
- /** The headline call as the model sees it: each eval by its short id. */
1486
- type ModelHeadlineInput = Omit<HeadlineInput, 'evals'> & {
1487
- evals: Array<HeadlineInput['evals'][number] & {
1488
- id: string;
1489
- }>;
1490
- };
1491
- //#endregion
1492
1035
  //#region src/types.d.ts
1036
+ /** More than zero and at most 596h: `90s`, `15m`, `1h`, or a number of milliseconds. */
1493
1037
  type Duration = number | `${number}${'ms' | 's' | 'm' | 'h'}`;
1494
1038
  /** When an event started: wall clock, and ms since its turn started. */
1495
1039
  type EventTime = {
@@ -1500,13 +1044,17 @@ type FileEditEvent = {
1500
1044
  type: 'file-edit';
1501
1045
  toolCallId: string;
1502
1046
  path: string;
1503
- kind: 'create' | 'modify' | 'delete';
1047
+ /**
1048
+ * `write` is a whole-file write whose result doesn't say whether the file existed before
1049
+ * (Antigravity's, fx's, Goose's, Grok Build's and pi's write tools, for example).
1050
+ */
1051
+ kind: 'create' | 'modify' | 'write' | 'delete';
1504
1052
  /** Line counts, or null when the harness didn't send the content (e.g. deletes). */
1505
1053
  added: number | null;
1506
1054
  removed: number | null;
1507
1055
  };
1508
1056
  /**
1509
- * Normalised agent events. Each tool call is one event, merged with its result; `durationMs`
1057
+ * Normalized agent events. Each tool call is one event, merged with its result; `durationMs`
1510
1058
  * is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
1511
1059
  * when the harness never reported the end. Long shell or edit calls set `stalled: true`.
1512
1060
  */
@@ -1599,16 +1147,21 @@ type Wait = CaptureWait | {
1599
1147
  type CaptureInstruction = string | {
1600
1148
  wait: Wait;
1601
1149
  };
1150
+ /** The keys a capture's driver presses. In a focused `<select>`, the arrows change the choice. */
1151
+ type CaptureKey = 'Tab' | 'Enter' | 'Escape' | 'Backspace' | 'Space' | 'ArrowUp' | 'ArrowDown' | 'ArrowLeft' | 'ArrowRight';
1602
1152
  type CaptureUseAction = {
1603
1153
  type: 'click';
1604
1154
  x: number;
1605
1155
  y: number;
1606
- } | {
1156
+ } |
1157
+ /** `replace` selects what the focused field holds first, so the text replaces it. */
1158
+ {
1607
1159
  type: 'type';
1608
1160
  text: string;
1161
+ replace?: boolean;
1609
1162
  } | {
1610
1163
  type: 'key';
1611
- key: 'Tab' | 'Enter';
1164
+ key: CaptureKey;
1612
1165
  } | {
1613
1166
  type: 'scroll';
1614
1167
  x: number;
@@ -1643,7 +1196,7 @@ type CaptureUse = {
1643
1196
  type CaptureOptions = {
1644
1197
  /** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
1645
1198
  viewport?: Viewport;
1646
- /** How long to poll before failing. Default `'60s'`. */
1199
+ /** How long to poll before failing; a 502, 503 or 504 is polled past until then. Default `'60s'`. */
1647
1200
  timeout?: Duration;
1648
1201
  /** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
1649
1202
  browser?: boolean;
@@ -1657,7 +1210,7 @@ type CaptureOptions = {
1657
1210
  type BackgroundHandle = {
1658
1211
  kill(): Promise<void>;
1659
1212
  };
1660
- /** `c.serve`: a background process plus the port it was given or allocated. */
1213
+ /** `c.serve`: a background process plus the port it listens on. */
1661
1214
  type ServeHandle = BackgroundHandle & {
1662
1215
  port: number;
1663
1216
  };
@@ -1683,6 +1236,7 @@ type Capture = {
1683
1236
  * default `name` is derived from the path. `wait` and `instructions` are call-only.
1684
1237
  */
1685
1238
  type CaptureCall = {
1239
+ /** The `artifacts/<name>.png` file name: 1 to 100 of `a-z`, `0-9`, `_` and `-`, starting with a letter or digit. Unique within the attempt. */
1686
1240
  name?: string;
1687
1241
  path?: string;
1688
1242
  viewport?: Viewport;
@@ -1769,10 +1323,10 @@ type JudgeState = JudgeJsonState | Array<{
1769
1323
  type: 'json';
1770
1324
  value: JudgeJsonState;
1771
1325
  } | JudgeFilePart>;
1772
- /** A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`. */
1326
+ /** A capture this judge call sent as a file part. Bytes live in the `artifacts/<name>.png` file. */
1773
1327
  type JudgeArtifactRef = {
1774
1328
  name: string;
1775
- /** `artifacts/<name>.png` under the attempt directory. */
1329
+ /** The `artifacts/<name>.png` path under the attempt directory. */
1776
1330
  path: string;
1777
1331
  sha256: string;
1778
1332
  bytes: number;
@@ -1804,11 +1358,6 @@ type JudgeOptions = {
1804
1358
  providerOptions?: Record<string, Record<string, JSONValue>>;
1805
1359
  /** The most judge calls to this model's provider at once, across the run. */
1806
1360
  maxConcurrent?: number;
1807
- /**
1808
- * The judge's state: what the judge is given beside the question, not the workspace's or the
1809
- * attempt's state. Merged key by key down the cascade.
1810
- */
1811
- state?: JudgeStateOptions;
1812
1361
  /**
1813
1362
  * The gateway the judge's calls go through, by name or by its helper. It skips the config's
1814
1363
  * `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
@@ -1816,43 +1365,61 @@ type JudgeOptions = {
1816
1365
  gateway?: GatewayName | GatewayDescriptor;
1817
1366
  };
1818
1367
  /**
1819
- * What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
1820
- * keeps its default, which is what the judge gets with no `state` option at all.
1368
+ * What one `c.judge()` call shows the judge beside the question. Every input is off unless the
1369
+ * call turns it on, and a call turns on at least one; each is a key of the state in
1370
+ * `judge-input.json`, or the captures sent as file parts.
1821
1371
  */
1822
- type JudgeStateOptions = {
1823
- /** The graded turn's prompt. Default true. */
1372
+ type JudgeInputs = {
1373
+ /** The graded turn's prompt. */
1824
1374
  task?: boolean;
1825
- /** The graded turn's final message, or `on`'s value. Default true. */
1375
+ /** The graded turn's final message, or `on`'s value. */
1826
1376
  output?: boolean;
1827
- /** `status path` lines for the changed files, under `diff`'s patterns. Default true. */
1377
+ /** `status path` lines for the changed files, under `diff`'s patterns. */
1828
1378
  files?: boolean;
1829
1379
  /**
1830
1380
  * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
1831
- * the last matching pattern wins and `!` excludes. Default true.
1381
+ * the last matching pattern wins and `!` excludes.
1832
1382
  */
1833
1383
  diff?: boolean | string[];
1834
1384
  /**
1835
1385
  * Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
1836
1386
  * patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
1837
1387
  * the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
1838
- * These files leave the diff. Default none.
1388
+ * These files leave the diff.
1839
1389
  */
1840
1390
  contents?: string[];
1841
1391
  /**
1842
1392
  * The graded turn's transcript as text lines: `true` is messages and tool calls with tool
1843
1393
  * output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
1844
- * Default false.
1845
1394
  */
1846
1395
  transcript?: boolean | 'tools' | 'full';
1396
+ /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
1397
+ checks?: boolean | string[];
1398
+ /**
1399
+ * Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
1400
+ * capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
1401
+ */
1402
+ artifacts?: true | Array<Capture | string>;
1403
+ };
1404
+ /** What a judge call's inputs read: which turn, and whether the turns before it too. */
1405
+ type JudgeSubject = {
1406
+ /** The turn `task`, `output`, `transcript` and `diff` read instead of the latest, or a value used as the output. */
1407
+ on?: Turn | JSONValue;
1847
1408
  /**
1848
1409
  * `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
1849
1410
  * `transcript`) transcript. Default `'last'`.
1850
1411
  */
1851
1412
  turns?: 'last' | 'all';
1852
- /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
1853
- checks?: boolean | string[];
1854
1413
  };
1855
- /** `JudgeStateOptions` with every key set: what one judge call saw. */
1414
+ /** One input turned on: `true`, a mode or a non-empty selection, not `false`. */
1415
+ type OneInput<I, K extends keyof I> = { [P in K]-?: Exclude<I[P], false | undefined>; };
1416
+ /** At least one of `I`'s keys turned on. */
1417
+ type AtLeastOneInput<I> = { [K in keyof I]-?: OneInput<I, K>; }[keyof I];
1418
+ /** `c.judge()`'s options, before the at-least-one-input rule that `JudgeCallOptions` adds. */
1419
+ type JudgeCallFields = JudgeOptions & JudgeInputs & JudgeSubject;
1420
+ /** `c.judge()`'s options: the judge options, the inputs (at least one) and what they read. */
1421
+ type JudgeCallOptions = JudgeCallFields & AtLeastOneInput<JudgeInputs>;
1422
+ /** The inputs one judge call saw, every key set (`false` when off); `artifacts` is recorded apart. */
1856
1423
  type ResolvedJudgeStateOptions = {
1857
1424
  task: boolean;
1858
1425
  output: boolean;
@@ -1863,16 +1430,6 @@ type ResolvedJudgeStateOptions = {
1863
1430
  turns: 'last' | 'all';
1864
1431
  checks: boolean | string[];
1865
1432
  };
1866
- type JudgeCallOptions = JudgeOptions & {
1867
- /** What to grade instead of the latest turn: another turn, or a value used as the output. */
1868
- on?: Turn | JSONValue;
1869
- /**
1870
- * Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
1871
- * `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
1872
- * captures; a provider that cannot take files fails the call.
1873
- */
1874
- artifacts?: true | Array<Capture | string>;
1875
- };
1876
1433
  type Verdict = {
1877
1434
  verdict: 'pass' | 'fail';
1878
1435
  /** The judge's probability that the answer is yes, from 0 to 1. */
@@ -1884,6 +1441,10 @@ type Verdict = {
1884
1441
  resolvedModel?: string;
1885
1442
  /** The gateway from the config's `gateway` list the call went through; unset without one. */
1886
1443
  gateway?: GatewayName;
1444
+ /** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
1445
+ upstream?: string;
1446
+ /** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
1447
+ costUsd?: number;
1887
1448
  /** Captures this call sent the judge as file parts, when any. */
1888
1449
  artifacts?: JudgeArtifactRef[];
1889
1450
  };
@@ -1891,7 +1452,7 @@ type Verdict = {
1891
1452
  * What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished,
1892
1453
  * so the judges and checks after it still run.
1893
1454
  */
1894
- type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
1455
+ type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel' | 'upstream' | 'costUsd'> & {
1895
1456
  verdict: 'error';
1896
1457
  score: null;
1897
1458
  error: string;
@@ -1904,10 +1465,11 @@ type AttemptContext = {
1904
1465
  /** Sends the next turn of this eval's agent session. */
1905
1466
  send(prompt: string): Promise<Turn>;
1906
1467
  /**
1907
- * Asks the judge a yes/no question about the latest turn (or `options.on`).
1908
- * `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
1468
+ * Asks the judge a yes/no question about what `options` turns on, and nothing else: at least
1469
+ * one of `task`, `output`, `files`, `diff`, `contents`, `transcript`, `checks` or `artifacts`.
1470
+ * They read the latest turn, or `options.on`.
1909
1471
  */
1910
- judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1472
+ judge(question: string, options: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1911
1473
  /**
1912
1474
  * Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
1913
1475
  * many extra times, `interval` ms apart; it resolves with the last try.
@@ -1922,8 +1484,10 @@ type AttemptContext = {
1922
1484
  background?: boolean;
1923
1485
  }): Promise<ExecResult | BackgroundHandle>;
1924
1486
  /**
1925
- * Starts `command` in the background and returns its port. A given `port` is used; omit it
1926
- * and reeval allocates one. Sets `PORT` so a bash string can use `$PORT`.
1487
+ * Starts `command` in the background with `PORT` set to `port`, or to a port reeval allocates,
1488
+ * and returns once it listens. Without `port`, a command that listens on one new port instead
1489
+ * (Vite's 5173) gets that one; several new ports, none of them `PORT`, or an exit first, throw
1490
+ * with its last output.
1927
1491
  */
1928
1492
  serve(command: string, port?: number): Promise<ServeHandle>;
1929
1493
  /**
@@ -1966,7 +1530,7 @@ type FakeHarnessDescriptor = {
1966
1530
  files?: Record<string, string>;
1967
1531
  /** The fake agent's final message on every turn; default `Done.`. */
1968
1532
  text?: string;
1969
- /** Simulated turn length, honours the abort signal. */
1533
+ /** Simulated turn length, honors the abort signal. */
1970
1534
  delayMs?: number;
1971
1535
  /** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
1972
1536
  ignoreAbort?: boolean;
@@ -2243,7 +1807,7 @@ type CodexHarnessDescriptor = {
2243
1807
  };
2244
1808
  /**
2245
1809
  * fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
2246
- * OpenAI's API by the model id, and reports each turn's tokens.
1810
+ * OpenAI's API by the model id, or on OpenRouter or Vertex AI, and reports each turn's tokens.
2247
1811
  */
2248
1812
  type FxHarnessDescriptor = {
2249
1813
  kind: 'fx';
@@ -2255,10 +1819,18 @@ type FxHarnessDescriptor = {
2255
1819
  /**
2256
1820
  * A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
2257
1821
  * `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
2258
- * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`.
1822
+ * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`. On `openrouter()`, either
1823
+ * runs on OpenRouter's Chat Completions API with `OPENROUTER_API_KEY`, a bare id as `openai/<id>`.
1824
+ * On `vertex()`, a Gemini model id, `google/<id>` or a bare `gemini-…` id, run on Vertex AI's
1825
+ * Chat Completions endpoint, e.g. `google/gemini-3.8-flash`.
2259
1826
  */
2260
1827
  model: string;
2261
- /** Cell id; defaults to `fx:<model>`. */
1828
+ /**
1829
+ * Reasoning effort, sent as `reasoning_effort` on each model call; only on `vertex()`, since fx
1830
+ * sends none itself. Values are what the model takes there, e.g. `low`, `medium` or `high`.
1831
+ */
1832
+ effort?: string;
1833
+ /** Cell id; defaults to `fx:<model>`, or `fx:<model>[effort=<value>]` when `effort` is set. */
2262
1834
  id?: string;
2263
1835
  /** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
2264
1836
  maxConcurrent?: number;
@@ -2298,12 +1870,21 @@ type GrokBuildHarnessDescriptor = {
2298
1870
  };
2299
1871
  /**
2300
1872
  * A gateway the config lists by name: `direct` is each provider's own key, as without `gateway`;
2301
- * `vercel` is Vercel AI Gateway.
1873
+ * `vercel` is Vercel AI Gateway; `openrouter` is OpenRouter; `vertex` is
1874
+ * Google Cloud's Vertex AI.
1875
+ */
1876
+ type GatewayName = 'direct' | 'vercel' | 'openrouter' | 'vertex';
1877
+ /**
1878
+ * Each provider's own key: made by `direct()` in `reeval/gateway`. Every `gateway` list ends with
1879
+ * it unless it's listed elsewhere, where it's tried in that place.
2302
1880
  */
2303
- type GatewayName = 'direct' | 'vercel';
2304
- /** Each provider's own key, as without `gateway`: made by `direct()` in `reeval/gateway`. */
2305
1881
  type DirectGatewayDescriptor = {
2306
1882
  kind: 'direct';
1883
+ /**
1884
+ * `false` turns off the fallback to each provider's own key, so a cell no other gateway in the
1885
+ * list serves is refused at plan time. Default `true`.
1886
+ */
1887
+ enabled?: boolean;
2307
1888
  };
2308
1889
  /**
2309
1890
  * Vercel AI Gateway, with `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`): made by `vercel()` in
@@ -2312,8 +1893,37 @@ type DirectGatewayDescriptor = {
2312
1893
  type VercelGatewayDescriptor = {
2313
1894
  kind: 'vercel';
2314
1895
  };
1896
+ /**
1897
+ * OpenRouter, with `OPENROUTER_API_KEY`: made by `openrouter()` in `reeval/gateway`. It serves
1898
+ * Claude Code, Codex, Goose, OpenCode, fx, Grok Build and pi.
1899
+ */
1900
+ type OpenRouterGatewayDescriptor = {
1901
+ kind: 'openrouter';
1902
+ /**
1903
+ * Let OpenRouter fail over to another upstream provider when the one it picked can't serve a
1904
+ * call. Off by default, so reeval's own calls (the judge, analysis, compare questions and
1905
+ * captures) send `allow_fallbacks: false`. The harnesses' calls take OpenRouter's own routing
1906
+ * either way.
1907
+ */
1908
+ fallbacks?: boolean;
1909
+ };
1910
+ /** A model publisher on Vertex AI, which may serve its models in a region of its own. */
1911
+ type VertexPublisher = 'anthropic' | 'google' | 'xai';
1912
+ /**
1913
+ * Google Cloud's Vertex AI, on Application Default Credentials: made by `vertex()` in
1914
+ * `reeval/gateway`.
1915
+ */
1916
+ type VertexGatewayDescriptor = {
1917
+ kind: 'vertex';
1918
+ /** The Google Cloud project the calls are billed to; defaults to `GOOGLE_CLOUD_PROJECT`. */
1919
+ project?: string;
1920
+ /** The region the models are called in, e.g. `us-east5`; defaults to `global`. */
1921
+ location?: string;
1922
+ /** A region per publisher, where it differs from `location`, e.g. `{ anthropic: 'us-east5' }`. */
1923
+ regions?: Partial<Record<VertexPublisher, string>>;
1924
+ };
2315
1925
  /** One entry of the config's `gateway` list. */
2316
- type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor;
1926
+ type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor | OpenRouterGatewayDescriptor | VertexGatewayDescriptor;
2317
1927
  /** A real coding agent, driven through its AI SDK harness adapter. */
2318
1928
  type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
2319
1929
  /** The agent an attempt runs: a real harness, or the fake one. */
@@ -2818,9 +2428,11 @@ type CompareOptions<E = Environment> = {
2818
2428
  * With `across`, it picks between two attempts that differ only in that factor, in both orders;
2819
2429
  * without it, it's asked of each attempt on its own. Without a model, the cascaded
2820
2430
  * `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
2821
- * `state` and `artifacts` choose what it's shown of each attempt.
2431
+ * It's shown only the inputs it turns on, at least one.
2822
2432
  */
2823
- type Question = string | {
2433
+ type Question = QuestionFields & AtLeastOneInput<QuestionInputs>;
2434
+ /** A question's options, before the at-least-one-input rule that `Question` adds. */
2435
+ type QuestionFields = QuestionInputs & {
2824
2436
  question: string;
2825
2437
  /** `'provider/model'` that answers it, over the cascaded `question.model`. */
2826
2438
  model?: string;
@@ -2831,43 +2443,39 @@ type Question = string | {
2831
2443
  across?: PairFactor | PairFactor[];
2832
2444
  /** The answers a question asked of each attempt picks from, with a short reason. */
2833
2445
  choices?: string[];
2834
- /** What it's shown of each attempt, over the cascaded `question.state`, key by key. */
2835
- state?: QuestionStateOptions;
2836
- /** Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
2837
- artifacts?: true | string[];
2446
+ /**
2447
+ * `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
2448
+ * transcript; `'last'`: the last turn only. Default `'all'`.
2449
+ */
2450
+ turns?: 'last' | 'all';
2838
2451
  };
2839
2452
  /**
2840
- * What a question is shown of each attempt: the keys of `judge.state` but `contents` (the
2841
- * attempts' workspaces are gone when questions are asked). An unset key keeps its default: a
2842
- * pairwise question shows the task and the diff; one asked of each attempt also shows the final
2843
- * message and the transcript.
2453
+ * What a question is shown of each attempt: `c.judge()`'s inputs but `contents` (the attempts'
2454
+ * workspaces are gone when questions are asked). Every input is off unless the question turns it
2455
+ * on, and a question turns on at least one.
2844
2456
  */
2845
- type QuestionStateOptions = {
2846
- /** The prompts the attempt was sent, once for both sides when they match. Default true. */
2457
+ type QuestionInputs = {
2458
+ /** The prompts the attempt was sent, once for both sides when they match. */
2847
2459
  task?: boolean;
2848
- /** The final message: the agent's own account of its work. Default false in a pair, true per attempt. */
2460
+ /** The final message: the agent's own account of its work. */
2849
2461
  output?: boolean;
2850
- /** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
2462
+ /** `status path` lines for the changed files, under `diff`'s patterns. */
2851
2463
  files?: boolean;
2852
2464
  /**
2853
2465
  * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
2854
- * the last matching pattern wins and `!` excludes. Default true.
2466
+ * the last matching pattern wins and `!` excludes.
2855
2467
  */
2856
2468
  diff?: boolean | string[];
2857
2469
  /**
2858
2470
  * The transcript as text lines: `true` is messages and tool calls with tool output clipped,
2859
2471
  * `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
2860
- * harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per
2861
- * attempt.
2472
+ * harness's tools, so a pair of two harnesses isn't blind.
2862
2473
  */
2863
2474
  transcript?: boolean | 'tools' | 'full';
2864
- /**
2865
- * `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
2866
- * transcript; `'last'`: the last turn only. Default `'all'`.
2867
- */
2868
- turns?: 'last' | 'all';
2869
- /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
2475
+ /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
2870
2476
  checks?: boolean | string[];
2477
+ /** Each attempt's own captures, by name; `true` sends all of them. */
2478
+ artifacts?: true | string[];
2871
2479
  };
2872
2480
  /** `QuestionStateOptions` with every key set: what each side of one question showed. */
2873
2481
  type ResolvedQuestionStateOptions = {
@@ -2879,20 +2487,16 @@ type ResolvedQuestionStateOptions = {
2879
2487
  turns: 'last' | 'all';
2880
2488
  checks: boolean | string[];
2881
2489
  };
2882
- /** Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key. */
2490
+ /** Defaults for every question below it. What a question is shown is its own, never a default. */
2883
2491
  type QuestionOptions = {
2884
2492
  /** `'provider/model'` that answers every question below it without its own `model`. */
2885
2493
  model?: string;
2886
- /** What it's shown of each attempt. */
2887
- state?: QuestionStateOptions;
2888
- /** Each attempt's own captures, by name; `true` sends all of them. Default none. */
2889
- artifacts?: true | string[];
2890
2494
  };
2891
2495
  /** A matrix factor a pairwise question varies; the others are held. */
2892
2496
  type PairFactor = 'environment' | 'harness';
2893
2497
  /** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
2894
2498
  type CompareCut = 'within' | 'across';
2895
- /** One judgement of one pair of attempts, in one order. */
2499
+ /** One judgment of one pair of attempts, in one order. */
2896
2500
  type PairVote = {
2897
2501
  question: string;
2898
2502
  cut: CompareCut;
@@ -2924,7 +2528,7 @@ type CompareAnswer = {
2924
2528
  model: string;
2925
2529
  /** The model id the provider says served the calls, when it reports one. */
2926
2530
  resolvedModel?: string;
2927
- /** What each side showed, every key resolved; unset on answers from before questions took `state`, which showed the defaults. */
2531
+ /** The inputs each side showed, every key resolved; unset on answers recorded before reeval wrote it, which showed the task and the diff. */
2928
2532
  stateOptions?: ResolvedQuestionStateOptions;
2929
2533
  /** The captures it sent of each side, as asked: `true` for all, or their names. */
2930
2534
  artifacts?: true | string[];
@@ -2977,8 +2581,21 @@ type GatePolicy = 'all' | {
2977
2581
  type CellGate = GatePolicy | 'none';
2978
2582
  /** A compare object: `baseline` and `variants`. Same as `CompareOptions`. */
2979
2583
  type CompareSpec<E = Environment> = CompareOptions<E>;
2584
+ type OddDigit = '1' | '3' | '5' | '7' | '9';
2585
+ /**
2586
+ * `N` when it's an odd whole number of 3 or more, else a sentence the compiler shows as the type
2587
+ * it expected. A `number` that isn't a literal stays `number`: only planning can check it.
2588
+ */
2589
+ type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${string}` | `${string}.${string}` | `${string}e${string}` | '1' ? 'an odd whole number of 3 or more, e.g. 3 or 5' : `${N}` extends `${string}${OddDigit}` ? N : 'an odd whole number of 3 or more, e.g. 3 or 5';
2590
+ /**
2591
+ * Attempts per cell: a fixed count, or `{ bestOf }`, an odd number of 3 or more, which stops a
2592
+ * cell once one side (pass or fail) has a majority: `{ bestOf: 5 }` runs 3 to 5 attempts.
2593
+ */
2594
+ type Runs<N extends number = number> = number | {
2595
+ bestOf: OddAtLeast3<N>;
2596
+ };
2980
2597
  /** Options other than `environment` and `compare`, which are exclusive of each other. */
2981
- type EvalOptionFields = {
2598
+ type EvalOptionFields<N extends number = number> = {
2982
2599
  /** Which agents run the eval; each harness is its own cell. */
2983
2600
  harness?: HarnessOption;
2984
2601
  /**
@@ -2986,8 +2603,13 @@ type EvalOptionFields = {
2986
2603
  * Not an environment: it does not invent skills, files or a name.
2987
2604
  */
2988
2605
  sandbox?: SandboxDescriptor;
2989
- /** Attempts per cell. */
2990
- runs?: number;
2606
+ /**
2607
+ * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
2608
+ * a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
2609
+ * Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
2610
+ * a verdict, and anything else runs once, a pooled `evaluate.each` case included.
2611
+ */
2612
+ runs?: Runs<N>;
2991
2613
  /** How each cell's attempts decide its outcome; default `'all'`. */
2992
2614
  gate?: GatePolicy;
2993
2615
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
@@ -3001,7 +2623,7 @@ type EvalOptionFields = {
3001
2623
  judge?: JudgeOptions;
3002
2624
  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
3003
2625
  capture?: CaptureOptions;
3004
- /** Defaults for every question: `model`, and `state` merged key by key, down the cascade. */
2626
+ /** The model every question that names none of its own is answered by, down the cascade. */
3005
2627
  question?: QuestionOptions;
3006
2628
  /**
3007
2629
  * What a language model is asked about the attempts, always a list; they never change pass,
@@ -3012,11 +2634,32 @@ type EvalOptionFields = {
3012
2634
  questions?: Question[];
3013
2635
  /** Per attempt. */
3014
2636
  timeout?: Duration;
2637
+ /**
2638
+ * When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
2639
+ * the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
2640
+ */
2641
+ snapshot?: SnapshotOption;
3015
2642
  /** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
3016
2643
  tags?: string[];
2644
+ /**
2645
+ * The eval's version, part of its fingerprint: bump it when a change the eval file doesn't show
2646
+ * (a helper, a fixture, data it reads) changes what it measures. A group's applies to the evals
2647
+ * inside it that set none.
2648
+ */
2649
+ revision?: Revision;
3017
2650
  /** Keys a profile's `limit` may not narrow for this group or eval. */
3018
2651
  pin?: PinnableKey[];
3019
2652
  };
2653
+ /** An eval's `revision`: a whole number or a non-empty string, compared exactly. */
2654
+ type Revision = number | string;
2655
+ /**
2656
+ * An eval's own source as hashes: its body and case data as declared, and its options and its
2657
+ * enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
2658
+ */
2659
+ type EvalSource = DeclaredSource & {
2660
+ /** sha256 of the canonical options. */
2661
+ options: string;
2662
+ };
3020
2663
  /**
3021
2664
  * `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
3022
2665
  * A compare with a `baseline` names every arm, so the cascaded `environment` is not applied, and
@@ -3042,8 +2685,9 @@ type EnvironmentOrCompare<E> = {
3042
2685
  baseline?: undefined;
3043
2686
  };
3044
2687
  };
3045
- type EvalOptions = EvalOptionFields & EnvironmentOrCompare<Environment>;
3046
- type GroupOptions = EvalOptions;
2688
+ /** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
2689
+ type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
2690
+ type GroupOptions<N extends number = number> = EvalOptions<N>;
3047
2691
  /**
3048
2692
  * `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
3049
2693
  * by name or a fake judge, and environments without `extend`. The same `environment` / compare
@@ -3065,7 +2709,7 @@ type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
3065
2709
  type HistoryConfig = {
3066
2710
  /** Committed history folder (default: `.history` in the directory the evals live in). */
3067
2711
  dir?: string;
3068
- /** Artifacts recorded next to each run: every eval's, failing evals' only, or none. */
2712
+ /** Artifacts recorded next to each run: every attempt's, failed attempts' only, or none. */
3069
2713
  artifacts?: ArtifactLevel;
3070
2714
  };
3071
2715
  /** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
@@ -3083,8 +2727,8 @@ type ProfileDefaults = {
3083
2727
  sandbox?: SandboxDescriptor;
3084
2728
  /** What every eval runs in when it names no environment. */
3085
2729
  environment?: Environment;
3086
- /** Attempts per cell. */
3087
- runs?: number;
2730
+ /** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
2731
+ runs?: Runs;
3088
2732
  /** How each cell's attempts decide its outcome. */
3089
2733
  gate?: GatePolicy;
3090
2734
  /** Variables and secrets for the sandbox, merged by name over the config's. */
@@ -3099,6 +2743,8 @@ type ProfileDefaults = {
3099
2743
  capture?: CaptureOptions;
3100
2744
  /** Defaults for every question, merged over the config's. */
3101
2745
  question?: QuestionOptions;
2746
+ /** When each attempt's workspace is copied; merged over the config's. */
2747
+ snapshot?: SnapshotOption;
3102
2748
  };
3103
2749
  /** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
3104
2750
  type ProfileLimit = {
@@ -3148,74 +2794,6 @@ type FakeAnalysisModel = {
3148
2794
  answer?: FakeAnswer | ((attempt: BlindAttempt) => FakeAnswer) | {
3149
2795
  error: string;
3150
2796
  };
3151
- /**
3152
- * What insights get back, in `reeval run` and `reeval analyze --insights`: fixed insights, a
3153
- * function of each call's input (`input.call` is `classify` or `insights`, once each per eval,
3154
- * or `headline` once; it may return a promise), or `{ error }` to fail every call. It counts a
3155
- * token per 4 characters of each call's input and answer, so `analysis.maxTokens` applies.
3156
- */
3157
- insights?: FakeInsights | ((input: FakeInsightsInput) => FakeInsights | Promise<FakeInsights>) | {
3158
- error: string;
3159
- };
3160
- };
3161
- /** What a fake insights function is given: one eval's scoped facts, or the run's evals for the headline. */
3162
- type FakeInsightsInput = ModelInsightInput | ModelHeadlineInput;
3163
- /**
3164
- * A fake insight answer. Each call takes its part: `intent` the classification, `answer`,
3165
- * `findings`, `rows`, `namedTools` and `remarks` the eval's, `headline` the run's. Whatever's
3166
- * left out is empty, and the intent defaults to `job: 'other'`.
3167
- */
3168
- type FakeInsights = {
3169
- intent?: {
3170
- job: 'choose' | 'finish' | 'other';
3171
- question?: string;
3172
- subject?: {
3173
- tool?: string;
3174
- change?: string;
3175
- };
3176
- name?: 'states-intent' | 'no-intent' | 'contradicts';
3177
- /** `low` falls back to `job: 'other'`, as a real classification does. */
3178
- confidence?: 'high' | 'medium' | 'low';
3179
- };
3180
- answer?: {
3181
- text: string;
3182
- cites: string[];
3183
- } | null;
3184
- findings?: Array<{
3185
- text: string;
3186
- cites: string[];
3187
- }>;
3188
- rows?: Array<{
3189
- row: string;
3190
- text: string;
3191
- cites: string[];
3192
- }>;
3193
- namedTools?: Array<{
3194
- attempt: string;
3195
- tools: string[];
3196
- }>;
3197
- /**
3198
- * The eval's remarks, as the model's JSON would have them: `source.at` and `fix.at` are
3199
- * `L<n>` lines of the eval's source, `evidence` fact ids (`F1`) or attempt lines (`A1#L3`).
3200
- * They go through the same gates as a model's, so one missing a part is dropped.
3201
- */
3202
- remarks?: Array<{
3203
- kind: string;
3204
- text: string;
3205
- source?: {
3206
- at: string;
3207
- quote: string;
3208
- } | null;
3209
- evidence?: string[];
3210
- fix?: {
3211
- at: string;
3212
- edit: string;
3213
- } | null;
3214
- }>;
3215
- headline?: {
3216
- text: string;
3217
- cites: string[];
3218
- } | null;
3219
2797
  };
3220
2798
  /** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
3221
2799
  type FakeVote = 'A' | 'B' | 'tie';
@@ -3243,15 +2821,15 @@ type BlindAttempt = {
3243
2821
  choices?: string[];
3244
2822
  attempt: BlindSide;
3245
2823
  };
3246
- /** A turn of one side, with the keys its question's `state` turned on. */
2824
+ /** A turn of one side, with the inputs its question turned on. */
3247
2825
  type BlindTurn = {
3248
2826
  task?: string;
3249
2827
  output?: string;
3250
2828
  transcript?: string;
3251
2829
  };
3252
2830
  /**
3253
- * One side of a blinded pair, or the attempt a per-attempt question is asked of: only the keys
3254
- * its question's `state` turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
2831
+ * One side of a blinded pair, or the attempt a per-attempt question is asked of: only the inputs
2832
+ * its question turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
3255
2833
  * under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
3256
2834
  */
3257
2835
  type BlindSide = BlindTurn & {
@@ -3269,7 +2847,7 @@ type BlindSide = BlindTurn & {
3269
2847
  command: string;
3270
2848
  notRun: true;
3271
2849
  }>;
3272
- /** Its captures, in the order sent, labelled by position; a name it lacks is `missing`. */
2850
+ /** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
3273
2851
  artifacts?: Array<{
3274
2852
  label: string;
3275
2853
  mediaType: 'image/png';
@@ -3279,24 +2857,13 @@ type BlindSide = BlindTurn & {
3279
2857
  missing: true;
3280
2858
  }>;
3281
2859
  };
3282
- /** Insights and `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
2860
+ /** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
3283
2861
  type AnalysisConfig = {
3284
2862
  /**
3285
- * `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze`
3286
- * and questions fall back to the judge's when it's a language model.
2863
+ * `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Unset, `reeval analyze` and questions
2864
+ * fall back to the judge's when it's a language model.
3287
2865
  */
3288
2866
  model?: string | FakeAnalysisModel;
3289
- /**
3290
- * `reeval run` classifies each eval and explains its result with `model` once the attempts
3291
- * finish (default true when `model` is set; `--no-insights` turns it off for one run).
3292
- */
3293
- insights?: boolean;
3294
- /**
3295
- * Tokens (in and out) that one run's insights may use, over every classification, insight and
3296
- * headline call (default 250000). Past it, the evals left print without insights; the run's
3297
- * outcome and exit code never change.
3298
- */
3299
- maxTokens?: number;
3300
2867
  /** Added to the prompt, e.g. "Fewer tool calls count as a win. The skill under test is reeval-good." */
3301
2868
  instructions?: string;
3302
2869
  /** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
@@ -3316,8 +2883,11 @@ type ReevalConfig = {
3316
2883
  sandbox?: SandboxDescriptor;
3317
2884
  /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
3318
2885
  environment?: Environment;
3319
- /** Attempts per cell; default 1. */
3320
- runs?: number;
2886
+ /**
2887
+ * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
2888
+ * variants runs 4 a side and anything else once.
2889
+ */
2890
+ runs?: Runs;
3321
2891
  /** How each cell's attempts decide its outcome; default `'all'`. */
3322
2892
  gate?: GatePolicy;
3323
2893
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
@@ -3335,6 +2905,11 @@ type ReevalConfig = {
3335
2905
  capture?: CaptureOptions;
3336
2906
  /** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
3337
2907
  question?: QuestionOptions;
2908
+ /**
2909
+ * When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
2910
+ * the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
2911
+ */
2912
+ snapshot?: SnapshotOption;
3338
2913
  /** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
3339
2914
  concurrency?: number;
3340
2915
  /** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
@@ -3350,7 +2925,7 @@ type ReevalConfig = {
3350
2925
  ci?: string;
3351
2926
  local?: string;
3352
2927
  };
3353
- /** `unbrokered: 'error'` refuses Docker secrets without `hosts` instead of warning (default `'warn'`). */
2928
+ /** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers written as plain strings, instead of warning (default `'warn'`). */
3354
2929
  secrets?: {
3355
2930
  unbrokered?: 'warn' | 'error';
3356
2931
  };
@@ -3362,11 +2937,12 @@ type ReevalConfig = {
3362
2937
  keep?: number;
3363
2938
  traceBufferMb?: number;
3364
2939
  };
3365
- /** Insights during `reeval run`, `reeval analyze`, and questions without a model. */
2940
+ /** `reeval analyze`, and questions without a model of their own. */
3366
2941
  analysis?: AnalysisConfig;
3367
2942
  /**
3368
2943
  * The gateways model calls may go through, in the order you prefer them: each harness takes
3369
- * the first that can serve it. Default `[direct()]`, each provider's own key.
2944
+ * the first that can serve it. `direct()`, each provider's own key, ends the list unless it's listed
2945
+ * in another place or turned off with `direct({ enabled: false })`. Default `[direct()]`.
3370
2946
  */
3371
2947
  gateway?: GatewayDescriptor[];
3372
2948
  };
@@ -3418,9 +2994,9 @@ type AttemptError = {
3418
2994
  };
3419
2995
  /**
3420
2996
  * One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
3421
- * reeval's MCP timer, which every stdio server runs through. An agent CLI that runs a process per
3422
- * turn (Antigravity, pi, Codex) starts its servers on every turn; one that keeps a session starts
3423
- * them once, on the first. A server that doesn't answer within 60s fails the attempt as an infra
2997
+ * reeval's MCP timer, which every stdio server runs through. An agent CLI that starts one process
2998
+ * per turn starts its servers on every turn; a CLI that keeps a session starts them on the first
2999
+ * turn only. A server that doesn't answer within 60s fails the attempt as an infra
3424
3000
  * error naming it; one whose client closes it first doesn't. `mcpStartup` is absent when the
3425
3001
  * environment has no stdio MCP server, or the sandbox has no `node` to run the timer.
3426
3002
  */
@@ -3433,7 +3009,7 @@ type McpStartup = {
3433
3009
  */
3434
3010
  ms: number | null;
3435
3011
  };
3436
- /** Where an eval's time went, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
3012
+ /** The duration of each evaluation phase, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
3437
3013
  type PhaseTimings = {
3438
3014
  setupMs: number;
3439
3015
  agentMs: number;
@@ -3443,16 +3019,28 @@ type PhaseTimings = {
3443
3019
  };
3444
3020
  type CellRole = 'control' | 'variant';
3445
3021
  /**
3446
- * One cell of an eval: a harness, an environment (`-` when none is set) and that environment's
3447
- * image (`sandbox` id: tag, dockerfile and install). With `compare`, the control is `baseline`
3448
- * (or the cascaded environment, or an empty `baseline`) and each variant is that environment on
3449
- * its own, including its sandbox.
3022
+ * A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
3023
+ * environment's sandbox id. With `compare`, the control is `baseline` (or the cascaded
3024
+ * environment, or an empty `baseline`) and each variant is that environment on its own,
3025
+ * including its sandbox.
3450
3026
  */
3451
3027
  type Cell = {
3028
+ /** The harness id, the environment name, and the sandbox id, joined with `/`. */
3452
3029
  key: string;
3030
+ /**
3031
+ * The harness id: `id` when set, otherwise `<kind>:<model>` (`<kind>` alone without a model),
3032
+ * with `[effort=<value>]` appended whenever `effort` is set.
3033
+ */
3453
3034
  harness: string;
3454
- /** The environment's own name, also on a variant cell (`with-house-greeting`, with no prefix). */
3035
+ /** The environment's own name, also on a variant cell (`with-skill`, with no prefix). */
3455
3036
  environment: string;
3037
+ /**
3038
+ * The sandbox id is `id` when set, or `fake` for the fake sandbox. A Docker sandbox starts with
3039
+ * the engine (`docker` by default). It adds `:` and the sanitized final component of `image` or
3040
+ * `from`, or for `dockerfile` the sanitized filename (the parent directory name when that
3041
+ * filename is `Dockerfile` and a parent is present). Non-empty `install` steps append `+` and an
3042
+ * eight-character hash.
3043
+ */
3456
3044
  sandbox: string;
3457
3045
  role: CellRole;
3458
3046
  /** The variant's name, for variant cells. */
@@ -3495,9 +3083,9 @@ type EnvironmentRecord = {
3495
3083
  name: string;
3496
3084
  /** The previous `extendEnvironment` layer's name, else null. */
3497
3085
  parent: string | null;
3498
- /** Keys a later layer replaced, e.g. `env.KEY`, `skills.genkit`, `mcp.docs`, `rules`. */
3086
+ /** Keys a later layer replaced, for example `env.KEY`, `skills.typescript`, `mcp.docs`, or `rules`. */
3499
3087
  overrides: string[];
3500
- /** Covers every field but the rules file's path; secret names are included, values never are. The image is the sandbox's. */
3088
+ /** Covers every field but the rules file's path; secret names are included; values never are. The image is the sandbox's. */
3501
3089
  digest: string;
3502
3090
  setup: string[];
3503
3091
  skills: Array<{
@@ -3511,7 +3099,7 @@ type EnvironmentRecord = {
3511
3099
  /** The plugin the skill came with; unset for a skill from `skills`. */
3512
3100
  plugin?: string;
3513
3101
  }>;
3514
- /** Every MCP server, a plugin's as `<plugin>:<server>`. */
3102
+ /** The names of the MCP servers the agent was given. */
3515
3103
  mcp: string[];
3516
3104
  /** The plugins placed, each with what it placed; in the digest. Unset when there are none. */
3517
3105
  plugins?: PluginRecord[];
@@ -3520,17 +3108,25 @@ type EnvironmentRecord = {
3520
3108
  sha256: string;
3521
3109
  }>;
3522
3110
  vars: string[];
3111
+ /**
3112
+ * Every secret, by the variable it's set as in the sandbox; an MCP header's (`secret.fromEnv`,
3113
+ * `secret.template`, a plugin's `${VAR}`) by each host variable it reads.
3114
+ */
3523
3115
  secrets: string[];
3524
- /** Secrets the sandbox only had placeholders for, swapped in by the egress gateway. */
3116
+ /**
3117
+ * Secrets the sandbox only had placeholders for, swapped in by the egress gateway, named as in
3118
+ * `secrets`. A variable it also got as its real value somewhere (an `http://` header, an env
3119
+ * secret without `hosts`) isn't listed.
3120
+ */
3525
3121
  brokered: string[];
3526
3122
  /** Optional secrets whose host variable was unset, so they weren't injected. */
3527
3123
  unset: string[];
3528
3124
  /** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
3529
3125
  egress: EgressPolicy;
3530
3126
  /**
3531
- * The git repository the workspace started as: the repo as written (a local one relative to the
3532
- * project root), the ref and path asked for (null when unset), the commit the ref resolved to
3533
- * and the hash of the tree that was copied in, which is in the digest.
3127
+ * This records the Git repository the workspace started as: the repo as written (a local one
3128
+ * relative to the project root), the ref and path asked for (null when unset), the commit the
3129
+ * ref resolved to, and the hash of the tree that was copied in, which is in the digest.
3534
3130
  */
3535
3131
  workspace?: {
3536
3132
  source: string;
@@ -3547,8 +3143,8 @@ type EnvironmentRecord = {
3547
3143
  * and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
3548
3144
  * can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
3549
3145
  * the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
3550
- * harness always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
3551
- * out.
3146
+ * harness always tries (its CLI's telemetry, say), kept out of `blocked`, so `blocked` contains
3147
+ * only unexpected refusals.
3552
3148
  */
3553
3149
  type EgressRecord = {
3554
3150
  policy: EgressPolicy;
@@ -3586,7 +3182,7 @@ type AttemptImage = {
3586
3182
  /** Content-addressed image id (`sha256:…`). */
3587
3183
  id: string;
3588
3184
  agentCli: string | null;
3589
- /** The harness image `install` and plugin builds were baked onto: `ref` itself without either. */
3185
+ /** The harness image the `install` layers were baked onto: `ref` itself without any. */
3590
3186
  base: string;
3591
3187
  /** The `docker({ install })` commands, in order. */
3592
3188
  install: string[];
@@ -3611,11 +3207,36 @@ type StartRetry = {
3611
3207
  /** How long it waited before giving up on that start. */
3612
3208
  waitedMs: number;
3613
3209
  };
3614
- /** One `expect` or `expect.soft` matcher call an attempt made. */
3210
+ /**
3211
+ * A process the setup steps left running in a Docker sandbox with a `to: 'checks'` secret without
3212
+ * `hosts` in its environment: it holds the real value while the agent works.
3213
+ */
3214
+ type SetupProcess = {
3215
+ /** Its command line, cut at 200 characters. */
3216
+ command: string;
3217
+ /** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
3218
+ secrets: string[];
3219
+ };
3220
+ /** One `c.serve` call that listened. */
3221
+ type ServeRecord = {
3222
+ /** The command as the eval passed it. */
3223
+ command: string;
3224
+ /** The port it listens on, which the handle and `c.capture` use. */
3225
+ port: number;
3226
+ /** The `PORT` it was given: the port the eval passed, or the one reeval allocated. */
3227
+ portEnv: number;
3228
+ /** It listened on a port other than `PORT`, the one that started listening after it did. */
3229
+ detected: boolean;
3230
+ /** The address it listens on there, e.g. `127.0.0.1`, `::1` or `0.0.0.0`; empty when the sandbox can't tell. */
3231
+ address: string;
3232
+ /** From the command's start until it listened. */
3233
+ readyMs: number;
3234
+ };
3235
+ /** One `expect` or `expect.require` matcher call an attempt made, or a check another language made. */
3615
3236
  type AssertionRecord = {
3616
3237
  /**
3617
- * `expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or
3618
- * the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt.
3238
+ * `label` is the supplied `expect` label. Without one, it is the matcher plus its first
3239
+ * argument or named subject, cut at 60 characters. Repeats get ` #2`, ` #3`, and later suffixes.
3619
3240
  */
3620
3241
  label: string;
3621
3242
  /** The eval gave the label. */
@@ -3624,12 +3245,61 @@ type AssertionRecord = {
3624
3245
  matcher: string;
3625
3246
  /** `error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report). */
3626
3247
  status: 'pass' | 'fail' | 'error';
3248
+ /**
3249
+ * A failure didn't end the eval's body: an `expect`. False for `expect.require` and for a check
3250
+ * that threw, such as Python's `assert`, which end the body when they fail.
3251
+ */
3627
3252
  soft: boolean;
3253
+ /** `path:line` of the call in the eval file, relative to the project root, when reeval saw it. */
3254
+ at?: string;
3628
3255
  /** Why it failed or couldn't answer, cut at 1 KB. */
3629
3256
  message?: string;
3630
3257
  /** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
3631
3258
  durationMs?: number;
3632
3259
  };
3260
+ /**
3261
+ * A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
3262
+ * folder), or why it wasn't.
3263
+ */
3264
+ type Snapshot = {
3265
+ /** `final` after the checks, or `turn-<n>` after the nth `c.send`. */
3266
+ name: string;
3267
+ /** A plain directory, or a gzipped tarball with `snapshot: { compress: true }`. */
3268
+ form: 'dir' | 'tar.gz';
3269
+ /**
3270
+ * Where it is, relative to the run's directory in `.reeval/runs`:
3271
+ * `<eval>/<cell>/<n>/snapshots/<name>` (with `.tar.gz` when compressed); null when it wasn't
3272
+ * taken.
3273
+ */
3274
+ path: string | null;
3275
+ /** Its size on disk: the directory's files, or the tarball; 0 when it wasn't taken. */
3276
+ bytes: number;
3277
+ /** Regular files and symbolic links in it. */
3278
+ files: number;
3279
+ /** Directories left out by name, such as `node_modules/` and `.git/`, at most 50. */
3280
+ excluded: string[];
3281
+ /** Text files whose secret-looking values were masked, at most 50. */
3282
+ masked: string[];
3283
+ /** Binary files left out because they held a secret-looking value, at most 50. */
3284
+ dropped: string[];
3285
+ /**
3286
+ * Why it wasn't taken: `size` when the workspace was over its form's cap, `error` when the
3287
+ * sandbox couldn't archive it; null when it was.
3288
+ */
3289
+ skipped: {
3290
+ cause: 'size' | 'error';
3291
+ message: string;
3292
+ } | null;
3293
+ };
3294
+ /** `snapshot`: the shorthand, or the object that also picks the form. */
3295
+ type SnapshotOption = boolean | 'turns' | SnapshotOptions;
3296
+ /** `snapshot` as an object, merged field by field over the levels above. */
3297
+ type SnapshotOptions = {
3298
+ /** `true` after the checks, `'turns'` after each `c.send` too, `false` never. Default `true`. */
3299
+ when?: boolean | 'turns';
3300
+ /** Write each as `<name>.tar.gz` instead of a directory. Default `false`. */
3301
+ compress?: boolean;
3302
+ };
3633
3303
  /** Which environment ran as the control, so the studio can show it. */
3634
3304
  type EvalCompareMeta = {
3635
3305
  baseline: string;
@@ -3640,16 +3310,20 @@ type EvalCompareMeta = {
3640
3310
  from?: 'environment' | 'empty';
3641
3311
  };
3642
3312
  /**
3643
- * A sandbox with more privilege than the default `reeval` user: `docker({ root: true })`, or
3644
- * `docker({ capabilities })`. One per level that set such a sandbox, however many evals inherit it.
3313
+ * A `SandboxWarning` describes a sandbox with more privilege than the default `reeval` user:
3314
+ * `docker({ root: true })` or `docker({ capabilities })`. One per level that set such a sandbox,
3315
+ * however many evaluations inherit it.
3645
3316
  */
3646
3317
  type SandboxWarning = {
3647
3318
  kind: 'root' | 'capabilities';
3648
3319
  /** Which level set the sandbox. */
3649
3320
  level: 'config' | 'profile' | 'group' | 'eval';
3650
- /** That level as reeval names it: `reeval.config.ts`, `profiles.pr.defaults`, `group("net")` or `evaluate("installs deps")`. */
3321
+ /**
3322
+ * The value is the level as reeval names it: the `reeval.config.ts` file, `profiles.pr.defaults`,
3323
+ * `group("net")`, or `evaluate("installs deps")`.
3324
+ */
3651
3325
  where: string;
3652
- /** The eval file that sets it, for a group or eval; null for the config or a profile. */
3326
+ /** The evaluation file that sets it, for a group or evaluation; null for the config or a profile. */
3653
3327
  file: string | null;
3654
3328
  /** The capabilities added back, for `capabilities`. */
3655
3329
  capabilities?: string[];
@@ -3665,5 +3339,5 @@ type ResolvedGateway = {
3665
3339
  }>;
3666
3340
  };
3667
3341
  //#endregion
3668
- export { FakeJudgeModel as $, RunRecord as $n, Verdict as $t, CursorHarnessDescriptor as A, SuiteTally as An, PluginPart as At, EnvironmentHelpers as B, suiteResult as Bn, SandboxDescriptor as Bt, ClaudeCodeHarnessDescriptor as C, Interval as Cn, effortVerdict as Cr, McpServer as Ct, CompareOptions as D, ResultFile as Dn, tokenValues as Dr, PairVote as Dt, CompareCut as E, QuestionResult as En, formatSeconds as Er, PairFactor as Et, Duration as F, evalAcross as Fn, Question as Ft, EvalOptionsData as G, FamilyLift as Gn, SetupStep as Gt, EnvironmentSpec as H, AttemptRecord as Hn, Secret as Ht, EgressPolicy as I, factSegment as In, QuestionOptions as It, FakeAnalysisModel as J, FileRef as Jn, TranscriptEvent as Jt, ExecResult as K, FamilyRate as Kn, SkillSource as Kt, EgressRecord as L, isNotGatedResult as Ln, QuestionStateOptions as Lt, DirectGatewayDescriptor as M, checkEffect as Mn, Profile as Mt, DirectoryOptions as N, countedOf as Nn, ProfileDefaults as Nt, ConfigHelpers as O, ResultOutcome as On, PiHarnessDescriptor as Ot, DockerSandboxDescriptor as P, effectOf as Pn, ProfileLimit as Pt, FakeInsightsInput as Q, Lift as Qn, VercelGatewayDescriptor as Qt, EnvVars as R, isStudy as Rn, ReevalConfig as Rt, ClaudeCodeEffort as S, EvalResult as Sn, effortSummary as Sr, JudgeStateOptions as St, CompareAnswer as T, PooledArm as Tn, formatRatio as Tr, Outcome as Tt, EvalFn as U, CellRecord as Un, SecretTemplate as Ut, EnvironmentInput as V, ArtifactRef as Vn, SandboxWarning as Vt, EvalOptions as W, EvalEntry as Wn, ServeHandle as Wt, FakeHarnessDescriptor as X, HistoryEntry as Xn, TurnData as Xt, FakeAnswer as Y, HistoryCell as Yn, Turn as Yt, FakeInsights as Z, JudgementRecord as Zn, TurnStatus as Zt, CaptureUseScript as _, CellResult as _n, directional as _r, JudgeCallOptions as _t, AttemptAnswer as a, Job as an, EffortContrast as ar, GatewayName as at, Cell as b, Effect as bn, effortOf as br, JudgeOptions as bt, BlindAttempt as c, REMARK_KINDS as cn, EffortVerdict as cr, GroupOptions as ct, Capture as d, Sentence as dn, VERDICT_TEXT as dr, HarnessEdit as dt, Viewport as en, StackSummary as er, FakeSandboxDescriptor as et, CaptureCall as f, fromModel as fn, bothUnderATenth as fr, HarnessOption as ft, CaptureUseAction as g, CellError as gn, contrastEffort as gr, JudgeArtifactRef as gt, CaptureUse as h, CaseResult as hn, cellSamples as hr, JSONValue as ht, AssertionRecord as i, INSIGHTS_FILE as in, EFFORT_MIN_RUNS as ir, GatewayDescriptor as it, Diff as j, cancelledOf as jn, PluginSource as jt, ConfigJudgeOptions as k, SuiteResult as kn, PinnableKey as kt, BlindPair as l, Remark as ln, Spread as lr, HarnessData as lt, CaptureOptions as m, AttemptQuestionResult as mn, cellEffort as mr, HostFile as mt, AnalysisConfig as n, WebEventType as nn, DIRECTIONAL_MIN as nr, FakeVote as nt, AttemptContext as o, MODEL_REMARK_KINDS as on, EffortMetric as or, GooseHarnessDescriptor as ot, CaptureInstruction as p, Across as pn, cellContrast as pr, HistoryConfig as pt, ExecRetry as q, FamilyRecord as qn, TokenUsage as qt, AntigravityHarnessDescriptor as r, EvalInsight as rn, EFFORT_ALPHA as rr, FxHarnessDescriptor as rt, BackgroundHandle as s, NO_MODEL as sn, EffortSample as sr, GrokBuildHarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, CellEffort as tr, FakeScriptStep as tt, Budget as u, RunInsights as un, TOKENS_UNREPORTED as ur, HarnessDescriptor as ut, CaptureWait as v, CheckOutcome as vn, effortChangeLabel as vr, JudgeFailure as vt, CodexHarnessDescriptor as w, JudgeResult as wn, formatCount as wr, OpenCodeHarnessDescriptor as wt, CellRole as x, EffortCounters as xn, effortSample as xr, JudgeState as xt, CaptureWindow as y, CheckResult as yn, effortLabel as yr, JudgeModel as yt, Environment as z, messageLine as zn, ResolvedGateway as zt };
3669
- //# sourceMappingURL=types-DXBbP9Mk.d.ts.map
3342
+ export { FakeScriptStep as $, Verdict as $t, CursorHarnessDescriptor as A, Spread as An, PluginPart as At, EnvironmentHelpers as B, effortLabel as Bn, SandboxDescriptor as Bt, ClaudeCodeHarnessDescriptor as C, DIRECTIONAL_MIN as Cn, OpenCodeHarnessDescriptor as Ct, CompareOptions as D, EffortMetric as Dn, PairVote as Dt, CompareCut as E, EffortContrast as En, PairFactor as Et, Duration as F, cellEffort as Fn, Question as Ft, EvalOptions as G, formatCount as Gn, SetupStep as Gt, EnvironmentSpec as H, effortSample as Hn, Secret as Ht, EgressPolicy as I, cellSamples as In, QuestionInputs as It, FakeAnalysisModel as J, tokenValues as Jn, TranscriptEvent as Jt, ExecResult as K, formatRatio as Kn, SkillSource as Kt, EgressRecord as L, contrastEffort as Ln, QuestionOptions as Lt, DirectGatewayDescriptor as M, VERDICT_TEXT as Mn, Profile as Mt, DirectoryOptions as N, bothUnderATenth as Nn, ProfileDefaults as Nt, ConfigHelpers as O, EffortSample as On, PiHarnessDescriptor as Ot, DockerSandboxDescriptor as P, cellContrast as Pn, ProfileLimit as Pt, FakeSandboxDescriptor as Q, VercelGatewayDescriptor as Qt, EnvVars as R, directional as Rn, ReevalConfig as Rt, ClaudeCodeEffort as S, CellEffort as Sn, McpServer as St, CompareAnswer as T, EFFORT_MIN_RUNS as Tn, Outcome as Tt, ErrorKind as U, effortSummary as Un, SecretTemplate as Ut, EnvironmentInput as V, effortOf as Vn, SandboxWarning as Vt, EvalFn as W, effortVerdict as Wn, ServeHandle as Wt, FakeHarnessDescriptor as X, DeclaredNode as Xn, TurnData as Xt, FakeAnswer as Y, DeclarationTree as Yn, Turn as Yt, FakeJudgeModel as Z, TurnStatus as Zt, CaptureUseScript as _, JudgmentRecord as _n, JudgeInputs as _t, AttemptContext as a, ArtifactRef as an, GrokBuildHarnessDescriptor as at, Cell as b, StackSummary as bn, JudgeState as bt, BlindPair as c, CheckTally as cn, HarnessDescriptor as ct, CaptureCall as d, FamilyLiftSummary as dn, HistoryConfig as dt, VertexGatewayDescriptor as en, FakeVote as et, CaptureInstruction as f, FamilyRate as fn, HostFile as ft, CaptureUseAction as g, HistoryEntry as gn, JudgeFailure as gt, CaptureUse as h, HistoryCell as hn, JudgeCallOptions as ht, AttemptAnswer as i, WebEventType as in, GooseHarnessDescriptor as it, Diff as j, TOKENS_UNREPORTED as jn, PluginSource as jt, ConfigJudgeOptions as k, EffortVerdict as kn, PinnableKey as kt, Budget as l, EvalEntry as ln, HarnessEdit as lt, CaptureOptions as m, FileRef as mn, JudgeArtifactRef as mt, AnalysisConfig as n, Viewport as nn, GatewayDescriptor as nt, BackgroundHandle as o, AttemptRecord as on, GroupOptions as ot, CaptureKey as p, FamilyRecord as pn, JSONValue as pt, ExecRetry as q, formatSeconds as qn, TokenUsage as qt, AntigravityHarnessDescriptor as r, Wait as rn, GatewayName as rt, BlindAttempt as s, CellRecord as sn, HarnessData as st, AgentHarnessDescriptor as t, VertexPublisher as tn, FxHarnessDescriptor as tt, Capture as u, FamilyLift as un, HarnessOption as ut, CaptureWait as v, Lift as vn, JudgeModel as vt, CodexHarnessDescriptor as w, EFFORT_ALPHA as wn, OpenRouterGatewayDescriptor as wt, CellRole as x, UpstreamCount as xn, JudgeSubject as xt, CaptureWindow as y, RunRecord as yn, JudgeOptions as yt, Environment as z, effortChangeLabel as zn, ResolvedGateway as zt };
3343
+ //# sourceMappingURL=types-DVvW66ci.d.ts.map