reeval 0.15.1 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. package/README.md +12 -12
  2. package/dist/{adapter-CbqbMR7y.js → adapter-CRKYPkRt.js} +3 -3
  3. package/dist/{adapter-CbqbMR7y.js.map → adapter-CRKYPkRt.js.map} +1 -1
  4. package/dist/{agent-Zgn-kiqq.js → agent-BgH4-kFc.js} +6 -6
  5. package/dist/{agent-Zgn-kiqq.js.map → agent-BgH4-kFc.js.map} +1 -1
  6. package/dist/{app-files-ChbvRx3P.js → app-files-BMVBr079.js} +3 -3
  7. package/dist/app-files-BMVBr079.js.map +1 -0
  8. package/dist/{assemble-SKQGW_QF.js → assemble-BVbI6NW-.js} +95 -29
  9. package/dist/assemble-BVbI6NW-.js.map +1 -0
  10. package/dist/assemble-SiHa4__F.js +1 -0
  11. package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
  12. package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
  13. package/dist/{attempt-host-BOp8JklD.js → attempt-host-CIxjs2Be.js} +13 -6
  14. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  15. package/dist/{capture-BM_3100g.js → capture-iQ6kYJV0.js} +987 -161
  16. package/dist/capture-iQ6kYJV0.js.map +1 -0
  17. package/dist/capture-px7Ey95_.js +1 -0
  18. package/dist/{cli-PdYrcoKM.js → cli-1YQU-66F.js} +98 -61
  19. package/dist/cli-1YQU-66F.js.map +1 -0
  20. package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
  21. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  22. package/dist/cli.js +2 -2
  23. package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
  24. package/dist/client-DMmOkbRA.js.map +1 -0
  25. package/dist/{command-4pJYVCT4.js → command-8_Cg5v6b.js} +12 -12
  26. package/dist/{command-4pJYVCT4.js.map → command-8_Cg5v6b.js.map} +1 -1
  27. package/dist/{command-DbSMcxZL.js → command-BlYCAqu-.js} +7 -7
  28. package/dist/{command-DbSMcxZL.js.map → command-BlYCAqu-.js.map} +1 -1
  29. package/dist/{command-9xm5ImqI.js → command-C-HeGlap.js} +16 -16
  30. package/dist/{command-9xm5ImqI.js.map → command-C-HeGlap.js.map} +1 -1
  31. package/dist/{command-BqBxLgTc.js → command-Cr1zjRvl.js} +13 -13
  32. package/dist/{command-BqBxLgTc.js.map → command-Cr1zjRvl.js.map} +1 -1
  33. package/dist/{commands-Bi2SEQtr.js → commands-Bj9NbmsO.js} +1 -1
  34. package/dist/{commands-BU6YJCz9.js → commands-BvTNZ2-b.js} +14 -9
  35. package/dist/commands-BvTNZ2-b.js.map +1 -0
  36. package/dist/{config-k6m7XbAz.js → config-CneBZB84.js} +430 -208
  37. package/dist/config-CneBZB84.js.map +1 -0
  38. package/dist/{contrasts-DLZF4dGF.js → contrasts-lhFsekco.js} +2 -2
  39. package/dist/{contrasts-DLZF4dGF.js.map → contrasts-lhFsekco.js.map} +1 -1
  40. package/dist/db-BnXyamuj.js +2 -0
  41. package/dist/{db-CpsifVln.js → db-Cnh3jIv_.js} +8 -8
  42. package/dist/{db-CpsifVln.js.map → db-Cnh3jIv_.js.map} +1 -1
  43. package/dist/debug-CAx3qcaP.js +1 -0
  44. package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
  45. package/dist/debug-CgG3T071.js.map +1 -0
  46. package/dist/{docker-51WvhN_X.js → docker-BihzJwbV.js} +9 -5
  47. package/dist/docker-BihzJwbV.js.map +1 -0
  48. package/dist/{docker-B4-rx4dA.js → docker-Cqq03H72.js} +5 -5
  49. package/dist/{docker-B4-rx4dA.js.map → docker-Cqq03H72.js.map} +1 -1
  50. package/dist/{docker-YFj4H1Cr.js → docker-DCAtFMo9.js} +1 -1
  51. package/dist/{doctor-BuQnfz4e.js → doctor-Dl4vMeqt.js} +15 -15
  52. package/dist/{doctor-BuQnfz4e.js.map → doctor-Dl4vMeqt.js.map} +1 -1
  53. package/dist/{environment-BOFgzFDg.js → environment-Cf5HMjib.js} +3 -3
  54. package/dist/{environment-BOFgzFDg.js.map → environment-Cf5HMjib.js.map} +1 -1
  55. package/dist/{errors-UjcLWelF.js → errors-CHebohjc.js} +9 -3
  56. package/dist/{errors-UjcLWelF.js.map → errors-CHebohjc.js.map} +1 -1
  57. package/dist/{eval-source-Dje3rERU.js → eval-source-CAE4y2Sr.js} +9 -8
  58. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  59. package/dist/{evaluate-BPP9s3s6.js → evaluate-CUtgyr43.js} +340 -45
  60. package/dist/evaluate-CUtgyr43.js.map +1 -0
  61. package/dist/{evaluate-dqXccn4k.d.ts → evaluate-D35T51AC.d.ts} +7 -6
  62. package/dist/files-BEnUJW1e.js +1 -0
  63. package/dist/{files-Bi461Be2.js → files-BRViE7m-.js} +20 -13
  64. package/dist/{files-Bi461Be2.js.map → files-BRViE7m-.js.map} +1 -1
  65. package/dist/{files-H24pz0hL.js → files-BSqA2sZE.js} +3 -3
  66. package/dist/{files-H24pz0hL.js.map → files-BSqA2sZE.js.map} +1 -1
  67. package/dist/files-C3tQtD26.js +1 -0
  68. package/dist/{files-Clipx4OW.js → files-CNiJ_fZf.js} +3 -3
  69. package/dist/{files-Clipx4OW.js.map → files-CNiJ_fZf.js.map} +1 -1
  70. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  71. package/dist/files-D1y8Pg4Z.js.map +1 -0
  72. package/dist/files-Dts7EY3H.js +1 -0
  73. package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
  74. package/dist/gate-BC6SdQZs.js.map +1 -0
  75. package/dist/gateway.d.ts +1 -1
  76. package/dist/{git-CsVZkNNL.js → git-UNxT_3z0.js} +2 -2
  77. package/dist/{git-CsVZkNNL.js.map → git-UNxT_3z0.js.map} +1 -1
  78. package/dist/harness.d.ts +1 -1
  79. package/dist/{images-4mwhNkvh.js → images-8A_LzVfw.js} +8 -8
  80. package/dist/{images-4mwhNkvh.js.map → images-8A_LzVfw.js.map} +1 -1
  81. package/dist/index.d.ts +63 -16
  82. package/dist/index.js +5 -5
  83. package/dist/init-okoALIah.js +1 -0
  84. package/dist/{init-cOk3D44b.js → init-sFBgODhu.js} +4 -4
  85. package/dist/{init-cOk3D44b.js.map → init-sFBgODhu.js.map} +1 -1
  86. package/dist/{invocations-DW-lTk2f.js → invocations-C9ejlOYl.js} +2 -2
  87. package/dist/{invocations-DW-lTk2f.js.map → invocations-C9ejlOYl.js.map} +1 -1
  88. package/dist/{judge-CKdgEXN7.js → judge-Bg4v4e_K.js} +2 -2
  89. package/dist/{judge-CKdgEXN7.js.map → judge-Bg4v4e_K.js.map} +1 -1
  90. package/dist/judge.d.ts +2 -2
  91. package/dist/judge.js +1 -1
  92. package/dist/{live-XJQ6nqK4.js → live-DeMvnDD5.js} +2 -2
  93. package/dist/{live-XJQ6nqK4.js.map → live-DeMvnDD5.js.map} +1 -1
  94. package/dist/{matchers-DT2TaGix.js → matchers-BKPuaXlj.js} +14 -13
  95. package/dist/matchers-BKPuaXlj.js.map +1 -0
  96. package/dist/{model-Bq3G0OG1.js → model-2rBZrdZA.js} +4 -4
  97. package/dist/{model-Bq3G0OG1.js.map → model-2rBZrdZA.js.map} +1 -1
  98. package/dist/{model-vRiUycWh.js → model-3MJtpho8.js} +4 -4
  99. package/dist/{model-vRiUycWh.js.map → model-3MJtpho8.js.map} +1 -1
  100. package/dist/{model-CTnyvh7H.js → model-Bz7-j71_.js} +3 -50
  101. package/dist/model-Bz7-j71_.js.map +1 -0
  102. package/dist/{model-g3XnzUv0.js → model-DkPwypN4.js} +1 -1
  103. package/dist/model-bREXVXwT.js +1 -0
  104. package/dist/{node-BVbbnFtv.js → node-fP-H2uxn.js} +2 -2
  105. package/dist/{node-BVbbnFtv.js.map → node-fP-H2uxn.js.map} +1 -1
  106. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  107. package/dist/object-BkXGPKWY.js.map +1 -0
  108. package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-Bo8PZ4Ew.js} +2 -2
  109. package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-Bo8PZ4Ew.js.map} +1 -1
  110. package/dist/{patch-DKJiC5Sq.js → patch-6T9XAipm.js} +4 -4
  111. package/dist/patch-6T9XAipm.js.map +1 -0
  112. package/dist/{plan-DdWkM6SW.js → plan-Dw2BjafJ.js} +20 -11
  113. package/dist/plan-Dw2BjafJ.js.map +1 -0
  114. package/dist/{plugins-D-CP6mMd.js → plugins-ngQEJBjJ.js} +24 -3
  115. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  116. package/dist/{preflight-DTiVv0gY.js → preflight-BYXkweNN.js} +1 -1
  117. package/dist/{preflight-9ixuXzsx.js → preflight-CDRpog9o.js} +3 -3
  118. package/dist/{preflight-9ixuXzsx.js.map → preflight-CDRpog9o.js.map} +1 -1
  119. package/dist/{prepare-tmBBf4W8.js → prepare-B3btUlT6.js} +2 -2
  120. package/dist/{prepare-BfyikQit.js → prepare-PkARUGoF.js} +7 -7
  121. package/dist/{prepare-BfyikQit.js.map → prepare-PkARUGoF.js.map} +1 -1
  122. package/dist/{provenance-BizwUn3h.js → provenance-7Hzg88f4.js} +2 -2
  123. package/dist/{provenance-BizwUn3h.js.map → provenance-7Hzg88f4.js.map} +1 -1
  124. package/dist/{providers-DkOupEBl.js → providers-DDNMbv1T.js} +1 -1
  125. package/dist/{providers-B6b9Tnyi.js → providers-PKoJtGK0.js} +5 -5
  126. package/dist/{providers-B6b9Tnyi.js.map → providers-PKoJtGK0.js.map} +1 -1
  127. package/dist/query-Bt6vLJ3m.js +1 -0
  128. package/dist/{query-DyM-VExS.js → query-nBXrgEwl.js} +2 -2
  129. package/dist/{query-DyM-VExS.js.map → query-nBXrgEwl.js.map} +1 -1
  130. package/dist/{questions-BRhU8MzB.js → questions-CkAwePPR.js} +16 -17
  131. package/dist/questions-CkAwePPR.js.map +1 -0
  132. package/dist/remarks-Bp6AwPgb.js +1 -0
  133. package/dist/{remarks-C7i2LMRb.js → remarks-Bt0zwiQB.js} +42 -8
  134. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  135. package/dist/{remarks-D_3qhBWU.js → remarks-IFFe_oty.js} +4 -3
  136. package/dist/{remarks-D_3qhBWU.js.map → remarks-IFFe_oty.js.map} +1 -1
  137. package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
  138. package/dist/remarks-eBcrIZRu.js.map +1 -0
  139. package/dist/{remote-DFIzcx_6.js → remote-DfJYiP-R.js} +5 -5
  140. package/dist/{remote-DFIzcx_6.js.map → remote-DfJYiP-R.js.map} +1 -1
  141. package/dist/resolve-CT4XZIfV.js +1 -0
  142. package/dist/{resolve-DWSJpKHX.js → resolve-D_cLa4tW.js} +5 -5
  143. package/dist/{resolve-DWSJpKHX.js.map → resolve-D_cLa4tW.js.map} +1 -1
  144. package/dist/{results-BC3pcG_Z.js → results-DrKJuIQF.js} +9 -9
  145. package/dist/{results-BC3pcG_Z.js.map → results-DrKJuIQF.js.map} +1 -1
  146. package/dist/{resume-j-rcIryi.js → resume-BmIM52MJ.js} +4 -4
  147. package/dist/{resume-j-rcIryi.js.map → resume-BmIM52MJ.js.map} +1 -1
  148. package/dist/{runtime-d56Gg3J5.js → runtime-CZVpkr04.js} +111 -37
  149. package/dist/runtime-CZVpkr04.js.map +1 -0
  150. package/dist/{sandbox-Cmln11gU.js → sandbox-Cmi6IQML.js} +2 -2
  151. package/dist/{sandbox-Cmln11gU.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  152. package/dist/sandbox.d.ts +1 -1
  153. package/dist/sandbox.js +1 -1
  154. package/dist/server-CKJOnIB0.js +1 -0
  155. package/dist/{server-BOoH10J8.js → server-DS8BGhGU.js} +34 -10
  156. package/dist/server-DS8BGhGU.js.map +1 -0
  157. package/dist/source-CgUqpumI.js +1 -0
  158. package/dist/{source-wD04-rRG.js → source-DcXv4de-.js} +17 -3
  159. package/dist/{source-wD04-rRG.js.map → source-DcXv4de-.js.map} +1 -1
  160. package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
  161. package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
  162. package/dist/{src-kYeaJJrl.js → src-BTqJbvj6.js} +1 -1
  163. package/dist/{src-Bp4aN3rA.js → src-BkiComGh.js} +1 -1
  164. package/dist/{src-dlX-Lmxg.js → src-BvgeZ4tS.js} +1 -1
  165. package/dist/{src-D91Hd6hT.js → src-C0ySiej5.js} +36 -17
  166. package/dist/src-C0ySiej5.js.map +1 -0
  167. package/dist/{src-HSxBazcg.js → src-CgJrz5p0.js} +1 -1
  168. package/dist/{src-BYPMM1x7.js → src-Cp-2Hdib.js} +1 -1
  169. package/dist/{src-6_zz7oJs.js → src-DMo2eCOJ.js} +1 -1
  170. package/dist/src-Dk2Na0b-.js +2 -0
  171. package/dist/{src-C7upnZSd.js → src-OESRtvOi.js} +1 -1
  172. package/dist/src-h5uDlk3m.js +1 -0
  173. package/dist/state-C97udoh3.js +1 -0
  174. package/dist/{state-DrJ687ut.js → state-C_kdhp5v.js} +5 -5
  175. package/dist/{state-DrJ687ut.js.map → state-C_kdhp5v.js.map} +1 -1
  176. package/dist/suite-ByT1lsdD.js +2 -0
  177. package/dist/{suite-Co8cwelb.js → suite-DieLXvCT.js} +15 -4
  178. package/dist/suite-DieLXvCT.js.map +1 -0
  179. package/dist/{summary-DLFHVoux.js → summary-CSzyBVsF.js} +40 -14
  180. package/dist/summary-CSzyBVsF.js.map +1 -0
  181. package/dist/tarball-CHcahyNK.js +258 -0
  182. package/dist/tarball-CHcahyNK.js.map +1 -0
  183. package/dist/testing.d.ts +52 -44
  184. package/dist/testing.js +4 -4
  185. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  186. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  187. package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
  188. package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
  189. package/dist/{types-coMWHaU2.d.ts → types-DVvW66ci.d.ts} +325 -171
  190. package/dist/view/index.d.ts +16 -4
  191. package/dist/view/index.js +6 -6
  192. package/dist/{views-BFO1BLKu.js → views-B9STsLB4.js} +5 -5
  193. package/dist/{views-BFO1BLKu.js.map → views-B9STsLB4.js.map} +1 -1
  194. package/dist/views-Ca-rTixp.js +2 -0
  195. package/dist/{worker-entry-CJtdsKBP.js → worker-entry-C5bLCv6s.js} +34 -13
  196. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  197. package/dist/workspace-8uHtBSve.js +1 -0
  198. package/dist/{workspace-BRxjfa8N.js → workspace-CBuI9Uds.js} +4 -4
  199. package/dist/{workspace-BRxjfa8N.js.map → workspace-CBuI9Uds.js.map} +1 -1
  200. package/package.json +1 -1
  201. package/skills/eval-rubric/SKILL.md +11 -0
  202. package/dist/app-files-ChbvRx3P.js.map +0 -1
  203. package/dist/assemble-SKQGW_QF.js.map +0 -1
  204. package/dist/assemble-XBWuK3Ed.js +0 -1
  205. package/dist/attempt-host-BOp8JklD.js.map +0 -1
  206. package/dist/capture-BM_3100g.js.map +0 -1
  207. package/dist/capture-DSK8O5bA.js +0 -1
  208. package/dist/cli-PdYrcoKM.js.map +0 -1
  209. package/dist/cli-spec-D0qyrPZi.js.map +0 -1
  210. package/dist/client-Ct6s6OnF.js.map +0 -1
  211. package/dist/commands-BU6YJCz9.js.map +0 -1
  212. package/dist/config-k6m7XbAz.js.map +0 -1
  213. package/dist/db-DkD-F-X-.js +0 -2
  214. package/dist/debug-BhizgAsU.js.map +0 -1
  215. package/dist/debug-DeM5pswK.js +0 -1
  216. package/dist/docker-51WvhN_X.js.map +0 -1
  217. package/dist/eval-source-Dje3rERU.js.map +0 -1
  218. package/dist/evaluate-BPP9s3s6.js.map +0 -1
  219. package/dist/files-BAt0i00X.js +0 -1
  220. package/dist/files-C2h2Cw4F.js +0 -1
  221. package/dist/files-Ca0d6Uq3.js.map +0 -1
  222. package/dist/files-DmUrRmmL.js +0 -1
  223. package/dist/init-BDEOopK7.js +0 -1
  224. package/dist/matchers-DT2TaGix.js.map +0 -1
  225. package/dist/model-CTnyvh7H.js.map +0 -1
  226. package/dist/model-D-SWrLZG.js +0 -1
  227. package/dist/object-CLf2-tNq.js.map +0 -1
  228. package/dist/pairs-puYi-zj2.js.map +0 -1
  229. package/dist/patch-DKJiC5Sq.js.map +0 -1
  230. package/dist/plan-DdWkM6SW.js.map +0 -1
  231. package/dist/plugins-D-CP6mMd.js.map +0 -1
  232. package/dist/query-Clm2mNSm.js +0 -1
  233. package/dist/questions-BRhU8MzB.js.map +0 -1
  234. package/dist/remarks-BwJ8pcTQ.js +0 -1
  235. package/dist/remarks-C7i2LMRb.js.map +0 -1
  236. package/dist/remarks-dA8LyoPc.js.map +0 -1
  237. package/dist/resolve--rQZ8u6Z.js +0 -1
  238. package/dist/runtime-d56Gg3J5.js.map +0 -1
  239. package/dist/server-BOoH10J8.js.map +0 -1
  240. package/dist/server-DpuVAbrO.js +0 -1
  241. package/dist/source-BFd6f0-R.js +0 -1
  242. package/dist/src-D91Hd6hT.js.map +0 -1
  243. package/dist/src-DH-9IO7f.js +0 -1
  244. package/dist/src-LD1BAJFa.js +0 -2
  245. package/dist/state-B5PLm5qY.js +0 -1
  246. package/dist/suite-22Kt9Grx.js +0 -2
  247. package/dist/suite-Co8cwelb.js.map +0 -1
  248. package/dist/summary-DLFHVoux.js.map +0 -1
  249. package/dist/views-BOf8qEL8.js +0 -2
  250. package/dist/worker-entry-CJtdsKBP.js.map +0 -1
  251. package/dist/workspace-DDHzkMjm.js +0 -1
@@ -187,7 +187,7 @@ type FileRef = {
187
187
  sha256: string;
188
188
  bytes: number;
189
189
  };
190
- /** A generic attempt file from `c.capture()` (and later video). */
190
+ /** A generic attempt file from `c.capture()`. */
191
191
  type ArtifactRef = FileRef & {
192
192
  name: string;
193
193
  mediaType: 'image/png';
@@ -217,15 +217,18 @@ type GitInfo = {
217
217
  */
218
218
  sha: string;
219
219
  branch: string | null;
220
- /** Uncommitted changes, outside `.reeval/` and the history folder, when the run started. */
220
+ /** Uncommitted changes, outside the `.reeval/` directory and the history folder, when the run started. */
221
221
  dirty: boolean;
222
222
  /** sha256 of the uncommitted changes (tracked and untracked, as a binary patch), when dirty. */
223
223
  diffHash: string | null;
224
- /** Those changes as `worktree.patch` in the artifact dir; apply on `sha` to reproduce. */
224
+ /**
225
+ * Those changes are stored as the `worktree.patch` file in the artifact directory. Applying it
226
+ * to `sha` reproduces the workspace.
227
+ */
225
228
  worktreePatch: FileRef | null;
226
229
  /**
227
- * Why a dirty run has no `worktree.patch`: too large, artifacts: 'none', or it looked like it held
228
- * a secret (then where, and which rule, never the value).
230
+ * This explains why a dirty run has no `worktree.patch` file: it was too large, `artifacts` was
231
+ * `'none'`, or it looked like it held a secret (then where, and which rule, never the value).
229
232
  */
230
233
  worktreePatchNote: string | null;
231
234
  commitTime: string;
@@ -267,7 +270,7 @@ type JudgmentRecord = {
267
270
  /** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
268
271
  costUsd?: number;
269
272
  /**
270
- * `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
273
+ * `<index>:<sha256(question) prefix>`: the same question at the same place in the evaluation, so a
271
274
  * verdict can be followed across runs (and an edited question starts a new series). `<index>`
272
275
  * is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
273
276
  * whichever finishes first.
@@ -279,14 +282,14 @@ type JudgmentRecord = {
279
282
  artifacts?: JudgeArtifactRef[];
280
283
  };
281
284
  type AttemptFiles = {
282
- /** Each environment setup step's command, exit code, duration and output. */
285
+ /** Each environment setup step's command, exit code, duration, and output. */
283
286
  setupLog?: FileRef;
284
287
  transcript?: FileRef;
285
288
  /** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
286
289
  rawTranscript?: FileRef;
287
290
  diff?: FileRef;
288
291
  judgeInput?: FileRef;
289
- /** PNGs from `c.capture()`, under `artifacts/<name>.png`. */
292
+ /** PNGs from `c.capture()`, under the `artifacts/<name>.png` path. */
290
293
  artifacts?: ArtifactRef[];
291
294
  };
292
295
  type ContextFile = {
@@ -301,16 +304,22 @@ type UpstreamCount = {
301
304
  type AttemptRecord = {
302
305
  /** `<eval key> # <cell key>#<n>`: unique in the run. */
303
306
  key: string;
304
- /** `<cell key>#<n>`: unique in its eval. */
307
+ /** `<cell key>#<n>`: unique in its evaluation. */
305
308
  id: string;
306
309
  /** The cell's key; see `EvalEntry.cells`. */
307
310
  cell: string;
308
311
  /** Run number within the cell, from 1. */
309
312
  n: number;
313
+ /**
314
+ * In a `runs: { bestOf }` cell: `added` when it ran past the majority's first attempts (the
315
+ * earlier ones split, or an infra error left them short), `skipped` when it didn't run because
316
+ * the earlier ones already had a majority. Unset on the attempts that always run.
317
+ */
318
+ bestOf?: 'added' | 'skipped';
310
319
  /** Start order across the run, from 1; null for attempts that never started. */
311
320
  seq: number | null;
312
321
  startedAt: string | null;
313
- /** The image the container started from, in `stack.images`; null for the fake sandbox. */
322
+ /** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
314
323
  imageId: string | null;
315
324
  /** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
316
325
  env: {
@@ -318,16 +327,17 @@ type AttemptRecord = {
318
327
  secrets: string[];
319
328
  brokered: string[];
320
329
  };
321
- /** Skills, rules and MCP config visible to the agent in its sandbox, by content hash. */
330
+ /** Skills, rules, and MCP config visible to the agent in its sandbox, by content hash. */
322
331
  context: ContextFile[];
323
332
  /** The egress policy and, when the sandbox records it, the hosts contacted and refused; null if it never ran. */
324
333
  egress: EgressRecord | null;
325
334
  /**
326
- * The models the harness's own API calls were served by, as the egress gateway saw them: named
327
- * in the request (e.g. `gemini-3.8-flash` for the id `gemini-3.8-flash-low`, plus any the agent
328
- * uses on the side) or, for Codex, in the response, with its effort
329
- * (`gpt-5.6-luna[effort=medium]`); null when neither names one (Cursor) or nothing
330
- * recorded them.
335
+ * The models served by the harness's API calls, as observed by the egress gateway. Values may
336
+ * come from requests or responses and include reported effort; null when no model was named or
337
+ * recorded. The gateway reads the request URL for Antigravity, Goose, pi, and OpenCode on a
338
+ * `google/` model, so a request for `<model>-low` records `<model>`, and the API response for
339
+ * Codex, Grok Build, and OpenCode on another provider. Null for Cursor and Claude Code, whose
340
+ * API requests name no model.
331
341
  */
332
342
  servedModels: string[] | null;
333
343
  /**
@@ -353,7 +363,7 @@ type AttemptRecord = {
353
363
  */
354
364
  effort: string | null;
355
365
  /**
356
- * The reasoning effort the harness's API said it ran at, from `servedModels` /
366
+ * The reasoning effort the harness's API said it ran at, from `servedModels` or
357
367
  * each turn's `model` (`medium` from `gpt-5.6-luna[effort=medium]`); null when those
358
368
  * labels name none. Fast (`[fast=true]`) is not this.
359
369
  */
@@ -362,7 +372,8 @@ type AttemptRecord = {
362
372
  error: AttemptError | null;
363
373
  durationMs: number;
364
374
  /**
365
- * `text` is the agent's final message, cut at 8 KB; the whole of it is in `transcript.jsonl`.
375
+ * `text` is the agent's final message, cut at 8 KB; the whole of it is in the `transcript.jsonl`
376
+ * file.
366
377
  * `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
367
378
  */
368
379
  turns: Array<{
@@ -385,8 +396,8 @@ type AttemptRecord = {
385
396
  /** Tokens across the attempt's turns; null when the harness reported none. */
386
397
  usage: TokenUsage | null;
387
398
  /**
388
- * Tokens the capture vision loop used. Infrastructure only: never folded
389
- * into `usage`, effort, compare or lift.
399
+ * Tokens the capture vision loop used. Infrastructure only: never included
400
+ * in `usage`, effort, compare, or lift.
390
401
  */
391
402
  captureUse?: {
392
403
  model: string;
@@ -394,7 +405,7 @@ type AttemptRecord = {
394
405
  usage: TokenUsage | null;
395
406
  };
396
407
  timings: PhaseTimings;
397
- /** Counts from the transcript, so a run's shape is visible without reading it. `webSearches` includes refused searches. */
408
+ /** Counts from the transcript, so the attempt's activity is available without reading the transcript. `webSearches` includes refused searches. */
398
409
  activity: {
399
410
  tools: number;
400
411
  fileEdits: number;
@@ -419,6 +430,22 @@ type AttemptRecord = {
419
430
  * names. Absent when there were none.
420
431
  */
421
432
  setupProcesses?: SetupProcess[];
433
+ /**
434
+ * Copies of the attempt's workspace in its directory in `.reeval/runs` on the machine that ran
435
+ * it (never in the history folder), in the order taken: `turn-<n>` after each `c.send` with
436
+ * `snapshot: 'turns'`, then `final` after the checks. Each has its form, path and size, its
437
+ * file count, the directories left out by name, the files whose secret-looking values were
438
+ * masked and the binary ones left out for holding one, or why it wasn't taken. Absent with
439
+ * `snapshot: false`, and when the sandbox was gone first (a timeout or a cancel).
440
+ */
441
+ snapshots?: Snapshot[];
442
+ /**
443
+ * Each `c.serve` call that listened, in call order: `{ command, port, portEnv, detected,
444
+ * address, readyMs }`. `port` is where it listens; `detected` is true when that isn't the
445
+ * `PORT` it was given (`portEnv`), so reeval took the one port that started listening after
446
+ * the command did. Absent when the attempt served nothing.
447
+ */
448
+ serves?: ServeRecord[];
422
449
  /**
423
450
  * Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
424
451
  * `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
@@ -433,9 +460,9 @@ type AttemptRecord = {
433
460
  /** Every try, in order, when it was given `retries`; its log has each try's output. */
434
461
  tries?: ExecTry[];
435
462
  }>;
436
- /** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
463
+ /** Every `expect` and `expect.require` matcher call that settled, in call order; unset with none. */
437
464
  assertions?: AssertionRecord[];
438
- /** Files under a dependency directory (node_modules/, .venv/, …) collapse to one entry with `count`. */
465
+ /** Files under a dependency directory, such as the `node_modules/` directory or `.venv/` directory, collapse to one entry with `count`. */
439
466
  changedFiles: Array<FileChange & {
440
467
  count?: number;
441
468
  }>;
@@ -451,7 +478,7 @@ type AttemptRecord = {
451
478
  type EvalEntry = {
452
479
  /** `file › group › … › name`, as shown. */
453
480
  id: string;
454
- /** What the eval is matched on across runs; currently the same as `id`. */
481
+ /** Used to match an evaluation across runs; equal to `id`. */
455
482
  key: string;
456
483
  /** Directory name inside the artifact directory, from `key`. */
457
484
  slug: string;
@@ -461,8 +488,8 @@ type EvalEntry = {
461
488
  /** Results are comparable only when fingerprints match. */
462
489
  fingerprint: string;
463
490
  /**
464
- * The eval file, by content hash: where the eval lives. Records from before `source` also list
465
- * the local files it imported, and were fingerprinted over them.
491
+ * The evaluation file, by content hash: where the evaluation lives. Records from before `source`
492
+ * also list the local files it imported, and were fingerprinted over them.
466
493
  */
467
494
  sources: Array<{
468
495
  path: string;
@@ -476,12 +503,12 @@ type EvalEntry = {
476
503
  /** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
477
504
  revision?: Revision;
478
505
  tags: string[];
479
- /** The baseline environment, when the eval uses `compare`. */
506
+ /** The baseline environment, when the evaluation uses `compare`. */
480
507
  compare?: EvalCompareMeta;
481
508
  /**
482
- * Over gated cells: `fail` if any failed, else `infra_error` if any is (a cell is `infra_error`
483
- * only when nothing counted), else `skipped` if all were, else `pass`. An ungated cell's
484
- * failures don't fail the eval.
509
+ * Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
510
+ * `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
511
+ * cell's failures don't fail the evaluation.
485
512
  */
486
513
  outcome: Outcome;
487
514
  /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
@@ -494,9 +521,9 @@ type EvalEntry = {
494
521
  * The eval's pairwise questions (those with `across`), answered after its attempts by votes,
495
522
  * one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
496
523
  * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
497
- * (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
498
- * `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
499
- * masked and screenshots as refs; unset where the eval's files weren't kept.
524
+ * (rates by cut, votes, ties, a failed call, and a sample of reasons). Each answer's `input` is
525
+ * the `<eval-slug>/questions/<n>.json` file: every call its model was sent, per pair and order,
526
+ * with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
500
527
  */
501
528
  answers?: CompareAnswer[];
502
529
  /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
@@ -513,7 +540,7 @@ type FamilyInterval = {
513
540
  hi: number;
514
541
  level: 0.95;
515
542
  /**
516
- * `wilson` / `newcombe-paired`: one counted attempt a case and side, so each case is one
543
+ * `wilson` or `newcombe-paired`: one counted attempt a case and side, so each case is one
517
544
  * Bernoulli trial (or pair). `t-cases`: several, so a t interval over the cases' pass@1, with
518
545
  * attempts within a case never counted as independent.
519
546
  */
@@ -588,17 +615,17 @@ type FamilyLift = {
588
615
  outcome: 'pass' | 'fail';
589
616
  effort?: FamilyEffort;
590
617
  };
591
- /** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an eval. */
618
+ /** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an evaluation. */
592
619
  type FamilyRecord = {
593
620
  /** `file › group › … › template`. */
594
621
  id: string;
595
622
  file: string;
596
623
  group: string[];
597
- /** The name template, e.g. `fixes $id`. */
624
+ /** The name template, for example `fixes $id`. */
598
625
  name: string;
599
626
  /** Cases in the list. */
600
627
  cases: number;
601
- /** Cases with an eval in this run; fewer when a filter or tag left some out. */
628
+ /** Cases with an evaluation in this run; fewer when a filter or tag left some out. */
602
629
  ran: number;
603
630
  /** One per cell. */
604
631
  rates: FamilyRate[];
@@ -643,7 +670,7 @@ type FingerprintParts = {
643
670
  environment: string | null;
644
671
  image: string | null;
645
672
  agentCli: string | null;
646
- /** Over the judge's model, threshold and version, and the config's `state` when it sets one. */
673
+ /** Over the judge's model, threshold and version. */
647
674
  judge: string;
648
675
  /** Over the attempt's context files. */
649
676
  context: string;
@@ -655,7 +682,7 @@ type FingerprintParts = {
655
682
  */
656
683
  gate?: string;
657
684
  };
658
- /** One harness and environment of an eval, and how its attempts went. */
685
+ /** One harness and environment of an evaluation, and how its attempts went. */
659
686
  type CellRecord = Cell & {
660
687
  /** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
661
688
  fingerprint: string;
@@ -678,10 +705,15 @@ type CellRecord = Cell & {
678
705
  * Fast is not this.
679
706
  */
680
707
  effort: string | null;
681
- /** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the eval. */
708
+ /** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the evaluation. */
682
709
  gate: CellGate;
683
710
  /** Environment keys the variant replaced on the control's (or a child on its parent's). */
684
711
  overrides: string[];
712
+ /**
713
+ * Set by `runs: { bestOf }`: the cell stopped once a majority of this many attempts agreed, and
714
+ * gates on that majority. `runs` still counts every planned attempt, the skipped ones included.
715
+ */
716
+ bestOf?: number;
685
717
  runs: number;
686
718
  pass: number;
687
719
  fail: number;
@@ -697,7 +729,7 @@ type CellRecord = Cell & {
697
729
  outcome: Outcome;
698
730
  lift?: Lift;
699
731
  /**
700
- * Each check (assertion label) seen in any attempt of the eval, over this cell's counted
732
+ * Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
701
733
  * attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
702
734
  */
703
735
  checks?: CellCheck[];
@@ -710,10 +742,10 @@ type CheckTally = {
710
742
  notRun: number;
711
743
  };
712
744
  /**
713
- * One check over a cell's counted attempts. `verdict` (variant cells), over the attempts that
714
- * reached the check: `discriminates` when it passes on every one of one side and none of the
715
- * other; `both-pass` or `both-fail` when it does the same on both sides, so it can't tell them
716
- * apart; `mixed` otherwise. No verdict when a side never reached it: not run is not a fail.
745
+ * One check over a cell's counted attempts. `verdict` (variant cells) is `discriminates` when
746
+ * every answered instance passes on one side and fails on the other, `both-pass` when every
747
+ * answered instance passes on both sides, `both-fail` when every answered instance fails on both
748
+ * sides, or `mixed` otherwise. It is absent when either side has no pass or fail.
717
749
  */
718
750
  type CellCheck = CheckTally & {
719
751
  label: string;
@@ -773,7 +805,10 @@ type RunRecord = {
773
805
  harness?: string[];
774
806
  environment?: string;
775
807
  sandbox?: string;
808
+ /** With `bestOf`, the most attempts a cell runs. */
776
809
  runs?: number;
810
+ /** Set by `defaults.runs: { bestOf }`. */
811
+ bestOf?: number;
777
812
  timeout?: string | number;
778
813
  env?: string[];
779
814
  };
@@ -807,8 +842,13 @@ type RunRecord = {
807
842
  config: {
808
843
  sandbox: string;
809
844
  environment: string | null;
810
- /** The config's `runs`; 1 when it sets none, though a compare's cells then run 4 each. */
845
+ /**
846
+ * The config's `runs` (with `bestOf`, the most a cell runs); 1 when it sets none, though a
847
+ * compare's cells then run 4 each.
848
+ */
811
849
  runs: number;
850
+ /** Set by the config's `runs: { bestOf }`. */
851
+ bestOf?: number;
812
852
  judgeModel: string | null;
813
853
  threshold: number;
814
854
  timeoutMs: number;
@@ -827,28 +867,26 @@ type RunRecord = {
827
867
  /** Each pooled `evaluate.each` that ran; unset when none did. */
828
868
  families?: FamilyRecord[];
829
869
  /**
830
- * Errors outside any attempt, redacted: an eval file that didn't load, a rejection or throw no
831
- * attempt awaited, a worker that died. Any one fails the run. Unset when there were none.
870
+ * These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
871
+ * or throw no attempt awaited, or a worker that died. Any one fails the run. Unset when there
872
+ * were none.
832
873
  */
833
874
  errors?: string[];
834
875
  };
835
- /** What ran and graded the agents: the machine, images, harnesses and judge. */
876
+ /** These fields describe what ran and graded the agents: the machine, images, harnesses, and judge. */
836
877
  type Stack = {
837
878
  /** The reeval version that wrote the run. */
838
879
  reeval: string;
839
- /** `process.versions.node`: under Bun or Deno, the Node version they're compatible with. */
880
+ /** `process.versions.node`, or the compatible Node version reported by another runtime. */
840
881
  node: string;
841
- /**
842
- * The runtime reeval ran on, `node`, `bun`, `deno` or another, and its own version. Absent in
843
- * runs recorded before it, which ran on Node.
844
- */
882
+ /** The runtime reeval ran on, `node`, `bun`, `deno`, or another, and its version. */
845
883
  runtime?: {
846
884
  name: string;
847
885
  version: string;
848
886
  };
849
887
  /**
850
- * The language the evals are written in, `typescript` for `*.eval.ts`, and the version of its
851
- * toolchain when the runner reports one. Absent in runs recorded before it.
888
+ * The evaluation language, `typescript` for `*.eval.ts` files, and the version of its toolchain
889
+ * when the runner reports one.
852
890
  */
853
891
  language?: {
854
892
  name: string;
@@ -856,16 +894,17 @@ type Stack = {
856
894
  };
857
895
  os: string;
858
896
  arch: string;
859
- /** Every image an attempt started from. `id` is content-addressed; `repoDigests` only exist once pushed. */
897
+ /** Every image an attempt started from. `id` is content-addressed; `repoDigests` exist only after the image is pushed. */
860
898
  images: Array<AttemptImage & {
861
899
  repoDigests: string[];
862
900
  }>;
863
901
  /** Exact versions of the packages that run and grade the agent. */
864
902
  packages: Record<string, string>;
865
903
  /**
866
- * Each harness: its configured `model` and requested `effort`, `resolvedModels` (what it
867
- * reported serving, with the variant it resolved), and `resolvedEffort` (reasoning-effort
868
- * suffixes those labels named). Fast (`[fast=true]`) is not `resolvedEffort`.
904
+ * For each harness, the record includes its configured `model` and requested `effort`,
905
+ * `resolvedModels` (what it reported serving, with the variant it resolved), and
906
+ * `resolvedEffort` (reasoning-effort suffixes those labels named). Fast (`[fast=true]`) is not
907
+ * `resolvedEffort`.
869
908
  */
870
909
  harnesses: Array<{
871
910
  id: string;
@@ -901,12 +940,12 @@ type Stack = {
901
940
  threshold: number;
902
941
  /** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
903
942
  version: string;
904
- /** What the judge sees by default, every key resolved; absent when the config and profile leave `state` at its defaults. */
943
+ /** The config's default judge inputs, every key resolved. Unset when the config names none, which reeval requires: each `c.judge()` call names its own. */
905
944
  state?: ResolvedJudgeStateOptions;
906
945
  };
907
946
  /** Union of every attempt's context files. */
908
947
  context: ContextFile[];
909
- /** What eval fingerprints hash; fingerprints from different scopes aren't comparable. */
948
+ /** What evaluation fingerprints hash; fingerprints from different scopes aren't comparable. */
910
949
  fingerprintScope: string[];
911
950
  };
912
951
  /** The parts of a run's stack that `reeval compare` checks for drift. */
@@ -1108,16 +1147,21 @@ type Wait = CaptureWait | {
1108
1147
  type CaptureInstruction = string | {
1109
1148
  wait: Wait;
1110
1149
  };
1150
+ /** The keys a capture's driver presses. In a focused `<select>`, the arrows change the choice. */
1151
+ type CaptureKey = 'Tab' | 'Enter' | 'Escape' | 'Backspace' | 'Space' | 'ArrowUp' | 'ArrowDown' | 'ArrowLeft' | 'ArrowRight';
1111
1152
  type CaptureUseAction = {
1112
1153
  type: 'click';
1113
1154
  x: number;
1114
1155
  y: number;
1115
- } | {
1156
+ } |
1157
+ /** `replace` selects what the focused field holds first, so the text replaces it. */
1158
+ {
1116
1159
  type: 'type';
1117
1160
  text: string;
1161
+ replace?: boolean;
1118
1162
  } | {
1119
1163
  type: 'key';
1120
- key: 'Tab' | 'Enter';
1164
+ key: CaptureKey;
1121
1165
  } | {
1122
1166
  type: 'scroll';
1123
1167
  x: number;
@@ -1152,7 +1196,7 @@ type CaptureUse = {
1152
1196
  type CaptureOptions = {
1153
1197
  /** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
1154
1198
  viewport?: Viewport;
1155
- /** How long to poll before failing. Default `'60s'`. */
1199
+ /** How long to poll before failing; a 502, 503 or 504 is polled past until then. Default `'60s'`. */
1156
1200
  timeout?: Duration;
1157
1201
  /** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
1158
1202
  browser?: boolean;
@@ -1166,7 +1210,7 @@ type CaptureOptions = {
1166
1210
  type BackgroundHandle = {
1167
1211
  kill(): Promise<void>;
1168
1212
  };
1169
- /** `c.serve`: a background process plus the port it was given or allocated. */
1213
+ /** `c.serve`: a background process plus the port it listens on. */
1170
1214
  type ServeHandle = BackgroundHandle & {
1171
1215
  port: number;
1172
1216
  };
@@ -1279,10 +1323,10 @@ type JudgeState = JudgeJsonState | Array<{
1279
1323
  type: 'json';
1280
1324
  value: JudgeJsonState;
1281
1325
  } | JudgeFilePart>;
1282
- /** A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`. */
1326
+ /** A capture this judge call sent as a file part. Bytes live in the `artifacts/<name>.png` file. */
1283
1327
  type JudgeArtifactRef = {
1284
1328
  name: string;
1285
- /** `artifacts/<name>.png` under the attempt directory. */
1329
+ /** The `artifacts/<name>.png` path under the attempt directory. */
1286
1330
  path: string;
1287
1331
  sha256: string;
1288
1332
  bytes: number;
@@ -1314,11 +1358,6 @@ type JudgeOptions = {
1314
1358
  providerOptions?: Record<string, Record<string, JSONValue>>;
1315
1359
  /** The most judge calls to this model's provider at once, across the run. */
1316
1360
  maxConcurrent?: number;
1317
- /**
1318
- * The judge's state: what the judge is given beside the question, not the workspace's or the
1319
- * attempt's state. Merged key by key down the cascade.
1320
- */
1321
- state?: JudgeStateOptions;
1322
1361
  /**
1323
1362
  * The gateway the judge's calls go through, by name or by its helper. It skips the config's
1324
1363
  * `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
@@ -1326,43 +1365,61 @@ type JudgeOptions = {
1326
1365
  gateway?: GatewayName | GatewayDescriptor;
1327
1366
  };
1328
1367
  /**
1329
- * What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
1330
- * keeps its default, which is what the judge gets with no `state` option at all.
1368
+ * What one `c.judge()` call shows the judge beside the question. Every input is off unless the
1369
+ * call turns it on, and a call turns on at least one; each is a key of the state in
1370
+ * `judge-input.json`, or the captures sent as file parts.
1331
1371
  */
1332
- type JudgeStateOptions = {
1333
- /** The graded turn's prompt. Default true. */
1372
+ type JudgeInputs = {
1373
+ /** The graded turn's prompt. */
1334
1374
  task?: boolean;
1335
- /** The graded turn's final message, or `on`'s value. Default true. */
1375
+ /** The graded turn's final message, or `on`'s value. */
1336
1376
  output?: boolean;
1337
- /** `status path` lines for the changed files, under `diff`'s patterns. Default true. */
1377
+ /** `status path` lines for the changed files, under `diff`'s patterns. */
1338
1378
  files?: boolean;
1339
1379
  /**
1340
1380
  * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
1341
- * the last matching pattern wins and `!` excludes. Default true.
1381
+ * the last matching pattern wins and `!` excludes.
1342
1382
  */
1343
1383
  diff?: boolean | string[];
1344
1384
  /**
1345
1385
  * Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
1346
1386
  * patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
1347
1387
  * the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
1348
- * These files leave the diff. Default none.
1388
+ * These files leave the diff.
1349
1389
  */
1350
1390
  contents?: string[];
1351
1391
  /**
1352
1392
  * The graded turn's transcript as text lines: `true` is messages and tool calls with tool
1353
1393
  * output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
1354
- * Default false.
1355
1394
  */
1356
1395
  transcript?: boolean | 'tools' | 'full';
1396
+ /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
1397
+ checks?: boolean | string[];
1398
+ /**
1399
+ * Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
1400
+ * capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
1401
+ */
1402
+ artifacts?: true | Array<Capture | string>;
1403
+ };
1404
+ /** What a judge call's inputs read: which turn, and whether the turns before it too. */
1405
+ type JudgeSubject = {
1406
+ /** The turn `task`, `output`, `transcript` and `diff` read instead of the latest, or a value used as the output. */
1407
+ on?: Turn | JSONValue;
1357
1408
  /**
1358
1409
  * `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
1359
1410
  * `transcript`) transcript. Default `'last'`.
1360
1411
  */
1361
1412
  turns?: 'last' | 'all';
1362
- /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
1363
- checks?: boolean | string[];
1364
1413
  };
1365
- /** `JudgeStateOptions` with every key set: what one judge call saw. */
1414
+ /** One input turned on: `true`, a mode or a non-empty selection, not `false`. */
1415
+ type OneInput<I, K extends keyof I> = { [P in K]-?: Exclude<I[P], false | undefined>; };
1416
+ /** At least one of `I`'s keys turned on. */
1417
+ type AtLeastOneInput<I> = { [K in keyof I]-?: OneInput<I, K>; }[keyof I];
1418
+ /** `c.judge()`'s options, before the at-least-one-input rule that `JudgeCallOptions` adds. */
1419
+ type JudgeCallFields = JudgeOptions & JudgeInputs & JudgeSubject;
1420
+ /** `c.judge()`'s options: the judge options, the inputs (at least one) and what they read. */
1421
+ type JudgeCallOptions = JudgeCallFields & AtLeastOneInput<JudgeInputs>;
1422
+ /** The inputs one judge call saw, every key set (`false` when off); `artifacts` is recorded apart. */
1366
1423
  type ResolvedJudgeStateOptions = {
1367
1424
  task: boolean;
1368
1425
  output: boolean;
@@ -1373,16 +1430,6 @@ type ResolvedJudgeStateOptions = {
1373
1430
  turns: 'last' | 'all';
1374
1431
  checks: boolean | string[];
1375
1432
  };
1376
- type JudgeCallOptions = JudgeOptions & {
1377
- /** What to grade instead of the latest turn: another turn, or a value used as the output. */
1378
- on?: Turn | JSONValue;
1379
- /**
1380
- * Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
1381
- * `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
1382
- * captures; a provider that cannot take files fails the call.
1383
- */
1384
- artifacts?: true | Array<Capture | string>;
1385
- };
1386
1433
  type Verdict = {
1387
1434
  verdict: 'pass' | 'fail';
1388
1435
  /** The judge's probability that the answer is yes, from 0 to 1. */
@@ -1418,10 +1465,11 @@ type AttemptContext = {
1418
1465
  /** Sends the next turn of this eval's agent session. */
1419
1466
  send(prompt: string): Promise<Turn>;
1420
1467
  /**
1421
- * Asks the judge a yes/no question about the latest turn (or `options.on`).
1422
- * `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
1468
+ * Asks the judge a yes/no question about what `options` turns on, and nothing else: at least
1469
+ * one of `task`, `output`, `files`, `diff`, `contents`, `transcript`, `checks` or `artifacts`.
1470
+ * They read the latest turn, or `options.on`.
1423
1471
  */
1424
- judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1472
+ judge(question: string, options: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1425
1473
  /**
1426
1474
  * Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
1427
1475
  * many extra times, `interval` ms apart; it resolves with the last try.
@@ -1436,8 +1484,10 @@ type AttemptContext = {
1436
1484
  background?: boolean;
1437
1485
  }): Promise<ExecResult | BackgroundHandle>;
1438
1486
  /**
1439
- * Starts `command` in the background and returns its port. A given `port` is used; omit it
1440
- * and reeval allocates one. Sets `PORT` so a bash string can use `$PORT`.
1487
+ * Starts `command` in the background with `PORT` set to `port`, or to a port reeval allocates,
1488
+ * and returns once it listens. Without `port`, a command that listens on one new port instead
1489
+ * (Vite's 5173) gets that one; several new ports, none of them `PORT`, or an exit first, throw
1490
+ * with its last output.
1441
1491
  */
1442
1492
  serve(command: string, port?: number): Promise<ServeHandle>;
1443
1493
  /**
@@ -2378,9 +2428,11 @@ type CompareOptions<E = Environment> = {
2378
2428
  * With `across`, it picks between two attempts that differ only in that factor, in both orders;
2379
2429
  * without it, it's asked of each attempt on its own. Without a model, the cascaded
2380
2430
  * `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
2381
- * `state` and `artifacts` choose what it's shown of each attempt.
2431
+ * It's shown only the inputs it turns on, at least one.
2382
2432
  */
2383
- type Question = string | {
2433
+ type Question = QuestionFields & AtLeastOneInput<QuestionInputs>;
2434
+ /** A question's options, before the at-least-one-input rule that `Question` adds. */
2435
+ type QuestionFields = QuestionInputs & {
2384
2436
  question: string;
2385
2437
  /** `'provider/model'` that answers it, over the cascaded `question.model`. */
2386
2438
  model?: string;
@@ -2391,43 +2443,39 @@ type Question = string | {
2391
2443
  across?: PairFactor | PairFactor[];
2392
2444
  /** The answers a question asked of each attempt picks from, with a short reason. */
2393
2445
  choices?: string[];
2394
- /** What it's shown of each attempt, over the cascaded `question.state`, key by key. */
2395
- state?: QuestionStateOptions;
2396
- /** Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
2397
- artifacts?: true | string[];
2446
+ /**
2447
+ * `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
2448
+ * transcript; `'last'`: the last turn only. Default `'all'`.
2449
+ */
2450
+ turns?: 'last' | 'all';
2398
2451
  };
2399
2452
  /**
2400
- * What a question is shown of each attempt: the keys of `judge.state` but `contents` (the
2401
- * attempts' workspaces are gone when questions are asked). An unset key keeps its default: a
2402
- * pairwise question shows the task and the diff; one asked of each attempt also shows the final
2403
- * message and the transcript.
2453
+ * What a question is shown of each attempt: `c.judge()`'s inputs but `contents` (the attempts'
2454
+ * workspaces are gone when questions are asked). Every input is off unless the question turns it
2455
+ * on, and a question turns on at least one.
2404
2456
  */
2405
- type QuestionStateOptions = {
2406
- /** The prompts the attempt was sent, once for both sides when they match. Default true. */
2457
+ type QuestionInputs = {
2458
+ /** The prompts the attempt was sent, once for both sides when they match. */
2407
2459
  task?: boolean;
2408
- /** The final message: the agent's own account of its work. Default false in a pair, true per attempt. */
2460
+ /** The final message: the agent's own account of its work. */
2409
2461
  output?: boolean;
2410
- /** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
2462
+ /** `status path` lines for the changed files, under `diff`'s patterns. */
2411
2463
  files?: boolean;
2412
2464
  /**
2413
2465
  * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
2414
- * the last matching pattern wins and `!` excludes. Default true.
2466
+ * the last matching pattern wins and `!` excludes.
2415
2467
  */
2416
2468
  diff?: boolean | string[];
2417
2469
  /**
2418
2470
  * The transcript as text lines: `true` is messages and tool calls with tool output clipped,
2419
2471
  * `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
2420
- * harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per
2421
- * attempt.
2472
+ * harness's tools, so a pair of two harnesses isn't blind.
2422
2473
  */
2423
2474
  transcript?: boolean | 'tools' | 'full';
2424
- /**
2425
- * `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
2426
- * transcript; `'last'`: the last turn only. Default `'all'`.
2427
- */
2428
- turns?: 'last' | 'all';
2429
- /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
2475
+ /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
2430
2476
  checks?: boolean | string[];
2477
+ /** Each attempt's own captures, by name; `true` sends all of them. */
2478
+ artifacts?: true | string[];
2431
2479
  };
2432
2480
  /** `QuestionStateOptions` with every key set: what each side of one question showed. */
2433
2481
  type ResolvedQuestionStateOptions = {
@@ -2439,14 +2487,10 @@ type ResolvedQuestionStateOptions = {
2439
2487
  turns: 'last' | 'all';
2440
2488
  checks: boolean | string[];
2441
2489
  };
2442
- /** Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key. */
2490
+ /** Defaults for every question below it. What a question is shown is its own, never a default. */
2443
2491
  type QuestionOptions = {
2444
2492
  /** `'provider/model'` that answers every question below it without its own `model`. */
2445
2493
  model?: string;
2446
- /** What it's shown of each attempt. */
2447
- state?: QuestionStateOptions;
2448
- /** Each attempt's own captures, by name; `true` sends all of them. Default none. */
2449
- artifacts?: true | string[];
2450
2494
  };
2451
2495
  /** A matrix factor a pairwise question varies; the others are held. */
2452
2496
  type PairFactor = 'environment' | 'harness';
@@ -2484,7 +2528,7 @@ type CompareAnswer = {
2484
2528
  model: string;
2485
2529
  /** The model id the provider says served the calls, when it reports one. */
2486
2530
  resolvedModel?: string;
2487
- /** What each side showed, every key resolved; unset on answers from before questions took `state`, which showed the defaults. */
2531
+ /** The inputs each side showed, every key resolved; unset on answers recorded before reeval wrote it, which showed the task and the diff. */
2488
2532
  stateOptions?: ResolvedQuestionStateOptions;
2489
2533
  /** The captures it sent of each side, as asked: `true` for all, or their names. */
2490
2534
  artifacts?: true | string[];
@@ -2537,8 +2581,21 @@ type GatePolicy = 'all' | {
2537
2581
  type CellGate = GatePolicy | 'none';
2538
2582
  /** A compare object: `baseline` and `variants`. Same as `CompareOptions`. */
2539
2583
  type CompareSpec<E = Environment> = CompareOptions<E>;
2584
+ type OddDigit = '1' | '3' | '5' | '7' | '9';
2585
+ /**
2586
+ * `N` when it's an odd whole number of 3 or more, else a sentence the compiler shows as the type
2587
+ * it expected. A `number` that isn't a literal stays `number`: only planning can check it.
2588
+ */
2589
+ type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${string}` | `${string}.${string}` | `${string}e${string}` | '1' ? 'an odd whole number of 3 or more, e.g. 3 or 5' : `${N}` extends `${string}${OddDigit}` ? N : 'an odd whole number of 3 or more, e.g. 3 or 5';
2590
+ /**
2591
+ * Attempts per cell: a fixed count, or `{ bestOf }`, an odd number of 3 or more, which stops a
2592
+ * cell once one side (pass or fail) has a majority: `{ bestOf: 5 }` runs 3 to 5 attempts.
2593
+ */
2594
+ type Runs<N extends number = number> = number | {
2595
+ bestOf: OddAtLeast3<N>;
2596
+ };
2540
2597
  /** Options other than `environment` and `compare`, which are exclusive of each other. */
2541
- type EvalOptionFields = {
2598
+ type EvalOptionFields<N extends number = number> = {
2542
2599
  /** Which agents run the eval; each harness is its own cell. */
2543
2600
  harness?: HarnessOption;
2544
2601
  /**
@@ -2547,11 +2604,12 @@ type EvalOptionFields = {
2547
2604
  */
2548
2605
  sandbox?: SandboxDescriptor;
2549
2606
  /**
2550
- * Attempts per cell. Unset at every level, a compare with variants runs 4 a side, the fewest
2551
- * whose lift can reach a verdict, and anything else runs once, a pooled `evaluate.each` case
2552
- * included.
2607
+ * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
2608
+ * a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
2609
+ * Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
2610
+ * a verdict, and anything else runs once, a pooled `evaluate.each` case included.
2553
2611
  */
2554
- runs?: number;
2612
+ runs?: Runs<N>;
2555
2613
  /** How each cell's attempts decide its outcome; default `'all'`. */
2556
2614
  gate?: GatePolicy;
2557
2615
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
@@ -2565,7 +2623,7 @@ type EvalOptionFields = {
2565
2623
  judge?: JudgeOptions;
2566
2624
  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
2567
2625
  capture?: CaptureOptions;
2568
- /** Defaults for every question: `model`, and `state` merged key by key, down the cascade. */
2626
+ /** The model every question that names none of its own is answered by, down the cascade. */
2569
2627
  question?: QuestionOptions;
2570
2628
  /**
2571
2629
  * What a language model is asked about the attempts, always a list; they never change pass,
@@ -2576,6 +2634,11 @@ type EvalOptionFields = {
2576
2634
  questions?: Question[];
2577
2635
  /** Per attempt. */
2578
2636
  timeout?: Duration;
2637
+ /**
2638
+ * When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
2639
+ * the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
2640
+ */
2641
+ snapshot?: SnapshotOption;
2579
2642
  /** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
2580
2643
  tags?: string[];
2581
2644
  /**
@@ -2622,8 +2685,9 @@ type EnvironmentOrCompare<E> = {
2622
2685
  baseline?: undefined;
2623
2686
  };
2624
2687
  };
2625
- type EvalOptions = EvalOptionFields & EnvironmentOrCompare<Environment>;
2626
- type GroupOptions = EvalOptions;
2688
+ /** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
2689
+ type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
2690
+ type GroupOptions<N extends number = number> = EvalOptions<N>;
2627
2691
  /**
2628
2692
  * `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
2629
2693
  * by name or a fake judge, and environments without `extend`. The same `environment` / compare
@@ -2663,8 +2727,8 @@ type ProfileDefaults = {
2663
2727
  sandbox?: SandboxDescriptor;
2664
2728
  /** What every eval runs in when it names no environment. */
2665
2729
  environment?: Environment;
2666
- /** Attempts per cell. */
2667
- runs?: number;
2730
+ /** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
2731
+ runs?: Runs;
2668
2732
  /** How each cell's attempts decide its outcome. */
2669
2733
  gate?: GatePolicy;
2670
2734
  /** Variables and secrets for the sandbox, merged by name over the config's. */
@@ -2679,6 +2743,8 @@ type ProfileDefaults = {
2679
2743
  capture?: CaptureOptions;
2680
2744
  /** Defaults for every question, merged over the config's. */
2681
2745
  question?: QuestionOptions;
2746
+ /** When each attempt's workspace is copied; merged over the config's. */
2747
+ snapshot?: SnapshotOption;
2682
2748
  };
2683
2749
  /** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
2684
2750
  type ProfileLimit = {
@@ -2755,15 +2821,15 @@ type BlindAttempt = {
2755
2821
  choices?: string[];
2756
2822
  attempt: BlindSide;
2757
2823
  };
2758
- /** A turn of one side, with the keys its question's `state` turned on. */
2824
+ /** A turn of one side, with the inputs its question turned on. */
2759
2825
  type BlindTurn = {
2760
2826
  task?: string;
2761
2827
  output?: string;
2762
2828
  transcript?: string;
2763
2829
  };
2764
2830
  /**
2765
- * One side of a blinded pair, or the attempt a per-attempt question is asked of: only the keys
2766
- * its question's `state` turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
2831
+ * One side of a blinded pair, or the attempt a per-attempt question is asked of: only the inputs
2832
+ * its question turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
2767
2833
  * under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
2768
2834
  */
2769
2835
  type BlindSide = BlindTurn & {
@@ -2817,8 +2883,11 @@ type ReevalConfig = {
2817
2883
  sandbox?: SandboxDescriptor;
2818
2884
  /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
2819
2885
  environment?: Environment;
2820
- /** Attempts per cell; unset, a compare with variants runs 4 a side and anything else once. */
2821
- runs?: number;
2886
+ /**
2887
+ * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
2888
+ * variants runs 4 a side and anything else once.
2889
+ */
2890
+ runs?: Runs;
2822
2891
  /** How each cell's attempts decide its outcome; default `'all'`. */
2823
2892
  gate?: GatePolicy;
2824
2893
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
@@ -2836,6 +2905,11 @@ type ReevalConfig = {
2836
2905
  capture?: CaptureOptions;
2837
2906
  /** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
2838
2907
  question?: QuestionOptions;
2908
+ /**
2909
+ * When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
2910
+ * the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
2911
+ */
2912
+ snapshot?: SnapshotOption;
2839
2913
  /** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
2840
2914
  concurrency?: number;
2841
2915
  /** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
@@ -2920,9 +2994,9 @@ type AttemptError = {
2920
2994
  };
2921
2995
  /**
2922
2996
  * One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
2923
- * reeval's MCP timer, which every stdio server runs through. An agent CLI that runs a process per
2924
- * turn (Antigravity, pi, Codex) starts its servers on every turn; one that keeps a session starts
2925
- * them once, on the first. A server that doesn't answer within 60s fails the attempt as an infra
2997
+ * reeval's MCP timer, which every stdio server runs through. An agent CLI that starts one process
2998
+ * per turn starts its servers on every turn; a CLI that keeps a session starts them on the first
2999
+ * turn only. A server that doesn't answer within 60s fails the attempt as an infra
2926
3000
  * error naming it; one whose client closes it first doesn't. `mcpStartup` is absent when the
2927
3001
  * environment has no stdio MCP server, or the sandbox has no `node` to run the timer.
2928
3002
  */
@@ -2935,7 +3009,7 @@ type McpStartup = {
2935
3009
  */
2936
3010
  ms: number | null;
2937
3011
  };
2938
- /** Where an eval's time went, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
3012
+ /** The duration of each evaluation phase, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
2939
3013
  type PhaseTimings = {
2940
3014
  setupMs: number;
2941
3015
  agentMs: number;
@@ -2945,16 +3019,28 @@ type PhaseTimings = {
2945
3019
  };
2946
3020
  type CellRole = 'control' | 'variant';
2947
3021
  /**
2948
- * One cell of an eval: a harness, an environment (`-` when none is set) and that environment's
2949
- * image (`sandbox` id: tag, dockerfile and install). With `compare`, the control is `baseline`
2950
- * (or the cascaded environment, or an empty `baseline`) and each variant is that environment on
2951
- * its own, including its sandbox.
3022
+ * A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
3023
+ * environment's sandbox id. With `compare`, the control is `baseline` (or the cascaded
3024
+ * environment, or an empty `baseline`) and each variant is that environment on its own,
3025
+ * including its sandbox.
2952
3026
  */
2953
3027
  type Cell = {
3028
+ /** The harness id, the environment name, and the sandbox id, joined with `/`. */
2954
3029
  key: string;
3030
+ /**
3031
+ * The harness id: `id` when set, otherwise `<kind>:<model>` (`<kind>` alone without a model),
3032
+ * with `[effort=<value>]` appended whenever `effort` is set.
3033
+ */
2955
3034
  harness: string;
2956
- /** The environment's own name, also on a variant cell (`with-house-greeting`, with no prefix). */
3035
+ /** The environment's own name, also on a variant cell (`with-skill`, with no prefix). */
2957
3036
  environment: string;
3037
+ /**
3038
+ * The sandbox id is `id` when set, or `fake` for the fake sandbox. A Docker sandbox starts with
3039
+ * the engine (`docker` by default). It adds `:` and the sanitized final component of `image` or
3040
+ * `from`, or for `dockerfile` the sanitized filename (the parent directory name when that
3041
+ * filename is `Dockerfile` and a parent is present). Non-empty `install` steps append `+` and an
3042
+ * eight-character hash.
3043
+ */
2958
3044
  sandbox: string;
2959
3045
  role: CellRole;
2960
3046
  /** The variant's name, for variant cells. */
@@ -2997,9 +3083,9 @@ type EnvironmentRecord = {
2997
3083
  name: string;
2998
3084
  /** The previous `extendEnvironment` layer's name, else null. */
2999
3085
  parent: string | null;
3000
- /** Keys a later layer replaced, e.g. `env.KEY`, `skills.genkit`, `mcp.docs`, `rules`. */
3086
+ /** Keys a later layer replaced, for example `env.KEY`, `skills.typescript`, `mcp.docs`, or `rules`. */
3001
3087
  overrides: string[];
3002
- /** Covers every field but the rules file's path; secret names are included, values never are. The image is the sandbox's. */
3088
+ /** Covers every field but the rules file's path; secret names are included; values never are. The image is the sandbox's. */
3003
3089
  digest: string;
3004
3090
  setup: string[];
3005
3091
  skills: Array<{
@@ -3013,7 +3099,7 @@ type EnvironmentRecord = {
3013
3099
  /** The plugin the skill came with; unset for a skill from `skills`. */
3014
3100
  plugin?: string;
3015
3101
  }>;
3016
- /** Every MCP server, a plugin's as `<plugin>:<server>`. */
3102
+ /** The names of the MCP servers the agent was given. */
3017
3103
  mcp: string[];
3018
3104
  /** The plugins placed, each with what it placed; in the digest. Unset when there are none. */
3019
3105
  plugins?: PluginRecord[];
@@ -3038,9 +3124,9 @@ type EnvironmentRecord = {
3038
3124
  /** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
3039
3125
  egress: EgressPolicy;
3040
3126
  /**
3041
- * The git repository the workspace started as: the repo as written (a local one relative to the
3042
- * project root), the ref and path asked for (null when unset), the commit the ref resolved to
3043
- * and the hash of the tree that was copied in, which is in the digest.
3127
+ * This records the Git repository the workspace started as: the repo as written (a local one
3128
+ * relative to the project root), the ref and path asked for (null when unset), the commit the
3129
+ * ref resolved to, and the hash of the tree that was copied in, which is in the digest.
3044
3130
  */
3045
3131
  workspace?: {
3046
3132
  source: string;
@@ -3057,8 +3143,8 @@ type EnvironmentRecord = {
3057
3143
  * and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
3058
3144
  * can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
3059
3145
  * the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
3060
- * harness always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
3061
- * out.
3146
+ * harness always tries (its CLI's telemetry, say), kept out of `blocked`, so `blocked` contains
3147
+ * only unexpected refusals.
3062
3148
  */
3063
3149
  type EgressRecord = {
3064
3150
  policy: EgressPolicy;
@@ -3096,7 +3182,7 @@ type AttemptImage = {
3096
3182
  /** Content-addressed image id (`sha256:…`). */
3097
3183
  id: string;
3098
3184
  agentCli: string | null;
3099
- /** The harness image `install` and plugin builds were baked onto: `ref` itself without either. */
3185
+ /** The harness image the `install` layers were baked onto: `ref` itself without any. */
3100
3186
  base: string;
3101
3187
  /** The `docker({ install })` commands, in order. */
3102
3188
  install: string[];
@@ -3131,11 +3217,26 @@ type SetupProcess = {
3131
3217
  /** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
3132
3218
  secrets: string[];
3133
3219
  };
3134
- /** One `expect` or `expect.soft` matcher call an attempt made. */
3220
+ /** One `c.serve` call that listened. */
3221
+ type ServeRecord = {
3222
+ /** The command as the eval passed it. */
3223
+ command: string;
3224
+ /** The port it listens on, which the handle and `c.capture` use. */
3225
+ port: number;
3226
+ /** The `PORT` it was given: the port the eval passed, or the one reeval allocated. */
3227
+ portEnv: number;
3228
+ /** It listened on a port other than `PORT`, the one that started listening after it did. */
3229
+ detected: boolean;
3230
+ /** The address it listens on there, e.g. `127.0.0.1`, `::1` or `0.0.0.0`; empty when the sandbox can't tell. */
3231
+ address: string;
3232
+ /** From the command's start until it listened. */
3233
+ readyMs: number;
3234
+ };
3235
+ /** One `expect` or `expect.require` matcher call an attempt made, or a check another language made. */
3135
3236
  type AssertionRecord = {
3136
3237
  /**
3137
- * `expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or
3138
- * the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt.
3238
+ * `label` is the supplied `expect` label. Without one, it is the matcher plus its first
3239
+ * argument or named subject, cut at 60 characters. Repeats get ` #2`, ` #3`, and later suffixes.
3139
3240
  */
3140
3241
  label: string;
3141
3242
  /** The eval gave the label. */
@@ -3144,12 +3245,61 @@ type AssertionRecord = {
3144
3245
  matcher: string;
3145
3246
  /** `error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report). */
3146
3247
  status: 'pass' | 'fail' | 'error';
3248
+ /**
3249
+ * A failure didn't end the eval's body: an `expect`. False for `expect.require` and for a check
3250
+ * that threw, such as Python's `assert`, which end the body when they fail.
3251
+ */
3147
3252
  soft: boolean;
3253
+ /** `path:line` of the call in the eval file, relative to the project root, when reeval saw it. */
3254
+ at?: string;
3148
3255
  /** Why it failed or couldn't answer, cut at 1 KB. */
3149
3256
  message?: string;
3150
3257
  /** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
3151
3258
  durationMs?: number;
3152
3259
  };
3260
+ /**
3261
+ * A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
3262
+ * folder), or why it wasn't.
3263
+ */
3264
+ type Snapshot = {
3265
+ /** `final` after the checks, or `turn-<n>` after the nth `c.send`. */
3266
+ name: string;
3267
+ /** A plain directory, or a gzipped tarball with `snapshot: { compress: true }`. */
3268
+ form: 'dir' | 'tar.gz';
3269
+ /**
3270
+ * Where it is, relative to the run's directory in `.reeval/runs`:
3271
+ * `<eval>/<cell>/<n>/snapshots/<name>` (with `.tar.gz` when compressed); null when it wasn't
3272
+ * taken.
3273
+ */
3274
+ path: string | null;
3275
+ /** Its size on disk: the directory's files, or the tarball; 0 when it wasn't taken. */
3276
+ bytes: number;
3277
+ /** Regular files and symbolic links in it. */
3278
+ files: number;
3279
+ /** Directories left out by name, such as `node_modules/` and `.git/`, at most 50. */
3280
+ excluded: string[];
3281
+ /** Text files whose secret-looking values were masked, at most 50. */
3282
+ masked: string[];
3283
+ /** Binary files left out because they held a secret-looking value, at most 50. */
3284
+ dropped: string[];
3285
+ /**
3286
+ * Why it wasn't taken: `size` when the workspace was over its form's cap, `error` when the
3287
+ * sandbox couldn't archive it; null when it was.
3288
+ */
3289
+ skipped: {
3290
+ cause: 'size' | 'error';
3291
+ message: string;
3292
+ } | null;
3293
+ };
3294
+ /** `snapshot`: the shorthand, or the object that also picks the form. */
3295
+ type SnapshotOption = boolean | 'turns' | SnapshotOptions;
3296
+ /** `snapshot` as an object, merged field by field over the levels above. */
3297
+ type SnapshotOptions = {
3298
+ /** `true` after the checks, `'turns'` after each `c.send` too, `false` never. Default `true`. */
3299
+ when?: boolean | 'turns';
3300
+ /** Write each as `<name>.tar.gz` instead of a directory. Default `false`. */
3301
+ compress?: boolean;
3302
+ };
3153
3303
  /** Which environment ran as the control, so the studio can show it. */
3154
3304
  type EvalCompareMeta = {
3155
3305
  baseline: string;
@@ -3160,16 +3310,20 @@ type EvalCompareMeta = {
3160
3310
  from?: 'environment' | 'empty';
3161
3311
  };
3162
3312
  /**
3163
- * A sandbox with more privilege than the default `reeval` user: `docker({ root: true })`, or
3164
- * `docker({ capabilities })`. One per level that set such a sandbox, however many evals inherit it.
3313
+ * A `SandboxWarning` describes a sandbox with more privilege than the default `reeval` user:
3314
+ * `docker({ root: true })` or `docker({ capabilities })`. One per level that set such a sandbox,
3315
+ * however many evaluations inherit it.
3165
3316
  */
3166
3317
  type SandboxWarning = {
3167
3318
  kind: 'root' | 'capabilities';
3168
3319
  /** Which level set the sandbox. */
3169
3320
  level: 'config' | 'profile' | 'group' | 'eval';
3170
- /** That level as reeval names it: `reeval.config.ts`, `profiles.pr.defaults`, `group("net")` or `evaluate("installs deps")`. */
3321
+ /**
3322
+ * The value is the level as reeval names it: the `reeval.config.ts` file, `profiles.pr.defaults`,
3323
+ * `group("net")`, or `evaluate("installs deps")`.
3324
+ */
3171
3325
  where: string;
3172
- /** The eval file that sets it, for a group or eval; null for the config or a profile. */
3326
+ /** The evaluation file that sets it, for a group or evaluation; null for the config or a profile. */
3173
3327
  file: string | null;
3174
3328
  /** The capabilities added back, for `capabilities`. */
3175
3329
  capabilities?: string[];
@@ -3185,5 +3339,5 @@ type ResolvedGateway = {
3185
3339
  }>;
3186
3340
  };
3187
3341
  //#endregion
3188
- export { FakeVote as $, VertexPublisher as $t, Diff as A, VERDICT_TEXT as An, Profile as At, EnvironmentInput as B, effortSample as Bn, Secret as Bt, CodexHarnessDescriptor as C, EFFORT_MIN_RUNS as Cn, Outcome as Ct, ConfigHelpers as D, EffortVerdict as Dn, PinnableKey as Dt, CompareOptions as E, EffortSample as En, PiHarnessDescriptor as Et, EgressPolicy as F, contrastEffort as Fn, QuestionStateOptions as Ft, ExecResult as G, formatSeconds as Gn, TokenUsage as Gt, ErrorKind as H, effortVerdict as Hn, ServeHandle as Ht, EgressRecord as I, directional as In, ReevalConfig as It, FakeAnswer as J, DeclaredNode as Jn, TurnData as Jt, ExecRetry as K, tokenValues as Kn, TranscriptEvent as Kt, EnvVars as L, effortChangeLabel as Ln, ResolvedGateway as Lt, DirectoryOptions as M, cellContrast as Mn, ProfileLimit as Mt, DockerSandboxDescriptor as N, cellEffort as Nn, Question as Nt, ConfigJudgeOptions as O, Spread as On, PluginPart as Ot, Duration as P, cellSamples as Pn, QuestionOptions as Pt, FakeScriptStep as Q, VertexGatewayDescriptor as Qt, Environment as R, effortLabel as Rn, SandboxDescriptor as Rt, ClaudeCodeHarnessDescriptor as S, EFFORT_ALPHA as Sn, OpenRouterGatewayDescriptor as St, CompareCut as T, EffortMetric as Tn, PairVote as Tt, EvalFn as U, formatCount as Un, SetupStep as Ut, EnvironmentSpec as V, effortSummary as Vn, SecretTemplate as Vt, EvalOptions as W, formatRatio as Wn, SkillSource as Wt, FakeJudgeModel as X, VercelGatewayDescriptor as Xt, FakeHarnessDescriptor as Y, TurnStatus as Yt, FakeSandboxDescriptor as Z, Verdict as Zt, CaptureWait as _, RunRecord as _n, JudgeOptions as _t, AttemptContext as a, CellRecord as an, GroupOptions as at, CellRole as b, CellEffort as bn, McpServer as bt, BlindPair as c, FamilyLift as cn, HarnessEdit as ct, CaptureCall as d, FamilyRecord as dn, HostFile as dt, Viewport as en, FxHarnessDescriptor as et, CaptureInstruction as f, FileRef as fn, JSONValue as ft, CaptureUseScript as g, Lift as gn, JudgeModel as gt, CaptureUseAction as h, JudgmentRecord as hn, JudgeFailure as ht, AttemptAnswer as i, AttemptRecord as in, GrokBuildHarnessDescriptor as it, DirectGatewayDescriptor as j, bothUnderATenth as jn, ProfileDefaults as jt, CursorHarnessDescriptor as k, TOKENS_UNREPORTED as kn, PluginSource as kt, Budget as l, FamilyLiftSummary as ln, HarnessOption as lt, CaptureUse as m, HistoryEntry as mn, JudgeCallOptions as mt, AnalysisConfig as n, WebEventType as nn, GatewayName as nt, BackgroundHandle as o, CheckTally as on, HarnessData as ot, CaptureOptions as p, HistoryCell as pn, JudgeArtifactRef as pt, FakeAnalysisModel as q, DeclarationTree as qn, Turn as qt, AntigravityHarnessDescriptor as r, ArtifactRef as rn, GooseHarnessDescriptor as rt, BlindAttempt as s, EvalEntry as sn, HarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, GatewayDescriptor as tt, Capture as u, FamilyRate as un, HistoryConfig as ut, CaptureWindow as v, StackSummary as vn, JudgeState as vt, CompareAnswer as w, EffortContrast as wn, PairFactor as wt, ClaudeCodeEffort as x, DIRECTIONAL_MIN as xn, OpenCodeHarnessDescriptor as xt, Cell as y, UpstreamCount as yn, JudgeStateOptions as yt, EnvironmentHelpers as z, effortOf as zn, SandboxWarning as zt };
3189
- //# sourceMappingURL=types-coMWHaU2.d.ts.map
3342
+ export { FakeScriptStep as $, Verdict as $t, CursorHarnessDescriptor as A, Spread as An, PluginPart as At, EnvironmentHelpers as B, effortLabel as Bn, SandboxDescriptor as Bt, ClaudeCodeHarnessDescriptor as C, DIRECTIONAL_MIN as Cn, OpenCodeHarnessDescriptor as Ct, CompareOptions as D, EffortMetric as Dn, PairVote as Dt, CompareCut as E, EffortContrast as En, PairFactor as Et, Duration as F, cellEffort as Fn, Question as Ft, EvalOptions as G, formatCount as Gn, SetupStep as Gt, EnvironmentSpec as H, effortSample as Hn, Secret as Ht, EgressPolicy as I, cellSamples as In, QuestionInputs as It, FakeAnalysisModel as J, tokenValues as Jn, TranscriptEvent as Jt, ExecResult as K, formatRatio as Kn, SkillSource as Kt, EgressRecord as L, contrastEffort as Ln, QuestionOptions as Lt, DirectGatewayDescriptor as M, VERDICT_TEXT as Mn, Profile as Mt, DirectoryOptions as N, bothUnderATenth as Nn, ProfileDefaults as Nt, ConfigHelpers as O, EffortSample as On, PiHarnessDescriptor as Ot, DockerSandboxDescriptor as P, cellContrast as Pn, ProfileLimit as Pt, FakeSandboxDescriptor as Q, VercelGatewayDescriptor as Qt, EnvVars as R, directional as Rn, ReevalConfig as Rt, ClaudeCodeEffort as S, CellEffort as Sn, McpServer as St, CompareAnswer as T, EFFORT_MIN_RUNS as Tn, Outcome as Tt, ErrorKind as U, effortSummary as Un, SecretTemplate as Ut, EnvironmentInput as V, effortOf as Vn, SandboxWarning as Vt, EvalFn as W, effortVerdict as Wn, ServeHandle as Wt, FakeHarnessDescriptor as X, DeclaredNode as Xn, TurnData as Xt, FakeAnswer as Y, DeclarationTree as Yn, Turn as Yt, FakeJudgeModel as Z, TurnStatus as Zt, CaptureUseScript as _, JudgmentRecord as _n, JudgeInputs as _t, AttemptContext as a, ArtifactRef as an, GrokBuildHarnessDescriptor as at, Cell as b, StackSummary as bn, JudgeState as bt, BlindPair as c, CheckTally as cn, HarnessDescriptor as ct, CaptureCall as d, FamilyLiftSummary as dn, HistoryConfig as dt, VertexGatewayDescriptor as en, FakeVote as et, CaptureInstruction as f, FamilyRate as fn, HostFile as ft, CaptureUseAction as g, HistoryEntry as gn, JudgeFailure as gt, CaptureUse as h, HistoryCell as hn, JudgeCallOptions as ht, AttemptAnswer as i, WebEventType as in, GooseHarnessDescriptor as it, Diff as j, TOKENS_UNREPORTED as jn, PluginSource as jt, ConfigJudgeOptions as k, EffortVerdict as kn, PinnableKey as kt, Budget as l, EvalEntry as ln, HarnessEdit as lt, CaptureOptions as m, FileRef as mn, JudgeArtifactRef as mt, AnalysisConfig as n, Viewport as nn, GatewayDescriptor as nt, BackgroundHandle as o, AttemptRecord as on, GroupOptions as ot, CaptureKey as p, FamilyRecord as pn, JSONValue as pt, ExecRetry as q, formatSeconds as qn, TokenUsage as qt, AntigravityHarnessDescriptor as r, Wait as rn, GatewayName as rt, BlindAttempt as s, CellRecord as sn, HarnessData as st, AgentHarnessDescriptor as t, VertexPublisher as tn, FxHarnessDescriptor as tt, Capture as u, FamilyLift as un, HarnessOption as ut, CaptureWait as v, Lift as vn, JudgeModel as vt, CodexHarnessDescriptor as w, EFFORT_ALPHA as wn, OpenRouterGatewayDescriptor as wt, CellRole as x, UpstreamCount as xn, JudgeSubject as xt, CaptureWindow as y, RunRecord as yn, JudgeOptions as yt, Environment as z, effortChangeLabel as zn, ResolvedGateway as zt };
3343
+ //# sourceMappingURL=types-DVvW66ci.d.ts.map