reeval 0.15.1 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. package/README.md +21 -14
  2. package/dist/{adapter-CbqbMR7y.js → adapter-CqUt4A0w.js} +3 -3
  3. package/dist/{adapter-CbqbMR7y.js.map → adapter-CqUt4A0w.js.map} +1 -1
  4. package/dist/{agent-Zgn-kiqq.js → agent-BtAbRTFq.js} +6 -6
  5. package/dist/{agent-Zgn-kiqq.js.map → agent-BtAbRTFq.js.map} +1 -1
  6. package/dist/{app-files-ChbvRx3P.js → app-files-CDttHPOz.js} +3 -3
  7. package/dist/app-files-CDttHPOz.js.map +1 -0
  8. package/dist/assemble-Bb7zqpRw.js +1 -0
  9. package/dist/{assemble-SKQGW_QF.js → assemble-J9rkE29m.js} +114 -31
  10. package/dist/assemble-J9rkE29m.js.map +1 -0
  11. package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
  12. package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
  13. package/dist/{attempt-host-BOp8JklD.js → attempt-host-DDedP7Qv.js} +17 -8
  14. package/dist/attempt-host-DDedP7Qv.js.map +1 -0
  15. package/dist/capture-Bvc4FtWT.js +15 -0
  16. package/dist/capture-Bvc4FtWT.js.map +1 -0
  17. package/dist/capture-D68TiQQy.js +1 -0
  18. package/dist/{capture-BM_3100g.js → capture-DheNqeYS.js} +1495 -203
  19. package/dist/capture-DheNqeYS.js.map +1 -0
  20. package/dist/{cli-PdYrcoKM.js → cli-KTHjNFoo.js} +329 -74
  21. package/dist/cli-KTHjNFoo.js.map +1 -0
  22. package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{colors-QV9dFVhN.js → colors-Y6O3mtl_.js} +20 -2
  28. package/dist/colors-Y6O3mtl_.js.map +1 -0
  29. package/dist/{command-9xm5ImqI.js → command-1FBECMs-.js} +16 -16
  30. package/dist/{command-9xm5ImqI.js.map → command-1FBECMs-.js.map} +1 -1
  31. package/dist/{command-BqBxLgTc.js → command-Ce03LLtg.js} +14 -14
  32. package/dist/{command-BqBxLgTc.js.map → command-Ce03LLtg.js.map} +1 -1
  33. package/dist/{command-4pJYVCT4.js → command-CztRuwUM.js} +12 -12
  34. package/dist/{command-4pJYVCT4.js.map → command-CztRuwUM.js.map} +1 -1
  35. package/dist/{command-DbSMcxZL.js → command-nNN0mx65.js} +7 -7
  36. package/dist/{command-DbSMcxZL.js.map → command-nNN0mx65.js.map} +1 -1
  37. package/dist/{commands-BU6YJCz9.js → commands-DP13JX9i.js} +24 -12
  38. package/dist/commands-DP13JX9i.js.map +1 -0
  39. package/dist/{commands-Bi2SEQtr.js → commands-ZwQUezI1.js} +1 -1
  40. package/dist/{config-k6m7XbAz.js → config-BjjDryd5.js} +464 -220
  41. package/dist/config-BjjDryd5.js.map +1 -0
  42. package/dist/{contrasts-DLZF4dGF.js → contrasts-DSsRZpR8.js} +2 -2
  43. package/dist/{contrasts-DLZF4dGF.js.map → contrasts-DSsRZpR8.js.map} +1 -1
  44. package/dist/cursor-catalog-D7bFLEAD.js +143 -0
  45. package/dist/cursor-catalog-D7bFLEAD.js.map +1 -0
  46. package/dist/{db-CpsifVln.js → db-B9Ityuit.js} +8 -8
  47. package/dist/{db-CpsifVln.js.map → db-B9Ityuit.js.map} +1 -1
  48. package/dist/db-Bk6LWwza.js +2 -0
  49. package/dist/debug-CAx3qcaP.js +1 -0
  50. package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
  51. package/dist/debug-CgG3T071.js.map +1 -0
  52. package/dist/default-effort-DK8gospa.js +91 -0
  53. package/dist/default-effort-DK8gospa.js.map +1 -0
  54. package/dist/default-effort-DmXHpqo9.js +1 -0
  55. package/dist/{docker-YFj4H1Cr.js → docker-BLCNfUie.js} +1 -1
  56. package/dist/{docker-B4-rx4dA.js → docker-SKpYOlGi.js} +5 -5
  57. package/dist/{docker-B4-rx4dA.js.map → docker-SKpYOlGi.js.map} +1 -1
  58. package/dist/{docker-51WvhN_X.js → docker-hwp3EucJ.js} +9 -5
  59. package/dist/docker-hwp3EucJ.js.map +1 -0
  60. package/dist/{doctor-BuQnfz4e.js → doctor-CT0Sgjng.js} +16 -16
  61. package/dist/{doctor-BuQnfz4e.js.map → doctor-CT0Sgjng.js.map} +1 -1
  62. package/dist/{environment-BOFgzFDg.js → environment-CZdercoj.js} +3 -3
  63. package/dist/{environment-BOFgzFDg.js.map → environment-CZdercoj.js.map} +1 -1
  64. package/dist/{errors-UjcLWelF.js → errors-BZVKVKYd.js} +15 -4
  65. package/dist/errors-BZVKVKYd.js.map +1 -0
  66. package/dist/{eval-source-Dje3rERU.js → eval-source-nZD7-FSz.js} +9 -8
  67. package/dist/eval-source-nZD7-FSz.js.map +1 -0
  68. package/dist/{evaluate-dqXccn4k.d.ts → evaluate-CNi0lZ7n.d.ts} +7 -6
  69. package/dist/{evaluate-BPP9s3s6.js → evaluate-fhC1PCd_.js} +345 -47
  70. package/dist/evaluate-fhC1PCd_.js.map +1 -0
  71. package/dist/{files-Clipx4OW.js → files-BGy05Dlr.js} +4 -4
  72. package/dist/{files-Clipx4OW.js.map → files-BGy05Dlr.js.map} +1 -1
  73. package/dist/{files-Bi461Be2.js → files-BIGxpL08.js} +25 -16
  74. package/dist/{files-Bi461Be2.js.map → files-BIGxpL08.js.map} +1 -1
  75. package/dist/files-C2RC7K4m.js +1 -0
  76. package/dist/files-C3tQtD26.js +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/{files-H24pz0hL.js → files-I1i6Xyh1.js} +4 -4
  80. package/dist/{files-H24pz0hL.js.map → files-I1i6Xyh1.js.map} +1 -1
  81. package/dist/files-dJmrfmOM.js +1 -0
  82. package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
  83. package/dist/gate-BC6SdQZs.js.map +1 -0
  84. package/dist/gateway.d.ts +1 -1
  85. package/dist/{git-CsVZkNNL.js → git-DR02tHtU.js} +2 -2
  86. package/dist/{git-CsVZkNNL.js.map → git-DR02tHtU.js.map} +1 -1
  87. package/dist/{harness-3NVyTSGK.js → harness-ylJLvrbp.js} +39 -5
  88. package/dist/{harness-3NVyTSGK.js.map → harness-ylJLvrbp.js.map} +1 -1
  89. package/dist/harness.d.ts +1 -1
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-4mwhNkvh.js → images---X8Wu9d.js} +8 -8
  92. package/dist/{images-4mwhNkvh.js.map → images---X8Wu9d.js.map} +1 -1
  93. package/dist/index.d.ts +71 -16
  94. package/dist/index.js +5 -5
  95. package/dist/init-BNTXrtMO.js +1 -0
  96. package/dist/{init-cOk3D44b.js → init-CA95ZnkV.js} +4 -4
  97. package/dist/{init-cOk3D44b.js.map → init-CA95ZnkV.js.map} +1 -1
  98. package/dist/{invocations-DW-lTk2f.js → invocations-BdoROAfb.js} +3 -3
  99. package/dist/{invocations-DW-lTk2f.js.map → invocations-BdoROAfb.js.map} +1 -1
  100. package/dist/{judge-CKdgEXN7.js → judge-Dp7FPJ1P.js} +3 -3
  101. package/dist/{judge-CKdgEXN7.js.map → judge-Dp7FPJ1P.js.map} +1 -1
  102. package/dist/judge.d.ts +2 -2
  103. package/dist/judge.js +1 -1
  104. package/dist/{live-XJQ6nqK4.js → live-CBFkdK1K.js} +3 -3
  105. package/dist/{live-XJQ6nqK4.js.map → live-CBFkdK1K.js.map} +1 -1
  106. package/dist/{matchers-DT2TaGix.js → matchers-Cv74mJRJ.js} +43 -13
  107. package/dist/matchers-Cv74mJRJ.js.map +1 -0
  108. package/dist/{model-Bq3G0OG1.js → model-6dfKFkQd.js} +4 -4
  109. package/dist/{model-Bq3G0OG1.js.map → model-6dfKFkQd.js.map} +1 -1
  110. package/dist/{model-CTnyvh7H.js → model-Be7fPvgi.js} +4 -51
  111. package/dist/model-Be7fPvgi.js.map +1 -0
  112. package/dist/{model-g3XnzUv0.js → model-CN4ccjNz.js} +1 -1
  113. package/dist/model-C_5TspZS.js +1 -0
  114. package/dist/{model-vRiUycWh.js → model-DkJVmVd2.js} +4 -4
  115. package/dist/{model-vRiUycWh.js.map → model-DkJVmVd2.js.map} +1 -1
  116. package/dist/{node-BVbbnFtv.js → node-D_Zkwhq2.js} +2 -2
  117. package/dist/{node-BVbbnFtv.js.map → node-D_Zkwhq2.js.map} +1 -1
  118. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  119. package/dist/object-BkXGPKWY.js.map +1 -0
  120. package/dist/{opencode-providers-Bz0l7OZI.js → opencode-providers-DPgdqRfg.js} +26 -2
  121. package/dist/{opencode-providers-Bz0l7OZI.js.map → opencode-providers-DPgdqRfg.js.map} +1 -1
  122. package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-CR00FWAa.js} +2 -2
  123. package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-CR00FWAa.js.map} +1 -1
  124. package/dist/{patch-DKJiC5Sq.js → patch-BJNdOZsf.js} +4 -4
  125. package/dist/patch-BJNdOZsf.js.map +1 -0
  126. package/dist/{plan-DdWkM6SW.js → plan-C1UpGZTk.js} +76 -12
  127. package/dist/plan-C1UpGZTk.js.map +1 -0
  128. package/dist/{plugins-D-CP6mMd.js → plugins-Cx6ReJWE.js} +24 -3
  129. package/dist/plugins-Cx6ReJWE.js.map +1 -0
  130. package/dist/{preflight-9ixuXzsx.js → preflight-BZ3L7P3a.js} +3 -3
  131. package/dist/{preflight-9ixuXzsx.js.map → preflight-BZ3L7P3a.js.map} +1 -1
  132. package/dist/{preflight-DTiVv0gY.js → preflight-CUooKgtT.js} +1 -1
  133. package/dist/{prepare-BfyikQit.js → prepare-BbBoTeDj.js} +7 -7
  134. package/dist/{prepare-BfyikQit.js.map → prepare-BbBoTeDj.js.map} +1 -1
  135. package/dist/{prepare-tmBBf4W8.js → prepare-CEZaEwaq.js} +2 -2
  136. package/dist/{provenance-BizwUn3h.js → provenance-BWvTIlQM.js} +2 -2
  137. package/dist/{provenance-BizwUn3h.js.map → provenance-BWvTIlQM.js.map} +1 -1
  138. package/dist/{providers-B6b9Tnyi.js → providers-B3yqakc7.js} +5 -5
  139. package/dist/{providers-B6b9Tnyi.js.map → providers-B3yqakc7.js.map} +1 -1
  140. package/dist/{providers-DkOupEBl.js → providers-DdyZW_ru.js} +1 -1
  141. package/dist/query-BkwQ0W-c.js +1 -0
  142. package/dist/{query-DyM-VExS.js → query-CZcqpOI9.js} +2 -2
  143. package/dist/{query-DyM-VExS.js.map → query-CZcqpOI9.js.map} +1 -1
  144. package/dist/{questions-BRhU8MzB.js → questions-CFU7-crW.js} +20 -19
  145. package/dist/questions-CFU7-crW.js.map +1 -0
  146. package/dist/{record-DZBCjuwm.js → record-Bu1Xi6MQ.js} +8 -4
  147. package/dist/record-Bu1Xi6MQ.js.map +1 -0
  148. package/dist/{remarks-C7i2LMRb.js → remarks-A3j-SkJ_.js} +42 -8
  149. package/dist/remarks-A3j-SkJ_.js.map +1 -0
  150. package/dist/remarks-Bp6AwPgb.js +1 -0
  151. package/dist/{remarks-D_3qhBWU.js → remarks-Dj_x9l5n.js} +4 -3
  152. package/dist/{remarks-D_3qhBWU.js.map → remarks-Dj_x9l5n.js.map} +1 -1
  153. package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
  154. package/dist/remarks-eBcrIZRu.js.map +1 -0
  155. package/dist/{remote-DFIzcx_6.js → remote-EIFHdMd8.js} +5 -5
  156. package/dist/{remote-DFIzcx_6.js.map → remote-EIFHdMd8.js.map} +1 -1
  157. package/dist/{resolve-DWSJpKHX.js → resolve-8H5rz8mi.js} +5 -5
  158. package/dist/{resolve-DWSJpKHX.js.map → resolve-8H5rz8mi.js.map} +1 -1
  159. package/dist/resolve-Tm9pX7Hv.js +1 -0
  160. package/dist/{results-BC3pcG_Z.js → results-E7JdVjq2.js} +9 -9
  161. package/dist/{results-BC3pcG_Z.js.map → results-E7JdVjq2.js.map} +1 -1
  162. package/dist/{resume-j-rcIryi.js → resume-Blu4yIcq.js} +4 -4
  163. package/dist/{resume-j-rcIryi.js.map → resume-Blu4yIcq.js.map} +1 -1
  164. package/dist/{runtime-d56Gg3J5.js → runtime-CNbsMJQ7.js} +111 -37
  165. package/dist/runtime-CNbsMJQ7.js.map +1 -0
  166. package/dist/{sandbox-Cmln11gU.js → sandbox-RButGfIE.js} +2 -2
  167. package/dist/{sandbox-Cmln11gU.js.map → sandbox-RButGfIE.js.map} +1 -1
  168. package/dist/sandbox.d.ts +1 -1
  169. package/dist/sandbox.js +1 -1
  170. package/dist/server-Dqrq-Sia.js +1 -0
  171. package/dist/{server-BOoH10J8.js → server-XgCYs_MF.js} +35 -11
  172. package/dist/server-XgCYs_MF.js.map +1 -0
  173. package/dist/{source-wD04-rRG.js → source-BOkbJCD2.js} +17 -3
  174. package/dist/{source-wD04-rRG.js.map → source-BOkbJCD2.js.map} +1 -1
  175. package/dist/source-DooSTtyz.js +1 -0
  176. package/dist/src-1M0srjji.js +1 -0
  177. package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
  178. package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
  179. package/dist/{src-D91Hd6hT.js → src-BJCKWacl.js} +38 -18
  180. package/dist/src-BJCKWacl.js.map +1 -0
  181. package/dist/src-BZlhaDhE.js +2 -0
  182. package/dist/{src-BO8AYqJ5.js → src-BkhEPVJZ.js} +12 -3
  183. package/dist/{src-BO8AYqJ5.js.map → src-BkhEPVJZ.js.map} +1 -1
  184. package/dist/{src-C7upnZSd.js → src-Bp6X4193.js} +1 -1
  185. package/dist/{src-BYLM3hEz.js → src-C2Japg_I.js} +21 -10
  186. package/dist/{src-BYLM3hEz.js.map → src-C2Japg_I.js.map} +1 -1
  187. package/dist/src-C5q3_fRV.js +3 -0
  188. package/dist/src-C9EHSYyP.js +2 -0
  189. package/dist/src-CJdpsrXi.js +2 -0
  190. package/dist/{src-1lpTocbd.js → src-D1ET6oPL.js} +3 -27
  191. package/dist/{src-1lpTocbd.js.map → src-D1ET6oPL.js.map} +1 -1
  192. package/dist/{src-Bp4aN3rA.js → src-D4CZjMQ4.js} +1 -1
  193. package/dist/{src-BYPMM1x7.js → src-GkJ-1Rdy.js} +1 -1
  194. package/dist/{src-kYeaJJrl.js → src-OOKE-5GA.js} +1 -1
  195. package/dist/state-BCyNaz13.js +1 -0
  196. package/dist/{state-DrJ687ut.js → state-D6-tvqtp.js} +8 -8
  197. package/dist/{state-DrJ687ut.js.map → state-D6-tvqtp.js.map} +1 -1
  198. package/dist/suite-BiWajSlA.js +2 -0
  199. package/dist/{suite-Co8cwelb.js → suite-DYI1k2Ie.js} +22 -5
  200. package/dist/suite-DYI1k2Ie.js.map +1 -0
  201. package/dist/{summary-DLFHVoux.js → summary-CN2aqAUz.js} +59 -17
  202. package/dist/summary-CN2aqAUz.js.map +1 -0
  203. package/dist/tarball-CHcahyNK.js +258 -0
  204. package/dist/tarball-CHcahyNK.js.map +1 -0
  205. package/dist/testing.d.ts +96 -44
  206. package/dist/testing.js +4 -4
  207. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  208. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  209. package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
  210. package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
  211. package/dist/{types-coMWHaU2.d.ts → types-DDeCnNmp.d.ts} +437 -181
  212. package/dist/view/index.d.ts +34 -5
  213. package/dist/view/index.js +8 -7
  214. package/dist/views-7bNY17Z6.js +2 -0
  215. package/dist/{views-BFO1BLKu.js → views-Bd-H71_t.js} +5 -5
  216. package/dist/{views-BFO1BLKu.js.map → views-Bd-H71_t.js.map} +1 -1
  217. package/dist/{worker-entry-CJtdsKBP.js → worker-entry-BIHJqE_z.js} +34 -13
  218. package/dist/worker-entry-BIHJqE_z.js.map +1 -0
  219. package/dist/workspace-CxwHv8mG.js +1 -0
  220. package/dist/{workspace-BRxjfa8N.js → workspace-Sy5g8zW9.js} +4 -4
  221. package/dist/{workspace-BRxjfa8N.js.map → workspace-Sy5g8zW9.js.map} +1 -1
  222. package/package.json +1 -1
  223. package/skills/eval-rubric/SKILL.md +11 -0
  224. package/dist/app-files-ChbvRx3P.js.map +0 -1
  225. package/dist/assemble-SKQGW_QF.js.map +0 -1
  226. package/dist/assemble-XBWuK3Ed.js +0 -1
  227. package/dist/attempt-host-BOp8JklD.js.map +0 -1
  228. package/dist/capture-BM_3100g.js.map +0 -1
  229. package/dist/capture-DSK8O5bA.js +0 -1
  230. package/dist/cli-PdYrcoKM.js.map +0 -1
  231. package/dist/cli-spec-D0qyrPZi.js.map +0 -1
  232. package/dist/client-Ct6s6OnF.js.map +0 -1
  233. package/dist/colors-QV9dFVhN.js.map +0 -1
  234. package/dist/commands-BU6YJCz9.js.map +0 -1
  235. package/dist/config-k6m7XbAz.js.map +0 -1
  236. package/dist/db-DkD-F-X-.js +0 -2
  237. package/dist/debug-BhizgAsU.js.map +0 -1
  238. package/dist/debug-DeM5pswK.js +0 -1
  239. package/dist/docker-51WvhN_X.js.map +0 -1
  240. package/dist/errors-UjcLWelF.js.map +0 -1
  241. package/dist/eval-source-Dje3rERU.js.map +0 -1
  242. package/dist/evaluate-BPP9s3s6.js.map +0 -1
  243. package/dist/files-BAt0i00X.js +0 -1
  244. package/dist/files-C2h2Cw4F.js +0 -1
  245. package/dist/files-Ca0d6Uq3.js.map +0 -1
  246. package/dist/files-DmUrRmmL.js +0 -1
  247. package/dist/init-BDEOopK7.js +0 -1
  248. package/dist/matchers-DT2TaGix.js.map +0 -1
  249. package/dist/model-CTnyvh7H.js.map +0 -1
  250. package/dist/model-D-SWrLZG.js +0 -1
  251. package/dist/object-CLf2-tNq.js.map +0 -1
  252. package/dist/pairs-puYi-zj2.js.map +0 -1
  253. package/dist/patch-DKJiC5Sq.js.map +0 -1
  254. package/dist/plan-DdWkM6SW.js.map +0 -1
  255. package/dist/plugins-D-CP6mMd.js.map +0 -1
  256. package/dist/query-Clm2mNSm.js +0 -1
  257. package/dist/questions-BRhU8MzB.js.map +0 -1
  258. package/dist/record-DZBCjuwm.js.map +0 -1
  259. package/dist/remarks-BwJ8pcTQ.js +0 -1
  260. package/dist/remarks-C7i2LMRb.js.map +0 -1
  261. package/dist/remarks-dA8LyoPc.js.map +0 -1
  262. package/dist/resolve--rQZ8u6Z.js +0 -1
  263. package/dist/runtime-d56Gg3J5.js.map +0 -1
  264. package/dist/server-BOoH10J8.js.map +0 -1
  265. package/dist/server-DpuVAbrO.js +0 -1
  266. package/dist/source-BFd6f0-R.js +0 -1
  267. package/dist/src-6_zz7oJs.js +0 -2
  268. package/dist/src-D91Hd6hT.js.map +0 -1
  269. package/dist/src-DH-9IO7f.js +0 -1
  270. package/dist/src-HSxBazcg.js +0 -3
  271. package/dist/src-LD1BAJFa.js +0 -2
  272. package/dist/src-dlX-Lmxg.js +0 -2
  273. package/dist/state-B5PLm5qY.js +0 -1
  274. package/dist/suite-22Kt9Grx.js +0 -2
  275. package/dist/suite-Co8cwelb.js.map +0 -1
  276. package/dist/summary-DLFHVoux.js.map +0 -1
  277. package/dist/views-BOf8qEL8.js +0 -2
  278. package/dist/worker-entry-CJtdsKBP.js.map +0 -1
  279. package/dist/workspace-DDHzkMjm.js +0 -1
@@ -187,7 +187,7 @@ type FileRef = {
187
187
  sha256: string;
188
188
  bytes: number;
189
189
  };
190
- /** A generic attempt file from `c.capture()` (and later video). */
190
+ /** A generic attempt file from `c.capture()`. */
191
191
  type ArtifactRef = FileRef & {
192
192
  name: string;
193
193
  mediaType: 'image/png';
@@ -208,6 +208,14 @@ type ArtifactRef = FileRef & {
208
208
  model: string;
209
209
  moves: number;
210
210
  };
211
+ /** The HTTP status of the page's document, after redirects; absent when no response came. */
212
+ status?: number;
213
+ /** How many uncaught exceptions and unhandled rejections the page had; absent when none. */
214
+ pageErrors?: number;
215
+ /** `artifacts/<name>.capture.json` beside the PNG: what the browser saw, a `CapturePage`. */
216
+ page?: FileRef;
217
+ /** Set on the screenshot taken as one of the capture's instructions ended. */
218
+ step?: CaptureStep;
211
219
  };
212
220
  };
213
221
  type GitInfo = {
@@ -217,15 +225,18 @@ type GitInfo = {
217
225
  */
218
226
  sha: string;
219
227
  branch: string | null;
220
- /** Uncommitted changes, outside `.reeval/` and the history folder, when the run started. */
228
+ /** Uncommitted changes, outside the `.reeval/` directory and the history folder, when the run started. */
221
229
  dirty: boolean;
222
230
  /** sha256 of the uncommitted changes (tracked and untracked, as a binary patch), when dirty. */
223
231
  diffHash: string | null;
224
- /** Those changes as `worktree.patch` in the artifact dir; apply on `sha` to reproduce. */
232
+ /**
233
+ * Those changes are stored as the `worktree.patch` file in the artifact directory. Applying it
234
+ * to `sha` reproduces the workspace.
235
+ */
225
236
  worktreePatch: FileRef | null;
226
237
  /**
227
- * Why a dirty run has no `worktree.patch`: too large, artifacts: 'none', or it looked like it held
228
- * a secret (then where, and which rule, never the value).
238
+ * This explains why a dirty run has no `worktree.patch` file: it was too large, `artifacts` was
239
+ * `'none'`, or it looked like it held a secret (then where, and which rule, never the value).
229
240
  */
230
241
  worktreePatchNote: string | null;
231
242
  commitTime: string;
@@ -267,7 +278,7 @@ type JudgmentRecord = {
267
278
  /** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
268
279
  costUsd?: number;
269
280
  /**
270
- * `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
281
+ * `<index>:<sha256(question) prefix>`: the same question at the same place in the evaluation, so a
271
282
  * verdict can be followed across runs (and an edited question starts a new series). `<index>`
272
283
  * is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
273
284
  * whichever finishes first.
@@ -279,14 +290,17 @@ type JudgmentRecord = {
279
290
  artifacts?: JudgeArtifactRef[];
280
291
  };
281
292
  type AttemptFiles = {
282
- /** Each environment setup step's command, exit code, duration and output. */
293
+ /** Each environment setup step's command, exit code, duration, and output. */
283
294
  setupLog?: FileRef;
284
295
  transcript?: FileRef;
285
296
  /** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
286
297
  rawTranscript?: FileRef;
287
298
  diff?: FileRef;
288
299
  judgeInput?: FileRef;
289
- /** PNGs from `c.capture()`, under `artifacts/<name>.png`. */
300
+ /**
301
+ * PNGs from `c.capture()`, under the `artifacts/<name>.png` path, then the page as each of its
302
+ * instructions ended, `artifacts/<name>.step-<n>.png`, kept when the capture fails too.
303
+ */
290
304
  artifacts?: ArtifactRef[];
291
305
  };
292
306
  type ContextFile = {
@@ -301,16 +315,22 @@ type UpstreamCount = {
301
315
  type AttemptRecord = {
302
316
  /** `<eval key> # <cell key>#<n>`: unique in the run. */
303
317
  key: string;
304
- /** `<cell key>#<n>`: unique in its eval. */
318
+ /** `<cell key>#<n>`: unique in its evaluation. */
305
319
  id: string;
306
320
  /** The cell's key; see `EvalEntry.cells`. */
307
321
  cell: string;
308
322
  /** Run number within the cell, from 1. */
309
323
  n: number;
324
+ /**
325
+ * In a `runs: { bestOf }` cell: `added` when it ran past the majority's first attempts (the
326
+ * earlier ones split, or an infra error left them short), `skipped` when it didn't run because
327
+ * the earlier ones already had a majority. Unset on the attempts that always run.
328
+ */
329
+ bestOf?: 'added' | 'skipped';
310
330
  /** Start order across the run, from 1; null for attempts that never started. */
311
331
  seq: number | null;
312
332
  startedAt: string | null;
313
- /** The image the container started from, in `stack.images`; null for the fake sandbox. */
333
+ /** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
314
334
  imageId: string | null;
315
335
  /** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
316
336
  env: {
@@ -318,16 +338,17 @@ type AttemptRecord = {
318
338
  secrets: string[];
319
339
  brokered: string[];
320
340
  };
321
- /** Skills, rules and MCP config visible to the agent in its sandbox, by content hash. */
341
+ /** Skills, rules, and MCP config visible to the agent in its sandbox, by content hash. */
322
342
  context: ContextFile[];
323
343
  /** The egress policy and, when the sandbox records it, the hosts contacted and refused; null if it never ran. */
324
344
  egress: EgressRecord | null;
325
345
  /**
326
- * The models the harness's own API calls were served by, as the egress gateway saw them: named
327
- * in the request (e.g. `gemini-3.8-flash` for the id `gemini-3.8-flash-low`, plus any the agent
328
- * uses on the side) or, for Codex, in the response, with its effort
329
- * (`gpt-5.6-luna[effort=medium]`); null when neither names one (Cursor) or nothing
330
- * recorded them.
346
+ * The models served by the harness's API calls, as observed by the egress gateway. Values may
347
+ * come from requests or responses and include reported effort; null when no model was named or
348
+ * recorded. The gateway reads the request URL for Antigravity, Goose, pi, and OpenCode on a
349
+ * `google/` model, so a request for `<model>-low` records `<model>`, and the API response for
350
+ * Codex, Grok Build, and OpenCode on another provider. Null for Cursor and Claude Code, whose
351
+ * API requests name no model.
331
352
  */
332
353
  servedModels: string[] | null;
333
354
  /**
@@ -348,12 +369,14 @@ type AttemptRecord = {
348
369
  */
349
370
  reportedCostUsd?: number;
350
371
  /**
351
- * The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
352
- * null when the factory left it unset. Fast is not this.
372
+ * The reasoning effort reeval asked the harness for: the factory's
373
+ * (`claudeCode({ effort: 'high' })`), or `medium` where it set none and the harness and model
374
+ * take one; null when reeval asked for none, leaving the CLI's own default. Fast is not
375
+ * this.
353
376
  */
354
377
  effort: string | null;
355
378
  /**
356
- * The reasoning effort the harness's API said it ran at, from `servedModels` /
379
+ * The reasoning effort the harness's API said it ran at, from `servedModels` or
357
380
  * each turn's `model` (`medium` from `gpt-5.6-luna[effort=medium]`); null when those
358
381
  * labels name none. Fast (`[fast=true]`) is not this.
359
382
  */
@@ -362,7 +385,8 @@ type AttemptRecord = {
362
385
  error: AttemptError | null;
363
386
  durationMs: number;
364
387
  /**
365
- * `text` is the agent's final message, cut at 8 KB; the whole of it is in `transcript.jsonl`.
388
+ * `text` is the agent's final message, cut at 8 KB; the whole of it is in the `transcript.jsonl`
389
+ * file.
366
390
  * `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
367
391
  */
368
392
  turns: Array<{
@@ -385,8 +409,8 @@ type AttemptRecord = {
385
409
  /** Tokens across the attempt's turns; null when the harness reported none. */
386
410
  usage: TokenUsage | null;
387
411
  /**
388
- * Tokens the capture vision loop used. Infrastructure only: never folded
389
- * into `usage`, effort, compare or lift.
412
+ * Tokens the capture vision loop used. Infrastructure only: never included
413
+ * in `usage`, effort, compare, or lift.
390
414
  */
391
415
  captureUse?: {
392
416
  model: string;
@@ -394,7 +418,7 @@ type AttemptRecord = {
394
418
  usage: TokenUsage | null;
395
419
  };
396
420
  timings: PhaseTimings;
397
- /** Counts from the transcript, so a run's shape is visible without reading it. `webSearches` includes refused searches. */
421
+ /** Counts from the transcript, so the attempt's activity is available without reading the transcript. `webSearches` includes refused searches. */
398
422
  activity: {
399
423
  tools: number;
400
424
  fileEdits: number;
@@ -419,6 +443,22 @@ type AttemptRecord = {
419
443
  * names. Absent when there were none.
420
444
  */
421
445
  setupProcesses?: SetupProcess[];
446
+ /**
447
+ * Copies of the attempt's workspace in its directory in `.reeval/runs` on the machine that ran
448
+ * it (never in the history folder), in the order taken: `turn-<n>` after each `c.send` with
449
+ * `snapshot: 'turns'`, then `final` after the checks. Each has its form, path and size, its
450
+ * file count, the directories left out by name, the files whose secret-looking values were
451
+ * masked and the binary ones left out for holding one, or why it wasn't taken. Absent with
452
+ * `snapshot: false`, and when the sandbox was gone first (a timeout or a cancel).
453
+ */
454
+ snapshots?: Snapshot[];
455
+ /**
456
+ * Each `c.serve` call that listened, in call order: `{ command, port, portEnv, detected,
457
+ * address, readyMs }`. `port` is where it listens; `detected` is true when that isn't the
458
+ * `PORT` it was given (`portEnv`), so reeval took the one port that started listening after
459
+ * the command did. Absent when the attempt served nothing.
460
+ */
461
+ serves?: ServeRecord[];
422
462
  /**
423
463
  * Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
424
464
  * `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
@@ -433,9 +473,9 @@ type AttemptRecord = {
433
473
  /** Every try, in order, when it was given `retries`; its log has each try's output. */
434
474
  tries?: ExecTry[];
435
475
  }>;
436
- /** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
476
+ /** Every `expect` and `expect.require` matcher call that settled, in call order; unset with none. */
437
477
  assertions?: AssertionRecord[];
438
- /** Files under a dependency directory (node_modules/, .venv/, …) collapse to one entry with `count`. */
478
+ /** Files under a dependency directory, such as the `node_modules/` directory or `.venv/` directory, collapse to one entry with `count`. */
439
479
  changedFiles: Array<FileChange & {
440
480
  count?: number;
441
481
  }>;
@@ -451,7 +491,7 @@ type AttemptRecord = {
451
491
  type EvalEntry = {
452
492
  /** `file › group › … › name`, as shown. */
453
493
  id: string;
454
- /** What the eval is matched on across runs; currently the same as `id`. */
494
+ /** Used to match an evaluation across runs; equal to `id`. */
455
495
  key: string;
456
496
  /** Directory name inside the artifact directory, from `key`. */
457
497
  slug: string;
@@ -461,8 +501,8 @@ type EvalEntry = {
461
501
  /** Results are comparable only when fingerprints match. */
462
502
  fingerprint: string;
463
503
  /**
464
- * The eval file, by content hash: where the eval lives. Records from before `source` also list
465
- * the local files it imported, and were fingerprinted over them.
504
+ * The evaluation file, by content hash: where the evaluation lives. Records from before `source`
505
+ * also list the local files it imported, and were fingerprinted over them.
466
506
  */
467
507
  sources: Array<{
468
508
  path: string;
@@ -476,12 +516,12 @@ type EvalEntry = {
476
516
  /** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
477
517
  revision?: Revision;
478
518
  tags: string[];
479
- /** The baseline environment, when the eval uses `compare`. */
519
+ /** The baseline environment, when the evaluation uses `compare`. */
480
520
  compare?: EvalCompareMeta;
481
521
  /**
482
- * Over gated cells: `fail` if any failed, else `infra_error` if any is (a cell is `infra_error`
483
- * only when nothing counted), else `skipped` if all were, else `pass`. An ungated cell's
484
- * failures don't fail the eval.
522
+ * Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
523
+ * `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
524
+ * cell's failures don't fail the evaluation.
485
525
  */
486
526
  outcome: Outcome;
487
527
  /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
@@ -494,9 +534,9 @@ type EvalEntry = {
494
534
  * The eval's pairwise questions (those with `across`), answered after its attempts by votes,
495
535
  * one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
496
536
  * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
497
- * (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
498
- * `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
499
- * masked and screenshots as refs; unset where the eval's files weren't kept.
537
+ * (rates by cut, votes, ties, a failed call, and a sample of reasons). Each answer's `input` is
538
+ * the `<eval-slug>/questions/<n>.json` file: every call its model was sent, per pair and order,
539
+ * with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
500
540
  */
501
541
  answers?: CompareAnswer[];
502
542
  /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
@@ -513,7 +553,7 @@ type FamilyInterval = {
513
553
  hi: number;
514
554
  level: 0.95;
515
555
  /**
516
- * `wilson` / `newcombe-paired`: one counted attempt a case and side, so each case is one
556
+ * `wilson` or `newcombe-paired`: one counted attempt a case and side, so each case is one
517
557
  * Bernoulli trial (or pair). `t-cases`: several, so a t interval over the cases' pass@1, with
518
558
  * attempts within a case never counted as independent.
519
559
  */
@@ -588,17 +628,17 @@ type FamilyLift = {
588
628
  outcome: 'pass' | 'fail';
589
629
  effort?: FamilyEffort;
590
630
  };
591
- /** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an eval. */
631
+ /** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an evaluation. */
592
632
  type FamilyRecord = {
593
633
  /** `file › group › … › template`. */
594
634
  id: string;
595
635
  file: string;
596
636
  group: string[];
597
- /** The name template, e.g. `fixes $id`. */
637
+ /** The name template, for example `fixes $id`. */
598
638
  name: string;
599
639
  /** Cases in the list. */
600
640
  cases: number;
601
- /** Cases with an eval in this run; fewer when a filter or tag left some out. */
641
+ /** Cases with an evaluation in this run; fewer when a filter or tag left some out. */
602
642
  ran: number;
603
643
  /** One per cell. */
604
644
  rates: FamilyRate[];
@@ -643,7 +683,7 @@ type FingerprintParts = {
643
683
  environment: string | null;
644
684
  image: string | null;
645
685
  agentCli: string | null;
646
- /** Over the judge's model, threshold and version, and the config's `state` when it sets one. */
686
+ /** Over the judge's model, threshold and version. */
647
687
  judge: string;
648
688
  /** Over the attempt's context files. */
649
689
  context: string;
@@ -655,7 +695,7 @@ type FingerprintParts = {
655
695
  */
656
696
  gate?: string;
657
697
  };
658
- /** One harness and environment of an eval, and how its attempts went. */
698
+ /** One harness and environment of an evaluation, and how its attempts went. */
659
699
  type CellRecord = Cell & {
660
700
  /** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
661
701
  fingerprint: string;
@@ -674,14 +714,20 @@ type CellRecord = Cell & {
674
714
  /** The configured harness model, when it has one. */
675
715
  model: string | null;
676
716
  /**
677
- * The reasoning effort the harness factory asked for; null when it left it unset.
678
- * Fast is not this.
717
+ * The reasoning effort reeval asked the harness for: the factory's, or `medium` where it set
718
+ * none and the harness and model take one; null when reeval asked for none, leaving the CLI's
719
+ * own default. Fast is not this.
679
720
  */
680
721
  effort: string | null;
681
- /** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the eval. */
722
+ /** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the evaluation. */
682
723
  gate: CellGate;
683
724
  /** Environment keys the variant replaced on the control's (or a child on its parent's). */
684
725
  overrides: string[];
726
+ /**
727
+ * Set by `runs: { bestOf }`: the cell stopped once a majority of this many attempts agreed, and
728
+ * gates on that majority. `runs` still counts every planned attempt, the skipped ones included.
729
+ */
730
+ bestOf?: number;
685
731
  runs: number;
686
732
  pass: number;
687
733
  fail: number;
@@ -697,7 +743,7 @@ type CellRecord = Cell & {
697
743
  outcome: Outcome;
698
744
  lift?: Lift;
699
745
  /**
700
- * Each check (assertion label) seen in any attempt of the eval, over this cell's counted
746
+ * Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
701
747
  * attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
702
748
  */
703
749
  checks?: CellCheck[];
@@ -710,10 +756,10 @@ type CheckTally = {
710
756
  notRun: number;
711
757
  };
712
758
  /**
713
- * One check over a cell's counted attempts. `verdict` (variant cells), over the attempts that
714
- * reached the check: `discriminates` when it passes on every one of one side and none of the
715
- * other; `both-pass` or `both-fail` when it does the same on both sides, so it can't tell them
716
- * apart; `mixed` otherwise. No verdict when a side never reached it: not run is not a fail.
759
+ * One check over a cell's counted attempts. `verdict` (variant cells) is `discriminates` when
760
+ * every answered instance passes on one side and fails on the other, `both-pass` when every
761
+ * answered instance passes on both sides, `both-fail` when every answered instance fails on both
762
+ * sides, or `mixed` otherwise. It is absent when either side has no pass or fail.
717
763
  */
718
764
  type CellCheck = CheckTally & {
719
765
  label: string;
@@ -773,7 +819,10 @@ type RunRecord = {
773
819
  harness?: string[];
774
820
  environment?: string;
775
821
  sandbox?: string;
822
+ /** With `bestOf`, the most attempts a cell runs. */
776
823
  runs?: number;
824
+ /** Set by `defaults.runs: { bestOf }`. */
825
+ bestOf?: number;
777
826
  timeout?: string | number;
778
827
  env?: string[];
779
828
  };
@@ -807,8 +856,13 @@ type RunRecord = {
807
856
  config: {
808
857
  sandbox: string;
809
858
  environment: string | null;
810
- /** The config's `runs`; 1 when it sets none, though a compare's cells then run 4 each. */
859
+ /**
860
+ * The config's `runs` (with `bestOf`, the most a cell runs); 1 when it sets none, though a
861
+ * compare's cells then run 4 each.
862
+ */
811
863
  runs: number;
864
+ /** Set by the config's `runs: { bestOf }`. */
865
+ bestOf?: number;
812
866
  judgeModel: string | null;
813
867
  threshold: number;
814
868
  timeoutMs: number;
@@ -827,28 +881,26 @@ type RunRecord = {
827
881
  /** Each pooled `evaluate.each` that ran; unset when none did. */
828
882
  families?: FamilyRecord[];
829
883
  /**
830
- * Errors outside any attempt, redacted: an eval file that didn't load, a rejection or throw no
831
- * attempt awaited, a worker that died. Any one fails the run. Unset when there were none.
884
+ * These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
885
+ * or throw no attempt awaited, or a worker that died. Any one fails the run. Unset when there
886
+ * were none.
832
887
  */
833
888
  errors?: string[];
834
889
  };
835
- /** What ran and graded the agents: the machine, images, harnesses and judge. */
890
+ /** These fields describe what ran and graded the agents: the machine, images, harnesses, and judge. */
836
891
  type Stack = {
837
892
  /** The reeval version that wrote the run. */
838
893
  reeval: string;
839
- /** `process.versions.node`: under Bun or Deno, the Node version they're compatible with. */
894
+ /** `process.versions.node`, or the compatible Node version reported by another runtime. */
840
895
  node: string;
841
- /**
842
- * The runtime reeval ran on, `node`, `bun`, `deno` or another, and its own version. Absent in
843
- * runs recorded before it, which ran on Node.
844
- */
896
+ /** The runtime reeval ran on, `node`, `bun`, `deno`, or another, and its version. */
845
897
  runtime?: {
846
898
  name: string;
847
899
  version: string;
848
900
  };
849
901
  /**
850
- * The language the evals are written in, `typescript` for `*.eval.ts`, and the version of its
851
- * toolchain when the runner reports one. Absent in runs recorded before it.
902
+ * The evaluation language, `typescript` for `*.eval.ts` files, and the version of its toolchain
903
+ * when the runner reports one.
852
904
  */
853
905
  language?: {
854
906
  name: string;
@@ -856,16 +908,17 @@ type Stack = {
856
908
  };
857
909
  os: string;
858
910
  arch: string;
859
- /** Every image an attempt started from. `id` is content-addressed; `repoDigests` only exist once pushed. */
911
+ /** Every image an attempt started from. `id` is content-addressed; `repoDigests` exist only after the image is pushed. */
860
912
  images: Array<AttemptImage & {
861
913
  repoDigests: string[];
862
914
  }>;
863
915
  /** Exact versions of the packages that run and grade the agent. */
864
916
  packages: Record<string, string>;
865
917
  /**
866
- * Each harness: its configured `model` and requested `effort`, `resolvedModels` (what it
867
- * reported serving, with the variant it resolved), and `resolvedEffort` (reasoning-effort
868
- * suffixes those labels named). Fast (`[fast=true]`) is not `resolvedEffort`.
918
+ * For each harness, the record includes its configured `model` and the `effort` it ran at,
919
+ * `resolvedModels` (what it reported serving, with the variant it resolved), and
920
+ * `resolvedEffort` (reasoning-effort suffixes those labels named). Fast (`[fast=true]`) is not
921
+ * `resolvedEffort`.
869
922
  */
870
923
  harnesses: Array<{
871
924
  id: string;
@@ -873,8 +926,9 @@ type Stack = {
873
926
  /** The configured model, when the harness has one. */
874
927
  model: string | null;
875
928
  /**
876
- * The reasoning effort the harness factory asked for; null when it left it unset.
877
- * Fast is not this.
929
+ * The reasoning effort reeval asked the harness for: the factory's, or `medium` where it set
930
+ * none and the harness and model take one; null when reeval asked for none, leaving the
931
+ * CLI's own default. Fast is not this.
878
932
  */
879
933
  effort: string | null;
880
934
  /**
@@ -901,12 +955,12 @@ type Stack = {
901
955
  threshold: number;
902
956
  /** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
903
957
  version: string;
904
- /** What the judge sees by default, every key resolved; absent when the config and profile leave `state` at its defaults. */
958
+ /** The config's default judge inputs, every key resolved. Unset when the config names none, which reeval requires: each `c.judge()` call names its own. */
905
959
  state?: ResolvedJudgeStateOptions;
906
960
  };
907
961
  /** Union of every attempt's context files. */
908
962
  context: ContextFile[];
909
- /** What eval fingerprints hash; fingerprints from different scopes aren't comparable. */
963
+ /** What evaluation fingerprints hash; fingerprints from different scopes aren't comparable. */
910
964
  fingerprintScope: string[];
911
965
  };
912
966
  /** The parts of a run's stack that `reeval compare` checks for drift. */
@@ -1108,16 +1162,22 @@ type Wait = CaptureWait | {
1108
1162
  type CaptureInstruction = string | {
1109
1163
  wait: Wait;
1110
1164
  };
1111
- type CaptureUseAction = {
1165
+ /** The keys a capture's driver presses. In a focused `<select>`, the arrows change the choice. */
1166
+ type CaptureKey = 'Tab' | 'Enter' | 'Escape' | 'Backspace' | 'Space' | 'ArrowUp' | 'ArrowDown' | 'ArrowLeft' | 'ArrowRight';
1167
+ /** A move on the page. */
1168
+ type CaptureUseMove = ({
1112
1169
  type: 'click';
1113
1170
  x: number;
1114
1171
  y: number;
1115
- } | {
1172
+ } |
1173
+ /** `replace` selects what the focused field holds first, so the text replaces it. */
1174
+ {
1116
1175
  type: 'type';
1117
1176
  text: string;
1177
+ replace?: boolean;
1118
1178
  } | {
1119
1179
  type: 'key';
1120
- key: 'Tab' | 'Enter';
1180
+ key: CaptureKey;
1121
1181
  } | {
1122
1182
  type: 'scroll';
1123
1183
  x: number;
@@ -1133,7 +1193,15 @@ type CaptureUseAction = {
1133
1193
  type: 'hover';
1134
1194
  x: number;
1135
1195
  y: number;
1136
- } | {
1196
+ }) & {
1197
+ /**
1198
+ * The move finishes its instruction: once the page settles, the journey goes on to the next.
1199
+ * A model's final move that changed nothing on the page is asked about again instead.
1200
+ */
1201
+ final?: boolean;
1202
+ };
1203
+ /** A move, or `done`: the instruction's goal is met without another move. */
1204
+ type CaptureUseAction = CaptureUseMove | {
1137
1205
  type: 'done';
1138
1206
  };
1139
1207
  /** One list per instruction string. Tests; never downloads a model. */
@@ -1152,7 +1220,7 @@ type CaptureUse = {
1152
1220
  type CaptureOptions = {
1153
1221
  /** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
1154
1222
  viewport?: Viewport;
1155
- /** How long to poll before failing. Default `'60s'`. */
1223
+ /** How long to poll before failing; a 502, 503 or 504 is polled past until then. Default `'60s'`. */
1156
1224
  timeout?: Duration;
1157
1225
  /** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
1158
1226
  browser?: boolean;
@@ -1166,12 +1234,15 @@ type CaptureOptions = {
1166
1234
  type BackgroundHandle = {
1167
1235
  kill(): Promise<void>;
1168
1236
  };
1169
- /** `c.serve`: a background process plus the port it was given or allocated. */
1237
+ /** `c.serve`: a background process plus the port it listens on. */
1170
1238
  type ServeHandle = BackgroundHandle & {
1171
1239
  port: number;
1172
1240
  };
1173
- /** What `c.capture()` returns: the PNG is also in the attempt record under `artifacts/`. */
1174
- type Capture = {
1241
+ /**
1242
+ * What `c.capture()` returns: the PNG is also in the attempt record under `artifacts/`, and the
1243
+ * page facts in `artifacts/<name>.capture.json` beside it.
1244
+ */
1245
+ type Capture = CapturePage & {
1175
1246
  name: string;
1176
1247
  /** Container port that was tunneled. */
1177
1248
  port: number;
@@ -1186,6 +1257,61 @@ type Capture = {
1186
1257
  bytes: number;
1187
1258
  };
1188
1259
  };
1260
+ /** One message the page's console showed while it was captured. */
1261
+ type CaptureConsoleMessage = {
1262
+ level: 'error' | 'warning' | 'info' | 'log' | 'debug';
1263
+ /** The message, at most 2000 characters, with the app's own URLs as paths. */
1264
+ text: string;
1265
+ /**
1266
+ * `console-api` for the page's `console.*` calls; otherwise the browser's own source, such as
1267
+ * `network` (a request that failed), `javascript`, `security` or `deprecation`.
1268
+ */
1269
+ source: string;
1270
+ /** The script or resource it came from, as a path when it's on the app. */
1271
+ url?: string;
1272
+ };
1273
+ /** A request the page made that failed or answered 400 or above. */
1274
+ type CaptureFailedRequest = {
1275
+ method: string;
1276
+ /** As a path when it's on the app. */
1277
+ url: string;
1278
+ /** The response's status; absent when no response came. */
1279
+ status?: number;
1280
+ /** The browser's error, such as `net::ERR_CONNECTION_REFUSED`; absent when a response came. */
1281
+ error?: string;
1282
+ };
1283
+ /** A class of HTTP status for `toHaveStatus()`: `'2xx'` is 200 to 299. */
1284
+ type StatusRange = `${1 | 2 | 3 | 4 | 5}xx`;
1285
+ /**
1286
+ * What the browser saw while it loaded and captured a page: `<name>.capture.json` beside the
1287
+ * capture's PNG, and these fields of the `Capture` that `c.capture()` returns. On the fake
1288
+ * sandbox, which has no browser, `status` and `finalUrl` come from an HTTP GET and the lists are
1289
+ * empty.
1290
+ */
1291
+ type CapturePage = {
1292
+ /**
1293
+ * The HTTP status of the main frame's latest document, after redirects. A full navigation
1294
+ * replaces it; a client-side route change keeps it. Absent when no response came.
1295
+ */
1296
+ status?: number;
1297
+ /** Where the page ended up, after redirects and client-side routing: a path on the app. */
1298
+ finalUrl: string;
1299
+ /** Each message the console showed, in order; at most 100 of each level. */
1300
+ console: CaptureConsoleMessage[];
1301
+ /**
1302
+ * The page errors: each uncaught exception and unhandled promise rejection, with its stack, at
1303
+ * most 100.
1304
+ */
1305
+ errors: string[];
1306
+ /** Requests that failed or answered 400 or above, in order, at most 100. */
1307
+ failedRequests: CaptureFailedRequest[];
1308
+ /** How many messages, errors and requests the caps left out; absent when none. */
1309
+ dropped?: number;
1310
+ };
1311
+ /** A capture's `artifacts/<name>.capture.json`: its page facts, versioned. */
1312
+ type CapturePageFile = {
1313
+ schemaVersion: 1;
1314
+ } & CapturePage;
1189
1315
  /**
1190
1316
  * Optional camera and path for `c.capture(port | ServeHandle, options?)`.
1191
1317
  * `port` is the first argument, not a field here. Default `path` is `/`;
@@ -1279,10 +1405,10 @@ type JudgeState = JudgeJsonState | Array<{
1279
1405
  type: 'json';
1280
1406
  value: JudgeJsonState;
1281
1407
  } | JudgeFilePart>;
1282
- /** A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`. */
1408
+ /** A capture this judge call sent as a file part. Bytes live in the `artifacts/<name>.png` file. */
1283
1409
  type JudgeArtifactRef = {
1284
1410
  name: string;
1285
- /** `artifacts/<name>.png` under the attempt directory. */
1411
+ /** The `artifacts/<name>.png` path under the attempt directory. */
1286
1412
  path: string;
1287
1413
  sha256: string;
1288
1414
  bytes: number;
@@ -1314,11 +1440,6 @@ type JudgeOptions = {
1314
1440
  providerOptions?: Record<string, Record<string, JSONValue>>;
1315
1441
  /** The most judge calls to this model's provider at once, across the run. */
1316
1442
  maxConcurrent?: number;
1317
- /**
1318
- * The judge's state: what the judge is given beside the question, not the workspace's or the
1319
- * attempt's state. Merged key by key down the cascade.
1320
- */
1321
- state?: JudgeStateOptions;
1322
1443
  /**
1323
1444
  * The gateway the judge's calls go through, by name or by its helper. It skips the config's
1324
1445
  * `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
@@ -1326,43 +1447,61 @@ type JudgeOptions = {
1326
1447
  gateway?: GatewayName | GatewayDescriptor;
1327
1448
  };
1328
1449
  /**
1329
- * What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
1330
- * keeps its default, which is what the judge gets with no `state` option at all.
1450
+ * What one `c.judge()` call shows the judge beside the question. Every input is off unless the
1451
+ * call turns it on, and a call turns on at least one; each is a key of the state in
1452
+ * `judge-input.json`, or the captures sent as file parts.
1331
1453
  */
1332
- type JudgeStateOptions = {
1333
- /** The graded turn's prompt. Default true. */
1454
+ type JudgeInputs = {
1455
+ /** The graded turn's prompt. */
1334
1456
  task?: boolean;
1335
- /** The graded turn's final message, or `on`'s value. Default true. */
1457
+ /** The graded turn's final message, or `on`'s value. */
1336
1458
  output?: boolean;
1337
- /** `status path` lines for the changed files, under `diff`'s patterns. Default true. */
1459
+ /** `status path` lines for the changed files, under `diff`'s patterns. */
1338
1460
  files?: boolean;
1339
1461
  /**
1340
1462
  * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
1341
- * the last matching pattern wins and `!` excludes. Default true.
1463
+ * the last matching pattern wins and `!` excludes.
1342
1464
  */
1343
1465
  diff?: boolean | string[];
1344
1466
  /**
1345
1467
  * Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
1346
1468
  * patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
1347
1469
  * the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
1348
- * These files leave the diff. Default none.
1470
+ * These files leave the diff.
1349
1471
  */
1350
1472
  contents?: string[];
1351
1473
  /**
1352
1474
  * The graded turn's transcript as text lines: `true` is messages and tool calls with tool
1353
1475
  * output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
1354
- * Default false.
1355
1476
  */
1356
1477
  transcript?: boolean | 'tools' | 'full';
1478
+ /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
1479
+ checks?: boolean | string[];
1480
+ /**
1481
+ * Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
1482
+ * capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
1483
+ */
1484
+ artifacts?: true | Array<Capture | string>;
1485
+ };
1486
+ /** What a judge call's inputs read: which turn, and whether the turns before it too. */
1487
+ type JudgeSubject = {
1488
+ /** The turn `task`, `output`, `transcript` and `diff` read instead of the latest, or a value used as the output. */
1489
+ on?: Turn | JSONValue;
1357
1490
  /**
1358
1491
  * `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
1359
1492
  * `transcript`) transcript. Default `'last'`.
1360
1493
  */
1361
1494
  turns?: 'last' | 'all';
1362
- /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
1363
- checks?: boolean | string[];
1364
1495
  };
1365
- /** `JudgeStateOptions` with every key set: what one judge call saw. */
1496
+ /** One input turned on: `true`, a mode or a non-empty selection, not `false`. */
1497
+ type OneInput<I, K extends keyof I> = { [P in K]-?: Exclude<I[P], false | undefined>; };
1498
+ /** At least one of `I`'s keys turned on. */
1499
+ type AtLeastOneInput<I> = { [K in keyof I]-?: OneInput<I, K>; }[keyof I];
1500
+ /** `c.judge()`'s options, before the at-least-one-input rule that `JudgeCallOptions` adds. */
1501
+ type JudgeCallFields = JudgeOptions & JudgeInputs & JudgeSubject;
1502
+ /** `c.judge()`'s options: the judge options, the inputs (at least one) and what they read. */
1503
+ type JudgeCallOptions = JudgeCallFields & AtLeastOneInput<JudgeInputs>;
1504
+ /** The inputs one judge call saw, every key set (`false` when off); `artifacts` is recorded apart. */
1366
1505
  type ResolvedJudgeStateOptions = {
1367
1506
  task: boolean;
1368
1507
  output: boolean;
@@ -1373,16 +1512,6 @@ type ResolvedJudgeStateOptions = {
1373
1512
  turns: 'last' | 'all';
1374
1513
  checks: boolean | string[];
1375
1514
  };
1376
- type JudgeCallOptions = JudgeOptions & {
1377
- /** What to grade instead of the latest turn: another turn, or a value used as the output. */
1378
- on?: Turn | JSONValue;
1379
- /**
1380
- * Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
1381
- * `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
1382
- * captures; a provider that cannot take files fails the call.
1383
- */
1384
- artifacts?: true | Array<Capture | string>;
1385
- };
1386
1515
  type Verdict = {
1387
1516
  verdict: 'pass' | 'fail';
1388
1517
  /** The judge's probability that the answer is yes, from 0 to 1. */
@@ -1418,10 +1547,11 @@ type AttemptContext = {
1418
1547
  /** Sends the next turn of this eval's agent session. */
1419
1548
  send(prompt: string): Promise<Turn>;
1420
1549
  /**
1421
- * Asks the judge a yes/no question about the latest turn (or `options.on`).
1422
- * `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
1550
+ * Asks the judge a yes/no question about what `options` turns on, and nothing else: at least
1551
+ * one of `task`, `output`, `files`, `diff`, `contents`, `transcript`, `checks` or `artifacts`.
1552
+ * They read the latest turn, or `options.on`.
1423
1553
  */
1424
- judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1554
+ judge(question: string, options: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1425
1555
  /**
1426
1556
  * Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
1427
1557
  * many extra times, `interval` ms apart; it resolves with the last try.
@@ -1436,8 +1566,10 @@ type AttemptContext = {
1436
1566
  background?: boolean;
1437
1567
  }): Promise<ExecResult | BackgroundHandle>;
1438
1568
  /**
1439
- * Starts `command` in the background and returns its port. A given `port` is used; omit it
1440
- * and reeval allocates one. Sets `PORT` so a bash string can use `$PORT`.
1569
+ * Starts `command` in the background with `PORT` set to `port`, or to a port reeval allocates,
1570
+ * and returns once it listens. Without `port`, a command that listens on one new port instead
1571
+ * (Vite's 5173) gets that one; several new ports, none of them `PORT`, or an exit first, throw
1572
+ * with its last output.
1441
1573
  */
1442
1574
  serve(command: string, port?: number): Promise<ServeHandle>;
1443
1575
  /**
@@ -2378,9 +2510,11 @@ type CompareOptions<E = Environment> = {
2378
2510
  * With `across`, it picks between two attempts that differ only in that factor, in both orders;
2379
2511
  * without it, it's asked of each attempt on its own. Without a model, the cascaded
2380
2512
  * `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
2381
- * `state` and `artifacts` choose what it's shown of each attempt.
2513
+ * It's shown only the inputs it turns on, at least one.
2382
2514
  */
2383
- type Question = string | {
2515
+ type Question = QuestionFields & AtLeastOneInput<QuestionInputs>;
2516
+ /** A question's options, before the at-least-one-input rule that `Question` adds. */
2517
+ type QuestionFields = QuestionInputs & {
2384
2518
  question: string;
2385
2519
  /** `'provider/model'` that answers it, over the cascaded `question.model`. */
2386
2520
  model?: string;
@@ -2391,43 +2525,39 @@ type Question = string | {
2391
2525
  across?: PairFactor | PairFactor[];
2392
2526
  /** The answers a question asked of each attempt picks from, with a short reason. */
2393
2527
  choices?: string[];
2394
- /** What it's shown of each attempt, over the cascaded `question.state`, key by key. */
2395
- state?: QuestionStateOptions;
2396
- /** Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
2397
- artifacts?: true | string[];
2528
+ /**
2529
+ * `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
2530
+ * transcript; `'last'`: the last turn only. Default `'all'`.
2531
+ */
2532
+ turns?: 'last' | 'all';
2398
2533
  };
2399
2534
  /**
2400
- * What a question is shown of each attempt: the keys of `judge.state` but `contents` (the
2401
- * attempts' workspaces are gone when questions are asked). An unset key keeps its default: a
2402
- * pairwise question shows the task and the diff; one asked of each attempt also shows the final
2403
- * message and the transcript.
2535
+ * What a question is shown of each attempt: `c.judge()`'s inputs but `contents` (the attempts'
2536
+ * workspaces are gone when questions are asked). Every input is off unless the question turns it
2537
+ * on, and a question turns on at least one.
2404
2538
  */
2405
- type QuestionStateOptions = {
2406
- /** The prompts the attempt was sent, once for both sides when they match. Default true. */
2539
+ type QuestionInputs = {
2540
+ /** The prompts the attempt was sent, once for both sides when they match. */
2407
2541
  task?: boolean;
2408
- /** The final message: the agent's own account of its work. Default false in a pair, true per attempt. */
2542
+ /** The final message: the agent's own account of its work. */
2409
2543
  output?: boolean;
2410
- /** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
2544
+ /** `status path` lines for the changed files, under `diff`'s patterns. */
2411
2545
  files?: boolean;
2412
2546
  /**
2413
2547
  * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
2414
- * the last matching pattern wins and `!` excludes. Default true.
2548
+ * the last matching pattern wins and `!` excludes.
2415
2549
  */
2416
2550
  diff?: boolean | string[];
2417
2551
  /**
2418
2552
  * The transcript as text lines: `true` is messages and tool calls with tool output clipped,
2419
2553
  * `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
2420
- * harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per
2421
- * attempt.
2554
+ * harness's tools, so a pair of two harnesses isn't blind.
2422
2555
  */
2423
2556
  transcript?: boolean | 'tools' | 'full';
2424
- /**
2425
- * `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
2426
- * transcript; `'last'`: the last turn only. Default `'all'`.
2427
- */
2428
- turns?: 'last' | 'all';
2429
- /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
2557
+ /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
2430
2558
  checks?: boolean | string[];
2559
+ /** Each attempt's own captures, by name; `true` sends all of them. */
2560
+ artifacts?: true | string[];
2431
2561
  };
2432
2562
  /** `QuestionStateOptions` with every key set: what each side of one question showed. */
2433
2563
  type ResolvedQuestionStateOptions = {
@@ -2439,14 +2569,10 @@ type ResolvedQuestionStateOptions = {
2439
2569
  turns: 'last' | 'all';
2440
2570
  checks: boolean | string[];
2441
2571
  };
2442
- /** Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key. */
2572
+ /** Defaults for every question below it. What a question is shown is its own, never a default. */
2443
2573
  type QuestionOptions = {
2444
2574
  /** `'provider/model'` that answers every question below it without its own `model`. */
2445
2575
  model?: string;
2446
- /** What it's shown of each attempt. */
2447
- state?: QuestionStateOptions;
2448
- /** Each attempt's own captures, by name; `true` sends all of them. Default none. */
2449
- artifacts?: true | string[];
2450
2576
  };
2451
2577
  /** A matrix factor a pairwise question varies; the others are held. */
2452
2578
  type PairFactor = 'environment' | 'harness';
@@ -2484,7 +2610,7 @@ type CompareAnswer = {
2484
2610
  model: string;
2485
2611
  /** The model id the provider says served the calls, when it reports one. */
2486
2612
  resolvedModel?: string;
2487
- /** What each side showed, every key resolved; unset on answers from before questions took `state`, which showed the defaults. */
2613
+ /** The inputs each side showed, every key resolved; unset on answers recorded before reeval wrote it, which showed the task and the diff. */
2488
2614
  stateOptions?: ResolvedQuestionStateOptions;
2489
2615
  /** The captures it sent of each side, as asked: `true` for all, or their names. */
2490
2616
  artifacts?: true | string[];
@@ -2537,8 +2663,21 @@ type GatePolicy = 'all' | {
2537
2663
  type CellGate = GatePolicy | 'none';
2538
2664
  /** A compare object: `baseline` and `variants`. Same as `CompareOptions`. */
2539
2665
  type CompareSpec<E = Environment> = CompareOptions<E>;
2666
+ type OddDigit = '1' | '3' | '5' | '7' | '9';
2667
+ /**
2668
+ * `N` when it's an odd whole number of 3 or more, else a sentence the compiler shows as the type
2669
+ * it expected. A `number` that isn't a literal stays `number`: only planning can check it.
2670
+ */
2671
+ type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${string}` | `${string}.${string}` | `${string}e${string}` | '1' ? 'an odd whole number of 3 or more, e.g. 3 or 5' : `${N}` extends `${string}${OddDigit}` ? N : 'an odd whole number of 3 or more, e.g. 3 or 5';
2672
+ /**
2673
+ * Attempts per cell: a fixed count, or `{ bestOf }`, an odd number of 3 or more, which stops a
2674
+ * cell once one side (pass or fail) has a majority: `{ bestOf: 5 }` runs 3 to 5 attempts.
2675
+ */
2676
+ type Runs<N extends number = number> = number | {
2677
+ bestOf: OddAtLeast3<N>;
2678
+ };
2540
2679
  /** Options other than `environment` and `compare`, which are exclusive of each other. */
2541
- type EvalOptionFields = {
2680
+ type EvalOptionFields<N extends number = number> = {
2542
2681
  /** Which agents run the eval; each harness is its own cell. */
2543
2682
  harness?: HarnessOption;
2544
2683
  /**
@@ -2547,11 +2686,12 @@ type EvalOptionFields = {
2547
2686
  */
2548
2687
  sandbox?: SandboxDescriptor;
2549
2688
  /**
2550
- * Attempts per cell. Unset at every level, a compare with variants runs 4 a side, the fewest
2551
- * whose lift can reach a verdict, and anything else runs once, a pooled `evaluate.each` case
2552
- * included.
2689
+ * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
2690
+ * a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
2691
+ * Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
2692
+ * a verdict, and anything else runs once, a pooled `evaluate.each` case included.
2553
2693
  */
2554
- runs?: number;
2694
+ runs?: Runs<N>;
2555
2695
  /** How each cell's attempts decide its outcome; default `'all'`. */
2556
2696
  gate?: GatePolicy;
2557
2697
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
@@ -2565,7 +2705,7 @@ type EvalOptionFields = {
2565
2705
  judge?: JudgeOptions;
2566
2706
  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
2567
2707
  capture?: CaptureOptions;
2568
- /** Defaults for every question: `model`, and `state` merged key by key, down the cascade. */
2708
+ /** The model every question that names none of its own is answered by, down the cascade. */
2569
2709
  question?: QuestionOptions;
2570
2710
  /**
2571
2711
  * What a language model is asked about the attempts, always a list; they never change pass,
@@ -2576,6 +2716,11 @@ type EvalOptionFields = {
2576
2716
  questions?: Question[];
2577
2717
  /** Per attempt. */
2578
2718
  timeout?: Duration;
2719
+ /**
2720
+ * When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
2721
+ * the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
2722
+ */
2723
+ snapshot?: SnapshotOption;
2579
2724
  /** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
2580
2725
  tags?: string[];
2581
2726
  /**
@@ -2622,8 +2767,9 @@ type EnvironmentOrCompare<E> = {
2622
2767
  baseline?: undefined;
2623
2768
  };
2624
2769
  };
2625
- type EvalOptions = EvalOptionFields & EnvironmentOrCompare<Environment>;
2626
- type GroupOptions = EvalOptions;
2770
+ /** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
2771
+ type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
2772
+ type GroupOptions<N extends number = number> = EvalOptions<N>;
2627
2773
  /**
2628
2774
  * `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
2629
2775
  * by name or a fake judge, and environments without `extend`. The same `environment` / compare
@@ -2663,8 +2809,8 @@ type ProfileDefaults = {
2663
2809
  sandbox?: SandboxDescriptor;
2664
2810
  /** What every eval runs in when it names no environment. */
2665
2811
  environment?: Environment;
2666
- /** Attempts per cell. */
2667
- runs?: number;
2812
+ /** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
2813
+ runs?: Runs;
2668
2814
  /** How each cell's attempts decide its outcome. */
2669
2815
  gate?: GatePolicy;
2670
2816
  /** Variables and secrets for the sandbox, merged by name over the config's. */
@@ -2679,6 +2825,8 @@ type ProfileDefaults = {
2679
2825
  capture?: CaptureOptions;
2680
2826
  /** Defaults for every question, merged over the config's. */
2681
2827
  question?: QuestionOptions;
2828
+ /** When each attempt's workspace is copied; merged over the config's. */
2829
+ snapshot?: SnapshotOption;
2682
2830
  };
2683
2831
  /** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
2684
2832
  type ProfileLimit = {
@@ -2755,15 +2903,15 @@ type BlindAttempt = {
2755
2903
  choices?: string[];
2756
2904
  attempt: BlindSide;
2757
2905
  };
2758
- /** A turn of one side, with the keys its question's `state` turned on. */
2906
+ /** A turn of one side, with the inputs its question turned on. */
2759
2907
  type BlindTurn = {
2760
2908
  task?: string;
2761
2909
  output?: string;
2762
2910
  transcript?: string;
2763
2911
  };
2764
2912
  /**
2765
- * One side of a blinded pair, or the attempt a per-attempt question is asked of: only the keys
2766
- * its question's `state` turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
2913
+ * One side of a blinded pair, or the attempt a per-attempt question is asked of: only the inputs
2914
+ * its question turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
2767
2915
  * under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
2768
2916
  */
2769
2917
  type BlindSide = BlindTurn & {
@@ -2817,8 +2965,11 @@ type ReevalConfig = {
2817
2965
  sandbox?: SandboxDescriptor;
2818
2966
  /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
2819
2967
  environment?: Environment;
2820
- /** Attempts per cell; unset, a compare with variants runs 4 a side and anything else once. */
2821
- runs?: number;
2968
+ /**
2969
+ * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
2970
+ * variants runs 4 a side and anything else once.
2971
+ */
2972
+ runs?: Runs;
2822
2973
  /** How each cell's attempts decide its outcome; default `'all'`. */
2823
2974
  gate?: GatePolicy;
2824
2975
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
@@ -2836,6 +2987,11 @@ type ReevalConfig = {
2836
2987
  capture?: CaptureOptions;
2837
2988
  /** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
2838
2989
  question?: QuestionOptions;
2990
+ /**
2991
+ * When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
2992
+ * the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
2993
+ */
2994
+ snapshot?: SnapshotOption;
2839
2995
  /** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
2840
2996
  concurrency?: number;
2841
2997
  /** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
@@ -2920,9 +3076,9 @@ type AttemptError = {
2920
3076
  };
2921
3077
  /**
2922
3078
  * One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
2923
- * reeval's MCP timer, which every stdio server runs through. An agent CLI that runs a process per
2924
- * turn (Antigravity, pi, Codex) starts its servers on every turn; one that keeps a session starts
2925
- * them once, on the first. A server that doesn't answer within 60s fails the attempt as an infra
3079
+ * reeval's MCP timer, which every stdio server runs through. An agent CLI that starts one process
3080
+ * per turn starts its servers on every turn; a CLI that keeps a session starts them on the first
3081
+ * turn only. A server that doesn't answer within 60s fails the attempt as an infra
2926
3082
  * error naming it; one whose client closes it first doesn't. `mcpStartup` is absent when the
2927
3083
  * environment has no stdio MCP server, or the sandbox has no `node` to run the timer.
2928
3084
  */
@@ -2935,7 +3091,7 @@ type McpStartup = {
2935
3091
  */
2936
3092
  ms: number | null;
2937
3093
  };
2938
- /** Where an eval's time went, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
3094
+ /** The duration of each evaluation phase, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
2939
3095
  type PhaseTimings = {
2940
3096
  setupMs: number;
2941
3097
  agentMs: number;
@@ -2945,16 +3101,28 @@ type PhaseTimings = {
2945
3101
  };
2946
3102
  type CellRole = 'control' | 'variant';
2947
3103
  /**
2948
- * One cell of an eval: a harness, an environment (`-` when none is set) and that environment's
2949
- * image (`sandbox` id: tag, dockerfile and install). With `compare`, the control is `baseline`
2950
- * (or the cascaded environment, or an empty `baseline`) and each variant is that environment on
2951
- * its own, including its sandbox.
3104
+ * A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
3105
+ * environment's sandbox id. With `compare`, the control is `baseline` (or the cascaded
3106
+ * environment, or an empty `baseline`) and each variant is that environment on its own,
3107
+ * including its sandbox.
2952
3108
  */
2953
3109
  type Cell = {
3110
+ /** The harness id, the environment name, and the sandbox id, joined with `/`. */
2954
3111
  key: string;
3112
+ /**
3113
+ * The harness id: `id` when set, otherwise `<kind>:<model>` (`<kind>` alone without a model),
3114
+ * with `[effort=<value>]` appended whenever `effort` is set.
3115
+ */
2955
3116
  harness: string;
2956
- /** The environment's own name, also on a variant cell (`with-house-greeting`, with no prefix). */
3117
+ /** The environment's own name, also on a variant cell (`with-skill`, with no prefix). */
2957
3118
  environment: string;
3119
+ /**
3120
+ * The sandbox id is `id` when set, or `fake` for the fake sandbox. A Docker sandbox starts with
3121
+ * the engine (`docker` by default). It adds `:` and the sanitized final component of `image` or
3122
+ * `from`, or for `dockerfile` the sanitized filename (the parent directory name when that
3123
+ * filename is `Dockerfile` and a parent is present). Non-empty `install` steps append `+` and an
3124
+ * eight-character hash.
3125
+ */
2958
3126
  sandbox: string;
2959
3127
  role: CellRole;
2960
3128
  /** The variant's name, for variant cells. */
@@ -2997,9 +3165,9 @@ type EnvironmentRecord = {
2997
3165
  name: string;
2998
3166
  /** The previous `extendEnvironment` layer's name, else null. */
2999
3167
  parent: string | null;
3000
- /** Keys a later layer replaced, e.g. `env.KEY`, `skills.genkit`, `mcp.docs`, `rules`. */
3168
+ /** Keys a later layer replaced, for example `env.KEY`, `skills.typescript`, `mcp.docs`, or `rules`. */
3001
3169
  overrides: string[];
3002
- /** Covers every field but the rules file's path; secret names are included, values never are. The image is the sandbox's. */
3170
+ /** Covers every field but the rules file's path; secret names are included; values never are. The image is the sandbox's. */
3003
3171
  digest: string;
3004
3172
  setup: string[];
3005
3173
  skills: Array<{
@@ -3013,7 +3181,7 @@ type EnvironmentRecord = {
3013
3181
  /** The plugin the skill came with; unset for a skill from `skills`. */
3014
3182
  plugin?: string;
3015
3183
  }>;
3016
- /** Every MCP server, a plugin's as `<plugin>:<server>`. */
3184
+ /** The names of the MCP servers the agent was given. */
3017
3185
  mcp: string[];
3018
3186
  /** The plugins placed, each with what it placed; in the digest. Unset when there are none. */
3019
3187
  plugins?: PluginRecord[];
@@ -3038,9 +3206,9 @@ type EnvironmentRecord = {
3038
3206
  /** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
3039
3207
  egress: EgressPolicy;
3040
3208
  /**
3041
- * The git repository the workspace started as: the repo as written (a local one relative to the
3042
- * project root), the ref and path asked for (null when unset), the commit the ref resolved to
3043
- * and the hash of the tree that was copied in, which is in the digest.
3209
+ * This records the Git repository the workspace started as: the repo as written (a local one
3210
+ * relative to the project root), the ref and path asked for (null when unset), the commit the
3211
+ * ref resolved to, and the hash of the tree that was copied in, which is in the digest.
3044
3212
  */
3045
3213
  workspace?: {
3046
3214
  source: string;
@@ -3057,8 +3225,8 @@ type EnvironmentRecord = {
3057
3225
  * and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
3058
3226
  * can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
3059
3227
  * the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
3060
- * harness always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
3061
- * out.
3228
+ * harness always tries (its CLI's telemetry, say), kept out of `blocked`, so `blocked` contains
3229
+ * only unexpected refusals.
3062
3230
  */
3063
3231
  type EgressRecord = {
3064
3232
  policy: EgressPolicy;
@@ -3096,7 +3264,7 @@ type AttemptImage = {
3096
3264
  /** Content-addressed image id (`sha256:…`). */
3097
3265
  id: string;
3098
3266
  agentCli: string | null;
3099
- /** The harness image `install` and plugin builds were baked onto: `ref` itself without either. */
3267
+ /** The harness image the `install` layers were baked onto: `ref` itself without any. */
3100
3268
  base: string;
3101
3269
  /** The `docker({ install })` commands, in order. */
3102
3270
  install: string[];
@@ -3131,11 +3299,26 @@ type SetupProcess = {
3131
3299
  /** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
3132
3300
  secrets: string[];
3133
3301
  };
3134
- /** One `expect` or `expect.soft` matcher call an attempt made. */
3302
+ /** One `c.serve` call that listened. */
3303
+ type ServeRecord = {
3304
+ /** The command as the eval passed it. */
3305
+ command: string;
3306
+ /** The port it listens on, which the handle and `c.capture` use. */
3307
+ port: number;
3308
+ /** The `PORT` it was given: the port the eval passed, or the one reeval allocated. */
3309
+ portEnv: number;
3310
+ /** It listened on a port other than `PORT`, the one that started listening after it did. */
3311
+ detected: boolean;
3312
+ /** The address it listens on there, e.g. `127.0.0.1`, `::1` or `0.0.0.0`; empty when the sandbox can't tell. */
3313
+ address: string;
3314
+ /** From the command's start until it listened. */
3315
+ readyMs: number;
3316
+ };
3317
+ /** One `expect` or `expect.require` matcher call an attempt made, or a check another language made. */
3135
3318
  type AssertionRecord = {
3136
3319
  /**
3137
- * `expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or
3138
- * the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt.
3320
+ * `label` is the supplied `expect` label. Without one, it is the matcher plus its first
3321
+ * argument or named subject, cut at 60 characters. Repeats get ` #2`, ` #3`, and later suffixes.
3139
3322
  */
3140
3323
  label: string;
3141
3324
  /** The eval gave the label. */
@@ -3144,12 +3327,81 @@ type AssertionRecord = {
3144
3327
  matcher: string;
3145
3328
  /** `error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report). */
3146
3329
  status: 'pass' | 'fail' | 'error';
3330
+ /**
3331
+ * A failure didn't end the eval's body: an `expect`. False for `expect.require` and for a check
3332
+ * that threw, such as Python's `assert`, which end the body when they fail.
3333
+ */
3147
3334
  soft: boolean;
3335
+ /** `path:line` of the call in the eval file, relative to the project root, when reeval saw it. */
3336
+ at?: string;
3148
3337
  /** Why it failed or couldn't answer, cut at 1 KB. */
3149
3338
  message?: string;
3150
3339
  /** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
3151
3340
  durationMs?: number;
3152
3341
  };
3342
+ /**
3343
+ * A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
3344
+ * folder), or why it wasn't.
3345
+ */
3346
+ type Snapshot = {
3347
+ /** `final` after the checks, or `turn-<n>` after the nth `c.send`. */
3348
+ name: string;
3349
+ /** A plain directory, or a gzipped tarball with `snapshot: { compress: true }`. */
3350
+ form: 'dir' | 'tar.gz';
3351
+ /**
3352
+ * Where it is, relative to the run's directory in `.reeval/runs`:
3353
+ * `<eval>/<cell>/<n>/snapshots/<name>` (with `.tar.gz` when compressed); null when it wasn't
3354
+ * taken.
3355
+ */
3356
+ path: string | null;
3357
+ /** Its size on disk: the directory's files, or the tarball; 0 when it wasn't taken. */
3358
+ bytes: number;
3359
+ /** Regular files and symbolic links in it. */
3360
+ files: number;
3361
+ /** Directories left out by name, such as `node_modules/` and `.git/`, at most 50. */
3362
+ excluded: string[];
3363
+ /** Text files whose secret-looking values were masked, at most 50. */
3364
+ masked: string[];
3365
+ /** Binary files left out because they held a secret-looking value, at most 50. */
3366
+ dropped: string[];
3367
+ /**
3368
+ * Why it wasn't taken: `size` when the workspace was over its form's cap, `error` when the
3369
+ * sandbox couldn't archive it; null when it was.
3370
+ */
3371
+ skipped: {
3372
+ cause: 'size' | 'error';
3373
+ message: string;
3374
+ } | null;
3375
+ };
3376
+ /** `snapshot`: the shorthand, or the object that also picks the form. */
3377
+ type SnapshotOption = boolean | 'turns' | SnapshotOptions;
3378
+ /** `snapshot` as an object, merged field by field over the levels above. */
3379
+ type SnapshotOptions = {
3380
+ /** `true` after the checks, `'turns'` after each `c.send` too, `false` never. Default `true`. */
3381
+ when?: boolean | 'turns';
3382
+ /** Write each as `<name>.tar.gz` instead of a directory. Default `false`. */
3383
+ compress?: boolean;
3384
+ };
3385
+ /**
3386
+ * One instruction of a capture, as it ended. Its screenshot is `artifacts/<name>.step-<index>.png`,
3387
+ * beside the capture's own `artifacts/<name>.png`.
3388
+ */
3389
+ type CaptureStep = {
3390
+ /** The capture's name. */
3391
+ of: string;
3392
+ /** Its place among the capture's goal strings, from 1. */
3393
+ index: number;
3394
+ instruction: string;
3395
+ /** Moves made on the page for it. */
3396
+ moves: number;
3397
+ /** Milliseconds from its first look at the page to its screenshot. */
3398
+ ms: number;
3399
+ /**
3400
+ * `final`: the model marked a move as finishing it. `done`: the model said the goal was met.
3401
+ * `stopped`: it stopped the capture (its timeout, the move cap, a repeated move or an error).
3402
+ */
3403
+ ended: 'final' | 'done' | 'stopped';
3404
+ };
3153
3405
  /** Which environment ran as the control, so the studio can show it. */
3154
3406
  type EvalCompareMeta = {
3155
3407
  baseline: string;
@@ -3160,16 +3412,20 @@ type EvalCompareMeta = {
3160
3412
  from?: 'environment' | 'empty';
3161
3413
  };
3162
3414
  /**
3163
- * A sandbox with more privilege than the default `reeval` user: `docker({ root: true })`, or
3164
- * `docker({ capabilities })`. One per level that set such a sandbox, however many evals inherit it.
3415
+ * A `SandboxWarning` describes a sandbox with more privilege than the default `reeval` user:
3416
+ * `docker({ root: true })` or `docker({ capabilities })`. One per level that set such a sandbox,
3417
+ * however many evaluations inherit it.
3165
3418
  */
3166
3419
  type SandboxWarning = {
3167
3420
  kind: 'root' | 'capabilities';
3168
3421
  /** Which level set the sandbox. */
3169
3422
  level: 'config' | 'profile' | 'group' | 'eval';
3170
- /** That level as reeval names it: `reeval.config.ts`, `profiles.pr.defaults`, `group("net")` or `evaluate("installs deps")`. */
3423
+ /**
3424
+ * The value is the level as reeval names it: the `reeval.config.ts` file, `profiles.pr.defaults`,
3425
+ * `group("net")`, or `evaluate("installs deps")`.
3426
+ */
3171
3427
  where: string;
3172
- /** The eval file that sets it, for a group or eval; null for the config or a profile. */
3428
+ /** The evaluation file that sets it, for a group or evaluation; null for the config or a profile. */
3173
3429
  file: string | null;
3174
3430
  /** The capabilities added back, for `capabilities`. */
3175
3431
  capabilities?: string[];
@@ -3185,5 +3441,5 @@ type ResolvedGateway = {
3185
3441
  }>;
3186
3442
  };
3187
3443
  //#endregion
3188
- export { FakeVote as $, VertexPublisher as $t, Diff as A, VERDICT_TEXT as An, Profile as At, EnvironmentInput as B, effortSample as Bn, Secret as Bt, CodexHarnessDescriptor as C, EFFORT_MIN_RUNS as Cn, Outcome as Ct, ConfigHelpers as D, EffortVerdict as Dn, PinnableKey as Dt, CompareOptions as E, EffortSample as En, PiHarnessDescriptor as Et, EgressPolicy as F, contrastEffort as Fn, QuestionStateOptions as Ft, ExecResult as G, formatSeconds as Gn, TokenUsage as Gt, ErrorKind as H, effortVerdict as Hn, ServeHandle as Ht, EgressRecord as I, directional as In, ReevalConfig as It, FakeAnswer as J, DeclaredNode as Jn, TurnData as Jt, ExecRetry as K, tokenValues as Kn, TranscriptEvent as Kt, EnvVars as L, effortChangeLabel as Ln, ResolvedGateway as Lt, DirectoryOptions as M, cellContrast as Mn, ProfileLimit as Mt, DockerSandboxDescriptor as N, cellEffort as Nn, Question as Nt, ConfigJudgeOptions as O, Spread as On, PluginPart as Ot, Duration as P, cellSamples as Pn, QuestionOptions as Pt, FakeScriptStep as Q, VertexGatewayDescriptor as Qt, Environment as R, effortLabel as Rn, SandboxDescriptor as Rt, ClaudeCodeHarnessDescriptor as S, EFFORT_ALPHA as Sn, OpenRouterGatewayDescriptor as St, CompareCut as T, EffortMetric as Tn, PairVote as Tt, EvalFn as U, formatCount as Un, SetupStep as Ut, EnvironmentSpec as V, effortSummary as Vn, SecretTemplate as Vt, EvalOptions as W, formatRatio as Wn, SkillSource as Wt, FakeJudgeModel as X, VercelGatewayDescriptor as Xt, FakeHarnessDescriptor as Y, TurnStatus as Yt, FakeSandboxDescriptor as Z, Verdict as Zt, CaptureWait as _, RunRecord as _n, JudgeOptions as _t, AttemptContext as a, CellRecord as an, GroupOptions as at, CellRole as b, CellEffort as bn, McpServer as bt, BlindPair as c, FamilyLift as cn, HarnessEdit as ct, CaptureCall as d, FamilyRecord as dn, HostFile as dt, Viewport as en, FxHarnessDescriptor as et, CaptureInstruction as f, FileRef as fn, JSONValue as ft, CaptureUseScript as g, Lift as gn, JudgeModel as gt, CaptureUseAction as h, JudgmentRecord as hn, JudgeFailure as ht, AttemptAnswer as i, AttemptRecord as in, GrokBuildHarnessDescriptor as it, DirectGatewayDescriptor as j, bothUnderATenth as jn, ProfileDefaults as jt, CursorHarnessDescriptor as k, TOKENS_UNREPORTED as kn, PluginSource as kt, Budget as l, FamilyLiftSummary as ln, HarnessOption as lt, CaptureUse as m, HistoryEntry as mn, JudgeCallOptions as mt, AnalysisConfig as n, WebEventType as nn, GatewayName as nt, BackgroundHandle as o, CheckTally as on, HarnessData as ot, CaptureOptions as p, HistoryCell as pn, JudgeArtifactRef as pt, FakeAnalysisModel as q, DeclarationTree as qn, Turn as qt, AntigravityHarnessDescriptor as r, ArtifactRef as rn, GooseHarnessDescriptor as rt, BlindAttempt as s, EvalEntry as sn, HarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, GatewayDescriptor as tt, Capture as u, FamilyRate as un, HistoryConfig as ut, CaptureWindow as v, StackSummary as vn, JudgeState as vt, CompareAnswer as w, EffortContrast as wn, PairFactor as wt, ClaudeCodeEffort as x, DIRECTIONAL_MIN as xn, OpenCodeHarnessDescriptor as xt, Cell as y, UpstreamCount as yn, JudgeStateOptions as yt, EnvironmentHelpers as z, effortOf as zn, SandboxWarning as zt };
3189
- //# sourceMappingURL=types-coMWHaU2.d.ts.map
3444
+ export { FakeAnswer as $, tokenValues as $n, TranscriptEvent as $t, CompareCut as A, EFFORT_MIN_RUNS as An, PairFactor as At, EgressPolicy as B, cellEffort as Bn, QuestionInputs as Bt, CaptureWindow as C, Lift as Cn, JudgeOptions as Ct, ClaudeCodeHarnessDescriptor as D, CellEffort as Dn, OpenCodeHarnessDescriptor as Dt, ClaudeCodeEffort as E, UpstreamCount as En, McpServer as Et, Diff as F, Spread as Fn, PluginSource as Ft, EnvironmentInput as G, effortLabel as Gn, SandboxWarning as Gt, EnvVars as H, contrastEffort as Hn, ReevalConfig as Ht, DirectGatewayDescriptor as I, TOKENS_UNREPORTED as In, Profile as It, EvalFn as J, effortSummary as Jn, ServeHandle as Jt, EnvironmentSpec as K, effortOf as Kn, Secret as Kt, DirectoryOptions as L, VERDICT_TEXT as Ln, ProfileDefaults as Lt, ConfigHelpers as M, EffortMetric as Mn, PiHarnessDescriptor as Mt, ConfigJudgeOptions as N, EffortSample as Nn, PinnableKey as Nt, CodexHarnessDescriptor as O, DIRECTIONAL_MIN as On, OpenRouterGatewayDescriptor as Ot, CursorHarnessDescriptor as P, EffortVerdict as Pn, PluginPart as Pt, FakeAnalysisModel as Q, formatSeconds as Qn, TokenUsage as Qt, DockerSandboxDescriptor as R, bothUnderATenth as Rn, ProfileLimit as Rt, CaptureWait as S, JudgmentRecord as Sn, JudgeModel as St, CellRole as T, StackSummary as Tn, JudgeSubject as Tt, Environment as U, directional as Un, ResolvedGateway as Ut, EgressRecord as V, cellSamples as Vn, QuestionOptions as Vt, EnvironmentHelpers as W, effortChangeLabel as Wn, SandboxDescriptor as Wt, ExecResult as X, formatCount as Xn, SkillSource as Xt, EvalOptions as Y, effortVerdict as Yn, SetupStep as Yt, ExecRetry as Z, formatRatio as Zn, StatusRange as Zt, CapturePage as _, FamilyRate as _n, JSONValue as _t, AttemptContext as a, VertexGatewayDescriptor as an, FxHarnessDescriptor as at, CaptureUseAction as b, HistoryCell as bn, JudgeFailure as bt, BlindPair as c, Wait as cn, GooseHarnessDescriptor as ct, CaptureCall as d, AttemptRecord as dn, HarnessData as dt, Turn as en, DeclarationTree as er, FakeHarnessDescriptor as et, CaptureConsoleMessage as f, CellRecord as fn, HarnessDescriptor as ft, CaptureOptions as g, FamilyLiftSummary as gn, HostFile as gt, CaptureKey as h, FamilyLift as hn, HistoryConfig as ht, AttemptAnswer as i, Verdict as in, FakeVote as it, CompareOptions as j, EffortContrast as jn, PairVote as jt, CompareAnswer as k, EFFORT_ALPHA as kn, Outcome as kt, Budget as l, WebEventType as ln, GrokBuildHarnessDescriptor as lt, CaptureInstruction as m, EvalEntry as mn, HarnessOption as mt, AnalysisConfig as n, TurnStatus as nn, FakeSandboxDescriptor as nt, BackgroundHandle as o, VertexPublisher as on, GatewayDescriptor as ot, CaptureFailedRequest as p, CheckTally as pn, HarnessEdit as pt, ErrorKind as q, effortSample as qn, SecretTemplate as qt, AntigravityHarnessDescriptor as r, VercelGatewayDescriptor as rn, FakeScriptStep as rt, BlindAttempt as s, Viewport as sn, GatewayName as st, AgentHarnessDescriptor as t, TurnData as tn, DeclaredNode as tr, FakeJudgeModel as tt, Capture as u, ArtifactRef as un, GroupOptions as ut, CapturePageFile as v, FamilyRecord as vn, JudgeArtifactRef as vt, Cell as w, RunRecord as wn, JudgeState as wt, CaptureUseScript as x, HistoryEntry as xn, JudgeInputs as xt, CaptureUse as y, FileRef as yn, JudgeCallOptions as yt, Duration as z, cellContrast as zn, Question as zt };
3445
+ //# sourceMappingURL=types-DDeCnNmp.d.ts.map