reeval 0.15.1 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. package/README.md +21 -14
  2. package/dist/{adapter-CbqbMR7y.js → adapter-CqUt4A0w.js} +3 -3
  3. package/dist/{adapter-CbqbMR7y.js.map → adapter-CqUt4A0w.js.map} +1 -1
  4. package/dist/{agent-Zgn-kiqq.js → agent-BtAbRTFq.js} +6 -6
  5. package/dist/{agent-Zgn-kiqq.js.map → agent-BtAbRTFq.js.map} +1 -1
  6. package/dist/{app-files-ChbvRx3P.js → app-files-CDttHPOz.js} +3 -3
  7. package/dist/app-files-CDttHPOz.js.map +1 -0
  8. package/dist/assemble-Bb7zqpRw.js +1 -0
  9. package/dist/{assemble-SKQGW_QF.js → assemble-J9rkE29m.js} +114 -31
  10. package/dist/assemble-J9rkE29m.js.map +1 -0
  11. package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
  12. package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
  13. package/dist/{attempt-host-BOp8JklD.js → attempt-host-DDedP7Qv.js} +17 -8
  14. package/dist/attempt-host-DDedP7Qv.js.map +1 -0
  15. package/dist/capture-Bvc4FtWT.js +15 -0
  16. package/dist/capture-Bvc4FtWT.js.map +1 -0
  17. package/dist/capture-D68TiQQy.js +1 -0
  18. package/dist/{capture-BM_3100g.js → capture-DheNqeYS.js} +1495 -203
  19. package/dist/capture-DheNqeYS.js.map +1 -0
  20. package/dist/{cli-PdYrcoKM.js → cli-KTHjNFoo.js} +329 -74
  21. package/dist/cli-KTHjNFoo.js.map +1 -0
  22. package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{colors-QV9dFVhN.js → colors-Y6O3mtl_.js} +20 -2
  28. package/dist/colors-Y6O3mtl_.js.map +1 -0
  29. package/dist/{command-9xm5ImqI.js → command-1FBECMs-.js} +16 -16
  30. package/dist/{command-9xm5ImqI.js.map → command-1FBECMs-.js.map} +1 -1
  31. package/dist/{command-BqBxLgTc.js → command-Ce03LLtg.js} +14 -14
  32. package/dist/{command-BqBxLgTc.js.map → command-Ce03LLtg.js.map} +1 -1
  33. package/dist/{command-4pJYVCT4.js → command-CztRuwUM.js} +12 -12
  34. package/dist/{command-4pJYVCT4.js.map → command-CztRuwUM.js.map} +1 -1
  35. package/dist/{command-DbSMcxZL.js → command-nNN0mx65.js} +7 -7
  36. package/dist/{command-DbSMcxZL.js.map → command-nNN0mx65.js.map} +1 -1
  37. package/dist/{commands-BU6YJCz9.js → commands-DP13JX9i.js} +24 -12
  38. package/dist/commands-DP13JX9i.js.map +1 -0
  39. package/dist/{commands-Bi2SEQtr.js → commands-ZwQUezI1.js} +1 -1
  40. package/dist/{config-k6m7XbAz.js → config-BjjDryd5.js} +464 -220
  41. package/dist/config-BjjDryd5.js.map +1 -0
  42. package/dist/{contrasts-DLZF4dGF.js → contrasts-DSsRZpR8.js} +2 -2
  43. package/dist/{contrasts-DLZF4dGF.js.map → contrasts-DSsRZpR8.js.map} +1 -1
  44. package/dist/cursor-catalog-D7bFLEAD.js +143 -0
  45. package/dist/cursor-catalog-D7bFLEAD.js.map +1 -0
  46. package/dist/{db-CpsifVln.js → db-B9Ityuit.js} +8 -8
  47. package/dist/{db-CpsifVln.js.map → db-B9Ityuit.js.map} +1 -1
  48. package/dist/db-Bk6LWwza.js +2 -0
  49. package/dist/debug-CAx3qcaP.js +1 -0
  50. package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
  51. package/dist/debug-CgG3T071.js.map +1 -0
  52. package/dist/default-effort-DK8gospa.js +91 -0
  53. package/dist/default-effort-DK8gospa.js.map +1 -0
  54. package/dist/default-effort-DmXHpqo9.js +1 -0
  55. package/dist/{docker-YFj4H1Cr.js → docker-BLCNfUie.js} +1 -1
  56. package/dist/{docker-B4-rx4dA.js → docker-SKpYOlGi.js} +5 -5
  57. package/dist/{docker-B4-rx4dA.js.map → docker-SKpYOlGi.js.map} +1 -1
  58. package/dist/{docker-51WvhN_X.js → docker-hwp3EucJ.js} +9 -5
  59. package/dist/docker-hwp3EucJ.js.map +1 -0
  60. package/dist/{doctor-BuQnfz4e.js → doctor-CT0Sgjng.js} +16 -16
  61. package/dist/{doctor-BuQnfz4e.js.map → doctor-CT0Sgjng.js.map} +1 -1
  62. package/dist/{environment-BOFgzFDg.js → environment-CZdercoj.js} +3 -3
  63. package/dist/{environment-BOFgzFDg.js.map → environment-CZdercoj.js.map} +1 -1
  64. package/dist/{errors-UjcLWelF.js → errors-BZVKVKYd.js} +15 -4
  65. package/dist/errors-BZVKVKYd.js.map +1 -0
  66. package/dist/{eval-source-Dje3rERU.js → eval-source-nZD7-FSz.js} +9 -8
  67. package/dist/eval-source-nZD7-FSz.js.map +1 -0
  68. package/dist/{evaluate-dqXccn4k.d.ts → evaluate-CNi0lZ7n.d.ts} +7 -6
  69. package/dist/{evaluate-BPP9s3s6.js → evaluate-fhC1PCd_.js} +345 -47
  70. package/dist/evaluate-fhC1PCd_.js.map +1 -0
  71. package/dist/{files-Clipx4OW.js → files-BGy05Dlr.js} +4 -4
  72. package/dist/{files-Clipx4OW.js.map → files-BGy05Dlr.js.map} +1 -1
  73. package/dist/{files-Bi461Be2.js → files-BIGxpL08.js} +25 -16
  74. package/dist/{files-Bi461Be2.js.map → files-BIGxpL08.js.map} +1 -1
  75. package/dist/files-C2RC7K4m.js +1 -0
  76. package/dist/files-C3tQtD26.js +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/{files-H24pz0hL.js → files-I1i6Xyh1.js} +4 -4
  80. package/dist/{files-H24pz0hL.js.map → files-I1i6Xyh1.js.map} +1 -1
  81. package/dist/files-dJmrfmOM.js +1 -0
  82. package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
  83. package/dist/gate-BC6SdQZs.js.map +1 -0
  84. package/dist/gateway.d.ts +1 -1
  85. package/dist/{git-CsVZkNNL.js → git-DR02tHtU.js} +2 -2
  86. package/dist/{git-CsVZkNNL.js.map → git-DR02tHtU.js.map} +1 -1
  87. package/dist/{harness-3NVyTSGK.js → harness-ylJLvrbp.js} +39 -5
  88. package/dist/{harness-3NVyTSGK.js.map → harness-ylJLvrbp.js.map} +1 -1
  89. package/dist/harness.d.ts +1 -1
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-4mwhNkvh.js → images---X8Wu9d.js} +8 -8
  92. package/dist/{images-4mwhNkvh.js.map → images---X8Wu9d.js.map} +1 -1
  93. package/dist/index.d.ts +71 -16
  94. package/dist/index.js +5 -5
  95. package/dist/init-BNTXrtMO.js +1 -0
  96. package/dist/{init-cOk3D44b.js → init-CA95ZnkV.js} +4 -4
  97. package/dist/{init-cOk3D44b.js.map → init-CA95ZnkV.js.map} +1 -1
  98. package/dist/{invocations-DW-lTk2f.js → invocations-BdoROAfb.js} +3 -3
  99. package/dist/{invocations-DW-lTk2f.js.map → invocations-BdoROAfb.js.map} +1 -1
  100. package/dist/{judge-CKdgEXN7.js → judge-Dp7FPJ1P.js} +3 -3
  101. package/dist/{judge-CKdgEXN7.js.map → judge-Dp7FPJ1P.js.map} +1 -1
  102. package/dist/judge.d.ts +2 -2
  103. package/dist/judge.js +1 -1
  104. package/dist/{live-XJQ6nqK4.js → live-CBFkdK1K.js} +3 -3
  105. package/dist/{live-XJQ6nqK4.js.map → live-CBFkdK1K.js.map} +1 -1
  106. package/dist/{matchers-DT2TaGix.js → matchers-Cv74mJRJ.js} +43 -13
  107. package/dist/matchers-Cv74mJRJ.js.map +1 -0
  108. package/dist/{model-Bq3G0OG1.js → model-6dfKFkQd.js} +4 -4
  109. package/dist/{model-Bq3G0OG1.js.map → model-6dfKFkQd.js.map} +1 -1
  110. package/dist/{model-CTnyvh7H.js → model-Be7fPvgi.js} +4 -51
  111. package/dist/model-Be7fPvgi.js.map +1 -0
  112. package/dist/{model-g3XnzUv0.js → model-CN4ccjNz.js} +1 -1
  113. package/dist/model-C_5TspZS.js +1 -0
  114. package/dist/{model-vRiUycWh.js → model-DkJVmVd2.js} +4 -4
  115. package/dist/{model-vRiUycWh.js.map → model-DkJVmVd2.js.map} +1 -1
  116. package/dist/{node-BVbbnFtv.js → node-D_Zkwhq2.js} +2 -2
  117. package/dist/{node-BVbbnFtv.js.map → node-D_Zkwhq2.js.map} +1 -1
  118. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  119. package/dist/object-BkXGPKWY.js.map +1 -0
  120. package/dist/{opencode-providers-Bz0l7OZI.js → opencode-providers-DPgdqRfg.js} +26 -2
  121. package/dist/{opencode-providers-Bz0l7OZI.js.map → opencode-providers-DPgdqRfg.js.map} +1 -1
  122. package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-CR00FWAa.js} +2 -2
  123. package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-CR00FWAa.js.map} +1 -1
  124. package/dist/{patch-DKJiC5Sq.js → patch-BJNdOZsf.js} +4 -4
  125. package/dist/patch-BJNdOZsf.js.map +1 -0
  126. package/dist/{plan-DdWkM6SW.js → plan-C1UpGZTk.js} +76 -12
  127. package/dist/plan-C1UpGZTk.js.map +1 -0
  128. package/dist/{plugins-D-CP6mMd.js → plugins-Cx6ReJWE.js} +24 -3
  129. package/dist/plugins-Cx6ReJWE.js.map +1 -0
  130. package/dist/{preflight-9ixuXzsx.js → preflight-BZ3L7P3a.js} +3 -3
  131. package/dist/{preflight-9ixuXzsx.js.map → preflight-BZ3L7P3a.js.map} +1 -1
  132. package/dist/{preflight-DTiVv0gY.js → preflight-CUooKgtT.js} +1 -1
  133. package/dist/{prepare-BfyikQit.js → prepare-BbBoTeDj.js} +7 -7
  134. package/dist/{prepare-BfyikQit.js.map → prepare-BbBoTeDj.js.map} +1 -1
  135. package/dist/{prepare-tmBBf4W8.js → prepare-CEZaEwaq.js} +2 -2
  136. package/dist/{provenance-BizwUn3h.js → provenance-BWvTIlQM.js} +2 -2
  137. package/dist/{provenance-BizwUn3h.js.map → provenance-BWvTIlQM.js.map} +1 -1
  138. package/dist/{providers-B6b9Tnyi.js → providers-B3yqakc7.js} +5 -5
  139. package/dist/{providers-B6b9Tnyi.js.map → providers-B3yqakc7.js.map} +1 -1
  140. package/dist/{providers-DkOupEBl.js → providers-DdyZW_ru.js} +1 -1
  141. package/dist/query-BkwQ0W-c.js +1 -0
  142. package/dist/{query-DyM-VExS.js → query-CZcqpOI9.js} +2 -2
  143. package/dist/{query-DyM-VExS.js.map → query-CZcqpOI9.js.map} +1 -1
  144. package/dist/{questions-BRhU8MzB.js → questions-CFU7-crW.js} +20 -19
  145. package/dist/questions-CFU7-crW.js.map +1 -0
  146. package/dist/{record-DZBCjuwm.js → record-Bu1Xi6MQ.js} +8 -4
  147. package/dist/record-Bu1Xi6MQ.js.map +1 -0
  148. package/dist/{remarks-C7i2LMRb.js → remarks-A3j-SkJ_.js} +42 -8
  149. package/dist/remarks-A3j-SkJ_.js.map +1 -0
  150. package/dist/remarks-Bp6AwPgb.js +1 -0
  151. package/dist/{remarks-D_3qhBWU.js → remarks-Dj_x9l5n.js} +4 -3
  152. package/dist/{remarks-D_3qhBWU.js.map → remarks-Dj_x9l5n.js.map} +1 -1
  153. package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
  154. package/dist/remarks-eBcrIZRu.js.map +1 -0
  155. package/dist/{remote-DFIzcx_6.js → remote-EIFHdMd8.js} +5 -5
  156. package/dist/{remote-DFIzcx_6.js.map → remote-EIFHdMd8.js.map} +1 -1
  157. package/dist/{resolve-DWSJpKHX.js → resolve-8H5rz8mi.js} +5 -5
  158. package/dist/{resolve-DWSJpKHX.js.map → resolve-8H5rz8mi.js.map} +1 -1
  159. package/dist/resolve-Tm9pX7Hv.js +1 -0
  160. package/dist/{results-BC3pcG_Z.js → results-E7JdVjq2.js} +9 -9
  161. package/dist/{results-BC3pcG_Z.js.map → results-E7JdVjq2.js.map} +1 -1
  162. package/dist/{resume-j-rcIryi.js → resume-Blu4yIcq.js} +4 -4
  163. package/dist/{resume-j-rcIryi.js.map → resume-Blu4yIcq.js.map} +1 -1
  164. package/dist/{runtime-d56Gg3J5.js → runtime-CNbsMJQ7.js} +111 -37
  165. package/dist/runtime-CNbsMJQ7.js.map +1 -0
  166. package/dist/{sandbox-Cmln11gU.js → sandbox-RButGfIE.js} +2 -2
  167. package/dist/{sandbox-Cmln11gU.js.map → sandbox-RButGfIE.js.map} +1 -1
  168. package/dist/sandbox.d.ts +1 -1
  169. package/dist/sandbox.js +1 -1
  170. package/dist/server-Dqrq-Sia.js +1 -0
  171. package/dist/{server-BOoH10J8.js → server-XgCYs_MF.js} +35 -11
  172. package/dist/server-XgCYs_MF.js.map +1 -0
  173. package/dist/{source-wD04-rRG.js → source-BOkbJCD2.js} +17 -3
  174. package/dist/{source-wD04-rRG.js.map → source-BOkbJCD2.js.map} +1 -1
  175. package/dist/source-DooSTtyz.js +1 -0
  176. package/dist/src-1M0srjji.js +1 -0
  177. package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
  178. package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
  179. package/dist/{src-D91Hd6hT.js → src-BJCKWacl.js} +38 -18
  180. package/dist/src-BJCKWacl.js.map +1 -0
  181. package/dist/src-BZlhaDhE.js +2 -0
  182. package/dist/{src-BO8AYqJ5.js → src-BkhEPVJZ.js} +12 -3
  183. package/dist/{src-BO8AYqJ5.js.map → src-BkhEPVJZ.js.map} +1 -1
  184. package/dist/{src-C7upnZSd.js → src-Bp6X4193.js} +1 -1
  185. package/dist/{src-BYLM3hEz.js → src-C2Japg_I.js} +21 -10
  186. package/dist/{src-BYLM3hEz.js.map → src-C2Japg_I.js.map} +1 -1
  187. package/dist/src-C5q3_fRV.js +3 -0
  188. package/dist/src-C9EHSYyP.js +2 -0
  189. package/dist/src-CJdpsrXi.js +2 -0
  190. package/dist/{src-1lpTocbd.js → src-D1ET6oPL.js} +3 -27
  191. package/dist/{src-1lpTocbd.js.map → src-D1ET6oPL.js.map} +1 -1
  192. package/dist/{src-Bp4aN3rA.js → src-D4CZjMQ4.js} +1 -1
  193. package/dist/{src-BYPMM1x7.js → src-GkJ-1Rdy.js} +1 -1
  194. package/dist/{src-kYeaJJrl.js → src-OOKE-5GA.js} +1 -1
  195. package/dist/state-BCyNaz13.js +1 -0
  196. package/dist/{state-DrJ687ut.js → state-D6-tvqtp.js} +8 -8
  197. package/dist/{state-DrJ687ut.js.map → state-D6-tvqtp.js.map} +1 -1
  198. package/dist/suite-BiWajSlA.js +2 -0
  199. package/dist/{suite-Co8cwelb.js → suite-DYI1k2Ie.js} +22 -5
  200. package/dist/suite-DYI1k2Ie.js.map +1 -0
  201. package/dist/{summary-DLFHVoux.js → summary-CN2aqAUz.js} +59 -17
  202. package/dist/summary-CN2aqAUz.js.map +1 -0
  203. package/dist/tarball-CHcahyNK.js +258 -0
  204. package/dist/tarball-CHcahyNK.js.map +1 -0
  205. package/dist/testing.d.ts +96 -44
  206. package/dist/testing.js +4 -4
  207. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  208. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  209. package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
  210. package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
  211. package/dist/{types-coMWHaU2.d.ts → types-DDeCnNmp.d.ts} +437 -181
  212. package/dist/view/index.d.ts +34 -5
  213. package/dist/view/index.js +8 -7
  214. package/dist/views-7bNY17Z6.js +2 -0
  215. package/dist/{views-BFO1BLKu.js → views-Bd-H71_t.js} +5 -5
  216. package/dist/{views-BFO1BLKu.js.map → views-Bd-H71_t.js.map} +1 -1
  217. package/dist/{worker-entry-CJtdsKBP.js → worker-entry-BIHJqE_z.js} +34 -13
  218. package/dist/worker-entry-BIHJqE_z.js.map +1 -0
  219. package/dist/workspace-CxwHv8mG.js +1 -0
  220. package/dist/{workspace-BRxjfa8N.js → workspace-Sy5g8zW9.js} +4 -4
  221. package/dist/{workspace-BRxjfa8N.js.map → workspace-Sy5g8zW9.js.map} +1 -1
  222. package/package.json +1 -1
  223. package/skills/eval-rubric/SKILL.md +11 -0
  224. package/dist/app-files-ChbvRx3P.js.map +0 -1
  225. package/dist/assemble-SKQGW_QF.js.map +0 -1
  226. package/dist/assemble-XBWuK3Ed.js +0 -1
  227. package/dist/attempt-host-BOp8JklD.js.map +0 -1
  228. package/dist/capture-BM_3100g.js.map +0 -1
  229. package/dist/capture-DSK8O5bA.js +0 -1
  230. package/dist/cli-PdYrcoKM.js.map +0 -1
  231. package/dist/cli-spec-D0qyrPZi.js.map +0 -1
  232. package/dist/client-Ct6s6OnF.js.map +0 -1
  233. package/dist/colors-QV9dFVhN.js.map +0 -1
  234. package/dist/commands-BU6YJCz9.js.map +0 -1
  235. package/dist/config-k6m7XbAz.js.map +0 -1
  236. package/dist/db-DkD-F-X-.js +0 -2
  237. package/dist/debug-BhizgAsU.js.map +0 -1
  238. package/dist/debug-DeM5pswK.js +0 -1
  239. package/dist/docker-51WvhN_X.js.map +0 -1
  240. package/dist/errors-UjcLWelF.js.map +0 -1
  241. package/dist/eval-source-Dje3rERU.js.map +0 -1
  242. package/dist/evaluate-BPP9s3s6.js.map +0 -1
  243. package/dist/files-BAt0i00X.js +0 -1
  244. package/dist/files-C2h2Cw4F.js +0 -1
  245. package/dist/files-Ca0d6Uq3.js.map +0 -1
  246. package/dist/files-DmUrRmmL.js +0 -1
  247. package/dist/init-BDEOopK7.js +0 -1
  248. package/dist/matchers-DT2TaGix.js.map +0 -1
  249. package/dist/model-CTnyvh7H.js.map +0 -1
  250. package/dist/model-D-SWrLZG.js +0 -1
  251. package/dist/object-CLf2-tNq.js.map +0 -1
  252. package/dist/pairs-puYi-zj2.js.map +0 -1
  253. package/dist/patch-DKJiC5Sq.js.map +0 -1
  254. package/dist/plan-DdWkM6SW.js.map +0 -1
  255. package/dist/plugins-D-CP6mMd.js.map +0 -1
  256. package/dist/query-Clm2mNSm.js +0 -1
  257. package/dist/questions-BRhU8MzB.js.map +0 -1
  258. package/dist/record-DZBCjuwm.js.map +0 -1
  259. package/dist/remarks-BwJ8pcTQ.js +0 -1
  260. package/dist/remarks-C7i2LMRb.js.map +0 -1
  261. package/dist/remarks-dA8LyoPc.js.map +0 -1
  262. package/dist/resolve--rQZ8u6Z.js +0 -1
  263. package/dist/runtime-d56Gg3J5.js.map +0 -1
  264. package/dist/server-BOoH10J8.js.map +0 -1
  265. package/dist/server-DpuVAbrO.js +0 -1
  266. package/dist/source-BFd6f0-R.js +0 -1
  267. package/dist/src-6_zz7oJs.js +0 -2
  268. package/dist/src-D91Hd6hT.js.map +0 -1
  269. package/dist/src-DH-9IO7f.js +0 -1
  270. package/dist/src-HSxBazcg.js +0 -3
  271. package/dist/src-LD1BAJFa.js +0 -2
  272. package/dist/src-dlX-Lmxg.js +0 -2
  273. package/dist/state-B5PLm5qY.js +0 -1
  274. package/dist/suite-22Kt9Grx.js +0 -2
  275. package/dist/suite-Co8cwelb.js.map +0 -1
  276. package/dist/summary-DLFHVoux.js.map +0 -1
  277. package/dist/views-BOf8qEL8.js +0 -2
  278. package/dist/worker-entry-CJtdsKBP.js.map +0 -1
  279. package/dist/workspace-DDHzkMjm.js +0 -1
@@ -1,29 +1,29 @@
1
- import { n as isOneOf, t as isObjectLike } from "./object-CLf2-tNq.js";
1
+ import { n as isOneOf, t as isObjectLike } from "./object-BkXGPKWY.js";
2
2
  import { n as UsageError } from "./errors-B6BSMs03.js";
3
3
  import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-B_lvGqWz.js";
4
- import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-BOFgzFDg.js";
5
- import { B as gatePolicySchema, D as pluginServerName, L as egressPolicySchema, V as gateProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-D-CP6mMd.js";
6
- import { _ as gatewayDescriptorSchema, f as FACTORY, g as directOffPin, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayEntryDataSchema, y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
4
+ import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-CZdercoj.js";
5
+ import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-Cx6ReJWE.js";
6
+ import { _ as GATEWAY_NAMES, b as gatewayEntryDataSchema, f as FACTORY, g as harnessProblem, h as harnessDescriptorSchema, u as opencodeModelError, v as directOffPin, x as gatewayNameSchema, y as gatewayDescriptorSchema } from "./harness-ylJLvrbp.js";
7
7
  import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
8
- import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
9
- import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-CKdgEXN7.js";
8
+ import { a as providerHosts, i as providerEndpoints, l as resolveOpenCodeProvider, o as providerModelResponses, s as providerServedModel } from "./opencode-providers-DPgdqRfg.js";
9
+ import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-Dp7FPJ1P.js";
10
10
  import { a as VERTEX_ROUTES, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-DypYjx71.js";
11
11
  import { r as badCharacterAt } from "./redact-DVIaahLv.js";
12
12
  import { s as withoutBom } from "./text-DTigcA64.js";
13
13
  import { t as inCI } from "./env-CARj5Axq.js";
14
14
  import { u as ownFileURL } from "./root-Dlff-zp8.js";
15
- import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, dt as run, ft as shellQuote } from "./src-D91Hd6hT.js";
15
+ import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, ft as run, pt as shellQuote } from "./src-BJCKWacl.js";
16
16
  import { t as InfraError } from "./types-C5vI3g1C.js";
17
- import { i as pairFactors } from "./pairs-puYi-zj2.js";
17
+ import { n as gateLabel, u as pairFactors } from "./gate-BC6SdQZs.js";
18
18
  import "./stats-hfpuYsAM.js";
19
- import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CbqbMR7y.js";
20
- import { t as docker } from "./sandbox-Cmln11gU.js";
19
+ import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CqUt4A0w.js";
20
+ import { t as docker } from "./sandbox-RButGfIE.js";
21
21
  import { existsSync, readFileSync } from "node:fs";
22
22
  import { availableParallelism, tmpdir, totalmem } from "node:os";
23
23
  import { dirname, join, resolve, win32 } from "node:path";
24
24
  import { parseDocument } from "yaml";
25
25
  import { z } from "zod";
26
- import { isIP } from "node:net";
26
+ import { connect, isIP } from "node:net";
27
27
  import { createHash, randomUUID } from "node:crypto";
28
28
  import { spawn } from "node:child_process";
29
29
  import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
@@ -84,21 +84,43 @@ function decideState(json, files) {
84
84
  }
85
85
  //#endregion
86
86
  //#region src/contract/judge.ts
87
- const judgeStateSchema = z.strictObject({
87
+ /** The inputs every judge call and question shares; `c.judge()` adds `contents` and its own `artifacts`. */
88
+ const sharedInputFields = {
88
89
  task: z.boolean().optional(),
89
90
  output: z.boolean().optional(),
90
91
  files: z.boolean().optional(),
91
92
  diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
92
- contents: z.array(nonEmpty()).optional(),
93
93
  transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
94
- turns: z.enum(["last", "all"]).optional(),
95
94
  checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
95
+ };
96
+ const turnsSchema = z.enum(["last", "all"]);
97
+ const { task, output, files, diff, transcript, checks } = sharedInputFields;
98
+ const judgeInputsSchema = z.strictObject({
99
+ task,
100
+ output,
101
+ files,
102
+ diff,
103
+ contents: z.array(nonEmpty()).optional(),
104
+ transcript,
105
+ checks,
106
+ artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
96
107
  });
108
+ const judgeSubjectSchema = z.strictObject({
109
+ on: z.custom().optional(),
110
+ turns: turnsSchema.optional()
111
+ });
112
+ const INPUT_KEYS = Object.keys(judgeInputsSchema.shape);
113
+ /** True when an input's value turns it on: not `false`, unset or an empty list. */
114
+ const inputOn = (value) => value !== void 0 && value !== false && !(Array.isArray(value) && value.length === 0);
115
+ /** The input keys an options object turns on, in `keys`' order. */
116
+ const inputsOn = (value, keys) => keys.filter((k) => inputOn(value[k]));
117
+ /** `a`, `b` and `c` as prose: "a, b or c". */
118
+ const orList = (items) => items.length < 2 ? items[0] ?? "" : `${items.slice(0, -1).join(", ")} or ${items.at(-1)}`;
119
+ const NO_JUDGE_INPUT = `c.judge() shows the judge nothing: turn on at least one input, such as { diff: true } or { contents: ['src/**'] }. The inputs are ${orList(INPUT_KEYS)}.`;
97
120
  const judgeFields = {
98
121
  threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional(),
99
122
  providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional(),
100
123
  maxConcurrent: z.number().int().min(1).optional(),
101
- state: judgeStateSchema.optional(),
102
124
  gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional()
103
125
  };
104
126
  const fakeJudgeDataSchema = z.strictObject({
@@ -114,18 +136,13 @@ const judgeOptionsSchema = z.strictObject({
114
136
  model: z.custom().optional(),
115
137
  ...judgeFields
116
138
  });
117
- const judgeCallSchema = judgeOptionsSchema.extend({
118
- on: z.custom().optional(),
119
- artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
120
- });
121
- const STATE_KEYS$1 = Object.keys(judgeStateSchema.shape);
122
- const JUDGE_HINTS = {
123
- see: "state",
124
- shows: "state",
125
- show: "state",
126
- input: "state"
127
- };
128
- const VALUES = {
139
+ const judgeCallSchema = judgeOptionsSchema.extend(judgeInputsSchema.shape).extend(judgeSubjectSchema.shape);
140
+ const CALL_ONLY_KEYS = [
141
+ ...INPUT_KEYS,
142
+ "on",
143
+ "turns"
144
+ ];
145
+ const VALUES$1 = {
129
146
  task: "true or false",
130
147
  output: "true or false",
131
148
  files: "true or false",
@@ -162,7 +179,10 @@ function judgeProblem(judge, site) {
162
179
  return `${key ? `${base}.${key}` : base} in ${site.at === "config" ? site.file : site.where}`;
163
180
  };
164
181
  const loc = site.at === "call" ? " in c.judge()" : site.at === "options" ? ` in ${site.where}` : ` in ${site.path === "judge" ? "" : `${site.path.replace(/\.judge$/, "")} in `}${site.file}`;
165
- if (typeof judge !== "object" || judge === null || Array.isArray(judge)) return `${field()} must be an object such as { threshold: 0.8 } (got ${shown(judge)})`;
182
+ if (typeof judge !== "object" || judge === null || Array.isArray(judge)) {
183
+ const example = site.at === "call" ? "{ diff: true }" : "{ threshold: 0.8 }";
184
+ return `${field()} must be an object such as ${example} (got ${shown(judge)})`;
185
+ }
166
186
  const value = judge;
167
187
  const model = value.model;
168
188
  if (site.at === "config" && typeof model === "string") {
@@ -171,7 +191,7 @@ function judgeProblem(judge, site) {
171
191
  }
172
192
  const schema = site.at === "call" ? judgeCallSchema : site.at === "config" ? configJudgeDataSchema : judgeOptionsSchema;
173
193
  const parsed = schema.safeParse(judge);
174
- if (parsed.success) return void 0;
194
+ if (parsed.success) return site.at === "call" && !inputsOn(value, INPUT_KEYS).length ? NO_JUDGE_INPUT : void 0;
175
195
  const numericFake = site.at === "config" && typeof model === "object" && model?.kind === "fake-judge" && typeof model.score === "number";
176
196
  const issues = parsed.error.issues.filter((i) => !(numericFake && i.path[0] === "model"));
177
197
  if (!issues.length) return void 0;
@@ -194,29 +214,22 @@ function judgeProblem(judge, site) {
194
214
  const name = issues.find((i) => i.code === "unrecognized_keys" && i.path.length === 0)?.keys[0];
195
215
  if (name === void 0) return void 0;
196
216
  const allowed = Object.keys(schema.shape);
197
- const hint = JUDGE_HINTS[name] ?? (STATE_KEYS$1.includes(name) ? `state.${name}` : nearest(name, allowed));
198
- const why = site.at !== "call" && (name === "on" || name === "artifacts") ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
217
+ const hint = nearest(name, allowed);
218
+ const why = site.at !== "call" && CALL_ONLY_KEYS.includes(name) ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
199
219
  return `Unknown judge option "${name}"${loc}.${why}`;
200
220
  };
201
- const stateProblem = () => {
202
- const issue = issues.find((i) => i.path[0] === "state");
221
+ const inputProblem = () => {
222
+ const issue = issues.find((i) => typeof i.path[0] === "string" && i.path[0] in VALUES$1);
203
223
  if (!issue) return void 0;
204
- const { state } = value;
205
- if (typeof state !== "object" || state === null || Array.isArray(state)) return `${field("state")} must be an object such as { transcript: 'tools' } (got ${shown(state)})`;
206
- if (issue.code === "unrecognized_keys" && issue.path.length === 1) {
207
- const key = issue.keys[0] ?? "";
208
- const hint = nearest(key, STATE_KEYS$1);
209
- return `Unknown judge.state key "${key}"${loc}.${hint ? ` Did you mean "${hint}"?` : ""}`;
210
- }
211
- const key = issue.path[1];
212
- return `${field(`state.${String(key)}`)} must be ${VALUES[key]} (got ${shown(state[key])})`;
224
+ const key = issue.path[0];
225
+ return `${field(key)} must be ${VALUES$1[key]} (got ${shown(value[key])})`;
213
226
  };
214
227
  const providerOptionsProblem = () => at("providerOptions") ? `${field("providerOptions")} must map each provider to an object of JSON values, e.g. { google: { thinkingConfig: { thinkingBudget: 0 } } } (got ${shown(value.providerOptions)})` : void 0;
215
228
  const artifactsProblem = () => at("artifacts") ? `c.judge() artifacts ${badJudgeArtifacts(value.artifacts)}` : void 0;
216
229
  const gatewayProblem = () => at("gateway") ? `${field("gateway")} names a gateway in the config's gateway list: ${GATEWAY_NAMES.map((g) => `'${g}'`).join(" or ")}, or is its helper (got ${shown(value.gateway)})` : void 0;
217
- if (site.at === "config") order.push(modelProblem, keyProblem, stateProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
218
- else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, stateProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
219
- else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, stateProblem, providerOptionsProblem, gatewayProblem);
230
+ if (site.at === "config") order.push(modelProblem, keyProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
231
+ else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, inputProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
232
+ else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, providerOptionsProblem, gatewayProblem);
220
233
  for (const check of order) {
221
234
  const problem = check();
222
235
  if (problem) return problem;
@@ -228,22 +241,15 @@ function judgeProblem(judge, site) {
228
241
  const pairFactorSchema = z.enum(["environment", "harness"]);
229
242
  /** Evaluation models score yes/no questions only, so they can't answer a question. */
230
243
  const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
231
- const questionStateSchema = z.strictObject({
232
- task: z.boolean().optional(),
233
- output: z.boolean().optional(),
234
- files: z.boolean().optional(),
235
- diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
236
- transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
237
- turns: z.enum(["last", "all"]).optional(),
238
- checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
239
- });
240
244
  const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
241
- const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
242
- const questionOptionsSchema = z.strictObject({
243
- model: questionModelSchema.optional(),
244
- state: questionStateSchema.optional(),
245
+ const questionInputFields = {
246
+ ...sharedInputFields,
245
247
  artifacts: questionArtifactsSchema.optional()
246
- });
248
+ };
249
+ z.strictObject(questionInputFields);
250
+ const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
251
+ const questionOptionsSchema = z.strictObject({ model: questionModelSchema.optional() });
252
+ const QUESTION_INPUT_KEYS = Object.keys(questionInputFields);
247
253
  /** The first item a list holds twice. */
248
254
  const twice = (items) => items.find((k, i) => items.indexOf(k) !== i);
249
255
  const unique = (list, ctx) => {
@@ -254,35 +260,38 @@ const unique = (list, ctx) => {
254
260
  };
255
261
  const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
256
262
  const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
257
- const questionObjectSchema = z.strictObject({
263
+ const questionSchema = z.strictObject({
258
264
  question: nonEmpty(),
259
265
  model: questionModelSchema.optional(),
260
266
  across: acrossSchema.optional(),
261
267
  choices: choicesSchema.optional(),
262
- state: questionStateSchema.optional(),
263
- artifacts: questionArtifactsSchema.optional()
268
+ turns: turnsSchema.optional(),
269
+ ...questionInputFields
264
270
  }).superRefine((q, ctx) => {
265
271
  if (q.across !== void 0 && q.choices !== void 0) ctx.addIssue({
266
272
  code: "custom",
267
273
  path: ["choices"],
268
274
  message: "with across"
269
275
  });
276
+ if (!inputsOn(q, QUESTION_INPUT_KEYS).length) ctx.addIssue({
277
+ code: "custom",
278
+ message: "no input"
279
+ });
270
280
  });
271
- const questionSchema = z.union([nonEmpty(), questionObjectSchema]);
272
- const questionText = (q) => typeof q === "string" ? q : q.question;
281
+ const questionText = (q) => q.question;
273
282
  const questionsSchema = z.array(questionSchema).superRefine((list, ctx) => {
274
283
  if (twice(list.map(questionText)) !== void 0) ctx.addIssue({
275
284
  code: "custom",
276
285
  message: "twice"
277
286
  });
278
287
  });
279
- const QUESTION_EXAMPLE = `questions: ['How did the agent find the failing test?']`;
288
+ const QUESTION_EXAMPLE = `questions: [{ question: 'How did the agent find the failing test?', transcript: true }]`;
280
289
  const ACROSS_HINT = `'environment' (vary the environment, same harness) or 'harness' (vary the harness, same environment)`;
281
290
  const ACROSS_MEAN = " Did you mean \"across\"? It is the factor a pairwise question varies: across: 'environment'.";
282
- const QUESTION_KEYS = Object.keys(questionObjectSchema.shape);
283
- const QUESTION_SHAPE = "{ question, model?, across?, choices?, state?, artifacts? }";
284
- const STATE_KEYS = Object.keys(questionStateSchema.shape);
285
- const STATE_VALUES = {
291
+ const QUESTION_KEYS = Object.keys(questionSchema.shape);
292
+ const INPUTS = orList(QUESTION_INPUT_KEYS);
293
+ const QUESTION_SHAPE = `{ question, model?, across?, choices?, turns? } with at least one of ${INPUTS}`;
294
+ const VALUES = {
286
295
  task: "true or false",
287
296
  output: "true or false",
288
297
  files: "true or false",
@@ -299,41 +308,31 @@ const describe$2 = (value) => {
299
308
  }
300
309
  return JSON.stringify(value) ?? String(value);
301
310
  };
302
- /**
303
- * What's wrong with a question's `state` or `artifacts`, as one sentence, or undefined. `name`
304
- * turns a key into how the sentence names it: `question.state` on the option,
305
- * `questions "…" state` on an entry.
306
- */
307
- function questionPartsProblem(value, name, where) {
308
- const { state, artifacts } = value;
309
- if (state !== void 0) {
310
- if (!isObjectLike(state) || Array.isArray(state)) return `${name("state")} in ${where} must be an object such as { transcript: 'tools' } (got ${describe$2(state)})`;
311
- for (const [key, v] of Object.entries(state)) {
312
- if (key === "contents") return `${name("state.contents")} in ${where} can't be set: questions are asked after the attempts, when their workspaces are gone. Narrow state.diff instead.`;
313
- const schema = questionStateSchema.shape[key];
314
- if (!schema) {
315
- const hint = closest(key, STATE_KEYS);
316
- return `Unknown ${name("state")} key "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""}`;
317
- }
318
- if (!schema.safeParse(v).success) return `${name(`state.${key}`)} in ${where} must be ${STATE_VALUES[key]} (got ${describe$2(v)})`;
311
+ /** What's wrong with an entry's inputs and `turns`, as one sentence, or undefined. */
312
+ function questionInputsProblem(value, at, where) {
313
+ for (const [key, v] of Object.entries(value)) {
314
+ if (key === "artifacts") {
315
+ if (!questionArtifactsSchema.safeParse(v).success) return `${at} artifacts in ${where} must be true or a non-empty list of capture names (got ${describe$2(v)})`;
316
+ continue;
319
317
  }
318
+ if (!(key in VALUES)) continue;
319
+ if (!(key === "turns" ? turnsSchema : sharedInputFields[key]).safeParse(v).success) return `${at} ${key} in ${where} must be ${VALUES[key]} (got ${describe$2(v)})`;
320
320
  }
321
- if (artifacts !== void 0 && !questionArtifactsSchema.safeParse(artifacts).success) return `${name("artifacts")} in ${where} must be true or a non-empty list of capture names (got ${describe$2(artifacts)})`;
321
+ if (!inputsOn(value, QUESTION_INPUT_KEYS).length) return `${at} in ${where} shows the model nothing: turn on at least one of ${INPUTS}, e.g. { question: '…', diff: true }.`;
322
322
  }
323
323
  /** What's wrong with the `question` option set in `where`, as one sentence, or undefined. */
324
324
  function questionOptionsProblem(value, where) {
325
- if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model?, state?, artifacts? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
325
+ if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
326
326
  const keys = Object.keys(questionOptionsSchema.shape);
327
327
  for (const key of Object.keys(value)) {
328
328
  if (keys.includes(key)) continue;
329
- if (key === "across" || key === "choices") return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: it says what kind of question that entry is.`;
329
+ if (key === "across" || key === "choices" || key === "turns" || key in questionInputFields) return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: what a question is and what it's shown are its own.`;
330
330
  const hint = closest(key, keys);
331
- return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model, state and artifacts.`;
331
+ return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model.`;
332
332
  }
333
333
  const { model } = value;
334
334
  if (model !== void 0 && typeof model === "string" && isEvaluationModel(model)) return `question.model in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
335
335
  if (model !== void 0 && !questionModelSchema.safeParse(model).success) return `question.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
336
- return questionPartsProblem(value, (key) => `question.${key}`, where);
337
336
  }
338
337
  /** What's wrong with an entry's `across`, or undefined. */
339
338
  function acrossProblem(question, across, where) {
@@ -352,11 +351,11 @@ function choicesProblem(question, choices, where) {
352
351
  if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
353
352
  return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
354
353
  }
355
- /** One question's sentence: the union refuses it whole, so the branch it meant says why. */
354
+ /** One question's sentence. */
356
355
  function questionProblem(item, where) {
357
- if (typeof item === "string") return `questions in ${where} has an empty string. Ask something, e.g. ${QUESTION_EXAMPLE}.`;
358
- if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes strings or ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
356
+ if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
359
357
  const unknown = Object.keys(item).filter((k) => !QUESTION_KEYS.includes(k));
358
+ if (unknown.includes("contents")) return `questions in ${where} can't set contents: questions are asked after the attempts, when their workspaces are gone. Narrow diff instead.`;
360
359
  if (unknown.length) {
361
360
  const hint = unknown.some((k) => k === "cut" || k === "cuts") ? ACROSS_MEAN : "";
362
361
  return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
@@ -373,7 +372,7 @@ function questionProblem(item, where) {
373
372
  if (problem) return problem;
374
373
  }
375
374
  if (across !== void 0 && choices !== void 0) return `questions "${text}" in ${where} sets both across and choices. choices are the answers a question asked of each attempt picks from, and across makes it pick between two attempts instead: drop choices to vote between pairs, or drop across to ask each attempt.`;
376
- const parts = questionPartsProblem(item, (key) => `questions "${text}" ${key}`, where);
375
+ const parts = questionInputsProblem(item, `questions "${text}"`, where);
377
376
  if (parts) return parts;
378
377
  if (typeof model === "string" && isEvaluationModel(model)) return `questions "${text}" in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
379
378
  return `questions "${text}" in ${where}: model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
@@ -389,6 +388,8 @@ function questionsProblem(value, where) {
389
388
  //#endregion
390
389
  //#region src/contract/config.ts
391
390
  const count = () => z.number().int().min(1);
391
+ /** `runs`: a count, or `{ bestOf }` with an odd count of 3 or more. */
392
+ const runs = () => z.union([count(), z.strictObject({ bestOf: z.number().int().min(3).refine((n) => n % 2 === 1, "must be odd") })]);
392
393
  const captureWindowSchema = z.strictObject({
393
394
  width: count(),
394
395
  height: count()
@@ -421,14 +422,15 @@ const profileDefaultsDataSchema = z.strictObject({
421
422
  harness: harnessList.optional(),
422
423
  sandbox: sandboxDescriptorSchema.optional(),
423
424
  environment: environmentDataSchema.optional(),
424
- runs: count().optional(),
425
+ runs: runs().optional(),
425
426
  gate: gatePolicySchema.optional(),
426
427
  env: envVarsSchema.optional(),
427
428
  egress: egressPolicySchema.optional(),
428
429
  judge: configJudgeDataSchema.optional(),
429
430
  timeout: durationSchema.optional(),
430
431
  capture: captureOptionsSchema.optional(),
431
- question: questionOptionsSchema.optional()
432
+ question: questionOptionsSchema.optional(),
433
+ snapshot: snapshotOptionSchema.optional()
432
434
  });
433
435
  const profileLimitSchema = z.strictObject({
434
436
  runs: count().optional(),
@@ -481,7 +483,7 @@ const reevalConfigDataSchema = z.strictObject({
481
483
  harness: harnessList.optional(),
482
484
  sandbox: sandboxDescriptorSchema.optional(),
483
485
  environment: environmentDataSchema.optional(),
484
- runs: count().optional(),
486
+ runs: runs().optional(),
485
487
  gate: gatePolicySchema.optional(),
486
488
  env: envVarsSchema.optional(),
487
489
  egress: egressPolicySchema.optional(),
@@ -489,6 +491,7 @@ const reevalConfigDataSchema = z.strictObject({
489
491
  timeout: durationSchema.optional(),
490
492
  capture: captureOptionsSchema.optional(),
491
493
  question: questionOptionsSchema.optional(),
494
+ snapshot: snapshotOptionSchema.optional(),
492
495
  concurrency: count().optional(),
493
496
  record: recordMode.optional(),
494
497
  history: historyConfigSchema.optional(),
@@ -643,12 +646,18 @@ const CURSOR_EXCHANGE = {
643
646
  path: { exact: "/auth/exchange_user_api_key" },
644
647
  method: ["POST"]
645
648
  };
646
- /** What Codex sends the key on: the Responses API. */
647
- const OPENAI_ENDPOINTS = [{
649
+ /** Codex's model calls: the Responses API, whose response names the served model. */
650
+ const OPENAI_MODEL_ENDPOINTS = [{
648
651
  host: "api.openai.com",
649
652
  path: { exact: "/v1/responses" },
650
653
  method: ["POST"]
651
654
  }];
655
+ /** What Codex sends the key on: its model calls, and its web search tool's own endpoint. */
656
+ const OPENAI_ENDPOINTS = [...OPENAI_MODEL_ENDPOINTS, {
657
+ host: "api.openai.com",
658
+ path: { exact: "/v1/alpha/search" },
659
+ method: ["POST"]
660
+ }];
652
661
  /**
653
662
  * The model the Responses API says served a request, from its response (a dated snapshot for some
654
663
  * models), and the reasoning effort Codex resolved for it, e.g. `gpt-5.6-luna[effort=medium]`.
@@ -1125,7 +1134,7 @@ const AGENTS = {
1125
1134
  },
1126
1135
  unreported: cursorUnreported,
1127
1136
  mcpServers: cursorMcpServers,
1128
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-6_zz7oJs.js")).createCursorSdk({
1137
+ adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-C9EHSYyP.js")).createCursorSdk({
1129
1138
  auth,
1130
1139
  ...mcpServers ? { mcpServers } : {},
1131
1140
  ...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
@@ -1192,7 +1201,7 @@ const AGENTS = {
1192
1201
  },
1193
1202
  servedModel: geminiServedModel,
1194
1203
  mcpServers: antigravityMcpServers,
1195
- adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-C7upnZSd.js")).createAntigravity({
1204
+ adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-Bp6X4193.js")).createAntigravity({
1196
1205
  auth,
1197
1206
  ...vertex ? { vertex } : {},
1198
1207
  ...mcpServers ? { mcpServers } : {},
@@ -1219,7 +1228,7 @@ const AGENTS = {
1219
1228
  servedModel: geminiServedModel,
1220
1229
  forModel: opencodeForModel,
1221
1230
  mcpServers: opencodeMcpServers,
1222
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-HSxBazcg.js")).createOpenCode({
1231
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-C5q3_fRV.js")).createOpenCode({
1223
1232
  auth,
1224
1233
  model,
1225
1234
  ...vertex ? { vertex } : { vars: opencodeVars(model) },
@@ -1241,14 +1250,14 @@ const AGENTS = {
1241
1250
  },
1242
1251
  servedModel: geminiServedModel,
1243
1252
  mcpServers: gooseMcpServers,
1244
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-kYeaJJrl.js")).createGoose({
1253
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-OOKE-5GA.js")).createGoose({
1245
1254
  ...vertex ? { vertex } : {},
1246
1255
  ...auth["AI_GATEWAY_API_KEY"] ? {
1247
1256
  auth,
1248
- provider: (await import("./src-kYeaJJrl.js")).AI_GATEWAY_PROVIDER
1257
+ provider: (await import("./src-OOKE-5GA.js")).AI_GATEWAY_PROVIDER
1249
1258
  } : auth["OPENROUTER_API_KEY"] ? {
1250
1259
  auth,
1251
- provider: (await import("./src-kYeaJJrl.js")).OPENROUTER_PROVIDER
1260
+ provider: (await import("./src-OOKE-5GA.js")).OPENROUTER_PROVIDER
1252
1261
  } : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
1253
1262
  model,
1254
1263
  ...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
@@ -1269,13 +1278,13 @@ const AGENTS = {
1269
1278
  endpoints: GEMINI_ENDPOINTS,
1270
1279
  servedModel: geminiServedModel,
1271
1280
  mcpServers: piMcpServers,
1272
- adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-Bp4aN3rA.js")).createPi({
1281
+ adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-D4CZjMQ4.js")).createPi({
1273
1282
  ...auth["AI_GATEWAY_API_KEY"] ? {
1274
1283
  auth,
1275
- gateway: (await import("./src-Bp4aN3rA.js")).PI_AI_GATEWAY
1284
+ gateway: (await import("./src-D4CZjMQ4.js")).PI_AI_GATEWAY
1276
1285
  } : auth["OPENROUTER_API_KEY"] ? {
1277
1286
  auth,
1278
- gateway: (await import("./src-Bp4aN3rA.js")).PI_OPENROUTER
1287
+ gateway: (await import("./src-D4CZjMQ4.js")).PI_OPENROUTER
1279
1288
  } : {
1280
1289
  auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
1281
1290
  ...vertex ? { vertex } : {}
@@ -1296,7 +1305,7 @@ const AGENTS = {
1296
1305
  factory: "codex",
1297
1306
  credential: "OPENAI_API_KEY",
1298
1307
  endpoints: OPENAI_ENDPOINTS,
1299
- modelResponses: OPENAI_ENDPOINTS,
1308
+ modelResponses: OPENAI_MODEL_ENDPOINTS,
1300
1309
  expectedBlocked: {
1301
1310
  hosts: [
1302
1311
  "github.com",
@@ -1308,9 +1317,9 @@ const AGENTS = {
1308
1317
  },
1309
1318
  servedModel: openaiServedModel,
1310
1319
  mcpServers: codexMcpServers,
1311
- adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-dlX-Lmxg.js")).createCodex({
1320
+ adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-CJdpsrXi.js")).createCodex({
1312
1321
  auth,
1313
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-dlX-Lmxg.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-dlX-Lmxg.js")).OPENROUTER_PROVIDER } : {},
1322
+ ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-CJdpsrXi.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-CJdpsrXi.js")).OPENROUTER_PROVIDER } : {},
1314
1323
  ...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
1315
1324
  ...mcpServers ? { mcpServers } : {},
1316
1325
  ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
@@ -1328,7 +1337,7 @@ const AGENTS = {
1328
1337
  ...fxForModel("provider/model"),
1329
1338
  forModel: fxForModel,
1330
1339
  mcpServers: acpMcpServers,
1331
- adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-LD1BAJFa.js")).createFx({
1340
+ adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-BZlhaDhE.js")).createFx({
1332
1341
  auth,
1333
1342
  model,
1334
1343
  ...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
@@ -1353,7 +1362,7 @@ const AGENTS = {
1353
1362
  servedModel: xaiServedModel,
1354
1363
  mcpServers: acpMcpServers,
1355
1364
  adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
1356
- const grok = await import("./src-BYPMM1x7.js");
1365
+ const grok = await import("./src-GkJ-1Rdy.js");
1357
1366
  return grok.createGrokBuild({
1358
1367
  ...vertexChat ? {
1359
1368
  auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
@@ -1661,7 +1670,8 @@ function harnessEffort(h) {
1661
1670
  }
1662
1671
  /**
1663
1672
  * The reasoning effort a served-model label recorded, e.g. `gpt-5.6-luna[effort=medium]` →
1664
- * `medium`. Fast (`[fast=true]`) is not effort and is ignored.
1673
+ * `medium`, or Cursor's `[reasoning=medium]`. Fast (`[fast=true]`) and an on/off `thinking=true`
1674
+ * are not effort and are ignored.
1665
1675
  */
1666
1676
  function effortFromModelLabel(model) {
1667
1677
  const inner = model?.match(/\[([^\]]+)]/)?.[1];
@@ -1671,9 +1681,9 @@ function effortFromModelLabel(model) {
1671
1681
  if (eq < 0) continue;
1672
1682
  const key = part.slice(0, eq).trim();
1673
1683
  if (key === "fast") continue;
1674
- if (/^(?:reasoning_)?effort$|^thinking$/.test(key)) {
1684
+ if (/^(?:reasoning[_-]?)?effort$|^reasoning$|^thinking$/.test(key)) {
1675
1685
  const value = part.slice(eq + 1).trim();
1676
- if (value) return value;
1686
+ if (value && value !== "true" && value !== "false") return value;
1677
1687
  }
1678
1688
  }
1679
1689
  }
@@ -1704,11 +1714,10 @@ function resolvedEffortOf(models) {
1704
1714
  /** The error a per-attempt answer carries when the run stopped before a model answered it. */
1705
1715
  const STOPPED_FIRST = "the run stopped before it was answered";
1706
1716
  /**
1707
- * What each side of a pairwise question shows with no `state`: the task and the diff, every
1708
- * turn's prompt. The final message stays out, since a better-written account of the work would
1709
- * win a side-by-side pick on its wording; the changed files repeat what the diff shows.
1717
+ * What a recorded pairwise answer without `stateOptions` showed each side: the task and the diff,
1718
+ * every turn's prompt.
1710
1719
  */
1711
- const DEFAULT_QUESTION_STATE = {
1720
+ const UNRECORDED_QUESTION_STATE = {
1712
1721
  task: true,
1713
1722
  output: false,
1714
1723
  files: false,
@@ -1717,62 +1726,39 @@ const DEFAULT_QUESTION_STATE = {
1717
1726
  turns: "all",
1718
1727
  checks: false
1719
1728
  };
1729
+ const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
1720
1730
  /**
1721
- * What a question asked of each attempt on its own shows with no `state`: the task, the diff,
1722
- * the final message and the transcript. Nothing is weighed against another side, so how the
1723
- * agent went about the work is the point.
1731
+ * Every key set, in the record's key order, so equal selections serialize the same: an input the
1732
+ * question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
1724
1733
  */
1725
- const DEFAULT_ATTEMPT_QUESTION_STATE = {
1726
- ...DEFAULT_QUESTION_STATE,
1727
- output: true,
1728
- transcript: true
1729
- };
1730
- const KEYS = Object.keys(DEFAULT_QUESTION_STATE);
1731
- /** A pairwise question's defaults when `pairwise`, a per-attempt question's otherwise. */
1732
- const defaultQuestionState = (pairwise) => pairwise ? DEFAULT_QUESTION_STATE : DEFAULT_ATTEMPT_QUESTION_STATE;
1733
- /** Every key set, in `DEFAULT_QUESTION_STATE`'s order, so equal selections serialize the same. */
1734
- function resolveQuestionState(state, pairwise) {
1735
- const defaults = defaultQuestionState(pairwise);
1736
- const out = {};
1737
- for (const key of KEYS) out[key] = state?.[key] ?? defaults[key];
1738
- return out;
1734
+ function resolveQuestionState(state) {
1735
+ return {
1736
+ task: state?.task ?? false,
1737
+ output: state?.output ?? false,
1738
+ files: state?.files ?? false,
1739
+ diff: state?.diff ?? false,
1740
+ transcript: state?.transcript ?? false,
1741
+ turns: state?.turns ?? "all",
1742
+ checks: state?.checks ?? false
1743
+ };
1739
1744
  }
1740
- /** `inner` over `outer`: `state` key by key, `model` and `artifacts` replaced. */
1745
+ /** `inner` over `outer`, field by field. */
1741
1746
  function mergeQuestion(outer, inner) {
1742
- if (!inner) return outer;
1743
- const out = {
1747
+ return inner ? {
1744
1748
  ...outer,
1745
1749
  ...inner
1746
- };
1747
- if (outer.state || inner.state) out.state = {
1748
- ...outer.state,
1749
- ...inner.state
1750
- };
1751
- return out;
1750
+ } : outer;
1752
1751
  }
1753
- /** A planned question with the cascaded `question` defaults under its own `model`, `state` and `artifacts`. */
1752
+ /** A planned question under the cascaded `question` defaults: its own `model`, else theirs. */
1754
1753
  function withQuestionDefaults(q, defaults) {
1755
- const { state, artifacts } = mergeQuestion(defaults, {
1756
- ...q.state ? { state: q.state } : {},
1757
- ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
1758
- });
1759
- const { state: _state, artifacts: _artifacts, ...rest } = q;
1760
1754
  const model = q.model ?? defaults.model;
1761
- return {
1762
- ...rest,
1763
- ...model !== void 0 ? { model } : {},
1764
- ...state && Object.keys(state).length ? { state } : {},
1765
- ...artifacts !== void 0 ? { artifacts } : {}
1766
- };
1755
+ return model !== void 0 ? {
1756
+ ...q,
1757
+ model
1758
+ } : q;
1767
1759
  }
1768
1760
  const isOff = (value) => value === false || Array.isArray(value) && !value.length;
1769
- /** True when the resolved state shows nothing and no captures are sent. */
1770
- const showsNothing = (q) => {
1771
- if (q.artifacts !== void 0) return false;
1772
- const s = resolveQuestionState(q.state, q.across !== void 0);
1773
- return KEYS.every((k) => k === "turns" || isOff(s[k]));
1774
- };
1775
- /** The state keys a question shows, in order, plus `artifacts` when it sends captures. */
1761
+ /** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
1776
1762
  function questionShows(state, artifacts) {
1777
1763
  const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
1778
1764
  return artifacts === void 0 ? on : [...on, "artifacts"];
@@ -1838,6 +1824,189 @@ function compareProblem(value, where, environment) {
1838
1824
  if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness? }.`;
1839
1825
  return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
1840
1826
  }
1827
+ const POLL_MS = 200;
1828
+ /**
1829
+ * Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
1830
+ * (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
1831
+ */
1832
+ function probeScript(pid, tables = true) {
1833
+ return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
1834
+ }
1835
+ /** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
1836
+ function procAddress(hex) {
1837
+ const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
1838
+ if (bytes.length === 4) return bytes.join(".");
1839
+ if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
1840
+ const groups = [];
1841
+ for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
1842
+ const full = groups.join(":");
1843
+ const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
1844
+ return longest ? full.replace(longest, "::") : full;
1845
+ }
1846
+ /** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
1847
+ const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
1848
+ function parseProbe(stdout) {
1849
+ const lines = stdout.split("\n");
1850
+ const alive = lines.includes("alive");
1851
+ if (lines.includes("noproc")) return {
1852
+ proc: false,
1853
+ alive
1854
+ };
1855
+ const listeners = [];
1856
+ for (const l of lines) {
1857
+ const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
1858
+ const address = m ? procAddress(m[1]) : "";
1859
+ if (m && reachable(address)) listeners.push({
1860
+ port: Number.parseInt(m[2], 16),
1861
+ address
1862
+ });
1863
+ }
1864
+ return {
1865
+ proc: true,
1866
+ alive,
1867
+ listeners
1868
+ };
1869
+ }
1870
+ /** The ports listening now that weren't before the command started, ascending. */
1871
+ const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
1872
+ /** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
1873
+ const addressFor = (listeners, port) => {
1874
+ const all = listeners.filter((l) => l.port === port).map((l) => l.address);
1875
+ return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
1876
+ };
1877
+ /** `want` is taken whoever listens on it: reeval allocated it, or the eval named it. */
1878
+ function decide(s) {
1879
+ const { listeners, alive } = s.probe;
1880
+ const ports = newPorts(listeners, s.before);
1881
+ if (listeners.some((l) => l.port === s.want)) return {
1882
+ kind: "port",
1883
+ port: s.want,
1884
+ address: addressFor(listeners, s.want)
1885
+ };
1886
+ const late = s.elapsedMs >= s.timeoutMs;
1887
+ if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
1888
+ if (ports.length === 1) {
1889
+ const port = ports[0];
1890
+ return {
1891
+ kind: "port",
1892
+ port,
1893
+ address: addressFor(listeners, port)
1894
+ };
1895
+ }
1896
+ return {
1897
+ kind: "several",
1898
+ ports
1899
+ };
1900
+ }
1901
+ if (!alive) return { kind: "exited" };
1902
+ return late ? {
1903
+ kind: "timeout",
1904
+ ports
1905
+ } : { kind: "wait" };
1906
+ }
1907
+ const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
1908
+ /** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
1909
+ function printedUrls(output, ports) {
1910
+ const found = /* @__PURE__ */ new Map();
1911
+ for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
1912
+ const port = Number(m[1]);
1913
+ if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
1914
+ }
1915
+ return [...found].map(([url, port]) => ({
1916
+ url,
1917
+ port
1918
+ }));
1919
+ }
1920
+ /** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
1921
+ async function hostListening(port) {
1922
+ for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
1923
+ const socket = connect({
1924
+ port,
1925
+ host
1926
+ });
1927
+ socket.once("connect", () => {
1928
+ socket.destroy();
1929
+ resolve(true);
1930
+ });
1931
+ socket.once("error", () => resolve(false));
1932
+ })) return host;
1933
+ }
1934
+ const sleep = (ms, signal) => new Promise((resolve) => {
1935
+ const done = () => {
1936
+ clearTimeout(timer);
1937
+ signal?.removeEventListener("abort", done);
1938
+ resolve();
1939
+ };
1940
+ const timer = setTimeout(done, ms);
1941
+ signal?.addEventListener("abort", done, { once: true });
1942
+ });
1943
+ const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
1944
+ /**
1945
+ * Polls until something listens on `want`, or, without a given port, until the ports that are
1946
+ * new since the command started have settled on one. Throws, with the command's last output,
1947
+ * when they settle on several, it exits, or nothing listens in time.
1948
+ */
1949
+ async function awaitListening(command, w) {
1950
+ const timeoutMs = w.timeoutMs ?? 12e4;
1951
+ const started = Date.now();
1952
+ let last = "";
1953
+ let changed = started;
1954
+ const output = async () => plain(await w.output().catch(() => ""));
1955
+ const fail = async (why, out) => {
1956
+ const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
1957
+ return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
1958
+ };
1959
+ for (;;) {
1960
+ w.signal?.throwIfAborted();
1961
+ const probe = await w.probe();
1962
+ const now = Date.now();
1963
+ if (!probe.proc) {
1964
+ const address = await (w.hostListening ?? hostListening)(w.want);
1965
+ if (address !== void 0) return {
1966
+ port: w.want,
1967
+ address,
1968
+ detected: false,
1969
+ readyMs: now - started
1970
+ };
1971
+ if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
1972
+ if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
1973
+ } else {
1974
+ const key = newPorts(probe.listeners, w.before).join(",");
1975
+ if (key !== last) {
1976
+ last = key;
1977
+ changed = now;
1978
+ }
1979
+ const d = decide({
1980
+ want: w.want,
1981
+ given: w.given,
1982
+ probe,
1983
+ before: w.before,
1984
+ stableMs: now - changed,
1985
+ elapsedMs: now - started,
1986
+ timeoutMs
1987
+ });
1988
+ if (d.kind === "port") return {
1989
+ port: d.port,
1990
+ address: d.address,
1991
+ detected: d.port !== w.want,
1992
+ readyMs: now - started
1993
+ };
1994
+ if (d.kind === "several") {
1995
+ const out = await output();
1996
+ const urls = printedUrls(out, d.ports);
1997
+ const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
1998
+ const hint = urls[0]?.port ?? d.ports[0];
1999
+ throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
2000
+ }
2001
+ if (d.kind === "exited") throw await fail("exited before it listened on a port");
2002
+ if (d.kind === "timeout") {
2003
+ const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
2004
+ throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
2005
+ }
2006
+ }
2007
+ await sleep(POLL_MS, w.signal);
2008
+ }
2009
+ }
1841
2010
  //#endregion
1842
2011
  //#region src/sandbox/fake.ts
1843
2012
  /** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
@@ -1982,7 +2151,7 @@ async function createFakeSandbox(descriptor, env = {}) {
1982
2151
  },
1983
2152
  home: base,
1984
2153
  readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
1985
- getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://127.0.0.1:${port}` }),
2154
+ getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
1986
2155
  async writeFile(path, content) {
1987
2156
  const target = resolve(path);
1988
2157
  await mkdir(dirname(target), { recursive: true });
@@ -2063,13 +2232,16 @@ function resolveOutput(input) {
2063
2232
  "errors-only",
2064
2233
  "none"
2065
2234
  ]);
2235
+ const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only";
2236
+ const color = env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions;
2066
2237
  return {
2067
2238
  mode,
2068
2239
  statusLine,
2069
2240
  logOrder,
2070
2241
  logPrefix,
2071
- outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
2072
- color: env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions,
2242
+ outputLogs,
2243
+ color,
2244
+ rich: statusLine && color,
2073
2245
  githubActions,
2074
2246
  heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
2075
2247
  };
@@ -2200,34 +2372,28 @@ function egressLabel({ policy, auto }) {
2200
2372
  }
2201
2373
  //#endregion
2202
2374
  //#region src/judge/state-options.ts
2203
- const DEFAULT_STATE = {
2204
- task: true,
2205
- output: true,
2206
- files: true,
2207
- diff: true,
2208
- contents: [],
2209
- transcript: false,
2210
- turns: "last",
2211
- checks: false
2212
- };
2213
- /** Every key set, in `STATE_KEYS` order, so equal selections serialize the same. */
2214
- function resolveStateOptions(state) {
2215
- const out = {};
2216
- for (const key of STATE_KEYS$1) out[key] = state?.[key] ?? DEFAULT_STATE[key];
2217
- return out;
2375
+ /**
2376
+ * What one call turned on, every key set (`false` when off), in the record's key order, so equal
2377
+ * selections serialize the same.
2378
+ */
2379
+ function resolveStateOptions(call) {
2380
+ return {
2381
+ task: call.task ?? false,
2382
+ output: call.output ?? false,
2383
+ files: call.files ?? false,
2384
+ diff: call.diff ?? false,
2385
+ contents: call.contents ?? [],
2386
+ transcript: call.transcript ?? false,
2387
+ turns: call.turns ?? "last",
2388
+ checks: call.checks ?? false
2389
+ };
2218
2390
  }
2219
- /** `inner` over `outer`, field by field, and `state` key by key. */
2391
+ /** `inner` over `outer`, field by field. */
2220
2392
  function mergeJudge(outer, inner) {
2221
- if (!inner) return outer;
2222
- const out = {
2393
+ return inner ? {
2223
2394
  ...outer,
2224
2395
  ...inner
2225
- };
2226
- if (outer.state || inner.state) out.state = {
2227
- ...outer.state,
2228
- ...inner.state
2229
- };
2230
- return out;
2396
+ } : outer;
2231
2397
  }
2232
2398
  const KNOWN = [...[
2233
2399
  "harness",
@@ -2243,6 +2409,7 @@ const KNOWN = [...[
2243
2409
  "question",
2244
2410
  "questions",
2245
2411
  "timeout",
2412
+ "snapshot",
2246
2413
  "tags",
2247
2414
  "revision",
2248
2415
  "pin"
@@ -2257,6 +2424,7 @@ const SUGGEST = {
2257
2424
  repeats: "runs",
2258
2425
  iterations: "runs",
2259
2426
  trials: "runs",
2427
+ bestOf: "runs",
2260
2428
  secrets: "env",
2261
2429
  vars: "env",
2262
2430
  network: "egress",
@@ -2265,9 +2433,26 @@ const SUGGEST = {
2265
2433
  version: "revision",
2266
2434
  rev: "revision",
2267
2435
  pins: "pin",
2436
+ workspace: "snapshot",
2437
+ snapshots: "snapshot",
2268
2438
  passRate: "gate",
2269
2439
  threshold: "gate"
2270
2440
  };
2441
+ /**
2442
+ * `snapshot` over the levels above: the shorthand sets `when`, the object sets the fields it
2443
+ * names, and `compress` carries through either.
2444
+ */
2445
+ function mergeSnapshot(outer, inner) {
2446
+ if (inner === void 0) return outer;
2447
+ if (typeof inner !== "object") return {
2448
+ ...outer,
2449
+ when: inner
2450
+ };
2451
+ return {
2452
+ when: inner.when ?? outer.when,
2453
+ compress: inner.compress ?? outer.compress
2454
+ };
2455
+ }
2271
2456
  /** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
2272
2457
  function checkGate(value, where) {
2273
2458
  const problem = gateProblem(value, where);
@@ -2310,10 +2495,12 @@ function sandboxId(s) {
2310
2495
  return image ? `${engine}:${sanitize(image)}${install}` : `${engine}${install}`;
2311
2496
  }
2312
2497
  /** The planned attempts' sandboxes by id, once each, or `fallback`'s when none is planned. */
2313
- function plannedSandboxes(planned, fallback) {
2498
+ function plannedSandboxIds(planned, fallback) {
2314
2499
  const ids = new Set(planned.map(sandboxId));
2315
- return ids.size ? [...ids].join(", ") : sandboxId(fallback);
2500
+ return ids.size ? [...ids] : [sandboxId(fallback)];
2316
2501
  }
2502
+ /** {@link plannedSandboxIds}, comma-separated. */
2503
+ const plannedSandboxes = (planned, fallback) => plannedSandboxIds(planned, fallback).join(", ");
2317
2504
  const cellKey = (harness, environment, sandbox) => `${harness}/${environment}/${sandbox}`;
2318
2505
  const isHarness = (v) => isObjectLike(v) && (v.kind === "fake" || isAgentKind(v.kind));
2319
2506
  function checkMaxConcurrent(value, where) {
@@ -2335,19 +2522,44 @@ function refuseIllegalCompare(options, where) {
2335
2522
  if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
2336
2523
  }
2337
2524
  const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
2525
+ /** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
2526
+ function checkRuns(runs, where) {
2527
+ if (typeof runs === "number" && Number.isInteger(runs) && runs >= 1) return;
2528
+ if (typeof runs === "number") throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
2529
+ if (!isObjectLike(runs)) throw new UsageError(`runs in ${where} must be a whole number of at least 1, or { bestOf: n } (got ${JSON.stringify(runs)})`);
2530
+ const extra = Object.keys(runs).find((k) => k !== "bestOf");
2531
+ if (extra !== void 0) throw new UsageError(`runs in ${where} takes { bestOf: n } only (got the key "${extra}"), e.g. runs: { bestOf: 3 }`);
2532
+ const { bestOf } = runs;
2533
+ if (!(Number.isInteger(bestOf) && bestOf >= 3 && bestOf % 2 === 1)) throw new UsageError(`runs.bestOf in ${where} must be an odd whole number of 3 or more, so one side always has a majority, e.g. runs: { bestOf: 3 } (got ${JSON.stringify(bestOf)})`);
2534
+ }
2535
+ /** The most attempts `runs` plans a cell, and its `bestOf` when it has one. */
2536
+ const runsOf = (runs) => typeof runs === "number" ? { runs } : {
2537
+ runs: runs.bestOf,
2538
+ bestOf: runs.bestOf
2539
+ };
2540
+ /**
2541
+ * A best-of cell's gate: half or more of its counted attempts pass, which is its majority once it
2542
+ * has one (an infra error counts on neither side).
2543
+ */
2544
+ const MAJORITY_GATE = { passRate: .5 };
2545
+ /** How many of a best-of cell's attempts always run: the majority. */
2546
+ const majorityOf = (bestOf) => (bestOf + 1) / 2;
2338
2547
  /** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
2339
2548
  function checkOptions(options, where, allowed = KNOWN) {
2340
2549
  for (const key of Object.keys(options)) {
2341
2550
  if (allowed.includes(key)) continue;
2342
2551
  const near = SUGGEST[key] ?? closest(key, allowed);
2343
- const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : near ? ` Did you mean "${near}"?` : "";
2552
+ const value = options[key];
2553
+ const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
2344
2554
  throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
2345
2555
  }
2346
2556
  const o = options;
2347
- if (o.runs !== void 0 && !(Number.isInteger(o.runs) && o.runs >= 1)) throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(o.runs)})`);
2557
+ if (o.runs !== void 0) checkRuns(o.runs, where);
2348
2558
  if (o.gate !== void 0) checkGate(o.gate, where);
2349
2559
  if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
2350
2560
  if (o.revision !== void 0 && !isRevision(o.revision)) throw new UsageError(`revision in ${where} must be a whole number or a non-empty string, e.g. revision: 2 (got ${JSON.stringify(o.revision)})`);
2561
+ const snapshotIssue = snapshotProblem(o.snapshot, where);
2562
+ if (snapshotIssue) throw new UsageError(snapshotIssue);
2351
2563
  if (o.env !== void 0) checkEnv(o.env, where, UsageError);
2352
2564
  if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
2353
2565
  const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
@@ -2478,17 +2690,18 @@ function checkUnique(list, where) {
2478
2690
  seen.add(key);
2479
2691
  }
2480
2692
  }
2481
- const plannedQuestions = (value) => value.map((q, i) => typeof q === "string" ? {
2482
- n: i + 1,
2483
- question: q
2484
- } : {
2485
- n: i + 1,
2486
- question: q.question,
2487
- ...q.model !== void 0 ? { model: q.model } : {},
2488
- ...q.across !== void 0 ? { across: q.across } : {},
2489
- ...q.choices ? { choices: q.choices } : {},
2490
- ...q.state ? { state: q.state } : {},
2491
- ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
2693
+ const plannedQuestions = (value) => value.map((q, i) => {
2694
+ const { question, model, across, choices, artifacts, ...rest } = q;
2695
+ const state = Object.fromEntries(Object.entries(rest).filter(([, v]) => v !== void 0));
2696
+ return {
2697
+ n: i + 1,
2698
+ question,
2699
+ ...model !== void 0 ? { model } : {},
2700
+ ...across !== void 0 ? { across } : {},
2701
+ ...choices ? { choices } : {},
2702
+ state,
2703
+ ...artifacts !== void 0 ? { artifacts } : {}
2704
+ };
2492
2705
  });
2493
2706
  /** Whether the eval's matrix varies each factor a pairwise question can vary. */
2494
2707
  const variesBy = (r) => ({
@@ -2529,9 +2742,14 @@ function applyLevel(out, options, where, egress = "own") {
2529
2742
  if (options.sandbox !== void 0) out.sandbox = options.sandbox;
2530
2743
  if (options.environment !== void 0) out.environment = options.environment;
2531
2744
  if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
2532
- if (options.runs !== void 0) out.runs = options.runs;
2745
+ if (options.runs !== void 0) {
2746
+ const { runs, bestOf } = runsOf(options.runs);
2747
+ out.runs = runs;
2748
+ out.bestOf = bestOf;
2749
+ }
2533
2750
  if (options.gate !== void 0) out.gate = options.gate;
2534
2751
  if (options.timeout !== void 0) out.timeout = options.timeout;
2752
+ out.snapshot = mergeSnapshot(out.snapshot, options.snapshot);
2535
2753
  if (options.revision !== void 0) out.revision = options.revision;
2536
2754
  Object.assign(out.env, options.env);
2537
2755
  out.judge = mergeJudge(out.judge, options.judge);
@@ -2578,11 +2796,7 @@ function cascade(base, levels, where, options = {}) {
2578
2796
  baseline: out.environment,
2579
2797
  from: "environment"
2580
2798
  };
2581
- if (out.questions.length) {
2582
- out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
2583
- const blank = out.questions.find(showsNothing);
2584
- if (blank) throw new UsageError(`questions "${blank.question}" in ${where} shows the model nothing: every state key is off and it sends no artifacts. Turn on a state key, such as state: { diff: true }, or send artifacts.`);
2585
- }
2799
+ if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
2586
2800
  checkUnique(out.harness, where);
2587
2801
  const harnessFromProfile = profile?.defaults.harness !== void 0 && !levels.some((l) => l.options.harness);
2588
2802
  const authored = variesBy(out);
@@ -2648,8 +2862,10 @@ function cascade(base, levels, where, options = {}) {
2648
2862
  }
2649
2863
  defaultRuns(out, notes, options);
2650
2864
  if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && (out.runs ?? 1) > limit.runs) {
2651
- notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
2865
+ const was = out.bestOf !== void 0 ? `best of ${out.bestOf}` : String(out.runs);
2866
+ notes.push(`runs ${was} → ${limit.runs} (${tag})`);
2652
2867
  out.runs = limit.runs;
2868
+ out.bestOf = void 0;
2653
2869
  out.sources = {
2654
2870
  ...out.sources,
2655
2871
  runs: `profiles.${profile.name}.limit`
@@ -2678,6 +2894,16 @@ function cascade(base, levels, where, options = {}) {
2678
2894
  if (narrowed) notes.push(`${plural(narrowed)} narrowed (profile ${profile.name})`);
2679
2895
  out.questions = kept;
2680
2896
  }
2897
+ if (options.runs !== void 0) out.bestOf = void 0;
2898
+ if (out.bestOf !== void 0 && out.compare?.variants.length) {
2899
+ const from = out.sources?.runs ?? "the config";
2900
+ const set = from === where ? "" : ` from ${from}`;
2901
+ throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
2902
+ }
2903
+ if (out.bestOf !== void 0) {
2904
+ if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
2905
+ out.gate = MAJORITY_GATE;
2906
+ }
2681
2907
  defaultRuns(out, notes, options);
2682
2908
  return {
2683
2909
  resolved: {
@@ -2813,6 +3039,7 @@ function expand(resolved, vertex) {
2813
3039
  egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers, onVertex),
2814
3040
  gate,
2815
3041
  ...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
3042
+ ...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
2816
3043
  n,
2817
3044
  label,
2818
3045
  ...onVertex ? { vertex: onVertex } : {}
@@ -2941,6 +3168,7 @@ function evalPlan(r, attempts, origin) {
2941
3168
  } } : {},
2942
3169
  ...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
2943
3170
  runs: resolved.runs,
3171
+ ...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
2944
3172
  ...resolved.gate !== void 0 ? { gate: resolved.gate } : {},
2945
3173
  attempts: skip ? 0 : attempts.length,
2946
3174
  notes,
@@ -2964,7 +3192,7 @@ function evalPlan(r, attempts, origin) {
2964
3192
  /** A pairwise question that shows each side's transcript, asked of two harnesses: tool names unblind it. */
2965
3193
  function questionWarnings(resolved) {
2966
3194
  if (resolved.harness.length < 2) return [];
2967
- return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows state.transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
3195
+ return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows the transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
2968
3196
  }
2969
3197
  function captureNote(c) {
2970
3198
  const parts = [];
@@ -3011,6 +3239,16 @@ function fxEffortRefusal(h, gateway) {
3011
3239
  if (h.kind !== "fx" || effort === void 0) return void 0;
3012
3240
  return `fx sends no effort of its own, and reeval adds fx's effort (${JSON.stringify(effort)}) only to its calls on Vertex AI, not on ${gateway}: drop effort or run fx on vertex()`;
3013
3241
  }
3242
+ /**
3243
+ * Why `h` takes no effort on `gateway`, where an effort it set would be refused, or undefined
3244
+ * when it can take one there.
3245
+ */
3246
+ function effortRefusedOn(h, gateway) {
3247
+ if (h.kind === "fx" && gateway !== "vertex") return `fx sends no effort of its own, and reeval adds one only on vertex()`;
3248
+ if (h.kind === "fake" || !isRoutingGateway(gateway)) return void 0;
3249
+ const { name, unprovenEffort } = ROUTING_GATEWAYS[gateway];
3250
+ return unprovenEffort[h.kind]?.(h.model) ? `reeval doesn't send ${agentName(h)}'s effort to ${h.model} through ${name} yet` : void 0;
3251
+ }
3014
3252
  /** A routing gateway's entry: its route table decides whom it serves, and with what effort. */
3015
3253
  function routingEntry(name) {
3016
3254
  const gateway = ROUTING_GATEWAYS[name];
@@ -3157,6 +3395,7 @@ const CONFIG_KEYS = [
3157
3395
  "capture",
3158
3396
  "question",
3159
3397
  "timeout",
3398
+ "snapshot",
3160
3399
  "concurrency",
3161
3400
  "record",
3162
3401
  "history",
@@ -3187,7 +3426,8 @@ const DEFAULTS_KEYS = [
3187
3426
  "judge",
3188
3427
  "capture",
3189
3428
  "question",
3190
- "timeout"
3429
+ "timeout",
3430
+ "snapshot"
3191
3431
  ];
3192
3432
  const LIMIT_KEYS = [
3193
3433
  "runs",
@@ -3543,7 +3783,7 @@ function selectProfile(config, flag, env = process.env) {
3543
3783
  function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3544
3784
  const user = withHostPaths(input, root, file);
3545
3785
  if ("compare" in user) throw new UsageError(`compare can't be set in ${file}: it would multiply every eval. Put it on the group or eval that asks the question.`);
3546
- if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model, state, artifacts } in ${file} sets every question's defaults.`);
3786
+ if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model } in ${file} sets every question's model.`);
3547
3787
  checkOptions(user, file, CONFIG_KEYS);
3548
3788
  const include = user.include ?? defaultIncludeGlobs();
3549
3789
  if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
@@ -3588,6 +3828,10 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3588
3828
  capture: user.capture ?? {},
3589
3829
  question: user.question ?? {},
3590
3830
  timeoutMs: parseDuration(user.timeout ?? "15m"),
3831
+ snapshot: mergeSnapshot({
3832
+ when: true,
3833
+ compress: false
3834
+ }, user.snapshot),
3591
3835
  concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
3592
3836
  record,
3593
3837
  history: {
@@ -3772,6 +4016,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
3772
4016
  });
3773
4017
  }
3774
4018
  //#endregion
3775
- export { isDirect as $, harnessId as A, badJudgeArtifacts as At, sandboxSetBy as B, routedHarness as C, captureWindowSchema as Ct, checkOptions as D, judgeProblem as Dt, checkLimitHarness as E, judgeModelNameSchema as Et, plannedSandboxes as F, resolveStateOptions as G, unbrokered as H, profileJudge as I, allowedHosts as J, DEFAULT_EGRESS as K, refuseIllegalCompare as L, matchesTags as M, filePartsOf as Mt, plainHeaderError as N, resolveJudgeArtifacts as Nt, evalPlan as O, judgeStateSchema as Ot, plainHeaders as P, gatewayPolicy as Q, sandboxId as R, resolveGateway as S, writePluginFiles as St, cascade as T, judgeFields as Tt, unbrokeredError as U, secretVarsIn as V, mergeJudge as W, directOnCloud as X, ciRunner as Y, egressLabel as Z, fellBackToDirect as _, expectedBlockedFor as _t, configPathFor as a, questionShows as at, resolveModelGateway as b, refusedMcp as bt, findConfig as c, resolvedEffortOf as ct, malformedCredentials as d, agentFor as dt, unenforceable as et, missingCredentials as f, agentMcp as ft, selectProfile as g, credentialOf as gt, runJudgeModel as h, claudeMcpServers as ht, configLabel as i, STOPPED_FIRST as it, levelOf as j, decideState as jt, expand as k, artifactRefsOf as kt, hostCapacity as l, servedEffortOf as lt, resolveConcurrency as m, brokeringFor as mt, analysisModelOf as n, createFakeSandbox as nt, defaultConcurrency as o, resolveQuestionState as ot, requireConfig as p, authHint as pt, DEFAULT_EGRESS_ALLOW as q, configFilesPhrase as r, DEFAULT_QUESTION_STATE as rt, defineConfig as s, harnessEffort as st, CONFIG_FILES as t, resolveOutput as tt, loadConfig as u, agentAuth as ut, gatewayScope as v, isAgentKind as vt, brokeredIn as w, fakeJudgeDataSchema as wt, fallbackHint as x, refusedPlugins as xt, modelGatewayOrNone as y, layoutOf as yt, sandboxProvider as z };
4019
+ export { ciRunner as $, expand as A, captureWindowSchema as At, refuseIllegalCompare as B, artifactRefsOf as Bt, resolveGateway as C, credentialOf as Ct, checkLimitHarness as D, refusedMcp as Dt, cascade as E, layoutOf as Et, plainHeaderError as F, judgeFields as Ft, secretVarsIn as G, sandboxId as H, decideState as Ht, plainHeaders as I, judgeModelNameSchema as It, mergeJudge as J, unbrokered as K, plannedSandboxIds as L, judgeProblem as Lt, levelOf as M, NO_JUDGE_INPUT as Mt, majorityOf as N, fakeJudgeDataSchema as Nt, checkOptions as O, refusedPlugins as Ot, matchesTags as P, inputsOn as Pt, allowedHosts as Q, plannedSandboxes as R, sharedInputFields as Rt, fallbackHint as S, claudeMcpServers as St, brokeredIn as T, isAgentKind as Tt, sandboxProvider as U, filePartsOf as Ut, runsOf as V, badJudgeArtifacts as Vt, sandboxSetBy as W, resolveJudgeArtifacts as Wt, DEFAULT_EGRESS as X, resolveStateOptions as Y, DEFAULT_EGRESS_ALLOW as Z, fellBackToDirect as _, agentAuth as _t, configPathFor as a, resolveOutput as at, resolveModelGateway as b, authHint as bt, findConfig as c, parseProbe as ct, malformedCredentials as d, UNRECORDED_QUESTION_STATE as dt, directOnCloud as et, missingCredentials as f, questionShows as ft, selectProfile as g, servedEffortOf as gt, runJudgeModel as h, resolvedEffortOf as ht, configLabel as i, unenforceable as it, harnessId as j, INPUT_KEYS as jt, evalPlan as k, writePluginFiles as kt, hostCapacity as l, probeScript as lt, resolveConcurrency as m, harnessEffort as mt, analysisModelOf as n, gatewayPolicy as nt, defaultConcurrency as o, createFakeSandbox as ot, requireConfig as p, resolveQuestionState as pt, unbrokeredError as q, configFilesPhrase as r, isDirect as rt, defineConfig as s, awaitListening as st, CONFIG_FILES as t, egressLabel as tt, loadConfig as u, STOPPED_FIRST as ut, gatewayScope as v, agentFor as vt, routedHarness as w, expectedBlockedFor as wt, effortRefusedOn as x, brokeringFor as xt, modelGatewayOrNone as y, agentMcp as yt, profileJudge as z, turnsSchema as zt };
3776
4020
 
3777
- //# sourceMappingURL=config-k6m7XbAz.js.map
4021
+ //# sourceMappingURL=config-BjjDryd5.js.map