reeval 0.15.1 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. package/README.md +21 -14
  2. package/dist/{adapter-CbqbMR7y.js → adapter-CqUt4A0w.js} +3 -3
  3. package/dist/{adapter-CbqbMR7y.js.map → adapter-CqUt4A0w.js.map} +1 -1
  4. package/dist/{agent-Zgn-kiqq.js → agent-BtAbRTFq.js} +6 -6
  5. package/dist/{agent-Zgn-kiqq.js.map → agent-BtAbRTFq.js.map} +1 -1
  6. package/dist/{app-files-ChbvRx3P.js → app-files-CDttHPOz.js} +3 -3
  7. package/dist/app-files-CDttHPOz.js.map +1 -0
  8. package/dist/assemble-Bb7zqpRw.js +1 -0
  9. package/dist/{assemble-SKQGW_QF.js → assemble-J9rkE29m.js} +114 -31
  10. package/dist/assemble-J9rkE29m.js.map +1 -0
  11. package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
  12. package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
  13. package/dist/{attempt-host-BOp8JklD.js → attempt-host-DDedP7Qv.js} +17 -8
  14. package/dist/attempt-host-DDedP7Qv.js.map +1 -0
  15. package/dist/capture-Bvc4FtWT.js +15 -0
  16. package/dist/capture-Bvc4FtWT.js.map +1 -0
  17. package/dist/capture-D68TiQQy.js +1 -0
  18. package/dist/{capture-BM_3100g.js → capture-DheNqeYS.js} +1495 -203
  19. package/dist/capture-DheNqeYS.js.map +1 -0
  20. package/dist/{cli-PdYrcoKM.js → cli-KTHjNFoo.js} +329 -74
  21. package/dist/cli-KTHjNFoo.js.map +1 -0
  22. package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{colors-QV9dFVhN.js → colors-Y6O3mtl_.js} +20 -2
  28. package/dist/colors-Y6O3mtl_.js.map +1 -0
  29. package/dist/{command-9xm5ImqI.js → command-1FBECMs-.js} +16 -16
  30. package/dist/{command-9xm5ImqI.js.map → command-1FBECMs-.js.map} +1 -1
  31. package/dist/{command-BqBxLgTc.js → command-Ce03LLtg.js} +14 -14
  32. package/dist/{command-BqBxLgTc.js.map → command-Ce03LLtg.js.map} +1 -1
  33. package/dist/{command-4pJYVCT4.js → command-CztRuwUM.js} +12 -12
  34. package/dist/{command-4pJYVCT4.js.map → command-CztRuwUM.js.map} +1 -1
  35. package/dist/{command-DbSMcxZL.js → command-nNN0mx65.js} +7 -7
  36. package/dist/{command-DbSMcxZL.js.map → command-nNN0mx65.js.map} +1 -1
  37. package/dist/{commands-BU6YJCz9.js → commands-DP13JX9i.js} +24 -12
  38. package/dist/commands-DP13JX9i.js.map +1 -0
  39. package/dist/{commands-Bi2SEQtr.js → commands-ZwQUezI1.js} +1 -1
  40. package/dist/{config-k6m7XbAz.js → config-BjjDryd5.js} +464 -220
  41. package/dist/config-BjjDryd5.js.map +1 -0
  42. package/dist/{contrasts-DLZF4dGF.js → contrasts-DSsRZpR8.js} +2 -2
  43. package/dist/{contrasts-DLZF4dGF.js.map → contrasts-DSsRZpR8.js.map} +1 -1
  44. package/dist/cursor-catalog-D7bFLEAD.js +143 -0
  45. package/dist/cursor-catalog-D7bFLEAD.js.map +1 -0
  46. package/dist/{db-CpsifVln.js → db-B9Ityuit.js} +8 -8
  47. package/dist/{db-CpsifVln.js.map → db-B9Ityuit.js.map} +1 -1
  48. package/dist/db-Bk6LWwza.js +2 -0
  49. package/dist/debug-CAx3qcaP.js +1 -0
  50. package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
  51. package/dist/debug-CgG3T071.js.map +1 -0
  52. package/dist/default-effort-DK8gospa.js +91 -0
  53. package/dist/default-effort-DK8gospa.js.map +1 -0
  54. package/dist/default-effort-DmXHpqo9.js +1 -0
  55. package/dist/{docker-YFj4H1Cr.js → docker-BLCNfUie.js} +1 -1
  56. package/dist/{docker-B4-rx4dA.js → docker-SKpYOlGi.js} +5 -5
  57. package/dist/{docker-B4-rx4dA.js.map → docker-SKpYOlGi.js.map} +1 -1
  58. package/dist/{docker-51WvhN_X.js → docker-hwp3EucJ.js} +9 -5
  59. package/dist/docker-hwp3EucJ.js.map +1 -0
  60. package/dist/{doctor-BuQnfz4e.js → doctor-CT0Sgjng.js} +16 -16
  61. package/dist/{doctor-BuQnfz4e.js.map → doctor-CT0Sgjng.js.map} +1 -1
  62. package/dist/{environment-BOFgzFDg.js → environment-CZdercoj.js} +3 -3
  63. package/dist/{environment-BOFgzFDg.js.map → environment-CZdercoj.js.map} +1 -1
  64. package/dist/{errors-UjcLWelF.js → errors-BZVKVKYd.js} +15 -4
  65. package/dist/errors-BZVKVKYd.js.map +1 -0
  66. package/dist/{eval-source-Dje3rERU.js → eval-source-nZD7-FSz.js} +9 -8
  67. package/dist/eval-source-nZD7-FSz.js.map +1 -0
  68. package/dist/{evaluate-dqXccn4k.d.ts → evaluate-CNi0lZ7n.d.ts} +7 -6
  69. package/dist/{evaluate-BPP9s3s6.js → evaluate-fhC1PCd_.js} +345 -47
  70. package/dist/evaluate-fhC1PCd_.js.map +1 -0
  71. package/dist/{files-Clipx4OW.js → files-BGy05Dlr.js} +4 -4
  72. package/dist/{files-Clipx4OW.js.map → files-BGy05Dlr.js.map} +1 -1
  73. package/dist/{files-Bi461Be2.js → files-BIGxpL08.js} +25 -16
  74. package/dist/{files-Bi461Be2.js.map → files-BIGxpL08.js.map} +1 -1
  75. package/dist/files-C2RC7K4m.js +1 -0
  76. package/dist/files-C3tQtD26.js +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/{files-H24pz0hL.js → files-I1i6Xyh1.js} +4 -4
  80. package/dist/{files-H24pz0hL.js.map → files-I1i6Xyh1.js.map} +1 -1
  81. package/dist/files-dJmrfmOM.js +1 -0
  82. package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
  83. package/dist/gate-BC6SdQZs.js.map +1 -0
  84. package/dist/gateway.d.ts +1 -1
  85. package/dist/{git-CsVZkNNL.js → git-DR02tHtU.js} +2 -2
  86. package/dist/{git-CsVZkNNL.js.map → git-DR02tHtU.js.map} +1 -1
  87. package/dist/{harness-3NVyTSGK.js → harness-ylJLvrbp.js} +39 -5
  88. package/dist/{harness-3NVyTSGK.js.map → harness-ylJLvrbp.js.map} +1 -1
  89. package/dist/harness.d.ts +1 -1
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-4mwhNkvh.js → images---X8Wu9d.js} +8 -8
  92. package/dist/{images-4mwhNkvh.js.map → images---X8Wu9d.js.map} +1 -1
  93. package/dist/index.d.ts +71 -16
  94. package/dist/index.js +5 -5
  95. package/dist/init-BNTXrtMO.js +1 -0
  96. package/dist/{init-cOk3D44b.js → init-CA95ZnkV.js} +4 -4
  97. package/dist/{init-cOk3D44b.js.map → init-CA95ZnkV.js.map} +1 -1
  98. package/dist/{invocations-DW-lTk2f.js → invocations-BdoROAfb.js} +3 -3
  99. package/dist/{invocations-DW-lTk2f.js.map → invocations-BdoROAfb.js.map} +1 -1
  100. package/dist/{judge-CKdgEXN7.js → judge-Dp7FPJ1P.js} +3 -3
  101. package/dist/{judge-CKdgEXN7.js.map → judge-Dp7FPJ1P.js.map} +1 -1
  102. package/dist/judge.d.ts +2 -2
  103. package/dist/judge.js +1 -1
  104. package/dist/{live-XJQ6nqK4.js → live-CBFkdK1K.js} +3 -3
  105. package/dist/{live-XJQ6nqK4.js.map → live-CBFkdK1K.js.map} +1 -1
  106. package/dist/{matchers-DT2TaGix.js → matchers-Cv74mJRJ.js} +43 -13
  107. package/dist/matchers-Cv74mJRJ.js.map +1 -0
  108. package/dist/{model-Bq3G0OG1.js → model-6dfKFkQd.js} +4 -4
  109. package/dist/{model-Bq3G0OG1.js.map → model-6dfKFkQd.js.map} +1 -1
  110. package/dist/{model-CTnyvh7H.js → model-Be7fPvgi.js} +4 -51
  111. package/dist/model-Be7fPvgi.js.map +1 -0
  112. package/dist/{model-g3XnzUv0.js → model-CN4ccjNz.js} +1 -1
  113. package/dist/model-C_5TspZS.js +1 -0
  114. package/dist/{model-vRiUycWh.js → model-DkJVmVd2.js} +4 -4
  115. package/dist/{model-vRiUycWh.js.map → model-DkJVmVd2.js.map} +1 -1
  116. package/dist/{node-BVbbnFtv.js → node-D_Zkwhq2.js} +2 -2
  117. package/dist/{node-BVbbnFtv.js.map → node-D_Zkwhq2.js.map} +1 -1
  118. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  119. package/dist/object-BkXGPKWY.js.map +1 -0
  120. package/dist/{opencode-providers-Bz0l7OZI.js → opencode-providers-DPgdqRfg.js} +26 -2
  121. package/dist/{opencode-providers-Bz0l7OZI.js.map → opencode-providers-DPgdqRfg.js.map} +1 -1
  122. package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-CR00FWAa.js} +2 -2
  123. package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-CR00FWAa.js.map} +1 -1
  124. package/dist/{patch-DKJiC5Sq.js → patch-BJNdOZsf.js} +4 -4
  125. package/dist/patch-BJNdOZsf.js.map +1 -0
  126. package/dist/{plan-DdWkM6SW.js → plan-C1UpGZTk.js} +76 -12
  127. package/dist/plan-C1UpGZTk.js.map +1 -0
  128. package/dist/{plugins-D-CP6mMd.js → plugins-Cx6ReJWE.js} +24 -3
  129. package/dist/plugins-Cx6ReJWE.js.map +1 -0
  130. package/dist/{preflight-9ixuXzsx.js → preflight-BZ3L7P3a.js} +3 -3
  131. package/dist/{preflight-9ixuXzsx.js.map → preflight-BZ3L7P3a.js.map} +1 -1
  132. package/dist/{preflight-DTiVv0gY.js → preflight-CUooKgtT.js} +1 -1
  133. package/dist/{prepare-BfyikQit.js → prepare-BbBoTeDj.js} +7 -7
  134. package/dist/{prepare-BfyikQit.js.map → prepare-BbBoTeDj.js.map} +1 -1
  135. package/dist/{prepare-tmBBf4W8.js → prepare-CEZaEwaq.js} +2 -2
  136. package/dist/{provenance-BizwUn3h.js → provenance-BWvTIlQM.js} +2 -2
  137. package/dist/{provenance-BizwUn3h.js.map → provenance-BWvTIlQM.js.map} +1 -1
  138. package/dist/{providers-B6b9Tnyi.js → providers-B3yqakc7.js} +5 -5
  139. package/dist/{providers-B6b9Tnyi.js.map → providers-B3yqakc7.js.map} +1 -1
  140. package/dist/{providers-DkOupEBl.js → providers-DdyZW_ru.js} +1 -1
  141. package/dist/query-BkwQ0W-c.js +1 -0
  142. package/dist/{query-DyM-VExS.js → query-CZcqpOI9.js} +2 -2
  143. package/dist/{query-DyM-VExS.js.map → query-CZcqpOI9.js.map} +1 -1
  144. package/dist/{questions-BRhU8MzB.js → questions-CFU7-crW.js} +20 -19
  145. package/dist/questions-CFU7-crW.js.map +1 -0
  146. package/dist/{record-DZBCjuwm.js → record-Bu1Xi6MQ.js} +8 -4
  147. package/dist/record-Bu1Xi6MQ.js.map +1 -0
  148. package/dist/{remarks-C7i2LMRb.js → remarks-A3j-SkJ_.js} +42 -8
  149. package/dist/remarks-A3j-SkJ_.js.map +1 -0
  150. package/dist/remarks-Bp6AwPgb.js +1 -0
  151. package/dist/{remarks-D_3qhBWU.js → remarks-Dj_x9l5n.js} +4 -3
  152. package/dist/{remarks-D_3qhBWU.js.map → remarks-Dj_x9l5n.js.map} +1 -1
  153. package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
  154. package/dist/remarks-eBcrIZRu.js.map +1 -0
  155. package/dist/{remote-DFIzcx_6.js → remote-EIFHdMd8.js} +5 -5
  156. package/dist/{remote-DFIzcx_6.js.map → remote-EIFHdMd8.js.map} +1 -1
  157. package/dist/{resolve-DWSJpKHX.js → resolve-8H5rz8mi.js} +5 -5
  158. package/dist/{resolve-DWSJpKHX.js.map → resolve-8H5rz8mi.js.map} +1 -1
  159. package/dist/resolve-Tm9pX7Hv.js +1 -0
  160. package/dist/{results-BC3pcG_Z.js → results-E7JdVjq2.js} +9 -9
  161. package/dist/{results-BC3pcG_Z.js.map → results-E7JdVjq2.js.map} +1 -1
  162. package/dist/{resume-j-rcIryi.js → resume-Blu4yIcq.js} +4 -4
  163. package/dist/{resume-j-rcIryi.js.map → resume-Blu4yIcq.js.map} +1 -1
  164. package/dist/{runtime-d56Gg3J5.js → runtime-CNbsMJQ7.js} +111 -37
  165. package/dist/runtime-CNbsMJQ7.js.map +1 -0
  166. package/dist/{sandbox-Cmln11gU.js → sandbox-RButGfIE.js} +2 -2
  167. package/dist/{sandbox-Cmln11gU.js.map → sandbox-RButGfIE.js.map} +1 -1
  168. package/dist/sandbox.d.ts +1 -1
  169. package/dist/sandbox.js +1 -1
  170. package/dist/server-Dqrq-Sia.js +1 -0
  171. package/dist/{server-BOoH10J8.js → server-XgCYs_MF.js} +35 -11
  172. package/dist/server-XgCYs_MF.js.map +1 -0
  173. package/dist/{source-wD04-rRG.js → source-BOkbJCD2.js} +17 -3
  174. package/dist/{source-wD04-rRG.js.map → source-BOkbJCD2.js.map} +1 -1
  175. package/dist/source-DooSTtyz.js +1 -0
  176. package/dist/src-1M0srjji.js +1 -0
  177. package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
  178. package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
  179. package/dist/{src-D91Hd6hT.js → src-BJCKWacl.js} +38 -18
  180. package/dist/src-BJCKWacl.js.map +1 -0
  181. package/dist/src-BZlhaDhE.js +2 -0
  182. package/dist/{src-BO8AYqJ5.js → src-BkhEPVJZ.js} +12 -3
  183. package/dist/{src-BO8AYqJ5.js.map → src-BkhEPVJZ.js.map} +1 -1
  184. package/dist/{src-C7upnZSd.js → src-Bp6X4193.js} +1 -1
  185. package/dist/{src-BYLM3hEz.js → src-C2Japg_I.js} +21 -10
  186. package/dist/{src-BYLM3hEz.js.map → src-C2Japg_I.js.map} +1 -1
  187. package/dist/src-C5q3_fRV.js +3 -0
  188. package/dist/src-C9EHSYyP.js +2 -0
  189. package/dist/src-CJdpsrXi.js +2 -0
  190. package/dist/{src-1lpTocbd.js → src-D1ET6oPL.js} +3 -27
  191. package/dist/{src-1lpTocbd.js.map → src-D1ET6oPL.js.map} +1 -1
  192. package/dist/{src-Bp4aN3rA.js → src-D4CZjMQ4.js} +1 -1
  193. package/dist/{src-BYPMM1x7.js → src-GkJ-1Rdy.js} +1 -1
  194. package/dist/{src-kYeaJJrl.js → src-OOKE-5GA.js} +1 -1
  195. package/dist/state-BCyNaz13.js +1 -0
  196. package/dist/{state-DrJ687ut.js → state-D6-tvqtp.js} +8 -8
  197. package/dist/{state-DrJ687ut.js.map → state-D6-tvqtp.js.map} +1 -1
  198. package/dist/suite-BiWajSlA.js +2 -0
  199. package/dist/{suite-Co8cwelb.js → suite-DYI1k2Ie.js} +22 -5
  200. package/dist/suite-DYI1k2Ie.js.map +1 -0
  201. package/dist/{summary-DLFHVoux.js → summary-CN2aqAUz.js} +59 -17
  202. package/dist/summary-CN2aqAUz.js.map +1 -0
  203. package/dist/tarball-CHcahyNK.js +258 -0
  204. package/dist/tarball-CHcahyNK.js.map +1 -0
  205. package/dist/testing.d.ts +96 -44
  206. package/dist/testing.js +4 -4
  207. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  208. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  209. package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
  210. package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
  211. package/dist/{types-coMWHaU2.d.ts → types-DDeCnNmp.d.ts} +437 -181
  212. package/dist/view/index.d.ts +34 -5
  213. package/dist/view/index.js +8 -7
  214. package/dist/views-7bNY17Z6.js +2 -0
  215. package/dist/{views-BFO1BLKu.js → views-Bd-H71_t.js} +5 -5
  216. package/dist/{views-BFO1BLKu.js.map → views-Bd-H71_t.js.map} +1 -1
  217. package/dist/{worker-entry-CJtdsKBP.js → worker-entry-BIHJqE_z.js} +34 -13
  218. package/dist/worker-entry-BIHJqE_z.js.map +1 -0
  219. package/dist/workspace-CxwHv8mG.js +1 -0
  220. package/dist/{workspace-BRxjfa8N.js → workspace-Sy5g8zW9.js} +4 -4
  221. package/dist/{workspace-BRxjfa8N.js.map → workspace-Sy5g8zW9.js.map} +1 -1
  222. package/package.json +1 -1
  223. package/skills/eval-rubric/SKILL.md +11 -0
  224. package/dist/app-files-ChbvRx3P.js.map +0 -1
  225. package/dist/assemble-SKQGW_QF.js.map +0 -1
  226. package/dist/assemble-XBWuK3Ed.js +0 -1
  227. package/dist/attempt-host-BOp8JklD.js.map +0 -1
  228. package/dist/capture-BM_3100g.js.map +0 -1
  229. package/dist/capture-DSK8O5bA.js +0 -1
  230. package/dist/cli-PdYrcoKM.js.map +0 -1
  231. package/dist/cli-spec-D0qyrPZi.js.map +0 -1
  232. package/dist/client-Ct6s6OnF.js.map +0 -1
  233. package/dist/colors-QV9dFVhN.js.map +0 -1
  234. package/dist/commands-BU6YJCz9.js.map +0 -1
  235. package/dist/config-k6m7XbAz.js.map +0 -1
  236. package/dist/db-DkD-F-X-.js +0 -2
  237. package/dist/debug-BhizgAsU.js.map +0 -1
  238. package/dist/debug-DeM5pswK.js +0 -1
  239. package/dist/docker-51WvhN_X.js.map +0 -1
  240. package/dist/errors-UjcLWelF.js.map +0 -1
  241. package/dist/eval-source-Dje3rERU.js.map +0 -1
  242. package/dist/evaluate-BPP9s3s6.js.map +0 -1
  243. package/dist/files-BAt0i00X.js +0 -1
  244. package/dist/files-C2h2Cw4F.js +0 -1
  245. package/dist/files-Ca0d6Uq3.js.map +0 -1
  246. package/dist/files-DmUrRmmL.js +0 -1
  247. package/dist/init-BDEOopK7.js +0 -1
  248. package/dist/matchers-DT2TaGix.js.map +0 -1
  249. package/dist/model-CTnyvh7H.js.map +0 -1
  250. package/dist/model-D-SWrLZG.js +0 -1
  251. package/dist/object-CLf2-tNq.js.map +0 -1
  252. package/dist/pairs-puYi-zj2.js.map +0 -1
  253. package/dist/patch-DKJiC5Sq.js.map +0 -1
  254. package/dist/plan-DdWkM6SW.js.map +0 -1
  255. package/dist/plugins-D-CP6mMd.js.map +0 -1
  256. package/dist/query-Clm2mNSm.js +0 -1
  257. package/dist/questions-BRhU8MzB.js.map +0 -1
  258. package/dist/record-DZBCjuwm.js.map +0 -1
  259. package/dist/remarks-BwJ8pcTQ.js +0 -1
  260. package/dist/remarks-C7i2LMRb.js.map +0 -1
  261. package/dist/remarks-dA8LyoPc.js.map +0 -1
  262. package/dist/resolve--rQZ8u6Z.js +0 -1
  263. package/dist/runtime-d56Gg3J5.js.map +0 -1
  264. package/dist/server-BOoH10J8.js.map +0 -1
  265. package/dist/server-DpuVAbrO.js +0 -1
  266. package/dist/source-BFd6f0-R.js +0 -1
  267. package/dist/src-6_zz7oJs.js +0 -2
  268. package/dist/src-D91Hd6hT.js.map +0 -1
  269. package/dist/src-DH-9IO7f.js +0 -1
  270. package/dist/src-HSxBazcg.js +0 -3
  271. package/dist/src-LD1BAJFa.js +0 -2
  272. package/dist/src-dlX-Lmxg.js +0 -2
  273. package/dist/state-B5PLm5qY.js +0 -1
  274. package/dist/suite-22Kt9Grx.js +0 -2
  275. package/dist/suite-Co8cwelb.js.map +0 -1
  276. package/dist/summary-DLFHVoux.js.map +0 -1
  277. package/dist/views-BOf8qEL8.js +0 -2
  278. package/dist/worker-entry-CJtdsKBP.js.map +0 -1
  279. package/dist/workspace-DDHzkMjm.js +0 -1
@@ -1,36 +1,40 @@
1
- import { t as isObjectLike } from "./object-CLf2-tNq.js";
2
- import { r as withLease } from "./client-Ct6s6OnF.js";
1
+ import { t as isObjectLike } from "./object-BkXGPKWY.js";
2
+ import { r as withLease } from "./client-DMmOkbRA.js";
3
3
  import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
4
4
  import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
5
- import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-BOFgzFDg.js";
6
- import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-D-CP6mMd.js";
5
+ import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-CZdercoj.js";
6
+ import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-Cx6ReJWE.js";
7
7
  import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
8
- import { $ as isDirect, A as harnessId, At as badJudgeArtifacts, B as sandboxSetBy, D as checkOptions, Dt as judgeProblem, G as resolveStateOptions, H as unbrokered, J as allowedHosts, L as refuseIllegalCompare, M as matchesTags, Mt as filePartsOf, N as plainHeaderError, Nt as resolveJudgeArtifacts, O as evalPlan, P as plainHeaders, R as sandboxId, S as resolveGateway, St as writePluginFiles, T as cascade, U as unbrokeredError, V as secretVarsIn, W as mergeJudge, Z as egressLabel, _ as fellBackToDirect, _t as expectedBlockedFor, b as resolveModelGateway, bt as refusedMcp, et as unenforceable, ft as agentMcp, ht as claudeMcpServers, jt as decideState, k as expand, kt as artifactRefsOf, mt as brokeringFor, nt as createFakeSandbox, v as gatewayScope, w as brokeredIn, xt as refusedPlugins, yt as layoutOf, z as sandboxProvider } from "./config-k6m7XbAz.js";
9
- import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-CKdgEXN7.js";
8
+ import { A as expand, B as refuseIllegalCompare, Bt as artifactRefsOf, C as resolveGateway, Dt as refusedMcp, E as cascade, Et as layoutOf, F as plainHeaderError, G as secretVarsIn, H as sandboxId, Ht as decideState, I as plainHeaders, J as mergeJudge, K as unbrokered, Lt as judgeProblem, Mt as NO_JUDGE_INPUT, N as majorityOf, O as checkOptions, Ot as refusedPlugins, P as matchesTags, Pt as inputsOn, Q as allowedHosts, St as claudeMcpServers, T as brokeredIn, U as sandboxProvider, Ut as filePartsOf, V as runsOf, Vt as badJudgeArtifacts, W as sandboxSetBy, Wt as resolveJudgeArtifacts, Y as resolveStateOptions, _ as fellBackToDirect, b as resolveModelGateway, it as unenforceable, j as harnessId, jt as INPUT_KEYS, k as evalPlan, kt as writePluginFiles, ot as createFakeSandbox, q as unbrokeredError, rt as isDirect, tt as egressLabel, v as gatewayScope, wt as expectedBlockedFor, xt as brokeringFor, yt as agentMcp } from "./config-BjjDryd5.js";
9
+ import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-Dp7FPJ1P.js";
10
10
  import { v as vertexSettings } from "./vertex-DypYjx71.js";
11
- import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
11
+ import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-DVIaahLv.js";
12
12
  import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
13
13
  import { t as inCI } from "./env-CARj5Axq.js";
14
- import { ft as shellQuote, t as MCP_TIMER } from "./src-D91Hd6hT.js";
14
+ import { pt as shellQuote, t as MCP_TIMER, ut as OPAQUE_OUTPUT } from "./src-BJCKWacl.js";
15
15
  import { t as InfraError } from "./types-C5vI3g1C.js";
16
- import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-d56Gg3J5.js";
17
- import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-Ba2QerOK.js";
18
- import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
19
- import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
20
- import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-BM_3100g.js";
16
+ import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CNbsMJQ7.js";
17
+ import { c as formatBytes } from "./debug-CgG3T071.js";
18
+ import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CpbZ9arV.js";
19
+ import { t as attemptDebugDir } from "./files-D1y8Pg4Z.js";
20
+ import { r as openRouterFallbacks } from "./openrouter-models-CR00FWAa.js";
21
+ import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-DheNqeYS.js";
21
22
  import { t as contextTaker } from "./context-Dgn48BqT.js";
22
- import { a as skillFiles, i as resolveEnvironment } from "./resolve-DWSJpKHX.js";
23
+ import { a as skillFiles, i as resolveEnvironment } from "./resolve-8H5rz8mi.js";
23
24
  import { t as PrepareError } from "./provider-CmAYTQQs.js";
24
- import { i as preparedWorkspace } from "./workspace-BRxjfa8N.js";
25
+ import { i as preparedWorkspace } from "./workspace-Sy5g8zW9.js";
25
26
  import { t as patchPath } from "./patch-CquGAGX5.js";
26
- import { n as judgeDiff } from "./patch-DKJiC5Sq.js";
27
- import { t as transcriptForJudge } from "./transcript-DxPaWHTZ.js";
28
- import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-Dje3rERU.js";
27
+ import { n as judgeDiff } from "./patch-BJNdOZsf.js";
28
+ import { t as transcriptForJudge } from "./transcript-BJrKfgRW.js";
29
+ import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CHcahyNK.js";
30
+ import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-nZD7-FSz.js";
31
+ import { i as withDefaultEffort } from "./default-effort-DK8gospa.js";
29
32
  import { dirname, join } from "node:path";
30
33
  import { parse, stringify } from "yaml";
31
34
  import { z } from "zod";
32
35
  import { setTimeout as setTimeout$1 } from "node:timers/promises";
33
- import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
36
+ import { gunzipSync, gzipSync } from "node:zlib";
37
+ import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
34
38
  //#region src/attempt/assertions.ts
35
39
  const MESSAGE_LIMIT = 1024;
36
40
  const LABEL_LIMIT = 60;
@@ -76,6 +80,7 @@ function assertionRecords(raw) {
76
80
  matcher: r.matcher,
77
81
  status: r.status,
78
82
  soft: r.soft,
83
+ ...r.at ? { at: r.at } : {},
79
84
  ...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
80
85
  ...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
81
86
  };
@@ -84,7 +89,7 @@ function assertionRecords(raw) {
84
89
  const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
85
90
  /**
86
91
  * What fails the attempt at its end for the checks that failed without ending the body: the first
87
- * that didn't pass, an `expect.soft` as what it threw, a hard check the eval caught with
92
+ * that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
88
93
  * `CAUGHT_HINT` after its message.
89
94
  */
90
95
  function checkFailure(raw) {
@@ -436,7 +441,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
436
441
  //#region src/harness/index.ts
437
442
  async function createHarness(descriptor, vertex) {
438
443
  if (descriptor.kind === "fake") return createFakeHarness(descriptor);
439
- const { createAgentHarness } = await import("./agent-Zgn-kiqq.js");
444
+ const { createAgentHarness } = await import("./agent-BtAbRTFq.js");
440
445
  return createAgentHarness(descriptor, vertex);
441
446
  }
442
447
  /** Where each harness's agent looks for repo skills, relative to the workspace. */
@@ -466,8 +471,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
466
471
  if (debug) traceEnvironment(debug, spec, env);
467
472
  record.image = null;
468
473
  if (spec.sandbox.kind === "docker") {
469
- const { preparedFor } = await import("./preflight-DTiVv0gY.js");
470
- const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DkOupEBl.js");
474
+ const { preparedFor } = await import("./preflight-CUooKgtT.js");
475
+ const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DdyZW_ru.js");
471
476
  signal.throwIfAborted();
472
477
  const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
473
478
  const builds = builtPlugins(spec.environment.layers);
@@ -536,16 +541,16 @@ function describeSize(state, question) {
536
541
  return `${p.name} ${k(tokensOf(p.chars))}${top.length ? `, of which ${top.join(", ")}` : ""}`;
537
542
  }).join("; ");
538
543
  }
539
- /** `Narrow state.diff, state.transcript or state.output.`: the state's parts, largest first. */
544
+ /** `Narrow diff, transcript or output.`: the inputs that were on, largest first. */
540
545
  function stateHint(state, question) {
541
- const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => `state.${p.name}`);
546
+ const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => p.name);
542
547
  if (!keys.length) return "";
543
548
  const last = keys.pop();
544
549
  return ` Narrow ${keys.length ? `${keys.join(", ")} or ${last}` : last}.`;
545
550
  }
546
551
  /**
547
552
  * A provider's error, as the provider worded it, then what the judge's input held: its estimated
548
- * size and biggest parts and files, and the `state` keys that were on, largest first. reeval knows
553
+ * size and biggest parts and files, and the inputs that were on, largest first. reeval knows
549
554
  * no model's limit, so this follows every provider error.
550
555
  */
551
556
  const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
@@ -585,7 +590,7 @@ const workspacePath = (path) => path.replace(/^\.\//, "");
585
590
  /** A `contents` entry that names one file rather than matching many. */
586
591
  const isExactPath = (entry) => !entry.startsWith("!") && !/[*?[\]{}]/.test(entry);
587
592
  /**
588
- * The judge's state for what the `state` option chose: the graded turn's task and output (or every turn's),
593
+ * The judge's state for the inputs the call turned on: the graded turn's task and output (or every turn's),
589
594
  * its changed files and diff as `judgeDiff` shows them, files whole, `c.exec` results and the
590
595
  * transcript.
591
596
  */
@@ -633,7 +638,7 @@ function checksShown(which, ran) {
633
638
  if (which === false) return void 0;
634
639
  if (which !== true) {
635
640
  const missing = which.find((command) => !ran.some((r) => r.command === command));
636
- if (missing !== void 0) throw new Error(`c.judge() state.checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
641
+ if (missing !== void 0) throw new Error(`c.judge() checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
637
642
  }
638
643
  return ran.filter((r) => which === true || which.includes(r.command)).map(({ command, exitCode, stdout, stderr }) => ({
639
644
  command,
@@ -657,24 +662,24 @@ function placeFor(calls, order) {
657
662
  * checks after it still run.
658
663
  */
659
664
  function judgeCaller(a, readFile) {
660
- const ask = async (order, question, { on, ...call } = {}) => {
661
- if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
665
+ const ask = async (order, question, { on, ...call }) => {
662
666
  const problem = judgeProblem(call, { at: "call" });
663
667
  if (problem) throw new Error(problem);
664
668
  const badArts = badJudgeArtifacts(call.artifacts);
665
669
  if (badArts) throw new Error(`c.judge() artifacts ${badArts}`);
666
670
  const subject = on && "turn" in on ? a.turns[on.turn] : a.turns.at(-1);
667
671
  if (!subject) throw new Error(on && "turn" in on ? `c.judge() on names turn ${on.turn}, but this attempt has ${a.turns.length}` : "c.judge() has no turn to grade: call c.send() first");
668
- const options = mergeJudge(a.spec.judge, call);
672
+ const wanted = resolveStateOptions(call);
673
+ if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives a value to grade as the output, but output is off. Pass output: true.");
674
+ const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, artifacts, ...callJudge } = call;
675
+ const options = mergeJudge(a.spec.judge, callJudge);
669
676
  const { model } = options;
670
677
  if (model === void 0) {
671
678
  const message = noJudgeModel(getRunContext().config.file);
672
679
  a.judgeThrown.add(message);
673
680
  throw new Error(message);
674
681
  }
675
- const wanted = resolveStateOptions(options.state);
676
- if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
677
- const { state: _state, maxConcurrent, ...judgeOptions } = options;
682
+ const { maxConcurrent, ...judgeOptions } = options;
678
683
  const output = on && "value" in on ? on.value : subject.text;
679
684
  const { draft, dropped } = await draftState({
680
685
  turns: a.turns,
@@ -687,7 +692,8 @@ function judgeCaller(a, readFile) {
687
692
  });
688
693
  const name = judgeModelName(model);
689
694
  const state = redactValue(stateOf(draft));
690
- const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
695
+ const resolved = resolveJudgeArtifacts(artifacts, a.artifactBlobs);
696
+ if (artifacts === true && !resolved.length && inputsOn(call, INPUT_KEYS).length === 1) throw new Error("c.judge() artifacts: true shows the judge nothing yet: this attempt has no captures. Call c.capture() first, or turn on another input.");
691
697
  const refs = artifactRefsOf(resolved);
692
698
  const decide = decideState(state, filePartsOf(resolved));
693
699
  const input = {
@@ -789,7 +795,10 @@ function judgeCaller(a, readFile) {
789
795
  return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
790
796
  }
791
797
  };
792
- return async (question, { on, ...options } = {}) => {
798
+ return async (question, given) => {
799
+ if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
800
+ if (given === null || typeof given !== "object" || Array.isArray(given)) throw new Error(judgeProblem(given, { at: "call" }) ?? NO_JUDGE_INPUT);
801
+ const { on, ...options } = given;
793
802
  const order = a.judgeCalls++;
794
803
  const call = on === void 0 ? options : {
795
804
  ...options,
@@ -805,7 +814,7 @@ async function createSandbox(descriptor, options) {
805
814
  switch (descriptor.kind) {
806
815
  case "fake": return await createFakeSandbox(descriptor, options.env);
807
816
  case "docker": {
808
- const { createDockerSandbox } = await import("./docker-YFj4H1Cr.js");
817
+ const { createDockerSandbox } = await import("./docker-BLCNfUie.js");
809
818
  return await createDockerSandbox(descriptor, options);
810
819
  }
811
820
  }
@@ -849,8 +858,8 @@ function isJudges(a, error, raw) {
849
858
  }
850
859
  /**
851
860
  * The error that fails the attempt: a judge's gives way to the first other check that didn't hold
852
- * (an `expect.soft`, or a hard check the eval caught), whichever came first, so the attempt fails
853
- * as that check.
861
+ * (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
862
+ * fails as that check.
854
863
  */
855
864
  function attemptError(a, error, raw) {
856
865
  if (!isJudges(a, error, raw)) return error;
@@ -954,7 +963,7 @@ async function writeRecord(a, stagingDir, id) {
954
963
  const { record, spec } = a;
955
964
  const recordPath = join(stagingDir, "evals", recordFileName(attemptKey(id, spec.cell.key, spec.n)));
956
965
  await mkdir(join(stagingDir, "evals"), { recursive: true });
957
- await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
966
+ await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), new Map([...a.artifactBlobs, ...a.stepBlobs]), a.capturePages);
958
967
  const tmp = `${recordPath}.${process.pid}.tmp`;
959
968
  await writeFile(tmp, stagedRecordJson(record));
960
969
  await rename(tmp, recordPath);
@@ -968,12 +977,23 @@ async function writeRecord(a, stagingDir, id) {
968
977
  question: clip(j.question, 200)
969
978
  })),
970
979
  turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
980
+ ...failedChecks(record),
971
981
  ...record.error?.prepare ? { prepare: {
972
982
  ref: record.error.prepare.ref,
973
983
  stage: record.error.prepare.stage
974
984
  } } : {}
975
985
  };
976
986
  }
987
+ /** The record's checks that didn't pass, for the terminal; unset when all did. */
988
+ function failedChecks(record) {
989
+ const failed = (record.assertions ?? []).filter((x) => x.status !== "pass");
990
+ if (!failed.length) return {};
991
+ return { failedChecks: failed.map((x) => ({
992
+ label: x.label,
993
+ ...x.at ? { at: x.at } : {},
994
+ message: clip(x.message?.split("\n")[0] ?? "", 500)
995
+ })) };
996
+ }
977
997
  /** `turn N error: <message>` for each turn with an error event. */
978
998
  function turnErrors(transcripts) {
979
999
  return transcripts.flatMap((events, i) => {
@@ -1190,7 +1210,7 @@ const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
1190
1210
  //#region src/attempt/state.ts
1191
1211
  /** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
1192
1212
  const SETUP_BUDGET_MS = 12e5;
1193
- function createAttempt(spec, signal, progress, start) {
1213
+ function createAttempt(spec, signal, progress, start, snapshotDir) {
1194
1214
  const record = {
1195
1215
  startedAt: (/* @__PURE__ */ new Date()).toISOString(),
1196
1216
  outcome: "pass",
@@ -1245,7 +1265,10 @@ function createAttempt(spec, signal, progress, start) {
1245
1265
  judgeInputCalls: [],
1246
1266
  captureNames: /* @__PURE__ */ new Set(),
1247
1267
  backgrounds: [],
1248
- artifactBlobs: /* @__PURE__ */ new Map()
1268
+ artifactBlobs: /* @__PURE__ */ new Map(),
1269
+ capturePages: /* @__PURE__ */ new Map(),
1270
+ stepBlobs: /* @__PURE__ */ new Map(),
1271
+ snapshotDir
1249
1272
  };
1250
1273
  return a;
1251
1274
  }
@@ -1638,6 +1661,237 @@ async function startSession(a, runId) {
1638
1661
  return [box, session];
1639
1662
  }
1640
1663
  //#endregion
1664
+ //#region src/attempt/snapshot.ts
1665
+ /** The most a snapshot directory's files may add up to; a larger one isn't taken. */
1666
+ const SNAPSHOT_DIR_LIMIT = 50 * 2 ** 20;
1667
+ /** The most a compressed snapshot may be. */
1668
+ const SNAPSHOT_TAR_LIMIT = 10 * 2 ** 20;
1669
+ /** The most a compressed snapshot may be uncompressed, which the host holds in memory to mask. */
1670
+ const SNAPSHOT_RAW_LIMIT = 200 * 2 ** 20;
1671
+ /** How long one snapshot may take before the attempt goes on without it. */
1672
+ const SNAPSHOT_TIMEOUT_MS = 12e4;
1673
+ /** Installed dependencies, caches and git's objects: left out by directory name, at any depth. */
1674
+ const EXCLUDED_DIRS = [
1675
+ ".git",
1676
+ "node_modules",
1677
+ ".pnpm-store",
1678
+ ".venv",
1679
+ "venv",
1680
+ ".tox",
1681
+ "__pycache__",
1682
+ ".pytest_cache",
1683
+ ".mypy_cache",
1684
+ ".ruff_cache",
1685
+ ".next",
1686
+ ".nuxt",
1687
+ ".svelte-kit",
1688
+ ".turbo",
1689
+ ".parcel-cache",
1690
+ ".cache",
1691
+ ".dart_tool",
1692
+ ".gradle"
1693
+ ];
1694
+ const LISTED = 50;
1695
+ const SIZE_MARK = "--reeval-size--";
1696
+ const excluded = (path) => path.split("/").some((part) => EXCLUDED_DIRS.includes(part));
1697
+ const utf8 = new TextDecoder("utf-8", { fatal: true });
1698
+ function asText(data) {
1699
+ if (data.subarray(0, 8192).includes(0)) return null;
1700
+ try {
1701
+ return utf8.decode(data);
1702
+ } catch {
1703
+ return null;
1704
+ }
1705
+ }
1706
+ /** The sandbox's archive, with the excluded directories gone and secret-looking values masked. */
1707
+ function cleanArchive(tar) {
1708
+ const entries = [];
1709
+ const masked = [];
1710
+ const dropped = [];
1711
+ const left = /* @__PURE__ */ new Set();
1712
+ for (const e of readTar(tar)) {
1713
+ if (excluded(e.path)) {
1714
+ const parts = e.path.split("/");
1715
+ const at = parts.findIndex((p) => EXCLUDED_DIRS.includes(p));
1716
+ left.add(redactSecrets(`${parts.slice(0, at + 1).join("/")}/`));
1717
+ continue;
1718
+ }
1719
+ if (findSecrets(e.path).length || e.type === "symlink" && findSecrets(e.target).length) {
1720
+ dropped.push(redactSecrets(e.path));
1721
+ continue;
1722
+ }
1723
+ if (e.type !== "file") {
1724
+ entries.push(e);
1725
+ continue;
1726
+ }
1727
+ const text = asText(e.data);
1728
+ if (text === null) {
1729
+ if (findSecrets(e.data.toString("latin1")).length) dropped.push(e.path);
1730
+ else entries.push(e);
1731
+ continue;
1732
+ }
1733
+ const clean = redactSecrets(text);
1734
+ if (clean === text) entries.push(e);
1735
+ else {
1736
+ masked.push(e.path);
1737
+ entries.push({
1738
+ ...e,
1739
+ data: Buffer.from(clean)
1740
+ });
1741
+ }
1742
+ }
1743
+ return {
1744
+ entries,
1745
+ masked,
1746
+ dropped,
1747
+ excluded: [...left]
1748
+ };
1749
+ }
1750
+ const tarCommand = (workdir) => {
1751
+ const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
1752
+ return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
1753
+ };
1754
+ const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
1755
+ const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
1756
+ const listed = (paths) => paths.slice(0, LISTED);
1757
+ const LIMITS = {
1758
+ dir: SNAPSHOT_DIR_LIMIT,
1759
+ tar: SNAPSHOT_TAR_LIMIT,
1760
+ raw: SNAPSHOT_RAW_LIMIT
1761
+ };
1762
+ /**
1763
+ * Archives the sandbox's workspace and returns what to record and the masked entries to write,
1764
+ * or why none: over its form's cap, or an archive the sandbox couldn't make.
1765
+ */
1766
+ async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
1767
+ const none = (cause, message, dirs = []) => ({
1768
+ taken: {
1769
+ bytes: 0,
1770
+ files: 0,
1771
+ excluded: listed(dirs),
1772
+ masked: [],
1773
+ dropped: [],
1774
+ skipped: {
1775
+ cause,
1776
+ message
1777
+ }
1778
+ },
1779
+ entries: null
1780
+ });
1781
+ const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
1782
+ const tar = tarCommand(box.workdir);
1783
+ const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
1784
+ const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
1785
+ const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
1786
+ const rawBytes = Number(raw.trim());
1787
+ const gzBytes = Number(size.trim());
1788
+ if (sized.exitCode !== 0 || !(rawBytes > 0) || !(gzBytes > 0)) return none("error", `couldn't archive the workspace: ${(sized.stderr || sized.stdout).trim()}`);
1789
+ const without = dirs.length ? dirs.join(", ") : "its dependency and cache directories";
1790
+ const rawLimit = compress ? limits.raw : limits.dir;
1791
+ if (rawBytes > rawLimit) return none("size", `the workspace is ${formatBytes(rawBytes)} without ${without}, over the ${formatBytes(rawLimit)} cap${compress ? "" : ` (compressed, the cap is ${formatBytes(limits.tar)})`}`, dirs);
1792
+ if (compress && gzBytes > limits.tar) return none("size", `the workspace is ${formatBytes(gzBytes)} compressed without ${without}, over the ${formatBytes(limits.tar)} cap`, dirs);
1793
+ const r = await box.execOwn(`${NEEDS_TAR}; echo ${OPAQUE_OUTPUT}; ${tar} | { ${GZIP}; } | base64`, { signal });
1794
+ const encoded = r.stdout.startsWith(`--opaque-output--\n`) ? r.stdout.slice(OPAQUE_OUTPUT.length + 1) : "";
1795
+ const fetched = Buffer.from(encoded, "base64");
1796
+ if (r.exitCode !== 0 || fetched.length === 0) return none("error", `couldn't archive the workspace: ${r.stderr.trim() || "no output"}`);
1797
+ const clean = cleanArchive(fetched[0] === 31 && fetched[1] === 139 ? gunzipSync(fetched, { maxOutputLength: rawLimit + 2 ** 20 }) : fetched);
1798
+ const all = [.../* @__PURE__ */ new Set([...dirs, ...clean.excluded])].sort();
1799
+ return {
1800
+ taken: {
1801
+ bytes: 0,
1802
+ files: clean.entries.filter((e) => e.type !== "dir").length,
1803
+ excluded: listed(all),
1804
+ masked: listed(clean.masked),
1805
+ dropped: listed(clean.dropped),
1806
+ skipped: null
1807
+ },
1808
+ entries: clean.entries
1809
+ };
1810
+ }
1811
+ /** Where an attempt's snapshots wait, beside its staging record, until the run is assembled. */
1812
+ const snapshotSidecar = (stagingDir, key) => join(stagingDir, "evals", recordFileName(key).replace(/\.json$/, ".snapshots"));
1813
+ /**
1814
+ * Takes snapshot `name` of the workspace into the attempt's sidecar and adds it to the record;
1815
+ * never fails the attempt. Adds nothing when the attempt's signal stopped it.
1816
+ */
1817
+ async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SNAPSHOT_TIMEOUT_MS) {
1818
+ const { compress } = a.spec.snapshot;
1819
+ const form = compress ? "tar.gz" : "dir";
1820
+ const dir = a.snapshotDir;
1821
+ const target = join(dir, compress ? `${name}.tar.gz` : name);
1822
+ const timeout = AbortSignal.timeout(timeoutMs);
1823
+ const add = (taken) => {
1824
+ a.record.snapshots ??= [];
1825
+ a.record.snapshots.push({
1826
+ name,
1827
+ form,
1828
+ path: null,
1829
+ ...taken
1830
+ });
1831
+ };
1832
+ const failed = (message) => {
1833
+ debugLog(phase, `no ${name} snapshot: ${message}`);
1834
+ add({
1835
+ bytes: 0,
1836
+ files: 0,
1837
+ excluded: [],
1838
+ masked: [],
1839
+ dropped: [],
1840
+ skipped: {
1841
+ cause: "error",
1842
+ message: `couldn't archive the workspace: ${message}`
1843
+ }
1844
+ });
1845
+ };
1846
+ try {
1847
+ const { taken, entries } = await archiveWorkspace(box, AbortSignal.any([a.signal, timeout]), compress, limits);
1848
+ if (!entries) {
1849
+ debugLog(phase, `no ${name} snapshot: ${taken.skipped?.message}`);
1850
+ add(taken);
1851
+ return;
1852
+ }
1853
+ await rm(target, {
1854
+ recursive: true,
1855
+ force: true
1856
+ });
1857
+ await mkdir(dir, { recursive: true });
1858
+ if (compress) {
1859
+ const archive = gzipSync(writeTar(entries));
1860
+ if (archive.length > limits.tar) {
1861
+ const message = `the workspace is ${formatBytes(archive.length)} compressed, over the ${formatBytes(limits.tar)} cap`;
1862
+ debugLog(phase, `no ${name} snapshot: ${message}`);
1863
+ add({
1864
+ ...taken,
1865
+ files: 0,
1866
+ skipped: {
1867
+ cause: "size",
1868
+ message
1869
+ }
1870
+ });
1871
+ return;
1872
+ }
1873
+ await writeFile(target, archive);
1874
+ taken.bytes = archive.length;
1875
+ } else {
1876
+ const skipped = await extractTar(entries, target);
1877
+ taken.bytes = entries.reduce((n, e) => n + (e.type === "file" ? e.data.length : 0), 0);
1878
+ taken.dropped = listed([...taken.dropped, ...skipped]);
1879
+ }
1880
+ const masked = taken.masked.length ? ` · masked ${taken.masked.length}` : "";
1881
+ const left = taken.excluded.length ? ` · left out ${taken.excluded.join(", ")}` : "";
1882
+ debugLog(phase, `${name} snapshot: ${taken.files} files, ${formatBytes(taken.bytes)} (${form})${masked}${left}`);
1883
+ add(taken);
1884
+ } catch (error) {
1885
+ await rm(target, {
1886
+ recursive: true,
1887
+ force: true
1888
+ }).catch(() => {});
1889
+ if (a.signal.aborted) return;
1890
+ if (timeout.aborted) return failed(`it took over ${Math.round(timeoutMs / 1e3)}s`);
1891
+ failed(error instanceof Error ? error.message : String(error));
1892
+ }
1893
+ }
1894
+ //#endregion
1641
1895
  //#region src/attempt/turns.ts
1642
1896
  /** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
1643
1897
  const TURN_STOP_GRACE_MS = 3e3;
@@ -1820,6 +2074,7 @@ function turnSender(a, takeContext, checks) {
1820
2074
  a.signal.throwIfAborted();
1821
2075
  const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
1822
2076
  a.record.diff = diff;
2077
+ if (a.spec.snapshot.when === "turns") await takeSnapshot(a, box, `turn-${turnNumber}`, "harness");
1823
2078
  const data = {
1824
2079
  prompt,
1825
2080
  ...result,
@@ -1863,7 +2118,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1863
2118
  } finally {
1864
2119
  clock?.resume();
1865
2120
  }
1866
- });
2121
+ }, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
1867
2122
  const destroy = () => void a.sandbox?.destroy();
1868
2123
  testSignal.addEventListener("abort", destroy, { once: true });
1869
2124
  const takeContext = contextTaker(a);
@@ -1932,6 +2187,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1932
2187
  if (a.session) debug.log("teardown", "harness session closed");
1933
2188
  recordEgress(a, sandbox);
1934
2189
  oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
2190
+ if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
1935
2191
  await sandbox?.destroy();
1936
2192
  if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
1937
2193
  const { record } = a;
@@ -1953,6 +2209,30 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1953
2209
  if (failure) throw failure.error;
1954
2210
  }
1955
2211
  //#endregion
2212
+ //#region src/best-of.ts
2213
+ /** The side an attempt that settled counts on: its record's outcome, else the engine's state. */
2214
+ function sideOf({ state, meta }) {
2215
+ const outcome = meta?.outcome ?? (state === "passed" ? "pass" : state === "failed" ? "fail" : null);
2216
+ return outcome === "pass" || outcome === "fail" ? outcome : null;
2217
+ }
2218
+ /** Whether a best-of cell still needs another attempt after these. */
2219
+ function undecided(sides, bestOf) {
2220
+ const majority = majorityOf(bestOf);
2221
+ const pass = sides.filter((s) => s === "pass").length;
2222
+ const fail = sides.filter((s) => s === "fail").length;
2223
+ return pass < majority && fail < majority;
2224
+ }
2225
+ /** One best-of cell: the `ready` and `settled` hooks of its attempt `n`, from 1 to `bestOf`. */
2226
+ function bestOfCell(bestOf) {
2227
+ const sides = [];
2228
+ const settle = [];
2229
+ for (let n = 1; n <= bestOf; n++) sides.push(new Promise((resolve) => settle.push(resolve)));
2230
+ return (n) => ({
2231
+ ...n > majorityOf(bestOf) ? { ready: async () => undecided(await Promise.all(sides.slice(0, n - 1)), bestOf) } : {},
2232
+ settled: (settled) => settle[n - 1]?.(sideOf(settled))
2233
+ });
2234
+ }
2235
+ //#endregion
1956
2236
  //#region src/contract/declaration.ts
1957
2237
  /** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
1958
2238
  const evalOptionsDataSchema = z.record(z.string(), z.unknown());
@@ -2014,12 +2294,14 @@ const declarationProblem = (children) => dataProblem(declarationTreeSchema, { ch
2014
2294
  /** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
2015
2295
  const GRACE_MS = 5e3;
2016
2296
  function configLevel(config) {
2297
+ const { runs, bestOf } = config.runs !== void 0 ? runsOf(config.runs) : { runs: void 0 };
2017
2298
  return {
2018
2299
  harness: config.harness,
2019
2300
  sandbox: config.sandbox,
2020
2301
  environment: config.environment,
2021
2302
  compare: void 0,
2022
- runs: config.runs,
2303
+ runs,
2304
+ bestOf,
2023
2305
  gate: config.gate,
2024
2306
  env: config.env,
2025
2307
  egress: config.egress !== void 0 ? { inherited: config.egress } : {},
@@ -2028,6 +2310,7 @@ function configLevel(config) {
2028
2310
  question: config.question,
2029
2311
  questions: [],
2030
2312
  timeout: void 0,
2313
+ snapshot: config.snapshot,
2031
2314
  tags: [],
2032
2315
  revision: void 0,
2033
2316
  pin: [],
@@ -2101,7 +2384,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
2101
2384
  ...harness,
2102
2385
  gateway: route.name
2103
2386
  };
2104
- });
2387
+ }).map((harness) => withDefaultEffort(harness, ctx.cursorEfforts));
2105
2388
  const plannedAttempts = resolvedEval.skip ? [] : expand({
2106
2389
  ...resolved,
2107
2390
  harness: routed
@@ -2210,6 +2493,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
2210
2493
  setBy,
2211
2494
  gate: planned.gate,
2212
2495
  ...planned.minLift !== void 0 ? { minLift: planned.minLift } : {},
2496
+ ...planned.bestOf !== void 0 ? { bestOf: planned.bestOf } : {},
2213
2497
  brokered: brokeredIn(resolved.env, planned),
2214
2498
  secretVars: secretVarsIn(resolved.env, planned),
2215
2499
  egress: planned.egress,
@@ -2249,6 +2533,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
2249
2533
  env: resolved.env,
2250
2534
  judge: resolved.judge,
2251
2535
  capture: resolved.capture,
2536
+ snapshot: resolved.snapshot,
2252
2537
  sources: {
2253
2538
  ...resolved.sources,
2254
2539
  sandbox: setBy.sandbox
@@ -2487,7 +2772,20 @@ function register(mode, name, own, fn, source, hint = "", family) {
2487
2772
  declareHarnesses(harnesses);
2488
2773
  if (!tests) return;
2489
2774
  registered++;
2490
- for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
2775
+ const bestOfCells = /* @__PURE__ */ new Map();
2776
+ for (const { name: testName, spec, ...options } of tests) {
2777
+ const { cell, n, bestOf } = options.attempt;
2778
+ let hooks = {};
2779
+ if (bestOf !== void 0) {
2780
+ const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
2781
+ bestOfCells.set(cell.key, of);
2782
+ hooks = of(n);
2783
+ }
2784
+ registerTest(testName, {
2785
+ ...options,
2786
+ ...hooks
2787
+ }, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
2788
+ }
2491
2789
  }
2492
2790
  /** Evals registered so far in this worker, for the scheduler's tie-break. */
2493
2791
  let registered = 0;
@@ -2601,4 +2899,4 @@ function optionsFrom(written, where, base) {
2601
2899
  //#endregion
2602
2900
  export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
2603
2901
 
2604
- //# sourceMappingURL=evaluate-BPP9s3s6.js.map
2902
+ //# sourceMappingURL=evaluate-fhC1PCd_.js.map