reeval 0.15.1 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. package/README.md +12 -12
  2. package/dist/{adapter-CbqbMR7y.js → adapter-CRKYPkRt.js} +3 -3
  3. package/dist/{adapter-CbqbMR7y.js.map → adapter-CRKYPkRt.js.map} +1 -1
  4. package/dist/{agent-Zgn-kiqq.js → agent-BgH4-kFc.js} +6 -6
  5. package/dist/{agent-Zgn-kiqq.js.map → agent-BgH4-kFc.js.map} +1 -1
  6. package/dist/{app-files-ChbvRx3P.js → app-files-BMVBr079.js} +3 -3
  7. package/dist/app-files-BMVBr079.js.map +1 -0
  8. package/dist/{assemble-SKQGW_QF.js → assemble-BVbI6NW-.js} +95 -29
  9. package/dist/assemble-BVbI6NW-.js.map +1 -0
  10. package/dist/assemble-SiHa4__F.js +1 -0
  11. package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
  12. package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
  13. package/dist/{attempt-host-BOp8JklD.js → attempt-host-CIxjs2Be.js} +13 -6
  14. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  15. package/dist/{capture-BM_3100g.js → capture-iQ6kYJV0.js} +987 -161
  16. package/dist/capture-iQ6kYJV0.js.map +1 -0
  17. package/dist/capture-px7Ey95_.js +1 -0
  18. package/dist/{cli-PdYrcoKM.js → cli-1YQU-66F.js} +98 -61
  19. package/dist/cli-1YQU-66F.js.map +1 -0
  20. package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
  21. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  22. package/dist/cli.js +2 -2
  23. package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
  24. package/dist/client-DMmOkbRA.js.map +1 -0
  25. package/dist/{command-4pJYVCT4.js → command-8_Cg5v6b.js} +12 -12
  26. package/dist/{command-4pJYVCT4.js.map → command-8_Cg5v6b.js.map} +1 -1
  27. package/dist/{command-DbSMcxZL.js → command-BlYCAqu-.js} +7 -7
  28. package/dist/{command-DbSMcxZL.js.map → command-BlYCAqu-.js.map} +1 -1
  29. package/dist/{command-9xm5ImqI.js → command-C-HeGlap.js} +16 -16
  30. package/dist/{command-9xm5ImqI.js.map → command-C-HeGlap.js.map} +1 -1
  31. package/dist/{command-BqBxLgTc.js → command-Cr1zjRvl.js} +13 -13
  32. package/dist/{command-BqBxLgTc.js.map → command-Cr1zjRvl.js.map} +1 -1
  33. package/dist/{commands-Bi2SEQtr.js → commands-Bj9NbmsO.js} +1 -1
  34. package/dist/{commands-BU6YJCz9.js → commands-BvTNZ2-b.js} +14 -9
  35. package/dist/commands-BvTNZ2-b.js.map +1 -0
  36. package/dist/{config-k6m7XbAz.js → config-CneBZB84.js} +430 -208
  37. package/dist/config-CneBZB84.js.map +1 -0
  38. package/dist/{contrasts-DLZF4dGF.js → contrasts-lhFsekco.js} +2 -2
  39. package/dist/{contrasts-DLZF4dGF.js.map → contrasts-lhFsekco.js.map} +1 -1
  40. package/dist/db-BnXyamuj.js +2 -0
  41. package/dist/{db-CpsifVln.js → db-Cnh3jIv_.js} +8 -8
  42. package/dist/{db-CpsifVln.js.map → db-Cnh3jIv_.js.map} +1 -1
  43. package/dist/debug-CAx3qcaP.js +1 -0
  44. package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
  45. package/dist/debug-CgG3T071.js.map +1 -0
  46. package/dist/{docker-51WvhN_X.js → docker-BihzJwbV.js} +9 -5
  47. package/dist/docker-BihzJwbV.js.map +1 -0
  48. package/dist/{docker-B4-rx4dA.js → docker-Cqq03H72.js} +5 -5
  49. package/dist/{docker-B4-rx4dA.js.map → docker-Cqq03H72.js.map} +1 -1
  50. package/dist/{docker-YFj4H1Cr.js → docker-DCAtFMo9.js} +1 -1
  51. package/dist/{doctor-BuQnfz4e.js → doctor-Dl4vMeqt.js} +15 -15
  52. package/dist/{doctor-BuQnfz4e.js.map → doctor-Dl4vMeqt.js.map} +1 -1
  53. package/dist/{environment-BOFgzFDg.js → environment-Cf5HMjib.js} +3 -3
  54. package/dist/{environment-BOFgzFDg.js.map → environment-Cf5HMjib.js.map} +1 -1
  55. package/dist/{errors-UjcLWelF.js → errors-CHebohjc.js} +9 -3
  56. package/dist/{errors-UjcLWelF.js.map → errors-CHebohjc.js.map} +1 -1
  57. package/dist/{eval-source-Dje3rERU.js → eval-source-CAE4y2Sr.js} +9 -8
  58. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  59. package/dist/{evaluate-BPP9s3s6.js → evaluate-CUtgyr43.js} +340 -45
  60. package/dist/evaluate-CUtgyr43.js.map +1 -0
  61. package/dist/{evaluate-dqXccn4k.d.ts → evaluate-D35T51AC.d.ts} +7 -6
  62. package/dist/files-BEnUJW1e.js +1 -0
  63. package/dist/{files-Bi461Be2.js → files-BRViE7m-.js} +20 -13
  64. package/dist/{files-Bi461Be2.js.map → files-BRViE7m-.js.map} +1 -1
  65. package/dist/{files-H24pz0hL.js → files-BSqA2sZE.js} +3 -3
  66. package/dist/{files-H24pz0hL.js.map → files-BSqA2sZE.js.map} +1 -1
  67. package/dist/files-C3tQtD26.js +1 -0
  68. package/dist/{files-Clipx4OW.js → files-CNiJ_fZf.js} +3 -3
  69. package/dist/{files-Clipx4OW.js.map → files-CNiJ_fZf.js.map} +1 -1
  70. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  71. package/dist/files-D1y8Pg4Z.js.map +1 -0
  72. package/dist/files-Dts7EY3H.js +1 -0
  73. package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
  74. package/dist/gate-BC6SdQZs.js.map +1 -0
  75. package/dist/gateway.d.ts +1 -1
  76. package/dist/{git-CsVZkNNL.js → git-UNxT_3z0.js} +2 -2
  77. package/dist/{git-CsVZkNNL.js.map → git-UNxT_3z0.js.map} +1 -1
  78. package/dist/harness.d.ts +1 -1
  79. package/dist/{images-4mwhNkvh.js → images-8A_LzVfw.js} +8 -8
  80. package/dist/{images-4mwhNkvh.js.map → images-8A_LzVfw.js.map} +1 -1
  81. package/dist/index.d.ts +63 -16
  82. package/dist/index.js +5 -5
  83. package/dist/init-okoALIah.js +1 -0
  84. package/dist/{init-cOk3D44b.js → init-sFBgODhu.js} +4 -4
  85. package/dist/{init-cOk3D44b.js.map → init-sFBgODhu.js.map} +1 -1
  86. package/dist/{invocations-DW-lTk2f.js → invocations-C9ejlOYl.js} +2 -2
  87. package/dist/{invocations-DW-lTk2f.js.map → invocations-C9ejlOYl.js.map} +1 -1
  88. package/dist/{judge-CKdgEXN7.js → judge-Bg4v4e_K.js} +2 -2
  89. package/dist/{judge-CKdgEXN7.js.map → judge-Bg4v4e_K.js.map} +1 -1
  90. package/dist/judge.d.ts +2 -2
  91. package/dist/judge.js +1 -1
  92. package/dist/{live-XJQ6nqK4.js → live-DeMvnDD5.js} +2 -2
  93. package/dist/{live-XJQ6nqK4.js.map → live-DeMvnDD5.js.map} +1 -1
  94. package/dist/{matchers-DT2TaGix.js → matchers-BKPuaXlj.js} +14 -13
  95. package/dist/matchers-BKPuaXlj.js.map +1 -0
  96. package/dist/{model-Bq3G0OG1.js → model-2rBZrdZA.js} +4 -4
  97. package/dist/{model-Bq3G0OG1.js.map → model-2rBZrdZA.js.map} +1 -1
  98. package/dist/{model-vRiUycWh.js → model-3MJtpho8.js} +4 -4
  99. package/dist/{model-vRiUycWh.js.map → model-3MJtpho8.js.map} +1 -1
  100. package/dist/{model-CTnyvh7H.js → model-Bz7-j71_.js} +3 -50
  101. package/dist/model-Bz7-j71_.js.map +1 -0
  102. package/dist/{model-g3XnzUv0.js → model-DkPwypN4.js} +1 -1
  103. package/dist/model-bREXVXwT.js +1 -0
  104. package/dist/{node-BVbbnFtv.js → node-fP-H2uxn.js} +2 -2
  105. package/dist/{node-BVbbnFtv.js.map → node-fP-H2uxn.js.map} +1 -1
  106. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  107. package/dist/object-BkXGPKWY.js.map +1 -0
  108. package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-Bo8PZ4Ew.js} +2 -2
  109. package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-Bo8PZ4Ew.js.map} +1 -1
  110. package/dist/{patch-DKJiC5Sq.js → patch-6T9XAipm.js} +4 -4
  111. package/dist/patch-6T9XAipm.js.map +1 -0
  112. package/dist/{plan-DdWkM6SW.js → plan-Dw2BjafJ.js} +20 -11
  113. package/dist/plan-Dw2BjafJ.js.map +1 -0
  114. package/dist/{plugins-D-CP6mMd.js → plugins-ngQEJBjJ.js} +24 -3
  115. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  116. package/dist/{preflight-DTiVv0gY.js → preflight-BYXkweNN.js} +1 -1
  117. package/dist/{preflight-9ixuXzsx.js → preflight-CDRpog9o.js} +3 -3
  118. package/dist/{preflight-9ixuXzsx.js.map → preflight-CDRpog9o.js.map} +1 -1
  119. package/dist/{prepare-tmBBf4W8.js → prepare-B3btUlT6.js} +2 -2
  120. package/dist/{prepare-BfyikQit.js → prepare-PkARUGoF.js} +7 -7
  121. package/dist/{prepare-BfyikQit.js.map → prepare-PkARUGoF.js.map} +1 -1
  122. package/dist/{provenance-BizwUn3h.js → provenance-7Hzg88f4.js} +2 -2
  123. package/dist/{provenance-BizwUn3h.js.map → provenance-7Hzg88f4.js.map} +1 -1
  124. package/dist/{providers-DkOupEBl.js → providers-DDNMbv1T.js} +1 -1
  125. package/dist/{providers-B6b9Tnyi.js → providers-PKoJtGK0.js} +5 -5
  126. package/dist/{providers-B6b9Tnyi.js.map → providers-PKoJtGK0.js.map} +1 -1
  127. package/dist/query-Bt6vLJ3m.js +1 -0
  128. package/dist/{query-DyM-VExS.js → query-nBXrgEwl.js} +2 -2
  129. package/dist/{query-DyM-VExS.js.map → query-nBXrgEwl.js.map} +1 -1
  130. package/dist/{questions-BRhU8MzB.js → questions-CkAwePPR.js} +16 -17
  131. package/dist/questions-CkAwePPR.js.map +1 -0
  132. package/dist/remarks-Bp6AwPgb.js +1 -0
  133. package/dist/{remarks-C7i2LMRb.js → remarks-Bt0zwiQB.js} +42 -8
  134. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  135. package/dist/{remarks-D_3qhBWU.js → remarks-IFFe_oty.js} +4 -3
  136. package/dist/{remarks-D_3qhBWU.js.map → remarks-IFFe_oty.js.map} +1 -1
  137. package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
  138. package/dist/remarks-eBcrIZRu.js.map +1 -0
  139. package/dist/{remote-DFIzcx_6.js → remote-DfJYiP-R.js} +5 -5
  140. package/dist/{remote-DFIzcx_6.js.map → remote-DfJYiP-R.js.map} +1 -1
  141. package/dist/resolve-CT4XZIfV.js +1 -0
  142. package/dist/{resolve-DWSJpKHX.js → resolve-D_cLa4tW.js} +5 -5
  143. package/dist/{resolve-DWSJpKHX.js.map → resolve-D_cLa4tW.js.map} +1 -1
  144. package/dist/{results-BC3pcG_Z.js → results-DrKJuIQF.js} +9 -9
  145. package/dist/{results-BC3pcG_Z.js.map → results-DrKJuIQF.js.map} +1 -1
  146. package/dist/{resume-j-rcIryi.js → resume-BmIM52MJ.js} +4 -4
  147. package/dist/{resume-j-rcIryi.js.map → resume-BmIM52MJ.js.map} +1 -1
  148. package/dist/{runtime-d56Gg3J5.js → runtime-CZVpkr04.js} +111 -37
  149. package/dist/runtime-CZVpkr04.js.map +1 -0
  150. package/dist/{sandbox-Cmln11gU.js → sandbox-Cmi6IQML.js} +2 -2
  151. package/dist/{sandbox-Cmln11gU.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  152. package/dist/sandbox.d.ts +1 -1
  153. package/dist/sandbox.js +1 -1
  154. package/dist/server-CKJOnIB0.js +1 -0
  155. package/dist/{server-BOoH10J8.js → server-DS8BGhGU.js} +34 -10
  156. package/dist/server-DS8BGhGU.js.map +1 -0
  157. package/dist/source-CgUqpumI.js +1 -0
  158. package/dist/{source-wD04-rRG.js → source-DcXv4de-.js} +17 -3
  159. package/dist/{source-wD04-rRG.js.map → source-DcXv4de-.js.map} +1 -1
  160. package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
  161. package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
  162. package/dist/{src-kYeaJJrl.js → src-BTqJbvj6.js} +1 -1
  163. package/dist/{src-Bp4aN3rA.js → src-BkiComGh.js} +1 -1
  164. package/dist/{src-dlX-Lmxg.js → src-BvgeZ4tS.js} +1 -1
  165. package/dist/{src-D91Hd6hT.js → src-C0ySiej5.js} +36 -17
  166. package/dist/src-C0ySiej5.js.map +1 -0
  167. package/dist/{src-HSxBazcg.js → src-CgJrz5p0.js} +1 -1
  168. package/dist/{src-BYPMM1x7.js → src-Cp-2Hdib.js} +1 -1
  169. package/dist/{src-6_zz7oJs.js → src-DMo2eCOJ.js} +1 -1
  170. package/dist/src-Dk2Na0b-.js +2 -0
  171. package/dist/{src-C7upnZSd.js → src-OESRtvOi.js} +1 -1
  172. package/dist/src-h5uDlk3m.js +1 -0
  173. package/dist/state-C97udoh3.js +1 -0
  174. package/dist/{state-DrJ687ut.js → state-C_kdhp5v.js} +5 -5
  175. package/dist/{state-DrJ687ut.js.map → state-C_kdhp5v.js.map} +1 -1
  176. package/dist/suite-ByT1lsdD.js +2 -0
  177. package/dist/{suite-Co8cwelb.js → suite-DieLXvCT.js} +15 -4
  178. package/dist/suite-DieLXvCT.js.map +1 -0
  179. package/dist/{summary-DLFHVoux.js → summary-CSzyBVsF.js} +40 -14
  180. package/dist/summary-CSzyBVsF.js.map +1 -0
  181. package/dist/tarball-CHcahyNK.js +258 -0
  182. package/dist/tarball-CHcahyNK.js.map +1 -0
  183. package/dist/testing.d.ts +52 -44
  184. package/dist/testing.js +4 -4
  185. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  186. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  187. package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
  188. package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
  189. package/dist/{types-coMWHaU2.d.ts → types-DVvW66ci.d.ts} +325 -171
  190. package/dist/view/index.d.ts +16 -4
  191. package/dist/view/index.js +6 -6
  192. package/dist/{views-BFO1BLKu.js → views-B9STsLB4.js} +5 -5
  193. package/dist/{views-BFO1BLKu.js.map → views-B9STsLB4.js.map} +1 -1
  194. package/dist/views-Ca-rTixp.js +2 -0
  195. package/dist/{worker-entry-CJtdsKBP.js → worker-entry-C5bLCv6s.js} +34 -13
  196. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  197. package/dist/workspace-8uHtBSve.js +1 -0
  198. package/dist/{workspace-BRxjfa8N.js → workspace-CBuI9Uds.js} +4 -4
  199. package/dist/{workspace-BRxjfa8N.js.map → workspace-CBuI9Uds.js.map} +1 -1
  200. package/package.json +1 -1
  201. package/skills/eval-rubric/SKILL.md +11 -0
  202. package/dist/app-files-ChbvRx3P.js.map +0 -1
  203. package/dist/assemble-SKQGW_QF.js.map +0 -1
  204. package/dist/assemble-XBWuK3Ed.js +0 -1
  205. package/dist/attempt-host-BOp8JklD.js.map +0 -1
  206. package/dist/capture-BM_3100g.js.map +0 -1
  207. package/dist/capture-DSK8O5bA.js +0 -1
  208. package/dist/cli-PdYrcoKM.js.map +0 -1
  209. package/dist/cli-spec-D0qyrPZi.js.map +0 -1
  210. package/dist/client-Ct6s6OnF.js.map +0 -1
  211. package/dist/commands-BU6YJCz9.js.map +0 -1
  212. package/dist/config-k6m7XbAz.js.map +0 -1
  213. package/dist/db-DkD-F-X-.js +0 -2
  214. package/dist/debug-BhizgAsU.js.map +0 -1
  215. package/dist/debug-DeM5pswK.js +0 -1
  216. package/dist/docker-51WvhN_X.js.map +0 -1
  217. package/dist/eval-source-Dje3rERU.js.map +0 -1
  218. package/dist/evaluate-BPP9s3s6.js.map +0 -1
  219. package/dist/files-BAt0i00X.js +0 -1
  220. package/dist/files-C2h2Cw4F.js +0 -1
  221. package/dist/files-Ca0d6Uq3.js.map +0 -1
  222. package/dist/files-DmUrRmmL.js +0 -1
  223. package/dist/init-BDEOopK7.js +0 -1
  224. package/dist/matchers-DT2TaGix.js.map +0 -1
  225. package/dist/model-CTnyvh7H.js.map +0 -1
  226. package/dist/model-D-SWrLZG.js +0 -1
  227. package/dist/object-CLf2-tNq.js.map +0 -1
  228. package/dist/pairs-puYi-zj2.js.map +0 -1
  229. package/dist/patch-DKJiC5Sq.js.map +0 -1
  230. package/dist/plan-DdWkM6SW.js.map +0 -1
  231. package/dist/plugins-D-CP6mMd.js.map +0 -1
  232. package/dist/query-Clm2mNSm.js +0 -1
  233. package/dist/questions-BRhU8MzB.js.map +0 -1
  234. package/dist/remarks-BwJ8pcTQ.js +0 -1
  235. package/dist/remarks-C7i2LMRb.js.map +0 -1
  236. package/dist/remarks-dA8LyoPc.js.map +0 -1
  237. package/dist/resolve--rQZ8u6Z.js +0 -1
  238. package/dist/runtime-d56Gg3J5.js.map +0 -1
  239. package/dist/server-BOoH10J8.js.map +0 -1
  240. package/dist/server-DpuVAbrO.js +0 -1
  241. package/dist/source-BFd6f0-R.js +0 -1
  242. package/dist/src-D91Hd6hT.js.map +0 -1
  243. package/dist/src-DH-9IO7f.js +0 -1
  244. package/dist/src-LD1BAJFa.js +0 -2
  245. package/dist/state-B5PLm5qY.js +0 -1
  246. package/dist/suite-22Kt9Grx.js +0 -2
  247. package/dist/suite-Co8cwelb.js.map +0 -1
  248. package/dist/summary-DLFHVoux.js.map +0 -1
  249. package/dist/views-BOf8qEL8.js +0 -2
  250. package/dist/worker-entry-CJtdsKBP.js.map +0 -1
  251. package/dist/workspace-DDHzkMjm.js +0 -1
@@ -1,36 +1,39 @@
1
- import { t as isObjectLike } from "./object-CLf2-tNq.js";
2
- import { r as withLease } from "./client-Ct6s6OnF.js";
1
+ import { t as isObjectLike } from "./object-BkXGPKWY.js";
2
+ import { r as withLease } from "./client-DMmOkbRA.js";
3
3
  import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
4
4
  import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
5
- import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-BOFgzFDg.js";
6
- import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-D-CP6mMd.js";
5
+ import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-Cf5HMjib.js";
6
+ import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-ngQEJBjJ.js";
7
7
  import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
8
- import { $ as isDirect, A as harnessId, At as badJudgeArtifacts, B as sandboxSetBy, D as checkOptions, Dt as judgeProblem, G as resolveStateOptions, H as unbrokered, J as allowedHosts, L as refuseIllegalCompare, M as matchesTags, Mt as filePartsOf, N as plainHeaderError, Nt as resolveJudgeArtifacts, O as evalPlan, P as plainHeaders, R as sandboxId, S as resolveGateway, St as writePluginFiles, T as cascade, U as unbrokeredError, V as secretVarsIn, W as mergeJudge, Z as egressLabel, _ as fellBackToDirect, _t as expectedBlockedFor, b as resolveModelGateway, bt as refusedMcp, et as unenforceable, ft as agentMcp, ht as claudeMcpServers, jt as decideState, k as expand, kt as artifactRefsOf, mt as brokeringFor, nt as createFakeSandbox, v as gatewayScope, w as brokeredIn, xt as refusedPlugins, yt as layoutOf, z as sandboxProvider } from "./config-k6m7XbAz.js";
9
- import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-CKdgEXN7.js";
8
+ import { $ as egressLabel, A as harnessId, At as NO_JUDGE_INPUT, B as sandboxId, Bt as decideState, D as checkOptions, Dt as writePluginFiles, Et as refusedPlugins, F as plainHeaders, Ft as judgeProblem, G as unbrokeredError, H as sandboxSetBy, Ht as resolveJudgeArtifacts, K as mergeJudge, M as majorityOf, Mt as inputsOn, N as matchesTags, O as evalPlan, P as plainHeaderError, R as refuseIllegalCompare, Rt as artifactRefsOf, S as resolveGateway, St as expectedBlockedFor, T as cascade, Tt as refusedMcp, U as secretVarsIn, V as sandboxProvider, Vt as filePartsOf, W as unbrokered, X as allowedHosts, _ as fellBackToDirect, _t as agentMcp, b as resolveModelGateway, bt as claudeMcpServers, it as createFakeSandbox, k as expand, kt as INPUT_KEYS, nt as unenforceable, q as resolveStateOptions, tt as isDirect, v as gatewayScope, w as brokeredIn, wt as layoutOf, yt as brokeringFor, z as runsOf, zt as badJudgeArtifacts } from "./config-CneBZB84.js";
9
+ import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-Bg4v4e_K.js";
10
10
  import { v as vertexSettings } from "./vertex-DypYjx71.js";
11
- import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
11
+ import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-DVIaahLv.js";
12
12
  import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
13
13
  import { t as inCI } from "./env-CARj5Axq.js";
14
- import { ft as shellQuote, t as MCP_TIMER } from "./src-D91Hd6hT.js";
14
+ import { pt as shellQuote, t as MCP_TIMER, ut as OPAQUE_OUTPUT } from "./src-C0ySiej5.js";
15
15
  import { t as InfraError } from "./types-C5vI3g1C.js";
16
- import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-d56Gg3J5.js";
17
- import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-Ba2QerOK.js";
18
- import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
19
- import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
20
- import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-BM_3100g.js";
16
+ import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CZVpkr04.js";
17
+ import { c as formatBytes } from "./debug-CgG3T071.js";
18
+ import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CpbZ9arV.js";
19
+ import { t as attemptDebugDir } from "./files-D1y8Pg4Z.js";
20
+ import { r as openRouterFallbacks } from "./openrouter-models-Bo8PZ4Ew.js";
21
+ import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-iQ6kYJV0.js";
21
22
  import { t as contextTaker } from "./context-Dgn48BqT.js";
22
- import { a as skillFiles, i as resolveEnvironment } from "./resolve-DWSJpKHX.js";
23
+ import { a as skillFiles, i as resolveEnvironment } from "./resolve-D_cLa4tW.js";
23
24
  import { t as PrepareError } from "./provider-CmAYTQQs.js";
24
- import { i as preparedWorkspace } from "./workspace-BRxjfa8N.js";
25
+ import { i as preparedWorkspace } from "./workspace-CBuI9Uds.js";
25
26
  import { t as patchPath } from "./patch-CquGAGX5.js";
26
- import { n as judgeDiff } from "./patch-DKJiC5Sq.js";
27
- import { t as transcriptForJudge } from "./transcript-DxPaWHTZ.js";
28
- import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-Dje3rERU.js";
27
+ import { n as judgeDiff } from "./patch-6T9XAipm.js";
28
+ import { t as transcriptForJudge } from "./transcript-BJrKfgRW.js";
29
+ import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CHcahyNK.js";
30
+ import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-CAE4y2Sr.js";
29
31
  import { dirname, join } from "node:path";
30
32
  import { parse, stringify } from "yaml";
31
33
  import { z } from "zod";
32
34
  import { setTimeout as setTimeout$1 } from "node:timers/promises";
33
- import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
35
+ import { gunzipSync, gzipSync } from "node:zlib";
36
+ import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
34
37
  //#region src/attempt/assertions.ts
35
38
  const MESSAGE_LIMIT = 1024;
36
39
  const LABEL_LIMIT = 60;
@@ -76,6 +79,7 @@ function assertionRecords(raw) {
76
79
  matcher: r.matcher,
77
80
  status: r.status,
78
81
  soft: r.soft,
82
+ ...r.at ? { at: r.at } : {},
79
83
  ...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
80
84
  ...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
81
85
  };
@@ -84,7 +88,7 @@ function assertionRecords(raw) {
84
88
  const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
85
89
  /**
86
90
  * What fails the attempt at its end for the checks that failed without ending the body: the first
87
- * that didn't pass, an `expect.soft` as what it threw, a hard check the eval caught with
91
+ * that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
88
92
  * `CAUGHT_HINT` after its message.
89
93
  */
90
94
  function checkFailure(raw) {
@@ -436,7 +440,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
436
440
  //#region src/harness/index.ts
437
441
  async function createHarness(descriptor, vertex) {
438
442
  if (descriptor.kind === "fake") return createFakeHarness(descriptor);
439
- const { createAgentHarness } = await import("./agent-Zgn-kiqq.js");
443
+ const { createAgentHarness } = await import("./agent-BgH4-kFc.js");
440
444
  return createAgentHarness(descriptor, vertex);
441
445
  }
442
446
  /** Where each harness's agent looks for repo skills, relative to the workspace. */
@@ -466,8 +470,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
466
470
  if (debug) traceEnvironment(debug, spec, env);
467
471
  record.image = null;
468
472
  if (spec.sandbox.kind === "docker") {
469
- const { preparedFor } = await import("./preflight-DTiVv0gY.js");
470
- const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DkOupEBl.js");
473
+ const { preparedFor } = await import("./preflight-BYXkweNN.js");
474
+ const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DDNMbv1T.js");
471
475
  signal.throwIfAborted();
472
476
  const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
473
477
  const builds = builtPlugins(spec.environment.layers);
@@ -536,16 +540,16 @@ function describeSize(state, question) {
536
540
  return `${p.name} ${k(tokensOf(p.chars))}${top.length ? `, of which ${top.join(", ")}` : ""}`;
537
541
  }).join("; ");
538
542
  }
539
- /** `Narrow state.diff, state.transcript or state.output.`: the state's parts, largest first. */
543
+ /** `Narrow diff, transcript or output.`: the inputs that were on, largest first. */
540
544
  function stateHint(state, question) {
541
- const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => `state.${p.name}`);
545
+ const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => p.name);
542
546
  if (!keys.length) return "";
543
547
  const last = keys.pop();
544
548
  return ` Narrow ${keys.length ? `${keys.join(", ")} or ${last}` : last}.`;
545
549
  }
546
550
  /**
547
551
  * A provider's error, as the provider worded it, then what the judge's input held: its estimated
548
- * size and biggest parts and files, and the `state` keys that were on, largest first. reeval knows
552
+ * size and biggest parts and files, and the inputs that were on, largest first. reeval knows
549
553
  * no model's limit, so this follows every provider error.
550
554
  */
551
555
  const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
@@ -585,7 +589,7 @@ const workspacePath = (path) => path.replace(/^\.\//, "");
585
589
  /** A `contents` entry that names one file rather than matching many. */
586
590
  const isExactPath = (entry) => !entry.startsWith("!") && !/[*?[\]{}]/.test(entry);
587
591
  /**
588
- * The judge's state for what the `state` option chose: the graded turn's task and output (or every turn's),
592
+ * The judge's state for the inputs the call turned on: the graded turn's task and output (or every turn's),
589
593
  * its changed files and diff as `judgeDiff` shows them, files whole, `c.exec` results and the
590
594
  * transcript.
591
595
  */
@@ -633,7 +637,7 @@ function checksShown(which, ran) {
633
637
  if (which === false) return void 0;
634
638
  if (which !== true) {
635
639
  const missing = which.find((command) => !ran.some((r) => r.command === command));
636
- if (missing !== void 0) throw new Error(`c.judge() state.checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
640
+ if (missing !== void 0) throw new Error(`c.judge() checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
637
641
  }
638
642
  return ran.filter((r) => which === true || which.includes(r.command)).map(({ command, exitCode, stdout, stderr }) => ({
639
643
  command,
@@ -657,24 +661,24 @@ function placeFor(calls, order) {
657
661
  * checks after it still run.
658
662
  */
659
663
  function judgeCaller(a, readFile) {
660
- const ask = async (order, question, { on, ...call } = {}) => {
661
- if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
664
+ const ask = async (order, question, { on, ...call }) => {
662
665
  const problem = judgeProblem(call, { at: "call" });
663
666
  if (problem) throw new Error(problem);
664
667
  const badArts = badJudgeArtifacts(call.artifacts);
665
668
  if (badArts) throw new Error(`c.judge() artifacts ${badArts}`);
666
669
  const subject = on && "turn" in on ? a.turns[on.turn] : a.turns.at(-1);
667
670
  if (!subject) throw new Error(on && "turn" in on ? `c.judge() on names turn ${on.turn}, but this attempt has ${a.turns.length}` : "c.judge() has no turn to grade: call c.send() first");
668
- const options = mergeJudge(a.spec.judge, call);
671
+ const wanted = resolveStateOptions(call);
672
+ if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives a value to grade as the output, but output is off. Pass output: true.");
673
+ const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, artifacts, ...callJudge } = call;
674
+ const options = mergeJudge(a.spec.judge, callJudge);
669
675
  const { model } = options;
670
676
  if (model === void 0) {
671
677
  const message = noJudgeModel(getRunContext().config.file);
672
678
  a.judgeThrown.add(message);
673
679
  throw new Error(message);
674
680
  }
675
- const wanted = resolveStateOptions(options.state);
676
- if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
677
- const { state: _state, maxConcurrent, ...judgeOptions } = options;
681
+ const { maxConcurrent, ...judgeOptions } = options;
678
682
  const output = on && "value" in on ? on.value : subject.text;
679
683
  const { draft, dropped } = await draftState({
680
684
  turns: a.turns,
@@ -687,7 +691,8 @@ function judgeCaller(a, readFile) {
687
691
  });
688
692
  const name = judgeModelName(model);
689
693
  const state = redactValue(stateOf(draft));
690
- const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
694
+ const resolved = resolveJudgeArtifacts(artifacts, a.artifactBlobs);
695
+ if (artifacts === true && !resolved.length && inputsOn(call, INPUT_KEYS).length === 1) throw new Error("c.judge() artifacts: true shows the judge nothing yet: this attempt has no captures. Call c.capture() first, or turn on another input.");
691
696
  const refs = artifactRefsOf(resolved);
692
697
  const decide = decideState(state, filePartsOf(resolved));
693
698
  const input = {
@@ -789,7 +794,10 @@ function judgeCaller(a, readFile) {
789
794
  return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
790
795
  }
791
796
  };
792
- return async (question, { on, ...options } = {}) => {
797
+ return async (question, given) => {
798
+ if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
799
+ if (given === null || typeof given !== "object" || Array.isArray(given)) throw new Error(judgeProblem(given, { at: "call" }) ?? NO_JUDGE_INPUT);
800
+ const { on, ...options } = given;
793
801
  const order = a.judgeCalls++;
794
802
  const call = on === void 0 ? options : {
795
803
  ...options,
@@ -805,7 +813,7 @@ async function createSandbox(descriptor, options) {
805
813
  switch (descriptor.kind) {
806
814
  case "fake": return await createFakeSandbox(descriptor, options.env);
807
815
  case "docker": {
808
- const { createDockerSandbox } = await import("./docker-YFj4H1Cr.js");
816
+ const { createDockerSandbox } = await import("./docker-DCAtFMo9.js");
809
817
  return await createDockerSandbox(descriptor, options);
810
818
  }
811
819
  }
@@ -849,8 +857,8 @@ function isJudges(a, error, raw) {
849
857
  }
850
858
  /**
851
859
  * The error that fails the attempt: a judge's gives way to the first other check that didn't hold
852
- * (an `expect.soft`, or a hard check the eval caught), whichever came first, so the attempt fails
853
- * as that check.
860
+ * (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
861
+ * fails as that check.
854
862
  */
855
863
  function attemptError(a, error, raw) {
856
864
  if (!isJudges(a, error, raw)) return error;
@@ -968,12 +976,23 @@ async function writeRecord(a, stagingDir, id) {
968
976
  question: clip(j.question, 200)
969
977
  })),
970
978
  turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
979
+ ...failedChecks(record),
971
980
  ...record.error?.prepare ? { prepare: {
972
981
  ref: record.error.prepare.ref,
973
982
  stage: record.error.prepare.stage
974
983
  } } : {}
975
984
  };
976
985
  }
986
+ /** The record's checks that didn't pass, for the terminal; unset when all did. */
987
+ function failedChecks(record) {
988
+ const failed = (record.assertions ?? []).filter((x) => x.status !== "pass");
989
+ if (!failed.length) return {};
990
+ return { failedChecks: failed.map((x) => ({
991
+ label: x.label,
992
+ ...x.at ? { at: x.at } : {},
993
+ message: clip(x.message?.split("\n")[0] ?? "", 500)
994
+ })) };
995
+ }
977
996
  /** `turn N error: <message>` for each turn with an error event. */
978
997
  function turnErrors(transcripts) {
979
998
  return transcripts.flatMap((events, i) => {
@@ -1190,7 +1209,7 @@ const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
1190
1209
  //#region src/attempt/state.ts
1191
1210
  /** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
1192
1211
  const SETUP_BUDGET_MS = 12e5;
1193
- function createAttempt(spec, signal, progress, start) {
1212
+ function createAttempt(spec, signal, progress, start, snapshotDir) {
1194
1213
  const record = {
1195
1214
  startedAt: (/* @__PURE__ */ new Date()).toISOString(),
1196
1215
  outcome: "pass",
@@ -1245,7 +1264,8 @@ function createAttempt(spec, signal, progress, start) {
1245
1264
  judgeInputCalls: [],
1246
1265
  captureNames: /* @__PURE__ */ new Set(),
1247
1266
  backgrounds: [],
1248
- artifactBlobs: /* @__PURE__ */ new Map()
1267
+ artifactBlobs: /* @__PURE__ */ new Map(),
1268
+ snapshotDir
1249
1269
  };
1250
1270
  return a;
1251
1271
  }
@@ -1638,6 +1658,237 @@ async function startSession(a, runId) {
1638
1658
  return [box, session];
1639
1659
  }
1640
1660
  //#endregion
1661
+ //#region src/attempt/snapshot.ts
1662
+ /** The most a snapshot directory's files may add up to; a larger one isn't taken. */
1663
+ const SNAPSHOT_DIR_LIMIT = 50 * 2 ** 20;
1664
+ /** The most a compressed snapshot may be. */
1665
+ const SNAPSHOT_TAR_LIMIT = 10 * 2 ** 20;
1666
+ /** The most a compressed snapshot may be uncompressed, which the host holds in memory to mask. */
1667
+ const SNAPSHOT_RAW_LIMIT = 200 * 2 ** 20;
1668
+ /** How long one snapshot may take before the attempt goes on without it. */
1669
+ const SNAPSHOT_TIMEOUT_MS = 12e4;
1670
+ /** Installed dependencies, caches and git's objects: left out by directory name, at any depth. */
1671
+ const EXCLUDED_DIRS = [
1672
+ ".git",
1673
+ "node_modules",
1674
+ ".pnpm-store",
1675
+ ".venv",
1676
+ "venv",
1677
+ ".tox",
1678
+ "__pycache__",
1679
+ ".pytest_cache",
1680
+ ".mypy_cache",
1681
+ ".ruff_cache",
1682
+ ".next",
1683
+ ".nuxt",
1684
+ ".svelte-kit",
1685
+ ".turbo",
1686
+ ".parcel-cache",
1687
+ ".cache",
1688
+ ".dart_tool",
1689
+ ".gradle"
1690
+ ];
1691
+ const LISTED = 50;
1692
+ const SIZE_MARK = "--reeval-size--";
1693
+ const excluded = (path) => path.split("/").some((part) => EXCLUDED_DIRS.includes(part));
1694
+ const utf8 = new TextDecoder("utf-8", { fatal: true });
1695
+ function asText(data) {
1696
+ if (data.subarray(0, 8192).includes(0)) return null;
1697
+ try {
1698
+ return utf8.decode(data);
1699
+ } catch {
1700
+ return null;
1701
+ }
1702
+ }
1703
+ /** The sandbox's archive, with the excluded directories gone and secret-looking values masked. */
1704
+ function cleanArchive(tar) {
1705
+ const entries = [];
1706
+ const masked = [];
1707
+ const dropped = [];
1708
+ const left = /* @__PURE__ */ new Set();
1709
+ for (const e of readTar(tar)) {
1710
+ if (excluded(e.path)) {
1711
+ const parts = e.path.split("/");
1712
+ const at = parts.findIndex((p) => EXCLUDED_DIRS.includes(p));
1713
+ left.add(redactSecrets(`${parts.slice(0, at + 1).join("/")}/`));
1714
+ continue;
1715
+ }
1716
+ if (findSecrets(e.path).length || e.type === "symlink" && findSecrets(e.target).length) {
1717
+ dropped.push(redactSecrets(e.path));
1718
+ continue;
1719
+ }
1720
+ if (e.type !== "file") {
1721
+ entries.push(e);
1722
+ continue;
1723
+ }
1724
+ const text = asText(e.data);
1725
+ if (text === null) {
1726
+ if (findSecrets(e.data.toString("latin1")).length) dropped.push(e.path);
1727
+ else entries.push(e);
1728
+ continue;
1729
+ }
1730
+ const clean = redactSecrets(text);
1731
+ if (clean === text) entries.push(e);
1732
+ else {
1733
+ masked.push(e.path);
1734
+ entries.push({
1735
+ ...e,
1736
+ data: Buffer.from(clean)
1737
+ });
1738
+ }
1739
+ }
1740
+ return {
1741
+ entries,
1742
+ masked,
1743
+ dropped,
1744
+ excluded: [...left]
1745
+ };
1746
+ }
1747
+ const tarCommand = (workdir) => {
1748
+ const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
1749
+ return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
1750
+ };
1751
+ const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
1752
+ const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
1753
+ const listed = (paths) => paths.slice(0, LISTED);
1754
+ const LIMITS = {
1755
+ dir: SNAPSHOT_DIR_LIMIT,
1756
+ tar: SNAPSHOT_TAR_LIMIT,
1757
+ raw: SNAPSHOT_RAW_LIMIT
1758
+ };
1759
+ /**
1760
+ * Archives the sandbox's workspace and returns what to record and the masked entries to write,
1761
+ * or why none: over its form's cap, or an archive the sandbox couldn't make.
1762
+ */
1763
+ async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
1764
+ const none = (cause, message, dirs = []) => ({
1765
+ taken: {
1766
+ bytes: 0,
1767
+ files: 0,
1768
+ excluded: listed(dirs),
1769
+ masked: [],
1770
+ dropped: [],
1771
+ skipped: {
1772
+ cause,
1773
+ message
1774
+ }
1775
+ },
1776
+ entries: null
1777
+ });
1778
+ const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
1779
+ const tar = tarCommand(box.workdir);
1780
+ const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
1781
+ const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
1782
+ const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
1783
+ const rawBytes = Number(raw.trim());
1784
+ const gzBytes = Number(size.trim());
1785
+ if (sized.exitCode !== 0 || !(rawBytes > 0) || !(gzBytes > 0)) return none("error", `couldn't archive the workspace: ${(sized.stderr || sized.stdout).trim()}`);
1786
+ const without = dirs.length ? dirs.join(", ") : "its dependency and cache directories";
1787
+ const rawLimit = compress ? limits.raw : limits.dir;
1788
+ if (rawBytes > rawLimit) return none("size", `the workspace is ${formatBytes(rawBytes)} without ${without}, over the ${formatBytes(rawLimit)} cap${compress ? "" : ` (compressed, the cap is ${formatBytes(limits.tar)})`}`, dirs);
1789
+ if (compress && gzBytes > limits.tar) return none("size", `the workspace is ${formatBytes(gzBytes)} compressed without ${without}, over the ${formatBytes(limits.tar)} cap`, dirs);
1790
+ const r = await box.execOwn(`${NEEDS_TAR}; echo ${OPAQUE_OUTPUT}; ${tar} | { ${GZIP}; } | base64`, { signal });
1791
+ const encoded = r.stdout.startsWith(`--opaque-output--\n`) ? r.stdout.slice(OPAQUE_OUTPUT.length + 1) : "";
1792
+ const fetched = Buffer.from(encoded, "base64");
1793
+ if (r.exitCode !== 0 || fetched.length === 0) return none("error", `couldn't archive the workspace: ${r.stderr.trim() || "no output"}`);
1794
+ const clean = cleanArchive(fetched[0] === 31 && fetched[1] === 139 ? gunzipSync(fetched, { maxOutputLength: rawLimit + 2 ** 20 }) : fetched);
1795
+ const all = [.../* @__PURE__ */ new Set([...dirs, ...clean.excluded])].sort();
1796
+ return {
1797
+ taken: {
1798
+ bytes: 0,
1799
+ files: clean.entries.filter((e) => e.type !== "dir").length,
1800
+ excluded: listed(all),
1801
+ masked: listed(clean.masked),
1802
+ dropped: listed(clean.dropped),
1803
+ skipped: null
1804
+ },
1805
+ entries: clean.entries
1806
+ };
1807
+ }
1808
+ /** Where an attempt's snapshots wait, beside its staging record, until the run is assembled. */
1809
+ const snapshotSidecar = (stagingDir, key) => join(stagingDir, "evals", recordFileName(key).replace(/\.json$/, ".snapshots"));
1810
+ /**
1811
+ * Takes snapshot `name` of the workspace into the attempt's sidecar and adds it to the record;
1812
+ * never fails the attempt. Adds nothing when the attempt's signal stopped it.
1813
+ */
1814
+ async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SNAPSHOT_TIMEOUT_MS) {
1815
+ const { compress } = a.spec.snapshot;
1816
+ const form = compress ? "tar.gz" : "dir";
1817
+ const dir = a.snapshotDir;
1818
+ const target = join(dir, compress ? `${name}.tar.gz` : name);
1819
+ const timeout = AbortSignal.timeout(timeoutMs);
1820
+ const add = (taken) => {
1821
+ a.record.snapshots ??= [];
1822
+ a.record.snapshots.push({
1823
+ name,
1824
+ form,
1825
+ path: null,
1826
+ ...taken
1827
+ });
1828
+ };
1829
+ const failed = (message) => {
1830
+ debugLog(phase, `no ${name} snapshot: ${message}`);
1831
+ add({
1832
+ bytes: 0,
1833
+ files: 0,
1834
+ excluded: [],
1835
+ masked: [],
1836
+ dropped: [],
1837
+ skipped: {
1838
+ cause: "error",
1839
+ message: `couldn't archive the workspace: ${message}`
1840
+ }
1841
+ });
1842
+ };
1843
+ try {
1844
+ const { taken, entries } = await archiveWorkspace(box, AbortSignal.any([a.signal, timeout]), compress, limits);
1845
+ if (!entries) {
1846
+ debugLog(phase, `no ${name} snapshot: ${taken.skipped?.message}`);
1847
+ add(taken);
1848
+ return;
1849
+ }
1850
+ await rm(target, {
1851
+ recursive: true,
1852
+ force: true
1853
+ });
1854
+ await mkdir(dir, { recursive: true });
1855
+ if (compress) {
1856
+ const archive = gzipSync(writeTar(entries));
1857
+ if (archive.length > limits.tar) {
1858
+ const message = `the workspace is ${formatBytes(archive.length)} compressed, over the ${formatBytes(limits.tar)} cap`;
1859
+ debugLog(phase, `no ${name} snapshot: ${message}`);
1860
+ add({
1861
+ ...taken,
1862
+ files: 0,
1863
+ skipped: {
1864
+ cause: "size",
1865
+ message
1866
+ }
1867
+ });
1868
+ return;
1869
+ }
1870
+ await writeFile(target, archive);
1871
+ taken.bytes = archive.length;
1872
+ } else {
1873
+ const skipped = await extractTar(entries, target);
1874
+ taken.bytes = entries.reduce((n, e) => n + (e.type === "file" ? e.data.length : 0), 0);
1875
+ taken.dropped = listed([...taken.dropped, ...skipped]);
1876
+ }
1877
+ const masked = taken.masked.length ? ` · masked ${taken.masked.length}` : "";
1878
+ const left = taken.excluded.length ? ` · left out ${taken.excluded.join(", ")}` : "";
1879
+ debugLog(phase, `${name} snapshot: ${taken.files} files, ${formatBytes(taken.bytes)} (${form})${masked}${left}`);
1880
+ add(taken);
1881
+ } catch (error) {
1882
+ await rm(target, {
1883
+ recursive: true,
1884
+ force: true
1885
+ }).catch(() => {});
1886
+ if (a.signal.aborted) return;
1887
+ if (timeout.aborted) return failed(`it took over ${Math.round(timeoutMs / 1e3)}s`);
1888
+ failed(error instanceof Error ? error.message : String(error));
1889
+ }
1890
+ }
1891
+ //#endregion
1641
1892
  //#region src/attempt/turns.ts
1642
1893
  /** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
1643
1894
  const TURN_STOP_GRACE_MS = 3e3;
@@ -1820,6 +2071,7 @@ function turnSender(a, takeContext, checks) {
1820
2071
  a.signal.throwIfAborted();
1821
2072
  const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
1822
2073
  a.record.diff = diff;
2074
+ if (a.spec.snapshot.when === "turns") await takeSnapshot(a, box, `turn-${turnNumber}`, "harness");
1823
2075
  const data = {
1824
2076
  prompt,
1825
2077
  ...result,
@@ -1863,7 +2115,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1863
2115
  } finally {
1864
2116
  clock?.resume();
1865
2117
  }
1866
- });
2118
+ }, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
1867
2119
  const destroy = () => void a.sandbox?.destroy();
1868
2120
  testSignal.addEventListener("abort", destroy, { once: true });
1869
2121
  const takeContext = contextTaker(a);
@@ -1932,6 +2184,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1932
2184
  if (a.session) debug.log("teardown", "harness session closed");
1933
2185
  recordEgress(a, sandbox);
1934
2186
  oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
2187
+ if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
1935
2188
  await sandbox?.destroy();
1936
2189
  if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
1937
2190
  const { record } = a;
@@ -1953,6 +2206,30 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1953
2206
  if (failure) throw failure.error;
1954
2207
  }
1955
2208
  //#endregion
2209
+ //#region src/best-of.ts
2210
+ /** The side an attempt that settled counts on: its record's outcome, else the engine's state. */
2211
+ function sideOf({ state, meta }) {
2212
+ const outcome = meta?.outcome ?? (state === "passed" ? "pass" : state === "failed" ? "fail" : null);
2213
+ return outcome === "pass" || outcome === "fail" ? outcome : null;
2214
+ }
2215
+ /** Whether a best-of cell still needs another attempt after these. */
2216
+ function undecided(sides, bestOf) {
2217
+ const majority = majorityOf(bestOf);
2218
+ const pass = sides.filter((s) => s === "pass").length;
2219
+ const fail = sides.filter((s) => s === "fail").length;
2220
+ return pass < majority && fail < majority;
2221
+ }
2222
+ /** One best-of cell: the `ready` and `settled` hooks of its attempt `n`, from 1 to `bestOf`. */
2223
+ function bestOfCell(bestOf) {
2224
+ const sides = [];
2225
+ const settle = [];
2226
+ for (let n = 1; n <= bestOf; n++) sides.push(new Promise((resolve) => settle.push(resolve)));
2227
+ return (n) => ({
2228
+ ...n > majorityOf(bestOf) ? { ready: async () => undecided(await Promise.all(sides.slice(0, n - 1)), bestOf) } : {},
2229
+ settled: (settled) => settle[n - 1]?.(sideOf(settled))
2230
+ });
2231
+ }
2232
+ //#endregion
1956
2233
  //#region src/contract/declaration.ts
1957
2234
  /** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
1958
2235
  const evalOptionsDataSchema = z.record(z.string(), z.unknown());
@@ -2014,12 +2291,14 @@ const declarationProblem = (children) => dataProblem(declarationTreeSchema, { ch
2014
2291
  /** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
2015
2292
  const GRACE_MS = 5e3;
2016
2293
  function configLevel(config) {
2294
+ const { runs, bestOf } = config.runs !== void 0 ? runsOf(config.runs) : { runs: void 0 };
2017
2295
  return {
2018
2296
  harness: config.harness,
2019
2297
  sandbox: config.sandbox,
2020
2298
  environment: config.environment,
2021
2299
  compare: void 0,
2022
- runs: config.runs,
2300
+ runs,
2301
+ bestOf,
2023
2302
  gate: config.gate,
2024
2303
  env: config.env,
2025
2304
  egress: config.egress !== void 0 ? { inherited: config.egress } : {},
@@ -2028,6 +2307,7 @@ function configLevel(config) {
2028
2307
  question: config.question,
2029
2308
  questions: [],
2030
2309
  timeout: void 0,
2310
+ snapshot: config.snapshot,
2031
2311
  tags: [],
2032
2312
  revision: void 0,
2033
2313
  pin: [],
@@ -2210,6 +2490,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
2210
2490
  setBy,
2211
2491
  gate: planned.gate,
2212
2492
  ...planned.minLift !== void 0 ? { minLift: planned.minLift } : {},
2493
+ ...planned.bestOf !== void 0 ? { bestOf: planned.bestOf } : {},
2213
2494
  brokered: brokeredIn(resolved.env, planned),
2214
2495
  secretVars: secretVarsIn(resolved.env, planned),
2215
2496
  egress: planned.egress,
@@ -2249,6 +2530,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
2249
2530
  env: resolved.env,
2250
2531
  judge: resolved.judge,
2251
2532
  capture: resolved.capture,
2533
+ snapshot: resolved.snapshot,
2252
2534
  sources: {
2253
2535
  ...resolved.sources,
2254
2536
  sandbox: setBy.sandbox
@@ -2487,7 +2769,20 @@ function register(mode, name, own, fn, source, hint = "", family) {
2487
2769
  declareHarnesses(harnesses);
2488
2770
  if (!tests) return;
2489
2771
  registered++;
2490
- for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
2772
+ const bestOfCells = /* @__PURE__ */ new Map();
2773
+ for (const { name: testName, spec, ...options } of tests) {
2774
+ const { cell, n, bestOf } = options.attempt;
2775
+ let hooks = {};
2776
+ if (bestOf !== void 0) {
2777
+ const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
2778
+ bestOfCells.set(cell.key, of);
2779
+ hooks = of(n);
2780
+ }
2781
+ registerTest(testName, {
2782
+ ...options,
2783
+ ...hooks
2784
+ }, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
2785
+ }
2491
2786
  }
2492
2787
  /** Evals registered so far in this worker, for the scheduler's tie-break. */
2493
2788
  let registered = 0;
@@ -2601,4 +2896,4 @@ function optionsFrom(written, where, base) {
2601
2896
  //#endregion
2602
2897
  export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
2603
2898
 
2604
- //# sourceMappingURL=evaluate-BPP9s3s6.js.map
2899
+ //# sourceMappingURL=evaluate-CUtgyr43.js.map