reeval 0.15.1 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. package/README.md +12 -12
  2. package/dist/{adapter-CbqbMR7y.js → adapter-CRKYPkRt.js} +3 -3
  3. package/dist/{adapter-CbqbMR7y.js.map → adapter-CRKYPkRt.js.map} +1 -1
  4. package/dist/{agent-Zgn-kiqq.js → agent-BgH4-kFc.js} +6 -6
  5. package/dist/{agent-Zgn-kiqq.js.map → agent-BgH4-kFc.js.map} +1 -1
  6. package/dist/{app-files-ChbvRx3P.js → app-files-BMVBr079.js} +3 -3
  7. package/dist/app-files-BMVBr079.js.map +1 -0
  8. package/dist/{assemble-SKQGW_QF.js → assemble-BVbI6NW-.js} +95 -29
  9. package/dist/assemble-BVbI6NW-.js.map +1 -0
  10. package/dist/assemble-SiHa4__F.js +1 -0
  11. package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
  12. package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
  13. package/dist/{attempt-host-BOp8JklD.js → attempt-host-CIxjs2Be.js} +13 -6
  14. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  15. package/dist/{capture-BM_3100g.js → capture-iQ6kYJV0.js} +987 -161
  16. package/dist/capture-iQ6kYJV0.js.map +1 -0
  17. package/dist/capture-px7Ey95_.js +1 -0
  18. package/dist/{cli-PdYrcoKM.js → cli-1YQU-66F.js} +98 -61
  19. package/dist/cli-1YQU-66F.js.map +1 -0
  20. package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
  21. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  22. package/dist/cli.js +2 -2
  23. package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
  24. package/dist/client-DMmOkbRA.js.map +1 -0
  25. package/dist/{command-4pJYVCT4.js → command-8_Cg5v6b.js} +12 -12
  26. package/dist/{command-4pJYVCT4.js.map → command-8_Cg5v6b.js.map} +1 -1
  27. package/dist/{command-DbSMcxZL.js → command-BlYCAqu-.js} +7 -7
  28. package/dist/{command-DbSMcxZL.js.map → command-BlYCAqu-.js.map} +1 -1
  29. package/dist/{command-9xm5ImqI.js → command-C-HeGlap.js} +16 -16
  30. package/dist/{command-9xm5ImqI.js.map → command-C-HeGlap.js.map} +1 -1
  31. package/dist/{command-BqBxLgTc.js → command-Cr1zjRvl.js} +13 -13
  32. package/dist/{command-BqBxLgTc.js.map → command-Cr1zjRvl.js.map} +1 -1
  33. package/dist/{commands-Bi2SEQtr.js → commands-Bj9NbmsO.js} +1 -1
  34. package/dist/{commands-BU6YJCz9.js → commands-BvTNZ2-b.js} +14 -9
  35. package/dist/commands-BvTNZ2-b.js.map +1 -0
  36. package/dist/{config-k6m7XbAz.js → config-CneBZB84.js} +430 -208
  37. package/dist/config-CneBZB84.js.map +1 -0
  38. package/dist/{contrasts-DLZF4dGF.js → contrasts-lhFsekco.js} +2 -2
  39. package/dist/{contrasts-DLZF4dGF.js.map → contrasts-lhFsekco.js.map} +1 -1
  40. package/dist/db-BnXyamuj.js +2 -0
  41. package/dist/{db-CpsifVln.js → db-Cnh3jIv_.js} +8 -8
  42. package/dist/{db-CpsifVln.js.map → db-Cnh3jIv_.js.map} +1 -1
  43. package/dist/debug-CAx3qcaP.js +1 -0
  44. package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
  45. package/dist/debug-CgG3T071.js.map +1 -0
  46. package/dist/{docker-51WvhN_X.js → docker-BihzJwbV.js} +9 -5
  47. package/dist/docker-BihzJwbV.js.map +1 -0
  48. package/dist/{docker-B4-rx4dA.js → docker-Cqq03H72.js} +5 -5
  49. package/dist/{docker-B4-rx4dA.js.map → docker-Cqq03H72.js.map} +1 -1
  50. package/dist/{docker-YFj4H1Cr.js → docker-DCAtFMo9.js} +1 -1
  51. package/dist/{doctor-BuQnfz4e.js → doctor-Dl4vMeqt.js} +15 -15
  52. package/dist/{doctor-BuQnfz4e.js.map → doctor-Dl4vMeqt.js.map} +1 -1
  53. package/dist/{environment-BOFgzFDg.js → environment-Cf5HMjib.js} +3 -3
  54. package/dist/{environment-BOFgzFDg.js.map → environment-Cf5HMjib.js.map} +1 -1
  55. package/dist/{errors-UjcLWelF.js → errors-CHebohjc.js} +9 -3
  56. package/dist/{errors-UjcLWelF.js.map → errors-CHebohjc.js.map} +1 -1
  57. package/dist/{eval-source-Dje3rERU.js → eval-source-CAE4y2Sr.js} +9 -8
  58. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  59. package/dist/{evaluate-BPP9s3s6.js → evaluate-CUtgyr43.js} +340 -45
  60. package/dist/evaluate-CUtgyr43.js.map +1 -0
  61. package/dist/{evaluate-dqXccn4k.d.ts → evaluate-D35T51AC.d.ts} +7 -6
  62. package/dist/files-BEnUJW1e.js +1 -0
  63. package/dist/{files-Bi461Be2.js → files-BRViE7m-.js} +20 -13
  64. package/dist/{files-Bi461Be2.js.map → files-BRViE7m-.js.map} +1 -1
  65. package/dist/{files-H24pz0hL.js → files-BSqA2sZE.js} +3 -3
  66. package/dist/{files-H24pz0hL.js.map → files-BSqA2sZE.js.map} +1 -1
  67. package/dist/files-C3tQtD26.js +1 -0
  68. package/dist/{files-Clipx4OW.js → files-CNiJ_fZf.js} +3 -3
  69. package/dist/{files-Clipx4OW.js.map → files-CNiJ_fZf.js.map} +1 -1
  70. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  71. package/dist/files-D1y8Pg4Z.js.map +1 -0
  72. package/dist/files-Dts7EY3H.js +1 -0
  73. package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
  74. package/dist/gate-BC6SdQZs.js.map +1 -0
  75. package/dist/gateway.d.ts +1 -1
  76. package/dist/{git-CsVZkNNL.js → git-UNxT_3z0.js} +2 -2
  77. package/dist/{git-CsVZkNNL.js.map → git-UNxT_3z0.js.map} +1 -1
  78. package/dist/harness.d.ts +1 -1
  79. package/dist/{images-4mwhNkvh.js → images-8A_LzVfw.js} +8 -8
  80. package/dist/{images-4mwhNkvh.js.map → images-8A_LzVfw.js.map} +1 -1
  81. package/dist/index.d.ts +63 -16
  82. package/dist/index.js +5 -5
  83. package/dist/init-okoALIah.js +1 -0
  84. package/dist/{init-cOk3D44b.js → init-sFBgODhu.js} +4 -4
  85. package/dist/{init-cOk3D44b.js.map → init-sFBgODhu.js.map} +1 -1
  86. package/dist/{invocations-DW-lTk2f.js → invocations-C9ejlOYl.js} +2 -2
  87. package/dist/{invocations-DW-lTk2f.js.map → invocations-C9ejlOYl.js.map} +1 -1
  88. package/dist/{judge-CKdgEXN7.js → judge-Bg4v4e_K.js} +2 -2
  89. package/dist/{judge-CKdgEXN7.js.map → judge-Bg4v4e_K.js.map} +1 -1
  90. package/dist/judge.d.ts +2 -2
  91. package/dist/judge.js +1 -1
  92. package/dist/{live-XJQ6nqK4.js → live-DeMvnDD5.js} +2 -2
  93. package/dist/{live-XJQ6nqK4.js.map → live-DeMvnDD5.js.map} +1 -1
  94. package/dist/{matchers-DT2TaGix.js → matchers-BKPuaXlj.js} +14 -13
  95. package/dist/matchers-BKPuaXlj.js.map +1 -0
  96. package/dist/{model-Bq3G0OG1.js → model-2rBZrdZA.js} +4 -4
  97. package/dist/{model-Bq3G0OG1.js.map → model-2rBZrdZA.js.map} +1 -1
  98. package/dist/{model-vRiUycWh.js → model-3MJtpho8.js} +4 -4
  99. package/dist/{model-vRiUycWh.js.map → model-3MJtpho8.js.map} +1 -1
  100. package/dist/{model-CTnyvh7H.js → model-Bz7-j71_.js} +3 -50
  101. package/dist/model-Bz7-j71_.js.map +1 -0
  102. package/dist/{model-g3XnzUv0.js → model-DkPwypN4.js} +1 -1
  103. package/dist/model-bREXVXwT.js +1 -0
  104. package/dist/{node-BVbbnFtv.js → node-fP-H2uxn.js} +2 -2
  105. package/dist/{node-BVbbnFtv.js.map → node-fP-H2uxn.js.map} +1 -1
  106. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  107. package/dist/object-BkXGPKWY.js.map +1 -0
  108. package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-Bo8PZ4Ew.js} +2 -2
  109. package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-Bo8PZ4Ew.js.map} +1 -1
  110. package/dist/{patch-DKJiC5Sq.js → patch-6T9XAipm.js} +4 -4
  111. package/dist/patch-6T9XAipm.js.map +1 -0
  112. package/dist/{plan-DdWkM6SW.js → plan-Dw2BjafJ.js} +20 -11
  113. package/dist/plan-Dw2BjafJ.js.map +1 -0
  114. package/dist/{plugins-D-CP6mMd.js → plugins-ngQEJBjJ.js} +24 -3
  115. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  116. package/dist/{preflight-DTiVv0gY.js → preflight-BYXkweNN.js} +1 -1
  117. package/dist/{preflight-9ixuXzsx.js → preflight-CDRpog9o.js} +3 -3
  118. package/dist/{preflight-9ixuXzsx.js.map → preflight-CDRpog9o.js.map} +1 -1
  119. package/dist/{prepare-tmBBf4W8.js → prepare-B3btUlT6.js} +2 -2
  120. package/dist/{prepare-BfyikQit.js → prepare-PkARUGoF.js} +7 -7
  121. package/dist/{prepare-BfyikQit.js.map → prepare-PkARUGoF.js.map} +1 -1
  122. package/dist/{provenance-BizwUn3h.js → provenance-7Hzg88f4.js} +2 -2
  123. package/dist/{provenance-BizwUn3h.js.map → provenance-7Hzg88f4.js.map} +1 -1
  124. package/dist/{providers-DkOupEBl.js → providers-DDNMbv1T.js} +1 -1
  125. package/dist/{providers-B6b9Tnyi.js → providers-PKoJtGK0.js} +5 -5
  126. package/dist/{providers-B6b9Tnyi.js.map → providers-PKoJtGK0.js.map} +1 -1
  127. package/dist/query-Bt6vLJ3m.js +1 -0
  128. package/dist/{query-DyM-VExS.js → query-nBXrgEwl.js} +2 -2
  129. package/dist/{query-DyM-VExS.js.map → query-nBXrgEwl.js.map} +1 -1
  130. package/dist/{questions-BRhU8MzB.js → questions-CkAwePPR.js} +16 -17
  131. package/dist/questions-CkAwePPR.js.map +1 -0
  132. package/dist/remarks-Bp6AwPgb.js +1 -0
  133. package/dist/{remarks-C7i2LMRb.js → remarks-Bt0zwiQB.js} +42 -8
  134. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  135. package/dist/{remarks-D_3qhBWU.js → remarks-IFFe_oty.js} +4 -3
  136. package/dist/{remarks-D_3qhBWU.js.map → remarks-IFFe_oty.js.map} +1 -1
  137. package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
  138. package/dist/remarks-eBcrIZRu.js.map +1 -0
  139. package/dist/{remote-DFIzcx_6.js → remote-DfJYiP-R.js} +5 -5
  140. package/dist/{remote-DFIzcx_6.js.map → remote-DfJYiP-R.js.map} +1 -1
  141. package/dist/resolve-CT4XZIfV.js +1 -0
  142. package/dist/{resolve-DWSJpKHX.js → resolve-D_cLa4tW.js} +5 -5
  143. package/dist/{resolve-DWSJpKHX.js.map → resolve-D_cLa4tW.js.map} +1 -1
  144. package/dist/{results-BC3pcG_Z.js → results-DrKJuIQF.js} +9 -9
  145. package/dist/{results-BC3pcG_Z.js.map → results-DrKJuIQF.js.map} +1 -1
  146. package/dist/{resume-j-rcIryi.js → resume-BmIM52MJ.js} +4 -4
  147. package/dist/{resume-j-rcIryi.js.map → resume-BmIM52MJ.js.map} +1 -1
  148. package/dist/{runtime-d56Gg3J5.js → runtime-CZVpkr04.js} +111 -37
  149. package/dist/runtime-CZVpkr04.js.map +1 -0
  150. package/dist/{sandbox-Cmln11gU.js → sandbox-Cmi6IQML.js} +2 -2
  151. package/dist/{sandbox-Cmln11gU.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  152. package/dist/sandbox.d.ts +1 -1
  153. package/dist/sandbox.js +1 -1
  154. package/dist/server-CKJOnIB0.js +1 -0
  155. package/dist/{server-BOoH10J8.js → server-DS8BGhGU.js} +34 -10
  156. package/dist/server-DS8BGhGU.js.map +1 -0
  157. package/dist/source-CgUqpumI.js +1 -0
  158. package/dist/{source-wD04-rRG.js → source-DcXv4de-.js} +17 -3
  159. package/dist/{source-wD04-rRG.js.map → source-DcXv4de-.js.map} +1 -1
  160. package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
  161. package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
  162. package/dist/{src-kYeaJJrl.js → src-BTqJbvj6.js} +1 -1
  163. package/dist/{src-Bp4aN3rA.js → src-BkiComGh.js} +1 -1
  164. package/dist/{src-dlX-Lmxg.js → src-BvgeZ4tS.js} +1 -1
  165. package/dist/{src-D91Hd6hT.js → src-C0ySiej5.js} +36 -17
  166. package/dist/src-C0ySiej5.js.map +1 -0
  167. package/dist/{src-HSxBazcg.js → src-CgJrz5p0.js} +1 -1
  168. package/dist/{src-BYPMM1x7.js → src-Cp-2Hdib.js} +1 -1
  169. package/dist/{src-6_zz7oJs.js → src-DMo2eCOJ.js} +1 -1
  170. package/dist/src-Dk2Na0b-.js +2 -0
  171. package/dist/{src-C7upnZSd.js → src-OESRtvOi.js} +1 -1
  172. package/dist/src-h5uDlk3m.js +1 -0
  173. package/dist/state-C97udoh3.js +1 -0
  174. package/dist/{state-DrJ687ut.js → state-C_kdhp5v.js} +5 -5
  175. package/dist/{state-DrJ687ut.js.map → state-C_kdhp5v.js.map} +1 -1
  176. package/dist/suite-ByT1lsdD.js +2 -0
  177. package/dist/{suite-Co8cwelb.js → suite-DieLXvCT.js} +15 -4
  178. package/dist/suite-DieLXvCT.js.map +1 -0
  179. package/dist/{summary-DLFHVoux.js → summary-CSzyBVsF.js} +40 -14
  180. package/dist/summary-CSzyBVsF.js.map +1 -0
  181. package/dist/tarball-CHcahyNK.js +258 -0
  182. package/dist/tarball-CHcahyNK.js.map +1 -0
  183. package/dist/testing.d.ts +52 -44
  184. package/dist/testing.js +4 -4
  185. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  186. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  187. package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
  188. package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
  189. package/dist/{types-coMWHaU2.d.ts → types-DVvW66ci.d.ts} +325 -171
  190. package/dist/view/index.d.ts +16 -4
  191. package/dist/view/index.js +6 -6
  192. package/dist/{views-BFO1BLKu.js → views-B9STsLB4.js} +5 -5
  193. package/dist/{views-BFO1BLKu.js.map → views-B9STsLB4.js.map} +1 -1
  194. package/dist/views-Ca-rTixp.js +2 -0
  195. package/dist/{worker-entry-CJtdsKBP.js → worker-entry-C5bLCv6s.js} +34 -13
  196. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  197. package/dist/workspace-8uHtBSve.js +1 -0
  198. package/dist/{workspace-BRxjfa8N.js → workspace-CBuI9Uds.js} +4 -4
  199. package/dist/{workspace-BRxjfa8N.js.map → workspace-CBuI9Uds.js.map} +1 -1
  200. package/package.json +1 -1
  201. package/skills/eval-rubric/SKILL.md +11 -0
  202. package/dist/app-files-ChbvRx3P.js.map +0 -1
  203. package/dist/assemble-SKQGW_QF.js.map +0 -1
  204. package/dist/assemble-XBWuK3Ed.js +0 -1
  205. package/dist/attempt-host-BOp8JklD.js.map +0 -1
  206. package/dist/capture-BM_3100g.js.map +0 -1
  207. package/dist/capture-DSK8O5bA.js +0 -1
  208. package/dist/cli-PdYrcoKM.js.map +0 -1
  209. package/dist/cli-spec-D0qyrPZi.js.map +0 -1
  210. package/dist/client-Ct6s6OnF.js.map +0 -1
  211. package/dist/commands-BU6YJCz9.js.map +0 -1
  212. package/dist/config-k6m7XbAz.js.map +0 -1
  213. package/dist/db-DkD-F-X-.js +0 -2
  214. package/dist/debug-BhizgAsU.js.map +0 -1
  215. package/dist/debug-DeM5pswK.js +0 -1
  216. package/dist/docker-51WvhN_X.js.map +0 -1
  217. package/dist/eval-source-Dje3rERU.js.map +0 -1
  218. package/dist/evaluate-BPP9s3s6.js.map +0 -1
  219. package/dist/files-BAt0i00X.js +0 -1
  220. package/dist/files-C2h2Cw4F.js +0 -1
  221. package/dist/files-Ca0d6Uq3.js.map +0 -1
  222. package/dist/files-DmUrRmmL.js +0 -1
  223. package/dist/init-BDEOopK7.js +0 -1
  224. package/dist/matchers-DT2TaGix.js.map +0 -1
  225. package/dist/model-CTnyvh7H.js.map +0 -1
  226. package/dist/model-D-SWrLZG.js +0 -1
  227. package/dist/object-CLf2-tNq.js.map +0 -1
  228. package/dist/pairs-puYi-zj2.js.map +0 -1
  229. package/dist/patch-DKJiC5Sq.js.map +0 -1
  230. package/dist/plan-DdWkM6SW.js.map +0 -1
  231. package/dist/plugins-D-CP6mMd.js.map +0 -1
  232. package/dist/query-Clm2mNSm.js +0 -1
  233. package/dist/questions-BRhU8MzB.js.map +0 -1
  234. package/dist/remarks-BwJ8pcTQ.js +0 -1
  235. package/dist/remarks-C7i2LMRb.js.map +0 -1
  236. package/dist/remarks-dA8LyoPc.js.map +0 -1
  237. package/dist/resolve--rQZ8u6Z.js +0 -1
  238. package/dist/runtime-d56Gg3J5.js.map +0 -1
  239. package/dist/server-BOoH10J8.js.map +0 -1
  240. package/dist/server-DpuVAbrO.js +0 -1
  241. package/dist/source-BFd6f0-R.js +0 -1
  242. package/dist/src-D91Hd6hT.js.map +0 -1
  243. package/dist/src-DH-9IO7f.js +0 -1
  244. package/dist/src-LD1BAJFa.js +0 -2
  245. package/dist/state-B5PLm5qY.js +0 -1
  246. package/dist/suite-22Kt9Grx.js +0 -2
  247. package/dist/suite-Co8cwelb.js.map +0 -1
  248. package/dist/summary-DLFHVoux.js.map +0 -1
  249. package/dist/views-BOf8qEL8.js +0 -2
  250. package/dist/worker-entry-CJtdsKBP.js.map +0 -1
  251. package/dist/workspace-DDHzkMjm.js +0 -1
@@ -1,29 +1,29 @@
1
- import { n as isOneOf, t as isObjectLike } from "./object-CLf2-tNq.js";
1
+ import { n as isOneOf, t as isObjectLike } from "./object-BkXGPKWY.js";
2
2
  import { n as UsageError } from "./errors-B6BSMs03.js";
3
3
  import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-B_lvGqWz.js";
4
- import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-BOFgzFDg.js";
5
- import { B as gatePolicySchema, D as pluginServerName, L as egressPolicySchema, V as gateProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-D-CP6mMd.js";
4
+ import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-Cf5HMjib.js";
5
+ import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-ngQEJBjJ.js";
6
6
  import { _ as gatewayDescriptorSchema, f as FACTORY, g as directOffPin, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayEntryDataSchema, y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
7
7
  import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
8
8
  import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
9
- import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-CKdgEXN7.js";
9
+ import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-Bg4v4e_K.js";
10
10
  import { a as VERTEX_ROUTES, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-DypYjx71.js";
11
11
  import { r as badCharacterAt } from "./redact-DVIaahLv.js";
12
12
  import { s as withoutBom } from "./text-DTigcA64.js";
13
13
  import { t as inCI } from "./env-CARj5Axq.js";
14
14
  import { u as ownFileURL } from "./root-Dlff-zp8.js";
15
- import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, dt as run, ft as shellQuote } from "./src-D91Hd6hT.js";
15
+ import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, ft as run, pt as shellQuote } from "./src-C0ySiej5.js";
16
16
  import { t as InfraError } from "./types-C5vI3g1C.js";
17
- import { i as pairFactors } from "./pairs-puYi-zj2.js";
17
+ import { n as gateLabel, u as pairFactors } from "./gate-BC6SdQZs.js";
18
18
  import "./stats-hfpuYsAM.js";
19
- import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CbqbMR7y.js";
20
- import { t as docker } from "./sandbox-Cmln11gU.js";
19
+ import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CRKYPkRt.js";
20
+ import { t as docker } from "./sandbox-Cmi6IQML.js";
21
21
  import { existsSync, readFileSync } from "node:fs";
22
22
  import { availableParallelism, tmpdir, totalmem } from "node:os";
23
23
  import { dirname, join, resolve, win32 } from "node:path";
24
24
  import { parseDocument } from "yaml";
25
25
  import { z } from "zod";
26
- import { isIP } from "node:net";
26
+ import { connect, isIP } from "node:net";
27
27
  import { createHash, randomUUID } from "node:crypto";
28
28
  import { spawn } from "node:child_process";
29
29
  import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
@@ -84,21 +84,43 @@ function decideState(json, files) {
84
84
  }
85
85
  //#endregion
86
86
  //#region src/contract/judge.ts
87
- const judgeStateSchema = z.strictObject({
87
+ /** The inputs every judge call and question shares; `c.judge()` adds `contents` and its own `artifacts`. */
88
+ const sharedInputFields = {
88
89
  task: z.boolean().optional(),
89
90
  output: z.boolean().optional(),
90
91
  files: z.boolean().optional(),
91
92
  diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
92
- contents: z.array(nonEmpty()).optional(),
93
93
  transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
94
- turns: z.enum(["last", "all"]).optional(),
95
94
  checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
95
+ };
96
+ const turnsSchema = z.enum(["last", "all"]);
97
+ const { task, output, files, diff, transcript, checks } = sharedInputFields;
98
+ const judgeInputsSchema = z.strictObject({
99
+ task,
100
+ output,
101
+ files,
102
+ diff,
103
+ contents: z.array(nonEmpty()).optional(),
104
+ transcript,
105
+ checks,
106
+ artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
96
107
  });
108
+ const judgeSubjectSchema = z.strictObject({
109
+ on: z.custom().optional(),
110
+ turns: turnsSchema.optional()
111
+ });
112
+ const INPUT_KEYS = Object.keys(judgeInputsSchema.shape);
113
+ /** True when an input's value turns it on: not `false`, unset or an empty list. */
114
+ const inputOn = (value) => value !== void 0 && value !== false && !(Array.isArray(value) && value.length === 0);
115
+ /** The input keys an options object turns on, in `keys`' order. */
116
+ const inputsOn = (value, keys) => keys.filter((k) => inputOn(value[k]));
117
+ /** `a`, `b` and `c` as prose: "a, b or c". */
118
+ const orList = (items) => items.length < 2 ? items[0] ?? "" : `${items.slice(0, -1).join(", ")} or ${items.at(-1)}`;
119
+ const NO_JUDGE_INPUT = `c.judge() shows the judge nothing: turn on at least one input, such as { diff: true } or { contents: ['src/**'] }. The inputs are ${orList(INPUT_KEYS)}.`;
97
120
  const judgeFields = {
98
121
  threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional(),
99
122
  providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional(),
100
123
  maxConcurrent: z.number().int().min(1).optional(),
101
- state: judgeStateSchema.optional(),
102
124
  gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional()
103
125
  };
104
126
  const fakeJudgeDataSchema = z.strictObject({
@@ -114,18 +136,13 @@ const judgeOptionsSchema = z.strictObject({
114
136
  model: z.custom().optional(),
115
137
  ...judgeFields
116
138
  });
117
- const judgeCallSchema = judgeOptionsSchema.extend({
118
- on: z.custom().optional(),
119
- artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
120
- });
121
- const STATE_KEYS$1 = Object.keys(judgeStateSchema.shape);
122
- const JUDGE_HINTS = {
123
- see: "state",
124
- shows: "state",
125
- show: "state",
126
- input: "state"
127
- };
128
- const VALUES = {
139
+ const judgeCallSchema = judgeOptionsSchema.extend(judgeInputsSchema.shape).extend(judgeSubjectSchema.shape);
140
+ const CALL_ONLY_KEYS = [
141
+ ...INPUT_KEYS,
142
+ "on",
143
+ "turns"
144
+ ];
145
+ const VALUES$1 = {
129
146
  task: "true or false",
130
147
  output: "true or false",
131
148
  files: "true or false",
@@ -162,7 +179,10 @@ function judgeProblem(judge, site) {
162
179
  return `${key ? `${base}.${key}` : base} in ${site.at === "config" ? site.file : site.where}`;
163
180
  };
164
181
  const loc = site.at === "call" ? " in c.judge()" : site.at === "options" ? ` in ${site.where}` : ` in ${site.path === "judge" ? "" : `${site.path.replace(/\.judge$/, "")} in `}${site.file}`;
165
- if (typeof judge !== "object" || judge === null || Array.isArray(judge)) return `${field()} must be an object such as { threshold: 0.8 } (got ${shown(judge)})`;
182
+ if (typeof judge !== "object" || judge === null || Array.isArray(judge)) {
183
+ const example = site.at === "call" ? "{ diff: true }" : "{ threshold: 0.8 }";
184
+ return `${field()} must be an object such as ${example} (got ${shown(judge)})`;
185
+ }
166
186
  const value = judge;
167
187
  const model = value.model;
168
188
  if (site.at === "config" && typeof model === "string") {
@@ -171,7 +191,7 @@ function judgeProblem(judge, site) {
171
191
  }
172
192
  const schema = site.at === "call" ? judgeCallSchema : site.at === "config" ? configJudgeDataSchema : judgeOptionsSchema;
173
193
  const parsed = schema.safeParse(judge);
174
- if (parsed.success) return void 0;
194
+ if (parsed.success) return site.at === "call" && !inputsOn(value, INPUT_KEYS).length ? NO_JUDGE_INPUT : void 0;
175
195
  const numericFake = site.at === "config" && typeof model === "object" && model?.kind === "fake-judge" && typeof model.score === "number";
176
196
  const issues = parsed.error.issues.filter((i) => !(numericFake && i.path[0] === "model"));
177
197
  if (!issues.length) return void 0;
@@ -194,29 +214,22 @@ function judgeProblem(judge, site) {
194
214
  const name = issues.find((i) => i.code === "unrecognized_keys" && i.path.length === 0)?.keys[0];
195
215
  if (name === void 0) return void 0;
196
216
  const allowed = Object.keys(schema.shape);
197
- const hint = JUDGE_HINTS[name] ?? (STATE_KEYS$1.includes(name) ? `state.${name}` : nearest(name, allowed));
198
- const why = site.at !== "call" && (name === "on" || name === "artifacts") ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
217
+ const hint = nearest(name, allowed);
218
+ const why = site.at !== "call" && CALL_ONLY_KEYS.includes(name) ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
199
219
  return `Unknown judge option "${name}"${loc}.${why}`;
200
220
  };
201
- const stateProblem = () => {
202
- const issue = issues.find((i) => i.path[0] === "state");
221
+ const inputProblem = () => {
222
+ const issue = issues.find((i) => typeof i.path[0] === "string" && i.path[0] in VALUES$1);
203
223
  if (!issue) return void 0;
204
- const { state } = value;
205
- if (typeof state !== "object" || state === null || Array.isArray(state)) return `${field("state")} must be an object such as { transcript: 'tools' } (got ${shown(state)})`;
206
- if (issue.code === "unrecognized_keys" && issue.path.length === 1) {
207
- const key = issue.keys[0] ?? "";
208
- const hint = nearest(key, STATE_KEYS$1);
209
- return `Unknown judge.state key "${key}"${loc}.${hint ? ` Did you mean "${hint}"?` : ""}`;
210
- }
211
- const key = issue.path[1];
212
- return `${field(`state.${String(key)}`)} must be ${VALUES[key]} (got ${shown(state[key])})`;
224
+ const key = issue.path[0];
225
+ return `${field(key)} must be ${VALUES$1[key]} (got ${shown(value[key])})`;
213
226
  };
214
227
  const providerOptionsProblem = () => at("providerOptions") ? `${field("providerOptions")} must map each provider to an object of JSON values, e.g. { google: { thinkingConfig: { thinkingBudget: 0 } } } (got ${shown(value.providerOptions)})` : void 0;
215
228
  const artifactsProblem = () => at("artifacts") ? `c.judge() artifacts ${badJudgeArtifacts(value.artifacts)}` : void 0;
216
229
  const gatewayProblem = () => at("gateway") ? `${field("gateway")} names a gateway in the config's gateway list: ${GATEWAY_NAMES.map((g) => `'${g}'`).join(" or ")}, or is its helper (got ${shown(value.gateway)})` : void 0;
217
- if (site.at === "config") order.push(modelProblem, keyProblem, stateProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
218
- else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, stateProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
219
- else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, stateProblem, providerOptionsProblem, gatewayProblem);
230
+ if (site.at === "config") order.push(modelProblem, keyProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
231
+ else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, inputProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
232
+ else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, providerOptionsProblem, gatewayProblem);
220
233
  for (const check of order) {
221
234
  const problem = check();
222
235
  if (problem) return problem;
@@ -228,22 +241,15 @@ function judgeProblem(judge, site) {
228
241
  const pairFactorSchema = z.enum(["environment", "harness"]);
229
242
  /** Evaluation models score yes/no questions only, so they can't answer a question. */
230
243
  const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
231
- const questionStateSchema = z.strictObject({
232
- task: z.boolean().optional(),
233
- output: z.boolean().optional(),
234
- files: z.boolean().optional(),
235
- diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
236
- transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
237
- turns: z.enum(["last", "all"]).optional(),
238
- checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
239
- });
240
244
  const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
241
- const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
242
- const questionOptionsSchema = z.strictObject({
243
- model: questionModelSchema.optional(),
244
- state: questionStateSchema.optional(),
245
+ const questionInputFields = {
246
+ ...sharedInputFields,
245
247
  artifacts: questionArtifactsSchema.optional()
246
- });
248
+ };
249
+ z.strictObject(questionInputFields);
250
+ const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
251
+ const questionOptionsSchema = z.strictObject({ model: questionModelSchema.optional() });
252
+ const QUESTION_INPUT_KEYS = Object.keys(questionInputFields);
247
253
  /** The first item a list holds twice. */
248
254
  const twice = (items) => items.find((k, i) => items.indexOf(k) !== i);
249
255
  const unique = (list, ctx) => {
@@ -254,35 +260,38 @@ const unique = (list, ctx) => {
254
260
  };
255
261
  const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
256
262
  const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
257
- const questionObjectSchema = z.strictObject({
263
+ const questionSchema = z.strictObject({
258
264
  question: nonEmpty(),
259
265
  model: questionModelSchema.optional(),
260
266
  across: acrossSchema.optional(),
261
267
  choices: choicesSchema.optional(),
262
- state: questionStateSchema.optional(),
263
- artifacts: questionArtifactsSchema.optional()
268
+ turns: turnsSchema.optional(),
269
+ ...questionInputFields
264
270
  }).superRefine((q, ctx) => {
265
271
  if (q.across !== void 0 && q.choices !== void 0) ctx.addIssue({
266
272
  code: "custom",
267
273
  path: ["choices"],
268
274
  message: "with across"
269
275
  });
276
+ if (!inputsOn(q, QUESTION_INPUT_KEYS).length) ctx.addIssue({
277
+ code: "custom",
278
+ message: "no input"
279
+ });
270
280
  });
271
- const questionSchema = z.union([nonEmpty(), questionObjectSchema]);
272
- const questionText = (q) => typeof q === "string" ? q : q.question;
281
+ const questionText = (q) => q.question;
273
282
  const questionsSchema = z.array(questionSchema).superRefine((list, ctx) => {
274
283
  if (twice(list.map(questionText)) !== void 0) ctx.addIssue({
275
284
  code: "custom",
276
285
  message: "twice"
277
286
  });
278
287
  });
279
- const QUESTION_EXAMPLE = `questions: ['How did the agent find the failing test?']`;
288
+ const QUESTION_EXAMPLE = `questions: [{ question: 'How did the agent find the failing test?', transcript: true }]`;
280
289
  const ACROSS_HINT = `'environment' (vary the environment, same harness) or 'harness' (vary the harness, same environment)`;
281
290
  const ACROSS_MEAN = " Did you mean \"across\"? It is the factor a pairwise question varies: across: 'environment'.";
282
- const QUESTION_KEYS = Object.keys(questionObjectSchema.shape);
283
- const QUESTION_SHAPE = "{ question, model?, across?, choices?, state?, artifacts? }";
284
- const STATE_KEYS = Object.keys(questionStateSchema.shape);
285
- const STATE_VALUES = {
291
+ const QUESTION_KEYS = Object.keys(questionSchema.shape);
292
+ const INPUTS = orList(QUESTION_INPUT_KEYS);
293
+ const QUESTION_SHAPE = `{ question, model?, across?, choices?, turns? } with at least one of ${INPUTS}`;
294
+ const VALUES = {
286
295
  task: "true or false",
287
296
  output: "true or false",
288
297
  files: "true or false",
@@ -299,41 +308,31 @@ const describe$2 = (value) => {
299
308
  }
300
309
  return JSON.stringify(value) ?? String(value);
301
310
  };
302
- /**
303
- * What's wrong with a question's `state` or `artifacts`, as one sentence, or undefined. `name`
304
- * turns a key into how the sentence names it: `question.state` on the option,
305
- * `questions "…" state` on an entry.
306
- */
307
- function questionPartsProblem(value, name, where) {
308
- const { state, artifacts } = value;
309
- if (state !== void 0) {
310
- if (!isObjectLike(state) || Array.isArray(state)) return `${name("state")} in ${where} must be an object such as { transcript: 'tools' } (got ${describe$2(state)})`;
311
- for (const [key, v] of Object.entries(state)) {
312
- if (key === "contents") return `${name("state.contents")} in ${where} can't be set: questions are asked after the attempts, when their workspaces are gone. Narrow state.diff instead.`;
313
- const schema = questionStateSchema.shape[key];
314
- if (!schema) {
315
- const hint = closest(key, STATE_KEYS);
316
- return `Unknown ${name("state")} key "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""}`;
317
- }
318
- if (!schema.safeParse(v).success) return `${name(`state.${key}`)} in ${where} must be ${STATE_VALUES[key]} (got ${describe$2(v)})`;
311
+ /** What's wrong with an entry's inputs and `turns`, as one sentence, or undefined. */
312
+ function questionInputsProblem(value, at, where) {
313
+ for (const [key, v] of Object.entries(value)) {
314
+ if (key === "artifacts") {
315
+ if (!questionArtifactsSchema.safeParse(v).success) return `${at} artifacts in ${where} must be true or a non-empty list of capture names (got ${describe$2(v)})`;
316
+ continue;
319
317
  }
318
+ if (!(key in VALUES)) continue;
319
+ if (!(key === "turns" ? turnsSchema : sharedInputFields[key]).safeParse(v).success) return `${at} ${key} in ${where} must be ${VALUES[key]} (got ${describe$2(v)})`;
320
320
  }
321
- if (artifacts !== void 0 && !questionArtifactsSchema.safeParse(artifacts).success) return `${name("artifacts")} in ${where} must be true or a non-empty list of capture names (got ${describe$2(artifacts)})`;
321
+ if (!inputsOn(value, QUESTION_INPUT_KEYS).length) return `${at} in ${where} shows the model nothing: turn on at least one of ${INPUTS}, e.g. { question: '…', diff: true }.`;
322
322
  }
323
323
  /** What's wrong with the `question` option set in `where`, as one sentence, or undefined. */
324
324
  function questionOptionsProblem(value, where) {
325
- if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model?, state?, artifacts? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
325
+ if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
326
326
  const keys = Object.keys(questionOptionsSchema.shape);
327
327
  for (const key of Object.keys(value)) {
328
328
  if (keys.includes(key)) continue;
329
- if (key === "across" || key === "choices") return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: it says what kind of question that entry is.`;
329
+ if (key === "across" || key === "choices" || key === "turns" || key in questionInputFields) return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: what a question is and what it's shown are its own.`;
330
330
  const hint = closest(key, keys);
331
- return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model, state and artifacts.`;
331
+ return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model.`;
332
332
  }
333
333
  const { model } = value;
334
334
  if (model !== void 0 && typeof model === "string" && isEvaluationModel(model)) return `question.model in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
335
335
  if (model !== void 0 && !questionModelSchema.safeParse(model).success) return `question.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
336
- return questionPartsProblem(value, (key) => `question.${key}`, where);
337
336
  }
338
337
  /** What's wrong with an entry's `across`, or undefined. */
339
338
  function acrossProblem(question, across, where) {
@@ -352,11 +351,11 @@ function choicesProblem(question, choices, where) {
352
351
  if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
353
352
  return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
354
353
  }
355
- /** One question's sentence: the union refuses it whole, so the branch it meant says why. */
354
+ /** One question's sentence. */
356
355
  function questionProblem(item, where) {
357
- if (typeof item === "string") return `questions in ${where} has an empty string. Ask something, e.g. ${QUESTION_EXAMPLE}.`;
358
- if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes strings or ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
356
+ if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
359
357
  const unknown = Object.keys(item).filter((k) => !QUESTION_KEYS.includes(k));
358
+ if (unknown.includes("contents")) return `questions in ${where} can't set contents: questions are asked after the attempts, when their workspaces are gone. Narrow diff instead.`;
360
359
  if (unknown.length) {
361
360
  const hint = unknown.some((k) => k === "cut" || k === "cuts") ? ACROSS_MEAN : "";
362
361
  return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
@@ -373,7 +372,7 @@ function questionProblem(item, where) {
373
372
  if (problem) return problem;
374
373
  }
375
374
  if (across !== void 0 && choices !== void 0) return `questions "${text}" in ${where} sets both across and choices. choices are the answers a question asked of each attempt picks from, and across makes it pick between two attempts instead: drop choices to vote between pairs, or drop across to ask each attempt.`;
376
- const parts = questionPartsProblem(item, (key) => `questions "${text}" ${key}`, where);
375
+ const parts = questionInputsProblem(item, `questions "${text}"`, where);
377
376
  if (parts) return parts;
378
377
  if (typeof model === "string" && isEvaluationModel(model)) return `questions "${text}" in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
379
378
  return `questions "${text}" in ${where}: model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
@@ -389,6 +388,8 @@ function questionsProblem(value, where) {
389
388
  //#endregion
390
389
  //#region src/contract/config.ts
391
390
  const count = () => z.number().int().min(1);
391
+ /** `runs`: a count, or `{ bestOf }` with an odd count of 3 or more. */
392
+ const runs = () => z.union([count(), z.strictObject({ bestOf: z.number().int().min(3).refine((n) => n % 2 === 1, "must be odd") })]);
392
393
  const captureWindowSchema = z.strictObject({
393
394
  width: count(),
394
395
  height: count()
@@ -421,14 +422,15 @@ const profileDefaultsDataSchema = z.strictObject({
421
422
  harness: harnessList.optional(),
422
423
  sandbox: sandboxDescriptorSchema.optional(),
423
424
  environment: environmentDataSchema.optional(),
424
- runs: count().optional(),
425
+ runs: runs().optional(),
425
426
  gate: gatePolicySchema.optional(),
426
427
  env: envVarsSchema.optional(),
427
428
  egress: egressPolicySchema.optional(),
428
429
  judge: configJudgeDataSchema.optional(),
429
430
  timeout: durationSchema.optional(),
430
431
  capture: captureOptionsSchema.optional(),
431
- question: questionOptionsSchema.optional()
432
+ question: questionOptionsSchema.optional(),
433
+ snapshot: snapshotOptionSchema.optional()
432
434
  });
433
435
  const profileLimitSchema = z.strictObject({
434
436
  runs: count().optional(),
@@ -481,7 +483,7 @@ const reevalConfigDataSchema = z.strictObject({
481
483
  harness: harnessList.optional(),
482
484
  sandbox: sandboxDescriptorSchema.optional(),
483
485
  environment: environmentDataSchema.optional(),
484
- runs: count().optional(),
486
+ runs: runs().optional(),
485
487
  gate: gatePolicySchema.optional(),
486
488
  env: envVarsSchema.optional(),
487
489
  egress: egressPolicySchema.optional(),
@@ -489,6 +491,7 @@ const reevalConfigDataSchema = z.strictObject({
489
491
  timeout: durationSchema.optional(),
490
492
  capture: captureOptionsSchema.optional(),
491
493
  question: questionOptionsSchema.optional(),
494
+ snapshot: snapshotOptionSchema.optional(),
492
495
  concurrency: count().optional(),
493
496
  record: recordMode.optional(),
494
497
  history: historyConfigSchema.optional(),
@@ -1125,7 +1128,7 @@ const AGENTS = {
1125
1128
  },
1126
1129
  unreported: cursorUnreported,
1127
1130
  mcpServers: cursorMcpServers,
1128
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-6_zz7oJs.js")).createCursorSdk({
1131
+ adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-DMo2eCOJ.js")).createCursorSdk({
1129
1132
  auth,
1130
1133
  ...mcpServers ? { mcpServers } : {},
1131
1134
  ...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
@@ -1192,7 +1195,7 @@ const AGENTS = {
1192
1195
  },
1193
1196
  servedModel: geminiServedModel,
1194
1197
  mcpServers: antigravityMcpServers,
1195
- adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-C7upnZSd.js")).createAntigravity({
1198
+ adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-OESRtvOi.js")).createAntigravity({
1196
1199
  auth,
1197
1200
  ...vertex ? { vertex } : {},
1198
1201
  ...mcpServers ? { mcpServers } : {},
@@ -1219,7 +1222,7 @@ const AGENTS = {
1219
1222
  servedModel: geminiServedModel,
1220
1223
  forModel: opencodeForModel,
1221
1224
  mcpServers: opencodeMcpServers,
1222
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-HSxBazcg.js")).createOpenCode({
1225
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-CgJrz5p0.js")).createOpenCode({
1223
1226
  auth,
1224
1227
  model,
1225
1228
  ...vertex ? { vertex } : { vars: opencodeVars(model) },
@@ -1241,14 +1244,14 @@ const AGENTS = {
1241
1244
  },
1242
1245
  servedModel: geminiServedModel,
1243
1246
  mcpServers: gooseMcpServers,
1244
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-kYeaJJrl.js")).createGoose({
1247
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BTqJbvj6.js")).createGoose({
1245
1248
  ...vertex ? { vertex } : {},
1246
1249
  ...auth["AI_GATEWAY_API_KEY"] ? {
1247
1250
  auth,
1248
- provider: (await import("./src-kYeaJJrl.js")).AI_GATEWAY_PROVIDER
1251
+ provider: (await import("./src-BTqJbvj6.js")).AI_GATEWAY_PROVIDER
1249
1252
  } : auth["OPENROUTER_API_KEY"] ? {
1250
1253
  auth,
1251
- provider: (await import("./src-kYeaJJrl.js")).OPENROUTER_PROVIDER
1254
+ provider: (await import("./src-BTqJbvj6.js")).OPENROUTER_PROVIDER
1252
1255
  } : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
1253
1256
  model,
1254
1257
  ...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
@@ -1269,13 +1272,13 @@ const AGENTS = {
1269
1272
  endpoints: GEMINI_ENDPOINTS,
1270
1273
  servedModel: geminiServedModel,
1271
1274
  mcpServers: piMcpServers,
1272
- adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-Bp4aN3rA.js")).createPi({
1275
+ adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-BkiComGh.js")).createPi({
1273
1276
  ...auth["AI_GATEWAY_API_KEY"] ? {
1274
1277
  auth,
1275
- gateway: (await import("./src-Bp4aN3rA.js")).PI_AI_GATEWAY
1278
+ gateway: (await import("./src-BkiComGh.js")).PI_AI_GATEWAY
1276
1279
  } : auth["OPENROUTER_API_KEY"] ? {
1277
1280
  auth,
1278
- gateway: (await import("./src-Bp4aN3rA.js")).PI_OPENROUTER
1281
+ gateway: (await import("./src-BkiComGh.js")).PI_OPENROUTER
1279
1282
  } : {
1280
1283
  auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
1281
1284
  ...vertex ? { vertex } : {}
@@ -1308,9 +1311,9 @@ const AGENTS = {
1308
1311
  },
1309
1312
  servedModel: openaiServedModel,
1310
1313
  mcpServers: codexMcpServers,
1311
- adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-dlX-Lmxg.js")).createCodex({
1314
+ adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-BvgeZ4tS.js")).createCodex({
1312
1315
  auth,
1313
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-dlX-Lmxg.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-dlX-Lmxg.js")).OPENROUTER_PROVIDER } : {},
1316
+ ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-BvgeZ4tS.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-BvgeZ4tS.js")).OPENROUTER_PROVIDER } : {},
1314
1317
  ...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
1315
1318
  ...mcpServers ? { mcpServers } : {},
1316
1319
  ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
@@ -1328,7 +1331,7 @@ const AGENTS = {
1328
1331
  ...fxForModel("provider/model"),
1329
1332
  forModel: fxForModel,
1330
1333
  mcpServers: acpMcpServers,
1331
- adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-LD1BAJFa.js")).createFx({
1334
+ adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-Dk2Na0b-.js")).createFx({
1332
1335
  auth,
1333
1336
  model,
1334
1337
  ...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
@@ -1353,7 +1356,7 @@ const AGENTS = {
1353
1356
  servedModel: xaiServedModel,
1354
1357
  mcpServers: acpMcpServers,
1355
1358
  adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
1356
- const grok = await import("./src-BYPMM1x7.js");
1359
+ const grok = await import("./src-Cp-2Hdib.js");
1357
1360
  return grok.createGrokBuild({
1358
1361
  ...vertexChat ? {
1359
1362
  auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
@@ -1704,11 +1707,10 @@ function resolvedEffortOf(models) {
1704
1707
  /** The error a per-attempt answer carries when the run stopped before a model answered it. */
1705
1708
  const STOPPED_FIRST = "the run stopped before it was answered";
1706
1709
  /**
1707
- * What each side of a pairwise question shows with no `state`: the task and the diff, every
1708
- * turn's prompt. The final message stays out, since a better-written account of the work would
1709
- * win a side-by-side pick on its wording; the changed files repeat what the diff shows.
1710
+ * What a recorded pairwise answer without `stateOptions` showed each side: the task and the diff,
1711
+ * every turn's prompt.
1710
1712
  */
1711
- const DEFAULT_QUESTION_STATE = {
1713
+ const UNRECORDED_QUESTION_STATE = {
1712
1714
  task: true,
1713
1715
  output: false,
1714
1716
  files: false,
@@ -1717,62 +1719,39 @@ const DEFAULT_QUESTION_STATE = {
1717
1719
  turns: "all",
1718
1720
  checks: false
1719
1721
  };
1722
+ const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
1720
1723
  /**
1721
- * What a question asked of each attempt on its own shows with no `state`: the task, the diff,
1722
- * the final message and the transcript. Nothing is weighed against another side, so how the
1723
- * agent went about the work is the point.
1724
+ * Every key set, in the record's key order, so equal selections serialize the same: an input the
1725
+ * question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
1724
1726
  */
1725
- const DEFAULT_ATTEMPT_QUESTION_STATE = {
1726
- ...DEFAULT_QUESTION_STATE,
1727
- output: true,
1728
- transcript: true
1729
- };
1730
- const KEYS = Object.keys(DEFAULT_QUESTION_STATE);
1731
- /** A pairwise question's defaults when `pairwise`, a per-attempt question's otherwise. */
1732
- const defaultQuestionState = (pairwise) => pairwise ? DEFAULT_QUESTION_STATE : DEFAULT_ATTEMPT_QUESTION_STATE;
1733
- /** Every key set, in `DEFAULT_QUESTION_STATE`'s order, so equal selections serialize the same. */
1734
- function resolveQuestionState(state, pairwise) {
1735
- const defaults = defaultQuestionState(pairwise);
1736
- const out = {};
1737
- for (const key of KEYS) out[key] = state?.[key] ?? defaults[key];
1738
- return out;
1727
+ function resolveQuestionState(state) {
1728
+ return {
1729
+ task: state?.task ?? false,
1730
+ output: state?.output ?? false,
1731
+ files: state?.files ?? false,
1732
+ diff: state?.diff ?? false,
1733
+ transcript: state?.transcript ?? false,
1734
+ turns: state?.turns ?? "all",
1735
+ checks: state?.checks ?? false
1736
+ };
1739
1737
  }
1740
- /** `inner` over `outer`: `state` key by key, `model` and `artifacts` replaced. */
1738
+ /** `inner` over `outer`, field by field. */
1741
1739
  function mergeQuestion(outer, inner) {
1742
- if (!inner) return outer;
1743
- const out = {
1740
+ return inner ? {
1744
1741
  ...outer,
1745
1742
  ...inner
1746
- };
1747
- if (outer.state || inner.state) out.state = {
1748
- ...outer.state,
1749
- ...inner.state
1750
- };
1751
- return out;
1743
+ } : outer;
1752
1744
  }
1753
- /** A planned question with the cascaded `question` defaults under its own `model`, `state` and `artifacts`. */
1745
+ /** A planned question under the cascaded `question` defaults: its own `model`, else theirs. */
1754
1746
  function withQuestionDefaults(q, defaults) {
1755
- const { state, artifacts } = mergeQuestion(defaults, {
1756
- ...q.state ? { state: q.state } : {},
1757
- ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
1758
- });
1759
- const { state: _state, artifacts: _artifacts, ...rest } = q;
1760
1747
  const model = q.model ?? defaults.model;
1761
- return {
1762
- ...rest,
1763
- ...model !== void 0 ? { model } : {},
1764
- ...state && Object.keys(state).length ? { state } : {},
1765
- ...artifacts !== void 0 ? { artifacts } : {}
1766
- };
1748
+ return model !== void 0 ? {
1749
+ ...q,
1750
+ model
1751
+ } : q;
1767
1752
  }
1768
1753
  const isOff = (value) => value === false || Array.isArray(value) && !value.length;
1769
- /** True when the resolved state shows nothing and no captures are sent. */
1770
- const showsNothing = (q) => {
1771
- if (q.artifacts !== void 0) return false;
1772
- const s = resolveQuestionState(q.state, q.across !== void 0);
1773
- return KEYS.every((k) => k === "turns" || isOff(s[k]));
1774
- };
1775
- /** The state keys a question shows, in order, plus `artifacts` when it sends captures. */
1754
+ /** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
1776
1755
  function questionShows(state, artifacts) {
1777
1756
  const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
1778
1757
  return artifacts === void 0 ? on : [...on, "artifacts"];
@@ -1838,6 +1817,189 @@ function compareProblem(value, where, environment) {
1838
1817
  if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness? }.`;
1839
1818
  return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
1840
1819
  }
1820
+ const POLL_MS = 200;
1821
+ /**
1822
+ * Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
1823
+ * (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
1824
+ */
1825
+ function probeScript(pid, tables = true) {
1826
+ return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
1827
+ }
1828
+ /** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
1829
+ function procAddress(hex) {
1830
+ const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
1831
+ if (bytes.length === 4) return bytes.join(".");
1832
+ if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
1833
+ const groups = [];
1834
+ for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
1835
+ const full = groups.join(":");
1836
+ const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
1837
+ return longest ? full.replace(longest, "::") : full;
1838
+ }
1839
+ /** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
1840
+ const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
1841
+ function parseProbe(stdout) {
1842
+ const lines = stdout.split("\n");
1843
+ const alive = lines.includes("alive");
1844
+ if (lines.includes("noproc")) return {
1845
+ proc: false,
1846
+ alive
1847
+ };
1848
+ const listeners = [];
1849
+ for (const l of lines) {
1850
+ const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
1851
+ const address = m ? procAddress(m[1]) : "";
1852
+ if (m && reachable(address)) listeners.push({
1853
+ port: Number.parseInt(m[2], 16),
1854
+ address
1855
+ });
1856
+ }
1857
+ return {
1858
+ proc: true,
1859
+ alive,
1860
+ listeners
1861
+ };
1862
+ }
1863
+ /** The ports listening now that weren't before the command started, ascending. */
1864
+ const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
1865
+ /** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
1866
+ const addressFor = (listeners, port) => {
1867
+ const all = listeners.filter((l) => l.port === port).map((l) => l.address);
1868
+ return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
1869
+ };
1870
+ /** `want` is taken whoever listens on it: reeval allocated it, or the eval named it. */
1871
+ function decide(s) {
1872
+ const { listeners, alive } = s.probe;
1873
+ const ports = newPorts(listeners, s.before);
1874
+ if (listeners.some((l) => l.port === s.want)) return {
1875
+ kind: "port",
1876
+ port: s.want,
1877
+ address: addressFor(listeners, s.want)
1878
+ };
1879
+ const late = s.elapsedMs >= s.timeoutMs;
1880
+ if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
1881
+ if (ports.length === 1) {
1882
+ const port = ports[0];
1883
+ return {
1884
+ kind: "port",
1885
+ port,
1886
+ address: addressFor(listeners, port)
1887
+ };
1888
+ }
1889
+ return {
1890
+ kind: "several",
1891
+ ports
1892
+ };
1893
+ }
1894
+ if (!alive) return { kind: "exited" };
1895
+ return late ? {
1896
+ kind: "timeout",
1897
+ ports
1898
+ } : { kind: "wait" };
1899
+ }
1900
+ const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
1901
+ /** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
1902
+ function printedUrls(output, ports) {
1903
+ const found = /* @__PURE__ */ new Map();
1904
+ for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
1905
+ const port = Number(m[1]);
1906
+ if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
1907
+ }
1908
+ return [...found].map(([url, port]) => ({
1909
+ url,
1910
+ port
1911
+ }));
1912
+ }
1913
+ /** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
1914
+ async function hostListening(port) {
1915
+ for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
1916
+ const socket = connect({
1917
+ port,
1918
+ host
1919
+ });
1920
+ socket.once("connect", () => {
1921
+ socket.destroy();
1922
+ resolve(true);
1923
+ });
1924
+ socket.once("error", () => resolve(false));
1925
+ })) return host;
1926
+ }
1927
+ const sleep = (ms, signal) => new Promise((resolve) => {
1928
+ const done = () => {
1929
+ clearTimeout(timer);
1930
+ signal?.removeEventListener("abort", done);
1931
+ resolve();
1932
+ };
1933
+ const timer = setTimeout(done, ms);
1934
+ signal?.addEventListener("abort", done, { once: true });
1935
+ });
1936
+ const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
1937
+ /**
1938
+ * Polls until something listens on `want`, or, without a given port, until the ports that are
1939
+ * new since the command started have settled on one. Throws, with the command's last output,
1940
+ * when they settle on several, it exits, or nothing listens in time.
1941
+ */
1942
+ async function awaitListening(command, w) {
1943
+ const timeoutMs = w.timeoutMs ?? 12e4;
1944
+ const started = Date.now();
1945
+ let last = "";
1946
+ let changed = started;
1947
+ const output = async () => plain(await w.output().catch(() => ""));
1948
+ const fail = async (why, out) => {
1949
+ const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
1950
+ return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
1951
+ };
1952
+ for (;;) {
1953
+ w.signal?.throwIfAborted();
1954
+ const probe = await w.probe();
1955
+ const now = Date.now();
1956
+ if (!probe.proc) {
1957
+ const address = await (w.hostListening ?? hostListening)(w.want);
1958
+ if (address !== void 0) return {
1959
+ port: w.want,
1960
+ address,
1961
+ detected: false,
1962
+ readyMs: now - started
1963
+ };
1964
+ if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
1965
+ if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
1966
+ } else {
1967
+ const key = newPorts(probe.listeners, w.before).join(",");
1968
+ if (key !== last) {
1969
+ last = key;
1970
+ changed = now;
1971
+ }
1972
+ const d = decide({
1973
+ want: w.want,
1974
+ given: w.given,
1975
+ probe,
1976
+ before: w.before,
1977
+ stableMs: now - changed,
1978
+ elapsedMs: now - started,
1979
+ timeoutMs
1980
+ });
1981
+ if (d.kind === "port") return {
1982
+ port: d.port,
1983
+ address: d.address,
1984
+ detected: d.port !== w.want,
1985
+ readyMs: now - started
1986
+ };
1987
+ if (d.kind === "several") {
1988
+ const out = await output();
1989
+ const urls = printedUrls(out, d.ports);
1990
+ const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
1991
+ const hint = urls[0]?.port ?? d.ports[0];
1992
+ throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
1993
+ }
1994
+ if (d.kind === "exited") throw await fail("exited before it listened on a port");
1995
+ if (d.kind === "timeout") {
1996
+ const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
1997
+ throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
1998
+ }
1999
+ }
2000
+ await sleep(POLL_MS, w.signal);
2001
+ }
2002
+ }
1841
2003
  //#endregion
1842
2004
  //#region src/sandbox/fake.ts
1843
2005
  /** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
@@ -1982,7 +2144,7 @@ async function createFakeSandbox(descriptor, env = {}) {
1982
2144
  },
1983
2145
  home: base,
1984
2146
  readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
1985
- getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://127.0.0.1:${port}` }),
2147
+ getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
1986
2148
  async writeFile(path, content) {
1987
2149
  const target = resolve(path);
1988
2150
  await mkdir(dirname(target), { recursive: true });
@@ -2200,34 +2362,28 @@ function egressLabel({ policy, auto }) {
2200
2362
  }
2201
2363
  //#endregion
2202
2364
  //#region src/judge/state-options.ts
2203
- const DEFAULT_STATE = {
2204
- task: true,
2205
- output: true,
2206
- files: true,
2207
- diff: true,
2208
- contents: [],
2209
- transcript: false,
2210
- turns: "last",
2211
- checks: false
2212
- };
2213
- /** Every key set, in `STATE_KEYS` order, so equal selections serialize the same. */
2214
- function resolveStateOptions(state) {
2215
- const out = {};
2216
- for (const key of STATE_KEYS$1) out[key] = state?.[key] ?? DEFAULT_STATE[key];
2217
- return out;
2365
+ /**
2366
+ * What one call turned on, every key set (`false` when off), in the record's key order, so equal
2367
+ * selections serialize the same.
2368
+ */
2369
+ function resolveStateOptions(call) {
2370
+ return {
2371
+ task: call.task ?? false,
2372
+ output: call.output ?? false,
2373
+ files: call.files ?? false,
2374
+ diff: call.diff ?? false,
2375
+ contents: call.contents ?? [],
2376
+ transcript: call.transcript ?? false,
2377
+ turns: call.turns ?? "last",
2378
+ checks: call.checks ?? false
2379
+ };
2218
2380
  }
2219
- /** `inner` over `outer`, field by field, and `state` key by key. */
2381
+ /** `inner` over `outer`, field by field. */
2220
2382
  function mergeJudge(outer, inner) {
2221
- if (!inner) return outer;
2222
- const out = {
2383
+ return inner ? {
2223
2384
  ...outer,
2224
2385
  ...inner
2225
- };
2226
- if (outer.state || inner.state) out.state = {
2227
- ...outer.state,
2228
- ...inner.state
2229
- };
2230
- return out;
2386
+ } : outer;
2231
2387
  }
2232
2388
  const KNOWN = [...[
2233
2389
  "harness",
@@ -2243,6 +2399,7 @@ const KNOWN = [...[
2243
2399
  "question",
2244
2400
  "questions",
2245
2401
  "timeout",
2402
+ "snapshot",
2246
2403
  "tags",
2247
2404
  "revision",
2248
2405
  "pin"
@@ -2257,6 +2414,7 @@ const SUGGEST = {
2257
2414
  repeats: "runs",
2258
2415
  iterations: "runs",
2259
2416
  trials: "runs",
2417
+ bestOf: "runs",
2260
2418
  secrets: "env",
2261
2419
  vars: "env",
2262
2420
  network: "egress",
@@ -2265,9 +2423,26 @@ const SUGGEST = {
2265
2423
  version: "revision",
2266
2424
  rev: "revision",
2267
2425
  pins: "pin",
2426
+ workspace: "snapshot",
2427
+ snapshots: "snapshot",
2268
2428
  passRate: "gate",
2269
2429
  threshold: "gate"
2270
2430
  };
2431
+ /**
2432
+ * `snapshot` over the levels above: the shorthand sets `when`, the object sets the fields it
2433
+ * names, and `compress` carries through either.
2434
+ */
2435
+ function mergeSnapshot(outer, inner) {
2436
+ if (inner === void 0) return outer;
2437
+ if (typeof inner !== "object") return {
2438
+ ...outer,
2439
+ when: inner
2440
+ };
2441
+ return {
2442
+ when: inner.when ?? outer.when,
2443
+ compress: inner.compress ?? outer.compress
2444
+ };
2445
+ }
2271
2446
  /** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
2272
2447
  function checkGate(value, where) {
2273
2448
  const problem = gateProblem(value, where);
@@ -2335,19 +2510,44 @@ function refuseIllegalCompare(options, where) {
2335
2510
  if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
2336
2511
  }
2337
2512
  const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
2513
+ /** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
2514
+ function checkRuns(runs, where) {
2515
+ if (typeof runs === "number" && Number.isInteger(runs) && runs >= 1) return;
2516
+ if (typeof runs === "number") throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
2517
+ if (!isObjectLike(runs)) throw new UsageError(`runs in ${where} must be a whole number of at least 1, or { bestOf: n } (got ${JSON.stringify(runs)})`);
2518
+ const extra = Object.keys(runs).find((k) => k !== "bestOf");
2519
+ if (extra !== void 0) throw new UsageError(`runs in ${where} takes { bestOf: n } only (got the key "${extra}"), e.g. runs: { bestOf: 3 }`);
2520
+ const { bestOf } = runs;
2521
+ if (!(Number.isInteger(bestOf) && bestOf >= 3 && bestOf % 2 === 1)) throw new UsageError(`runs.bestOf in ${where} must be an odd whole number of 3 or more, so one side always has a majority, e.g. runs: { bestOf: 3 } (got ${JSON.stringify(bestOf)})`);
2522
+ }
2523
+ /** The most attempts `runs` plans a cell, and its `bestOf` when it has one. */
2524
+ const runsOf = (runs) => typeof runs === "number" ? { runs } : {
2525
+ runs: runs.bestOf,
2526
+ bestOf: runs.bestOf
2527
+ };
2528
+ /**
2529
+ * A best-of cell's gate: half or more of its counted attempts pass, which is its majority once it
2530
+ * has one (an infra error counts on neither side).
2531
+ */
2532
+ const MAJORITY_GATE = { passRate: .5 };
2533
+ /** How many of a best-of cell's attempts always run: the majority. */
2534
+ const majorityOf = (bestOf) => (bestOf + 1) / 2;
2338
2535
  /** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
2339
2536
  function checkOptions(options, where, allowed = KNOWN) {
2340
2537
  for (const key of Object.keys(options)) {
2341
2538
  if (allowed.includes(key)) continue;
2342
2539
  const near = SUGGEST[key] ?? closest(key, allowed);
2343
- const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : near ? ` Did you mean "${near}"?` : "";
2540
+ const value = options[key];
2541
+ const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
2344
2542
  throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
2345
2543
  }
2346
2544
  const o = options;
2347
- if (o.runs !== void 0 && !(Number.isInteger(o.runs) && o.runs >= 1)) throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(o.runs)})`);
2545
+ if (o.runs !== void 0) checkRuns(o.runs, where);
2348
2546
  if (o.gate !== void 0) checkGate(o.gate, where);
2349
2547
  if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
2350
2548
  if (o.revision !== void 0 && !isRevision(o.revision)) throw new UsageError(`revision in ${where} must be a whole number or a non-empty string, e.g. revision: 2 (got ${JSON.stringify(o.revision)})`);
2549
+ const snapshotIssue = snapshotProblem(o.snapshot, where);
2550
+ if (snapshotIssue) throw new UsageError(snapshotIssue);
2351
2551
  if (o.env !== void 0) checkEnv(o.env, where, UsageError);
2352
2552
  if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
2353
2553
  const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
@@ -2478,17 +2678,18 @@ function checkUnique(list, where) {
2478
2678
  seen.add(key);
2479
2679
  }
2480
2680
  }
2481
- const plannedQuestions = (value) => value.map((q, i) => typeof q === "string" ? {
2482
- n: i + 1,
2483
- question: q
2484
- } : {
2485
- n: i + 1,
2486
- question: q.question,
2487
- ...q.model !== void 0 ? { model: q.model } : {},
2488
- ...q.across !== void 0 ? { across: q.across } : {},
2489
- ...q.choices ? { choices: q.choices } : {},
2490
- ...q.state ? { state: q.state } : {},
2491
- ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
2681
+ const plannedQuestions = (value) => value.map((q, i) => {
2682
+ const { question, model, across, choices, artifacts, ...rest } = q;
2683
+ const state = Object.fromEntries(Object.entries(rest).filter(([, v]) => v !== void 0));
2684
+ return {
2685
+ n: i + 1,
2686
+ question,
2687
+ ...model !== void 0 ? { model } : {},
2688
+ ...across !== void 0 ? { across } : {},
2689
+ ...choices ? { choices } : {},
2690
+ state,
2691
+ ...artifacts !== void 0 ? { artifacts } : {}
2692
+ };
2492
2693
  });
2493
2694
  /** Whether the eval's matrix varies each factor a pairwise question can vary. */
2494
2695
  const variesBy = (r) => ({
@@ -2529,9 +2730,14 @@ function applyLevel(out, options, where, egress = "own") {
2529
2730
  if (options.sandbox !== void 0) out.sandbox = options.sandbox;
2530
2731
  if (options.environment !== void 0) out.environment = options.environment;
2531
2732
  if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
2532
- if (options.runs !== void 0) out.runs = options.runs;
2733
+ if (options.runs !== void 0) {
2734
+ const { runs, bestOf } = runsOf(options.runs);
2735
+ out.runs = runs;
2736
+ out.bestOf = bestOf;
2737
+ }
2533
2738
  if (options.gate !== void 0) out.gate = options.gate;
2534
2739
  if (options.timeout !== void 0) out.timeout = options.timeout;
2740
+ out.snapshot = mergeSnapshot(out.snapshot, options.snapshot);
2535
2741
  if (options.revision !== void 0) out.revision = options.revision;
2536
2742
  Object.assign(out.env, options.env);
2537
2743
  out.judge = mergeJudge(out.judge, options.judge);
@@ -2578,11 +2784,7 @@ function cascade(base, levels, where, options = {}) {
2578
2784
  baseline: out.environment,
2579
2785
  from: "environment"
2580
2786
  };
2581
- if (out.questions.length) {
2582
- out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
2583
- const blank = out.questions.find(showsNothing);
2584
- if (blank) throw new UsageError(`questions "${blank.question}" in ${where} shows the model nothing: every state key is off and it sends no artifacts. Turn on a state key, such as state: { diff: true }, or send artifacts.`);
2585
- }
2787
+ if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
2586
2788
  checkUnique(out.harness, where);
2587
2789
  const harnessFromProfile = profile?.defaults.harness !== void 0 && !levels.some((l) => l.options.harness);
2588
2790
  const authored = variesBy(out);
@@ -2648,8 +2850,10 @@ function cascade(base, levels, where, options = {}) {
2648
2850
  }
2649
2851
  defaultRuns(out, notes, options);
2650
2852
  if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && (out.runs ?? 1) > limit.runs) {
2651
- notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
2853
+ const was = out.bestOf !== void 0 ? `best of ${out.bestOf}` : String(out.runs);
2854
+ notes.push(`runs ${was} → ${limit.runs} (${tag})`);
2652
2855
  out.runs = limit.runs;
2856
+ out.bestOf = void 0;
2653
2857
  out.sources = {
2654
2858
  ...out.sources,
2655
2859
  runs: `profiles.${profile.name}.limit`
@@ -2678,6 +2882,16 @@ function cascade(base, levels, where, options = {}) {
2678
2882
  if (narrowed) notes.push(`${plural(narrowed)} narrowed (profile ${profile.name})`);
2679
2883
  out.questions = kept;
2680
2884
  }
2885
+ if (options.runs !== void 0) out.bestOf = void 0;
2886
+ if (out.bestOf !== void 0 && out.compare?.variants.length) {
2887
+ const from = out.sources?.runs ?? "the config";
2888
+ const set = from === where ? "" : ` from ${from}`;
2889
+ throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
2890
+ }
2891
+ if (out.bestOf !== void 0) {
2892
+ if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
2893
+ out.gate = MAJORITY_GATE;
2894
+ }
2681
2895
  defaultRuns(out, notes, options);
2682
2896
  return {
2683
2897
  resolved: {
@@ -2813,6 +3027,7 @@ function expand(resolved, vertex) {
2813
3027
  egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers, onVertex),
2814
3028
  gate,
2815
3029
  ...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
3030
+ ...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
2816
3031
  n,
2817
3032
  label,
2818
3033
  ...onVertex ? { vertex: onVertex } : {}
@@ -2941,6 +3156,7 @@ function evalPlan(r, attempts, origin) {
2941
3156
  } } : {},
2942
3157
  ...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
2943
3158
  runs: resolved.runs,
3159
+ ...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
2944
3160
  ...resolved.gate !== void 0 ? { gate: resolved.gate } : {},
2945
3161
  attempts: skip ? 0 : attempts.length,
2946
3162
  notes,
@@ -2964,7 +3180,7 @@ function evalPlan(r, attempts, origin) {
2964
3180
  /** A pairwise question that shows each side's transcript, asked of two harnesses: tool names unblind it. */
2965
3181
  function questionWarnings(resolved) {
2966
3182
  if (resolved.harness.length < 2) return [];
2967
- return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows state.transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
3183
+ return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows the transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
2968
3184
  }
2969
3185
  function captureNote(c) {
2970
3186
  const parts = [];
@@ -3157,6 +3373,7 @@ const CONFIG_KEYS = [
3157
3373
  "capture",
3158
3374
  "question",
3159
3375
  "timeout",
3376
+ "snapshot",
3160
3377
  "concurrency",
3161
3378
  "record",
3162
3379
  "history",
@@ -3187,7 +3404,8 @@ const DEFAULTS_KEYS = [
3187
3404
  "judge",
3188
3405
  "capture",
3189
3406
  "question",
3190
- "timeout"
3407
+ "timeout",
3408
+ "snapshot"
3191
3409
  ];
3192
3410
  const LIMIT_KEYS = [
3193
3411
  "runs",
@@ -3543,7 +3761,7 @@ function selectProfile(config, flag, env = process.env) {
3543
3761
  function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3544
3762
  const user = withHostPaths(input, root, file);
3545
3763
  if ("compare" in user) throw new UsageError(`compare can't be set in ${file}: it would multiply every eval. Put it on the group or eval that asks the question.`);
3546
- if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model, state, artifacts } in ${file} sets every question's defaults.`);
3764
+ if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model } in ${file} sets every question's model.`);
3547
3765
  checkOptions(user, file, CONFIG_KEYS);
3548
3766
  const include = user.include ?? defaultIncludeGlobs();
3549
3767
  if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
@@ -3588,6 +3806,10 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3588
3806
  capture: user.capture ?? {},
3589
3807
  question: user.question ?? {},
3590
3808
  timeoutMs: parseDuration(user.timeout ?? "15m"),
3809
+ snapshot: mergeSnapshot({
3810
+ when: true,
3811
+ compress: false
3812
+ }, user.snapshot),
3591
3813
  concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
3592
3814
  record,
3593
3815
  history: {
@@ -3772,6 +3994,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
3772
3994
  });
3773
3995
  }
3774
3996
  //#endregion
3775
- export { isDirect as $, harnessId as A, badJudgeArtifacts as At, sandboxSetBy as B, routedHarness as C, captureWindowSchema as Ct, checkOptions as D, judgeProblem as Dt, checkLimitHarness as E, judgeModelNameSchema as Et, plannedSandboxes as F, resolveStateOptions as G, unbrokered as H, profileJudge as I, allowedHosts as J, DEFAULT_EGRESS as K, refuseIllegalCompare as L, matchesTags as M, filePartsOf as Mt, plainHeaderError as N, resolveJudgeArtifacts as Nt, evalPlan as O, judgeStateSchema as Ot, plainHeaders as P, gatewayPolicy as Q, sandboxId as R, resolveGateway as S, writePluginFiles as St, cascade as T, judgeFields as Tt, unbrokeredError as U, secretVarsIn as V, mergeJudge as W, directOnCloud as X, ciRunner as Y, egressLabel as Z, fellBackToDirect as _, expectedBlockedFor as _t, configPathFor as a, questionShows as at, resolveModelGateway as b, refusedMcp as bt, findConfig as c, resolvedEffortOf as ct, malformedCredentials as d, agentFor as dt, unenforceable as et, missingCredentials as f, agentMcp as ft, selectProfile as g, credentialOf as gt, runJudgeModel as h, claudeMcpServers as ht, configLabel as i, STOPPED_FIRST as it, levelOf as j, decideState as jt, expand as k, artifactRefsOf as kt, hostCapacity as l, servedEffortOf as lt, resolveConcurrency as m, brokeringFor as mt, analysisModelOf as n, createFakeSandbox as nt, defaultConcurrency as o, resolveQuestionState as ot, requireConfig as p, authHint as pt, DEFAULT_EGRESS_ALLOW as q, configFilesPhrase as r, DEFAULT_QUESTION_STATE as rt, defineConfig as s, harnessEffort as st, CONFIG_FILES as t, resolveOutput as tt, loadConfig as u, agentAuth as ut, gatewayScope as v, isAgentKind as vt, brokeredIn as w, fakeJudgeDataSchema as wt, fallbackHint as x, refusedPlugins as xt, modelGatewayOrNone as y, layoutOf as yt, sandboxProvider as z };
3997
+ export { egressLabel as $, harnessId as A, NO_JUDGE_INPUT as At, sandboxId as B, decideState as Bt, routedHarness as C, isAgentKind as Ct, checkOptions as D, writePluginFiles as Dt, checkLimitHarness as E, refusedPlugins as Et, plainHeaders as F, judgeProblem as Ft, unbrokeredError as G, sandboxSetBy as H, resolveJudgeArtifacts as Ht, plannedSandboxes as I, sharedInputFields as It, DEFAULT_EGRESS as J, mergeJudge as K, profileJudge as L, turnsSchema as Lt, majorityOf as M, inputsOn as Mt, matchesTags as N, judgeFields as Nt, evalPlan as O, captureWindowSchema as Ot, plainHeaderError as P, judgeModelNameSchema as Pt, directOnCloud as Q, refuseIllegalCompare as R, artifactRefsOf as Rt, resolveGateway as S, expectedBlockedFor as St, cascade as T, refusedMcp as Tt, secretVarsIn as U, sandboxProvider as V, filePartsOf as Vt, unbrokered as W, allowedHosts as X, DEFAULT_EGRESS_ALLOW as Y, ciRunner as Z, fellBackToDirect as _, agentMcp as _t, configPathFor as a, awaitListening as at, resolveModelGateway as b, claudeMcpServers as bt, findConfig as c, STOPPED_FIRST as ct, malformedCredentials as d, resolveQuestionState as dt, gatewayPolicy as et, missingCredentials as f, harnessEffort as ft, selectProfile as g, agentFor as gt, runJudgeModel as h, agentAuth as ht, configLabel as i, createFakeSandbox as it, levelOf as j, fakeJudgeDataSchema as jt, expand as k, INPUT_KEYS as kt, hostCapacity as l, UNRECORDED_QUESTION_STATE as lt, resolveConcurrency as m, servedEffortOf as mt, analysisModelOf as n, unenforceable as nt, defaultConcurrency as o, parseProbe as ot, requireConfig as p, resolvedEffortOf as pt, resolveStateOptions as q, configFilesPhrase as r, resolveOutput as rt, defineConfig as s, probeScript as st, CONFIG_FILES as t, isDirect as tt, loadConfig as u, questionShows as ut, gatewayScope as v, authHint as vt, brokeredIn as w, layoutOf as wt, fallbackHint as x, credentialOf as xt, modelGatewayOrNone as y, brokeringFor as yt, runsOf as z, badJudgeArtifacts as zt };
3776
3998
 
3777
- //# sourceMappingURL=config-k6m7XbAz.js.map
3999
+ //# sourceMappingURL=config-CneBZB84.js.map