reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -1,29 +1,31 @@
1
1
  import { t as isObjectLike } from "./object-CLf2-tNq.js";
2
+ import { r as withLease } from "./client-Ct6s6OnF.js";
2
3
  import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
3
- import { n as dataProblem, p as parseDuration } from "./shared-CA61BuPJ.js";
4
- import { A as resolveJudgeArtifacts, D as badJudgeArtifacts, E as artifactRefsOf, O as decideState, T as judgeProblem, _ as unbrokered, a as evalPlan, b as resolveStateOptions, f as refuseIllegalCompare, g as secretVarsIn, h as sandboxSetBy, i as checkOptions, k as filePartsOf, l as matchesTags, m as sandboxProvider, n as cascade, o as expand, p as sandboxId, s as harnessId, t as brokeredIn, v as unbrokeredError, y as mergeJudge } from "./options-CyAyLjgJ.js";
5
- import { N as workspaceOf, O as sandboxPaths, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-B4IuURK1.js";
4
+ import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
5
+ import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-BOFgzFDg.js";
6
+ import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-D-CP6mMd.js";
6
7
  import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
7
- import { s as redactSecrets } from "./redact-CRGdfV2s.js";
8
- import { c as judgeModelName, d as runJudge, f as isApiError } from "./judge-CEDIfRiU.js";
9
- import "./config-D7TFvUpL.js";
10
- import { t as inCI } from "./env-BQqj93gu.js";
11
- import { dt as shellQuote, t as MCP_TIMER } from "./src-CIpKL45U.js";
12
- import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-P-uh54VF.js";
8
+ import { $ as isDirect, A as harnessId, At as badJudgeArtifacts, B as sandboxSetBy, D as checkOptions, Dt as judgeProblem, G as resolveStateOptions, H as unbrokered, J as allowedHosts, L as refuseIllegalCompare, M as matchesTags, Mt as filePartsOf, N as plainHeaderError, Nt as resolveJudgeArtifacts, O as evalPlan, P as plainHeaders, R as sandboxId, S as resolveGateway, St as writePluginFiles, T as cascade, U as unbrokeredError, V as secretVarsIn, W as mergeJudge, Z as egressLabel, _ as fellBackToDirect, _t as expectedBlockedFor, b as resolveModelGateway, bt as refusedMcp, et as unenforceable, ft as agentMcp, ht as claudeMcpServers, jt as decideState, k as expand, kt as artifactRefsOf, mt as brokeringFor, nt as createFakeSandbox, v as gatewayScope, w as brokeredIn, xt as refusedPlugins, yt as layoutOf, z as sandboxProvider } from "./config-k6m7XbAz.js";
9
+ import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-CKdgEXN7.js";
10
+ import { v as vertexSettings } from "./vertex-DypYjx71.js";
11
+ import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
12
+ import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
13
+ import { t as inCI } from "./env-CARj5Axq.js";
14
+ import { ft as shellQuote, t as MCP_TIMER } from "./src-D91Hd6hT.js";
13
15
  import { t as InfraError } from "./types-C5vI3g1C.js";
14
- import { C as writePluginFiles, S as refusedPlugins, b as layoutOf, g as claudeMcpServers, h as brokeringFor, n as allowedHosts, o as unenforceable, p as agentMcp, r as egressLabel, s as createFakeSandbox, v as expectedBlockedFor, x as refusedMcp } from "./egress-DEAvq8Fl.js";
15
- import { i as registerTest, n as getRunContext, o as SUBJECT, r as registerGroup, t as collectingFile } from "./runtime-PX_oXiCS.js";
16
- import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-jI0OWxlz.js";
16
+ import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-d56Gg3J5.js";
17
+ import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-Ba2QerOK.js";
17
18
  import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
18
- import { i as formatDuration, n as clip, t as capMiddle } from "./text-CrVy5ioL.js";
19
- import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-dn2hjSLj.js";
19
+ import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
20
+ import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-BM_3100g.js";
20
21
  import { t as contextTaker } from "./context-Dgn48BqT.js";
21
- import { a as skillFiles, i as resolveEnvironment } from "./resolve-DzdWcIL6.js";
22
+ import { a as skillFiles, i as resolveEnvironment } from "./resolve-DWSJpKHX.js";
22
23
  import { t as PrepareError } from "./provider-CmAYTQQs.js";
23
- import { i as preparedWorkspace } from "./workspace-CLeOybkr.js";
24
- import { n as judgeDiff } from "./patch-uMq9rCl_.js";
25
- import { t as transcriptForJudge } from "./transcript-BTxJpW4R.js";
26
- import { r as withLease } from "./client-fcwgwOAD.js";
24
+ import { i as preparedWorkspace } from "./workspace-BRxjfa8N.js";
25
+ import { t as patchPath } from "./patch-CquGAGX5.js";
26
+ import { n as judgeDiff } from "./patch-DKJiC5Sq.js";
27
+ import { t as transcriptForJudge } from "./transcript-DxPaWHTZ.js";
28
+ import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-Dje3rERU.js";
27
29
  import { dirname, join } from "node:path";
28
30
  import { parse, stringify } from "yaml";
29
31
  import { z } from "zod";
@@ -32,7 +34,11 @@ import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
32
34
  //#region src/attempt/assertions.ts
33
35
  const MESSAGE_LIMIT = 1024;
34
36
  const LABEL_LIMIT = 60;
35
- const clipped = (s, limit) => s.length > limit ? `${s.slice(0, limit)}…` : s;
37
+ /** Masked first, so a secret the limit would cut through is masked whole. */
38
+ const clipped = (text, limit) => {
39
+ const s = redactSecrets(text);
40
+ return s.length > limit ? `${s.slice(0, limit)}…` : s;
41
+ };
36
42
  const text = (error) => error instanceof Error ? error.message : String(error);
37
43
  const shown = (value) => {
38
44
  if (typeof value === "string") return value;
@@ -46,17 +52,26 @@ const shown = (value) => {
46
52
  const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
47
53
  /**
48
54
  * The records of an attempt's matcher calls, in call order. Without a label, one is
49
- * `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`.
55
+ * `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
56
+ * any label already recorded, so each record's label is unique (a given `build #2` after two
57
+ * `build`s is `build #2 #2`).
50
58
  */
51
59
  function assertionRecords(raw) {
52
60
  const seen = /* @__PURE__ */ new Map();
61
+ const used = /* @__PURE__ */ new Set();
53
62
  return raw.map((r) => {
54
63
  const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
55
64
  const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
56
- const n = (seen.get(base) ?? 0) + 1;
65
+ let n = seen.get(base) ?? 0;
66
+ let label;
67
+ do {
68
+ n += 1;
69
+ label = n > 1 ? `${base} #${n}` : base;
70
+ } while (used.has(label));
57
71
  seen.set(base, n);
72
+ used.add(label);
58
73
  return {
59
- label: n > 1 ? `${base} #${n}` : base,
74
+ label,
60
75
  labelled: r.label !== void 0,
61
76
  matcher: r.matcher,
62
77
  status: r.status,
@@ -66,11 +81,22 @@ function assertionRecords(raw) {
66
81
  };
67
82
  });
68
83
  }
69
- /** What fails the attempt at its end for its `expect.soft` calls: the first that didn't pass. */
70
- function softFailure(raw) {
71
- const failed = raw.find((r) => r.soft && r.status !== "pass");
84
+ const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
85
+ /**
86
+ * What fails the attempt at its end for the checks that failed without ending the body: the first
87
+ * that didn't pass, an `expect.soft` as what it threw, a hard check the eval caught with
88
+ * `CAUGHT_HINT` after its message.
89
+ */
90
+ function checkFailure(raw) {
91
+ const failed = raw.find((r) => r.status !== "pass");
72
92
  if (!failed) return void 0;
73
- return failed.error instanceof Error ? failed.error : new Error(text(failed.error));
93
+ const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
94
+ if (failed.soft) return error;
95
+ const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
96
+ caught.name = error.name;
97
+ caught.message = `${error.message} ${CAUGHT_HINT}`;
98
+ caught.stack = error.stack?.replace(error.message, caught.message);
99
+ return caught;
74
100
  }
75
101
  //#endregion
76
102
  //#region src/attempt/debug.ts
@@ -86,7 +112,7 @@ function tracePlan(debug, spec, ctx) {
86
112
  const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
87
113
  const envFrom = spec.cell.role === "variant" ? from("compare") : from("environment", "none set");
88
114
  debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
89
- debug.log("plan", `egress ${egressLabel(spec.egress) || "allow-all"} ${from("egress")}`);
115
+ debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
90
116
  const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
91
117
  debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
92
118
  const judge = spec.judge;
@@ -122,7 +148,7 @@ function traceEnvironment(debug, spec, env) {
122
148
  const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
123
149
  debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
124
150
  for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
125
- const brokering = brokeringFor(spec.harness);
151
+ const brokering = brokeringFor(spec.harness, spec.vertex);
126
152
  if (brokering) {
127
153
  const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
128
154
  debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
@@ -408,10 +434,10 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
408
434
  }
409
435
  //#endregion
410
436
  //#region src/harness/index.ts
411
- async function createHarness(descriptor) {
437
+ async function createHarness(descriptor, vertex) {
412
438
  if (descriptor.kind === "fake") return createFakeHarness(descriptor);
413
- const { createAgentHarness } = await import("./agent-ClE-g9VL.js");
414
- return createAgentHarness(descriptor);
439
+ const { createAgentHarness } = await import("./agent-Zgn-kiqq.js");
440
+ return createAgentHarness(descriptor, vertex);
415
441
  }
416
442
  /** Where each harness's agent looks for repo skills, relative to the workspace. */
417
443
  const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
@@ -430,9 +456,9 @@ async function prepareEnvironment(a, root, runId, where, signal) {
430
456
  });
431
457
  a.env = env;
432
458
  record.environment = env.record;
433
- const { secrets, brokered } = env.record;
459
+ const { vars, secrets, brokered } = env.record;
434
460
  record.env = {
435
- vars: Object.keys(env.vars).filter((n) => !secrets.includes(n)).sort(),
461
+ vars,
436
462
  secrets,
437
463
  brokered
438
464
  };
@@ -440,8 +466,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
440
466
  if (debug) traceEnvironment(debug, spec, env);
441
467
  record.image = null;
442
468
  if (spec.sandbox.kind === "docker") {
443
- const { preparedFor } = await import("./preflight-BQlPTHVu.js");
444
- const { attemptImage, dockerProvider, preparedImage } = await import("./providers-CTBYvMxu.js");
469
+ const { preparedFor } = await import("./preflight-DTiVv0gY.js");
470
+ const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DkOupEBl.js");
445
471
  signal.throwIfAborted();
446
472
  const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
447
473
  const builds = builtPlugins(spec.environment.layers);
@@ -474,7 +500,7 @@ const CHARS_PER_TOKEN = 4;
474
500
  const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
475
501
  const k = (tokens) => tokens < 1e3 ? String(tokens) : `${Math.round(tokens / 1e3)}k`;
476
502
  const diffItems = (patch) => patch.split(/(?=^diff --git )/m).map((part) => ({
477
- name: /^diff --git a\/.+ b\/(.+)$/m.exec(part)?.[1] ?? "",
503
+ name: patchPath(part) ?? "",
478
504
  chars: part.length
479
505
  })).filter((item) => item.name);
480
506
  /** The state's parts by size, each with its biggest files where it has any. */
@@ -617,13 +643,21 @@ function checksShown(which, ran) {
617
643
  }));
618
644
  }
619
645
  /**
646
+ * Where `order` goes among `calls`, which are in ascending order: calls finish in any order, but
647
+ * what they record stays in the order they were made.
648
+ */
649
+ function placeFor(calls, order) {
650
+ const after = calls.findIndex((c) => c > order);
651
+ return after < 0 ? calls.length : after;
652
+ }
653
+ /**
620
654
  * Grades a turn (the last by default) or a value against a question, under
621
655
  * the judge provider's concurrency lease. Every call's input is recorded, including calls that
622
656
  * fail. A call that fails returns a `JudgeFailure` and fails the attempt as a judge error after the eval's body, so the judges and
623
657
  * checks after it still run.
624
658
  */
625
659
  function judgeCaller(a, readFile) {
626
- const ask = async (question, { on, ...call } = {}) => {
660
+ const ask = async (order, question, { on, ...call } = {}) => {
627
661
  if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
628
662
  const problem = judgeProblem(call, { at: "call" });
629
663
  if (problem) throw new Error(problem);
@@ -634,8 +668,9 @@ function judgeCaller(a, readFile) {
634
668
  const options = mergeJudge(a.spec.judge, call);
635
669
  const { model } = options;
636
670
  if (model === void 0) {
637
- a.judgeErrored = true;
638
- throw new Error(noJudgeModel(getRunContext().config.file));
671
+ const message = noJudgeModel(getRunContext().config.file);
672
+ a.judgeThrown.add(message);
673
+ throw new Error(message);
639
674
  }
640
675
  const wanted = resolveStateOptions(options.state);
641
676
  if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
@@ -651,7 +686,7 @@ function judgeCaller(a, readFile) {
651
686
  listFiles: async () => workspaceFiles(await sandboxFor(a, "c.judge()"), a.signal)
652
687
  });
653
688
  const name = judgeModelName(model);
654
- const state = stateOf(draft);
689
+ const state = redactValue(stateOf(draft));
655
690
  const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
656
691
  const refs = artifactRefsOf(resolved);
657
692
  const decide = decideState(state, filePartsOf(resolved));
@@ -663,7 +698,9 @@ function judgeCaller(a, readFile) {
663
698
  ...refs.length ? { artifacts: refs } : {},
664
699
  ...dropped.length ? { dropped } : {}
665
700
  };
666
- a.record.judgeInputs?.push(input);
701
+ const place = placeFor(a.judgeInputCalls, order);
702
+ a.judgeInputCalls.splice(place, 0, order);
703
+ a.record.judgeInputs?.splice(place, 0, input);
667
704
  const start = performance.now();
668
705
  const ms = () => Math.round(performance.now() - start);
669
706
  const atLeast = options.threshold ?? .8;
@@ -671,12 +708,14 @@ function judgeCaller(a, readFile) {
671
708
  kind: "judge-start",
672
709
  question
673
710
  });
674
- debugLog("judge", `asking ${name}: ${JSON.stringify(clip(question, 160))}`);
711
+ debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
675
712
  if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
676
713
  debugTrace("judge", "prompt", input, "judge prompt");
677
714
  const failed = (message, retries) => {
678
- a.judgeErrored = true;
679
- a.judgeErrors.push(`c.judge(${JSON.stringify(clip(question, 80))}) failed: ${message}`);
715
+ a.judgeErrors.push({
716
+ call: order,
717
+ message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`
718
+ });
680
719
  input.error = message;
681
720
  if (retries?.length) input.retries = retries;
682
721
  debugLog("judge", `${name} · ${formatDuration(ms())} · error: ${message}`);
@@ -704,8 +743,17 @@ function judgeCaller(a, readFile) {
704
743
  };
705
744
  const provider = name.split("/")[0];
706
745
  try {
746
+ const { config, profile } = getRunContext();
747
+ const scope = gatewayScope(config, profile);
748
+ const gateway = resolveModelGateway(model, judgeOptions.gateway, scope, "judge model");
749
+ const { gateway: _pin, ...unpinned } = judgeOptions;
750
+ const entry = gateway && scope.gateways.find((g) => g.kind === gateway);
751
+ const routed = gateway ? {
752
+ ...unpinned,
753
+ gateway: entry || gateway
754
+ } : unpinned;
707
755
  const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
708
- ...judgeOptions,
756
+ ...routed,
709
757
  model,
710
758
  threshold: atLeast
711
759
  }, a.signal), a.signal));
@@ -713,10 +761,12 @@ function judgeCaller(a, readFile) {
713
761
  ...verdict,
714
762
  artifacts: refs
715
763
  } : verdict;
716
- a.record.judgements.push(retries.length ? {
764
+ const { judgements: judgments } = a.record;
765
+ judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
717
766
  ...recorded,
718
- retries
719
- } : recorded);
767
+ ...retries.length ? { retries } : {},
768
+ call: order
769
+ });
720
770
  const { score } = verdict;
721
771
  debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
722
772
  debugTrace("judge", "response", {
@@ -733,21 +783,19 @@ function judgeCaller(a, readFile) {
733
783
  });
734
784
  return recorded;
735
785
  } catch (error) {
736
- if (a.signal.aborted) {
737
- a.judgeErrored = true;
738
- throw error;
739
- }
786
+ if (a.signal.aborted) throw error;
740
787
  const { retries } = error;
741
788
  const message = error instanceof Error ? error.message : String(error);
742
789
  return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
743
790
  }
744
791
  };
745
792
  return async (question, { on, ...options } = {}) => {
793
+ const order = a.judgeCalls++;
746
794
  const call = on === void 0 ? options : {
747
795
  ...options,
748
796
  on: judgeOn(a.turns, on)
749
797
  };
750
- return Object.defineProperty(await ask(question, call), SUBJECT, { value: question });
798
+ return Object.defineProperty(await ask(order, question, call), SUBJECT, { value: question });
751
799
  };
752
800
  }
753
801
  //#endregion
@@ -757,7 +805,7 @@ async function createSandbox(descriptor, options) {
757
805
  switch (descriptor.kind) {
758
806
  case "fake": return await createFakeSandbox(descriptor, options.env);
759
807
  case "docker": {
760
- const { createDockerSandbox } = await import("./docker-Dac6Zalz.js");
808
+ const { createDockerSandbox } = await import("./docker-YFj4H1Cr.js");
761
809
  return await createDockerSandbox(descriptor, options);
762
810
  }
763
811
  }
@@ -776,7 +824,7 @@ function classify(error, signal) {
776
824
  message: "The eval timed out"
777
825
  } : {
778
826
  kind: "aborted",
779
- message: "The run was cancelled"
827
+ message: "The run was canceled"
780
828
  };
781
829
  if (error instanceof InfraError) return {
782
830
  kind: "infra",
@@ -787,17 +835,39 @@ function classify(error, signal) {
787
835
  message
788
836
  };
789
837
  }
838
+ const messageOf = (error) => error instanceof Error ? error.message : String(error);
839
+ const judged = (r) => /(^|\.)toPass$/.test(r.matcher);
840
+ /**
841
+ * Whether `error` is a judge's: one a judge threw, or what a `toPass()` or `.not.toPass()` call
842
+ * failed with. A failing verdict the eval didn't assert, or asserted with `.not.toPass()`, isn't.
843
+ * By message too, since another language's runner hands back its own copy of the error.
844
+ */
845
+ function isJudges(a, error, raw) {
846
+ const message = messageOf(error);
847
+ if (a.judgeThrown.has(message)) return true;
848
+ return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
849
+ }
850
+ /**
851
+ * The error that fails the attempt: a judge's gives way to the first other check that didn't hold
852
+ * (an `expect.soft`, or a hard check the eval caught), whichever came first, so the attempt fails
853
+ * as that check.
854
+ */
855
+ function attemptError(a, error, raw) {
856
+ if (!isJudges(a, error, raw)) return error;
857
+ const check = raw.find((r) => r.status !== "pass" && !judged(r));
858
+ return check ? checkFailure([check]) : error;
859
+ }
790
860
  /**
791
861
  * Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
792
- * assertion, which is the judge's when a judge call failed or gave a failing verdict.
862
+ * assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
793
863
  */
794
- function recordFailure(a, error, timedOut) {
864
+ function recordFailure(a, error, timedOut, raw) {
795
865
  const { record } = a;
796
866
  record.error = timedOut ? {
797
867
  kind: "timeout",
798
868
  message: "The eval timed out"
799
869
  } : classify(error, a.signal);
800
- record.error.message = capMiddle(record.error.message);
870
+ record.error.message = capMiddle(redactSecrets(record.error.message));
801
871
  if (error instanceof PrepareError && !timedOut) {
802
872
  const { class: kind, stage, ref, cause, log, setBy } = error.failure;
803
873
  record.error.prepare = {
@@ -809,8 +879,7 @@ function recordFailure(a, error, timedOut) {
809
879
  setBy: setBy ?? []
810
880
  };
811
881
  }
812
- const judgeFailed = a.judgeErrored || record.judgements.some((j) => j.verdict !== "pass");
813
- if (record.error.kind === "assertion" && judgeFailed) record.error.kind = "judge";
882
+ if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = "judge";
814
883
  record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
815
884
  }
816
885
  /**
@@ -837,11 +906,11 @@ async function recordOom(a, sandbox) {
837
906
  /**
838
907
  * The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
839
908
  * Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
840
- * summarised in one `-v` line.
909
+ * summarized in one `-v` line.
841
910
  */
842
911
  function recordEgress(a, sandbox) {
843
912
  const traffic = sandbox?.traffic?.();
844
- const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0);
913
+ const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
845
914
  const refused = traffic?.blocked ?? [];
846
915
  const isExpected = (h) => !!expected?.hosts.includes(h.host);
847
916
  const expectedBlocked = refused.filter(isExpected);
@@ -849,6 +918,7 @@ function recordEgress(a, sandbox) {
849
918
  policy: a.spec.egress.policy,
850
919
  allowed: allowedHosts(a.spec.egress),
851
920
  recorded: !!traffic,
921
+ ...traffic && isDirect(a.spec.egress.policy) ? { partial: true } : {},
852
922
  contacted: traffic?.contacted ?? [],
853
923
  blocked: refused.filter((h) => !isExpected(h)),
854
924
  expectedBlocked
@@ -858,6 +928,26 @@ function recordEgress(a, sandbox) {
858
928
  debugLog("gateway", `expected blocks, not logged per request: ${hosts} · ${expected.why}`);
859
929
  }
860
930
  a.record.servedModels = sandbox?.servedModels?.() ?? null;
931
+ const asked = sandbox?.askedModels?.() ?? [];
932
+ if (asked.length) a.record.askedModels = asked;
933
+ const upstreams = sandbox?.upstreams?.() ?? [];
934
+ if (upstreams.length) a.record.upstreams = upstreams;
935
+ const cost = sandbox?.reportedCostUsd?.();
936
+ if (cost !== void 0) a.record.reportedCostUsd = cost;
937
+ }
938
+ /**
939
+ * The record as its staging file holds it: redacted, with a secret split across a turn's raw
940
+ * stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
941
+ */
942
+ function stagedRecordJson(record) {
943
+ const turns = record.turns.map((turn) => turn.raw ? {
944
+ ...turn,
945
+ raw: redactSplitSecrets(turn.raw.map((part) => JSON.stringify(part))).map((line) => JSON.parse(line))
946
+ } : turn);
947
+ return redactJson({
948
+ ...record,
949
+ turns
950
+ }, 2);
861
951
  }
862
952
  /** Writes the attempt's record, redacted, and returns what the runner reports. */
863
953
  async function writeRecord(a, stagingDir, id) {
@@ -866,7 +956,7 @@ async function writeRecord(a, stagingDir, id) {
866
956
  await mkdir(join(stagingDir, "evals"), { recursive: true });
867
957
  await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
868
958
  const tmp = `${recordPath}.${process.pid}.tmp`;
869
- await writeFile(tmp, redactSecrets(JSON.stringify(record, null, 2)));
959
+ await writeFile(tmp, stagedRecordJson(record));
870
960
  await rename(tmp, recordPath);
871
961
  return {
872
962
  outcome: record.outcome,
@@ -888,7 +978,7 @@ async function writeRecord(a, stagingDir, id) {
888
978
  function turnErrors(transcripts) {
889
979
  return transcripts.flatMap((events, i) => {
890
980
  const found = events.find((e) => e.type === "error");
891
- return found?.type === "error" ? [`turn ${i + 1} error: ${redactSecrets(clip(found.message.trim(), 500))}`] : [];
981
+ return found?.type === "error" ? [`turn ${i + 1} error: ${clip(redactSecrets(found.message.trim()), 500)}`] : [];
892
982
  });
893
983
  }
894
984
  //#endregion
@@ -920,7 +1010,7 @@ function hookHandlers(plugin, text, hooks) {
920
1010
  const what = `the ${plugin.name} plugin's hooks/hooks.json`;
921
1011
  let raw;
922
1012
  try {
923
- raw = JSON.parse(text);
1013
+ raw = JSON.parse(withoutBom(text));
924
1014
  } catch {
925
1015
  throw new InfraError(`${what} isn't valid JSON`);
926
1016
  }
@@ -981,7 +1071,7 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
981
1071
  if (!hooks) {
982
1072
  let own = {};
983
1073
  try {
984
- own = hooksFile === null ? {} : JSON.parse(hooksFile);
1074
+ own = hooksFile === null ? {} : JSON.parse(withoutBom(hooksFile));
985
1075
  } catch {
986
1076
  throw new InfraError(`The workspace's ${unpack.hooks.file} isn't valid JSON, so reeval can't add the plugins' hooks to it`);
987
1077
  }
@@ -1149,8 +1239,10 @@ function createAttempt(spec, signal, progress, start) {
1149
1239
  }
1150
1240
  },
1151
1241
  signal,
1152
- judgeErrored: false,
1242
+ judgeThrown: /* @__PURE__ */ new Set(),
1153
1243
  judgeErrors: [],
1244
+ judgeCalls: 0,
1245
+ judgeInputCalls: [],
1154
1246
  captureNames: /* @__PURE__ */ new Set(),
1155
1247
  backgrounds: [],
1156
1248
  artifactBlobs: /* @__PURE__ */ new Map()
@@ -1198,7 +1290,7 @@ async function runSetup(a, box, env) {
1198
1290
  const steps = env.setup;
1199
1291
  record.setup = [];
1200
1292
  for (const [i, step] of steps.entries()) {
1201
- const what = `setup step ${i + 1}/${steps.length} "${clip(step.run, 80)}" in environment ${spec.environment.name}`;
1293
+ const what = `setup step ${i + 1}/${steps.length} "${clip(redactSecrets(step.run), 80)}" in environment ${spec.environment.name}`;
1202
1294
  const start = performance.now();
1203
1295
  const stepSignal = AbortSignal.any([a.signal, AbortSignal.timeout(step.timeoutMs)]);
1204
1296
  const r = await asCheck(a, () => box.exec(step.run, {
@@ -1218,7 +1310,7 @@ async function runSetup(a, box, env) {
1218
1310
  durationMs,
1219
1311
  output
1220
1312
  });
1221
- debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(step.run, 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
1313
+ debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(redactSecrets(step.run), 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
1222
1314
  debugTrace("env", "setup", {
1223
1315
  run: step.run,
1224
1316
  cwd: step.cwd,
@@ -1231,6 +1323,54 @@ async function runSetup(a, box, env) {
1231
1323
  throw new InfraError(r ? `${what} exited ${r.exitCode} after ${formatDuration(durationMs)} (see setup.log):\n${tailOf(r.stderr || r.stdout)}` : `${what} timed out after ${formatDuration(step.timeoutMs)} (see setup.log)`);
1232
1324
  }
1233
1325
  }
1326
+ /** The `to: 'checks'` secrets the setup steps got as their real value: those without `hosts`. */
1327
+ function unbrokeredChecks(env) {
1328
+ const brokered = new Set(env.substitutions.filter((s) => s.phase === "checks").map((s) => s.name));
1329
+ return Object.keys(env.checkVars).filter((name) => !brokered.has(name));
1330
+ }
1331
+ /** How many of the processes setup left running the record keeps, and how much of each command line. */
1332
+ const SETUP_PROCESSES = 20;
1333
+ const SETUP_COMMAND_CHARS = 200;
1334
+ /**
1335
+ * The processes the setup steps left running with an unbrokered `to: 'checks'` secret in their
1336
+ * environment, by the secret's variable name: a copy under another name or in a process's
1337
+ * arguments isn't found. The listing runs as the agent's user, with no checks secrets of its own,
1338
+ * so it sees the environments the agent could read, and prints names, never values. Docker only:
1339
+ * the fake sandbox runs on the host, with no separation to report on.
1340
+ */
1341
+ async function leftRunning(a, box, env) {
1342
+ const names = unbrokeredChecks(env);
1343
+ if (a.spec.sandbox.kind !== "docker" || !env.setup.length || !names.length) return [];
1344
+ const r = await box.exec([
1345
+ "for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
1346
+ "for p in /proc/[0-9]*; do",
1347
+ " [ \"$p\" = \"/proc/$$\" ] && continue",
1348
+ ` n=$(tr '\\0' '\\n' < "$p/environ" 2>/dev/null | sed -n -E 's/^(${names.join("|")})=.*/\\1/p' | sort -u | tr '\\n' ' ')`,
1349
+ " [ -n \"$n\" ] || continue",
1350
+ ` c=$(tr '\\0\\t\\n' ' ' < "$p/cmdline" 2>/dev/null)`,
1351
+ ` [ -n "$c" ] || c=$(tr '\\t\\n' ' ' < "$p/comm" 2>/dev/null)`,
1352
+ ` printf '%s\\t%s\\n' "$n" "$c"`,
1353
+ "done",
1354
+ "true"
1355
+ ].join("\n"), { signal: a.signal });
1356
+ if (r.exitCode !== 0) {
1357
+ debugLog("env", `couldn't list the processes setup left running: ${tailOf(r.stderr)}`);
1358
+ return [];
1359
+ }
1360
+ const known = new Set(names);
1361
+ const found = r.stdout.split("\n").flatMap((line) => {
1362
+ const tab = line.indexOf(" ");
1363
+ if (tab < 0) return [];
1364
+ const secrets = line.slice(0, tab).split(" ").filter((n) => known.has(n));
1365
+ const command = clip(redactSecrets(line.slice(tab + 1).trim()), SETUP_COMMAND_CHARS);
1366
+ return secrets.length ? [{
1367
+ command,
1368
+ secrets
1369
+ }] : [];
1370
+ });
1371
+ if (found.length > SETUP_PROCESSES) debugLog("env", `setup left ${found.length} processes running with a checks secret; the record keeps the first ${SETUP_PROCESSES}`);
1372
+ return found.slice(0, SETUP_PROCESSES);
1373
+ }
1234
1374
  const WORKSPACE_TAR = ".reeval-workspace.tar";
1235
1375
  /** Unpacks the git workspace's tarball into the (empty) workspace. */
1236
1376
  async function copyWorkspace(box, workspace, signal) {
@@ -1426,8 +1566,8 @@ async function startSession(a, runId) {
1426
1566
  signal: a.signal,
1427
1567
  env: env.vars,
1428
1568
  image: a.image,
1429
- brokering: brokeringFor(spec.harness),
1430
- expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0)?.hosts,
1569
+ brokering: brokeringFor(spec.harness, spec.vertex),
1570
+ expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
1431
1571
  substitutions: env.substitutions,
1432
1572
  egress: spec.egress
1433
1573
  });
@@ -1465,6 +1605,11 @@ async function startSession(a, runId) {
1465
1605
  for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
1466
1606
  for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
1467
1607
  await runSetup(a, box, env);
1608
+ const running = await leftRunning(a, box, env);
1609
+ if (running.length) {
1610
+ a.record.setupProcesses = running;
1611
+ debugLog("env", `setup left ${running.length} process${running.length === 1 ? "" : "es"} running with ${[...new Set(running.flatMap((p) => p.secrets))].join(", ")} (to: 'checks', no hosts) in the environment: ${running.map((p) => clip(p.command, 80)).join("; ")}`);
1612
+ }
1468
1613
  await snapshot(box, a.signal);
1469
1614
  debugLog("env", "workspace snapshot taken: the diff starts here");
1470
1615
  const timed = await timeMcpServers(box, env.mcp, a.signal);
@@ -1473,7 +1618,7 @@ async function startSession(a, runId) {
1473
1618
  debugLog("env", "stdio MCP servers start through reeval's MCP timer");
1474
1619
  } else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
1475
1620
  if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
1476
- const harness = await createHarness(spec.harness);
1621
+ const harness = await createHarness(spec.harness, spec.vertex);
1477
1622
  const harnessStart = performance.now();
1478
1623
  debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
1479
1624
  const session = await startHarness(a, harness, box, {
@@ -1506,7 +1651,7 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
1506
1651
  const cutoff = new Promise((resolve) => {
1507
1652
  onAbort = () => {
1508
1653
  timer = setTimeout(() => {
1509
- const message = `The agent didn't stop within ${grace / 1e3}s of being cancelled, so reeval stopped waiting for its turn`;
1654
+ const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
1510
1655
  const at = Date.now() - Date.parse(startedAt);
1511
1656
  resolve({
1512
1657
  text: "",
@@ -1602,7 +1747,7 @@ function turnSender(a, takeContext, checks) {
1602
1747
  kind: "turn-start",
1603
1748
  turn: turnNumber
1604
1749
  });
1605
- debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(prompt, 160))}`);
1750
+ debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(redactSecrets(prompt), 160))}`);
1606
1751
  debugTrace("harness", "prompt", {
1607
1752
  turn: turnNumber,
1608
1753
  prompt
@@ -1663,10 +1808,9 @@ function turnSender(a, takeContext, checks) {
1663
1808
  ms: result.durationMs,
1664
1809
  tools: count("tool", "web-search", "web-fetch"),
1665
1810
  fileEdits: count("file-edit"),
1666
- ...failure?.type === "error" ? { error: redactSecrets(clip(failure.message.trim(), 300)) } : {}
1811
+ ...failure?.type === "error" ? { error: clip(redactSecrets(failure.message.trim()), 300) } : {}
1667
1812
  });
1668
- const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
1669
- debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
1813
+ debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
1670
1814
  for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
1671
1815
  if (timedOut) {
1672
1816
  const error = new InfraError(timedOut);
@@ -1760,13 +1904,19 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1760
1904
  throw a.startFailure ? a.startFailure.error : error;
1761
1905
  });
1762
1906
  a.signal.throwIfAborted();
1763
- const missed = await unawaited() ?? softFailure(assertions()) ?? expectError();
1907
+ const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
1764
1908
  if (a.startFailure) throw a.startFailure.error;
1765
1909
  if (missed) throw missed;
1766
- const [only, ...more] = a.judgeErrors;
1767
- if (only) throw new Error(more.length ? `${more.length + 1} judge calls failed:\n${a.judgeErrors.join("\n")}` : only);
1768
- } catch (error) {
1769
- recordFailure(a, error, timedOut());
1910
+ const judgeErrors = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
1911
+ const [only, ...more] = judgeErrors;
1912
+ if (only) {
1913
+ const message = more.length ? `${more.length + 1} judge calls failed:\n${judgeErrors.join("\n")}` : only;
1914
+ a.judgeThrown.add(message);
1915
+ throw new Error(message);
1916
+ }
1917
+ } catch (thrown) {
1918
+ const error = attemptError(a, thrown, assertions());
1919
+ recordFailure(a, error, timedOut(), assertions());
1770
1920
  failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
1771
1921
  } finally {
1772
1922
  if (!a.signal.aborted) await unawaited().catch(() => {});
@@ -1788,7 +1938,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1788
1938
  const checked = assertionRecords(assertions());
1789
1939
  if (checked.length) record.assertions = checked;
1790
1940
  for (const x of checked) {
1791
- const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
1941
+ const why = x.message ? ` · ${clip(redactSecrets(x.message.split("\n")[0] ?? ""), 160)}` : "";
1792
1942
  debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
1793
1943
  }
1794
1944
  const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
@@ -1810,39 +1960,45 @@ const declaredModeSchema = z.enum([
1810
1960
  "run",
1811
1961
  "skip",
1812
1962
  "only"
1813
- ]).describe("How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`.");
1963
+ ]);
1814
1964
  const declaredFamilySchema = z.strictObject({
1815
- name: z.string().describe("The name template passed to `evaluate.each`."),
1816
- case: z.number().int().min(0).describe("The case's index, from 0."),
1817
- cases: z.number().int().min(2).describe("How many cases the list has.")
1818
- }).describe("Where a case of a pooled `evaluate.each` sits in its list.");
1965
+ name: z.string(),
1966
+ case: z.number().int().min(0),
1967
+ cases: z.number().int().min(2)
1968
+ });
1819
1969
  const declaredTodoSchema = z.strictObject({
1820
1970
  type: z.literal("todo"),
1821
- name: z.string().describe("The name passed to `.todo`.")
1822
- }).describe("`evaluate.todo` or `group.todo`.");
1971
+ name: z.string()
1972
+ });
1973
+ const hash = () => z.string().regex(/^[0-9a-f]{64}$/, "must be a sha256 hash: 64 lowercase hex digits");
1974
+ const declaredSourceSchema = z.strictObject({
1975
+ body: hash(),
1976
+ case: hash().optional()
1977
+ });
1823
1978
  const declaredEvalSchema = z.strictObject({
1824
1979
  type: z.literal("eval"),
1825
- name: z.string().describe("The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`)."),
1980
+ name: z.string(),
1826
1981
  mode: declaredModeSchema,
1827
- options: evalOptionsDataSchema.describe("The eval's own options, before the cascade."),
1828
- body: z.string().describe("Names the eval's body to whoever runs it."),
1829
- family: declaredFamilySchema.optional().describe("Set on each case of a pooled `evaluate.each`: its template, index and the list's length.")
1830
- }).describe("An `evaluate`: one eval, planned into a test per attempt.");
1982
+ options: evalOptionsDataSchema,
1983
+ body: z.string(),
1984
+ source: declaredSourceSchema,
1985
+ family: declaredFamilySchema.optional()
1986
+ });
1831
1987
  const declaredGroupSchema = z.strictObject({
1832
1988
  type: z.literal("group"),
1833
- name: z.string().describe("The group's name, as passed to `group`."),
1989
+ name: z.string(),
1834
1990
  mode: declaredModeSchema,
1835
- options: evalOptionsDataSchema.describe("The group's own options, before the cascade."),
1991
+ options: evalOptionsDataSchema,
1836
1992
  get children() {
1837
1993
  return z.array(declaredNodeSchema);
1838
1994
  }
1839
- }).describe("A `group`: options its evals and groups inherit, and its children in declaration order.");
1995
+ });
1840
1996
  const declaredNodeSchema = z.discriminatedUnion("type", [
1841
1997
  declaredEvalSchema,
1842
1998
  declaredGroupSchema,
1843
1999
  declaredTodoSchema
1844
- ]).describe("One node of the tree: a group, an eval or a todo.");
1845
- const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) }).describe("An eval file's top-level groups, evals and todos, in declaration order.");
2000
+ ]);
2001
+ const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) });
1846
2002
  const NODE_SCHEMAS = {
1847
2003
  eval: declaredEvalSchema,
1848
2004
  group: declaredGroupSchema,
@@ -1870,20 +2026,18 @@ function configLevel(config) {
1870
2026
  judge: config.judge,
1871
2027
  capture: config.capture,
1872
2028
  question: config.question,
2029
+ questions: [],
1873
2030
  timeout: void 0,
1874
2031
  tags: [],
2032
+ revision: void 0,
1875
2033
  pin: [],
1876
2034
  sources: {}
1877
2035
  };
1878
2036
  }
1879
- /**
1880
- * The tests an eval registers, or `null` when the run's tags leave it out (it isn't counted in
1881
- * `order`). Throws a UsageError for options the plan refuses.
1882
- */
2037
+ /** The tests an eval registers. Throws a UsageError for options the plan refuses. */
1883
2038
  function planEval(declaration, ctx) {
1884
- const { name, mode, groups: enclosing, options: own } = declaration;
1885
- const groups = enclosing.map((g) => g.name);
1886
- const where = `evaluate("${[...groups, name].join(" › ")}")`;
2039
+ const { name, groups: enclosing, options: own } = declaration;
2040
+ const where = `evaluate("${[...enclosing.map((g) => g.name), name].join(" › ")}")`;
1887
2041
  checkOptions(own, where);
1888
2042
  const levels = [...enclosing.map((g) => ({
1889
2043
  options: g.options,
@@ -1894,15 +2048,82 @@ function planEval(declaration, ctx) {
1894
2048
  }];
1895
2049
  const resolvedEval = cascade(configLevel(ctx.config), levels, where, {
1896
2050
  profile: ctx.profile,
1897
- runs: ctx.runs
2051
+ runs: ctx.runs,
2052
+ skipped: declaration.mode === "skip" || declaration.mode === "todo",
2053
+ pooled: declaration.family !== void 0
1898
2054
  });
2055
+ const { harnesses } = resolvedEval;
2056
+ if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
2057
+ tests: null,
2058
+ harnesses
2059
+ };
2060
+ return {
2061
+ tests: planTests(declaration, ctx, where, resolvedEval),
2062
+ harnesses
2063
+ };
2064
+ }
2065
+ function planTests(declaration, ctx, where, resolvedEval) {
2066
+ const { name, mode, groups: enclosing } = declaration;
2067
+ const groups = enclosing.map((g) => g.name);
1899
2068
  const { resolved } = resolvedEval;
1900
- if (!matchesTags(resolved.tags, ctx.tags)) return null;
2069
+ const source = {
2070
+ ...declaration.source,
2071
+ options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
2072
+ };
1901
2073
  const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
1902
- const plannedAttempts = resolvedEval.skip ? [] : expand(resolved);
2074
+ const gateways = ctx.profile?.gateway ?? ctx.config.gateway;
2075
+ const gatewaySet = ctx.config.gatewaySet || ctx.profile?.gateway !== void 0;
2076
+ const routes = /* @__PURE__ */ new Map();
2077
+ const gatewayNotes = [];
2078
+ let vertex;
2079
+ const routed = resolvedEval.skip ? resolved.harness : resolved.harness.map((harness) => {
2080
+ const route = resolveGateway(gateways, harness, where);
2081
+ const hid = harnessId(harness);
2082
+ if (gatewaySet || harness.gateway !== void 0) {
2083
+ routes.set(hid, route);
2084
+ gatewayNotes.push(`gateway: ${hid} → ${route.name}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
2085
+ }
2086
+ if (route.name === "direct") return harness;
2087
+ if (harness.kind !== "fake") try {
2088
+ if (route.name === "vercel") {
2089
+ const problem = vercelCredentialProblem();
2090
+ if (problem) throw new UsageError(problem);
2091
+ }
2092
+ if (route.name === "vertex") {
2093
+ const entry = gateways.find((g) => g.kind === "vertex");
2094
+ vertex ??= vertexSettings(entry?.kind === "vertex" ? entry : {});
2095
+ }
2096
+ } catch (error) {
2097
+ if (error instanceof UsageError) throw new UsageError(`${where} on ${hid}: ${error.message}`);
2098
+ throw error;
2099
+ }
2100
+ return {
2101
+ ...harness,
2102
+ gateway: route.name
2103
+ };
2104
+ });
2105
+ const plannedAttempts = resolvedEval.skip ? [] : expand({
2106
+ ...resolved,
2107
+ harness: routed
2108
+ }, vertex);
2109
+ let judge;
2110
+ if (plannedAttempts.length) try {
2111
+ const gateway = resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
2112
+ const { model } = resolved.judge;
2113
+ if (typeof model === "string") judge = {
2114
+ model,
2115
+ ...gateway ? { gateway } : {}
2116
+ };
2117
+ if (gatewaySet && fellBackToDirect(gateway, resolved.judge.gateway, { gateways })) gatewayNotes.push(`gateway: judge ${model} → direct (fallback)`);
2118
+ } catch (error) {
2119
+ if (error instanceof UsageError) throw new UsageError(`${where}: ${error.message}`);
2120
+ throw error;
2121
+ }
1903
2122
  if (ctx.config.secrets.unbrokered === "error") {
1904
2123
  const [first] = unbrokered(resolved.env, plannedAttempts);
1905
2124
  if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
2125
+ const [plain] = plainHeaders(plannedAttempts);
2126
+ if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
1906
2127
  }
1907
2128
  const pluginNotes = [];
1908
2129
  const pluginLabels = /* @__PURE__ */ new Map();
@@ -1928,7 +2149,8 @@ function planEval(declaration, ctx) {
1928
2149
  configFile: ctx.config.file,
1929
2150
  file: path
1930
2151
  });
1931
- plan.notes.push(...pluginNotes);
2152
+ if (gatewayNotes.length && openRouterFallbacks(gateways)) gatewayNotes.push("gateway: openrouter() fallbacks on: reeval's own calls may fail over to another upstream provider; the harnesses' calls take OpenRouter's own routing");
2153
+ plan.notes.push(...gatewayNotes, ...pluginNotes);
1932
2154
  const setByOf = (planned) => ({
1933
2155
  sandbox: sandboxSetBy(planned, resolved, ctx.config.file),
1934
2156
  harness: harnessBy
@@ -1940,6 +2162,8 @@ function planEval(declaration, ctx) {
1940
2162
  name,
1941
2163
  groups,
1942
2164
  tags: resolved.tags,
2165
+ ...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
2166
+ source,
1943
2167
  cell: first.cell,
1944
2168
  n: 1,
1945
2169
  label: "",
@@ -1976,6 +2200,8 @@ function planEval(declaration, ctx) {
1976
2200
  name,
1977
2201
  groups,
1978
2202
  tags: resolved.tags,
2203
+ ...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
2204
+ source,
1979
2205
  cell: planned.cell,
1980
2206
  n: planned.n,
1981
2207
  label: planned.label,
@@ -1996,13 +2222,15 @@ function planEval(declaration, ctx) {
1996
2222
  skip
1997
2223
  } : plan,
1998
2224
  ...skip ? { skip } : {},
1999
- ...family ? { family } : {}
2225
+ ...family ? { family } : {},
2226
+ ...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {},
2227
+ ...judge ? { judge } : {}
2000
2228
  };
2001
2229
  return {
2002
2230
  name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
2003
2231
  mode: skip ? "skip" : mode,
2004
2232
  attempt,
2005
- timeout: skip ? void 0 : timeoutMs + setupAllowance(planned) + GRACE_MS,
2233
+ timeout: skip ? void 0 : Math.min(timeoutMs + setupAllowance(planned) + GRACE_MS, MAX_TIMER_MS),
2006
2234
  gate: skip ? void 0 : {
2007
2235
  limits: attemptLimits(planned, ctx.concurrency),
2008
2236
  rank: {
@@ -2033,7 +2261,7 @@ function planEval(declaration, ctx) {
2033
2261
  /**
2034
2262
  * What the plan can tell of an attempt's plugins before any is fetched: a build outside Docker, a
2035
2263
  * local plugin's content, and the harness's refusal, as a refusal; the commands that fetch a
2036
- * package at start, as notes under `allow-all`; and a label for each plugin. Git and npm plugins
2264
+ * package at start, as notes under `proxy` and `allow-all`; and a label for each plugin. Git and npm plugins
2037
2265
  * are read when the attempt resolves its environment, which refuses them the same way.
2038
2266
  */
2039
2267
  function planPlugins(planned, root) {
@@ -2182,16 +2410,54 @@ function registerTodo(name) {
2182
2410
  const tags = [...new Set(groupStack.flatMap((g) => g.options.tags ?? []))];
2183
2411
  if (matchesTags(tags, getRunContext().tags)) registerTest(name, { mode: "todo" }, async () => {});
2184
2412
  }
2185
- /** The evals declared so far, by file and name path, so a second one with the same id is refused. */
2186
- const declared = /* @__PURE__ */ new Set();
2187
- function checkUnique(where, groups, name, hint) {
2188
- const key = `${collectingFile() ?? ""}\0${[...groups, name].join("\0")}`;
2189
- if (declared.has(key)) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}`);
2190
- declared.add(key);
2413
+ /**
2414
+ * The evals declared so far, by id, with their name paths and whether each is a family's case,
2415
+ * so a second id is refused.
2416
+ */
2417
+ const declared = /* @__PURE__ */ new Map();
2418
+ /** The pooled families declared so far, by id: their name path, length and the cases seen. */
2419
+ const families = /* @__PURE__ */ new Map();
2420
+ /** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
2421
+ const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
2422
+ /** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
2423
+ const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as the pooled evaluate.each("${family.join(" › ")}") in this file: a family's id is its name template. Give the eval or the template another name.${separatorHint(eval_, family)}`);
2424
+ function checkUnique(where, groups, name, hint, inFamily) {
2425
+ const path = [...groups, name];
2426
+ const id = evalId(collectingFile() ?? "", path);
2427
+ const earlier = declared.get(id);
2428
+ if (earlier) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}${separatorHint(path, earlier.path)}`);
2429
+ const family = inFamily ? void 0 : families.get(id);
2430
+ if (family) throw namedLikeFamily(path, family.path);
2431
+ declared.set(id, {
2432
+ path,
2433
+ inFamily
2434
+ });
2191
2435
  }
2192
- function register(mode, name, own, fn, hint = "", family) {
2436
+ /**
2437
+ * A family's cases pool by its id, the groups and the template, so each case may be declared
2438
+ * once and every case must agree on the list's length.
2439
+ */
2440
+ function checkFamily(groups, family) {
2441
+ const path = [...groups, family.name];
2442
+ const id = evalId(collectingFile() ?? "", path);
2443
+ const earlier = families.get(id);
2444
+ if (!earlier) {
2445
+ const plain = declared.get(id);
2446
+ if (plain && !plain.inFamily) throw namedLikeFamily(plain.path, path);
2447
+ families.set(id, {
2448
+ path,
2449
+ cases: family.cases,
2450
+ seen: /* @__PURE__ */ new Set([family.case])
2451
+ });
2452
+ return;
2453
+ }
2454
+ if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate.each("${path.join(" › ")}") is declared twice in this file. Each pooled evaluate.each needs its own name, or their cases are pooled into one family. Give each its own name template, or pass evaluate.each(cases, { pool: false }) to keep the cases apart.${separatorHint(path, earlier.path)}`);
2455
+ earlier.seen.add(family.case);
2456
+ }
2457
+ function register(mode, name, own, fn, source, hint = "", family) {
2193
2458
  const groups = groupStack.map((g) => g.name);
2194
- checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint);
2459
+ if (family) checkFamily(groups, family);
2460
+ checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint, !!family);
2195
2461
  const body = `b${bodies.size + 1}`;
2196
2462
  bodies.set(body, fn);
2197
2463
  declare({
@@ -2200,9 +2466,10 @@ function register(mode, name, own, fn, hint = "", family) {
2200
2466
  mode,
2201
2467
  options: data(own),
2202
2468
  body,
2469
+ source,
2203
2470
  ...family ? { family } : {}
2204
2471
  });
2205
- const tests = planEval({
2472
+ const { tests, harnesses } = planEval({
2206
2473
  name,
2207
2474
  mode,
2208
2475
  groups: groupStack.map(({ name, options }) => ({
@@ -2210,22 +2477,25 @@ function register(mode, name, own, fn, hint = "", family) {
2210
2477
  options
2211
2478
  })),
2212
2479
  options: own,
2480
+ source,
2213
2481
  ...family ? { family } : {}
2214
2482
  }, {
2215
2483
  ...getRunContext(),
2216
2484
  file: collectingFile(),
2217
2485
  order: registered
2218
2486
  });
2487
+ declareHarnesses(harnesses);
2219
2488
  if (!tests) return;
2220
2489
  registered++;
2221
2490
  for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
2222
2491
  }
2223
2492
  /** Evals registered so far in this worker, for the scheduler's tie-break. */
2224
2493
  let registered = 0;
2225
- const defineEval = (mode) => (name, a, b) => {
2494
+ const defineEval = (mode, declared) => (name, a, b) => {
2226
2495
  checkOnly(mode, "evaluate");
2227
2496
  const [own, fn] = splitArgs("evaluate", name, a, b);
2228
- register(mode, name, withPaths(own, `evaluate("${name}")`), fn);
2497
+ const where = `evaluate("${name}")`;
2498
+ register(mode, name, withPaths(own, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
2229
2499
  };
2230
2500
  /** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
2231
2501
  function formatName(template, item, index) {
@@ -2251,13 +2521,19 @@ const evalEach = (cases, options) => (name, a, b) => {
2251
2521
  const own = withPaths(written, `evaluate.each("${name}")`);
2252
2522
  const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
2253
2523
  const pooled = pool && cases.length >= 2;
2524
+ const file = collectingFile();
2525
+ const body = bodyHash(fn, `evaluate.each("${name}")`, file);
2254
2526
  cases.forEach((item, i) => {
2255
2527
  const family = pooled ? {
2256
2528
  name,
2257
2529
  case: i,
2258
2530
  cases: cases.length
2259
2531
  } : void 0;
2260
- register("run", formatName(name, item, i), own, (c) => fn(c, item), hint, family);
2532
+ const source = {
2533
+ body,
2534
+ case: caseHash(item, `evaluate.each("${name}") case ${i}`, file)
2535
+ };
2536
+ register("run", formatName(name, item, i), own, (c) => fn(c, item), source, hint, family);
2261
2537
  });
2262
2538
  };
2263
2539
  const evaluate = Object.assign(defineEval("run"), {
@@ -2287,8 +2563,8 @@ function declareNodes(nodes, bodyFor, base) {
2287
2563
  } else if (node.family) {
2288
2564
  const options = optionsFrom(node.options, `evaluate("${node.name}")`, base);
2289
2565
  checkOnly(node.mode, "evaluate");
2290
- register(node.mode, node.name, options, bodyFor(node.body), "", node.family);
2291
- } else defineEval(node.mode)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
2566
+ register(node.mode, node.name, options, bodyFor(node.body), node.source, "", node.family);
2567
+ } else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
2292
2568
  }
2293
2569
  /**
2294
2570
  * Options from their data forms, with host paths absolute against `base`: environments become
@@ -2325,4 +2601,4 @@ function optionsFrom(written, where, base) {
2325
2601
  //#endregion
2326
2602
  export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
2327
2603
 
2328
- //# sourceMappingURL=evaluate-Djk1YPFL.js.map
2604
+ //# sourceMappingURL=evaluate-BPP9s3s6.js.map