reeval 0.14.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/README.md +3 -3
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
  9. package/dist/app-files-ChbvRx3P.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
  11. package/dist/assemble-SKQGW_QF.js.map +1 -0
  12. package/dist/assemble-XBWuK3Ed.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
  16. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
  18. package/dist/capture-BM_3100g.js.map +1 -0
  19. package/dist/capture-DSK8O5bA.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
  21. package/dist/cli-PdYrcoKM.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
  23. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
  26. package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
  27. package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
  28. package/dist/command-4pJYVCT4.js.map +1 -0
  29. package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
  30. package/dist/command-9xm5ImqI.js.map +1 -0
  31. package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
  32. package/dist/command-BqBxLgTc.js.map +1 -0
  33. package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
  34. package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
  35. package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
  36. package/dist/commands-BU6YJCz9.js.map +1 -0
  37. package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
  38. package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
  39. package/dist/config-k6m7XbAz.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-CpsifVln.js +1530 -0
  46. package/dist/db-CpsifVln.js.map +1 -0
  47. package/dist/db-DkD-F-X-.js +2 -0
  48. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  49. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  50. package/dist/debug-DeM5pswK.js +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
  52. package/dist/docker-51WvhN_X.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
  54. package/dist/docker-B4-rx4dA.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
  57. package/dist/doctor-BuQnfz4e.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
  61. package/dist/environment-BOFgzFDg.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  63. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  64. package/dist/eval-source-Dje3rERU.js +145 -0
  65. package/dist/eval-source-Dje3rERU.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
  67. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  68. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  69. package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
  70. package/dist/files-Bi461Be2.js.map +1 -0
  71. package/dist/files-C2h2Cw4F.js +1 -0
  72. package/dist/files-Clipx4OW.js +37 -0
  73. package/dist/files-Clipx4OW.js.map +1 -0
  74. package/dist/files-DmUrRmmL.js +1 -0
  75. package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
  76. package/dist/files-H24pz0hL.js.map +1 -0
  77. package/dist/gateway.d.ts +22 -9
  78. package/dist/gateway.js +30 -8
  79. package/dist/gateway.js.map +1 -1
  80. package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
  81. package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
  82. package/dist/harness-3NVyTSGK.js +392 -0
  83. package/dist/harness-3NVyTSGK.js.map +1 -0
  84. package/dist/harness.d.ts +12 -8
  85. package/dist/harness.js +1 -1
  86. package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
  87. package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
  88. package/dist/index.d.ts +51 -10
  89. package/dist/index.js +5 -5
  90. package/dist/init-BDEOopK7.js +1 -0
  91. package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
  92. package/dist/init-cOk3D44b.js.map +1 -0
  93. package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
  94. package/dist/invocations-DW-lTk2f.js.map +1 -0
  95. package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
  96. package/dist/judge-CKdgEXN7.js.map +1 -0
  97. package/dist/judge.d.ts +5 -7
  98. package/dist/judge.js +4 -8
  99. package/dist/judge.js.map +1 -1
  100. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  101. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  102. package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
  103. package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
  104. package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
  105. package/dist/model-Bq3G0OG1.js.map +1 -0
  106. package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
  107. package/dist/model-CTnyvh7H.js.map +1 -0
  108. package/dist/model-D-SWrLZG.js +1 -0
  109. package/dist/model-g3XnzUv0.js +1 -0
  110. package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
  111. package/dist/model-vRiUycWh.js.map +1 -0
  112. package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
  113. package/dist/node-BVbbnFtv.js.map +1 -0
  114. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  115. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  116. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  117. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  118. package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
  119. package/dist/pairs-puYi-zj2.js.map +1 -0
  120. package/dist/patch-CquGAGX5.js +76 -0
  121. package/dist/patch-CquGAGX5.js.map +1 -0
  122. package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
  123. package/dist/patch-DKJiC5Sq.js.map +1 -0
  124. package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
  125. package/dist/plan-DdWkM6SW.js.map +1 -0
  126. package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
  127. package/dist/plugins-D-CP6mMd.js.map +1 -0
  128. package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
  129. package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
  130. package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
  131. package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
  132. package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
  133. package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
  134. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  135. package/dist/prices-QaS-kjSQ.js.map +1 -0
  136. package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
  137. package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
  138. package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
  139. package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
  140. package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
  141. package/dist/query-Clm2mNSm.js +1 -0
  142. package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
  143. package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
  144. package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
  145. package/dist/questions-BRhU8MzB.js.map +1 -0
  146. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  147. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  148. package/dist/redact-DVIaahLv.js +658 -0
  149. package/dist/redact-DVIaahLv.js.map +1 -0
  150. package/dist/remarks-BwJ8pcTQ.js +1 -0
  151. package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
  152. package/dist/remarks-C7i2LMRb.js.map +1 -0
  153. package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
  154. package/dist/remarks-D_3qhBWU.js.map +1 -0
  155. package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
  156. package/dist/remarks-dA8LyoPc.js.map +1 -0
  157. package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
  158. package/dist/remote-DFIzcx_6.js.map +1 -0
  159. package/dist/resolve--rQZ8u6Z.js +1 -0
  160. package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
  161. package/dist/resolve-DWSJpKHX.js.map +1 -0
  162. package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
  163. package/dist/results-BC3pcG_Z.js.map +1 -0
  164. package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
  165. package/dist/resume-j-rcIryi.js.map +1 -0
  166. package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
  167. package/dist/runtime-d56Gg3J5.js.map +1 -0
  168. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
  169. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
  170. package/dist/sandbox.d.ts +1 -1
  171. package/dist/sandbox.js +1 -1
  172. package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
  173. package/dist/server-BOoH10J8.js.map +1 -0
  174. package/dist/server-DpuVAbrO.js +1 -0
  175. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  176. package/dist/shared-B_lvGqWz.js.map +1 -0
  177. package/dist/source-BFd6f0-R.js +1 -0
  178. package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
  179. package/dist/source-wD04-rRG.js.map +1 -0
  180. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  181. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  182. package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
  183. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  184. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  185. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  186. package/dist/src-BO8AYqJ5.js.map +1 -0
  187. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  188. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  189. package/dist/src-BYPMM1x7.js +2 -0
  190. package/dist/src-Bp4aN3rA.js +2 -0
  191. package/dist/src-C7upnZSd.js +2 -0
  192. package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
  193. package/dist/src-D91Hd6hT.js.map +1 -0
  194. package/dist/src-DH-9IO7f.js +1 -0
  195. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  196. package/dist/src-DSZyg0Uq.js.map +1 -0
  197. package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
  198. package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
  199. package/dist/src-HSxBazcg.js +3 -0
  200. package/dist/src-LD1BAJFa.js +2 -0
  201. package/dist/src-dlX-Lmxg.js +2 -0
  202. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  203. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  204. package/dist/src-kYeaJJrl.js +2 -0
  205. package/dist/state-B5PLm5qY.js +1 -0
  206. package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
  207. package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
  208. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  209. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  210. package/dist/suite-22Kt9Grx.js +2 -0
  211. package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
  212. package/dist/suite-Co8cwelb.js.map +1 -0
  213. package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
  214. package/dist/summary-DLFHVoux.js.map +1 -0
  215. package/dist/testing.d.ts +417 -142
  216. package/dist/testing.js +12 -10
  217. package/dist/testing.js.map +1 -1
  218. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  219. package/dist/text-DTigcA64.js.map +1 -0
  220. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  221. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  222. package/dist/types-C5vI3g1C.js.map +1 -1
  223. package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
  224. package/dist/vertex-Bv7vhMgs.js +1 -0
  225. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  226. package/dist/vertex-DypYjx71.js +456 -0
  227. package/dist/vertex-DypYjx71.js.map +1 -0
  228. package/dist/view/index.d.ts +536 -121
  229. package/dist/view/index.js +9 -9
  230. package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
  231. package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
  232. package/dist/views-BOf8qEL8.js +2 -0
  233. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
  234. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  235. package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
  236. package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
  237. package/dist/workspace-DDHzkMjm.js +1 -0
  238. package/docker/seccomp-default.json +1 -0
  239. package/package.json +5 -1
  240. package/skills/eval-rubric/SKILL.md +35 -104
  241. package/dist/adapter-BloGVZ_a.js.map +0 -1
  242. package/dist/agent-m-_t-IuW.js.map +0 -1
  243. package/dist/app-files-C-jwhRDt.js.map +0 -1
  244. package/dist/assemble-BDEpelpi.js.map +0 -1
  245. package/dist/assemble-C5bfncHU.js +0 -1
  246. package/dist/capture-CVRwRTnb.js.map +0 -1
  247. package/dist/capture-CherbjzB.js +0 -1
  248. package/dist/cli-C_4gaJ50.js.map +0 -1
  249. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  250. package/dist/command-BWAuwQNk.js +0 -91
  251. package/dist/command-BWAuwQNk.js.map +0 -1
  252. package/dist/command-BXLUfw4h.js.map +0 -1
  253. package/dist/command-Be6BSE2L.js +0 -335
  254. package/dist/command-Be6BSE2L.js.map +0 -1
  255. package/dist/command-Bxdy62dW.js.map +0 -1
  256. package/dist/commands-jzigbNFb.js.map +0 -1
  257. package/dist/config-BrdHA9bJ.js.map +0 -1
  258. package/dist/cost-D4elx933.js.map +0 -1
  259. package/dist/cost-DmnAod9x.js +0 -1
  260. package/dist/db-CAfksinX.js +0 -2
  261. package/dist/db-DO0p7xgP.js +0 -619
  262. package/dist/db-DO0p7xgP.js.map +0 -1
  263. package/dist/debug-D7bEv_UV.js +0 -1
  264. package/dist/docker-BnRlTpZA.js.map +0 -1
  265. package/dist/docker-D01slDF3.js.map +0 -1
  266. package/dist/doctor-DQ2p0euG.js.map +0 -1
  267. package/dist/env-BQqj93gu.js.map +0 -1
  268. package/dist/environment-CO0P0soF.js.map +0 -1
  269. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  270. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  271. package/dist/facts-DzOUTreB.js +0 -906
  272. package/dist/facts-DzOUTreB.js.map +0 -1
  273. package/dist/files-B-CJeeEh.js.map +0 -1
  274. package/dist/files-Bp5j5BqI.js +0 -1
  275. package/dist/files-C2sjmXhk.js +0 -1
  276. package/dist/files-C_gQNVB-.js +0 -65
  277. package/dist/files-C_gQNVB-.js.map +0 -1
  278. package/dist/files-kUCrswI6.js.map +0 -1
  279. package/dist/generate-DZ7vp69F.js +0 -1647
  280. package/dist/generate-DZ7vp69F.js.map +0 -1
  281. package/dist/harness-COVXSwJ6.js +0 -354
  282. package/dist/harness-COVXSwJ6.js.map +0 -1
  283. package/dist/header-BLPP6aCU.js.map +0 -1
  284. package/dist/init-RgCQXZhb.js +0 -1
  285. package/dist/init-SRalM5xF.js.map +0 -1
  286. package/dist/insights-TyAvz9aV.js +0 -68
  287. package/dist/insights-TyAvz9aV.js.map +0 -1
  288. package/dist/invocations-DzyKSJBa.js.map +0 -1
  289. package/dist/judge-yLvjC5A3.js.map +0 -1
  290. package/dist/match-Bz0nSdzN.js.map +0 -1
  291. package/dist/model-1ZLr9Dcv.js.map +0 -1
  292. package/dist/model-B45lDUV1.js.map +0 -1
  293. package/dist/model-BTZq8HIZ.js +0 -1
  294. package/dist/model-CgnOpss4.js.map +0 -1
  295. package/dist/model-Ch8kX87u.js +0 -1
  296. package/dist/node-BSq5qHnQ.js.map +0 -1
  297. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  298. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  299. package/dist/pairs-CHfZNRT6.js.map +0 -1
  300. package/dist/patch-DWf-I_Q_.js.map +0 -1
  301. package/dist/plan-BIwUzlB8.js.map +0 -1
  302. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  303. package/dist/prices-BcLEXvdY.js.map +0 -1
  304. package/dist/query-Cvbwl804.js +0 -1
  305. package/dist/questions-bAudIc04.js.map +0 -1
  306. package/dist/redact-CRGdfV2s.js +0 -240
  307. package/dist/redact-CRGdfV2s.js.map +0 -1
  308. package/dist/remarks-CXwSnSjR.js +0 -278
  309. package/dist/remarks-CXwSnSjR.js.map +0 -1
  310. package/dist/remarks-CyAu2DiB.js +0 -1
  311. package/dist/remarks-mujJv1Ey.js.map +0 -1
  312. package/dist/remote-CKcEKja4.js.map +0 -1
  313. package/dist/resolve-B5-uqmJ6.js +0 -1
  314. package/dist/resolve-CBoKzQMP.js.map +0 -1
  315. package/dist/results-CMgZEANy.js.map +0 -1
  316. package/dist/resume-B9SOSnK9.js.map +0 -1
  317. package/dist/rules-VhftPOOT.js.map +0 -1
  318. package/dist/run-CwIEK16q.js +0 -88
  319. package/dist/run-CwIEK16q.js.map +0 -1
  320. package/dist/runtime-BgQRyCLl.js.map +0 -1
  321. package/dist/server-CHRSgKRp.js +0 -1
  322. package/dist/server-TGW4WyAz.js.map +0 -1
  323. package/dist/shared-BDUzStad.js.map +0 -1
  324. package/dist/source-B1WvBIQt.js.map +0 -1
  325. package/dist/source-DAzztZwU.js +0 -1
  326. package/dist/src-1jHbFaGW.js +0 -1
  327. package/dist/src-8zoX_qrb.js.map +0 -1
  328. package/dist/src-C_9wgowC.js +0 -2
  329. package/dist/src-ChhtI6BP.js.map +0 -1
  330. package/dist/src-D005Cbqz.js +0 -2
  331. package/dist/src-DIfRbp5a.js +0 -3
  332. package/dist/src-Dg8121Xy.js +0 -2
  333. package/dist/src-ElBqEN-S.js +0 -2
  334. package/dist/src-NSdQ9zI6.js.map +0 -1
  335. package/dist/src-YHGaUrGE.js +0 -2
  336. package/dist/src-mJVxw1Jm.js +0 -2
  337. package/dist/state-vznMh2wl.js +0 -1
  338. package/dist/suite-BsWGHzlG.js +0 -2
  339. package/dist/suite-WgT7m6Vs.js.map +0 -1
  340. package/dist/summary-DIGEVr08.js.map +0 -1
  341. package/dist/text-CrVy5ioL.js.map +0 -1
  342. package/dist/transcript-BTxJpW4R.js.map +0 -1
  343. package/dist/usage-DF3_p5pE.js +0 -1
  344. package/dist/validate-BhwqrVsm.js.map +0 -1
  345. package/dist/views-BW_C5eVH.js +0 -2
  346. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  347. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,27 +1,31 @@
1
1
  import { t as isObjectLike } from "./object-CLf2-tNq.js";
2
- import { r as withLease } from "./client-D5hNjpEx.js";
2
+ import { r as withLease } from "./client-Ct6s6OnF.js";
3
3
  import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
4
- import { n as dataProblem, p as parseDuration } from "./shared-BDUzStad.js";
5
- import { N as workspaceOf, O as sandboxPaths, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-CO0P0soF.js";
4
+ import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
5
+ import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-BOFgzFDg.js";
6
+ import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-D-CP6mMd.js";
6
7
  import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
7
- import { A as matchesTags, B as mergeJudge, C as brokeredIn, D as expand, Dt as filePartsOf, E as evalPlan, Et as decideState, F as sandboxProvider, I as sandboxSetBy, L as secretVarsIn, N as refuseIllegalCompare, O as harnessId, Ot as resolveJudgeArtifacts, P as sandboxId, Q as createFakeSandbox, R as unbrokered, St as judgeProblem, T as checkOptions, Tt as badJudgeArtifacts, V as resolveStateOptions, W as allowedHosts, X as unenforceable, Y as isDirect, _t as writePluginFiles, ct as agentMcp, dt as claudeMcpServers, gt as refusedPlugins, ht as refusedMcp, mt as layoutOf, pt as expectedBlockedFor, q as egressLabel, ut as brokeringFor, v as gatewayScope, w as cascade, wt as artifactRefsOf, x as resolveGateway, y as resolveModelGateway, z as unbrokeredError } from "./config-BrdHA9bJ.js";
8
- import { c as judgeModelName, d as runJudge, m as isApiError, w as vercelCredentialProblem } from "./judge-yLvjC5A3.js";
9
- import { s as redactSecrets } from "./redact-CRGdfV2s.js";
10
- import { t as inCI } from "./env-BQqj93gu.js";
11
- import { ft as shellQuote, t as MCP_TIMER } from "./src-8zoX_qrb.js";
12
- import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-Dkp_H3pB.js";
8
+ import { $ as isDirect, A as harnessId, At as badJudgeArtifacts, B as sandboxSetBy, D as checkOptions, Dt as judgeProblem, G as resolveStateOptions, H as unbrokered, J as allowedHosts, L as refuseIllegalCompare, M as matchesTags, Mt as filePartsOf, N as plainHeaderError, Nt as resolveJudgeArtifacts, O as evalPlan, P as plainHeaders, R as sandboxId, S as resolveGateway, St as writePluginFiles, T as cascade, U as unbrokeredError, V as secretVarsIn, W as mergeJudge, Z as egressLabel, _ as fellBackToDirect, _t as expectedBlockedFor, b as resolveModelGateway, bt as refusedMcp, et as unenforceable, ft as agentMcp, ht as claudeMcpServers, jt as decideState, k as expand, kt as artifactRefsOf, mt as brokeringFor, nt as createFakeSandbox, v as gatewayScope, w as brokeredIn, xt as refusedPlugins, yt as layoutOf, z as sandboxProvider } from "./config-k6m7XbAz.js";
9
+ import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-CKdgEXN7.js";
10
+ import { v as vertexSettings } from "./vertex-DypYjx71.js";
11
+ import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
12
+ import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
13
+ import { t as inCI } from "./env-CARj5Axq.js";
14
+ import { ft as shellQuote, t as MCP_TIMER } from "./src-D91Hd6hT.js";
13
15
  import { t as InfraError } from "./types-C5vI3g1C.js";
14
- import { i as formatDuration, n as clip, t as capMiddle } from "./text-CrVy5ioL.js";
15
- import { i as registerTest, n as getRunContext, o as SUBJECT, r as registerGroup, t as collectingFile } from "./runtime-BgQRyCLl.js";
16
- import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-jI0OWxlz.js";
16
+ import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-d56Gg3J5.js";
17
+ import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-Ba2QerOK.js";
17
18
  import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
18
- import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-CVRwRTnb.js";
19
+ import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
20
+ import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-BM_3100g.js";
19
21
  import { t as contextTaker } from "./context-Dgn48BqT.js";
20
- import { a as skillFiles, i as resolveEnvironment } from "./resolve-CBoKzQMP.js";
22
+ import { a as skillFiles, i as resolveEnvironment } from "./resolve-DWSJpKHX.js";
21
23
  import { t as PrepareError } from "./provider-CmAYTQQs.js";
22
- import { i as preparedWorkspace } from "./workspace-CDI68wtX.js";
23
- import { n as judgeDiff } from "./patch-DWf-I_Q_.js";
24
- import { t as transcriptForJudge } from "./transcript-BTxJpW4R.js";
24
+ import { i as preparedWorkspace } from "./workspace-BRxjfa8N.js";
25
+ import { t as patchPath } from "./patch-CquGAGX5.js";
26
+ import { n as judgeDiff } from "./patch-DKJiC5Sq.js";
27
+ import { t as transcriptForJudge } from "./transcript-DxPaWHTZ.js";
28
+ import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-Dje3rERU.js";
25
29
  import { dirname, join } from "node:path";
26
30
  import { parse, stringify } from "yaml";
27
31
  import { z } from "zod";
@@ -30,7 +34,11 @@ import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
30
34
  //#region src/attempt/assertions.ts
31
35
  const MESSAGE_LIMIT = 1024;
32
36
  const LABEL_LIMIT = 60;
33
- const clipped = (s, limit) => s.length > limit ? `${s.slice(0, limit)}…` : s;
37
+ /** Masked first, so a secret the limit would cut through is masked whole. */
38
+ const clipped = (text, limit) => {
39
+ const s = redactSecrets(text);
40
+ return s.length > limit ? `${s.slice(0, limit)}…` : s;
41
+ };
34
42
  const text = (error) => error instanceof Error ? error.message : String(error);
35
43
  const shown = (value) => {
36
44
  if (typeof value === "string") return value;
@@ -44,17 +52,26 @@ const shown = (value) => {
44
52
  const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
45
53
  /**
46
54
  * The records of an attempt's matcher calls, in call order. Without a label, one is
47
- * `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`.
55
+ * `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
56
+ * any label already recorded, so each record's label is unique (a given `build #2` after two
57
+ * `build`s is `build #2 #2`).
48
58
  */
49
59
  function assertionRecords(raw) {
50
60
  const seen = /* @__PURE__ */ new Map();
61
+ const used = /* @__PURE__ */ new Set();
51
62
  return raw.map((r) => {
52
63
  const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
53
64
  const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
54
- const n = (seen.get(base) ?? 0) + 1;
65
+ let n = seen.get(base) ?? 0;
66
+ let label;
67
+ do {
68
+ n += 1;
69
+ label = n > 1 ? `${base} #${n}` : base;
70
+ } while (used.has(label));
55
71
  seen.set(base, n);
72
+ used.add(label);
56
73
  return {
57
- label: n > 1 ? `${base} #${n}` : base,
74
+ label,
58
75
  labelled: r.label !== void 0,
59
76
  matcher: r.matcher,
60
77
  status: r.status,
@@ -64,11 +81,22 @@ function assertionRecords(raw) {
64
81
  };
65
82
  });
66
83
  }
67
- /** What fails the attempt at its end for its `expect.soft` calls: the first that didn't pass. */
68
- function softFailure(raw) {
69
- const failed = raw.find((r) => r.soft && r.status !== "pass");
84
+ const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
85
+ /**
86
+ * What fails the attempt at its end for the checks that failed without ending the body: the first
87
+ * that didn't pass, an `expect.soft` as what it threw, a hard check the eval caught with
88
+ * `CAUGHT_HINT` after its message.
89
+ */
90
+ function checkFailure(raw) {
91
+ const failed = raw.find((r) => r.status !== "pass");
70
92
  if (!failed) return void 0;
71
- return failed.error instanceof Error ? failed.error : new Error(text(failed.error));
93
+ const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
94
+ if (failed.soft) return error;
95
+ const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
96
+ caught.name = error.name;
97
+ caught.message = `${error.message} ${CAUGHT_HINT}`;
98
+ caught.stack = error.stack?.replace(error.message, caught.message);
99
+ return caught;
72
100
  }
73
101
  //#endregion
74
102
  //#region src/attempt/debug.ts
@@ -120,7 +148,7 @@ function traceEnvironment(debug, spec, env) {
120
148
  const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
121
149
  debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
122
150
  for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
123
- const brokering = brokeringFor(spec.harness);
151
+ const brokering = brokeringFor(spec.harness, spec.vertex);
124
152
  if (brokering) {
125
153
  const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
126
154
  debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
@@ -406,10 +434,10 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
406
434
  }
407
435
  //#endregion
408
436
  //#region src/harness/index.ts
409
- async function createHarness(descriptor) {
437
+ async function createHarness(descriptor, vertex) {
410
438
  if (descriptor.kind === "fake") return createFakeHarness(descriptor);
411
- const { createAgentHarness } = await import("./agent-m-_t-IuW.js");
412
- return createAgentHarness(descriptor);
439
+ const { createAgentHarness } = await import("./agent-Zgn-kiqq.js");
440
+ return createAgentHarness(descriptor, vertex);
413
441
  }
414
442
  /** Where each harness's agent looks for repo skills, relative to the workspace. */
415
443
  const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
@@ -428,9 +456,9 @@ async function prepareEnvironment(a, root, runId, where, signal) {
428
456
  });
429
457
  a.env = env;
430
458
  record.environment = env.record;
431
- const { secrets, brokered } = env.record;
459
+ const { vars, secrets, brokered } = env.record;
432
460
  record.env = {
433
- vars: Object.keys(env.vars).filter((n) => !secrets.includes(n)).sort(),
461
+ vars,
434
462
  secrets,
435
463
  brokered
436
464
  };
@@ -438,8 +466,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
438
466
  if (debug) traceEnvironment(debug, spec, env);
439
467
  record.image = null;
440
468
  if (spec.sandbox.kind === "docker") {
441
- const { preparedFor } = await import("./preflight-D9y-HvZQ.js");
442
- const { attemptImage, dockerProvider, preparedImage } = await import("./providers-D1Zxp2he.js");
469
+ const { preparedFor } = await import("./preflight-DTiVv0gY.js");
470
+ const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DkOupEBl.js");
443
471
  signal.throwIfAborted();
444
472
  const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
445
473
  const builds = builtPlugins(spec.environment.layers);
@@ -472,7 +500,7 @@ const CHARS_PER_TOKEN = 4;
472
500
  const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
473
501
  const k = (tokens) => tokens < 1e3 ? String(tokens) : `${Math.round(tokens / 1e3)}k`;
474
502
  const diffItems = (patch) => patch.split(/(?=^diff --git )/m).map((part) => ({
475
- name: /^diff --git a\/.+ b\/(.+)$/m.exec(part)?.[1] ?? "",
503
+ name: patchPath(part) ?? "",
476
504
  chars: part.length
477
505
  })).filter((item) => item.name);
478
506
  /** The state's parts by size, each with its biggest files where it has any. */
@@ -615,13 +643,21 @@ function checksShown(which, ran) {
615
643
  }));
616
644
  }
617
645
  /**
646
+ * Where `order` goes among `calls`, which are in ascending order: calls finish in any order, but
647
+ * what they record stays in the order they were made.
648
+ */
649
+ function placeFor(calls, order) {
650
+ const after = calls.findIndex((c) => c > order);
651
+ return after < 0 ? calls.length : after;
652
+ }
653
+ /**
618
654
  * Grades a turn (the last by default) or a value against a question, under
619
655
  * the judge provider's concurrency lease. Every call's input is recorded, including calls that
620
656
  * fail. A call that fails returns a `JudgeFailure` and fails the attempt as a judge error after the eval's body, so the judges and
621
657
  * checks after it still run.
622
658
  */
623
659
  function judgeCaller(a, readFile) {
624
- const ask = async (question, { on, ...call } = {}) => {
660
+ const ask = async (order, question, { on, ...call } = {}) => {
625
661
  if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
626
662
  const problem = judgeProblem(call, { at: "call" });
627
663
  if (problem) throw new Error(problem);
@@ -632,8 +668,9 @@ function judgeCaller(a, readFile) {
632
668
  const options = mergeJudge(a.spec.judge, call);
633
669
  const { model } = options;
634
670
  if (model === void 0) {
635
- a.judgeErrored = true;
636
- throw new Error(noJudgeModel(getRunContext().config.file));
671
+ const message = noJudgeModel(getRunContext().config.file);
672
+ a.judgeThrown.add(message);
673
+ throw new Error(message);
637
674
  }
638
675
  const wanted = resolveStateOptions(options.state);
639
676
  if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
@@ -649,7 +686,7 @@ function judgeCaller(a, readFile) {
649
686
  listFiles: async () => workspaceFiles(await sandboxFor(a, "c.judge()"), a.signal)
650
687
  });
651
688
  const name = judgeModelName(model);
652
- const state = stateOf(draft);
689
+ const state = redactValue(stateOf(draft));
653
690
  const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
654
691
  const refs = artifactRefsOf(resolved);
655
692
  const decide = decideState(state, filePartsOf(resolved));
@@ -661,7 +698,9 @@ function judgeCaller(a, readFile) {
661
698
  ...refs.length ? { artifacts: refs } : {},
662
699
  ...dropped.length ? { dropped } : {}
663
700
  };
664
- a.record.judgeInputs?.push(input);
701
+ const place = placeFor(a.judgeInputCalls, order);
702
+ a.judgeInputCalls.splice(place, 0, order);
703
+ a.record.judgeInputs?.splice(place, 0, input);
665
704
  const start = performance.now();
666
705
  const ms = () => Math.round(performance.now() - start);
667
706
  const atLeast = options.threshold ?? .8;
@@ -669,12 +708,14 @@ function judgeCaller(a, readFile) {
669
708
  kind: "judge-start",
670
709
  question
671
710
  });
672
- debugLog("judge", `asking ${name}: ${JSON.stringify(clip(question, 160))}`);
711
+ debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
673
712
  if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
674
713
  debugTrace("judge", "prompt", input, "judge prompt");
675
714
  const failed = (message, retries) => {
676
- a.judgeErrored = true;
677
- a.judgeErrors.push(`c.judge(${JSON.stringify(clip(question, 80))}) failed: ${message}`);
715
+ a.judgeErrors.push({
716
+ call: order,
717
+ message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`
718
+ });
678
719
  input.error = message;
679
720
  if (retries?.length) input.retries = retries;
680
721
  debugLog("judge", `${name} · ${formatDuration(ms())} · error: ${message}`);
@@ -703,11 +744,13 @@ function judgeCaller(a, readFile) {
703
744
  const provider = name.split("/")[0];
704
745
  try {
705
746
  const { config, profile } = getRunContext();
706
- const gateway = resolveModelGateway(model, judgeOptions.gateway, gatewayScope(config, profile), "judge model");
747
+ const scope = gatewayScope(config, profile);
748
+ const gateway = resolveModelGateway(model, judgeOptions.gateway, scope, "judge model");
707
749
  const { gateway: _pin, ...unpinned } = judgeOptions;
750
+ const entry = gateway && scope.gateways.find((g) => g.kind === gateway);
708
751
  const routed = gateway ? {
709
752
  ...unpinned,
710
- gateway
753
+ gateway: entry || gateway
711
754
  } : unpinned;
712
755
  const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
713
756
  ...routed,
@@ -718,10 +761,12 @@ function judgeCaller(a, readFile) {
718
761
  ...verdict,
719
762
  artifacts: refs
720
763
  } : verdict;
721
- a.record.judgements.push(retries.length ? {
764
+ const { judgements: judgments } = a.record;
765
+ judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
722
766
  ...recorded,
723
- retries
724
- } : recorded);
767
+ ...retries.length ? { retries } : {},
768
+ call: order
769
+ });
725
770
  const { score } = verdict;
726
771
  debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
727
772
  debugTrace("judge", "response", {
@@ -738,21 +783,19 @@ function judgeCaller(a, readFile) {
738
783
  });
739
784
  return recorded;
740
785
  } catch (error) {
741
- if (a.signal.aborted) {
742
- a.judgeErrored = true;
743
- throw error;
744
- }
786
+ if (a.signal.aborted) throw error;
745
787
  const { retries } = error;
746
788
  const message = error instanceof Error ? error.message : String(error);
747
789
  return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
748
790
  }
749
791
  };
750
792
  return async (question, { on, ...options } = {}) => {
793
+ const order = a.judgeCalls++;
751
794
  const call = on === void 0 ? options : {
752
795
  ...options,
753
796
  on: judgeOn(a.turns, on)
754
797
  };
755
- return Object.defineProperty(await ask(question, call), SUBJECT, { value: question });
798
+ return Object.defineProperty(await ask(order, question, call), SUBJECT, { value: question });
756
799
  };
757
800
  }
758
801
  //#endregion
@@ -762,7 +805,7 @@ async function createSandbox(descriptor, options) {
762
805
  switch (descriptor.kind) {
763
806
  case "fake": return await createFakeSandbox(descriptor, options.env);
764
807
  case "docker": {
765
- const { createDockerSandbox } = await import("./docker-DivfZl2u.js");
808
+ const { createDockerSandbox } = await import("./docker-YFj4H1Cr.js");
766
809
  return await createDockerSandbox(descriptor, options);
767
810
  }
768
811
  }
@@ -781,7 +824,7 @@ function classify(error, signal) {
781
824
  message: "The eval timed out"
782
825
  } : {
783
826
  kind: "aborted",
784
- message: "The run was cancelled"
827
+ message: "The run was canceled"
785
828
  };
786
829
  if (error instanceof InfraError) return {
787
830
  kind: "infra",
@@ -792,17 +835,39 @@ function classify(error, signal) {
792
835
  message
793
836
  };
794
837
  }
838
+ const messageOf = (error) => error instanceof Error ? error.message : String(error);
839
+ const judged = (r) => /(^|\.)toPass$/.test(r.matcher);
840
+ /**
841
+ * Whether `error` is a judge's: one a judge threw, or what a `toPass()` or `.not.toPass()` call
842
+ * failed with. A failing verdict the eval didn't assert, or asserted with `.not.toPass()`, isn't.
843
+ * By message too, since another language's runner hands back its own copy of the error.
844
+ */
845
+ function isJudges(a, error, raw) {
846
+ const message = messageOf(error);
847
+ if (a.judgeThrown.has(message)) return true;
848
+ return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
849
+ }
850
+ /**
851
+ * The error that fails the attempt: a judge's gives way to the first other check that didn't hold
852
+ * (an `expect.soft`, or a hard check the eval caught), whichever came first, so the attempt fails
853
+ * as that check.
854
+ */
855
+ function attemptError(a, error, raw) {
856
+ if (!isJudges(a, error, raw)) return error;
857
+ const check = raw.find((r) => r.status !== "pass" && !judged(r));
858
+ return check ? checkFailure([check]) : error;
859
+ }
795
860
  /**
796
861
  * Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
797
- * assertion, which is the judge's when a judge call failed or gave a failing verdict.
862
+ * assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
798
863
  */
799
- function recordFailure(a, error, timedOut) {
864
+ function recordFailure(a, error, timedOut, raw) {
800
865
  const { record } = a;
801
866
  record.error = timedOut ? {
802
867
  kind: "timeout",
803
868
  message: "The eval timed out"
804
869
  } : classify(error, a.signal);
805
- record.error.message = capMiddle(record.error.message);
870
+ record.error.message = capMiddle(redactSecrets(record.error.message));
806
871
  if (error instanceof PrepareError && !timedOut) {
807
872
  const { class: kind, stage, ref, cause, log, setBy } = error.failure;
808
873
  record.error.prepare = {
@@ -814,8 +879,7 @@ function recordFailure(a, error, timedOut) {
814
879
  setBy: setBy ?? []
815
880
  };
816
881
  }
817
- const judgeFailed = a.judgeErrored || record.judgements.some((j) => j.verdict !== "pass");
818
- if (record.error.kind === "assertion" && judgeFailed) record.error.kind = "judge";
882
+ if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = "judge";
819
883
  record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
820
884
  }
821
885
  /**
@@ -842,11 +906,11 @@ async function recordOom(a, sandbox) {
842
906
  /**
843
907
  * The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
844
908
  * Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
845
- * summarised in one `-v` line.
909
+ * summarized in one `-v` line.
846
910
  */
847
911
  function recordEgress(a, sandbox) {
848
912
  const traffic = sandbox?.traffic?.();
849
- const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0);
913
+ const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
850
914
  const refused = traffic?.blocked ?? [];
851
915
  const isExpected = (h) => !!expected?.hosts.includes(h.host);
852
916
  const expectedBlocked = refused.filter(isExpected);
@@ -866,6 +930,24 @@ function recordEgress(a, sandbox) {
866
930
  a.record.servedModels = sandbox?.servedModels?.() ?? null;
867
931
  const asked = sandbox?.askedModels?.() ?? [];
868
932
  if (asked.length) a.record.askedModels = asked;
933
+ const upstreams = sandbox?.upstreams?.() ?? [];
934
+ if (upstreams.length) a.record.upstreams = upstreams;
935
+ const cost = sandbox?.reportedCostUsd?.();
936
+ if (cost !== void 0) a.record.reportedCostUsd = cost;
937
+ }
938
+ /**
939
+ * The record as its staging file holds it: redacted, with a secret split across a turn's raw
940
+ * stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
941
+ */
942
+ function stagedRecordJson(record) {
943
+ const turns = record.turns.map((turn) => turn.raw ? {
944
+ ...turn,
945
+ raw: redactSplitSecrets(turn.raw.map((part) => JSON.stringify(part))).map((line) => JSON.parse(line))
946
+ } : turn);
947
+ return redactJson({
948
+ ...record,
949
+ turns
950
+ }, 2);
869
951
  }
870
952
  /** Writes the attempt's record, redacted, and returns what the runner reports. */
871
953
  async function writeRecord(a, stagingDir, id) {
@@ -874,7 +956,7 @@ async function writeRecord(a, stagingDir, id) {
874
956
  await mkdir(join(stagingDir, "evals"), { recursive: true });
875
957
  await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
876
958
  const tmp = `${recordPath}.${process.pid}.tmp`;
877
- await writeFile(tmp, redactSecrets(JSON.stringify(record, null, 2)));
959
+ await writeFile(tmp, stagedRecordJson(record));
878
960
  await rename(tmp, recordPath);
879
961
  return {
880
962
  outcome: record.outcome,
@@ -896,7 +978,7 @@ async function writeRecord(a, stagingDir, id) {
896
978
  function turnErrors(transcripts) {
897
979
  return transcripts.flatMap((events, i) => {
898
980
  const found = events.find((e) => e.type === "error");
899
- return found?.type === "error" ? [`turn ${i + 1} error: ${redactSecrets(clip(found.message.trim(), 500))}`] : [];
981
+ return found?.type === "error" ? [`turn ${i + 1} error: ${clip(redactSecrets(found.message.trim()), 500)}`] : [];
900
982
  });
901
983
  }
902
984
  //#endregion
@@ -928,7 +1010,7 @@ function hookHandlers(plugin, text, hooks) {
928
1010
  const what = `the ${plugin.name} plugin's hooks/hooks.json`;
929
1011
  let raw;
930
1012
  try {
931
- raw = JSON.parse(text);
1013
+ raw = JSON.parse(withoutBom(text));
932
1014
  } catch {
933
1015
  throw new InfraError(`${what} isn't valid JSON`);
934
1016
  }
@@ -989,7 +1071,7 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
989
1071
  if (!hooks) {
990
1072
  let own = {};
991
1073
  try {
992
- own = hooksFile === null ? {} : JSON.parse(hooksFile);
1074
+ own = hooksFile === null ? {} : JSON.parse(withoutBom(hooksFile));
993
1075
  } catch {
994
1076
  throw new InfraError(`The workspace's ${unpack.hooks.file} isn't valid JSON, so reeval can't add the plugins' hooks to it`);
995
1077
  }
@@ -1157,8 +1239,10 @@ function createAttempt(spec, signal, progress, start) {
1157
1239
  }
1158
1240
  },
1159
1241
  signal,
1160
- judgeErrored: false,
1242
+ judgeThrown: /* @__PURE__ */ new Set(),
1161
1243
  judgeErrors: [],
1244
+ judgeCalls: 0,
1245
+ judgeInputCalls: [],
1162
1246
  captureNames: /* @__PURE__ */ new Set(),
1163
1247
  backgrounds: [],
1164
1248
  artifactBlobs: /* @__PURE__ */ new Map()
@@ -1206,7 +1290,7 @@ async function runSetup(a, box, env) {
1206
1290
  const steps = env.setup;
1207
1291
  record.setup = [];
1208
1292
  for (const [i, step] of steps.entries()) {
1209
- const what = `setup step ${i + 1}/${steps.length} "${clip(step.run, 80)}" in environment ${spec.environment.name}`;
1293
+ const what = `setup step ${i + 1}/${steps.length} "${clip(redactSecrets(step.run), 80)}" in environment ${spec.environment.name}`;
1210
1294
  const start = performance.now();
1211
1295
  const stepSignal = AbortSignal.any([a.signal, AbortSignal.timeout(step.timeoutMs)]);
1212
1296
  const r = await asCheck(a, () => box.exec(step.run, {
@@ -1226,7 +1310,7 @@ async function runSetup(a, box, env) {
1226
1310
  durationMs,
1227
1311
  output
1228
1312
  });
1229
- debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(step.run, 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
1313
+ debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(redactSecrets(step.run), 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
1230
1314
  debugTrace("env", "setup", {
1231
1315
  run: step.run,
1232
1316
  cwd: step.cwd,
@@ -1239,6 +1323,54 @@ async function runSetup(a, box, env) {
1239
1323
  throw new InfraError(r ? `${what} exited ${r.exitCode} after ${formatDuration(durationMs)} (see setup.log):\n${tailOf(r.stderr || r.stdout)}` : `${what} timed out after ${formatDuration(step.timeoutMs)} (see setup.log)`);
1240
1324
  }
1241
1325
  }
1326
+ /** The `to: 'checks'` secrets the setup steps got as their real value: those without `hosts`. */
1327
+ function unbrokeredChecks(env) {
1328
+ const brokered = new Set(env.substitutions.filter((s) => s.phase === "checks").map((s) => s.name));
1329
+ return Object.keys(env.checkVars).filter((name) => !brokered.has(name));
1330
+ }
1331
+ /** How many of the processes setup left running the record keeps, and how much of each command line. */
1332
+ const SETUP_PROCESSES = 20;
1333
+ const SETUP_COMMAND_CHARS = 200;
1334
+ /**
1335
+ * The processes the setup steps left running with an unbrokered `to: 'checks'` secret in their
1336
+ * environment, by the secret's variable name: a copy under another name or in a process's
1337
+ * arguments isn't found. The listing runs as the agent's user, with no checks secrets of its own,
1338
+ * so it sees the environments the agent could read, and prints names, never values. Docker only:
1339
+ * the fake sandbox runs on the host, with no separation to report on.
1340
+ */
1341
+ async function leftRunning(a, box, env) {
1342
+ const names = unbrokeredChecks(env);
1343
+ if (a.spec.sandbox.kind !== "docker" || !env.setup.length || !names.length) return [];
1344
+ const r = await box.exec([
1345
+ "for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
1346
+ "for p in /proc/[0-9]*; do",
1347
+ " [ \"$p\" = \"/proc/$$\" ] && continue",
1348
+ ` n=$(tr '\\0' '\\n' < "$p/environ" 2>/dev/null | sed -n -E 's/^(${names.join("|")})=.*/\\1/p' | sort -u | tr '\\n' ' ')`,
1349
+ " [ -n \"$n\" ] || continue",
1350
+ ` c=$(tr '\\0\\t\\n' ' ' < "$p/cmdline" 2>/dev/null)`,
1351
+ ` [ -n "$c" ] || c=$(tr '\\t\\n' ' ' < "$p/comm" 2>/dev/null)`,
1352
+ ` printf '%s\\t%s\\n' "$n" "$c"`,
1353
+ "done",
1354
+ "true"
1355
+ ].join("\n"), { signal: a.signal });
1356
+ if (r.exitCode !== 0) {
1357
+ debugLog("env", `couldn't list the processes setup left running: ${tailOf(r.stderr)}`);
1358
+ return [];
1359
+ }
1360
+ const known = new Set(names);
1361
+ const found = r.stdout.split("\n").flatMap((line) => {
1362
+ const tab = line.indexOf(" ");
1363
+ if (tab < 0) return [];
1364
+ const secrets = line.slice(0, tab).split(" ").filter((n) => known.has(n));
1365
+ const command = clip(redactSecrets(line.slice(tab + 1).trim()), SETUP_COMMAND_CHARS);
1366
+ return secrets.length ? [{
1367
+ command,
1368
+ secrets
1369
+ }] : [];
1370
+ });
1371
+ if (found.length > SETUP_PROCESSES) debugLog("env", `setup left ${found.length} processes running with a checks secret; the record keeps the first ${SETUP_PROCESSES}`);
1372
+ return found.slice(0, SETUP_PROCESSES);
1373
+ }
1242
1374
  const WORKSPACE_TAR = ".reeval-workspace.tar";
1243
1375
  /** Unpacks the git workspace's tarball into the (empty) workspace. */
1244
1376
  async function copyWorkspace(box, workspace, signal) {
@@ -1434,8 +1566,8 @@ async function startSession(a, runId) {
1434
1566
  signal: a.signal,
1435
1567
  env: env.vars,
1436
1568
  image: a.image,
1437
- brokering: brokeringFor(spec.harness),
1438
- expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0)?.hosts,
1569
+ brokering: brokeringFor(spec.harness, spec.vertex),
1570
+ expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
1439
1571
  substitutions: env.substitutions,
1440
1572
  egress: spec.egress
1441
1573
  });
@@ -1473,6 +1605,11 @@ async function startSession(a, runId) {
1473
1605
  for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
1474
1606
  for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
1475
1607
  await runSetup(a, box, env);
1608
+ const running = await leftRunning(a, box, env);
1609
+ if (running.length) {
1610
+ a.record.setupProcesses = running;
1611
+ debugLog("env", `setup left ${running.length} process${running.length === 1 ? "" : "es"} running with ${[...new Set(running.flatMap((p) => p.secrets))].join(", ")} (to: 'checks', no hosts) in the environment: ${running.map((p) => clip(p.command, 80)).join("; ")}`);
1612
+ }
1476
1613
  await snapshot(box, a.signal);
1477
1614
  debugLog("env", "workspace snapshot taken: the diff starts here");
1478
1615
  const timed = await timeMcpServers(box, env.mcp, a.signal);
@@ -1481,7 +1618,7 @@ async function startSession(a, runId) {
1481
1618
  debugLog("env", "stdio MCP servers start through reeval's MCP timer");
1482
1619
  } else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
1483
1620
  if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
1484
- const harness = await createHarness(spec.harness);
1621
+ const harness = await createHarness(spec.harness, spec.vertex);
1485
1622
  const harnessStart = performance.now();
1486
1623
  debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
1487
1624
  const session = await startHarness(a, harness, box, {
@@ -1514,7 +1651,7 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
1514
1651
  const cutoff = new Promise((resolve) => {
1515
1652
  onAbort = () => {
1516
1653
  timer = setTimeout(() => {
1517
- const message = `The agent didn't stop within ${grace / 1e3}s of being cancelled, so reeval stopped waiting for its turn`;
1654
+ const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
1518
1655
  const at = Date.now() - Date.parse(startedAt);
1519
1656
  resolve({
1520
1657
  text: "",
@@ -1610,7 +1747,7 @@ function turnSender(a, takeContext, checks) {
1610
1747
  kind: "turn-start",
1611
1748
  turn: turnNumber
1612
1749
  });
1613
- debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(prompt, 160))}`);
1750
+ debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(redactSecrets(prompt), 160))}`);
1614
1751
  debugTrace("harness", "prompt", {
1615
1752
  turn: turnNumber,
1616
1753
  prompt
@@ -1671,10 +1808,9 @@ function turnSender(a, takeContext, checks) {
1671
1808
  ms: result.durationMs,
1672
1809
  tools: count("tool", "web-search", "web-fetch"),
1673
1810
  fileEdits: count("file-edit"),
1674
- ...failure?.type === "error" ? { error: redactSecrets(clip(failure.message.trim(), 300)) } : {}
1811
+ ...failure?.type === "error" ? { error: clip(redactSecrets(failure.message.trim()), 300) } : {}
1675
1812
  });
1676
- const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
1677
- debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
1813
+ debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
1678
1814
  for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
1679
1815
  if (timedOut) {
1680
1816
  const error = new InfraError(timedOut);
@@ -1768,13 +1904,19 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1768
1904
  throw a.startFailure ? a.startFailure.error : error;
1769
1905
  });
1770
1906
  a.signal.throwIfAborted();
1771
- const missed = await unawaited() ?? softFailure(assertions()) ?? expectError();
1907
+ const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
1772
1908
  if (a.startFailure) throw a.startFailure.error;
1773
1909
  if (missed) throw missed;
1774
- const [only, ...more] = a.judgeErrors;
1775
- if (only) throw new Error(more.length ? `${more.length + 1} judge calls failed:\n${a.judgeErrors.join("\n")}` : only);
1776
- } catch (error) {
1777
- recordFailure(a, error, timedOut());
1910
+ const judgeErrors = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
1911
+ const [only, ...more] = judgeErrors;
1912
+ if (only) {
1913
+ const message = more.length ? `${more.length + 1} judge calls failed:\n${judgeErrors.join("\n")}` : only;
1914
+ a.judgeThrown.add(message);
1915
+ throw new Error(message);
1916
+ }
1917
+ } catch (thrown) {
1918
+ const error = attemptError(a, thrown, assertions());
1919
+ recordFailure(a, error, timedOut(), assertions());
1778
1920
  failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
1779
1921
  } finally {
1780
1922
  if (!a.signal.aborted) await unawaited().catch(() => {});
@@ -1796,7 +1938,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1796
1938
  const checked = assertionRecords(assertions());
1797
1939
  if (checked.length) record.assertions = checked;
1798
1940
  for (const x of checked) {
1799
- const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
1941
+ const why = x.message ? ` · ${clip(redactSecrets(x.message.split("\n")[0] ?? ""), 160)}` : "";
1800
1942
  debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
1801
1943
  }
1802
1944
  const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
@@ -1818,39 +1960,45 @@ const declaredModeSchema = z.enum([
1818
1960
  "run",
1819
1961
  "skip",
1820
1962
  "only"
1821
- ]).describe("How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`.");
1963
+ ]);
1822
1964
  const declaredFamilySchema = z.strictObject({
1823
- name: z.string().describe("The name template passed to `evaluate.each`."),
1824
- case: z.number().int().min(0).describe("The case's index, from 0."),
1825
- cases: z.number().int().min(2).describe("How many cases the list has.")
1826
- }).describe("Where a case of a pooled `evaluate.each` sits in its list.");
1965
+ name: z.string(),
1966
+ case: z.number().int().min(0),
1967
+ cases: z.number().int().min(2)
1968
+ });
1827
1969
  const declaredTodoSchema = z.strictObject({
1828
1970
  type: z.literal("todo"),
1829
- name: z.string().describe("The name passed to `.todo`.")
1830
- }).describe("`evaluate.todo` or `group.todo`.");
1971
+ name: z.string()
1972
+ });
1973
+ const hash = () => z.string().regex(/^[0-9a-f]{64}$/, "must be a sha256 hash: 64 lowercase hex digits");
1974
+ const declaredSourceSchema = z.strictObject({
1975
+ body: hash(),
1976
+ case: hash().optional()
1977
+ });
1831
1978
  const declaredEvalSchema = z.strictObject({
1832
1979
  type: z.literal("eval"),
1833
- name: z.string().describe("The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`)."),
1980
+ name: z.string(),
1834
1981
  mode: declaredModeSchema,
1835
- options: evalOptionsDataSchema.describe("The eval's own options, before the cascade."),
1836
- body: z.string().describe("Names the eval's body to whoever runs it."),
1837
- family: declaredFamilySchema.optional().describe("Set on each case of a pooled `evaluate.each`: its template, index and the list's length.")
1838
- }).describe("An `evaluate`: one eval, planned into a test per attempt.");
1982
+ options: evalOptionsDataSchema,
1983
+ body: z.string(),
1984
+ source: declaredSourceSchema,
1985
+ family: declaredFamilySchema.optional()
1986
+ });
1839
1987
  const declaredGroupSchema = z.strictObject({
1840
1988
  type: z.literal("group"),
1841
- name: z.string().describe("The group's name, as passed to `group`."),
1989
+ name: z.string(),
1842
1990
  mode: declaredModeSchema,
1843
- options: evalOptionsDataSchema.describe("The group's own options, before the cascade."),
1991
+ options: evalOptionsDataSchema,
1844
1992
  get children() {
1845
1993
  return z.array(declaredNodeSchema);
1846
1994
  }
1847
- }).describe("A `group`: options its evals and groups inherit, and its children in declaration order.");
1995
+ });
1848
1996
  const declaredNodeSchema = z.discriminatedUnion("type", [
1849
1997
  declaredEvalSchema,
1850
1998
  declaredGroupSchema,
1851
1999
  declaredTodoSchema
1852
- ]).describe("One node of the tree: a group, an eval or a todo.");
1853
- const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) }).describe("An eval file's top-level groups, evals and todos, in declaration order.");
2000
+ ]);
2001
+ const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) });
1854
2002
  const NODE_SCHEMAS = {
1855
2003
  eval: declaredEvalSchema,
1856
2004
  group: declaredGroupSchema,
@@ -1881,18 +2029,15 @@ function configLevel(config) {
1881
2029
  questions: [],
1882
2030
  timeout: void 0,
1883
2031
  tags: [],
2032
+ revision: void 0,
1884
2033
  pin: [],
1885
2034
  sources: {}
1886
2035
  };
1887
2036
  }
1888
- /**
1889
- * The tests an eval registers, or `null` when the run's tags leave it out (it isn't counted in
1890
- * `order`). Throws a UsageError for options the plan refuses.
1891
- */
2037
+ /** The tests an eval registers. Throws a UsageError for options the plan refuses. */
1892
2038
  function planEval(declaration, ctx) {
1893
- const { name, mode, groups: enclosing, options: own } = declaration;
1894
- const groups = enclosing.map((g) => g.name);
1895
- const where = `evaluate("${[...groups, name].join(" › ")}")`;
2039
+ const { name, groups: enclosing, options: own } = declaration;
2040
+ const where = `evaluate("${[...enclosing.map((g) => g.name), name].join(" › ")}")`;
1896
2041
  checkOptions(own, where);
1897
2042
  const levels = [...enclosing.map((g) => ({
1898
2043
  options: g.options,
@@ -1904,25 +2049,54 @@ function planEval(declaration, ctx) {
1904
2049
  const resolvedEval = cascade(configLevel(ctx.config), levels, where, {
1905
2050
  profile: ctx.profile,
1906
2051
  runs: ctx.runs,
1907
- skipped: mode === "skip" || mode === "todo"
2052
+ skipped: declaration.mode === "skip" || declaration.mode === "todo",
2053
+ pooled: declaration.family !== void 0
1908
2054
  });
2055
+ const { harnesses } = resolvedEval;
2056
+ if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
2057
+ tests: null,
2058
+ harnesses
2059
+ };
2060
+ return {
2061
+ tests: planTests(declaration, ctx, where, resolvedEval),
2062
+ harnesses
2063
+ };
2064
+ }
2065
+ function planTests(declaration, ctx, where, resolvedEval) {
2066
+ const { name, mode, groups: enclosing } = declaration;
2067
+ const groups = enclosing.map((g) => g.name);
1909
2068
  const { resolved } = resolvedEval;
1910
- if (!matchesTags(resolved.tags, ctx.tags)) return null;
2069
+ const source = {
2070
+ ...declaration.source,
2071
+ options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
2072
+ };
1911
2073
  const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
1912
2074
  const gateways = ctx.profile?.gateway ?? ctx.config.gateway;
1913
2075
  const gatewaySet = ctx.config.gatewaySet || ctx.profile?.gateway !== void 0;
1914
2076
  const routes = /* @__PURE__ */ new Map();
1915
2077
  const gatewayNotes = [];
2078
+ let vertex;
1916
2079
  const routed = resolvedEval.skip ? resolved.harness : resolved.harness.map((harness) => {
1917
2080
  const route = resolveGateway(gateways, harness, where);
1918
2081
  const hid = harnessId(harness);
1919
2082
  if (gatewaySet || harness.gateway !== void 0) {
1920
2083
  routes.set(hid, route);
1921
- gatewayNotes.push(`gateway: ${hid} → ${route.name}`);
2084
+ gatewayNotes.push(`gateway: ${hid} → ${route.name}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
1922
2085
  }
1923
2086
  if (route.name === "direct") return harness;
1924
- const problem = harness.kind === "fake" ? void 0 : vercelCredentialProblem();
1925
- if (problem) throw new UsageError(`${where} on ${hid}: ${problem}`);
2087
+ if (harness.kind !== "fake") try {
2088
+ if (route.name === "vercel") {
2089
+ const problem = vercelCredentialProblem();
2090
+ if (problem) throw new UsageError(problem);
2091
+ }
2092
+ if (route.name === "vertex") {
2093
+ const entry = gateways.find((g) => g.kind === "vertex");
2094
+ vertex ??= vertexSettings(entry?.kind === "vertex" ? entry : {});
2095
+ }
2096
+ } catch (error) {
2097
+ if (error instanceof UsageError) throw new UsageError(`${where} on ${hid}: ${error.message}`);
2098
+ throw error;
2099
+ }
1926
2100
  return {
1927
2101
  ...harness,
1928
2102
  gateway: route.name
@@ -1931,9 +2105,16 @@ function planEval(declaration, ctx) {
1931
2105
  const plannedAttempts = resolvedEval.skip ? [] : expand({
1932
2106
  ...resolved,
1933
2107
  harness: routed
1934
- });
2108
+ }, vertex);
2109
+ let judge;
1935
2110
  if (plannedAttempts.length) try {
1936
- resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
2111
+ const gateway = resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
2112
+ const { model } = resolved.judge;
2113
+ if (typeof model === "string") judge = {
2114
+ model,
2115
+ ...gateway ? { gateway } : {}
2116
+ };
2117
+ if (gatewaySet && fellBackToDirect(gateway, resolved.judge.gateway, { gateways })) gatewayNotes.push(`gateway: judge ${model} → direct (fallback)`);
1937
2118
  } catch (error) {
1938
2119
  if (error instanceof UsageError) throw new UsageError(`${where}: ${error.message}`);
1939
2120
  throw error;
@@ -1941,6 +2122,8 @@ function planEval(declaration, ctx) {
1941
2122
  if (ctx.config.secrets.unbrokered === "error") {
1942
2123
  const [first] = unbrokered(resolved.env, plannedAttempts);
1943
2124
  if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
2125
+ const [plain] = plainHeaders(plannedAttempts);
2126
+ if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
1944
2127
  }
1945
2128
  const pluginNotes = [];
1946
2129
  const pluginLabels = /* @__PURE__ */ new Map();
@@ -1966,6 +2149,7 @@ function planEval(declaration, ctx) {
1966
2149
  configFile: ctx.config.file,
1967
2150
  file: path
1968
2151
  });
2152
+ if (gatewayNotes.length && openRouterFallbacks(gateways)) gatewayNotes.push("gateway: openrouter() fallbacks on: reeval's own calls may fail over to another upstream provider; the harnesses' calls take OpenRouter's own routing");
1969
2153
  plan.notes.push(...gatewayNotes, ...pluginNotes);
1970
2154
  const setByOf = (planned) => ({
1971
2155
  sandbox: sandboxSetBy(planned, resolved, ctx.config.file),
@@ -1978,6 +2162,8 @@ function planEval(declaration, ctx) {
1978
2162
  name,
1979
2163
  groups,
1980
2164
  tags: resolved.tags,
2165
+ ...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
2166
+ source,
1981
2167
  cell: first.cell,
1982
2168
  n: 1,
1983
2169
  label: "",
@@ -2014,6 +2200,8 @@ function planEval(declaration, ctx) {
2014
2200
  name,
2015
2201
  groups,
2016
2202
  tags: resolved.tags,
2203
+ ...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
2204
+ source,
2017
2205
  cell: planned.cell,
2018
2206
  n: planned.n,
2019
2207
  label: planned.label,
@@ -2035,13 +2223,14 @@ function planEval(declaration, ctx) {
2035
2223
  } : plan,
2036
2224
  ...skip ? { skip } : {},
2037
2225
  ...family ? { family } : {},
2038
- ...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {}
2226
+ ...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {},
2227
+ ...judge ? { judge } : {}
2039
2228
  };
2040
2229
  return {
2041
2230
  name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
2042
2231
  mode: skip ? "skip" : mode,
2043
2232
  attempt,
2044
- timeout: skip ? void 0 : timeoutMs + setupAllowance(planned) + GRACE_MS,
2233
+ timeout: skip ? void 0 : Math.min(timeoutMs + setupAllowance(planned) + GRACE_MS, MAX_TIMER_MS),
2045
2234
  gate: skip ? void 0 : {
2046
2235
  limits: attemptLimits(planned, ctx.concurrency),
2047
2236
  rank: {
@@ -2221,16 +2410,54 @@ function registerTodo(name) {
2221
2410
  const tags = [...new Set(groupStack.flatMap((g) => g.options.tags ?? []))];
2222
2411
  if (matchesTags(tags, getRunContext().tags)) registerTest(name, { mode: "todo" }, async () => {});
2223
2412
  }
2224
- /** The evals declared so far, by file and name path, so a second one with the same id is refused. */
2225
- const declared = /* @__PURE__ */ new Set();
2226
- function checkUnique(where, groups, name, hint) {
2227
- const key = `${collectingFile() ?? ""}\0${[...groups, name].join("\0")}`;
2228
- if (declared.has(key)) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}`);
2229
- declared.add(key);
2413
+ /**
2414
+ * The evals declared so far, by id, with their name paths and whether each is a family's case,
2415
+ * so a second id is refused.
2416
+ */
2417
+ const declared = /* @__PURE__ */ new Map();
2418
+ /** The pooled families declared so far, by id: their name path, length and the cases seen. */
2419
+ const families = /* @__PURE__ */ new Map();
2420
+ /** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
2421
+ const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
2422
+ /** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
2423
+ const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as the pooled evaluate.each("${family.join(" › ")}") in this file: a family's id is its name template. Give the eval or the template another name.${separatorHint(eval_, family)}`);
2424
+ function checkUnique(where, groups, name, hint, inFamily) {
2425
+ const path = [...groups, name];
2426
+ const id = evalId(collectingFile() ?? "", path);
2427
+ const earlier = declared.get(id);
2428
+ if (earlier) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}${separatorHint(path, earlier.path)}`);
2429
+ const family = inFamily ? void 0 : families.get(id);
2430
+ if (family) throw namedLikeFamily(path, family.path);
2431
+ declared.set(id, {
2432
+ path,
2433
+ inFamily
2434
+ });
2230
2435
  }
2231
- function register(mode, name, own, fn, hint = "", family) {
2436
+ /**
2437
+ * A family's cases pool by its id, the groups and the template, so each case may be declared
2438
+ * once and every case must agree on the list's length.
2439
+ */
2440
+ function checkFamily(groups, family) {
2441
+ const path = [...groups, family.name];
2442
+ const id = evalId(collectingFile() ?? "", path);
2443
+ const earlier = families.get(id);
2444
+ if (!earlier) {
2445
+ const plain = declared.get(id);
2446
+ if (plain && !plain.inFamily) throw namedLikeFamily(plain.path, path);
2447
+ families.set(id, {
2448
+ path,
2449
+ cases: family.cases,
2450
+ seen: /* @__PURE__ */ new Set([family.case])
2451
+ });
2452
+ return;
2453
+ }
2454
+ if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate.each("${path.join(" › ")}") is declared twice in this file. Each pooled evaluate.each needs its own name, or their cases are pooled into one family. Give each its own name template, or pass evaluate.each(cases, { pool: false }) to keep the cases apart.${separatorHint(path, earlier.path)}`);
2455
+ earlier.seen.add(family.case);
2456
+ }
2457
+ function register(mode, name, own, fn, source, hint = "", family) {
2232
2458
  const groups = groupStack.map((g) => g.name);
2233
- checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint);
2459
+ if (family) checkFamily(groups, family);
2460
+ checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint, !!family);
2234
2461
  const body = `b${bodies.size + 1}`;
2235
2462
  bodies.set(body, fn);
2236
2463
  declare({
@@ -2239,9 +2466,10 @@ function register(mode, name, own, fn, hint = "", family) {
2239
2466
  mode,
2240
2467
  options: data(own),
2241
2468
  body,
2469
+ source,
2242
2470
  ...family ? { family } : {}
2243
2471
  });
2244
- const tests = planEval({
2472
+ const { tests, harnesses } = planEval({
2245
2473
  name,
2246
2474
  mode,
2247
2475
  groups: groupStack.map(({ name, options }) => ({
@@ -2249,22 +2477,25 @@ function register(mode, name, own, fn, hint = "", family) {
2249
2477
  options
2250
2478
  })),
2251
2479
  options: own,
2480
+ source,
2252
2481
  ...family ? { family } : {}
2253
2482
  }, {
2254
2483
  ...getRunContext(),
2255
2484
  file: collectingFile(),
2256
2485
  order: registered
2257
2486
  });
2487
+ declareHarnesses(harnesses);
2258
2488
  if (!tests) return;
2259
2489
  registered++;
2260
2490
  for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
2261
2491
  }
2262
2492
  /** Evals registered so far in this worker, for the scheduler's tie-break. */
2263
2493
  let registered = 0;
2264
- const defineEval = (mode) => (name, a, b) => {
2494
+ const defineEval = (mode, declared) => (name, a, b) => {
2265
2495
  checkOnly(mode, "evaluate");
2266
2496
  const [own, fn] = splitArgs("evaluate", name, a, b);
2267
- register(mode, name, withPaths(own, `evaluate("${name}")`), fn);
2497
+ const where = `evaluate("${name}")`;
2498
+ register(mode, name, withPaths(own, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
2268
2499
  };
2269
2500
  /** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
2270
2501
  function formatName(template, item, index) {
@@ -2290,13 +2521,19 @@ const evalEach = (cases, options) => (name, a, b) => {
2290
2521
  const own = withPaths(written, `evaluate.each("${name}")`);
2291
2522
  const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
2292
2523
  const pooled = pool && cases.length >= 2;
2524
+ const file = collectingFile();
2525
+ const body = bodyHash(fn, `evaluate.each("${name}")`, file);
2293
2526
  cases.forEach((item, i) => {
2294
2527
  const family = pooled ? {
2295
2528
  name,
2296
2529
  case: i,
2297
2530
  cases: cases.length
2298
2531
  } : void 0;
2299
- register("run", formatName(name, item, i), own, (c) => fn(c, item), hint, family);
2532
+ const source = {
2533
+ body,
2534
+ case: caseHash(item, `evaluate.each("${name}") case ${i}`, file)
2535
+ };
2536
+ register("run", formatName(name, item, i), own, (c) => fn(c, item), source, hint, family);
2300
2537
  });
2301
2538
  };
2302
2539
  const evaluate = Object.assign(defineEval("run"), {
@@ -2326,8 +2563,8 @@ function declareNodes(nodes, bodyFor, base) {
2326
2563
  } else if (node.family) {
2327
2564
  const options = optionsFrom(node.options, `evaluate("${node.name}")`, base);
2328
2565
  checkOnly(node.mode, "evaluate");
2329
- register(node.mode, node.name, options, bodyFor(node.body), "", node.family);
2330
- } else defineEval(node.mode)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
2566
+ register(node.mode, node.name, options, bodyFor(node.body), node.source, "", node.family);
2567
+ } else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
2331
2568
  }
2332
2569
  /**
2333
2570
  * Options from their data forms, with host paths absolute against `base`: environments become
@@ -2364,4 +2601,4 @@ function optionsFrom(written, where, base) {
2364
2601
  //#endregion
2365
2602
  export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
2366
2603
 
2367
- //# sourceMappingURL=evaluate-BUmsN7yG.js.map
2604
+ //# sourceMappingURL=evaluate-BPP9s3s6.js.map