reeval 0.14.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/README.md +3 -3
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
  9. package/dist/app-files-ChbvRx3P.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
  11. package/dist/assemble-SKQGW_QF.js.map +1 -0
  12. package/dist/assemble-XBWuK3Ed.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
  16. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
  18. package/dist/capture-BM_3100g.js.map +1 -0
  19. package/dist/capture-DSK8O5bA.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
  21. package/dist/cli-PdYrcoKM.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
  23. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
  26. package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
  27. package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
  28. package/dist/command-4pJYVCT4.js.map +1 -0
  29. package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
  30. package/dist/command-9xm5ImqI.js.map +1 -0
  31. package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
  32. package/dist/command-BqBxLgTc.js.map +1 -0
  33. package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
  34. package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
  35. package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
  36. package/dist/commands-BU6YJCz9.js.map +1 -0
  37. package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
  38. package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
  39. package/dist/config-k6m7XbAz.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-CpsifVln.js +1530 -0
  46. package/dist/db-CpsifVln.js.map +1 -0
  47. package/dist/db-DkD-F-X-.js +2 -0
  48. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  49. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  50. package/dist/debug-DeM5pswK.js +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
  52. package/dist/docker-51WvhN_X.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
  54. package/dist/docker-B4-rx4dA.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
  57. package/dist/doctor-BuQnfz4e.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
  61. package/dist/environment-BOFgzFDg.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  63. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  64. package/dist/eval-source-Dje3rERU.js +145 -0
  65. package/dist/eval-source-Dje3rERU.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
  67. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  68. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  69. package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
  70. package/dist/files-Bi461Be2.js.map +1 -0
  71. package/dist/files-C2h2Cw4F.js +1 -0
  72. package/dist/files-Clipx4OW.js +37 -0
  73. package/dist/files-Clipx4OW.js.map +1 -0
  74. package/dist/files-DmUrRmmL.js +1 -0
  75. package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
  76. package/dist/files-H24pz0hL.js.map +1 -0
  77. package/dist/gateway.d.ts +22 -9
  78. package/dist/gateway.js +30 -8
  79. package/dist/gateway.js.map +1 -1
  80. package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
  81. package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
  82. package/dist/harness-3NVyTSGK.js +392 -0
  83. package/dist/harness-3NVyTSGK.js.map +1 -0
  84. package/dist/harness.d.ts +12 -8
  85. package/dist/harness.js +1 -1
  86. package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
  87. package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
  88. package/dist/index.d.ts +51 -10
  89. package/dist/index.js +5 -5
  90. package/dist/init-BDEOopK7.js +1 -0
  91. package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
  92. package/dist/init-cOk3D44b.js.map +1 -0
  93. package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
  94. package/dist/invocations-DW-lTk2f.js.map +1 -0
  95. package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
  96. package/dist/judge-CKdgEXN7.js.map +1 -0
  97. package/dist/judge.d.ts +5 -7
  98. package/dist/judge.js +4 -8
  99. package/dist/judge.js.map +1 -1
  100. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  101. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  102. package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
  103. package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
  104. package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
  105. package/dist/model-Bq3G0OG1.js.map +1 -0
  106. package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
  107. package/dist/model-CTnyvh7H.js.map +1 -0
  108. package/dist/model-D-SWrLZG.js +1 -0
  109. package/dist/model-g3XnzUv0.js +1 -0
  110. package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
  111. package/dist/model-vRiUycWh.js.map +1 -0
  112. package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
  113. package/dist/node-BVbbnFtv.js.map +1 -0
  114. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  115. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  116. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  117. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  118. package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
  119. package/dist/pairs-puYi-zj2.js.map +1 -0
  120. package/dist/patch-CquGAGX5.js +76 -0
  121. package/dist/patch-CquGAGX5.js.map +1 -0
  122. package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
  123. package/dist/patch-DKJiC5Sq.js.map +1 -0
  124. package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
  125. package/dist/plan-DdWkM6SW.js.map +1 -0
  126. package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
  127. package/dist/plugins-D-CP6mMd.js.map +1 -0
  128. package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
  129. package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
  130. package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
  131. package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
  132. package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
  133. package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
  134. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  135. package/dist/prices-QaS-kjSQ.js.map +1 -0
  136. package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
  137. package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
  138. package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
  139. package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
  140. package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
  141. package/dist/query-Clm2mNSm.js +1 -0
  142. package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
  143. package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
  144. package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
  145. package/dist/questions-BRhU8MzB.js.map +1 -0
  146. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  147. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  148. package/dist/redact-DVIaahLv.js +658 -0
  149. package/dist/redact-DVIaahLv.js.map +1 -0
  150. package/dist/remarks-BwJ8pcTQ.js +1 -0
  151. package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
  152. package/dist/remarks-C7i2LMRb.js.map +1 -0
  153. package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
  154. package/dist/remarks-D_3qhBWU.js.map +1 -0
  155. package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
  156. package/dist/remarks-dA8LyoPc.js.map +1 -0
  157. package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
  158. package/dist/remote-DFIzcx_6.js.map +1 -0
  159. package/dist/resolve--rQZ8u6Z.js +1 -0
  160. package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
  161. package/dist/resolve-DWSJpKHX.js.map +1 -0
  162. package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
  163. package/dist/results-BC3pcG_Z.js.map +1 -0
  164. package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
  165. package/dist/resume-j-rcIryi.js.map +1 -0
  166. package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
  167. package/dist/runtime-d56Gg3J5.js.map +1 -0
  168. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
  169. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
  170. package/dist/sandbox.d.ts +1 -1
  171. package/dist/sandbox.js +1 -1
  172. package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
  173. package/dist/server-BOoH10J8.js.map +1 -0
  174. package/dist/server-DpuVAbrO.js +1 -0
  175. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  176. package/dist/shared-B_lvGqWz.js.map +1 -0
  177. package/dist/source-BFd6f0-R.js +1 -0
  178. package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
  179. package/dist/source-wD04-rRG.js.map +1 -0
  180. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  181. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  182. package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
  183. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  184. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  185. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  186. package/dist/src-BO8AYqJ5.js.map +1 -0
  187. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  188. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  189. package/dist/src-BYPMM1x7.js +2 -0
  190. package/dist/src-Bp4aN3rA.js +2 -0
  191. package/dist/src-C7upnZSd.js +2 -0
  192. package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
  193. package/dist/src-D91Hd6hT.js.map +1 -0
  194. package/dist/src-DH-9IO7f.js +1 -0
  195. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  196. package/dist/src-DSZyg0Uq.js.map +1 -0
  197. package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
  198. package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
  199. package/dist/src-HSxBazcg.js +3 -0
  200. package/dist/src-LD1BAJFa.js +2 -0
  201. package/dist/src-dlX-Lmxg.js +2 -0
  202. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  203. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  204. package/dist/src-kYeaJJrl.js +2 -0
  205. package/dist/state-B5PLm5qY.js +1 -0
  206. package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
  207. package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
  208. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  209. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  210. package/dist/suite-22Kt9Grx.js +2 -0
  211. package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
  212. package/dist/suite-Co8cwelb.js.map +1 -0
  213. package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
  214. package/dist/summary-DLFHVoux.js.map +1 -0
  215. package/dist/testing.d.ts +417 -142
  216. package/dist/testing.js +12 -10
  217. package/dist/testing.js.map +1 -1
  218. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  219. package/dist/text-DTigcA64.js.map +1 -0
  220. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  221. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  222. package/dist/types-C5vI3g1C.js.map +1 -1
  223. package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
  224. package/dist/vertex-Bv7vhMgs.js +1 -0
  225. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  226. package/dist/vertex-DypYjx71.js +456 -0
  227. package/dist/vertex-DypYjx71.js.map +1 -0
  228. package/dist/view/index.d.ts +536 -121
  229. package/dist/view/index.js +9 -9
  230. package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
  231. package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
  232. package/dist/views-BOf8qEL8.js +2 -0
  233. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
  234. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  235. package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
  236. package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
  237. package/dist/workspace-DDHzkMjm.js +1 -0
  238. package/docker/seccomp-default.json +1 -0
  239. package/package.json +5 -1
  240. package/skills/eval-rubric/SKILL.md +35 -104
  241. package/dist/adapter-BloGVZ_a.js.map +0 -1
  242. package/dist/agent-m-_t-IuW.js.map +0 -1
  243. package/dist/app-files-C-jwhRDt.js.map +0 -1
  244. package/dist/assemble-BDEpelpi.js.map +0 -1
  245. package/dist/assemble-C5bfncHU.js +0 -1
  246. package/dist/capture-CVRwRTnb.js.map +0 -1
  247. package/dist/capture-CherbjzB.js +0 -1
  248. package/dist/cli-C_4gaJ50.js.map +0 -1
  249. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  250. package/dist/command-BWAuwQNk.js +0 -91
  251. package/dist/command-BWAuwQNk.js.map +0 -1
  252. package/dist/command-BXLUfw4h.js.map +0 -1
  253. package/dist/command-Be6BSE2L.js +0 -335
  254. package/dist/command-Be6BSE2L.js.map +0 -1
  255. package/dist/command-Bxdy62dW.js.map +0 -1
  256. package/dist/commands-jzigbNFb.js.map +0 -1
  257. package/dist/config-BrdHA9bJ.js.map +0 -1
  258. package/dist/cost-D4elx933.js.map +0 -1
  259. package/dist/cost-DmnAod9x.js +0 -1
  260. package/dist/db-CAfksinX.js +0 -2
  261. package/dist/db-DO0p7xgP.js +0 -619
  262. package/dist/db-DO0p7xgP.js.map +0 -1
  263. package/dist/debug-D7bEv_UV.js +0 -1
  264. package/dist/docker-BnRlTpZA.js.map +0 -1
  265. package/dist/docker-D01slDF3.js.map +0 -1
  266. package/dist/doctor-DQ2p0euG.js.map +0 -1
  267. package/dist/env-BQqj93gu.js.map +0 -1
  268. package/dist/environment-CO0P0soF.js.map +0 -1
  269. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  270. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  271. package/dist/facts-DzOUTreB.js +0 -906
  272. package/dist/facts-DzOUTreB.js.map +0 -1
  273. package/dist/files-B-CJeeEh.js.map +0 -1
  274. package/dist/files-Bp5j5BqI.js +0 -1
  275. package/dist/files-C2sjmXhk.js +0 -1
  276. package/dist/files-C_gQNVB-.js +0 -65
  277. package/dist/files-C_gQNVB-.js.map +0 -1
  278. package/dist/files-kUCrswI6.js.map +0 -1
  279. package/dist/generate-DZ7vp69F.js +0 -1647
  280. package/dist/generate-DZ7vp69F.js.map +0 -1
  281. package/dist/harness-COVXSwJ6.js +0 -354
  282. package/dist/harness-COVXSwJ6.js.map +0 -1
  283. package/dist/header-BLPP6aCU.js.map +0 -1
  284. package/dist/init-RgCQXZhb.js +0 -1
  285. package/dist/init-SRalM5xF.js.map +0 -1
  286. package/dist/insights-TyAvz9aV.js +0 -68
  287. package/dist/insights-TyAvz9aV.js.map +0 -1
  288. package/dist/invocations-DzyKSJBa.js.map +0 -1
  289. package/dist/judge-yLvjC5A3.js.map +0 -1
  290. package/dist/match-Bz0nSdzN.js.map +0 -1
  291. package/dist/model-1ZLr9Dcv.js.map +0 -1
  292. package/dist/model-B45lDUV1.js.map +0 -1
  293. package/dist/model-BTZq8HIZ.js +0 -1
  294. package/dist/model-CgnOpss4.js.map +0 -1
  295. package/dist/model-Ch8kX87u.js +0 -1
  296. package/dist/node-BSq5qHnQ.js.map +0 -1
  297. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  298. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  299. package/dist/pairs-CHfZNRT6.js.map +0 -1
  300. package/dist/patch-DWf-I_Q_.js.map +0 -1
  301. package/dist/plan-BIwUzlB8.js.map +0 -1
  302. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  303. package/dist/prices-BcLEXvdY.js.map +0 -1
  304. package/dist/query-Cvbwl804.js +0 -1
  305. package/dist/questions-bAudIc04.js.map +0 -1
  306. package/dist/redact-CRGdfV2s.js +0 -240
  307. package/dist/redact-CRGdfV2s.js.map +0 -1
  308. package/dist/remarks-CXwSnSjR.js +0 -278
  309. package/dist/remarks-CXwSnSjR.js.map +0 -1
  310. package/dist/remarks-CyAu2DiB.js +0 -1
  311. package/dist/remarks-mujJv1Ey.js.map +0 -1
  312. package/dist/remote-CKcEKja4.js.map +0 -1
  313. package/dist/resolve-B5-uqmJ6.js +0 -1
  314. package/dist/resolve-CBoKzQMP.js.map +0 -1
  315. package/dist/results-CMgZEANy.js.map +0 -1
  316. package/dist/resume-B9SOSnK9.js.map +0 -1
  317. package/dist/rules-VhftPOOT.js.map +0 -1
  318. package/dist/run-CwIEK16q.js +0 -88
  319. package/dist/run-CwIEK16q.js.map +0 -1
  320. package/dist/runtime-BgQRyCLl.js.map +0 -1
  321. package/dist/server-CHRSgKRp.js +0 -1
  322. package/dist/server-TGW4WyAz.js.map +0 -1
  323. package/dist/shared-BDUzStad.js.map +0 -1
  324. package/dist/source-B1WvBIQt.js.map +0 -1
  325. package/dist/source-DAzztZwU.js +0 -1
  326. package/dist/src-1jHbFaGW.js +0 -1
  327. package/dist/src-8zoX_qrb.js.map +0 -1
  328. package/dist/src-C_9wgowC.js +0 -2
  329. package/dist/src-ChhtI6BP.js.map +0 -1
  330. package/dist/src-D005Cbqz.js +0 -2
  331. package/dist/src-DIfRbp5a.js +0 -3
  332. package/dist/src-Dg8121Xy.js +0 -2
  333. package/dist/src-ElBqEN-S.js +0 -2
  334. package/dist/src-NSdQ9zI6.js.map +0 -1
  335. package/dist/src-YHGaUrGE.js +0 -2
  336. package/dist/src-mJVxw1Jm.js +0 -2
  337. package/dist/state-vznMh2wl.js +0 -1
  338. package/dist/suite-BsWGHzlG.js +0 -2
  339. package/dist/suite-WgT7m6Vs.js.map +0 -1
  340. package/dist/summary-DIGEVr08.js.map +0 -1
  341. package/dist/text-CrVy5ioL.js.map +0 -1
  342. package/dist/transcript-BTxJpW4R.js.map +0 -1
  343. package/dist/usage-DF3_p5pE.js +0 -1
  344. package/dist/validate-BhwqrVsm.js.map +0 -1
  345. package/dist/views-BW_C5eVH.js +0 -2
  346. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  347. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,33 +1,35 @@
1
1
  import { n as UsageError } from "./errors-B6BSMs03.js";
2
- import { p as parseDuration } from "./shared-BDUzStad.js";
3
- import { M as workspaceLabel, T as profileSandbox, p as environmentSandbox } from "./environment-CO0P0soF.js";
2
+ import { h as parseDuration } from "./shared-B_lvGqWz.js";
3
+ import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-BOFgzFDg.js";
4
+ import { C as pluginBuildsLabel } from "./plugins-D-CP6mMd.js";
4
5
  import { a as fromCwd, o as newRunId } from "./ids-Bp6ut7ZD.js";
5
- import { G as ciRunner, K as directOnCloud, O as harnessId, P as sandboxId, Y as isDirect, Z as resolveOutput, _ as selectProfile, a as configPathFor, c as findConfig, d as loadConfig, f as malformedCredentials, g as runJudgeModel, h as resolveConcurrency, i as configLabel, j as plannedSandboxes, l as hasAnalysisModel, m as requireConfig, p as missingCredentials, u as hostCapacity, v as gatewayScope, y as resolveModelGateway } from "./config-BrdHA9bJ.js";
6
- import { c as judgeModelName, p as unusedGatewayKey } from "./judge-yLvjC5A3.js";
7
- import { c as registerSecret, s as redactSecrets } from "./redact-CRGdfV2s.js";
8
- import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
9
- import { G as LABEL, ot as imageInfo } from "./src-8zoX_qrb.js";
6
+ import { $ as isDirect, A as harnessId, E as checkLimitHarness, F as plannedSandboxes, R as sandboxId, X as directOnCloud, Y as ciRunner, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, tt as resolveOutput, u as loadConfig, v as gatewayScope } from "./config-k6m7XbAz.js";
7
+ import { l as judgeModelName, m as unusedGatewayKey } from "./judge-CKdgEXN7.js";
8
+ import { y as vertexSettingsIn } from "./vertex-DypYjx71.js";
9
+ import { _ as registerSecret, m as redactSecrets } from "./redact-DVIaahLv.js";
10
+ import { a as plural, i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-DTigcA64.js";
11
+ import { n as loadEnvFiles, t as inCI } from "./env-CARj5Axq.js";
10
12
  import { o as PACKAGE_VERSION } from "./root-Dlff-zp8.js";
11
- import { C as pluginBuildsLabel } from "./plugins-Dkp_H3pB.js";
12
- import { i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-CrVy5ioL.js";
13
- import { s as debugTime } from "./debug-D24NOQH1.js";
13
+ import { U as LABEL, ot as imageInfo } from "./src-D91Hd6hT.js";
14
+ import { s as debugTime } from "./debug-BhizgAsU.js";
15
+ import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
14
16
  import { t as PrepareError } from "./provider-CmAYTQQs.js";
15
- import { D as runOutcome, K as triesNote, _ as formatUsage } from "./model-CgnOpss4.js";
16
- import { u as suiteResult } from "./suite-WgT7m6Vs.js";
17
- import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-CpnbATtI.js";
18
- import { p as render } from "./summary-DIGEVr08.js";
17
+ import { D as runOutcome, G as triesNote, _ as formatUsage } from "./model-CTnyvh7H.js";
18
+ import { u as suiteResult } from "./suite-Co8cwelb.js";
19
+ import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-D0qyrPZi.js";
19
20
  import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
20
- import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-CfZpFpAh.js";
21
+ import { f as render } from "./summary-DLFHVoux.js";
22
+ import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-BizwUn3h.js";
21
23
  import { t as budgetMarker } from "./budget-DBUrH4jX.js";
22
- import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-B9SOSnK9.js";
23
- import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-DJCD_BDn.js";
24
- import { n as imageHarness } from "./prepare-CIxCGG0M.js";
25
- import { i as createProviders, n as attemptImage } from "./providers-DFk_6W4K.js";
26
- import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DzyKSJBa.js";
24
+ import { a as prepareResume, c as writeCurrent, i as loadPrevious, l as writeLiveProgress, r as keptAttempts, s as resolveResume, t as completedKeysFor } from "./resume-j-rcIryi.js";
25
+ import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-9ixuXzsx.js";
26
+ import { n as imageHarness } from "./prepare-BfyikQit.js";
27
+ import { i as createProviders, n as attemptImage } from "./providers-B6b9Tnyi.js";
28
+ import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DW-lTk2f.js";
27
29
  import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
28
30
  import { dirname, join } from "node:path";
29
- import { appendFile, readFile, rm } from "node:fs/promises";
30
31
  import { inspect } from "node:util";
32
+ import { appendFile, readFile, rm } from "node:fs/promises";
31
33
  import { detectAgent } from "std-env";
32
34
  import { Command, Option } from "commander";
33
35
  //#region src/cli-parse.ts
@@ -244,14 +246,13 @@ function offset(ms) {
244
246
  }
245
247
  function outcomeOf(e) {
246
248
  if (e.state === "skipped") return "skipped";
247
- if (e.meta?.errorKind === "aborted") return "cancelled";
249
+ if (e.meta?.errorKind === "aborted") return "canceled";
248
250
  return e.meta?.outcome ?? (e.state === "passed" ? "pass" : "fail");
249
251
  }
250
252
  /** What tells an eval's attempts apart: the axes that vary (`cursor:composer-2.5 · with-skill`), else the harness. */
251
253
  function attemptAxes(a, fallback) {
252
254
  return a?.label.replace(/\s*#\d+$/, "") || a?.harness.kind || fallback;
253
255
  }
254
- const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
255
256
  /** A command on one line: the first line, cut to 80 characters at a word boundary. */
256
257
  function oneLine(text, max = 80) {
257
258
  const [first = ""] = text.trim().split("\n");
@@ -334,7 +335,7 @@ function tailLines(text, max) {
334
335
  */
335
336
  function failureLines(c, e, options) {
336
337
  const outcome = outcomeOf(e);
337
- if (outcome === "cancelled") return [c.yellow("cancelled")];
338
+ if (outcome === "canceled") return [c.yellow("canceled")];
338
339
  const message = e.errors[0] ?? "";
339
340
  const turns = (e.meta?.turnErrors ?? []).filter((t) => !message.includes(t.replace(/^turn \d+ error: /, ""))).map((t) => c.yellow(t));
340
341
  return [...failureBody(c, e, outcome, message, options), ...turns];
@@ -382,18 +383,17 @@ function resultLines(c, e, verbose) {
382
383
  const tokens = formatUsage(record.cost.usage);
383
384
  const notes = [
384
385
  ...record.status === "budget" ? [c.yellow("Budget reached: attempts that hadn’t started were skipped")] : [],
385
- ...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were cancelled")] : [],
386
+ ...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were canceled")] : [],
386
387
  c.dim([
387
388
  formatDuration(record.durationMs),
388
389
  tokens,
389
390
  `results ${e.summaryPath}`
390
391
  ].filter(Boolean).join(" · "))
391
392
  ];
392
- return render(suiteResult(record), e.insights ?? null, {
393
+ return render(suiteResult(record), e.remarks ?? null, {
393
394
  color: c.enabled,
394
395
  verbose,
395
- notes: [...(e.insightLines ?? []).map((l) => c.yellow(l)), ...notes],
396
- hint: e.hint
396
+ notes
397
397
  });
398
398
  }
399
399
  //#endregion
@@ -557,13 +557,11 @@ var PlainRenderer = class {
557
557
  #evals = /* @__PURE__ */ new Set();
558
558
  #tries = /* @__PURE__ */ new Set();
559
559
  #ended = /* @__PURE__ */ new Set();
560
- /** Insights after the last attempt, once they've started: they take the status line. */
561
- #insights = null;
562
560
  constructor(options) {
563
561
  this.#o = options;
564
562
  this.#c = options.colors;
565
563
  this.#now = options.now ?? Date.now;
566
- this.#live = options.output.requested === "live";
564
+ this.#live = options.output.statusLine;
567
565
  const { heartbeatMs } = options.output;
568
566
  if (options.timers !== false && heartbeatMs > 0) {
569
567
  this.#timer = setInterval(() => this.tick(), Math.min(heartbeatMs, 5e3));
@@ -575,16 +573,11 @@ var PlainRenderer = class {
575
573
  this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
576
574
  this.#lastWrite = this.#now();
577
575
  }
578
- /** `7 evals · 12/28 tries`: what's collected so far and how many tries ended; then `insights 3/7…`. */
576
+ /** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
579
577
  #drawStatus() {
580
- if (!this.#live) return;
581
- let status;
582
- if (this.#insights) status = this.#c.dim(`insights ${this.#insights.done}/${this.#insights.total}…`);
583
- else {
584
- if (!this.#tries.size) return;
585
- const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
586
- status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
587
- }
578
+ if (!this.#live || !this.#tries.size) return;
579
+ const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
580
+ const status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
588
581
  if (status === this.#status) return;
589
582
  this.#o.out.write(`${CLEAR}${status}`);
590
583
  this.#status = status;
@@ -712,14 +705,6 @@ var PlainRenderer = class {
712
705
  this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
713
706
  }
714
707
  return;
715
- case "insights-progress":
716
- this.#insights = {
717
- done: e.done,
718
- total: e.total,
719
- startedAt: this.#insights?.startedAt ?? e.at
720
- };
721
- this.#drawStatus();
722
- return;
723
708
  case "notice": {
724
709
  const paint = {
725
710
  info: c.dim,
@@ -780,18 +765,12 @@ var PlainRenderer = class {
780
765
  this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
781
766
  }
782
767
  }
783
- /** Prints the heartbeat if nothing was written for `heartbeatMs` and evals or insights are running. */
768
+ /** Prints the heartbeat if nothing was written for `heartbeatMs` and evals are running. */
784
769
  tick(now = this.#now()) {
785
770
  const { heartbeatMs } = this.#o.output;
786
771
  if (this.#live || heartbeatMs <= 0) return;
787
772
  if (now - this.#lastWrite < heartbeatMs) return;
788
773
  const c = this.#c;
789
- const insights = this.#insights;
790
- if (insights) {
791
- const elapsed = formatDuration(now - insights.startedAt);
792
- if (insights.done < insights.total) this.#write(c.dim(`… insights ${insights.done}/${insights.total} after ${elapsed}`));
793
- return;
794
- }
795
774
  if (this.#running.size === 0) return;
796
775
  const running = [...this.#running.values()].map((a) => {
797
776
  const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
@@ -857,13 +836,13 @@ const invocation = {
857
836
  };
858
837
  /** Set by the first Ctrl-C or SIGTERM of `reeval run`. */
859
838
  let interrupted = false;
860
- /** The last signal that cancelled the run: 130 after SIGINT, 143 after SIGTERM. */
839
+ /** The last signal that canceled the run: 130 after SIGINT, 143 after SIGTERM. */
861
840
  let stopCode = EXIT.interrupted;
862
841
  /**
863
- * Set while insights run, after the attempts and the exit code: Ctrl-C or SIGTERM then stops the
864
- * insights instead of cancelling the run, which has nothing left to cancel.
842
+ * Set once the attempts and questions are done and whether the run stopped is decided, as
843
+ * live.json turns `finishing`: a signal then has nothing left to cancel.
865
844
  */
866
- let onInsightsSignal = null;
845
+ let finishing = false;
867
846
  async function run(argv) {
868
847
  const { values, positionals } = parseCli(argv);
869
848
  invocation.flags = values;
@@ -881,7 +860,7 @@ async function run(argv) {
881
860
  return values.help ? EXIT.ok : EXIT.usage;
882
861
  }
883
862
  if (command === "init") {
884
- const { initCommand } = await import("./init-RgCQXZhb.js");
863
+ const { initCommand } = await import("./init-BDEOopK7.js");
885
864
  return initCommand(values, filters, {
886
865
  out: (s) => console.log(s),
887
866
  input: process.stdin,
@@ -890,7 +869,7 @@ async function run(argv) {
890
869
  });
891
870
  }
892
871
  if (command === "doctor") {
893
- const { doctorCommand } = await import("./doctor-DQ2p0euG.js");
872
+ const { doctorCommand } = await import("./doctor-BuQnfz4e.js");
894
873
  return doctorCommand(values, {
895
874
  out: (s) => console.log(s),
896
875
  color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -898,7 +877,7 @@ async function run(argv) {
898
877
  });
899
878
  }
900
879
  if (command === "report") {
901
- const { reportCommand } = await import("./command-BXLUfw4h.js");
880
+ const { reportCommand } = await import("./command-4pJYVCT4.js");
902
881
  return reportCommand(values, filters, {
903
882
  out: (s) => console.log(s),
904
883
  err: (s) => console.error(dim(s)),
@@ -913,7 +892,7 @@ async function run(argv) {
913
892
  }
914
893
  if (command === "studio") {
915
894
  const config = await loadConfig(configPath, "studio");
916
- const { studioCommand } = await import("./command-I1euLLio.js");
895
+ const { studioCommand } = await import("./command-DbSMcxZL.js");
917
896
  return studioCommand({
918
897
  config,
919
898
  flags: values,
@@ -924,7 +903,7 @@ async function run(argv) {
924
903
  if (command === "mcp") {
925
904
  if (!values.http) console.log = console.info = console.debug = console.error;
926
905
  const config = await loadConfig(configPath, "mcp");
927
- const { mcpCommand } = await import("./command-Bxdy62dW.js");
906
+ const { mcpCommand } = await import("./command-BqBxLgTc.js");
928
907
  return mcpCommand({
929
908
  config,
930
909
  flags: values,
@@ -933,7 +912,7 @@ async function run(argv) {
933
912
  }
934
913
  if (command in HISTORY_COMMANDS) {
935
914
  const config = await loadConfig(configPath, command);
936
- const commands = await import("./commands-p1S4nw_H.js");
915
+ const commands = await import("./commands-Bi2SEQtr.js");
937
916
  const ctx = {
938
917
  config,
939
918
  flags: values,
@@ -946,7 +925,7 @@ async function run(argv) {
946
925
  case "compare": return commands.compareCommand(ctx, filters);
947
926
  case "checkout": return commands.checkoutCommand(ctx, filters);
948
927
  case "query": {
949
- const { queryCommand } = await import("./query-Cvbwl804.js");
928
+ const { queryCommand } = await import("./query-Clm2mNSm.js");
950
929
  return queryCommand({
951
930
  ...ctx,
952
931
  err: (s) => console.error(dim(s))
@@ -954,7 +933,7 @@ async function run(argv) {
954
933
  }
955
934
  case "analyze": {
956
935
  loadEnvFiles(configPath);
957
- const { analyzeCommand } = await import("./command-Be6BSE2L.js");
936
+ const { analyzeCommand } = await import("./command-9xm5ImqI.js");
958
937
  return analyzeCommand({
959
938
  ...ctx,
960
939
  err: (s) => console.error(dim(s))
@@ -968,12 +947,12 @@ async function run(argv) {
968
947
  }
969
948
  /** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
970
949
  async function dryRunPrices(planned, judgeModel) {
971
- const { needsPrices } = await import("./cost-DmnAod9x.js");
950
+ const { needsPrices } = await import("./cost-C5RBTTJr.js");
972
951
  if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
973
952
  table: null,
974
953
  why: "nothing to price"
975
954
  };
976
- const { loadPrices } = await import("./prices-BcLEXvdY.js");
955
+ const { loadPrices } = await import("./prices-QaS-kjSQ.js");
977
956
  return loadPrices();
978
957
  }
979
958
  /** Asks a yes/no question on the terminal; anything but y or yes is no. */
@@ -998,13 +977,13 @@ async function cleanCommand(flags, configPath) {
998
977
  if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
999
978
  }
1000
979
  const { debugRoot, pruneDebug } = await import("./files-BAt0i00X.js");
1001
- const { formatBytes } = await import("./debug-D7bEv_UV.js");
980
+ const { formatBytes } = await import("./debug-DeM5pswK.js");
1002
981
  const { runs, bytes } = await pruneDebug(config.root, keep);
1003
982
  const where = fromCwd(debugRoot(config.root)) || ".";
1004
983
  console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
1005
984
  if (flags.images) {
1006
- const { dockerAvailable, pruneImages } = await import("./src-1jHbFaGW.js");
1007
- const { requireDocker } = await import("./providers-D1Zxp2he.js");
985
+ const { dockerAvailable, pruneImages } = await import("./src-DH-9IO7f.js");
986
+ const { requireDocker } = await import("./providers-DkOupEBl.js");
1008
987
  const engines = [];
1009
988
  for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
1010
989
  if (!engines.length) await requireDocker();
@@ -1025,7 +1004,7 @@ function verbosityOf(flags) {
1025
1004
  * does. Needs a config, as `reeval run` does: there is no default image.
1026
1005
  */
1027
1006
  async function imageBuildCommand(flags, configPath, filters) {
1028
- const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-DzLeFjLx.js");
1007
+ const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-4mwhNkvh.js");
1029
1008
  const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
1030
1009
  ...flags,
1031
1010
  "dry-run": true
@@ -1085,7 +1064,7 @@ async function runCommand(flags, configPath, filters) {
1085
1064
  if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
1086
1065
  const stepStart = process.env.GITHUB_STEP_SUMMARY;
1087
1066
  if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
1088
- const { engineVersions } = await import("./node-BSq5qHnQ.js");
1067
+ const { engineVersions } = await import("./node-BVbbnFtv.js");
1089
1068
  const renderer = createRenderer({
1090
1069
  out: setup.json ? setup.dryRun ? { write: () => true } : process.stderr : process.stdout,
1091
1070
  output: setup.output,
@@ -1106,8 +1085,8 @@ async function runCommand(flags, configPath, filters) {
1106
1085
  const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
1107
1086
  invocation.unhandledErrors = engineRun.unhandledErrors;
1108
1087
  const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
1109
- if ((interrupted || engineRun.cancelled) && engines.length) {
1110
- const { removeRunContainers } = await import("./docker-DivfZl2u.js");
1088
+ if ((interrupted || engineRun.canceled) && engines.length) {
1089
+ const { removeRunContainers } = await import("./docker-YFj4H1Cr.js");
1111
1090
  await removeRunContainers(ctx.runId, engines);
1112
1091
  }
1113
1092
  if (engineRun.stopped || !plan) {
@@ -1115,15 +1094,17 @@ async function runCommand(flags, configPath, filters) {
1115
1094
  renderer.close();
1116
1095
  return EXIT.ok;
1117
1096
  }
1118
- let stopped = cutShort || engineRun.cancelled;
1097
+ let stopped = cutShort || engineRun.canceled;
1119
1098
  if (stopped) questions?.cancel();
1120
1099
  let answers = stopped ? await questions?.answered() : await questions?.settle();
1121
1100
  if (!stopped && interrupted && questions) {
1122
1101
  stopped = true;
1123
1102
  answers = await questions?.answered();
1124
1103
  }
1125
- const { lockfiles } = await import("./state-vznMh2wl.js");
1126
- const { writeScratchRun } = await import("./results-CMgZEANy.js");
1104
+ finishing = true;
1105
+ live?.phase("finishing");
1106
+ const { lockfiles } = await import("./state-B5PLm5qY.js");
1107
+ const { writeScratchRun } = await import("./results-BC3pcG_Z.js");
1127
1108
  const run = await writeScratchRun({
1128
1109
  ctx,
1129
1110
  author: ctx.author,
@@ -1149,55 +1130,6 @@ async function runCommand(flags, configPath, filters) {
1149
1130
  const { record } = run;
1150
1131
  const outcome = runOutcome(record);
1151
1132
  const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
1152
- let insights = run.insights;
1153
- let insightLines = [];
1154
- const { runInsights, wantsInsights } = await import("./run-CwIEK16q.js");
1155
- const wanted = !stopped && wantsInsights(ctx.config, !flags["no-insights"]);
1156
- if (wanted && interrupted) insightLines = [`insights skipped (${stopCode === EXIT.terminated ? "SIGTERM" : "Ctrl-C"}): reeval analyze ${record.runId} --insights to write them`];
1157
- else if (wanted) {
1158
- const { scratchDir } = await import("./files-C2sjmXhk.js");
1159
- const stop = new AbortController();
1160
- let stopSignal = "SIGINT";
1161
- onInsightsSignal = (signal) => {
1162
- if (stop.signal.aborted) {
1163
- renderer.close();
1164
- process.exit(code);
1165
- }
1166
- stopSignal = signal;
1167
- stop.abort(/* @__PURE__ */ new Error(signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"));
1168
- };
1169
- try {
1170
- const done = await runInsights({
1171
- config: ctx.config,
1172
- record,
1173
- runDir: run.artifactDir ?? join(scratchDir(ctx.config.root), record.runId),
1174
- remarks: run.insights,
1175
- verbose: (ctx.verbosity ?? 0) > 0,
1176
- signal: stop.signal,
1177
- stopSignal: () => stopSignal,
1178
- onStart: (text) => {
1179
- renderer.onEvent({
1180
- type: "notice",
1181
- at: Date.now(),
1182
- level: "info",
1183
- text
1184
- });
1185
- },
1186
- onProgress: (n, total) => {
1187
- renderer.onEvent({
1188
- type: "insights-progress",
1189
- at: Date.now(),
1190
- done: n,
1191
- total
1192
- });
1193
- }
1194
- });
1195
- insights = done.insights ?? insights;
1196
- insightLines = done.lines;
1197
- } finally {
1198
- onInsightsSignal = null;
1199
- }
1200
- }
1201
1133
  const { pruneDebug } = await import("./files-BAt0i00X.js");
1202
1134
  await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
1203
1135
  const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
@@ -1207,9 +1139,7 @@ async function runCommand(flags, configPath, filters) {
1207
1139
  record,
1208
1140
  summaryPath: fromCwd(summary),
1209
1141
  results: engineRun.evals,
1210
- hint: !hasAnalysisModel(ctx.config),
1211
- insights,
1212
- insightLines
1142
+ remarks: run.remarks
1213
1143
  });
1214
1144
  renderer.close();
1215
1145
  const stepSummary = process.env.GITHUB_STEP_SUMMARY;
@@ -1217,7 +1147,7 @@ async function runCommand(flags, configPath, filters) {
1217
1147
  const { config } = ctx;
1218
1148
  const mode = setup.record;
1219
1149
  if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
1220
- const { record: recordCmd } = await import("./commands-p1S4nw_H.js");
1150
+ const { record: recordCmd } = await import("./commands-Bi2SEQtr.js");
1221
1151
  const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
1222
1152
  const line = await recordCmd(config, record.runId, {
1223
1153
  ...flags,
@@ -1227,7 +1157,7 @@ async function runCommand(flags, configPath, filters) {
1227
1157
  else console.log(line);
1228
1158
  }
1229
1159
  if (setup.json) {
1230
- const { suiteResult } = await import("./suite-BsWGHzlG.js");
1160
+ const { suiteResult } = await import("./suite-22Kt9Grx.js");
1231
1161
  process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
1232
1162
  }
1233
1163
  return code;
@@ -1260,7 +1190,7 @@ async function prepareRun(flags, configPath) {
1260
1190
  const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
1261
1191
  const container = sandboxes.find((s) => s.kind === "docker");
1262
1192
  if (!container) return hostCapacity();
1263
- const { dockerCapacity } = await import("./src-1jHbFaGW.js");
1193
+ const { dockerCapacity } = await import("./src-DH-9IO7f.js");
1264
1194
  return await dockerCapacity(container.engine) ?? hostCapacity();
1265
1195
  });
1266
1196
  return {
@@ -1275,8 +1205,6 @@ async function prepareRun(flags, configPath) {
1275
1205
  dryRun: !!flags["dry-run"],
1276
1206
  json: !!flags.json,
1277
1207
  force: !!flags.force,
1278
- /** `--no-insights` turns them off for this run. */
1279
- insights: !flags["no-insights"],
1280
1208
  verbosity: verbosityOf(flags),
1281
1209
  configFile: configLabel(path),
1282
1210
  /** Named in the plan line when the project root isn't the current directory. */
@@ -1289,11 +1217,11 @@ async function prepareRun(flags, configPath) {
1289
1217
  async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
1290
1218
  const startedAt = /* @__PURE__ */ new Date();
1291
1219
  const git = await gitInfo(config.root, [config.history.dir]);
1292
- const { worktreePatch } = await import("./state-vznMh2wl.js");
1220
+ const { worktreePatch } = await import("./state-B5PLm5qY.js");
1293
1221
  const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
1294
1222
  const who = await author(config.root);
1295
- const { stagingDir, sweepStaging } = await import("./results-CMgZEANy.js");
1296
- const { scratchDir } = await import("./files-C2sjmXhk.js");
1223
+ const { stagingDir, sweepStaging } = await import("./results-BC3pcG_Z.js");
1224
+ const { scratchDir } = await import("./files-DmUrRmmL.js");
1297
1225
  const scratch = scratchDir(config.root);
1298
1226
  if (resume !== void 0) {
1299
1227
  const target = await resolveResume(config.root, resume || void 0);
@@ -1304,7 +1232,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
1304
1232
  artifacts: target.previousArtifacts
1305
1233
  } : await loadPrevious(stagingDir(config.root, target.runId));
1306
1234
  if (!dryRun) try {
1307
- const { removeRunContainers } = await import("./docker-DivfZl2u.js");
1235
+ const { removeRunContainers } = await import("./docker-YFj4H1Cr.js");
1308
1236
  await removeRunContainers(target.runId, ["docker", "podman"]);
1309
1237
  } catch {}
1310
1238
  return {
@@ -1375,8 +1303,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1375
1303
  configFile,
1376
1304
  completed: []
1377
1305
  }));
1378
- const { runEngine } = await import("./node-BSq5qHnQ.js");
1379
- const { buildPlan, checkAttemptBudget, planDocument, planInsights, planLine, planTable } = await import("./plan-BIwUzlB8.js");
1306
+ const { runEngine } = await import("./node-BVbbnFtv.js");
1307
+ const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-DdWkM6SW.js");
1380
1308
  let plan;
1381
1309
  let sandboxVersions = { images: [] };
1382
1310
  let preflightStop;
@@ -1384,10 +1312,13 @@ async function executeRun(ctx, setup, filters, renderer) {
1384
1312
  let budgetTimer;
1385
1313
  let toRun = [];
1386
1314
  let live;
1315
+ let kept = /* @__PURE__ */ new Map();
1387
1316
  let questions;
1388
- const onPlan = async ({ planned, excluded, matched, unmatched }) => {
1317
+ const onPlan = async (collected) => {
1318
+ const { planned, excluded, matched, unmatched } = collected;
1389
1319
  if (unmatched.length) throw unmatchedFilter(config, unmatched);
1390
1320
  if (matched === 0) throw noMatch(config, filters, tags);
1321
+ checkLimitHarness(profile, config.file, collected);
1391
1322
  const completed = completedKeysFor(planned, stagingDir, ctx.previous);
1392
1323
  const skipped = planned.map((p) => completed.has(p.key) ? {
1393
1324
  ...p,
@@ -1402,7 +1333,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1402
1333
  } : p);
1403
1334
  toRun = skipped;
1404
1335
  for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
1405
- const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-Ch8kX87u.js") : void 0;
1336
+ const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-g3XnzUv0.js") : void 0;
1406
1337
  const judge = runJudgeModel(config, profile);
1407
1338
  const judgeModel = judge === void 0 ? null : judgeModelName(judge);
1408
1339
  plan = await buildPlan({
@@ -1413,16 +1344,10 @@ async function executeRun(ctx, setup, filters, renderer) {
1413
1344
  root: config.root,
1414
1345
  historyDir: config.history.dir,
1415
1346
  questionModel: models?.questionModelLabel(config, judge) ?? null,
1416
- insights: await planInsights(config, setup.insights),
1417
1347
  judgeModel,
1418
1348
  ...setup.dryRun ? { prices: await dryRunPrices(skipped, judgeModel) } : {}
1419
1349
  });
1420
1350
  plan.runId = runId;
1421
- if (plan.insights) {
1422
- const { analysisKeyProblem } = await import("./model-Ch8kX87u.js");
1423
- const problem = analysisKeyProblem(config);
1424
- if (problem) plan.warnings.push(`insights will be skipped: ${problem}`);
1425
- }
1426
1351
  const cloud = ciRunner(process.env);
1427
1352
  const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
1428
1353
  if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
@@ -1431,7 +1356,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1431
1356
  completed: completed.size
1432
1357
  };
1433
1358
  run.completed = [...completed];
1434
- const { imageStages, imageTargets } = await import("./images-DzLeFjLx.js");
1359
+ kept = keptAttempts(completed, stagingDir, ctx.previous);
1360
+ const { imageStages, imageTargets } = await import("./images-4mwhNkvh.js");
1435
1361
  const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
1436
1362
  id: p.id,
1437
1363
  harness: p.attempt.harness,
@@ -1453,8 +1379,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1453
1379
  if (setup.dryRun) {
1454
1380
  const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
1455
1381
  lines.push(...planTable(plan, todos));
1456
- const { imageStatusLines } = await import("./images-DzLeFjLx.js");
1457
- const { dockerAvailable, engineSpec } = await import("./src-1jHbFaGW.js");
1382
+ const { imageStatusLines } = await import("./images-4mwhNkvh.js");
1383
+ const { dockerAvailable, engineSpec } = await import("./src-DH-9IO7f.js");
1458
1384
  const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
1459
1385
  const down = [];
1460
1386
  for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
@@ -1463,14 +1389,10 @@ async function executeRun(ctx, setup, filters, renderer) {
1463
1389
  const names = down.map((e) => engineSpec(e).name).join(" and ");
1464
1390
  lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
1465
1391
  }
1466
- const { costLine } = await import("./cost-DmnAod9x.js");
1392
+ const { costLine } = await import("./cost-C5RBTTJr.js");
1467
1393
  const cost = plan.cost && costLine(plan.cost);
1468
1394
  if (cost) lines.push(cost);
1469
1395
  }
1470
- if (setup.dryRun && plan.insights) {
1471
- const { formatTokens } = await import("./usage-DF3_p5pE.js");
1472
- lines.push(`insights ${plan.insights.model} · up to ${formatTokens(plan.insights.maxTokens)} tokens`);
1473
- }
1474
1396
  lines.push(...plan.warnings.map((w) => `warning: ${w}`));
1475
1397
  lines.push(...plan.hints.map((h) => `hint: ${h}`));
1476
1398
  renderer.onEvent({
@@ -1494,6 +1416,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1494
1416
  question: q.question,
1495
1417
  model,
1496
1418
  ...gateway ? { gateway } : {},
1419
+ ...gateway === "openrouter" && openRouterFallbacks(scope.gateways) ? { fallbacks: true } : {},
1420
+ ...gateway === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {},
1497
1421
  ...q.across !== void 0 ? { across: q.across } : {},
1498
1422
  ...q.choices ? { choices: q.choices } : {},
1499
1423
  ...q.state ? { state: q.state } : {},
@@ -1506,10 +1430,16 @@ async function executeRun(ctx, setup, filters, renderer) {
1506
1430
  questions
1507
1431
  });
1508
1432
  }
1433
+ const judges = /* @__PURE__ */ new Map();
1434
+ for (const p of skipped) {
1435
+ const j = p.attempt.judge;
1436
+ if (j && !p.attempt.plan.skip) judges.set(`${j.model}\0${j.gateway ?? ""}`, j);
1437
+ }
1509
1438
  const missing = missingCredentials(config, process.env, {
1510
1439
  harnesses: plan.harnessDescriptors,
1511
1440
  profile,
1512
- questionModels: [...questionModels]
1441
+ questionModels: [...questionModels],
1442
+ judges: [...judges.values()]
1513
1443
  });
1514
1444
  if (missing.length) {
1515
1445
  const hint = missing.some((m) => m.includes("(for direct())")) ? unusedGatewayKey(process.env) : "";
@@ -1518,15 +1448,21 @@ async function executeRun(ctx, setup, filters, renderer) {
1518
1448
  const malformed = malformedCredentials(config, process.env, {
1519
1449
  harnesses: plan.harnessDescriptors,
1520
1450
  profile,
1521
- questionModels: [...questionModels]
1451
+ questionModels: [...questionModels],
1452
+ judges: [...judges.values()]
1522
1453
  });
1523
1454
  if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
1524
1455
  if (asked.length) {
1525
- const { questionRunner } = await import("./questions-bAudIc04.js");
1456
+ const { questionRunner } = await import("./questions-BRhU8MzB.js");
1526
1457
  const warned = /* @__PURE__ */ new Set();
1527
1458
  questions = questionRunner({
1528
1459
  evals: asked,
1529
1460
  stagingDir,
1461
+ resumed: {
1462
+ completed: new Set(run.completed),
1463
+ previous: ctx.previous,
1464
+ artifacts: ctx.previousArtifacts
1465
+ },
1530
1466
  maxInputTokens: config.analysis.maxInputTokens,
1531
1467
  maxOutputTokens: config.analysis.maxOutputTokens,
1532
1468
  onAnswer: (id, answer) => {
@@ -1559,7 +1495,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1559
1495
  }
1560
1496
  });
1561
1497
  }
1562
- const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-QZCacP8A.js");
1498
+ const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-DDHzkMjm.js");
1563
1499
  const live = skipped.filter((p) => !p.attempt.plan.skip);
1564
1500
  const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
1565
1501
  workspace: a.workspace,
@@ -1662,7 +1598,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1662
1598
  dir: stagingDir,
1663
1599
  runId,
1664
1600
  startedAt: ctx.startedAt,
1665
- planned: toRun
1601
+ planned: toRun,
1602
+ kept
1666
1603
  });
1667
1604
  renderer.onEvent({
1668
1605
  type: "run-start",
@@ -1687,22 +1624,22 @@ async function executeRun(ctx, setup, filters, renderer) {
1687
1624
  if (interrupted && e.type === "eval-end" && e.result.state === "skipped") return;
1688
1625
  renderer.onEvent(e);
1689
1626
  },
1690
- onCancellable: (cancel) => {
1627
+ onCancelable: (cancel) => {
1691
1628
  const onSignal = (signal) => {
1692
- if (onInsightsSignal) return onInsightsSignal(signal === "SIGTERM" ? "SIGTERM" : "SIGINT");
1693
1629
  stopCode = signal === "SIGTERM" ? EXIT.terminated : EXIT.interrupted;
1694
1630
  if (interrupted) {
1695
1631
  renderer.close();
1696
1632
  process.exit(stopCode);
1697
1633
  }
1698
1634
  interrupted = true;
1635
+ if (finishing) return;
1699
1636
  questions?.cancel();
1700
1637
  live?.phase("cancelling");
1701
1638
  renderer.onEvent({
1702
1639
  type: "notice",
1703
1640
  at: Date.now(),
1704
1641
  level: "warn",
1705
- text: "Cancelling: stopping agents and destroying sandboxes (Ctrl-C again to force)"
1642
+ text: "Canceling: stopping agents and destroying sandboxes (Ctrl-C again to force)"
1706
1643
  });
1707
1644
  cancel();
1708
1645
  };
@@ -1711,15 +1648,14 @@ async function executeRun(ctx, setup, filters, renderer) {
1711
1648
  }
1712
1649
  });
1713
1650
  cutShort = interrupted;
1714
- live?.phase("finishing");
1715
1651
  } catch (error) {
1716
1652
  live?.remove();
1717
1653
  throw error;
1718
1654
  } finally {
1719
1655
  clearTimeout(budgetTimer);
1720
- const { removePrepared } = await import("./preflight-D9y-HvZQ.js");
1656
+ const { removePrepared } = await import("./preflight-DTiVv0gY.js");
1721
1657
  removePrepared(config.root, runId);
1722
- const { removePreparedWorkspaces } = await import("./workspace-QZCacP8A.js");
1658
+ const { removePreparedWorkspaces } = await import("./workspace-DDHzkMjm.js");
1723
1659
  removePreparedWorkspaces(config.root, runId);
1724
1660
  }
1725
1661
  if (preflightStop) throw preflightStop;
@@ -1802,4 +1738,4 @@ run(process.argv.slice(2)).finally(() => {
1802
1738
  //#endregion
1803
1739
  export {};
1804
1740
 
1805
- //# sourceMappingURL=cli-C_4gaJ50.js.map
1741
+ //# sourceMappingURL=cli-PdYrcoKM.js.map