reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -1,30 +1,31 @@
1
- import { n as isOneOf } from "./object-CLf2-tNq.js";
2
1
  import { n as UsageError } from "./errors-B6BSMs03.js";
3
- import { p as parseDuration } from "./shared-CA61BuPJ.js";
4
- import { p as sandboxId, s as harnessId, u as plannedSandboxes } from "./options-CyAyLjgJ.js";
5
- import { M as workspaceLabel, T as profileSandbox, p as environmentSandbox } from "./environment-B4IuURK1.js";
2
+ import { h as parseDuration } from "./shared-B_lvGqWz.js";
3
+ import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-BOFgzFDg.js";
4
+ import { C as pluginBuildsLabel } from "./plugins-D-CP6mMd.js";
6
5
  import { a as fromCwd, o as newRunId } from "./ids-Bp6ut7ZD.js";
7
- import { c as registerSecret, s as redactSecrets } from "./redact-CRGdfV2s.js";
8
- import { a as configPathFor, c as findConfig, d as loadConfig, f as malformedCredentials, g as selectProfile, h as resolveConcurrency, i as configLabel, l as hasAnalysisModel, m as requireConfig, p as missingCredentials, u as hostCapacity } from "./config-D7TFvUpL.js";
9
- import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
10
- import { W as LABEL, at as imageInfo } from "./src-CIpKL45U.js";
11
- import { o as PACKAGE_VERSION } from "./root-DNuLhaA5.js";
12
- import { C as pluginBuildsLabel } from "./plugins-P-uh54VF.js";
13
- import { s as debugTime } from "./debug-D24NOQH1.js";
14
- import { i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-CrVy5ioL.js";
6
+ import { $ as isDirect, A as harnessId, E as checkLimitHarness, F as plannedSandboxes, R as sandboxId, X as directOnCloud, Y as ciRunner, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, tt as resolveOutput, u as loadConfig, v as gatewayScope } from "./config-k6m7XbAz.js";
7
+ import { l as judgeModelName, m as unusedGatewayKey } from "./judge-CKdgEXN7.js";
8
+ import { y as vertexSettingsIn } from "./vertex-DypYjx71.js";
9
+ import { _ as registerSecret, m as redactSecrets } from "./redact-DVIaahLv.js";
10
+ import { a as plural, i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-DTigcA64.js";
11
+ import { n as loadEnvFiles, t as inCI } from "./env-CARj5Axq.js";
12
+ import { o as PACKAGE_VERSION } from "./root-Dlff-zp8.js";
13
+ import { U as LABEL, ot as imageInfo } from "./src-D91Hd6hT.js";
14
+ import { s as debugTime } from "./debug-BhizgAsU.js";
15
+ import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
15
16
  import { t as PrepareError } from "./provider-CmAYTQQs.js";
16
- import { T as runOutcome, g as formatUsage } from "./model-UgHDMhI1.js";
17
- import { c as suiteResult } from "./suite-DmuF_ApO.js";
18
- import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-BXIBUlaK.js";
19
- import { p as render } from "./summary-BFMHMTz2.js";
17
+ import { D as runOutcome, G as triesNote, _ as formatUsage } from "./model-CTnyvh7H.js";
18
+ import { u as suiteResult } from "./suite-Co8cwelb.js";
19
+ import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-D0qyrPZi.js";
20
20
  import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
21
- import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-C_-8y-x3.js";
21
+ import { f as render } from "./summary-DLFHVoux.js";
22
+ import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-BizwUn3h.js";
22
23
  import { t as budgetMarker } from "./budget-DBUrH4jX.js";
23
- import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-8-nnW8iO.js";
24
- import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-D3HvKeAx.js";
25
- import { n as imageHarness } from "./prepare-Dx7dOnJB.js";
26
- import { i as createProviders, n as attemptImage } from "./providers-CF5UEaP8.js";
27
- import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-D_ePcwyL.js";
24
+ import { a as prepareResume, c as writeCurrent, i as loadPrevious, l as writeLiveProgress, r as keptAttempts, s as resolveResume, t as completedKeysFor } from "./resume-j-rcIryi.js";
25
+ import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-9ixuXzsx.js";
26
+ import { n as imageHarness } from "./prepare-BfyikQit.js";
27
+ import { i as createProviders, n as attemptImage } from "./providers-B6b9Tnyi.js";
28
+ import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DW-lTk2f.js";
28
29
  import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
29
30
  import { dirname, join } from "node:path";
30
31
  import { inspect } from "node:util";
@@ -245,14 +246,13 @@ function offset(ms) {
245
246
  }
246
247
  function outcomeOf(e) {
247
248
  if (e.state === "skipped") return "skipped";
248
- if (e.meta?.errorKind === "aborted") return "cancelled";
249
+ if (e.meta?.errorKind === "aborted") return "canceled";
249
250
  return e.meta?.outcome ?? (e.state === "passed" ? "pass" : "fail");
250
251
  }
251
252
  /** What tells an eval's attempts apart: the axes that vary (`cursor:composer-2.5 · with-skill`), else the harness. */
252
253
  function attemptAxes(a, fallback) {
253
254
  return a?.label.replace(/\s*#\d+$/, "") || a?.harness.kind || fallback;
254
255
  }
255
- const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
256
256
  /** A command on one line: the first line, cut to 80 characters at a word boundary. */
257
257
  function oneLine(text, max = 80) {
258
258
  const [first = ""] = text.trim().split("\n");
@@ -274,7 +274,8 @@ function progressLine(c, p) {
274
274
  case "check-start": return c.dim(`$ ${oneLine(p.command)}`);
275
275
  case "check-end": {
276
276
  const exit = p.exitCode === 0 ? c.dim("exit 0") : c.red(`exit ${p.exitCode}`);
277
- return `$ ${oneLine(p.command)} ${exit} ${c.dim(formatDuration(p.ms))}`;
277
+ const tries = p.try && p.tries ? triesNote(p.exitCode, p.try, p.tries) : null;
278
+ return `$ ${oneLine(p.command)} ${exit} ${c.dim([formatDuration(p.ms), tries].filter(Boolean).join(" · "))}`;
278
279
  }
279
280
  case "judge-start": return c.dim(`judging: ${oneLine(p.question, 120)}`);
280
281
  case "judge-end": return judgeLine(c, p);
@@ -334,7 +335,7 @@ function tailLines(text, max) {
334
335
  */
335
336
  function failureLines(c, e, options) {
336
337
  const outcome = outcomeOf(e);
337
- if (outcome === "cancelled") return [c.yellow("cancelled")];
338
+ if (outcome === "canceled") return [c.yellow("canceled")];
338
339
  const message = e.errors[0] ?? "";
339
340
  const turns = (e.meta?.turnErrors ?? []).filter((t) => !message.includes(t.replace(/^turn \d+ error: /, ""))).map((t) => c.yellow(t));
340
341
  return [...failureBody(c, e, outcome, message, options), ...turns];
@@ -382,18 +383,17 @@ function resultLines(c, e, verbose) {
382
383
  const tokens = formatUsage(record.cost.usage);
383
384
  const notes = [
384
385
  ...record.status === "budget" ? [c.yellow("Budget reached: attempts that hadn’t started were skipped")] : [],
385
- ...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were cancelled")] : [],
386
+ ...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were canceled")] : [],
386
387
  c.dim([
387
388
  formatDuration(record.durationMs),
388
389
  tokens,
389
390
  `results ${e.summaryPath}`
390
391
  ].filter(Boolean).join(" · "))
391
392
  ];
392
- return render(suiteResult(record), e.insights ?? null, {
393
+ return render(suiteResult(record), e.remarks ?? null, {
393
394
  color: c.enabled,
394
395
  verbose,
395
- notes: [...(e.insightLines ?? []).map((l) => c.yellow(l)), ...notes],
396
- hint: e.hint
396
+ notes
397
397
  });
398
398
  }
399
399
  //#endregion
@@ -557,13 +557,11 @@ var PlainRenderer = class {
557
557
  #evals = /* @__PURE__ */ new Set();
558
558
  #tries = /* @__PURE__ */ new Set();
559
559
  #ended = /* @__PURE__ */ new Set();
560
- /** Insights after the last attempt, once they've started: they take the status line. */
561
- #insights = null;
562
560
  constructor(options) {
563
561
  this.#o = options;
564
562
  this.#c = options.colors;
565
563
  this.#now = options.now ?? Date.now;
566
- this.#live = options.output.requested === "live";
564
+ this.#live = options.output.statusLine;
567
565
  const { heartbeatMs } = options.output;
568
566
  if (options.timers !== false && heartbeatMs > 0) {
569
567
  this.#timer = setInterval(() => this.tick(), Math.min(heartbeatMs, 5e3));
@@ -575,16 +573,11 @@ var PlainRenderer = class {
575
573
  this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
576
574
  this.#lastWrite = this.#now();
577
575
  }
578
- /** `7 evals · 12/28 tries`: what's collected so far and how many tries ended; then `insights 3/7…`. */
576
+ /** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
579
577
  #drawStatus() {
580
- if (!this.#live) return;
581
- let status;
582
- if (this.#insights) status = this.#c.dim(`insights ${this.#insights.done}/${this.#insights.total}…`);
583
- else {
584
- if (!this.#tries.size) return;
585
- const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
586
- status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
587
- }
578
+ if (!this.#live || !this.#tries.size) return;
579
+ const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
580
+ const status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
588
581
  if (status === this.#status) return;
589
582
  this.#o.out.write(`${CLEAR}${status}`);
590
583
  this.#status = status;
@@ -712,14 +705,6 @@ var PlainRenderer = class {
712
705
  this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
713
706
  }
714
707
  return;
715
- case "insights-progress":
716
- this.#insights = {
717
- done: e.done,
718
- total: e.total,
719
- startedAt: this.#insights?.startedAt ?? e.at
720
- };
721
- this.#drawStatus();
722
- return;
723
708
  case "notice": {
724
709
  const paint = {
725
710
  info: c.dim,
@@ -780,18 +765,12 @@ var PlainRenderer = class {
780
765
  this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
781
766
  }
782
767
  }
783
- /** Prints the heartbeat if nothing was written for `heartbeatMs` and evals or insights are running. */
768
+ /** Prints the heartbeat if nothing was written for `heartbeatMs` and evals are running. */
784
769
  tick(now = this.#now()) {
785
770
  const { heartbeatMs } = this.#o.output;
786
771
  if (this.#live || heartbeatMs <= 0) return;
787
772
  if (now - this.#lastWrite < heartbeatMs) return;
788
773
  const c = this.#c;
789
- const insights = this.#insights;
790
- if (insights) {
791
- const elapsed = formatDuration(now - insights.startedAt);
792
- if (insights.done < insights.total) this.#write(c.dim(`… insights ${insights.done}/${insights.total} after ${elapsed}`));
793
- return;
794
- }
795
774
  if (this.#running.size === 0) return;
796
775
  const running = [...this.#running.values()].map((a) => {
797
776
  const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
@@ -807,73 +786,6 @@ var PlainRenderer = class {
807
786
  }
808
787
  };
809
788
  //#endregion
810
- //#region src/ui/mode.ts
811
- const set = (env, name) => {
812
- const value = env[name];
813
- return !!value && value !== "false" && value !== "0";
814
- };
815
- const CI_VARS = [
816
- "BUILD_ID",
817
- "BUILD_NUMBER",
818
- "CI",
819
- "CI_APP_ID",
820
- "CI_BUILD_ID",
821
- "CI_BUILD_NUMBER",
822
- "CI_NAME",
823
- "CONTINUOUS_INTEGRATION",
824
- "RUN_ID",
825
- "TEAMCITY_VERSION",
826
- "GITHUB_ACTIONS"
827
- ];
828
- const isCI = (env) => CI_VARS.some((name) => set(env, name));
829
- function pick(name, value, allowed) {
830
- if (value === void 0 || value === "") return void 0;
831
- if (value === "auto" || isOneOf(allowed, value)) return value;
832
- throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
833
- }
834
- function parseHeartbeat(value) {
835
- if (!value) return 6e4;
836
- const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
837
- if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
838
- const unit = {
839
- ms: 1,
840
- s: 1e3,
841
- m: 6e4
842
- }[match[2] ?? "s"] ?? 1e3;
843
- return Number(match[1]) * unit;
844
- }
845
- function resolveOutput(input) {
846
- const { flags, env } = input;
847
- const ci = isCI(env);
848
- const githubActions = set(env, "GITHUB_ACTIONS");
849
- const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
850
- "live",
851
- "plain",
852
- "agent"
853
- ]);
854
- const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
855
- const mode = requested === "live" ? "plain" : requested;
856
- const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
857
- const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
858
- const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
859
- const logPrefix = prefix && prefix !== "auto" ? prefix : githubActions && logOrder === "grouped" ? "none" : "task";
860
- const logs = pick("--output-logs", flags.outputLogs ?? env.REEVAL_OUTPUT_LOGS, [
861
- "full",
862
- "errors-only",
863
- "none"
864
- ]);
865
- return {
866
- requested,
867
- mode,
868
- logOrder,
869
- logPrefix,
870
- outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
871
- color: env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions,
872
- githubActions,
873
- heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
874
- };
875
- }
876
- //#endregion
877
789
  //#region src/ui/index.ts
878
790
  function createRenderer(options) {
879
791
  const out = { write: (text) => options.out.write(redactSecrets(text)) };
@@ -924,13 +836,13 @@ const invocation = {
924
836
  };
925
837
  /** Set by the first Ctrl-C or SIGTERM of `reeval run`. */
926
838
  let interrupted = false;
927
- /** The last signal that cancelled the run: 130 after SIGINT, 143 after SIGTERM. */
839
+ /** The last signal that canceled the run: 130 after SIGINT, 143 after SIGTERM. */
928
840
  let stopCode = EXIT.interrupted;
929
841
  /**
930
- * Set while insights run, after the attempts and the exit code: Ctrl-C or SIGTERM then stops the
931
- * insights instead of cancelling the run, which has nothing left to cancel.
842
+ * Set once the attempts and questions are done and whether the run stopped is decided, as
843
+ * live.json turns `finishing`: a signal then has nothing left to cancel.
932
844
  */
933
- let onInsightsSignal = null;
845
+ let finishing = false;
934
846
  async function run(argv) {
935
847
  const { values, positionals } = parseCli(argv);
936
848
  invocation.flags = values;
@@ -948,7 +860,7 @@ async function run(argv) {
948
860
  return values.help ? EXIT.ok : EXIT.usage;
949
861
  }
950
862
  if (command === "init") {
951
- const { initCommand } = await import("./init-kMkTgTZc.js");
863
+ const { initCommand } = await import("./init-BDEOopK7.js");
952
864
  return initCommand(values, filters, {
953
865
  out: (s) => console.log(s),
954
866
  input: process.stdin,
@@ -957,7 +869,7 @@ async function run(argv) {
957
869
  });
958
870
  }
959
871
  if (command === "doctor") {
960
- const { doctorCommand } = await import("./doctor-De_QlOTM.js");
872
+ const { doctorCommand } = await import("./doctor-BuQnfz4e.js");
961
873
  return doctorCommand(values, {
962
874
  out: (s) => console.log(s),
963
875
  color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -965,7 +877,7 @@ async function run(argv) {
965
877
  });
966
878
  }
967
879
  if (command === "report") {
968
- const { reportCommand } = await import("./command-CANun0m0.js");
880
+ const { reportCommand } = await import("./command-4pJYVCT4.js");
969
881
  return reportCommand(values, filters, {
970
882
  out: (s) => console.log(s),
971
883
  err: (s) => console.error(dim(s)),
@@ -980,7 +892,7 @@ async function run(argv) {
980
892
  }
981
893
  if (command === "studio") {
982
894
  const config = await loadConfig(configPath, "studio");
983
- const { studioCommand } = await import("./command-Bh3yhN_b.js");
895
+ const { studioCommand } = await import("./command-DbSMcxZL.js");
984
896
  return studioCommand({
985
897
  config,
986
898
  flags: values,
@@ -991,7 +903,7 @@ async function run(argv) {
991
903
  if (command === "mcp") {
992
904
  if (!values.http) console.log = console.info = console.debug = console.error;
993
905
  const config = await loadConfig(configPath, "mcp");
994
- const { mcpCommand } = await import("./command-D_kEnYcX.js");
906
+ const { mcpCommand } = await import("./command-BqBxLgTc.js");
995
907
  return mcpCommand({
996
908
  config,
997
909
  flags: values,
@@ -1000,7 +912,7 @@ async function run(argv) {
1000
912
  }
1001
913
  if (command in HISTORY_COMMANDS) {
1002
914
  const config = await loadConfig(configPath, command);
1003
- const commands = await import("./commands-BKGzBMgb.js");
915
+ const commands = await import("./commands-Bi2SEQtr.js");
1004
916
  const ctx = {
1005
917
  config,
1006
918
  flags: values,
@@ -1013,7 +925,7 @@ async function run(argv) {
1013
925
  case "compare": return commands.compareCommand(ctx, filters);
1014
926
  case "checkout": return commands.checkoutCommand(ctx, filters);
1015
927
  case "query": {
1016
- const { queryCommand } = await import("./query-D_1ARx5q.js");
928
+ const { queryCommand } = await import("./query-Clm2mNSm.js");
1017
929
  return queryCommand({
1018
930
  ...ctx,
1019
931
  err: (s) => console.error(dim(s))
@@ -1021,7 +933,7 @@ async function run(argv) {
1021
933
  }
1022
934
  case "analyze": {
1023
935
  loadEnvFiles(configPath);
1024
- const { analyzeCommand } = await import("./command-CBaHukyl.js");
936
+ const { analyzeCommand } = await import("./command-9xm5ImqI.js");
1025
937
  return analyzeCommand({
1026
938
  ...ctx,
1027
939
  err: (s) => console.error(dim(s))
@@ -1033,6 +945,16 @@ async function run(argv) {
1033
945
  if (command !== "run") throw new UsageError(`Unknown command "${command}". Run reeval --help.`);
1034
946
  return runCommand(values, configPath, filters);
1035
947
  }
948
+ /** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
949
+ async function dryRunPrices(planned, judgeModel) {
950
+ const { needsPrices } = await import("./cost-C5RBTTJr.js");
951
+ if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
952
+ table: null,
953
+ why: "nothing to price"
954
+ };
955
+ const { loadPrices } = await import("./prices-QaS-kjSQ.js");
956
+ return loadPrices();
957
+ }
1036
958
  /** Asks a yes/no question on the terminal; anything but y or yes is no. */
1037
959
  async function confirm(question) {
1038
960
  const { createInterface } = await import("node:readline/promises");
@@ -1055,13 +977,13 @@ async function cleanCommand(flags, configPath) {
1055
977
  if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
1056
978
  }
1057
979
  const { debugRoot, pruneDebug } = await import("./files-BAt0i00X.js");
1058
- const { formatBytes } = await import("./debug-D7bEv_UV.js");
980
+ const { formatBytes } = await import("./debug-DeM5pswK.js");
1059
981
  const { runs, bytes } = await pruneDebug(config.root, keep);
1060
982
  const where = fromCwd(debugRoot(config.root)) || ".";
1061
983
  console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
1062
984
  if (flags.images) {
1063
- const { dockerAvailable, pruneImages } = await import("./src-D9fSTxlc.js");
1064
- const { requireDocker } = await import("./providers-CTBYvMxu.js");
985
+ const { dockerAvailable, pruneImages } = await import("./src-DH-9IO7f.js");
986
+ const { requireDocker } = await import("./providers-DkOupEBl.js");
1065
987
  const engines = [];
1066
988
  for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
1067
989
  if (!engines.length) await requireDocker();
@@ -1082,7 +1004,7 @@ function verbosityOf(flags) {
1082
1004
  * does. Needs a config, as `reeval run` does: there is no default image.
1083
1005
  */
1084
1006
  async function imageBuildCommand(flags, configPath, filters) {
1085
- const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-m784ueGK.js");
1007
+ const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-4mwhNkvh.js");
1086
1008
  const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
1087
1009
  ...flags,
1088
1010
  "dry-run": true
@@ -1142,7 +1064,7 @@ async function runCommand(flags, configPath, filters) {
1142
1064
  if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
1143
1065
  const stepStart = process.env.GITHUB_STEP_SUMMARY;
1144
1066
  if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
1145
- const { engineVersions } = await import("./node-CGZYh4Fj.js");
1067
+ const { engineVersions } = await import("./node-BVbbnFtv.js");
1146
1068
  const renderer = createRenderer({
1147
1069
  out: setup.json ? setup.dryRun ? { write: () => true } : process.stderr : process.stdout,
1148
1070
  output: setup.output,
@@ -1160,11 +1082,11 @@ async function runCommand(flags, configPath, filters) {
1160
1082
  });
1161
1083
  throw error;
1162
1084
  }
1163
- const { engineRun, plan, sandboxVersions, budgetReached, live, questions } = executed;
1085
+ const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
1164
1086
  invocation.unhandledErrors = engineRun.unhandledErrors;
1165
1087
  const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
1166
- if ((interrupted || engineRun.cancelled) && engines.length) {
1167
- const { removeRunContainers } = await import("./docker-Dac6Zalz.js");
1088
+ if ((interrupted || engineRun.canceled) && engines.length) {
1089
+ const { removeRunContainers } = await import("./docker-YFj4H1Cr.js");
1168
1090
  await removeRunContainers(ctx.runId, engines);
1169
1091
  }
1170
1092
  if (engineRun.stopped || !plan) {
@@ -1172,10 +1094,17 @@ async function runCommand(flags, configPath, filters) {
1172
1094
  renderer.close();
1173
1095
  return EXIT.ok;
1174
1096
  }
1175
- const stopped = interrupted || engineRun.cancelled;
1176
- const answers = questions && !stopped ? await questions.settle() : void 0;
1177
- const { lockfiles } = await import("./state-BGWGAwdo.js");
1178
- const { writeScratchRun } = await import("./results-D-naai_Q.js");
1097
+ let stopped = cutShort || engineRun.canceled;
1098
+ if (stopped) questions?.cancel();
1099
+ let answers = stopped ? await questions?.answered() : await questions?.settle();
1100
+ if (!stopped && interrupted && questions) {
1101
+ stopped = true;
1102
+ answers = await questions?.answered();
1103
+ }
1104
+ finishing = true;
1105
+ live?.phase("finishing");
1106
+ const { lockfiles } = await import("./state-B5PLm5qY.js");
1107
+ const { writeScratchRun } = await import("./results-BC3pcG_Z.js");
1179
1108
  const run = await writeScratchRun({
1180
1109
  ctx,
1181
1110
  author: ctx.author,
@@ -1196,60 +1125,11 @@ async function runCommand(flags, configPath, filters) {
1196
1125
  previous: ctx.previous,
1197
1126
  previousArtifacts: ctx.previousArtifacts,
1198
1127
  runner: runnerInfo(),
1199
- ...answers?.size ? { answers } : {}
1128
+ ...answers && (answers.evals.size || answers.attempts.size) ? { answers } : {}
1200
1129
  }, () => live?.stop());
1201
1130
  const { record } = run;
1202
1131
  const outcome = runOutcome(record);
1203
1132
  const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
1204
- let insights = run.insights;
1205
- let insightLines = [];
1206
- const { runInsights, wantsInsights } = await import("./run-Bel7Vde4.js");
1207
- const wanted = !stopped && wantsInsights(ctx.config, !flags["no-insights"]);
1208
- if (wanted && interrupted) insightLines = [`insights skipped (${stopCode === EXIT.terminated ? "SIGTERM" : "Ctrl-C"}): reeval analyze ${record.runId} --insights to write them`];
1209
- else if (wanted) {
1210
- const { scratchDir } = await import("./files-CtjEXWXm.js");
1211
- const stop = new AbortController();
1212
- let stopSignal = "SIGINT";
1213
- onInsightsSignal = (signal) => {
1214
- if (stop.signal.aborted) {
1215
- renderer.close();
1216
- process.exit(code);
1217
- }
1218
- stopSignal = signal;
1219
- stop.abort(/* @__PURE__ */ new Error(signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"));
1220
- };
1221
- try {
1222
- const done = await runInsights({
1223
- config: ctx.config,
1224
- record,
1225
- runDir: run.artifactDir ?? join(scratchDir(ctx.config.root), record.runId),
1226
- remarks: run.insights,
1227
- verbose: (ctx.verbosity ?? 0) > 0,
1228
- signal: stop.signal,
1229
- stopSignal: () => stopSignal,
1230
- onStart: (text) => {
1231
- renderer.onEvent({
1232
- type: "notice",
1233
- at: Date.now(),
1234
- level: "info",
1235
- text
1236
- });
1237
- },
1238
- onProgress: (n, total) => {
1239
- renderer.onEvent({
1240
- type: "insights-progress",
1241
- at: Date.now(),
1242
- done: n,
1243
- total
1244
- });
1245
- }
1246
- });
1247
- insights = done.insights ?? insights;
1248
- insightLines = done.lines;
1249
- } finally {
1250
- onInsightsSignal = null;
1251
- }
1252
- }
1253
1133
  const { pruneDebug } = await import("./files-BAt0i00X.js");
1254
1134
  await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
1255
1135
  const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
@@ -1259,9 +1139,7 @@ async function runCommand(flags, configPath, filters) {
1259
1139
  record,
1260
1140
  summaryPath: fromCwd(summary),
1261
1141
  results: engineRun.evals,
1262
- hint: !hasAnalysisModel(ctx.config),
1263
- insights,
1264
- insightLines
1142
+ remarks: run.remarks
1265
1143
  });
1266
1144
  renderer.close();
1267
1145
  const stepSummary = process.env.GITHUB_STEP_SUMMARY;
@@ -1269,7 +1147,7 @@ async function runCommand(flags, configPath, filters) {
1269
1147
  const { config } = ctx;
1270
1148
  const mode = setup.record;
1271
1149
  if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
1272
- const { record: recordCmd } = await import("./commands-BKGzBMgb.js");
1150
+ const { record: recordCmd } = await import("./commands-Bi2SEQtr.js");
1273
1151
  const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
1274
1152
  const line = await recordCmd(config, record.runId, {
1275
1153
  ...flags,
@@ -1279,7 +1157,7 @@ async function runCommand(flags, configPath, filters) {
1279
1157
  else console.log(line);
1280
1158
  }
1281
1159
  if (setup.json) {
1282
- const { suiteResult } = await import("./suite-mn9g4veR.js");
1160
+ const { suiteResult } = await import("./suite-22Kt9Grx.js");
1283
1161
  process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
1284
1162
  }
1285
1163
  return code;
@@ -1312,7 +1190,7 @@ async function prepareRun(flags, configPath) {
1312
1190
  const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
1313
1191
  const container = sandboxes.find((s) => s.kind === "docker");
1314
1192
  if (!container) return hostCapacity();
1315
- const { dockerCapacity } = await import("./src-D9fSTxlc.js");
1193
+ const { dockerCapacity } = await import("./src-DH-9IO7f.js");
1316
1194
  return await dockerCapacity(container.engine) ?? hostCapacity();
1317
1195
  });
1318
1196
  return {
@@ -1327,8 +1205,6 @@ async function prepareRun(flags, configPath) {
1327
1205
  dryRun: !!flags["dry-run"],
1328
1206
  json: !!flags.json,
1329
1207
  force: !!flags.force,
1330
- /** `--no-insights` turns them off for this run. */
1331
- insights: !flags["no-insights"],
1332
1208
  verbosity: verbosityOf(flags),
1333
1209
  configFile: configLabel(path),
1334
1210
  /** Named in the plan line when the project root isn't the current directory. */
@@ -1341,11 +1217,11 @@ async function prepareRun(flags, configPath) {
1341
1217
  async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
1342
1218
  const startedAt = /* @__PURE__ */ new Date();
1343
1219
  const git = await gitInfo(config.root, [config.history.dir]);
1344
- const { worktreePatch } = await import("./state-BGWGAwdo.js");
1220
+ const { worktreePatch } = await import("./state-B5PLm5qY.js");
1345
1221
  const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
1346
1222
  const who = await author(config.root);
1347
- const { stagingDir, sweepStaging } = await import("./results-D-naai_Q.js");
1348
- const { scratchDir } = await import("./files-CtjEXWXm.js");
1223
+ const { stagingDir, sweepStaging } = await import("./results-BC3pcG_Z.js");
1224
+ const { scratchDir } = await import("./files-DmUrRmmL.js");
1349
1225
  const scratch = scratchDir(config.root);
1350
1226
  if (resume !== void 0) {
1351
1227
  const target = await resolveResume(config.root, resume || void 0);
@@ -1356,7 +1232,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
1356
1232
  artifacts: target.previousArtifacts
1357
1233
  } : await loadPrevious(stagingDir(config.root, target.runId));
1358
1234
  if (!dryRun) try {
1359
- const { removeRunContainers } = await import("./docker-Dac6Zalz.js");
1235
+ const { removeRunContainers } = await import("./docker-YFj4H1Cr.js");
1360
1236
  await removeRunContainers(target.runId, ["docker", "podman"]);
1361
1237
  } catch {}
1362
1238
  return {
@@ -1427,8 +1303,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1427
1303
  configFile,
1428
1304
  completed: []
1429
1305
  }));
1430
- const { runEngine } = await import("./node-CGZYh4Fj.js");
1431
- const { buildPlan, checkAttemptBudget, planDocument, planInsights, planLine, planTable } = await import("./plan-CZYgXnZh.js");
1306
+ const { runEngine } = await import("./node-BVbbnFtv.js");
1307
+ const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-DdWkM6SW.js");
1432
1308
  let plan;
1433
1309
  let sandboxVersions = { images: [] };
1434
1310
  let preflightStop;
@@ -1436,10 +1312,13 @@ async function executeRun(ctx, setup, filters, renderer) {
1436
1312
  let budgetTimer;
1437
1313
  let toRun = [];
1438
1314
  let live;
1315
+ let kept = /* @__PURE__ */ new Map();
1439
1316
  let questions;
1440
- const onPlan = async ({ planned, excluded, matched, unmatched }) => {
1317
+ const onPlan = async (collected) => {
1318
+ const { planned, excluded, matched, unmatched } = collected;
1441
1319
  if (unmatched.length) throw unmatchedFilter(config, unmatched);
1442
1320
  if (matched === 0) throw noMatch(config, filters, tags);
1321
+ checkLimitHarness(profile, config.file, collected);
1443
1322
  const completed = completedKeysFor(planned, stagingDir, ctx.previous);
1444
1323
  const skipped = planned.map((p) => completed.has(p.key) ? {
1445
1324
  ...p,
@@ -1454,7 +1333,9 @@ async function executeRun(ctx, setup, filters, renderer) {
1454
1333
  } : p);
1455
1334
  toRun = skipped;
1456
1335
  for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
1457
- const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-B8H3gL69.js") : void 0;
1336
+ const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-g3XnzUv0.js") : void 0;
1337
+ const judge = runJudgeModel(config, profile);
1338
+ const judgeModel = judge === void 0 ? null : judgeModelName(judge);
1458
1339
  plan = await buildPlan({
1459
1340
  planned: skipped,
1460
1341
  profile,
@@ -1462,21 +1343,21 @@ async function executeRun(ctx, setup, filters, renderer) {
1462
1343
  concurrency,
1463
1344
  root: config.root,
1464
1345
  historyDir: config.history.dir,
1465
- questionModel: models?.questionModelLabel(config) ?? null,
1466
- insights: await planInsights(config, setup.insights)
1346
+ questionModel: models?.questionModelLabel(config, judge) ?? null,
1347
+ judgeModel,
1348
+ ...setup.dryRun ? { prices: await dryRunPrices(skipped, judgeModel) } : {}
1467
1349
  });
1468
1350
  plan.runId = runId;
1469
- if (plan.insights) {
1470
- const { analysisKeyProblem } = await import("./model-B8H3gL69.js");
1471
- const problem = analysisKeyProblem(config);
1472
- if (problem) plan.warnings.push(`insights will be skipped: ${problem}`);
1473
- }
1351
+ const cloud = ciRunner(process.env);
1352
+ const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
1353
+ if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
1474
1354
  if (completed.size) plan.resume = {
1475
1355
  runId,
1476
1356
  completed: completed.size
1477
1357
  };
1478
1358
  run.completed = [...completed];
1479
- const { imageStages, imageTargets } = await import("./images-m784ueGK.js");
1359
+ kept = keptAttempts(completed, stagingDir, ctx.previous);
1360
+ const { imageStages, imageTargets } = await import("./images-4mwhNkvh.js");
1480
1361
  const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
1481
1362
  id: p.id,
1482
1363
  harness: p.attempt.harness,
@@ -1498,8 +1379,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1498
1379
  if (setup.dryRun) {
1499
1380
  const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
1500
1381
  lines.push(...planTable(plan, todos));
1501
- const { imageStatusLines } = await import("./images-m784ueGK.js");
1502
- const { dockerAvailable, engineSpec } = await import("./src-D9fSTxlc.js");
1382
+ const { imageStatusLines } = await import("./images-4mwhNkvh.js");
1383
+ const { dockerAvailable, engineSpec } = await import("./src-DH-9IO7f.js");
1503
1384
  const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
1504
1385
  const down = [];
1505
1386
  for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
@@ -1508,10 +1389,9 @@ async function executeRun(ctx, setup, filters, renderer) {
1508
1389
  const names = down.map((e) => engineSpec(e).name).join(" and ");
1509
1390
  lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
1510
1391
  }
1511
- }
1512
- if (setup.dryRun && plan.insights) {
1513
- const { formatTokens } = await import("./usage-BDo2q_yu.js");
1514
- lines.push(`insights ${plan.insights.model} · up to ${formatTokens(plan.insights.maxTokens)} tokens`);
1392
+ const { costLine } = await import("./cost-C5RBTTJr.js");
1393
+ const cost = plan.cost && costLine(plan.cost);
1394
+ if (cost) lines.push(cost);
1515
1395
  }
1516
1396
  lines.push(...plan.warnings.map((w) => `warning: ${w}`));
1517
1397
  lines.push(...plan.hints.map((h) => `hint: ${h}`));
@@ -1525,38 +1405,64 @@ async function executeRun(ctx, setup, filters, renderer) {
1525
1405
  checkAttemptBudget(plan, setup.force, config.file);
1526
1406
  const questionModels = /* @__PURE__ */ new Set();
1527
1407
  const asked = [];
1408
+ const scope = gatewayScope(config, profile);
1528
1409
  for (const row of plan.rows) {
1529
1410
  if (row.plan.skip || !models) continue;
1530
- const questions = (row.plan.questions ?? []).map((q) => ({
1531
- question: q.question,
1532
- model: models.questionModel(config, q, row.id),
1533
- ...q.across ? { across: q.across } : {},
1534
- ...q.state ? { state: q.state } : {},
1535
- ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
1536
- }));
1411
+ const questions = (row.plan.questions ?? []).map((q) => {
1412
+ const model = models.questionModel(config, q, row.id, judge);
1413
+ const gateway = resolveModelGateway(model, void 0, scope, "the question model");
1414
+ return {
1415
+ n: q.n,
1416
+ question: q.question,
1417
+ model,
1418
+ ...gateway ? { gateway } : {},
1419
+ ...gateway === "openrouter" && openRouterFallbacks(scope.gateways) ? { fallbacks: true } : {},
1420
+ ...gateway === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {},
1421
+ ...q.across !== void 0 ? { across: q.across } : {},
1422
+ ...q.choices ? { choices: q.choices } : {},
1423
+ ...q.state ? { state: q.state } : {},
1424
+ ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
1425
+ };
1426
+ });
1537
1427
  for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
1538
1428
  if (questions.length) asked.push({
1539
1429
  row,
1540
1430
  questions
1541
1431
  });
1542
1432
  }
1433
+ const judges = /* @__PURE__ */ new Map();
1434
+ for (const p of skipped) {
1435
+ const j = p.attempt.judge;
1436
+ if (j && !p.attempt.plan.skip) judges.set(`${j.model}\0${j.gateway ?? ""}`, j);
1437
+ }
1543
1438
  const missing = missingCredentials(config, process.env, {
1544
1439
  harnesses: plan.harnessDescriptors,
1545
1440
  profile,
1546
- questionModels: [...questionModels]
1441
+ questionModels: [...questionModels],
1442
+ judges: [...judges.values()]
1547
1443
  });
1548
- if (missing.length) throw new UsageError(`Missing credentials: ${missing.join("; ")}. Set them in the environment or .env.local.`);
1444
+ if (missing.length) {
1445
+ const hint = missing.some((m) => m.includes("(for direct())")) ? unusedGatewayKey(process.env) : "";
1446
+ throw new UsageError(`Missing credentials: ${missing.join("; ")}. Set them in the environment or .env.local.${hint}`);
1447
+ }
1549
1448
  const malformed = malformedCredentials(config, process.env, {
1550
1449
  harnesses: plan.harnessDescriptors,
1551
1450
  profile,
1552
- questionModels: [...questionModels]
1451
+ questionModels: [...questionModels],
1452
+ judges: [...judges.values()]
1553
1453
  });
1554
1454
  if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
1555
1455
  if (asked.length) {
1556
- const { questionRunner } = await import("./questions-KENpWnfV.js");
1456
+ const { questionRunner } = await import("./questions-BRhU8MzB.js");
1457
+ const warned = /* @__PURE__ */ new Set();
1557
1458
  questions = questionRunner({
1558
1459
  evals: asked,
1559
1460
  stagingDir,
1461
+ resumed: {
1462
+ completed: new Set(run.completed),
1463
+ previous: ctx.previous,
1464
+ artifacts: ctx.previousArtifacts
1465
+ },
1560
1466
  maxInputTokens: config.analysis.maxInputTokens,
1561
1467
  maxOutputTokens: config.analysis.maxOutputTokens,
1562
1468
  onAnswer: (id, answer) => {
@@ -1565,12 +1471,31 @@ async function executeRun(ctx, setup, filters, renderer) {
1565
1471
  type: "notice",
1566
1472
  at: Date.now(),
1567
1473
  level: "warn",
1568
- text: `compare question "${clip(answer.question, 80)}" on ${id} failed: ${answer.error}. Outcomes are unchanged.`
1474
+ text: `question "${clip(answer.question, 80)}" on ${id} failed: ${answer.error}. Outcomes are unchanged.`
1475
+ });
1476
+ },
1477
+ onAttemptAnswer: (id, attempt, answer) => {
1478
+ const once = `${id}\0${answer.question}`;
1479
+ if (answer.error === void 0 || warned.has(once)) return;
1480
+ warned.add(once);
1481
+ renderer.onEvent({
1482
+ type: "notice",
1483
+ at: Date.now(),
1484
+ level: "warn",
1485
+ text: `question "${clip(answer.question, 80)}" on ${attempt} failed: ${answer.error}. Outcomes are unchanged; later failures of it on ${id} aren't shown.`
1486
+ });
1487
+ },
1488
+ onUnshown: (id, error) => {
1489
+ renderer.onEvent({
1490
+ type: "notice",
1491
+ at: Date.now(),
1492
+ level: "warn",
1493
+ text: `${error} on ${id}, so each attempt's answer is that error. Outcomes are unchanged.`
1569
1494
  });
1570
1495
  }
1571
1496
  });
1572
1497
  }
1573
- const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-B1eGO46M.js");
1498
+ const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-DDHzkMjm.js");
1574
1499
  const live = skipped.filter((p) => !p.attempt.plan.skip);
1575
1500
  const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
1576
1501
  workspace: a.workspace,
@@ -1661,6 +1586,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1661
1586
  budgetTimer.unref();
1662
1587
  };
1663
1588
  let engineRun;
1589
+ let cutShort = false;
1664
1590
  try {
1665
1591
  engineRun = await runEngine({
1666
1592
  run,
@@ -1672,7 +1598,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1672
1598
  dir: stagingDir,
1673
1599
  runId,
1674
1600
  startedAt: ctx.startedAt,
1675
- planned: toRun
1601
+ planned: toRun,
1602
+ kept
1676
1603
  });
1677
1604
  renderer.onEvent({
1678
1605
  type: "run-start",
@@ -1697,22 +1624,22 @@ async function executeRun(ctx, setup, filters, renderer) {
1697
1624
  if (interrupted && e.type === "eval-end" && e.result.state === "skipped") return;
1698
1625
  renderer.onEvent(e);
1699
1626
  },
1700
- onCancellable: (cancel) => {
1627
+ onCancelable: (cancel) => {
1701
1628
  const onSignal = (signal) => {
1702
- if (onInsightsSignal) return onInsightsSignal(signal === "SIGTERM" ? "SIGTERM" : "SIGINT");
1703
1629
  stopCode = signal === "SIGTERM" ? EXIT.terminated : EXIT.interrupted;
1704
1630
  if (interrupted) {
1705
1631
  renderer.close();
1706
1632
  process.exit(stopCode);
1707
1633
  }
1708
1634
  interrupted = true;
1635
+ if (finishing) return;
1709
1636
  questions?.cancel();
1710
1637
  live?.phase("cancelling");
1711
1638
  renderer.onEvent({
1712
1639
  type: "notice",
1713
1640
  at: Date.now(),
1714
1641
  level: "warn",
1715
- text: "Cancelling: stopping agents and destroying sandboxes (Ctrl-C again to force)"
1642
+ text: "Canceling: stopping agents and destroying sandboxes (Ctrl-C again to force)"
1716
1643
  });
1717
1644
  cancel();
1718
1645
  };
@@ -1720,15 +1647,15 @@ async function executeRun(ctx, setup, filters, renderer) {
1720
1647
  process.on("SIGTERM", onSignal);
1721
1648
  }
1722
1649
  });
1723
- live?.phase("finishing");
1650
+ cutShort = interrupted;
1724
1651
  } catch (error) {
1725
1652
  live?.remove();
1726
1653
  throw error;
1727
1654
  } finally {
1728
1655
  clearTimeout(budgetTimer);
1729
- const { removePrepared } = await import("./preflight-BQlPTHVu.js");
1656
+ const { removePrepared } = await import("./preflight-DTiVv0gY.js");
1730
1657
  removePrepared(config.root, runId);
1731
- const { removePreparedWorkspaces } = await import("./workspace-B1eGO46M.js");
1658
+ const { removePreparedWorkspaces } = await import("./workspace-DDHzkMjm.js");
1732
1659
  removePreparedWorkspaces(config.root, runId);
1733
1660
  }
1734
1661
  if (preflightStop) throw preflightStop;
@@ -1740,6 +1667,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1740
1667
  plan,
1741
1668
  sandboxVersions,
1742
1669
  budgetReached: () => reached,
1670
+ cutShort,
1743
1671
  live,
1744
1672
  questions
1745
1673
  };
@@ -1810,4 +1738,4 @@ run(process.argv.slice(2)).finally(() => {
1810
1738
  //#endregion
1811
1739
  export {};
1812
1740
 
1813
- //# sourceMappingURL=cli-BCE1xlm4.js.map
1741
+ //# sourceMappingURL=cli-PdYrcoKM.js.map