reeval 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (313) hide show
  1. package/README.md +2 -0
  2. package/THIRD_PARTY_NOTICES.md +121 -121
  3. package/dist/{adapter-IKKo4sZj.js → adapter-BloGVZ_a.js} +20 -8
  4. package/dist/adapter-BloGVZ_a.js.map +1 -0
  5. package/dist/{agent-ClE-g9VL.js → agent-m-_t-IuW.js} +53 -25
  6. package/dist/{agent-ClE-g9VL.js.map → agent-m-_t-IuW.js.map} +1 -1
  7. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  8. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  9. package/dist/app-files-C-jwhRDt.js +11 -0
  10. package/dist/app-files-C-jwhRDt.js.map +1 -0
  11. package/dist/{assemble-BO93VAk1.js → assemble-BDEpelpi.js} +92 -35
  12. package/dist/assemble-BDEpelpi.js.map +1 -0
  13. package/dist/assemble-C5bfncHU.js +1 -0
  14. package/dist/{capture-dn2hjSLj.js → capture-CVRwRTnb.js} +572 -36
  15. package/dist/capture-CVRwRTnb.js.map +1 -0
  16. package/dist/capture-CherbjzB.js +1 -0
  17. package/dist/{cli-BCE1xlm4.js → cli-C_4gaJ50.js} +132 -140
  18. package/dist/cli-C_4gaJ50.js.map +1 -0
  19. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
  20. package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
  21. package/dist/cli.js +2 -2
  22. package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
  23. package/dist/client-D5hNjpEx.js.map +1 -0
  24. package/dist/{command-C6vwmjS7.js → command-BWAuwQNk.js} +12 -12
  25. package/dist/{command-C6vwmjS7.js.map → command-BWAuwQNk.js.map} +1 -1
  26. package/dist/{command-CANun0m0.js → command-BXLUfw4h.js} +12 -12
  27. package/dist/{command-CANun0m0.js.map → command-BXLUfw4h.js.map} +1 -1
  28. package/dist/{command-CBaHukyl.js → command-Be6BSE2L.js} +12 -12
  29. package/dist/{command-CBaHukyl.js.map → command-Be6BSE2L.js.map} +1 -1
  30. package/dist/{command-D_kEnYcX.js → command-Bxdy62dW.js} +14 -13
  31. package/dist/command-Bxdy62dW.js.map +1 -0
  32. package/dist/{command-Bh3yhN_b.js → command-I1euLLio.js} +35 -15
  33. package/dist/command-I1euLLio.js.map +1 -0
  34. package/dist/{commands-7tH3EsVu.js → commands-jzigbNFb.js} +23 -14
  35. package/dist/commands-jzigbNFb.js.map +1 -0
  36. package/dist/{commands-BKGzBMgb.js → commands-p1S4nw_H.js} +1 -1
  37. package/dist/config-BrdHA9bJ.js +3549 -0
  38. package/dist/config-BrdHA9bJ.js.map +1 -0
  39. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DRxMQQTC.js} +3 -2
  40. package/dist/contrasts-DRxMQQTC.js.map +1 -0
  41. package/dist/cost-D4elx933.js +359 -0
  42. package/dist/cost-D4elx933.js.map +1 -0
  43. package/dist/cost-DmnAod9x.js +1 -0
  44. package/dist/db-CAfksinX.js +2 -0
  45. package/dist/{db-DJUuFoWQ.js → db-DO0p7xgP.js} +25 -8
  46. package/dist/db-DO0p7xgP.js.map +1 -0
  47. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  48. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  49. package/dist/{docker-DuHcQidD.js → docker-BnRlTpZA.js} +2 -2
  50. package/dist/{docker-DuHcQidD.js.map → docker-BnRlTpZA.js.map} +1 -1
  51. package/dist/{docker-CQxucRPe.js → docker-D01slDF3.js} +26 -10
  52. package/dist/docker-D01slDF3.js.map +1 -0
  53. package/dist/{docker-Dac6Zalz.js → docker-DivfZl2u.js} +1 -1
  54. package/dist/{doctor-De_QlOTM.js → doctor-DQ2p0euG.js} +135 -36
  55. package/dist/doctor-DQ2p0euG.js.map +1 -0
  56. package/dist/{environment-B4IuURK1.js → environment-CO0P0soF.js} +8 -7
  57. package/dist/environment-CO0P0soF.js.map +1 -0
  58. package/dist/{evaluate-Djk1YPFL.js → evaluate-BUmsN7yG.js} +66 -27
  59. package/dist/evaluate-BUmsN7yG.js.map +1 -0
  60. package/dist/{evaluate-C9-ritA5.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
  61. package/dist/{facts-Cfx1H70y.js → facts-DzOUTreB.js} +12 -4
  62. package/dist/facts-DzOUTreB.js.map +1 -0
  63. package/dist/{files-CN1QfenC.js → files-B-CJeeEh.js} +13 -8
  64. package/dist/{files-CN1QfenC.js.map → files-B-CJeeEh.js.map} +1 -1
  65. package/dist/files-Bp5j5BqI.js +1 -0
  66. package/dist/files-C2sjmXhk.js +1 -0
  67. package/dist/{files-Bs6H3HiG.js → files-C_gQNVB-.js} +3 -3
  68. package/dist/{files-Bs6H3HiG.js.map → files-C_gQNVB-.js.map} +1 -1
  69. package/dist/{files-D27nUk0K.js → files-kUCrswI6.js} +2 -2
  70. package/dist/{files-D27nUk0K.js.map → files-kUCrswI6.js.map} +1 -1
  71. package/dist/gateway.d.ts +17 -0
  72. package/dist/gateway.js +17 -0
  73. package/dist/gateway.js.map +1 -0
  74. package/dist/{generate-BnrvBAzF.js → generate-DZ7vp69F.js} +18 -17
  75. package/dist/generate-DZ7vp69F.js.map +1 -0
  76. package/dist/{git-rPfl7AD1.js → git-Bk8DSwHy.js} +2 -2
  77. package/dist/{git-rPfl7AD1.js.map → git-Bk8DSwHy.js.map} +1 -1
  78. package/dist/{harness-1OkumOAm.js → harness-COVXSwJ6.js} +46 -15
  79. package/dist/harness-COVXSwJ6.js.map +1 -0
  80. package/dist/harness.d.ts +26 -18
  81. package/dist/harness.js +1 -1
  82. package/dist/{header-BVzbRnNv.js → header-BLPP6aCU.js} +2 -2
  83. package/dist/{header-BVzbRnNv.js.map → header-BLPP6aCU.js.map} +1 -1
  84. package/dist/{images-m784ueGK.js → images-DzLeFjLx.js} +8 -8
  85. package/dist/{images-m784ueGK.js.map → images-DzLeFjLx.js.map} +1 -1
  86. package/dist/index.d.ts +3 -45
  87. package/dist/index.js +5 -6
  88. package/dist/init-RgCQXZhb.js +1 -0
  89. package/dist/{init-hiTkiM4L.js → init-SRalM5xF.js} +6 -8
  90. package/dist/init-SRalM5xF.js.map +1 -0
  91. package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
  92. package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
  93. package/dist/{invocations-D_ePcwyL.js → invocations-DzyKSJBa.js} +2 -2
  94. package/dist/{invocations-D_ePcwyL.js.map → invocations-DzyKSJBa.js.map} +1 -1
  95. package/dist/judge-yLvjC5A3.js +385 -0
  96. package/dist/judge-yLvjC5A3.js.map +1 -0
  97. package/dist/judge.d.ts +7 -6
  98. package/dist/judge.js +7 -5
  99. package/dist/judge.js.map +1 -1
  100. package/dist/match-Bz0nSdzN.js +204 -0
  101. package/dist/match-Bz0nSdzN.js.map +1 -0
  102. package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
  103. package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
  104. package/dist/{model-DpkclE4r.js → model-1ZLr9Dcv.js} +40 -26
  105. package/dist/model-1ZLr9Dcv.js.map +1 -0
  106. package/dist/model-B45lDUV1.js +68 -0
  107. package/dist/model-B45lDUV1.js.map +1 -0
  108. package/dist/model-BTZq8HIZ.js +1 -0
  109. package/dist/{model-UgHDMhI1.js → model-CgnOpss4.js} +57 -318
  110. package/dist/model-CgnOpss4.js.map +1 -0
  111. package/dist/{model-B8H3gL69.js → model-Ch8kX87u.js} +1 -1
  112. package/dist/{node-CGZYh4Fj.js → node-BSq5qHnQ.js} +12 -5
  113. package/dist/node-BSq5qHnQ.js.map +1 -0
  114. package/dist/{opencode-providers-snfeWd6Z.js → opencode-providers-klkqgzvb.js} +23 -52
  115. package/dist/opencode-providers-klkqgzvb.js.map +1 -0
  116. package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
  117. package/dist/pairs-CHfZNRT6.js.map +1 -0
  118. package/dist/{patch-uMq9rCl_.js → patch-DWf-I_Q_.js} +2 -2
  119. package/dist/{patch-uMq9rCl_.js.map → patch-DWf-I_Q_.js.map} +1 -1
  120. package/dist/{plan-CZYgXnZh.js → plan-BIwUzlB8.js} +63 -26
  121. package/dist/plan-BIwUzlB8.js.map +1 -0
  122. package/dist/{plugins-P-uh54VF.js → plugins-Dkp_H3pB.js} +4 -4
  123. package/dist/{plugins-P-uh54VF.js.map → plugins-Dkp_H3pB.js.map} +1 -1
  124. package/dist/{preflight-BQlPTHVu.js → preflight-D9y-HvZQ.js} +1 -1
  125. package/dist/{preflight-D3HvKeAx.js → preflight-DJCD_BDn.js} +3 -3
  126. package/dist/{preflight-D3HvKeAx.js.map → preflight-DJCD_BDn.js.map} +1 -1
  127. package/dist/{prepare-DCwM2KKD.js → prepare-BJG_pD54.js} +2 -2
  128. package/dist/{prepare-Dx7dOnJB.js → prepare-CIxCGG0M.js} +8 -8
  129. package/dist/{prepare-Dx7dOnJB.js.map → prepare-CIxCGG0M.js.map} +1 -1
  130. package/dist/prices-BcLEXvdY.js +250 -0
  131. package/dist/prices-BcLEXvdY.js.map +1 -0
  132. package/dist/{provenance-C_-8y-x3.js → provenance-CfZpFpAh.js} +2 -2
  133. package/dist/{provenance-C_-8y-x3.js.map → provenance-CfZpFpAh.js.map} +1 -1
  134. package/dist/{providers-CTBYvMxu.js → providers-D1Zxp2he.js} +1 -1
  135. package/dist/{providers-CF5UEaP8.js → providers-DFk_6W4K.js} +5 -5
  136. package/dist/{providers-CF5UEaP8.js.map → providers-DFk_6W4K.js.map} +1 -1
  137. package/dist/{query-DXthKmq0.js → query-CemclveB.js} +2 -2
  138. package/dist/{query-DXthKmq0.js.map → query-CemclveB.js.map} +1 -1
  139. package/dist/query-Cvbwl804.js +1 -0
  140. package/dist/{questions-KENpWnfV.js → questions-bAudIc04.js} +308 -45
  141. package/dist/questions-bAudIc04.js.map +1 -0
  142. package/dist/{remarks-Ml6bcSdV.js → remarks-CXwSnSjR.js} +9 -9
  143. package/dist/{remarks-Ml6bcSdV.js.map → remarks-CXwSnSjR.js.map} +1 -1
  144. package/dist/remarks-CyAu2DiB.js +1 -0
  145. package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
  146. package/dist/remarks-mujJv1Ey.js.map +1 -0
  147. package/dist/{remote-Dkeghk8k.js → remote-CKcEKja4.js} +129 -174
  148. package/dist/remote-CKcEKja4.js.map +1 -0
  149. package/dist/resolve-B5-uqmJ6.js +1 -0
  150. package/dist/{resolve-DzdWcIL6.js → resolve-CBoKzQMP.js} +8 -9
  151. package/dist/resolve-CBoKzQMP.js.map +1 -0
  152. package/dist/{results-D-naai_Q.js → results-CMgZEANy.js} +8 -8
  153. package/dist/{results-D-naai_Q.js.map → results-CMgZEANy.js.map} +1 -1
  154. package/dist/{resume-8-nnW8iO.js → resume-B9SOSnK9.js} +3 -3
  155. package/dist/{resume-8-nnW8iO.js.map → resume-B9SOSnK9.js.map} +1 -1
  156. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  157. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  158. package/dist/{rules-CcV3zLDR.js → rules-VhftPOOT.js} +53 -5
  159. package/dist/rules-VhftPOOT.js.map +1 -0
  160. package/dist/{run-Bel7Vde4.js → run-CwIEK16q.js} +9 -9
  161. package/dist/{run-Bel7Vde4.js.map → run-CwIEK16q.js.map} +1 -1
  162. package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
  163. package/dist/runtime-BgQRyCLl.js.map +1 -0
  164. package/dist/{sandbox-YLbk9mO7.js → sandbox-BOYZ0hlo.js} +2 -2
  165. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
  166. package/dist/sandbox.d.ts +2 -1
  167. package/dist/sandbox.js +1 -1
  168. package/dist/server-CHRSgKRp.js +1 -0
  169. package/dist/{server-De8oH3bt.js → server-TGW4WyAz.js} +25 -26
  170. package/dist/{server-De8oH3bt.js.map → server-TGW4WyAz.js.map} +1 -1
  171. package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
  172. package/dist/shared-BDUzStad.js.map +1 -0
  173. package/dist/shared-CHvO7eLi.d.ts +1 -0
  174. package/dist/{source-C686eZpY.js → source-B1WvBIQt.js} +2 -2
  175. package/dist/{source-C686eZpY.js.map → source-B1WvBIQt.js.map} +1 -1
  176. package/dist/source-DAzztZwU.js +1 -0
  177. package/dist/src-1jHbFaGW.js +1 -0
  178. package/dist/{src-CIpKL45U.js → src-8zoX_qrb.js} +36 -26
  179. package/dist/{src-CIpKL45U.js.map → src-8zoX_qrb.js.map} +1 -1
  180. package/dist/{src-CzQrAcF8.js → src-CHIwOy0o.js} +84 -52
  181. package/dist/src-CHIwOy0o.js.map +1 -0
  182. package/dist/src-C_9wgowC.js +2 -0
  183. package/dist/{src-djNNYIdP.js → src-ChhtI6BP.js} +2 -2
  184. package/dist/{src-djNNYIdP.js.map → src-ChhtI6BP.js.map} +1 -1
  185. package/dist/{src-DrJSnoMR.js → src-CyaDNG9r.js} +1 -1
  186. package/dist/src-D005Cbqz.js +2 -0
  187. package/dist/src-DIfRbp5a.js +3 -0
  188. package/dist/{src-C7W6kg81.js → src-DKPVX2Us.js} +34 -16
  189. package/dist/{src-C7W6kg81.js.map → src-DKPVX2Us.js.map} +1 -1
  190. package/dist/src-Dg8121Xy.js +2 -0
  191. package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
  192. package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
  193. package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
  194. package/dist/src-DjRI-IeE.js.map +1 -0
  195. package/dist/{src-DeY52Gyf.js → src-DmUgeTlE.js} +4 -4
  196. package/dist/{src-DeY52Gyf.js.map → src-DmUgeTlE.js.map} +1 -1
  197. package/dist/src-ElBqEN-S.js +2 -0
  198. package/dist/{src-Xj_BZKbq.js → src-NSdQ9zI6.js} +31 -16
  199. package/dist/{src-Xj_BZKbq.js.map → src-NSdQ9zI6.js.map} +1 -1
  200. package/dist/src-YHGaUrGE.js +2 -0
  201. package/dist/src-mJVxw1Jm.js +2 -0
  202. package/dist/{state-C0i_ujQZ.js → state-B1t143wG.js} +18 -18
  203. package/dist/{state-C0i_ujQZ.js.map → state-B1t143wG.js.map} +1 -1
  204. package/dist/state-vznMh2wl.js +1 -0
  205. package/dist/stats-DSy5HfOv.js +315 -0
  206. package/dist/stats-DSy5HfOv.js.map +1 -0
  207. package/dist/suite-BsWGHzlG.js +2 -0
  208. package/dist/{suite-DmuF_ApO.js → suite-WgT7m6Vs.js} +76 -11
  209. package/dist/suite-WgT7m6Vs.js.map +1 -0
  210. package/dist/{summary-BFMHMTz2.js → summary-DIGEVr08.js} +58 -17
  211. package/dist/summary-DIGEVr08.js.map +1 -0
  212. package/dist/testing.d.ts +174 -2
  213. package/dist/testing.js +14 -3
  214. package/dist/testing.js.map +1 -0
  215. package/dist/types-C5vI3g1C.js.map +1 -1
  216. package/dist/{types-B-9b1MDV.d.ts → types-DXBbP9Mk.d.ts} +298 -70
  217. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  218. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  219. package/dist/usage-DF3_p5pE.js +1 -0
  220. package/dist/{validate-DWjkshl1.js → validate-BhwqrVsm.js} +44 -12
  221. package/dist/validate-BhwqrVsm.js.map +1 -0
  222. package/dist/view/index.d.ts +133 -86
  223. package/dist/view/index.js +7 -6
  224. package/dist/views-BW_C5eVH.js +2 -0
  225. package/dist/{views-Dk8MCQKr.js → views-sXpGE6a2.js} +10 -8
  226. package/dist/views-sXpGE6a2.js.map +1 -0
  227. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-Duo9oAeT.js} +8 -8
  228. package/dist/{worker-entry-CqrzGPF4.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
  229. package/dist/{workspace-CLeOybkr.js → workspace-CDI68wtX.js} +4 -4
  230. package/dist/{workspace-CLeOybkr.js.map → workspace-CDI68wtX.js.map} +1 -1
  231. package/dist/workspace-QZCacP8A.js +1 -0
  232. package/package.json +6 -2
  233. package/skills/eval-rubric/SKILL.md +9 -0
  234. package/dist/adapter-IKKo4sZj.js.map +0 -1
  235. package/dist/assemble-BO93VAk1.js.map +0 -1
  236. package/dist/assemble-VQxyoDxK.js +0 -1
  237. package/dist/assert-EUIoRdIt.js.map +0 -1
  238. package/dist/capture-dn2hjSLj.js.map +0 -1
  239. package/dist/capture-nIJ-7kUE.js +0 -1
  240. package/dist/cli-BCE1xlm4.js.map +0 -1
  241. package/dist/client-fcwgwOAD.js +0 -106
  242. package/dist/client-fcwgwOAD.js.map +0 -1
  243. package/dist/command-Bh3yhN_b.js.map +0 -1
  244. package/dist/command-D_kEnYcX.js.map +0 -1
  245. package/dist/commands-7tH3EsVu.js.map +0 -1
  246. package/dist/config-D7TFvUpL.js +0 -759
  247. package/dist/config-D7TFvUpL.js.map +0 -1
  248. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  249. package/dist/db-DJUuFoWQ.js.map +0 -1
  250. package/dist/db-W9EFlXnn.js +0 -2
  251. package/dist/docker-CQxucRPe.js.map +0 -1
  252. package/dist/doctor-De_QlOTM.js.map +0 -1
  253. package/dist/egress-DEAvq8Fl.js +0 -1258
  254. package/dist/egress-DEAvq8Fl.js.map +0 -1
  255. package/dist/environment-B4IuURK1.js.map +0 -1
  256. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  257. package/dist/facts-Cfx1H70y.js.map +0 -1
  258. package/dist/files-CtjEXWXm.js +0 -1
  259. package/dist/files-D3gJa91B.js +0 -1
  260. package/dist/generate-BnrvBAzF.js.map +0 -1
  261. package/dist/harness-1OkumOAm.js.map +0 -1
  262. package/dist/init-hiTkiM4L.js.map +0 -1
  263. package/dist/init-kMkTgTZc.js +0 -1
  264. package/dist/judge-CEDIfRiU.js +0 -186
  265. package/dist/judge-CEDIfRiU.js.map +0 -1
  266. package/dist/model-CTiuzkWw.js +0 -46
  267. package/dist/model-CTiuzkWw.js.map +0 -1
  268. package/dist/model-DcIbmggK.js +0 -1
  269. package/dist/model-DpkclE4r.js.map +0 -1
  270. package/dist/model-UgHDMhI1.js.map +0 -1
  271. package/dist/node-CGZYh4Fj.js.map +0 -1
  272. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  273. package/dist/options-CyAyLjgJ.js +0 -1129
  274. package/dist/options-CyAyLjgJ.js.map +0 -1
  275. package/dist/pairs-DP6XOJby.js.map +0 -1
  276. package/dist/plan-CZYgXnZh.js.map +0 -1
  277. package/dist/query-D_1ARx5q.js +0 -1
  278. package/dist/questions-KENpWnfV.js.map +0 -1
  279. package/dist/remarks-1521zwfW.js.map +0 -1
  280. package/dist/remarks-BhAV1H4V.js +0 -1
  281. package/dist/remote-Dkeghk8k.js.map +0 -1
  282. package/dist/resolve-CMmhACr6.js +0 -1
  283. package/dist/resolve-DzdWcIL6.js.map +0 -1
  284. package/dist/rules-CcV3zLDR.js.map +0 -1
  285. package/dist/runtime-PX_oXiCS.js.map +0 -1
  286. package/dist/server-C-ysqkLv.js +0 -1
  287. package/dist/shared-CA61BuPJ.js.map +0 -1
  288. package/dist/source-DjgypUHW.js +0 -1
  289. package/dist/src-3mP821Pt.js +0 -2
  290. package/dist/src-B3KCLZVk.js +0 -3
  291. package/dist/src-BAyg6CtQ.js +0 -2
  292. package/dist/src-BezgGX4X.js +0 -2
  293. package/dist/src-ChBhk06Q.js +0 -2
  294. package/dist/src-CoqZYspK.js +0 -2
  295. package/dist/src-CzQrAcF8.js.map +0 -1
  296. package/dist/src-D9fSTxlc.js +0 -1
  297. package/dist/src-DIL0jST0.js.map +0 -1
  298. package/dist/src-dhSTRN50.js +0 -2
  299. package/dist/state-BGWGAwdo.js +0 -1
  300. package/dist/state-CeR6oz0k.js +0 -66
  301. package/dist/state-CeR6oz0k.js.map +0 -1
  302. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  303. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  304. package/dist/studio-app/favicon.svg +0 -1
  305. package/dist/studio-app/index.html +0 -15
  306. package/dist/suite-DmuF_ApO.js.map +0 -1
  307. package/dist/suite-mn9g4veR.js +0 -2
  308. package/dist/summary-BFMHMTz2.js.map +0 -1
  309. package/dist/usage-BDo2q_yu.js +0 -1
  310. package/dist/validate-DWjkshl1.js.map +0 -1
  311. package/dist/views-DE2A1jEX.js +0 -2
  312. package/dist/views-Dk8MCQKr.js.map +0 -1
  313. package/dist/workspace-B1eGO46M.js +0 -1
@@ -1,34 +1,33 @@
1
- import { n as isOneOf } from "./object-CLf2-tNq.js";
2
1
  import { n as UsageError } from "./errors-B6BSMs03.js";
3
- import { p as parseDuration } from "./shared-CA61BuPJ.js";
4
- import { p as sandboxId, s as harnessId, u as plannedSandboxes } from "./options-CyAyLjgJ.js";
5
- import { M as workspaceLabel, T as profileSandbox, p as environmentSandbox } from "./environment-B4IuURK1.js";
2
+ import { p as parseDuration } from "./shared-BDUzStad.js";
3
+ import { M as workspaceLabel, T as profileSandbox, p as environmentSandbox } from "./environment-CO0P0soF.js";
6
4
  import { a as fromCwd, o as newRunId } from "./ids-Bp6ut7ZD.js";
5
+ import { G as ciRunner, K as directOnCloud, O as harnessId, P as sandboxId, Y as isDirect, Z as resolveOutput, _ as selectProfile, a as configPathFor, c as findConfig, d as loadConfig, f as malformedCredentials, g as runJudgeModel, h as resolveConcurrency, i as configLabel, j as plannedSandboxes, l as hasAnalysisModel, m as requireConfig, p as missingCredentials, u as hostCapacity, v as gatewayScope, y as resolveModelGateway } from "./config-BrdHA9bJ.js";
6
+ import { c as judgeModelName, p as unusedGatewayKey } from "./judge-yLvjC5A3.js";
7
7
  import { c as registerSecret, s as redactSecrets } from "./redact-CRGdfV2s.js";
8
- import { a as configPathFor, c as findConfig, d as loadConfig, f as malformedCredentials, g as selectProfile, h as resolveConcurrency, i as configLabel, l as hasAnalysisModel, m as requireConfig, p as missingCredentials, u as hostCapacity } from "./config-D7TFvUpL.js";
9
8
  import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
10
- import { W as LABEL, at as imageInfo } from "./src-CIpKL45U.js";
11
- import { o as PACKAGE_VERSION } from "./root-DNuLhaA5.js";
12
- import { C as pluginBuildsLabel } from "./plugins-P-uh54VF.js";
13
- import { s as debugTime } from "./debug-D24NOQH1.js";
9
+ import { G as LABEL, ot as imageInfo } from "./src-8zoX_qrb.js";
10
+ import { o as PACKAGE_VERSION } from "./root-Dlff-zp8.js";
11
+ import { C as pluginBuildsLabel } from "./plugins-Dkp_H3pB.js";
14
12
  import { i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-CrVy5ioL.js";
13
+ import { s as debugTime } from "./debug-D24NOQH1.js";
15
14
  import { t as PrepareError } from "./provider-CmAYTQQs.js";
16
- import { T as runOutcome, g as formatUsage } from "./model-UgHDMhI1.js";
17
- import { c as suiteResult } from "./suite-DmuF_ApO.js";
18
- import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-BXIBUlaK.js";
19
- import { p as render } from "./summary-BFMHMTz2.js";
15
+ import { D as runOutcome, K as triesNote, _ as formatUsage } from "./model-CgnOpss4.js";
16
+ import { u as suiteResult } from "./suite-WgT7m6Vs.js";
17
+ import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-CpnbATtI.js";
18
+ import { p as render } from "./summary-DIGEVr08.js";
20
19
  import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
21
- import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-C_-8y-x3.js";
20
+ import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-CfZpFpAh.js";
22
21
  import { t as budgetMarker } from "./budget-DBUrH4jX.js";
23
- import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-8-nnW8iO.js";
24
- import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-D3HvKeAx.js";
25
- import { n as imageHarness } from "./prepare-Dx7dOnJB.js";
26
- import { i as createProviders, n as attemptImage } from "./providers-CF5UEaP8.js";
27
- import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-D_ePcwyL.js";
22
+ import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-B9SOSnK9.js";
23
+ import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-DJCD_BDn.js";
24
+ import { n as imageHarness } from "./prepare-CIxCGG0M.js";
25
+ import { i as createProviders, n as attemptImage } from "./providers-DFk_6W4K.js";
26
+ import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DzyKSJBa.js";
28
27
  import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
29
28
  import { dirname, join } from "node:path";
30
- import { inspect } from "node:util";
31
29
  import { appendFile, readFile, rm } from "node:fs/promises";
30
+ import { inspect } from "node:util";
32
31
  import { detectAgent } from "std-env";
33
32
  import { Command, Option } from "commander";
34
33
  //#region src/cli-parse.ts
@@ -274,7 +273,8 @@ function progressLine(c, p) {
274
273
  case "check-start": return c.dim(`$ ${oneLine(p.command)}`);
275
274
  case "check-end": {
276
275
  const exit = p.exitCode === 0 ? c.dim("exit 0") : c.red(`exit ${p.exitCode}`);
277
- return `$ ${oneLine(p.command)} ${exit} ${c.dim(formatDuration(p.ms))}`;
276
+ const tries = p.try && p.tries ? triesNote(p.exitCode, p.try, p.tries) : null;
277
+ return `$ ${oneLine(p.command)} ${exit} ${c.dim([formatDuration(p.ms), tries].filter(Boolean).join(" · "))}`;
278
278
  }
279
279
  case "judge-start": return c.dim(`judging: ${oneLine(p.question, 120)}`);
280
280
  case "judge-end": return judgeLine(c, p);
@@ -807,73 +807,6 @@ var PlainRenderer = class {
807
807
  }
808
808
  };
809
809
  //#endregion
810
- //#region src/ui/mode.ts
811
- const set = (env, name) => {
812
- const value = env[name];
813
- return !!value && value !== "false" && value !== "0";
814
- };
815
- const CI_VARS = [
816
- "BUILD_ID",
817
- "BUILD_NUMBER",
818
- "CI",
819
- "CI_APP_ID",
820
- "CI_BUILD_ID",
821
- "CI_BUILD_NUMBER",
822
- "CI_NAME",
823
- "CONTINUOUS_INTEGRATION",
824
- "RUN_ID",
825
- "TEAMCITY_VERSION",
826
- "GITHUB_ACTIONS"
827
- ];
828
- const isCI = (env) => CI_VARS.some((name) => set(env, name));
829
- function pick(name, value, allowed) {
830
- if (value === void 0 || value === "") return void 0;
831
- if (value === "auto" || isOneOf(allowed, value)) return value;
832
- throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
833
- }
834
- function parseHeartbeat(value) {
835
- if (!value) return 6e4;
836
- const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
837
- if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
838
- const unit = {
839
- ms: 1,
840
- s: 1e3,
841
- m: 6e4
842
- }[match[2] ?? "s"] ?? 1e3;
843
- return Number(match[1]) * unit;
844
- }
845
- function resolveOutput(input) {
846
- const { flags, env } = input;
847
- const ci = isCI(env);
848
- const githubActions = set(env, "GITHUB_ACTIONS");
849
- const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
850
- "live",
851
- "plain",
852
- "agent"
853
- ]);
854
- const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
855
- const mode = requested === "live" ? "plain" : requested;
856
- const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
857
- const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
858
- const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
859
- const logPrefix = prefix && prefix !== "auto" ? prefix : githubActions && logOrder === "grouped" ? "none" : "task";
860
- const logs = pick("--output-logs", flags.outputLogs ?? env.REEVAL_OUTPUT_LOGS, [
861
- "full",
862
- "errors-only",
863
- "none"
864
- ]);
865
- return {
866
- requested,
867
- mode,
868
- logOrder,
869
- logPrefix,
870
- outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
871
- color: env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions,
872
- githubActions,
873
- heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
874
- };
875
- }
876
- //#endregion
877
810
  //#region src/ui/index.ts
878
811
  function createRenderer(options) {
879
812
  const out = { write: (text) => options.out.write(redactSecrets(text)) };
@@ -948,7 +881,7 @@ async function run(argv) {
948
881
  return values.help ? EXIT.ok : EXIT.usage;
949
882
  }
950
883
  if (command === "init") {
951
- const { initCommand } = await import("./init-kMkTgTZc.js");
884
+ const { initCommand } = await import("./init-RgCQXZhb.js");
952
885
  return initCommand(values, filters, {
953
886
  out: (s) => console.log(s),
954
887
  input: process.stdin,
@@ -957,7 +890,7 @@ async function run(argv) {
957
890
  });
958
891
  }
959
892
  if (command === "doctor") {
960
- const { doctorCommand } = await import("./doctor-De_QlOTM.js");
893
+ const { doctorCommand } = await import("./doctor-DQ2p0euG.js");
961
894
  return doctorCommand(values, {
962
895
  out: (s) => console.log(s),
963
896
  color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -965,7 +898,7 @@ async function run(argv) {
965
898
  });
966
899
  }
967
900
  if (command === "report") {
968
- const { reportCommand } = await import("./command-CANun0m0.js");
901
+ const { reportCommand } = await import("./command-BXLUfw4h.js");
969
902
  return reportCommand(values, filters, {
970
903
  out: (s) => console.log(s),
971
904
  err: (s) => console.error(dim(s)),
@@ -980,7 +913,7 @@ async function run(argv) {
980
913
  }
981
914
  if (command === "studio") {
982
915
  const config = await loadConfig(configPath, "studio");
983
- const { studioCommand } = await import("./command-Bh3yhN_b.js");
916
+ const { studioCommand } = await import("./command-I1euLLio.js");
984
917
  return studioCommand({
985
918
  config,
986
919
  flags: values,
@@ -991,7 +924,7 @@ async function run(argv) {
991
924
  if (command === "mcp") {
992
925
  if (!values.http) console.log = console.info = console.debug = console.error;
993
926
  const config = await loadConfig(configPath, "mcp");
994
- const { mcpCommand } = await import("./command-D_kEnYcX.js");
927
+ const { mcpCommand } = await import("./command-Bxdy62dW.js");
995
928
  return mcpCommand({
996
929
  config,
997
930
  flags: values,
@@ -1000,7 +933,7 @@ async function run(argv) {
1000
933
  }
1001
934
  if (command in HISTORY_COMMANDS) {
1002
935
  const config = await loadConfig(configPath, command);
1003
- const commands = await import("./commands-BKGzBMgb.js");
936
+ const commands = await import("./commands-p1S4nw_H.js");
1004
937
  const ctx = {
1005
938
  config,
1006
939
  flags: values,
@@ -1013,7 +946,7 @@ async function run(argv) {
1013
946
  case "compare": return commands.compareCommand(ctx, filters);
1014
947
  case "checkout": return commands.checkoutCommand(ctx, filters);
1015
948
  case "query": {
1016
- const { queryCommand } = await import("./query-D_1ARx5q.js");
949
+ const { queryCommand } = await import("./query-Cvbwl804.js");
1017
950
  return queryCommand({
1018
951
  ...ctx,
1019
952
  err: (s) => console.error(dim(s))
@@ -1021,7 +954,7 @@ async function run(argv) {
1021
954
  }
1022
955
  case "analyze": {
1023
956
  loadEnvFiles(configPath);
1024
- const { analyzeCommand } = await import("./command-CBaHukyl.js");
957
+ const { analyzeCommand } = await import("./command-Be6BSE2L.js");
1025
958
  return analyzeCommand({
1026
959
  ...ctx,
1027
960
  err: (s) => console.error(dim(s))
@@ -1033,6 +966,16 @@ async function run(argv) {
1033
966
  if (command !== "run") throw new UsageError(`Unknown command "${command}". Run reeval --help.`);
1034
967
  return runCommand(values, configPath, filters);
1035
968
  }
969
+ /** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
970
+ async function dryRunPrices(planned, judgeModel) {
971
+ const { needsPrices } = await import("./cost-DmnAod9x.js");
972
+ if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
973
+ table: null,
974
+ why: "nothing to price"
975
+ };
976
+ const { loadPrices } = await import("./prices-BcLEXvdY.js");
977
+ return loadPrices();
978
+ }
1036
979
  /** Asks a yes/no question on the terminal; anything but y or yes is no. */
1037
980
  async function confirm(question) {
1038
981
  const { createInterface } = await import("node:readline/promises");
@@ -1060,8 +1003,8 @@ async function cleanCommand(flags, configPath) {
1060
1003
  const where = fromCwd(debugRoot(config.root)) || ".";
1061
1004
  console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
1062
1005
  if (flags.images) {
1063
- const { dockerAvailable, pruneImages } = await import("./src-D9fSTxlc.js");
1064
- const { requireDocker } = await import("./providers-CTBYvMxu.js");
1006
+ const { dockerAvailable, pruneImages } = await import("./src-1jHbFaGW.js");
1007
+ const { requireDocker } = await import("./providers-D1Zxp2he.js");
1065
1008
  const engines = [];
1066
1009
  for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
1067
1010
  if (!engines.length) await requireDocker();
@@ -1082,7 +1025,7 @@ function verbosityOf(flags) {
1082
1025
  * does. Needs a config, as `reeval run` does: there is no default image.
1083
1026
  */
1084
1027
  async function imageBuildCommand(flags, configPath, filters) {
1085
- const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-m784ueGK.js");
1028
+ const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-DzLeFjLx.js");
1086
1029
  const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
1087
1030
  ...flags,
1088
1031
  "dry-run": true
@@ -1142,7 +1085,7 @@ async function runCommand(flags, configPath, filters) {
1142
1085
  if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
1143
1086
  const stepStart = process.env.GITHUB_STEP_SUMMARY;
1144
1087
  if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
1145
- const { engineVersions } = await import("./node-CGZYh4Fj.js");
1088
+ const { engineVersions } = await import("./node-BSq5qHnQ.js");
1146
1089
  const renderer = createRenderer({
1147
1090
  out: setup.json ? setup.dryRun ? { write: () => true } : process.stderr : process.stdout,
1148
1091
  output: setup.output,
@@ -1160,11 +1103,11 @@ async function runCommand(flags, configPath, filters) {
1160
1103
  });
1161
1104
  throw error;
1162
1105
  }
1163
- const { engineRun, plan, sandboxVersions, budgetReached, live, questions } = executed;
1106
+ const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
1164
1107
  invocation.unhandledErrors = engineRun.unhandledErrors;
1165
1108
  const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
1166
1109
  if ((interrupted || engineRun.cancelled) && engines.length) {
1167
- const { removeRunContainers } = await import("./docker-Dac6Zalz.js");
1110
+ const { removeRunContainers } = await import("./docker-DivfZl2u.js");
1168
1111
  await removeRunContainers(ctx.runId, engines);
1169
1112
  }
1170
1113
  if (engineRun.stopped || !plan) {
@@ -1172,10 +1115,15 @@ async function runCommand(flags, configPath, filters) {
1172
1115
  renderer.close();
1173
1116
  return EXIT.ok;
1174
1117
  }
1175
- const stopped = interrupted || engineRun.cancelled;
1176
- const answers = questions && !stopped ? await questions.settle() : void 0;
1177
- const { lockfiles } = await import("./state-BGWGAwdo.js");
1178
- const { writeScratchRun } = await import("./results-D-naai_Q.js");
1118
+ let stopped = cutShort || engineRun.cancelled;
1119
+ if (stopped) questions?.cancel();
1120
+ let answers = stopped ? await questions?.answered() : await questions?.settle();
1121
+ if (!stopped && interrupted && questions) {
1122
+ stopped = true;
1123
+ answers = await questions?.answered();
1124
+ }
1125
+ const { lockfiles } = await import("./state-vznMh2wl.js");
1126
+ const { writeScratchRun } = await import("./results-CMgZEANy.js");
1179
1127
  const run = await writeScratchRun({
1180
1128
  ctx,
1181
1129
  author: ctx.author,
@@ -1196,18 +1144,18 @@ async function runCommand(flags, configPath, filters) {
1196
1144
  previous: ctx.previous,
1197
1145
  previousArtifacts: ctx.previousArtifacts,
1198
1146
  runner: runnerInfo(),
1199
- ...answers?.size ? { answers } : {}
1147
+ ...answers && (answers.evals.size || answers.attempts.size) ? { answers } : {}
1200
1148
  }, () => live?.stop());
1201
1149
  const { record } = run;
1202
1150
  const outcome = runOutcome(record);
1203
1151
  const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
1204
1152
  let insights = run.insights;
1205
1153
  let insightLines = [];
1206
- const { runInsights, wantsInsights } = await import("./run-Bel7Vde4.js");
1154
+ const { runInsights, wantsInsights } = await import("./run-CwIEK16q.js");
1207
1155
  const wanted = !stopped && wantsInsights(ctx.config, !flags["no-insights"]);
1208
1156
  if (wanted && interrupted) insightLines = [`insights skipped (${stopCode === EXIT.terminated ? "SIGTERM" : "Ctrl-C"}): reeval analyze ${record.runId} --insights to write them`];
1209
1157
  else if (wanted) {
1210
- const { scratchDir } = await import("./files-CtjEXWXm.js");
1158
+ const { scratchDir } = await import("./files-C2sjmXhk.js");
1211
1159
  const stop = new AbortController();
1212
1160
  let stopSignal = "SIGINT";
1213
1161
  onInsightsSignal = (signal) => {
@@ -1269,7 +1217,7 @@ async function runCommand(flags, configPath, filters) {
1269
1217
  const { config } = ctx;
1270
1218
  const mode = setup.record;
1271
1219
  if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
1272
- const { record: recordCmd } = await import("./commands-BKGzBMgb.js");
1220
+ const { record: recordCmd } = await import("./commands-p1S4nw_H.js");
1273
1221
  const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
1274
1222
  const line = await recordCmd(config, record.runId, {
1275
1223
  ...flags,
@@ -1279,7 +1227,7 @@ async function runCommand(flags, configPath, filters) {
1279
1227
  else console.log(line);
1280
1228
  }
1281
1229
  if (setup.json) {
1282
- const { suiteResult } = await import("./suite-mn9g4veR.js");
1230
+ const { suiteResult } = await import("./suite-BsWGHzlG.js");
1283
1231
  process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
1284
1232
  }
1285
1233
  return code;
@@ -1312,7 +1260,7 @@ async function prepareRun(flags, configPath) {
1312
1260
  const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
1313
1261
  const container = sandboxes.find((s) => s.kind === "docker");
1314
1262
  if (!container) return hostCapacity();
1315
- const { dockerCapacity } = await import("./src-D9fSTxlc.js");
1263
+ const { dockerCapacity } = await import("./src-1jHbFaGW.js");
1316
1264
  return await dockerCapacity(container.engine) ?? hostCapacity();
1317
1265
  });
1318
1266
  return {
@@ -1341,11 +1289,11 @@ async function prepareRun(flags, configPath) {
1341
1289
  async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
1342
1290
  const startedAt = /* @__PURE__ */ new Date();
1343
1291
  const git = await gitInfo(config.root, [config.history.dir]);
1344
- const { worktreePatch } = await import("./state-BGWGAwdo.js");
1292
+ const { worktreePatch } = await import("./state-vznMh2wl.js");
1345
1293
  const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
1346
1294
  const who = await author(config.root);
1347
- const { stagingDir, sweepStaging } = await import("./results-D-naai_Q.js");
1348
- const { scratchDir } = await import("./files-CtjEXWXm.js");
1295
+ const { stagingDir, sweepStaging } = await import("./results-CMgZEANy.js");
1296
+ const { scratchDir } = await import("./files-C2sjmXhk.js");
1349
1297
  const scratch = scratchDir(config.root);
1350
1298
  if (resume !== void 0) {
1351
1299
  const target = await resolveResume(config.root, resume || void 0);
@@ -1356,7 +1304,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
1356
1304
  artifacts: target.previousArtifacts
1357
1305
  } : await loadPrevious(stagingDir(config.root, target.runId));
1358
1306
  if (!dryRun) try {
1359
- const { removeRunContainers } = await import("./docker-Dac6Zalz.js");
1307
+ const { removeRunContainers } = await import("./docker-DivfZl2u.js");
1360
1308
  await removeRunContainers(target.runId, ["docker", "podman"]);
1361
1309
  } catch {}
1362
1310
  return {
@@ -1427,8 +1375,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1427
1375
  configFile,
1428
1376
  completed: []
1429
1377
  }));
1430
- const { runEngine } = await import("./node-CGZYh4Fj.js");
1431
- const { buildPlan, checkAttemptBudget, planDocument, planInsights, planLine, planTable } = await import("./plan-CZYgXnZh.js");
1378
+ const { runEngine } = await import("./node-BSq5qHnQ.js");
1379
+ const { buildPlan, checkAttemptBudget, planDocument, planInsights, planLine, planTable } = await import("./plan-BIwUzlB8.js");
1432
1380
  let plan;
1433
1381
  let sandboxVersions = { images: [] };
1434
1382
  let preflightStop;
@@ -1454,7 +1402,9 @@ async function executeRun(ctx, setup, filters, renderer) {
1454
1402
  } : p);
1455
1403
  toRun = skipped;
1456
1404
  for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
1457
- const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-B8H3gL69.js") : void 0;
1405
+ const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-Ch8kX87u.js") : void 0;
1406
+ const judge = runJudgeModel(config, profile);
1407
+ const judgeModel = judge === void 0 ? null : judgeModelName(judge);
1458
1408
  plan = await buildPlan({
1459
1409
  planned: skipped,
1460
1410
  profile,
@@ -1462,21 +1412,26 @@ async function executeRun(ctx, setup, filters, renderer) {
1462
1412
  concurrency,
1463
1413
  root: config.root,
1464
1414
  historyDir: config.history.dir,
1465
- questionModel: models?.questionModelLabel(config) ?? null,
1466
- insights: await planInsights(config, setup.insights)
1415
+ questionModel: models?.questionModelLabel(config, judge) ?? null,
1416
+ insights: await planInsights(config, setup.insights),
1417
+ judgeModel,
1418
+ ...setup.dryRun ? { prices: await dryRunPrices(skipped, judgeModel) } : {}
1467
1419
  });
1468
1420
  plan.runId = runId;
1469
1421
  if (plan.insights) {
1470
- const { analysisKeyProblem } = await import("./model-B8H3gL69.js");
1422
+ const { analysisKeyProblem } = await import("./model-Ch8kX87u.js");
1471
1423
  const problem = analysisKeyProblem(config);
1472
1424
  if (problem) plan.warnings.push(`insights will be skipped: ${problem}`);
1473
1425
  }
1426
+ const cloud = ciRunner(process.env);
1427
+ const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
1428
+ if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
1474
1429
  if (completed.size) plan.resume = {
1475
1430
  runId,
1476
1431
  completed: completed.size
1477
1432
  };
1478
1433
  run.completed = [...completed];
1479
- const { imageStages, imageTargets } = await import("./images-m784ueGK.js");
1434
+ const { imageStages, imageTargets } = await import("./images-DzLeFjLx.js");
1480
1435
  const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
1481
1436
  id: p.id,
1482
1437
  harness: p.attempt.harness,
@@ -1498,8 +1453,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1498
1453
  if (setup.dryRun) {
1499
1454
  const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
1500
1455
  lines.push(...planTable(plan, todos));
1501
- const { imageStatusLines } = await import("./images-m784ueGK.js");
1502
- const { dockerAvailable, engineSpec } = await import("./src-D9fSTxlc.js");
1456
+ const { imageStatusLines } = await import("./images-DzLeFjLx.js");
1457
+ const { dockerAvailable, engineSpec } = await import("./src-1jHbFaGW.js");
1503
1458
  const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
1504
1459
  const down = [];
1505
1460
  for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
@@ -1508,9 +1463,12 @@ async function executeRun(ctx, setup, filters, renderer) {
1508
1463
  const names = down.map((e) => engineSpec(e).name).join(" and ");
1509
1464
  lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
1510
1465
  }
1466
+ const { costLine } = await import("./cost-DmnAod9x.js");
1467
+ const cost = plan.cost && costLine(plan.cost);
1468
+ if (cost) lines.push(cost);
1511
1469
  }
1512
1470
  if (setup.dryRun && plan.insights) {
1513
- const { formatTokens } = await import("./usage-BDo2q_yu.js");
1471
+ const { formatTokens } = await import("./usage-DF3_p5pE.js");
1514
1472
  lines.push(`insights ${plan.insights.model} · up to ${formatTokens(plan.insights.maxTokens)} tokens`);
1515
1473
  }
1516
1474
  lines.push(...plan.warnings.map((w) => `warning: ${w}`));
@@ -1525,15 +1483,23 @@ async function executeRun(ctx, setup, filters, renderer) {
1525
1483
  checkAttemptBudget(plan, setup.force, config.file);
1526
1484
  const questionModels = /* @__PURE__ */ new Set();
1527
1485
  const asked = [];
1486
+ const scope = gatewayScope(config, profile);
1528
1487
  for (const row of plan.rows) {
1529
1488
  if (row.plan.skip || !models) continue;
1530
- const questions = (row.plan.questions ?? []).map((q) => ({
1531
- question: q.question,
1532
- model: models.questionModel(config, q, row.id),
1533
- ...q.across ? { across: q.across } : {},
1534
- ...q.state ? { state: q.state } : {},
1535
- ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
1536
- }));
1489
+ const questions = (row.plan.questions ?? []).map((q) => {
1490
+ const model = models.questionModel(config, q, row.id, judge);
1491
+ const gateway = resolveModelGateway(model, void 0, scope, "the question model");
1492
+ return {
1493
+ n: q.n,
1494
+ question: q.question,
1495
+ model,
1496
+ ...gateway ? { gateway } : {},
1497
+ ...q.across !== void 0 ? { across: q.across } : {},
1498
+ ...q.choices ? { choices: q.choices } : {},
1499
+ ...q.state ? { state: q.state } : {},
1500
+ ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
1501
+ };
1502
+ });
1537
1503
  for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
1538
1504
  if (questions.length) asked.push({
1539
1505
  row,
@@ -1545,7 +1511,10 @@ async function executeRun(ctx, setup, filters, renderer) {
1545
1511
  profile,
1546
1512
  questionModels: [...questionModels]
1547
1513
  });
1548
- if (missing.length) throw new UsageError(`Missing credentials: ${missing.join("; ")}. Set them in the environment or .env.local.`);
1514
+ if (missing.length) {
1515
+ const hint = missing.some((m) => m.includes("(for direct())")) ? unusedGatewayKey(process.env) : "";
1516
+ throw new UsageError(`Missing credentials: ${missing.join("; ")}. Set them in the environment or .env.local.${hint}`);
1517
+ }
1549
1518
  const malformed = malformedCredentials(config, process.env, {
1550
1519
  harnesses: plan.harnessDescriptors,
1551
1520
  profile,
@@ -1553,7 +1522,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1553
1522
  });
1554
1523
  if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
1555
1524
  if (asked.length) {
1556
- const { questionRunner } = await import("./questions-KENpWnfV.js");
1525
+ const { questionRunner } = await import("./questions-bAudIc04.js");
1526
+ const warned = /* @__PURE__ */ new Set();
1557
1527
  questions = questionRunner({
1558
1528
  evals: asked,
1559
1529
  stagingDir,
@@ -1565,12 +1535,31 @@ async function executeRun(ctx, setup, filters, renderer) {
1565
1535
  type: "notice",
1566
1536
  at: Date.now(),
1567
1537
  level: "warn",
1568
- text: `compare question "${clip(answer.question, 80)}" on ${id} failed: ${answer.error}. Outcomes are unchanged.`
1538
+ text: `question "${clip(answer.question, 80)}" on ${id} failed: ${answer.error}. Outcomes are unchanged.`
1539
+ });
1540
+ },
1541
+ onAttemptAnswer: (id, attempt, answer) => {
1542
+ const once = `${id}\0${answer.question}`;
1543
+ if (answer.error === void 0 || warned.has(once)) return;
1544
+ warned.add(once);
1545
+ renderer.onEvent({
1546
+ type: "notice",
1547
+ at: Date.now(),
1548
+ level: "warn",
1549
+ text: `question "${clip(answer.question, 80)}" on ${attempt} failed: ${answer.error}. Outcomes are unchanged; later failures of it on ${id} aren't shown.`
1550
+ });
1551
+ },
1552
+ onUnshown: (id, error) => {
1553
+ renderer.onEvent({
1554
+ type: "notice",
1555
+ at: Date.now(),
1556
+ level: "warn",
1557
+ text: `${error} on ${id}, so each attempt's answer is that error. Outcomes are unchanged.`
1569
1558
  });
1570
1559
  }
1571
1560
  });
1572
1561
  }
1573
- const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-B1eGO46M.js");
1562
+ const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-QZCacP8A.js");
1574
1563
  const live = skipped.filter((p) => !p.attempt.plan.skip);
1575
1564
  const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
1576
1565
  workspace: a.workspace,
@@ -1661,6 +1650,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1661
1650
  budgetTimer.unref();
1662
1651
  };
1663
1652
  let engineRun;
1653
+ let cutShort = false;
1664
1654
  try {
1665
1655
  engineRun = await runEngine({
1666
1656
  run,
@@ -1720,15 +1710,16 @@ async function executeRun(ctx, setup, filters, renderer) {
1720
1710
  process.on("SIGTERM", onSignal);
1721
1711
  }
1722
1712
  });
1713
+ cutShort = interrupted;
1723
1714
  live?.phase("finishing");
1724
1715
  } catch (error) {
1725
1716
  live?.remove();
1726
1717
  throw error;
1727
1718
  } finally {
1728
1719
  clearTimeout(budgetTimer);
1729
- const { removePrepared } = await import("./preflight-BQlPTHVu.js");
1720
+ const { removePrepared } = await import("./preflight-D9y-HvZQ.js");
1730
1721
  removePrepared(config.root, runId);
1731
- const { removePreparedWorkspaces } = await import("./workspace-B1eGO46M.js");
1722
+ const { removePreparedWorkspaces } = await import("./workspace-QZCacP8A.js");
1732
1723
  removePreparedWorkspaces(config.root, runId);
1733
1724
  }
1734
1725
  if (preflightStop) throw preflightStop;
@@ -1740,6 +1731,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1740
1731
  plan,
1741
1732
  sandboxVersions,
1742
1733
  budgetReached: () => reached,
1734
+ cutShort,
1743
1735
  live,
1744
1736
  questions
1745
1737
  };
@@ -1810,4 +1802,4 @@ run(process.argv.slice(2)).finally(() => {
1810
1802
  //#endregion
1811
1803
  export {};
1812
1804
 
1813
- //# sourceMappingURL=cli-BCE1xlm4.js.map
1805
+ //# sourceMappingURL=cli-C_4gaJ50.js.map