reeval 0.14.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (364) hide show
  1. package/README.md +15 -15
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
  5. package/dist/adapter-CRKYPkRt.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
  7. package/dist/agent-BgH4-kFc.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
  9. package/dist/app-files-BMVBr079.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
  11. package/dist/assemble-BVbI6NW-.js.map +1 -0
  12. package/dist/assemble-SiHa4__F.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
  16. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
  18. package/dist/capture-iQ6kYJV0.js.map +1 -0
  19. package/dist/capture-px7Ey95_.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
  21. package/dist/cli-1YQU-66F.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
  28. package/dist/command-8_Cg5v6b.js.map +1 -0
  29. package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
  30. package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
  31. package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
  32. package/dist/command-C-HeGlap.js.map +1 -0
  33. package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
  34. package/dist/command-Cr1zjRvl.js.map +1 -0
  35. package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
  36. package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
  37. package/dist/commands-BvTNZ2-b.js.map +1 -0
  38. package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
  39. package/dist/config-CneBZB84.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-BnXyamuj.js +2 -0
  46. package/dist/db-Cnh3jIv_.js +1530 -0
  47. package/dist/db-Cnh3jIv_.js.map +1 -0
  48. package/dist/debug-CAx3qcaP.js +1 -0
  49. package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
  50. package/dist/debug-CgG3T071.js.map +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
  52. package/dist/docker-BihzJwbV.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
  54. package/dist/docker-Cqq03H72.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
  57. package/dist/doctor-Dl4vMeqt.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
  61. package/dist/environment-Cf5HMjib.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
  63. package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
  64. package/dist/eval-source-CAE4y2Sr.js +146 -0
  65. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
  67. package/dist/evaluate-CUtgyr43.js.map +1 -0
  68. package/dist/evaluate-D35T51AC.d.ts +45 -0
  69. package/dist/files-BEnUJW1e.js +1 -0
  70. package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
  71. package/dist/files-BRViE7m-.js.map +1 -0
  72. package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
  73. package/dist/files-BSqA2sZE.js.map +1 -0
  74. package/dist/files-C3tQtD26.js +1 -0
  75. package/dist/files-CNiJ_fZf.js +37 -0
  76. package/dist/files-CNiJ_fZf.js.map +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/files-Dts7EY3H.js +1 -0
  80. package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
  81. package/dist/gate-BC6SdQZs.js.map +1 -0
  82. package/dist/gateway.d.ts +22 -9
  83. package/dist/gateway.js +30 -8
  84. package/dist/gateway.js.map +1 -1
  85. package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
  86. package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
  87. package/dist/harness-3NVyTSGK.js +392 -0
  88. package/dist/harness-3NVyTSGK.js.map +1 -0
  89. package/dist/harness.d.ts +12 -8
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
  92. package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
  93. package/dist/index.d.ts +99 -11
  94. package/dist/index.js +5 -5
  95. package/dist/init-okoALIah.js +1 -0
  96. package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
  97. package/dist/init-sFBgODhu.js.map +1 -0
  98. package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
  99. package/dist/invocations-C9ejlOYl.js.map +1 -0
  100. package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
  101. package/dist/judge-Bg4v4e_K.js.map +1 -0
  102. package/dist/judge.d.ts +5 -7
  103. package/dist/judge.js +4 -8
  104. package/dist/judge.js.map +1 -1
  105. package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
  106. package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
  107. package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
  108. package/dist/matchers-BKPuaXlj.js.map +1 -0
  109. package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
  110. package/dist/model-2rBZrdZA.js.map +1 -0
  111. package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
  112. package/dist/model-3MJtpho8.js.map +1 -0
  113. package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
  114. package/dist/model-Bz7-j71_.js.map +1 -0
  115. package/dist/model-DkPwypN4.js +1 -0
  116. package/dist/model-bREXVXwT.js +1 -0
  117. package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
  118. package/dist/node-fP-H2uxn.js.map +1 -0
  119. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  120. package/dist/object-BkXGPKWY.js.map +1 -0
  121. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  122. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  123. package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
  124. package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
  125. package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
  126. package/dist/patch-6T9XAipm.js.map +1 -0
  127. package/dist/patch-CquGAGX5.js +76 -0
  128. package/dist/patch-CquGAGX5.js.map +1 -0
  129. package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
  130. package/dist/plan-Dw2BjafJ.js.map +1 -0
  131. package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
  132. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  133. package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
  134. package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
  135. package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
  136. package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
  137. package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
  138. package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
  139. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  140. package/dist/prices-QaS-kjSQ.js.map +1 -0
  141. package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
  142. package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
  143. package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
  144. package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
  145. package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
  146. package/dist/query-Bt6vLJ3m.js +1 -0
  147. package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
  148. package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
  149. package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
  150. package/dist/questions-CkAwePPR.js.map +1 -0
  151. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  152. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  153. package/dist/redact-DVIaahLv.js +658 -0
  154. package/dist/redact-DVIaahLv.js.map +1 -0
  155. package/dist/remarks-Bp6AwPgb.js +1 -0
  156. package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
  157. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  158. package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
  159. package/dist/remarks-IFFe_oty.js.map +1 -0
  160. package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
  161. package/dist/remarks-eBcrIZRu.js.map +1 -0
  162. package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
  163. package/dist/remote-DfJYiP-R.js.map +1 -0
  164. package/dist/resolve-CT4XZIfV.js +1 -0
  165. package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
  166. package/dist/resolve-D_cLa4tW.js.map +1 -0
  167. package/dist/results-DrKJuIQF.js +63 -0
  168. package/dist/results-DrKJuIQF.js.map +1 -0
  169. package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
  170. package/dist/resume-BmIM52MJ.js.map +1 -0
  171. package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
  172. package/dist/runtime-CZVpkr04.js.map +1 -0
  173. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
  174. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  175. package/dist/sandbox.d.ts +1 -1
  176. package/dist/sandbox.js +1 -1
  177. package/dist/server-CKJOnIB0.js +1 -0
  178. package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
  179. package/dist/server-DS8BGhGU.js.map +1 -0
  180. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  181. package/dist/shared-B_lvGqWz.js.map +1 -0
  182. package/dist/source-CgUqpumI.js +1 -0
  183. package/dist/source-DcXv4de-.js +59 -0
  184. package/dist/source-DcXv4de-.js.map +1 -0
  185. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  186. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  187. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  188. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  189. package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
  190. package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
  191. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  192. package/dist/src-BO8AYqJ5.js.map +1 -0
  193. package/dist/src-BTqJbvj6.js +2 -0
  194. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  195. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BkiComGh.js +2 -0
  197. package/dist/src-BvgeZ4tS.js +2 -0
  198. package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
  199. package/dist/src-C0ySiej5.js.map +1 -0
  200. package/dist/src-CgJrz5p0.js +3 -0
  201. package/dist/src-Cp-2Hdib.js +2 -0
  202. package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
  203. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  204. package/dist/src-DSZyg0Uq.js.map +1 -0
  205. package/dist/src-Dk2Na0b-.js +2 -0
  206. package/dist/src-OESRtvOi.js +2 -0
  207. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  208. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  209. package/dist/src-h5uDlk3m.js +1 -0
  210. package/dist/state-C97udoh3.js +1 -0
  211. package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
  212. package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
  213. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  214. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  215. package/dist/suite-ByT1lsdD.js +2 -0
  216. package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
  217. package/dist/suite-DieLXvCT.js.map +1 -0
  218. package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
  219. package/dist/summary-CSzyBVsF.js.map +1 -0
  220. package/dist/tarball-CHcahyNK.js +258 -0
  221. package/dist/tarball-CHcahyNK.js.map +1 -0
  222. package/dist/testing.d.ts +426 -143
  223. package/dist/testing.js +12 -10
  224. package/dist/testing.js.map +1 -1
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  228. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  229. package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
  230. package/dist/transcript-BJrKfgRW.js.map +1 -0
  231. package/dist/types-C5vI3g1C.js.map +1 -1
  232. package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +548 -121
  238. package/dist/view/index.js +9 -9
  239. package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
  240. package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
  241. package/dist/views-Ca-rTixp.js +2 -0
  242. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
  243. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  244. package/dist/workspace-8uHtBSve.js +1 -0
  245. package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
  246. package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +5 -1
  249. package/skills/eval-rubric/SKILL.md +46 -104
  250. package/dist/adapter-BloGVZ_a.js.map +0 -1
  251. package/dist/agent-m-_t-IuW.js.map +0 -1
  252. package/dist/app-files-C-jwhRDt.js.map +0 -1
  253. package/dist/assemble-BDEpelpi.js.map +0 -1
  254. package/dist/assemble-C5bfncHU.js +0 -1
  255. package/dist/capture-CVRwRTnb.js.map +0 -1
  256. package/dist/capture-CherbjzB.js +0 -1
  257. package/dist/cli-C_4gaJ50.js.map +0 -1
  258. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  259. package/dist/client-D5hNjpEx.js.map +0 -1
  260. package/dist/command-BWAuwQNk.js +0 -91
  261. package/dist/command-BWAuwQNk.js.map +0 -1
  262. package/dist/command-BXLUfw4h.js.map +0 -1
  263. package/dist/command-Be6BSE2L.js +0 -335
  264. package/dist/command-Be6BSE2L.js.map +0 -1
  265. package/dist/command-Bxdy62dW.js.map +0 -1
  266. package/dist/commands-jzigbNFb.js.map +0 -1
  267. package/dist/config-BrdHA9bJ.js.map +0 -1
  268. package/dist/cost-D4elx933.js.map +0 -1
  269. package/dist/cost-DmnAod9x.js +0 -1
  270. package/dist/db-CAfksinX.js +0 -2
  271. package/dist/db-DO0p7xgP.js +0 -619
  272. package/dist/db-DO0p7xgP.js.map +0 -1
  273. package/dist/debug-D24NOQH1.js.map +0 -1
  274. package/dist/debug-D7bEv_UV.js +0 -1
  275. package/dist/docker-BnRlTpZA.js.map +0 -1
  276. package/dist/docker-D01slDF3.js.map +0 -1
  277. package/dist/doctor-DQ2p0euG.js.map +0 -1
  278. package/dist/env-BQqj93gu.js.map +0 -1
  279. package/dist/environment-CO0P0soF.js.map +0 -1
  280. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  281. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  282. package/dist/facts-DzOUTreB.js +0 -906
  283. package/dist/facts-DzOUTreB.js.map +0 -1
  284. package/dist/files-B-CJeeEh.js.map +0 -1
  285. package/dist/files-BAt0i00X.js +0 -1
  286. package/dist/files-Bp5j5BqI.js +0 -1
  287. package/dist/files-C2sjmXhk.js +0 -1
  288. package/dist/files-C_gQNVB-.js +0 -65
  289. package/dist/files-C_gQNVB-.js.map +0 -1
  290. package/dist/files-Ca0d6Uq3.js.map +0 -1
  291. package/dist/files-kUCrswI6.js.map +0 -1
  292. package/dist/generate-DZ7vp69F.js +0 -1647
  293. package/dist/generate-DZ7vp69F.js.map +0 -1
  294. package/dist/harness-COVXSwJ6.js +0 -354
  295. package/dist/harness-COVXSwJ6.js.map +0 -1
  296. package/dist/header-BLPP6aCU.js.map +0 -1
  297. package/dist/init-RgCQXZhb.js +0 -1
  298. package/dist/init-SRalM5xF.js.map +0 -1
  299. package/dist/insights-TyAvz9aV.js +0 -68
  300. package/dist/insights-TyAvz9aV.js.map +0 -1
  301. package/dist/invocations-DzyKSJBa.js.map +0 -1
  302. package/dist/judge-yLvjC5A3.js.map +0 -1
  303. package/dist/match-Bz0nSdzN.js.map +0 -1
  304. package/dist/matchers-D4eUqx4j.js.map +0 -1
  305. package/dist/model-1ZLr9Dcv.js.map +0 -1
  306. package/dist/model-B45lDUV1.js.map +0 -1
  307. package/dist/model-BTZq8HIZ.js +0 -1
  308. package/dist/model-CgnOpss4.js.map +0 -1
  309. package/dist/model-Ch8kX87u.js +0 -1
  310. package/dist/node-BSq5qHnQ.js.map +0 -1
  311. package/dist/object-CLf2-tNq.js.map +0 -1
  312. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  313. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  314. package/dist/pairs-CHfZNRT6.js.map +0 -1
  315. package/dist/patch-DWf-I_Q_.js.map +0 -1
  316. package/dist/plan-BIwUzlB8.js.map +0 -1
  317. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  318. package/dist/prices-BcLEXvdY.js.map +0 -1
  319. package/dist/query-Cvbwl804.js +0 -1
  320. package/dist/questions-bAudIc04.js.map +0 -1
  321. package/dist/redact-CRGdfV2s.js +0 -240
  322. package/dist/redact-CRGdfV2s.js.map +0 -1
  323. package/dist/remarks-CXwSnSjR.js +0 -278
  324. package/dist/remarks-CXwSnSjR.js.map +0 -1
  325. package/dist/remarks-CyAu2DiB.js +0 -1
  326. package/dist/remarks-mujJv1Ey.js.map +0 -1
  327. package/dist/remote-CKcEKja4.js.map +0 -1
  328. package/dist/resolve-B5-uqmJ6.js +0 -1
  329. package/dist/resolve-CBoKzQMP.js.map +0 -1
  330. package/dist/results-CMgZEANy.js +0 -78
  331. package/dist/results-CMgZEANy.js.map +0 -1
  332. package/dist/resume-B9SOSnK9.js.map +0 -1
  333. package/dist/rules-VhftPOOT.js.map +0 -1
  334. package/dist/run-CwIEK16q.js +0 -88
  335. package/dist/run-CwIEK16q.js.map +0 -1
  336. package/dist/runtime-BgQRyCLl.js.map +0 -1
  337. package/dist/server-CHRSgKRp.js +0 -1
  338. package/dist/server-TGW4WyAz.js.map +0 -1
  339. package/dist/shared-BDUzStad.js.map +0 -1
  340. package/dist/source-B1WvBIQt.js +0 -40
  341. package/dist/source-B1WvBIQt.js.map +0 -1
  342. package/dist/source-DAzztZwU.js +0 -1
  343. package/dist/src-1jHbFaGW.js +0 -1
  344. package/dist/src-8zoX_qrb.js.map +0 -1
  345. package/dist/src-C_9wgowC.js +0 -2
  346. package/dist/src-ChhtI6BP.js.map +0 -1
  347. package/dist/src-D005Cbqz.js +0 -2
  348. package/dist/src-DIfRbp5a.js +0 -3
  349. package/dist/src-Dg8121Xy.js +0 -2
  350. package/dist/src-ElBqEN-S.js +0 -2
  351. package/dist/src-NSdQ9zI6.js.map +0 -1
  352. package/dist/src-YHGaUrGE.js +0 -2
  353. package/dist/src-mJVxw1Jm.js +0 -2
  354. package/dist/state-vznMh2wl.js +0 -1
  355. package/dist/suite-BsWGHzlG.js +0 -2
  356. package/dist/suite-WgT7m6Vs.js.map +0 -1
  357. package/dist/summary-DIGEVr08.js.map +0 -1
  358. package/dist/text-CrVy5ioL.js.map +0 -1
  359. package/dist/transcript-BTxJpW4R.js.map +0 -1
  360. package/dist/usage-DF3_p5pE.js +0 -1
  361. package/dist/validate-BhwqrVsm.js.map +0 -1
  362. package/dist/views-BW_C5eVH.js +0 -2
  363. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  364. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,67 +1,22 @@
1
- import { l as sha256 } from "./ids-Bp6ut7ZD.js";
2
- import { a as oneLine, i as formatDuration, n as clip, r as fenced } from "./text-CrVy5ioL.js";
1
+ import { a as plural, i as formatDuration, n as clip, r as fenced } from "./text-DTigcA64.js";
3
2
  import { n as formatTokens } from "./usage-BNY1ABrU.js";
4
- import { C as isUngatedMiss, I as checkLabels, L as checkTries, S as isNotGatedPass, V as commandLine, W as labelledJudgements, dt as formatCount, k as runtimeLabel, pt as formatSeconds, u as evalMarkOutcome, v as harnessLabel, w as languageLabel } from "./model-CgnOpss4.js";
5
- import { c as isStudy, r as countedOf, s as isNotGatedResult, t as cancelledOf, u as suiteResult } from "./suite-WgT7m6Vs.js";
6
- import { a as fromModel } from "./insights-TyAvz9aV.js";
7
- import { o as warnings } from "./state-B1t143wG.js";
3
+ import { B as commandLine, C as isUngatedMiss, F as checkLabels, I as checkTries, S as isNotGatedPass, U as labeledJudgments, ft as formatSeconds, k as runtimeLabel, u as evalMarkOutcome, ut as formatCount, v as harnessLabel, w as languageLabel } from "./model-Bz7-j71_.js";
4
+ import { c as isStudy, r as countedOf, s as isNotGatedResult, t as canceledOf, u as suiteResult } from "./suite-DieLXvCT.js";
5
+ import { o as warnings } from "./state-C_kdhp5v.js";
8
6
  import { t as colors } from "./colors-QV9dFVhN.js";
9
- import { r as remarkLines } from "./remarks-mujJv1Ey.js";
10
- //#region src/insights/hash.ts
11
- /** JSON with every object's keys sorted. */
12
- function canonicalJson(value) {
13
- return JSON.stringify(value, (_key, v) => v && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
14
- }
15
- const factsHash = (suite) => `sha256:${sha256(canonicalJson(suite))}`;
16
- //#endregion
7
+ import { r as remarkLines } from "./remarks-eBcrIZRu.js";
17
8
  //#region src/ui/layout.ts
18
- /** The block for an eval: a pure function of its job and its shape. */
19
- function layout(job, across, isFamily, isCompare) {
9
+ /** The block for an eval: a pure function of its shape. */
10
+ function layout(across, isFamily, isCompare) {
20
11
  if (isCompare) return {
21
12
  kind: "regressions",
22
13
  family: false
23
14
  };
24
15
  return {
25
- kind: job === "choose" && across === "none" ? "choose" : across === "change" ? "measure" : across === "harness" ? "harnesses" : across === "both" ? "both" : "results",
16
+ kind: across === "change" ? "measure" : across === "harness" ? "harnesses" : across === "both" ? "both" : "results",
26
17
  family: isFamily
27
18
  };
28
19
  }
29
- const moved = (c) => c.effect === "better" || c.effect === "worse";
30
- /** A cell the agent failed, or that errored; not one whose only fails the interrupt cancelled. */
31
- const failedCell = (c) => c.outcome === "fail" && c.failed > cancelledOf(c) || c.outcome === "infra_error";
32
- /**
33
- * The insight slots an eval's layout opens: which rows take a sentence (a failing case; an arm
34
- * that failed or moved; a failing harness), whether it has findings (a choose or finish eval that
35
- * measures a change), and whether it has named tools (a choose eval). An `other` eval takes only
36
- * its answer, unless it compares harnesses, which takes a sentence on any job. What the run's
37
- * interrupt cancelled opens nothing: the agent didn't fail it, and its effects aren't measured.
38
- */
39
- function slotsOf(job, e) {
40
- const known = job !== "other";
41
- const measured = e.outcome !== "cancelled";
42
- const harnesses = e.across === "harness" || e.across === "both";
43
- const rows = [];
44
- if (known || harnesses) {
45
- if (e.cases) {
46
- for (const k of e.cases) if (k.outcome !== "pass" && k.outcome !== "cancelled") rows.push(k.path);
47
- for (const p of e.pooled ?? []) {
48
- const pooledMoved = measured && (p.effect === "better" || p.effect === "worse");
49
- if (p.outcome === "fail" || pooledMoved) rows.push(p.path);
50
- }
51
- } else if (e.across === "harness") {
52
- for (const c of e.cells ?? []) if (failedCell(c)) rows.push(c.path);
53
- } else if (e.across === "change" || e.across === "both") {
54
- for (const c of e.cells ?? []) if (failedCell(c) || measured && moved(c)) rows.push(c.path);
55
- }
56
- }
57
- return {
58
- rows,
59
- findings: known && e.across === "change" && measured,
60
- namedTools: job === "choose"
61
- };
62
- }
63
- /** The last line of a run's output while no analysis model writes its answers. */
64
- const NO_MODEL_HINT = "Set analysis.model in reeval.config.ts for answers to what each eval asks.";
65
20
  /** The narrowest a row's label column is, so short labels line up under the name. */
66
21
  const LABEL = 17;
67
22
  const ASSERTION_MAX = 160;
@@ -72,7 +27,7 @@ const VERDICT = {
72
27
  pass: "pass",
73
28
  fail: "fail",
74
29
  infra_error: "infra error",
75
- cancelled: "cancelled",
30
+ canceled: "canceled",
76
31
  skipped: "skipped"
77
32
  };
78
33
  const EFFECT = {
@@ -95,9 +50,12 @@ function table(items, width = LABEL) {
95
50
  const rows = items.filter((i) => "cells" in i);
96
51
  const first = Math.max(width, ...rows.filter((r) => r.cells.length > 1).map((r) => r.indent + textOf(r.cells[0] ?? "").length));
97
52
  const widths = [];
98
- for (const r of rows) r.cells.forEach((cell, i) => {
99
- if (i > 0) widths[i] = Math.max(widths[i] ?? 0, textOf(cell).length);
100
- });
53
+ for (const r of rows) {
54
+ if (r.loose) continue;
55
+ r.cells.forEach((cell, i) => {
56
+ if (i > 0) widths[i] = Math.max(widths[i] ?? 0, textOf(cell).length);
57
+ });
58
+ }
101
59
  return items.map((item) => {
102
60
  const pad = " ".repeat(item.indent);
103
61
  if ("free" in item) return `${pad}${show(item.free)}`;
@@ -108,25 +66,28 @@ function table(items, width = LABEL) {
108
66
  }).join(" ")}`.trimEnd();
109
67
  });
110
68
  }
111
- const paintOutcome = (c, o) => o === "pass" ? c.green : o === "fail" ? c.red : o === "infra_error" || o === "cancelled" ? c.yellow : c.dim;
69
+ const paintOutcome = (c, o) => o === "pass" ? c.green : o === "fail" ? c.red : o === "infra_error" || o === "canceled" ? c.yellow : c.dim;
112
70
  function verdict(c, e) {
113
71
  if (e.todo) return t("todo", c.dim);
114
72
  if (e.outcome === "pass" && isStudy(e.cells ?? [])) return t("not gated", c.dim);
115
73
  return t(VERDICT[e.outcome], paintOutcome(c, e.outcome));
116
74
  }
117
- const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
118
75
  /**
119
76
  * `3 of 4`, with the attempts that didn't count named: `2 of 3 · 1 infra error · 1 skipped`. A
120
- * cancelled attempt is recorded as a fail, but the agent didn't fail it, so it's named apart.
77
+ * canceled attempt is recorded as a fail, but the agent didn't fail it, so it's named apart, and
78
+ * a best-of cell names the attempts its majority made unneeded: `0 of 3 · 2 not needed (best of 5)`.
121
79
  */
122
80
  function cellCount(cell) {
123
- const cancelled = cancelledOf(cell);
81
+ const canceled = canceledOf(cell);
124
82
  const counted = countedOf(cell);
83
+ const notNeeded = cell.notNeeded ?? 0;
84
+ const skipped = cell.skipped - notNeeded;
125
85
  const parts = [
126
86
  counted > 0 ? `${cell.passed} of ${counted}` : "",
127
87
  cell.infraError ? plural(cell.infraError, "infra error") : "",
128
- cancelled ? `${cancelled} cancelled` : "",
129
- cell.skipped ? `${cell.skipped} skipped` : ""
88
+ canceled ? `${canceled} canceled` : "",
89
+ skipped ? `${skipped} skipped` : "",
90
+ notNeeded ? `${notNeeded} not needed (best of ${cell.bestOf})` : ""
130
91
  ].filter(Boolean);
131
92
  if (!parts.length) return "not run";
132
93
  return parts.join(" · ");
@@ -145,6 +106,8 @@ function assertionOf(k) {
145
106
  const line = k.outcome.assertion.trim().split("\n")[0]?.trim() ?? "";
146
107
  return clip(line.startsWith(`${k.label}: `) ? line.slice(k.label.length + 2) : line, ASSERTION_MAX);
147
108
  }
109
+ /** Where the eval made the check's first failure, `evals/a.eval.ts:7`. */
110
+ const atOf = (k) => k.outcome.status === "ran" ? k.outcome.at ?? "" : "";
148
111
  /** A cell passed outright: every counted attempt, every check, nothing errored. */
149
112
  const clean = (cell) => cell.outcome === "pass" && cell.failed === 0 && cell.infraError === 0 && cell.errors.length === 0 && !cell.checks.some(failing);
150
113
  /** An error no check explains, by kind: a throw in the eval's body reads `error`. */
@@ -152,7 +115,7 @@ const ERROR_KIND = {
152
115
  assertion: "error",
153
116
  judge: "judge",
154
117
  timeout: "timeout",
155
- aborted: "cancelled",
118
+ aborted: "canceled",
156
119
  infra: "infra error",
157
120
  turn: "turn error"
158
121
  };
@@ -164,9 +127,14 @@ function evidence(c, cell, indent, more = true) {
164
127
  const items = [];
165
128
  for (const k of cell.checks.filter(failing)) {
166
129
  const count = checkCount(k, single);
130
+ const at = atOf(k);
167
131
  items.push({
168
132
  indent,
169
- cells: [checkLabel(k.label), count === "not run" ? t(count, c.dim) : t(count, c.red)]
133
+ cells: [
134
+ checkLabel(k.label),
135
+ count === "not run" ? t(count, c.dim) : t(count, c.red),
136
+ ...at ? [t(at, c.dim)] : []
137
+ ]
170
138
  });
171
139
  const assertion = assertionOf(k);
172
140
  if (assertion) items.push({
@@ -183,6 +151,15 @@ function evidence(c, cell, indent, more = true) {
183
151
  cells: [errorKind(err), t(errorText(err), c.dim)]
184
152
  });
185
153
  items.push(...retriedItems(c, cell, indent));
154
+ const [snapshot, ...others] = cell.snapshots ?? [];
155
+ if (snapshot && items.length) {
156
+ const more = others.length ? ` (+${others.length} more in summary.md)` : "";
157
+ items.push({
158
+ indent,
159
+ cells: ["snapshot", t(`${snapshot}${more}`, c.dim)],
160
+ loose: true
161
+ });
162
+ }
186
163
  const passed = cell.checks.filter((k) => !failing(k) && !k.retried).length;
187
164
  if (more && passed && items.length) items.push({
188
165
  indent,
@@ -198,7 +175,7 @@ const retriedItems = (c, cell, indent) => cell.checks.filter((k) => !failing(k)
198
175
  /** A control with no environment of its own is the baseline the variants change. */
199
176
  const armOf = (cell) => cell.variant ?? (cell.environment === "-" ? "baseline" : cell.environment);
200
177
  /** Each cell's label: the parts that vary between them (harness, arm, sandbox). */
201
- function labeller(cells, parts) {
178
+ function labeler(cells, parts) {
202
179
  const vary = parts.filter((p) => {
203
180
  const of = (cell) => p === "harness" ? cell.harness : p === "arm" ? armOf(cell) : cell.sandbox;
204
181
  return new Set(cells.map(of)).size > 1;
@@ -277,73 +254,9 @@ function effortTable(c, cells, label, indent, failed, width) {
277
254
  ]
278
255
  }))], width);
279
256
  }
280
- const NO_SLOTS = {
281
- row: () => null,
282
- named: () => [],
283
- subject: null
284
- };
285
- const triesOf = (cells) => cells.reduce((n, x) => n + countedOf(x), 0);
286
- /** `named`, then a row per tool with the tries that named it. */
287
- function namedItems(c, slots, cells, indent) {
288
- const named = slots.named(cells);
289
- if (!named.length) return [];
290
- const of = triesOf(cells);
291
- return [{
292
- indent,
293
- free: t("named", c.dim)
294
- }, ...named.map((n) => ({
295
- indent: indent + 2,
296
- cells: [n.tool, `${n.tries} of ${of}`]
297
- }))];
298
- }
299
- /** The tries that named the subject, over the cells' tries: `0 of 4`; empty with no subject. */
300
- function subjectTries(slots, cells) {
301
- if (!slots.subject) return "";
302
- const subject = slots.subject.toLowerCase();
303
- return `${slots.named(cells).find((x) => x.tool.toLowerCase() === subject)?.tries ?? 0} of ${triesOf(cells)}`;
304
- }
305
- /** Web searches or fetches over the tries, when every try reported a number. */
306
- function webItems(cells, indent) {
307
- const row = (name, counts) => counts.length && counts.every((x) => x !== null) ? [{
308
- indent,
309
- cells: [name, String(counts.reduce((a, b) => (a ?? 0) + (b ?? 0), 0))]
310
- }] : [];
311
- return [...row("searches", cells.flatMap((x) => x.searches)), ...row("fetches", cells.flatMap((x) => x.fetches))];
312
- }
313
- /** A choose case that missed: its failed checks, the tools named, its searches, then `+N more passed`. */
314
- function chooseDetail(c, cells, indent, slots) {
315
- const checks = cells.flatMap((cell) => evidence(c, cell, indent, false));
316
- const passed = cells.flatMap((x) => x.checks.filter((k) => !failing(k))).length;
317
- return [
318
- ...checks,
319
- ...namedItems(c, slots, cells, indent),
320
- ...webItems(cells, indent),
321
- ...passed ? [{
322
- indent,
323
- free: t(`+${passed} more passed`, c.dim)
324
- }] : []
325
- ];
326
- }
327
- /** Choose a tool, on a single eval: the subject's count, and on a miss the detail. */
328
- function chooseBody(c, cells, indent, slots) {
329
- const items = slots.subject ? [{
330
- indent,
331
- cells: [slots.subject, subjectTries(slots, cells)]
332
- }] : [];
333
- if (!cells.every(clean)) items.push(...chooseDetail(c, cells, indent, slots));
334
- return table(items);
335
- }
336
- /** A row's sentence, as a line under it. */
337
- const rowLine = (slots, path, indent) => {
338
- const text = slots.row(path);
339
- return text ? [{
340
- indent,
341
- free: text
342
- }] : [];
343
- };
344
257
  /** Measure a change: a row per arm with its effect, the check matrix and the effort table. */
345
258
  function measureBody(c, cells, indent, o) {
346
- const label = labeller(cells, ["arm", "sandbox"]);
259
+ const label = labeler(cells, ["arm", "sandbox"]);
347
260
  const arms = [{
348
261
  indent,
349
262
  cells: ["arm", t("pass", c.dim)]
@@ -358,7 +271,6 @@ function measureBody(c, cells, indent, o) {
358
271
  ...effect.text ? [effect] : []
359
272
  ]
360
273
  });
361
- arms.push(...rowLine(o.slots, cell.path, indent + 2), ...namedItems(c, o.slots, [cell], indent + 2));
362
274
  const stats = o.verbose && o.effects ? statsLine(cell, cells) : null;
363
275
  if (stats) arms.push({
364
276
  indent: indent + 2,
@@ -379,8 +291,8 @@ function measureBody(c, cells, indent, o) {
379
291
  ];
380
292
  }
381
293
  /** A row per cell (harness, or sandbox), the ones that didn't pass outright with their evidence. */
382
- function cellRows(c, cells, heading, indent, slots) {
383
- const label = labeller(cells, heading === "harness" ? ["harness", "sandbox"] : ["sandbox"]);
294
+ function cellRows(c, cells, heading, indent) {
295
+ const label = labeler(cells, heading === "harness" ? ["harness", "sandbox"] : ["sandbox"]);
384
296
  const items = [{
385
297
  indent,
386
298
  cells: [heading, t("pass", c.dim)]
@@ -390,14 +302,13 @@ function cellRows(c, cells, heading, indent, slots) {
390
302
  indent,
391
303
  cells: [label(cell), cellCount(cell)]
392
304
  });
393
- items.push(...rowLine(slots, cell.path, indent + 2));
394
- if (!clean(cell)) items.push(...namedItems(c, slots, [cell], indent + 2), ...evidence(c, cell, indent + 2));
305
+ if (!clean(cell)) items.push(...evidence(c, cell, indent + 2));
395
306
  }
396
307
  return table(items);
397
308
  }
398
309
  /** Compare harnesses with arms: a heading per harness, its arms as Measure a change under it. */
399
310
  function bothBody(c, cells, indent, o) {
400
- const harnessOf = labeller(cells, ["harness", "sandbox"]);
311
+ const harnessOf = labeler(cells, ["harness", "sandbox"]);
401
312
  const groups = /* @__PURE__ */ new Map();
402
313
  for (const cell of cells) groups.set(harnessOf(cell), [...groups.get(harnessOf(cell)) ?? [], cell]);
403
314
  return [...groups].flatMap(([harness, group]) => [`${" ".repeat(indent)}${harness}`, ...measureBody(c, group, indent + 2, o)]);
@@ -405,19 +316,19 @@ function bothBody(c, cells, indent, o) {
405
316
  /** The rows under an eval's head (or a family case's), by what its cells compare. */
406
317
  function cellsBody(c, kind, cells, indent, o) {
407
318
  if (kind === "measure") return measureBody(c, cells, indent, o);
408
- if (kind === "harnesses") return cellRows(c, cells, "harness", indent, o.slots);
319
+ if (kind === "harnesses") return cellRows(c, cells, "harness", indent);
409
320
  if (kind === "both") return bothBody(c, cells, indent, o);
410
- if (kind === "choose") return chooseBody(c, cells, indent, o.slots);
411
- if (cells.length > 1) return cellRows(c, cells, "sandbox", indent, o.slots);
321
+ if (cells.length > 1) return cellRows(c, cells, "sandbox", indent);
412
322
  return resultsBody(c, cells[0], indent);
413
323
  }
414
324
  /** A single eval with one cell: its tries, and on anything but a clean pass, the evidence. */
415
325
  function resultsBody(c, cell, indent) {
416
326
  if (clean(cell)) return table(retriedItems(c, cell, indent));
417
327
  const items = [];
418
- if (cell.attempts > 1) items.push({
328
+ const tries = cell.attempts - (cell.notNeeded ?? 0);
329
+ if (tries > 1) items.push({
419
330
  indent,
420
- cells: [plural(cell.attempts, "try", "tries"), cellCount(cell)]
331
+ cells: [plural(tries, "try", "tries"), cellCount(cell)]
421
332
  });
422
333
  return table([...items, ...evidence(c, cell, indent)]);
423
334
  }
@@ -491,10 +402,9 @@ function pooledStats(p, label) {
491
402
  /**
492
403
  * A family: its template, then a row per case. With one cell a case is a row with its count;
493
404
  * with more, the case is outer and its cells inner, laid out as a single eval's are, and a
494
- * pooled `all cases` row sums each cell's counts with its variant's pooled effect. A choose
495
- * family's case row carries the subject's count instead, and a missed case the tools named.
405
+ * pooled `all cases` row sums each cell's counts with its variant's pooled effect.
496
406
  */
497
- function familyBlock(c, e, kind, verbose, slots) {
407
+ function familyBlock(c, e, kind, verbose) {
498
408
  const cases = e.cases ?? [];
499
409
  const columns = [...new Map(cases.flatMap((k) => k.cells).map((cell) => [cell.key, cell])).values()];
500
410
  const pooled = e.pooled ?? [];
@@ -513,38 +423,6 @@ function familyBlock(c, e, kind, verbose, slots) {
513
423
  });
514
424
  const head = `${" ".repeat(17)}${e.template ?? e.name}`;
515
425
  const questions = questionLines(c, e.questions, 2);
516
- if (kind === "choose") {
517
- const subject = slots.subject ? [t(slots.subject, c.dim)] : [];
518
- const items = [{
519
- indent: 2,
520
- cells: [
521
- "case",
522
- t("pass", c.dim),
523
- ...subject
524
- ]
525
- }];
526
- for (const k of cases) {
527
- const passed = k.cells.reduce((n, x) => n + x.passed, 0);
528
- const tries = triesOf(k.cells);
529
- const count = k.cells.length === 1 && k.cells[0] ? cellCount(k.cells[0]) : tries ? `${passed} of ${tries}` : "not run";
530
- const named = slots.subject && tries ? [subjectTries(slots, k.cells)] : [];
531
- items.push({
532
- indent: 2,
533
- cells: [
534
- lead(caseWord(k), k.label),
535
- count,
536
- ...named
537
- ]
538
- });
539
- items.push(...rowLine(slots, k.path, 4));
540
- if (!k.cells.every(clean)) items.push(...chooseDetail(c, k.cells, 4, slots));
541
- }
542
- return [
543
- head,
544
- ...table(items),
545
- ...questions
546
- ];
547
- }
548
426
  if (columns.length <= 1) {
549
427
  const items = [];
550
428
  for (const k of cases) {
@@ -553,7 +431,6 @@ function familyBlock(c, e, kind, verbose, slots) {
553
431
  indent: 2,
554
432
  cells: [lead(caseWord(k), k.label), cell ? cellCount(cell) : ""]
555
433
  });
556
- items.push(...rowLine(slots, k.path, 4));
557
434
  if (cell && !clean(cell)) items.push(...evidence(c, cell, 4));
558
435
  }
559
436
  return [
@@ -565,18 +442,15 @@ function familyBlock(c, e, kind, verbose, slots) {
565
442
  const out = [head];
566
443
  for (const k of cases) {
567
444
  out.push(` ${show(lead(caseWord(k), k.label))}`);
568
- const sentence = slots.row(k.path);
569
- if (sentence) out.push(` ${sentence}`);
570
445
  const failed = k.outcome !== "pass";
571
446
  out.push(...cellsBody(c, kind, k.cells, 4, {
572
447
  failed,
573
448
  verbose,
574
- effects: false,
575
- slots
449
+ effects: false
576
450
  }));
577
451
  }
578
452
  if (pooled.length) {
579
- const label = labeller(columns, [
453
+ const label = labeler(columns, [
580
454
  "harness",
581
455
  "arm",
582
456
  "sandbox"
@@ -597,7 +471,6 @@ function familyBlock(c, e, kind, verbose, slots) {
597
471
  ...effect.text ? [effect] : []
598
472
  ]
599
473
  });
600
- if (p) items.push(...rowLine(slots, p.path, 6));
601
474
  if (p && verbose) items.push({
602
475
  indent: 6,
603
476
  free: t(pooledStats(p, label(col)), c.dim)
@@ -607,43 +480,40 @@ function familyBlock(c, e, kind, verbose, slots) {
607
480
  }
608
481
  return [...out, ...questions];
609
482
  }
610
- /** The slots an eval's insight fills, or none without one. */
611
- function slotsFor(insight) {
612
- if (!insight) return NO_SLOTS;
613
- const rows = new Map((insight.rows ?? []).map((r) => [r.row, oneLine(r.text)]));
614
- const choose = insight.intent.job === "choose";
615
- const named = (cells) => {
616
- if (!choose) return [];
617
- const counts = /* @__PURE__ */ new Map();
618
- for (const n of insight.namedTools ?? []) {
619
- if (!cells.some((cell) => n.attempt.startsWith(`${cell.path}#`))) continue;
620
- for (const tool of new Set(n.tools.map(oneLine))) counts.set(tool, (counts.get(tool) ?? 0) + 1);
621
- }
622
- return [...counts].map(([tool, tries]) => ({
623
- tool,
624
- tries
625
- })).sort((a, b) => b.tries - a.tries || (a.tool < b.tool ? -1 : 1));
626
- };
627
- return {
628
- row: (path) => rows.get(path) ?? null,
629
- named,
630
- subject: choose && insight.intent.subject.tool ? oneLine(insight.intent.subject.tool) : null
631
- };
483
+ /** `Anthropic 12 · Amazon Bedrock 3`: each upstream provider and how many calls it served, summed. */
484
+ function upstreamList(counts) {
485
+ const total = /* @__PURE__ */ new Map();
486
+ for (const u of counts) total.set(u.provider, (total.get(u.provider) ?? 0) + u.calls);
487
+ return [...total].map(([provider, calls]) => `${provider} ${calls}`).join(" · ");
488
+ }
489
+ /**
490
+ * Which upstream providers a gateway said served each harness's calls and its judge's, a line per
491
+ * harness, so a run OpenRouter spread over several shows it. None where the gateway named none.
492
+ */
493
+ function upstreamLines(c, cells, indent) {
494
+ const by = /* @__PURE__ */ new Map();
495
+ for (const cell of cells) {
496
+ if (!cell.upstreams && !cell.judgeUpstreams) continue;
497
+ const seen = by.get(cell.harness) ?? {
498
+ agent: [],
499
+ judge: []
500
+ };
501
+ seen.agent.push(...cell.upstreams ?? []);
502
+ seen.judge.push(...cell.judgeUpstreams ?? []);
503
+ by.set(cell.harness, seen);
504
+ }
505
+ return [...by].map(([harness, { agent, judge }]) => {
506
+ const parts = [...agent.length ? [`served by ${upstreamList(agent)}`] : [], ...judge.length ? [`judge served by ${upstreamList(judge)}`] : []];
507
+ const label = by.size > 1 ? `${harness} ` : "";
508
+ return `${" ".repeat(indent)}${c.dim(`${label}${parts.join("; ")}`)}`;
509
+ });
632
510
  }
633
- /** One eval's block, laid out as `layout()` says for its job, with its insight's sentences. */
634
- function evalBlock(c, e, verbose = false, insight = null) {
635
- const shape = layout(insight?.intent.job ?? "other", e.across, e.family, false);
636
- const slots = slotsFor(insight);
637
- const answer = insight?.answer ? [` ${oneLine(insight.answer.text)}`] : [];
638
- const findings = (insight?.findings ?? []).map((f) => ` ${oneLine(f.text)}`);
511
+ /** One eval's block, laid out as `layout()` says for its shape. */
512
+ function evalBlock(c, e, verbose = false) {
513
+ const shape = layout(e.across, e.family, false);
639
514
  if (shape.family) {
640
- const [head = "", ...rest] = familyBlock(c, e, shape.kind, verbose, slots);
641
- return [
642
- head,
643
- ...answer,
644
- ...rest,
645
- ...findings
646
- ];
515
+ const served = upstreamLines(c, (e.cases ?? []).flatMap((k) => k.cells), 2);
516
+ return [...familyBlock(c, e, shape.kind, verbose), ...served];
647
517
  }
648
518
  const v = verdict(c, e);
649
519
  const name = `${e.group.length ? `${e.group.join(" › ")} › ` : ""}${e.name}`;
@@ -651,22 +521,19 @@ function evalBlock(c, e, verbose = false, insight = null) {
651
521
  const cells = e.cells ?? [];
652
522
  if (e.todo || e.outcome === "skipped" || !cells.length) return [head];
653
523
  const failed = e.outcome !== "pass";
654
- const body = cellsBody(c, shape.kind, cells, 2, {
655
- failed,
656
- verbose,
657
- effects: true,
658
- slots
659
- });
660
524
  return [
661
525
  head,
662
- ...answer,
663
- ...body,
664
- ...findings,
526
+ ...cellsBody(c, shape.kind, cells, 2, {
527
+ failed,
528
+ verbose,
529
+ effects: true
530
+ }),
531
+ ...upstreamLines(c, cells, 2),
665
532
  ...questionLines(c, e.questions, 2)
666
533
  ];
667
534
  }
668
535
  /**
669
- * `5 fail · 2 pass · 1 not gated · 1 cancelled`: errors outside any eval and fails first, every
536
+ * `5 fail · 2 pass · 1 not gated · 1 canceled`: errors outside any eval and fails first, every
670
537
  * case counting one, zeros left out.
671
538
  */
672
539
  function tallyLine(c, tally) {
@@ -676,16 +543,15 @@ function tallyLine(c, tally) {
676
543
  tally.pass ? c.green(`${tally.pass} pass`) : "",
677
544
  tally.notGated ? c.dim(`${tally.notGated} not gated`) : "",
678
545
  tally.infraError ? c.yellow(plural(tally.infraError, "infra error")) : "",
679
- tally.cancelled ? c.yellow(`${tally.cancelled} cancelled`) : "",
546
+ tally.canceled ? c.yellow(`${tally.canceled} canceled`) : "",
680
547
  tally.skipped ? c.dim(`${tally.skipped} skipped`) : ""
681
548
  ].filter(Boolean);
682
549
  return parts.length ? parts.join(c.dim(" · ")) : "no evals ran";
683
550
  }
684
551
  /** Every eval's block in file-path order, then source order; a blank line around each longer one. */
685
- function renderBlocks(suite, opts, insights = null) {
552
+ function renderBlocks(suite, opts) {
686
553
  const c = colors(opts.color);
687
- const byEval = new Map((insights?.evals ?? []).map((x) => [x.evalId, x]));
688
- const blocks = suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).map((e) => evalBlock(c, e, opts.verbose, byEval.get(e.path) ?? null)));
554
+ const blocks = suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).map((e) => evalBlock(c, e, opts.verbose)));
689
555
  const out = [];
690
556
  blocks.forEach((block, i) => {
691
557
  const prev = blocks[i - 1];
@@ -694,51 +560,20 @@ function renderBlocks(suite, opts, insights = null) {
694
560
  });
695
561
  return out;
696
562
  }
697
- /** Insights for these facts: null when there are none, they failed, or they were written for other facts. */
698
- const freshInsights = (suite, insights) => insights && !insights.error && insights.factsHash === factsHash(suite) ? insights : null;
699
- /** Why a run's insights didn't print, or with `-v` what wrote them. */
700
- function insightNotes(c, suite, insights, fresh, verbose) {
701
- if (!insights || !fromModel(insights)) return [];
702
- if (insights.error) return [c.yellow(`insights failed (${clip(insights.error, 200)})`)];
703
- if (!fresh) return [c.dim(`insights left out: insights.json was written for other facts than this run's; delete it and run reeval analyze ${suite.runId} --insights`)];
704
- const skipped = insights.budget?.skipped.length ?? 0;
705
- const failed = insights.failed ?? [];
706
- const stopped = insights.stopped;
707
- const finish = `reeval analyze ${suite.runId} --insights to finish`;
708
- const budget = [
709
- ...skipped ? [c.yellow(`insights skipped for ${plural(skipped, "eval")}: analysis.maxTokens (${formatTokens(insights.budget?.maxTokens ?? 0)}) reached — raise it in reeval.config.ts, or ${finish}`)] : [],
710
- ...failed[0] ? [c.yellow(`insights failed for ${plural(failed.length, "eval")} (${clip(failed[0].error, 160)}), so they print their facts: ${finish}`)] : [],
711
- ...stopped ? [c.yellow(`insights stopped: ${plural(stopped.skipped.length, "eval")} skipped (${stopped.signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"}): ${finish}`)] : []
712
- ];
713
- if (!verbose) return budget;
714
- const { dropped, regenerated } = insights.validation;
715
- return [...budget, c.dim([
716
- `insights by ${insights.model} (${insights.promptVersion})`,
717
- `${plural(dropped, "sentence")} removed`,
718
- ...regenerated ? [`${regenerated} asked again`] : [],
719
- `${formatTokens(insights.usage.inputTokens)} in, ${formatTokens(insights.usage.outputTokens)} out${insights.usage.reasoningTokens ? ` (${formatTokens(insights.usage.reasoningTokens)} reasoning)` : ""}`
720
- ].join(" · "))];
721
- }
722
563
  /**
723
- * A run's result: its errors outside any eval, its blocks with their insights, the headline,
724
- * then the tally and, with no analysis model, the hint, then the remarks. Insights written for
725
- * other facts are left out with a note; remarks come from the record and history, not a model,
726
- * so they print either way.
564
+ * A run's result: its errors outside any eval, its blocks, then the tally and the remarks, which
565
+ * come from the record and the history.
727
566
  */
728
- function render(suite, insights, opts) {
567
+ function render(suite, remarks, opts) {
729
568
  const c = colors(opts.color);
730
- const fresh = freshInsights(suite, insights);
731
- const blocks = renderBlocks(suite, opts, fresh);
569
+ const blocks = renderBlocks(suite, opts);
732
570
  return [
733
571
  ...suite.errors.flatMap((error) => [c.red(`error: ${error}`), ""]),
734
572
  ...blocks,
735
573
  ...blocks.length ? [""] : [],
736
- ...fresh?.headline ? [oneLine(fresh.headline.text)] : [],
737
- ...insightNotes(c, suite, insights, fresh, opts.verbose),
738
574
  ...opts.notes ?? [],
739
575
  tallyLine(c, suite.tally),
740
- ...opts.hint ? [NO_MODEL_HINT] : [],
741
- ...remarkLines(c, suite, insights, opts.verbose)
576
+ ...remarkLines(c, suite, remarks, opts.verbose)
742
577
  ];
743
578
  }
744
579
  //#endregion
@@ -777,7 +612,7 @@ function stateRows(run) {
777
612
  return rows;
778
613
  }
779
614
  /** Markdown for people and agents landing on a run cold. Links are relative to the artifact dir. */
780
- function renderSummary(run, opts) {
615
+ function renderSummary(run, opts = {}) {
781
616
  const lines = [
782
617
  `# reeval run ${run.runId}`,
783
618
  "",
@@ -788,10 +623,9 @@ function renderSummary(run, opts) {
788
623
  ...run.plan.sandboxWarnings.map((w) => `> **Warning:** ${w.message}\n`),
789
624
  "## Results",
790
625
  "",
791
- ...fenced(render(suiteResult(run), opts.insights ?? null, {
626
+ ...fenced(render(suiteResult(run), opts.remarks ?? null, {
792
627
  color: false,
793
- verbose: false,
794
- hint: opts.hint
628
+ verbose: false
795
629
  }), "text"),
796
630
  "",
797
631
  "## State",
@@ -844,9 +678,9 @@ function attemptDetail(lines, a) {
844
678
  const act = a.activity;
845
679
  const web = (n, what) => n === null ? `${what} not reported` : `${n} ${what}`;
846
680
  lines.push(`- Agent: ${act.fileEdits} file edits · ${act.tools} other tool calls · ${web(act.webSearches, "web searches")} · ${web(act.webFetches, "fetches")} · ${formatDuration(act.reasoningMs)} reasoning${act.stalledTools ? ` · ${act.stalledTools} stalled tool${act.stalledTools === 1 ? "" : "s"} (≥2m)${act.maxToolMs ? `, longest ${formatDuration(act.maxToolMs)}` : ""}` : ""}`);
847
- for (const { judgement: j, label, labelled } of labelledJudgements(a)) {
681
+ for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
848
682
  const retried = j.retries?.length ? `, after ${j.retries.length} ${j.retries.length === 1 ? "retry" : "retries"}` : "";
849
- lines.push(`- Judge${labelled ? ` ${cell(label)}` : ""} **${j.verdict}** ${j.score.toFixed(2)} (threshold ${j.threshold}, ${j.model}${retried}): ${j.question}`);
683
+ lines.push(`- Judge${labeled ? ` ${cell(label)}` : ""} **${j.verdict}** ${j.score.toFixed(2)} (threshold ${j.threshold}, ${j.model}${retried}): ${j.question}`);
850
684
  }
851
685
  for (const q of a.answers ?? []) {
852
686
  const said = q.error ? `failed: ${clip(cell(q.error), 200)}` : `${q.choice !== void 0 ? `**${cell(q.choice)}** ` : ""}${clip(cell(q.text), 300)}`;
@@ -871,8 +705,10 @@ function attemptDetail(lines, a) {
871
705
  a.files.judgeInput && `[judge input](${a.files.judgeInput.path})`
872
706
  ].filter(Boolean);
873
707
  if (links.length) lines.push(`- Files: ${links.join(" · ")}`);
708
+ const snapshots = (a.snapshots ?? []).map((s) => s.path ? `[${s.name}](${s.path})` : `${s.name} not taken: ${cell(s.skipped?.message ?? "")}`);
709
+ if (snapshots.length) lines.push(`- Snapshots (in .reeval/runs, never recorded): ${snapshots.join(" · ")}`);
874
710
  }
875
711
  //#endregion
876
- export { t as _, assertionOf as a, errorKind as c, labeller as d, plural as f, slotsOf as g, show as h, stateRows as i, errorText as l, renderBlocks as m, renderSummary as n, checkCount as o, render as p, runTally as r, checkLabel as s, OUTCOME_MARK as t, evalBlock as u, table as v, factsHash as y };
712
+ export { table as _, assertionOf as a, checkLabel as c, evalBlock as d, labeler as f, t as g, show as h, stateRows as i, errorKind as l, renderBlocks as m, renderSummary as n, atOf as o, render as p, runTally as r, checkCount as s, OUTCOME_MARK as t, errorText as u };
877
713
 
878
- //# sourceMappingURL=summary-DIGEVr08.js.map
714
+ //# sourceMappingURL=summary-CSzyBVsF.js.map