reeval 0.17.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
  4. package/dist/adapter-C8xPuXYq.js.map +1 -0
  5. package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
  6. package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
  7. package/dist/agents-C6zhkxp3.js +1151 -0
  8. package/dist/agents-C6zhkxp3.js.map +1 -0
  9. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  10. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  11. package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
  12. package/dist/app-files-DT5mJyV4.js.map +1 -0
  13. package/dist/assemble-UPCRrtOz.js +1 -0
  14. package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
  15. package/dist/assemble-t6h018An.js.map +1 -0
  16. package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
  17. package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
  18. package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
  19. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
  20. package/dist/boat-BjLqFJIZ.js +108 -0
  21. package/dist/boat-BjLqFJIZ.js.map +1 -0
  22. package/dist/boat-CJCtzfB1.js +322 -0
  23. package/dist/boat-CJCtzfB1.js.map +1 -0
  24. package/dist/bridge/index.mjs +300 -78
  25. package/dist/capture-DoxFUN3J.js +1 -0
  26. package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
  27. package/dist/capture-iBzFiC0j.js.map +1 -0
  28. package/dist/cases-Dy0mifal.js +1 -0
  29. package/dist/cases-z9E6DCd5.js +639 -0
  30. package/dist/cases-z9E6DCd5.js.map +1 -0
  31. package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
  32. package/dist/cli-20Fpvs5H.js.map +1 -0
  33. package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
  34. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
  37. package/dist/client-KM4C5r5_.js.map +1 -0
  38. package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
  39. package/dist/command-B5OyHpOu.js.map +1 -0
  40. package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
  41. package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
  42. package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
  43. package/dist/command-CRh-cb6Z.js.map +1 -0
  44. package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
  45. package/dist/command-aUJvENwy.js.map +1 -0
  46. package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
  47. package/dist/commands-CWy-WndG.js.map +1 -0
  48. package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
  49. package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
  50. package/dist/config-B9mB1kF5.js.map +1 -0
  51. package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
  52. package/dist/context-Bkv3ndjq.js.map +1 -0
  53. package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
  54. package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
  55. package/dist/cost-Cr26Alhq.js +1 -0
  56. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
  57. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
  58. package/dist/db-P2fXw4nx.js +2 -0
  59. package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
  60. package/dist/db-zgC1tvKB.js.map +1 -0
  61. package/dist/debug-CF4p4Lp_.js +1 -0
  62. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  63. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  64. package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
  65. package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
  66. package/dist/default-effort-DGhQmu95.js.map +1 -0
  67. package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
  68. package/dist/dist-CAFD1yfv.js.map +1 -0
  69. package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
  70. package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
  71. package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
  72. package/dist/docker-DAMnm3L9.js.map +1 -0
  73. package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
  74. package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
  75. package/dist/doctor-B2PohneB.js.map +1 -0
  76. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  77. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  78. package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
  79. package/dist/environment-DB8o3hGi.js.map +1 -0
  80. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  81. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  82. package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
  83. package/dist/eval-source-D3sbAzt-.js.map +1 -0
  84. package/dist/evaluate-COdzf7Hl.d.ts +92 -0
  85. package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
  86. package/dist/evaluate-CwKeN_fo.js.map +1 -0
  87. package/dist/files-B4KQUiM7.js +1 -0
  88. package/dist/files-BEwfjD8J.js +64 -0
  89. package/dist/files-BEwfjD8J.js.map +1 -0
  90. package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
  91. package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
  92. package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
  93. package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
  94. package/dist/files-C1Rk9FYA.js +1 -0
  95. package/dist/files-C3mCC8Mn.js +1 -0
  96. package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
  97. package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
  98. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  99. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  100. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  101. package/dist/files-DlQshsks.js.map +1 -0
  102. package/dist/files-DnlA1A7N.js +1 -0
  103. package/dist/gate-CgYdaprG.js +49 -0
  104. package/dist/gate-CgYdaprG.js.map +1 -0
  105. package/dist/gateway.d.ts +1 -1
  106. package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
  107. package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
  108. package/dist/harness-C2berMiM.js +105 -0
  109. package/dist/harness-C2berMiM.js.map +1 -0
  110. package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
  111. package/dist/harness-DXv2gnaT.js.map +1 -0
  112. package/dist/harness.d.ts +1 -1
  113. package/dist/harness.js +1 -1
  114. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  115. package/dist/ids-C_advdJA.js.map +1 -0
  116. package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
  117. package/dist/images-BNgcR77K.js.map +1 -0
  118. package/dist/index.d.ts +23 -3
  119. package/dist/index.js +5 -5
  120. package/dist/init-4RPeD-K-.js +1 -0
  121. package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
  122. package/dist/init-DbuZVRrW.js.map +1 -0
  123. package/dist/ink-list-BpfidQ1h.js +274 -0
  124. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  125. package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
  126. package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
  127. package/dist/judge-COMXT1Hx.js +230 -0
  128. package/dist/judge-COMXT1Hx.js.map +1 -0
  129. package/dist/judge.d.ts +1 -1
  130. package/dist/judge.js +1 -1
  131. package/dist/judgments-CzgpsTqv.js +246 -0
  132. package/dist/judgments-CzgpsTqv.js.map +1 -0
  133. package/dist/kinds-C1oY3qyC.js +697 -0
  134. package/dist/kinds-C1oY3qyC.js.map +1 -0
  135. package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
  136. package/dist/layout-Cm85tQYk.js.map +1 -0
  137. package/dist/list-DuThsE28.js +1348 -0
  138. package/dist/list-DuThsE28.js.map +1 -0
  139. package/dist/live-CJCK11nh.js +1 -0
  140. package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
  141. package/dist/live-CgYbO4Te.js.map +1 -0
  142. package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
  143. package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
  144. package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
  145. package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
  146. package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
  147. package/dist/model-Cv4GuTsZ.js +1 -0
  148. package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
  149. package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
  150. package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
  151. package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
  152. package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
  153. package/dist/model-V3mMXp6x.js.map +1 -0
  154. package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
  155. package/dist/node-D7pAU--0.js.map +1 -0
  156. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
  157. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  158. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
  159. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  160. package/dist/options-CHubslQf.js +1 -0
  161. package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
  162. package/dist/pairs-BGb134Vu.js.map +1 -0
  163. package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
  164. package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
  165. package/dist/plain-BYUf_fUV.js +267 -0
  166. package/dist/plain-BYUf_fUV.js.map +1 -0
  167. package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
  168. package/dist/plan-Cgq9NCEX.js.map +1 -0
  169. package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
  170. package/dist/plugins-BkmPxDiY.js.map +1 -0
  171. package/dist/preflight-4gWSy0Pe.js +1 -0
  172. package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
  173. package/dist/preflight-BElE_q0u.js.map +1 -0
  174. package/dist/preflight-CtlloeOQ.js +68 -0
  175. package/dist/preflight-CtlloeOQ.js.map +1 -0
  176. package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
  177. package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
  178. package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
  179. package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
  180. package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
  181. package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
  182. package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
  183. package/dist/provenance-zsc4--dP.js.map +1 -0
  184. package/dist/provider-BnUoz4KB.js +1 -0
  185. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  186. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  187. package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
  188. package/dist/providers-B8xYMeah.js.map +1 -0
  189. package/dist/providers-CVu8e0xS.js +1 -0
  190. package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
  191. package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
  192. package/dist/query-DIfkUcPh.js +1 -0
  193. package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
  194. package/dist/questions-CWlR9ffu.js.map +1 -0
  195. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  196. package/dist/record-6IUzwP9K.js.map +1 -0
  197. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  198. package/dist/redact-BhjtnjrU.js.map +1 -0
  199. package/dist/remarks-CAvTSq16.js +1 -0
  200. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  201. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  202. package/dist/remarks-DPCONFxZ.js +138 -0
  203. package/dist/remarks-DPCONFxZ.js.map +1 -0
  204. package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
  205. package/dist/remarks-vtkPmQb9.js.map +1 -0
  206. package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
  207. package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
  208. package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
  209. package/dist/resolve-Do0ZaYH6.js.map +1 -0
  210. package/dist/resolve-amw_2Odz.js +1 -0
  211. package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
  212. package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
  213. package/dist/resume-AW5oXzUc.js +691 -0
  214. package/dist/resume-AW5oXzUc.js.map +1 -0
  215. package/dist/retry-ChzISxmy.js +105 -0
  216. package/dist/retry-ChzISxmy.js.map +1 -0
  217. package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
  218. package/dist/runtime-BpW1f1Mx.js.map +1 -0
  219. package/dist/sandbox/boat.d.ts +26 -0
  220. package/dist/sandbox/boat.js +37 -0
  221. package/dist/sandbox/boat.js.map +1 -0
  222. package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
  223. package/dist/sandbox-kSgr7vo5.js.map +1 -0
  224. package/dist/sandbox.d.ts +20 -2
  225. package/dist/sandbox.js +1 -1
  226. package/dist/server-2_gUnVvf.js +1 -0
  227. package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
  228. package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
  229. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  230. package/dist/shared-Cfi45Bvy.js.map +1 -0
  231. package/dist/slug-IHh3f770.js +136 -0
  232. package/dist/slug-IHh3f770.js.map +1 -0
  233. package/dist/source-CY3etKzw.js +1 -0
  234. package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
  235. package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
  236. package/dist/src-6Zzyg4Qd.js +2 -0
  237. package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
  238. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  239. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  240. package/dist/src-BRMsk9nh.js +2 -0
  241. package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
  242. package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
  243. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  244. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  245. package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
  246. package/dist/src-Ck2r_EMT.js.map +1 -0
  247. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  248. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  249. package/dist/src-DB7N8TYJ.js +2 -0
  250. package/dist/src-DPDaAYwh.js +2 -0
  251. package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
  252. package/dist/src-DtodMT59.js +1 -0
  253. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  254. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  255. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  256. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  257. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  258. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  259. package/dist/src-T-OeuJLr.js +2 -0
  260. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  261. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  262. package/dist/src-zntEDlnx.js +3 -0
  263. package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
  264. package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
  265. package/dist/state-kimtIhBP.js +1 -0
  266. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  267. package/dist/stats-C18W4xPX.js.map +1 -0
  268. package/dist/suite-CKBa6MKt.js +2 -0
  269. package/dist/suite-COglUhPd.js +1272 -0
  270. package/dist/suite-COglUhPd.js.map +1 -0
  271. package/dist/summary-DU5zaibx.js +148 -0
  272. package/dist/summary-DU5zaibx.js.map +1 -0
  273. package/dist/tart-BTVXp8rz.js +373 -0
  274. package/dist/tart-BTVXp8rz.js.map +1 -0
  275. package/dist/tart-cli-BIC9HqPg.js +37 -0
  276. package/dist/tart-cli-BIC9HqPg.js.map +1 -0
  277. package/dist/tart-cli-BMrkmBjI.js +1 -0
  278. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  279. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  280. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  281. package/dist/tart-ref-DYZijjrU.js +55 -0
  282. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  283. package/dist/testing.d.ts +47 -37
  284. package/dist/testing.js +4 -4
  285. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  286. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  287. package/dist/trace-Dx2NBE1O.js +311 -0
  288. package/dist/trace-Dx2NBE1O.js.map +1 -0
  289. package/dist/traits-DKnYkqeK.js +66 -0
  290. package/dist/traits-DKnYkqeK.js.map +1 -0
  291. package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
  292. package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
  293. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  294. package/dist/types-C2mLpph3.js.map +1 -0
  295. package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
  296. package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
  297. package/dist/vercel-CGlSBv63.js.map +1 -0
  298. package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
  299. package/dist/vertex-Eb2wqahk.js.map +1 -0
  300. package/dist/vertex-O88TCwgY.js +1 -0
  301. package/dist/view/index.d.ts +238 -90
  302. package/dist/view/index.js +11 -9
  303. package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
  304. package/dist/views-DAIiLiV3.js.map +1 -0
  305. package/dist/views-DtRhj2Tn.js +2 -0
  306. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
  307. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
  308. package/dist/workspace-CcQRT31C.js +1 -0
  309. package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
  310. package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
  311. package/package.json +26 -11
  312. package/skills/eval-rubric/SKILL.md +31 -4
  313. package/dist/adapter-DyvDj4ZN.js.map +0 -1
  314. package/dist/app-files-hpRT5lj_.js.map +0 -1
  315. package/dist/assemble-DTXwCL3m.js +0 -1
  316. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  317. package/dist/capture-BT2fAmpZ.js +0 -1
  318. package/dist/capture-Bdcx9Gap.js.map +0 -1
  319. package/dist/cli-CGd51thd.js.map +0 -1
  320. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  321. package/dist/client-CF-DOjOU.js.map +0 -1
  322. package/dist/command-CsfcwMyc.js.map +0 -1
  323. package/dist/command-CyIb9BSC.js.map +0 -1
  324. package/dist/command-TLmY-rYp.js.map +0 -1
  325. package/dist/commands-C54S45d7.js.map +0 -1
  326. package/dist/config-ImY6AlRE.js.map +0 -1
  327. package/dist/context-BajfiH0S.js.map +0 -1
  328. package/dist/contrasts-DtSX8E_V.js +0 -101
  329. package/dist/contrasts-DtSX8E_V.js.map +0 -1
  330. package/dist/cost-CMzY9VXv.js +0 -1
  331. package/dist/db-ZIk6S2CD.js +0 -2
  332. package/dist/db-sHGpaLo9.js.map +0 -1
  333. package/dist/debug-BnXV7JbX.js +0 -1
  334. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  335. package/dist/dist-ByYk_wAs.js.map +0 -1
  336. package/dist/docker--2ytOHcu.js.map +0 -1
  337. package/dist/doctor-eE65YpEv.js.map +0 -1
  338. package/dist/environment-Cy3FVLR5.js.map +0 -1
  339. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  340. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  341. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  342. package/dist/files-B3d-8Fe0.js +0 -1
  343. package/dist/files-COW5JBJM.js +0 -1
  344. package/dist/files-Do4Rp3yZ.js +0 -1
  345. package/dist/files-p-uTdOjA.js.map +0 -1
  346. package/dist/gate-BoDKeuJu.js.map +0 -1
  347. package/dist/harness-Ch2P81sx.js.map +0 -1
  348. package/dist/ids-DuK6xl4w.js.map +0 -1
  349. package/dist/images-BVZHDf_z.js.map +0 -1
  350. package/dist/init-BWnMbUnc.js.map +0 -1
  351. package/dist/init-s1jN-PCI.js +0 -1
  352. package/dist/judge-ByEUlzOM.js.map +0 -1
  353. package/dist/live-DmJix4c8.js +0 -1
  354. package/dist/live-d_5a5iUv.js.map +0 -1
  355. package/dist/model-8uZBYqV5.js +0 -1
  356. package/dist/model-DOiCiMCD.js.map +0 -1
  357. package/dist/node-CwP1G178.js.map +0 -1
  358. package/dist/opencode-providers-yNKyXD49.js.map +0 -1
  359. package/dist/plan-CIfOFhth.js.map +0 -1
  360. package/dist/plugins-BEs2Am3L.js.map +0 -1
  361. package/dist/preflight-50Qw5e5K.js.map +0 -1
  362. package/dist/provenance-Cc8fFE7Z.js.map +0 -1
  363. package/dist/provider-DCdTHQpk.js +0 -1
  364. package/dist/providers-CP_bBtYm.js +0 -1
  365. package/dist/providers-DZSTlC9R.js.map +0 -1
  366. package/dist/query-kFPuqBv1.js +0 -1
  367. package/dist/questions-C_Y-oXW6.js.map +0 -1
  368. package/dist/record-DhzwkSXc.js.map +0 -1
  369. package/dist/redact-BEEGc37e.js.map +0 -1
  370. package/dist/remarks-0EXMh6AD.js.map +0 -1
  371. package/dist/remarks-BL18Id09.js +0 -1
  372. package/dist/remarks-DeZZT8PK.js +0 -116
  373. package/dist/remarks-DeZZT8PK.js.map +0 -1
  374. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  375. package/dist/resolve-BJSay360.js +0 -1
  376. package/dist/resolve-B_JlQ0La.js.map +0 -1
  377. package/dist/resume-CexsWuGK.js +0 -434
  378. package/dist/resume-CexsWuGK.js.map +0 -1
  379. package/dist/runtime-CSZ1hjlN.js.map +0 -1
  380. package/dist/sandbox-B5W8lLL1.js.map +0 -1
  381. package/dist/server-SBzrf35T.js +0 -1
  382. package/dist/shared-ogqPJ0uo.js.map +0 -1
  383. package/dist/source-Dn1uqPon.js +0 -1
  384. package/dist/src-BKDZO3ra.js +0 -2
  385. package/dist/src-BxrHipV0.js +0 -3
  386. package/dist/src-C9elednc.js +0 -2
  387. package/dist/src-CM7xdJFA.js +0 -2
  388. package/dist/src-CWEGknZo.js +0 -2
  389. package/dist/src-Cpy9MNS3.js.map +0 -1
  390. package/dist/src-DVaAKCsH.js +0 -1
  391. package/dist/src-DurP--_D.js +0 -2
  392. package/dist/state-Cicc9Oa5.js +0 -1
  393. package/dist/stats-D-Gqwqt0.js.map +0 -1
  394. package/dist/suite-BYJ2Q0S-.js +0 -2
  395. package/dist/suite-BtHPBFYL.js +0 -544
  396. package/dist/suite-BtHPBFYL.js.map +0 -1
  397. package/dist/summary-zP3cCXDN.js.map +0 -1
  398. package/dist/types-C4OkYhZg.js.map +0 -1
  399. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  400. package/dist/vertex-BaAz44YN.js +0 -1
  401. package/dist/views-CKoN5NAx.js.map +0 -1
  402. package/dist/views-CmyX255j.js +0 -2
  403. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -1,9 +1,11 @@
1
+ import { a as plural } from "./text-B8-W3Cvn.js";
2
+ import { n as formatTokens, r as promptTokens } from "./usage-DJiDnh2D.js";
1
3
  import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-Dwmi7L4X.js";
2
- import { a as plural, n as clip } from "./text-D--Q5KsA.js";
3
- import { a as rescore, h as sampleReasons, n as gateLabel } from "./gate-BoDKeuJu.js";
4
- import { c as minimumP, d as round3, i as mannWhitney, n as containsZero, r as liftInterval, s as median, u as quantile } from "./stats-D-Gqwqt0.js";
5
- import { a as totalTokens, i as sumUsage, n as formatTokens, r as promptTokens } from "./usage-DJiDnh2D.js";
6
- import { u as readableTranscriptLine } from "./record-DhzwkSXc.js";
4
+ import { l as sampleReasons } from "./pairs-BGb134Vu.js";
5
+ import { a as rescore, n as gateLabel } from "./gate-CgYdaprG.js";
6
+ import { T as cellEffort, w as cellContrast } from "./cases-z9E6DCd5.js";
7
+ import { m as uniqueJudgeLabels, n as cellUsage, t as cellJudgeRates } from "./judgments-CzgpsTqv.js";
8
+ import { u as readableTranscriptLine } from "./record-6IUzwP9K.js";
7
9
  // src/view/compare.ts
8
10
  const STACK_FIELDS = [
9
11
  {
@@ -223,7 +225,7 @@ function canceledSides(outcomes, cells) {
223
225
  const sides = ["base", "head"].filter((s) => canceledOn(outcomes[s], cells[s] ?? []));
224
226
  return sides.length ? { canceled: sides } : {};
225
227
  }
226
- /** An eval only head has. */
228
+ /** An eval only head has, with each of its cases. */
227
229
  const addedRow = (h) => ({
228
230
  id: h.id,
229
231
  base: null,
@@ -232,8 +234,41 @@ const addedRow = (h) => ({
232
234
  ...canceledSides({
233
235
  base: null,
234
236
  head: h.outcome
235
- }, { head: h.cells })
237
+ }, { head: h.cells }),
238
+ ...verdictOf(h),
239
+ ...h.cases ? { cases: caseEvals(h).map(addedRow) } : {},
240
+ ...casesGate(null, h)
236
241
  });
242
+ /** An eval only base has, with each of its cases. */
243
+ const removedRow = (b) => ({
244
+ id: b.id,
245
+ base: b.outcome,
246
+ head: null,
247
+ status: "removed",
248
+ ...canceledSides({
249
+ base: b.outcome,
250
+ head: null
251
+ }, { base: b.cells }),
252
+ ...verdictOf(b),
253
+ ...b.cases ? { cases: caseEvals(b).map(removedRow) } : {},
254
+ ...casesGate(b, null)
255
+ });
256
+ /** A case its eval's rate gates, on the side that decides: head's, or base's when only it ran. */
257
+ const verdictOf = (e) => e?.verdict ? { verdict: e.verdict } : {};
258
+ /** An eval's cases, each as an eval of its own keyed by the case key. */
259
+ const caseEvals = (e) => (e.cases ?? []).map((k) => ({
260
+ id: k.key,
261
+ fingerprint: k.fingerprint,
262
+ outcome: k.outcome,
263
+ ...k.verdict ? { verdict: k.verdict } : {},
264
+ sources: e.sources,
265
+ ...e.source ? { source: {
266
+ ...e.source,
267
+ ...k.hash ? { case: k.hash } : {}
268
+ } } : {},
269
+ ...e.revision !== void 0 ? { revision: e.revision } : {},
270
+ cells: k.cells
271
+ }));
237
272
  /**
238
273
  * When two runs of an eval differ only in which cells ran (a filtered run, a harness added), its
239
274
  * outcome on the cells both have, and the cells only one has. Null when no cell is shared or a
@@ -276,13 +311,34 @@ function compareCells(base, head) {
276
311
  }];
277
312
  });
278
313
  }
279
- /** A family with a gate: some pooled lift has a `minLift`. One without can't fail, so has no row. */
280
- const gated = (f) => !!f?.lifts.some((l) => l.minLift !== void 0);
314
+ /**
315
+ * An eval whose cases pool a gate: some lift has a `minLift`, or some cell's rate over the cases
316
+ * a `passRate`. Its outcome is then over the cases that ran, so a run of other cases can't be
317
+ * compared with it.
318
+ */
319
+ const pooledGate = (e) => !!e && ((e.lifts ?? []).some((l) => l.minLift !== void 0) || !!e.rates?.length);
281
320
  const liftSide = ({ variant: _, vs: __, ...side }) => side;
282
- /** Each variant's pooled lift on both sides, head's order first. */
283
- function familyLifts(b, h) {
321
+ /**
322
+ * When both runs pooled lifts over different variants: the variants only head or only base has,
323
+ * and those both have, over which the eval's outcome is then read.
324
+ */
325
+ function liftScope(b, h) {
326
+ const of = (e) => (e.lifts ?? []).map((l) => l.variant);
327
+ const [base, head] = [of(b), of(h)];
328
+ if (!base.length || !head.length) return null;
329
+ const added = head.filter((v) => !base.includes(v));
330
+ const removed = base.filter((v) => !head.includes(v));
331
+ if (!added.length && !removed.length) return null;
332
+ return {
333
+ added,
334
+ removed,
335
+ shared: head.filter((v) => base.includes(v))
336
+ };
337
+ }
338
+ /** Each variant's lift paired by case on both sides, head's order first. */
339
+ function casesLifts(b, h) {
284
340
  const out = /* @__PURE__ */ new Map();
285
- for (const [side, f] of [["head", h], ["base", b]]) for (const l of f?.lifts ?? []) {
341
+ for (const [side, e] of [["head", h], ["base", b]]) for (const l of e?.lifts ?? []) {
286
342
  const row = out.get(l.variant) ?? {
287
343
  key: l.variant,
288
344
  vs: l.vs,
@@ -294,131 +350,142 @@ function familyLifts(b, h) {
294
350
  }
295
351
  return [...out.values()];
296
352
  }
297
- const liftsOutcome = (lifts) => lifts.some((l) => l.outcome === "fail") ? "fail" : "pass";
298
- /** A gated family only head has. */
299
- const addedFamilyRow = (h) => ({
300
- id: h.id,
301
- kind: "family",
302
- base: null,
303
- head: h.outcome,
304
- status: "added",
305
- lifts: familyLifts(null, h)
306
- });
307
- /** The rows of a run with nothing to compare it with: every eval and gated family is new. */
308
- const newRows = (head) => [...head.evals.map(addedRow), ...(head.families ?? []).filter(gated).map(addedFamilyRow)];
353
+ /** Each gated cell's rate over the cases on both sides, head's order first. */
354
+ function casesRates(b, h) {
355
+ const out = /* @__PURE__ */ new Map();
356
+ for (const [side, e] of [["head", h], ["base", b]]) for (const { cell, ...rate } of e?.rates ?? []) {
357
+ const row = out.get(cell) ?? {
358
+ key: cell,
359
+ base: null,
360
+ head: null
361
+ };
362
+ row[side] = rate;
363
+ out.set(cell, row);
364
+ }
365
+ return [...out.values()];
366
+ }
367
+ /** The lifts and gated rates of an eval with `cases`, on both sides; unset when it has none. */
368
+ function casesGate(b, h) {
369
+ const lifts = casesLifts(b, h);
370
+ const rates = casesRates(b, h);
371
+ return {
372
+ ...lifts.length ? { lifts } : {},
373
+ ...rates.length ? { rates } : {}
374
+ };
375
+ }
376
+ /** The rows of a run with nothing to compare it with: every eval is new. */
377
+ const newRows = (head) => head.evals.map(addedRow);
378
+ /** Each eval's row, an eval with `cases` with a row for each of its cases. */
379
+ function compare(base, head) {
380
+ const graderChanged = stackChanges(base, head).some((c) => c.outsideFingerprint);
381
+ return evalRows(base.evals, head.evals, graderChanged);
382
+ }
383
+ /** A case's outcome as its eval's verdict reads it: a failure its eval's rate gates passes. */
384
+ const caseOutcome = (o, verdict) => verdict === "family" && o === "fail" ? "pass" : o;
309
385
  /**
310
- * A gated family's row. Its pooled outcome is comparable only when both runs ran the same cases
311
- * and every case is: a case that changed or hit an infrastructure error makes the family `drift`
312
- * or `infra` as it does the case, and a failing side with a canceled attempt in any case reads
313
- * `canceled`, since the pooled lift counts it. With different variants, it's over the lifts both
314
- * have, and with none in common it can't compare them.
386
+ * An eval with `cases` in both runs. Over the same cases, its outcome is the eval's; over
387
+ * different ones, the worst of the cases both ran, as their eval reads them, unless its cases
388
+ * pool a gate, which a run of other cases can't be compared with. A case that changed or hit an
389
+ * infrastructure error makes the eval `drift` or `infra` too, and a case that regressed outside
390
+ * a pooled gate makes it `regressed`.
315
391
  */
316
- function familyRow(id, b, h, ctx) {
317
- if (!b) return addedFamilyRow(h);
318
- const lifts = familyLifts(b, h ?? null);
319
- if (!h) return {
320
- id,
321
- kind: "family",
322
- base: b.outcome,
323
- head: null,
324
- status: "removed",
325
- lifts
392
+ function casesRow(b, h, graderChanged, row) {
393
+ const caseRows = evalRows(caseEvals(b), caseEvals(h), graderChanged);
394
+ const shared = caseRows.filter((r) => r.base !== null && r.head !== null);
395
+ const added = caseRows.filter((r) => r.base === null).map((r) => r.id);
396
+ const removed = caseRows.filter((r) => r.head === null).map((r) => r.id);
397
+ const out = {
398
+ ...row,
399
+ cases: caseRows,
400
+ ...casesGate(b, h)
326
401
  };
327
- const shared = lifts.flatMap((l) => l.base && l.head ? [{
328
- base: l.base,
329
- head: l.head
330
- }] : []);
331
- const scoped = shared.length > 0 && shared.length < lifts.length;
332
- const disjoint = !shared.length && b.lifts.length > 0 && h.lifts.length > 0;
333
- const row = {
334
- id,
335
- kind: "family",
336
- base: scoped ? liftsOutcome(shared.map((l) => l.base)) : b.outcome,
337
- head: scoped ? liftsOutcome(shared.map((l) => l.head)) : h.outcome,
338
- lifts,
339
- ...scoped || disjoint ? { cells: {
340
- added: lifts.filter((l) => !l.base).map((l) => l.key),
341
- removed: lifts.filter((l) => !l.head).map((l) => l.key)
342
- } } : {}
402
+ const changes = [.../* @__PURE__ */ new Set([...row.changes ?? [], ...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])])];
403
+ const failing = caseRows.filter((r) => r.base === null && caseOutcome(r.head, r.verdict) === "fail" && !r.canceled?.includes("head"));
404
+ const over = (side) => side === "head" ? [...shared, ...failing] : shared;
405
+ if (added.length || removed.length) {
406
+ if (pooledGate(b) || pooledGate(h) || !shared.length) changes.push(...added.map((k) => `case ${k} added`), ...removed.map((k) => `case ${k} removed`));
407
+ else {
408
+ const worst = (side) => outcomeOf(over(side).map((r) => ({
409
+ gate: "all",
410
+ outcome: caseOutcome(r[side], r.verdict) ?? "skipped"
411
+ })));
412
+ out.base = worst("base");
413
+ out.head = worst("head");
414
+ out.caseScope = {
415
+ added,
416
+ removed
417
+ };
418
+ changes.push(...failing.map((r) => `case ${r.id} added`));
419
+ }
420
+ }
421
+ const variants = liftScope(b, h);
422
+ if (variants) {
423
+ out.cells = {
424
+ added: variants.added,
425
+ removed: variants.removed
426
+ };
427
+ if (!variants.shared.length) return {
428
+ ...out,
429
+ ...changes.length ? { changes } : {},
430
+ status: "drift"
431
+ };
432
+ const worst = (side, e) => outcomeOf([
433
+ ...(shared.length ? over(side) : caseRows).map((r) => ({
434
+ gate: "all",
435
+ outcome: caseOutcome(r[side], r.verdict) ?? "skipped"
436
+ })),
437
+ ...(e.lifts ?? []).filter((l) => variants.shared.includes(l.variant)).map((l) => ({
438
+ gate: "all",
439
+ outcome: l.outcome
440
+ })),
441
+ ...(e.rates ?? []).filter((r) => !variants.added.includes(r.cell) && !variants.removed.includes(r.cell)).map((r) => ({
442
+ gate: "all",
443
+ outcome: r.outcome
444
+ }))
445
+ ]);
446
+ out.base = worst("base", b);
447
+ out.head = worst("head", h);
448
+ }
449
+ if (changes.length) out.changes = changes;
450
+ if (out.base === "infra_error" || out.head === "infra_error") return {
451
+ ...out,
452
+ status: "infra"
343
453
  };
344
- const prefix = id.slice(0, id.lastIndexOf(" › ") + 3);
345
- const name = (caseId) => caseId.startsWith(prefix) ? caseId.slice(prefix.length) : caseId;
346
- const before = new Set(b.caseIds);
347
- const after = new Set(h.caseIds);
348
- const caseRows = [.../* @__PURE__ */ new Set([...h.caseIds, ...b.caseIds])].flatMap((c) => ctx.rows.get(c) ?? []);
349
- const changes = [.../* @__PURE__ */ new Set([
350
- ...h.caseIds.filter((c) => !before.has(c)).map((c) => `case ${name(c)} added`),
351
- ...b.caseIds.filter((c) => !after.has(c)).map((c) => `case ${name(c)} removed`),
352
- ...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])
353
- ])];
354
- const canceled = ["base", "head"].filter((s) => row[s] === "fail" && ctx.canceledIn(s, s === "base" ? b.caseIds : h.caseIds));
355
- if (canceled.length) row.canceled = canceled;
356
454
  if (caseRows.some((r) => r.status === "infra")) return {
357
- ...row,
455
+ ...out,
358
456
  status: "infra"
359
457
  };
360
- if (changes.length) return {
361
- ...row,
362
- status: "drift",
363
- changes
364
- };
365
- if (disjoint || row.canceled || caseRows.some((r) => r.status === "drift")) return {
366
- ...row,
458
+ if (changes.some((c) => c !== "gate changed") || out.canceled || caseRows.some((r) => r.status === "drift" || r.canceled)) return {
459
+ ...out,
367
460
  status: "drift"
368
461
  };
369
- if (ctx.graderChanged && row.base !== row.head) return {
370
- ...row,
462
+ const from = out.gate?.rescored ?? out.base;
463
+ if (graderChanged && from !== out.head) return {
464
+ ...out,
371
465
  status: "drift"
372
466
  };
373
- if (row.base === "pass" && row.head === "fail") return {
374
- ...row,
467
+ const caseRegressed = caseRows.some((r) => r.status === "regressed" && r.verdict !== "family");
468
+ if (from === "pass" && out.head === "fail" || caseRegressed) return {
469
+ ...out,
375
470
  status: "regressed"
376
471
  };
377
- if (row.base === "fail" && row.head === "pass") return {
378
- ...row,
472
+ if (from === "fail" && out.head === "pass") return {
473
+ ...out,
379
474
  status: "fixed"
380
475
  };
381
476
  return {
382
- ...row,
383
- status: "same"
477
+ ...out,
478
+ status: out.base === out.head ? "same" : "gate"
384
479
  };
385
480
  }
386
- /** Each eval's row, then each family's that either run gated. */
387
- function compare(base, head) {
388
- const graderChanged = stackChanges(base, head).some((c) => c.outsideFingerprint);
389
- const rows = evalRows(base, head, graderChanged);
390
- const before = new Map((base.families ?? []).map((f) => [f.id, f]));
391
- const after = new Map((head.families ?? []).map((f) => [f.id, f]));
392
- const families = [.../* @__PURE__ */ new Set([...after.keys(), ...before.keys()])].filter((id) => gated(before.get(id)) || gated(after.get(id)));
393
- const runs = {
394
- base,
395
- head
396
- };
397
- const ctx = {
398
- rows: new Map(rows.map((r) => [r.id, r])),
399
- graderChanged,
400
- canceledIn: (side, caseIds) => runs[side].evals.some((e) => caseIds.includes(e.id) && e.cells.some((c) => (c.canceled ?? 0) > 0))
401
- };
402
- return [...rows, ...families.map((id) => familyRow(id, before.get(id), after.get(id), ctx))];
403
- }
404
- /** The key that tells a family's row from an eval's, whose ids can coincide. */
405
- const rowKey = (row) => row.kind === "family" ? `family\0${row.id}` : row.id;
406
- function evalRows(base, head, graderChanged) {
407
- return [.../* @__PURE__ */ new Set([...head.evals.map((e) => e.id), ...base.evals.map((e) => e.id)])].map((id) => {
408
- const b = base.evals.find((e) => e.id === id);
409
- const h = head.evals.find((e) => e.id === id);
481
+ function evalRows(baseEvals, headEvals, graderChanged) {
482
+ return [.../* @__PURE__ */ new Set([...headEvals.map((e) => e.id), ...baseEvals.map((e) => e.id)])].map((id) => {
483
+ const b = baseEvals.find((e) => e.id === id);
484
+ const h = headEvals.find((e) => e.id === id);
410
485
  if (!b) return addedRow(h);
411
- if (!h) return {
412
- id,
413
- base: b.outcome,
414
- head: null,
415
- status: "removed",
416
- ...canceledSides({
417
- base: b.outcome,
418
- head: null
419
- }, { base: b.cells })
420
- };
421
- const comparable = sameEval(b, h);
486
+ if (!h) return removedRow(b);
487
+ const withCases = !!(b.cases || h.cases);
488
+ const comparable = withCases || sameEval(b, h);
422
489
  const scoped = comparable ? null : sharedScope(b, h);
423
490
  const outcomes = scoped ? {
424
491
  base: scoped.base,
@@ -434,19 +501,22 @@ function evalRows(base, head, graderChanged) {
434
501
  ...canceledSides(outcomes, scoped?.over ?? {
435
502
  base: b.cells,
436
503
  head: h.cells
437
- })
504
+ }),
505
+ ...verdictOf(h)
438
506
  };
439
507
  const perCell = compareCells(b, h);
440
508
  if (perCell.length) row.perCell = perCell;
441
- if (row.base === "infra_error" || row.head === "infra_error") return {
442
- ...row,
443
- status: "infra"
444
- };
445
- if (!comparable && !scoped) return {
446
- ...row,
447
- status: "drift",
448
- changes: whyChanged(b, h)
449
- };
509
+ if (!withCases) {
510
+ if (row.base === "infra_error" || row.head === "infra_error") return {
511
+ ...row,
512
+ status: "infra"
513
+ };
514
+ if (!comparable && !scoped) return {
515
+ ...row,
516
+ status: "drift",
517
+ changes: whyChanged(b, h)
518
+ };
519
+ }
450
520
  const over = scoped?.over ?? {
451
521
  base: b.cells ?? [],
452
522
  head: h.cells ?? []
@@ -456,6 +526,7 @@ function evalRows(base, head, graderChanged) {
456
526
  changes: ["gate changed"],
457
527
  gate
458
528
  });
529
+ if (withCases) return casesRow(b, h, graderChanged, row);
459
530
  const from = gate?.rescored ?? row.base;
460
531
  if (row.canceled) return {
461
532
  ...row,
@@ -479,426 +550,6 @@ function evalRows(base, head, graderChanged) {
479
550
  };
480
551
  });
481
552
  }
482
- // src/view/effort.ts
483
- /** Below this two-sided p, an effort difference is an effect. */
484
- const EFFORT_ALPHA = .05;
485
- /** Some counted attempts reported no usage. Not a verdict; never printed as one. */
486
- const TOKENS_UNREPORTED = "some cells did not report usage";
487
- const effortSample = (a) => ({
488
- tools: a.activity.tools,
489
- agentMs: a.timings.agentMs,
490
- fileEdits: a.activity.fileEdits,
491
- filesChanged: filesChanged(a.changedFiles),
492
- ...a.usage ? { tokens: totalTokens(a.usage) } : {}
493
- });
494
- /** The files the workspace diff changed, a collapsed directory counting each of its files. */
495
- const filesChanged = (files) => files.reduce((n, f) => n + (f.count ?? 1), 0);
496
- const counted$1 = (a) => a.outcome === "pass" || a.outcome === "fail";
497
- const spread = (values) => ({
498
- median: median(values) ?? 0,
499
- q1: quantile(values, .25) ?? 0,
500
- q3: quantile(values, .75) ?? 0
501
- });
502
- /** Token totals from the attempts that reported usage; empty when none did. */
503
- const tokenValues = (samples) => samples.flatMap((s) => s.tokens === void 0 ? [] : [s.tokens]);
504
- function effortOf(samples) {
505
- if (samples.length === 0) return null;
506
- const tokens = tokenValues(samples);
507
- return {
508
- n: samples.length,
509
- tools: spread(samples.map((s) => s.tools)),
510
- agentMs: spread(samples.map((s) => s.agentMs)),
511
- fileEdits: median(samples.map((s) => s.fileEdits)) ?? 0,
512
- filesChanged: median(samples.map((s) => s.filesChanged)) ?? 0,
513
- ...tokens.length ? { tokens: spread(tokens) } : {}
514
- };
515
- }
516
- /**
517
- * The verdict of an exact rank test of `xs` (variant) against `ys` (control). Its direction is
518
- * the test's (which side ranks lower), not the medians', which can be equal or disagree.
519
- */
520
- function effortVerdict(xs, ys) {
521
- const test = mannWhitney(xs, ys);
522
- const p = test?.p ?? null;
523
- if (test === null || test.floor >= .05) return {
524
- p,
525
- verdict: "insufficient"
526
- };
527
- if (test.p >= .05) return {
528
- p,
529
- verdict: "noise"
530
- };
531
- return {
532
- p,
533
- verdict: test.sign < 0 ? "fewer" : "more"
534
- };
535
- }
536
- const ratio = (a, b) => b === 0 ? null : a / b;
537
- /** Every value on one side is below every value on the other. */
538
- const apart = (xs, ys) => Math.max(...xs) < Math.min(...ys) || Math.min(...xs) > Math.max(...ys);
539
- /**
540
- * The fewest attempts a side for "no overlap" to say anything: at 3 against 3 one ordering in
541
- * ten separates by chance; at 1 against 1 every pair of unequal values does.
542
- */
543
- const DIRECTIONAL_MIN = 3;
544
- /**
545
- * The fewest runs per cell whose effort test can reach `EFFORT_ALPHA` (4 against 4: p = 2/70).
546
- */
547
- const EFFORT_MIN_RUNS = (() => {
548
- let n = 1;
549
- while (minimumP(n, n) >= EFFORT_ALPHA) n++;
550
- return n;
551
- })();
552
- /**
553
- * The metrics whose test is `insufficient` but whose ranges don't overlap at
554
- * `DIRECTIONAL_MIN` a side or more: a direction, not an effect.
555
- */
556
- function directional(c) {
557
- if (Math.min(c.variant.n, c.control.n) < 3) return [];
558
- const out = [];
559
- if (c.separated && c.verdict === "insufficient") out.push("tools");
560
- if (c.timeSeparated && c.timeVerdict === "insufficient") out.push("agentS");
561
- if (c.tokensSeparated && c.tokensVerdict === "insufficient") out.push("tokens");
562
- return out;
563
- }
564
- function contrastEffort(variant, control) {
565
- const v = effortOf(variant);
566
- const c = effortOf(control);
567
- if (!v || !c) return null;
568
- const tools = effortVerdict(variant.map((s) => s.tools), control.map((s) => s.tools));
569
- const time = effortVerdict(variant.map((s) => s.agentMs), control.map((s) => s.agentMs));
570
- const vTok = tokenValues(variant);
571
- const cTok = tokenValues(control);
572
- const tokens = vTok.length && cTok.length ? effortVerdict(vTok, cTok) : null;
573
- const reported = vTok.length + cTok.length;
574
- const unreported = reported > 0 && reported < variant.length + control.length;
575
- return {
576
- variant: v,
577
- control: c,
578
- toolsRatio: ratio(v.tools.median, c.tools.median),
579
- agentMsRatio: ratio(v.agentMs.median, c.agentMs.median),
580
- agentMsDelta: v.agentMs.median - c.agentMs.median,
581
- toolsP: tools.p,
582
- agentMsP: time.p,
583
- separated: apart(variant.map((s) => s.tools), control.map((s) => s.tools)),
584
- timeSeparated: apart(variant.map((s) => s.agentMs), control.map((s) => s.agentMs)),
585
- tokensSeparated: tokens ? apart(vTok, cTok) : null,
586
- verdict: tools.verdict,
587
- timeVerdict: time.verdict,
588
- tokensRatio: tokens && v.tokens && c.tokens ? ratio(v.tokens.median, c.tokens.median) : null,
589
- tokensP: tokens?.p ?? null,
590
- tokensVerdict: tokens?.verdict ?? null,
591
- ...unreported ? { tokensNote: TOKENS_UNREPORTED } : {}
592
- };
593
- }
594
- /** The counted attempts of one cell of an eval, as effort samples. */
595
- const cellSamples = (e, cellKey) => e.attempts.filter((a) => a.cell === cellKey && counted$1(a)).map(effortSample);
596
- const cellEffort = (e, cellKey) => effortOf(cellSamples(e, cellKey));
597
- /** A variant cell's effort against its control's, in the same run; null for controls. */
598
- function cellContrast(e, cellKey) {
599
- const cell = e.cells.find((c) => c.key === cellKey);
600
- if (!cell?.lift) return null;
601
- return contrastEffort(cellSamples(e, cellKey), cellSamples(e, cell.lift.vs));
602
- }
603
- /** `4`, `4.5`: medians of counts are whole or halves. */
604
- const formatCount = (n) => Number.isInteger(n) ? String(n) : n.toFixed(1);
605
- const formatRatio = (r) => r === null ? "×–" : `×${r.toFixed(2)}`;
606
- /** Below this, agent time rounds under a tenth of a second and prints `<0.1s`. */
607
- const TENTH_MS = 50;
608
- /** Two agent times that both print `<0.1s` have no ratio worth printing. */
609
- const bothUnderATenth = (a, b) => a < TENTH_MS && b < TENTH_MS;
610
- /** Seconds for agent time: `<0.1s`, `0.4s`, `21s`, `1m 05s`. */
611
- function formatSeconds(ms) {
612
- if (ms < 9950) return ms < TENTH_MS ? "<0.1s" : `${(ms / 1e3).toFixed(1)}s`;
613
- const s = Math.round(ms / 1e3);
614
- return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${String(s % 60).padStart(2, "0")}s`;
615
- }
616
- const formatP = (p) => p === null ? "" : ` (p = ${p < .001 ? "<0.001" : p.toFixed(3)})`;
617
- const VERDICT_TEXT = {
618
- fewer: (what) => `fewer ${what}`,
619
- more: (what) => `more ${what}`,
620
- noise: () => "within noise",
621
- insufficient: () => "too few attempts to test"
622
- };
623
- const toolsVerdictText = (c) => c.verdict === "fewer" || c.verdict === "more" ? `${VERDICT_TEXT[c.verdict]("tool calls")}${formatP(c.toolsP)}` : VERDICT_TEXT[c.verdict]("");
624
- /** Token verdict after the tools one; empty when omitted or already covered as insufficient. */
625
- const tokensVerdictText = (c) => {
626
- if (c.tokensVerdict === null) return "";
627
- if (c.verdict === "insufficient" && c.tokensVerdict === "insufficient") return "";
628
- if (c.tokensVerdict === "fewer" || c.tokensVerdict === "more") return `; ${VERDICT_TEXT[c.tokensVerdict]("tokens")}${formatP(c.tokensP)}`;
629
- if (c.tokensVerdict === "noise") return "; tokens within noise";
630
- return `; ${VERDICT_TEXT[c.tokensVerdict]("")}`;
631
- };
632
- /**
633
- * `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
634
- * against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
635
- * when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
636
- * A metric too small to test whose ranges don't overlap says so: `(×0.37, no overlap)`. Agent
637
- * times both under a tenth of a second have no ratio: `agent <0.1s vs <0.1s`.
638
- */
639
- function effortLabel(c) {
640
- const [va, ca] = [c.variant.agentMs.median, c.control.agentMs.median];
641
- const tiny = bothUnderATenth(va, ca);
642
- const apartOn = new Set(directional(c).filter((m) => !(tiny && m === "agentS")));
643
- const r = (metric, value) => `(${formatRatio(value)}${apartOn.has(metric) ? ", no overlap" : ""})`;
644
- const parts = [`tools ${formatCount(c.variant.tools.median)} vs ${formatCount(c.control.tools.median)} ${r("tools", c.toolsRatio)}`, `agent ${formatSeconds(va)} vs ${formatSeconds(ca)}${tiny ? "" : ` ${r("agentS", c.agentMsRatio)}`}`];
645
- if (c.tokensVerdict !== null && c.variant.tokens && c.control.tokens) parts.push(`tokens ${formatTokens(c.variant.tokens.median)} vs ${formatTokens(c.control.tokens.median)} ${r("tokens", c.tokensRatio)}`);
646
- const n = apartOn.size ? ` (n = ${c.variant.n} vs ${c.control.n})` : "";
647
- return `${parts.join(", ")}: ${toolsVerdictText(c)}${tokensVerdictText(c)}${n}`;
648
- }
649
- /** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
650
- const effortSummary = (e) => e.tokens ? `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)} · tokens ${formatTokens(e.tokens.median)}` : `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)}`;
651
- /** The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`. */
652
- function effortChangeLabel(base, head) {
653
- const t = ratio(head.tools.median, base.tools.median);
654
- const a = ratio(head.agentMs.median, base.agentMs.median);
655
- const [ba, ha] = [base.agentMs.median, head.agentMs.median];
656
- const agent = `agent ${formatSeconds(ba)} → ${formatSeconds(ha)}${bothUnderATenth(ba, ha) ? "" : ` (${formatRatio(a)})`}`;
657
- const core = `tools ${formatCount(base.tools.median)} → ${formatCount(head.tools.median)} (${formatRatio(t)}) · ${agent}`;
658
- if (!base.tokens || !head.tokens) return core;
659
- const tok = ratio(head.tokens.median, base.tokens.median);
660
- return `${core} · tokens ${formatTokens(base.tokens.median)} → ${formatTokens(head.tokens.median)} (${formatRatio(tok)})`;
661
- }
662
- // src/view/judgments.ts
663
- /** Show each key until this many; then the minimum score plus a count. */
664
- const COLLAPSE_AFTER = 2;
665
- const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
666
- /** Short names stay at this many characters; a collision takes the distinctive tail instead. */
667
- const LABEL_MAX = 20;
668
- const judgeText = (question) => question.replace(/\?+$/, "").trim();
669
- /** Last `max` characters, cut at a word boundary where there is one, marked with a leading `…`. */
670
- function clipTail(text, max) {
671
- if (text.length <= max) return text;
672
- const cut = text.slice(-(max - 1));
673
- const space = cut.search(/\s/);
674
- return `…${(space >= 0 && space < max / 2 ? cut.slice(space + 1) : cut).trimStart()}`;
675
- }
676
- /** Short name for a judge question, e.g. `login` from `login?`. */
677
- function judgeLabel(question) {
678
- return clip(judgeText(question), LABEL_MAX);
679
- }
680
- /**
681
- * One short label per key. Shared prefixes take the distinctive tail so
682
- * `Does this change…` does not collapse several checks; a tail that still
683
- * collides falls back to the key.
684
- */
685
- function uniqueJudgeLabels(items) {
686
- const first = /* @__PURE__ */ new Map();
687
- for (const item of items) if (!first.has(item.key)) first.set(item.key, item);
688
- const labels = /* @__PURE__ */ new Map();
689
- const used = /* @__PURE__ */ new Map();
690
- const setLabel = (key, label) => {
691
- const prev = labels.get(key);
692
- if (prev !== void 0) {
693
- const group = used.get(prev);
694
- if (group) {
695
- const next = group.filter((k) => k !== key);
696
- if (next.length) used.set(prev, next);
697
- else used.delete(prev);
698
- }
699
- }
700
- labels.set(key, label);
701
- const group = used.get(label) ?? [];
702
- group.push(key);
703
- used.set(label, group);
704
- };
705
- for (const row of first.values()) setLabel(row.key, judgeLabel(row.question));
706
- for (const [label, keys] of [...used]) {
707
- if (keys.length < 2) continue;
708
- const tails = /* @__PURE__ */ new Map();
709
- for (const key of keys) {
710
- const row = first.get(key);
711
- if (!row) continue;
712
- const tail = clipTail(judgeText(row.question), LABEL_MAX);
713
- const group = tails.get(tail) ?? [];
714
- group.push(key);
715
- tails.set(tail, group);
716
- }
717
- for (const [tail, tkeys] of tails) {
718
- const [only] = tkeys;
719
- const taken = (used.get(tail) ?? []).some((k) => !tkeys.includes(k));
720
- if (only && tkeys.length === 1 && tail !== label && !taken) setLabel(only, tail);
721
- else for (const key of tkeys) setLabel(key, key);
722
- }
723
- }
724
- for (const [, keys] of [...used]) {
725
- if (keys.length < 2) continue;
726
- for (const key of keys) setLabel(key, key);
727
- }
728
- return labels;
729
- }
730
- function keyTallies(e, cellKey) {
731
- const byKey = /* @__PURE__ */ new Map();
732
- const expectLabels = /* @__PURE__ */ new Map();
733
- for (const a of e.attempts) {
734
- if (a.cell !== cellKey || !counted(a)) continue;
735
- for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
736
- const row = byKey.get(j.key) ?? {
737
- label: "",
738
- labelled: false,
739
- question: j.question,
740
- pass: 0,
741
- counted: 0
742
- };
743
- row.counted++;
744
- if (j.verdict === "pass") row.pass++;
745
- byKey.set(j.key, row);
746
- if (labeled && !expectLabels.has(j.key)) expectLabels.set(j.key, label);
747
- }
748
- }
749
- const labels = uniqueJudgeLabels([...byKey].map(([key, row]) => ({
750
- key,
751
- question: row.question
752
- })));
753
- for (const [key, row] of byKey) {
754
- const label = expectLabels.get(key);
755
- row.labelled = label !== void 0;
756
- row.label = label ?? labels.get(key) ?? judgeLabel(row.question);
757
- }
758
- return byKey;
759
- }
760
- function pairedControl(e, cellKey) {
761
- const cell = e.cells.find((c) => c.key === cellKey);
762
- if (cell?.role !== "variant") return void 0;
763
- return e.cells.find((x) => x.role === "control" && x.harness === cell.harness && x.sandbox === cell.sandbox);
764
- }
765
- function liftForKey(variant, control) {
766
- if (!control || variant.counted < 4 || control.counted < 4) return null;
767
- const { points, low, high } = liftInterval({
768
- pass: variant.pass,
769
- n: variant.counted
770
- }, {
771
- pass: control.pass,
772
- n: control.counted
773
- });
774
- return {
775
- points: round3(points),
776
- control: {
777
- pass: control.pass,
778
- counted: control.counted
779
- },
780
- variant: {
781
- pass: variant.pass,
782
- counted: variant.counted
783
- },
784
- withinNoise: containsZero(low, high)
785
- };
786
- }
787
- /** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides qualify. */
788
- function cellJudgeRates(e, cellKey) {
789
- const tallies = keyTallies(e, cellKey);
790
- const control = pairedControl(e, cellKey);
791
- const controlTallies = control ? keyTallies(e, control.key) : void 0;
792
- return [...tallies].map(([key, row]) => ({
793
- key,
794
- ...row,
795
- lift: liftForKey(row, controlTallies?.get(key))
796
- }));
797
- }
798
- /**
799
- * The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
800
- * of the cell's checks against its control is already there.
801
- */
802
- function summaryJudgeRates(e, cell) {
803
- const checked = new Set((cell.checks ?? []).filter((k) => k.control && k.verdict).map((k) => k.label));
804
- return cellJudgeRates(e, cell.key).filter((r) => !(r.labelled && checked.has(r.label)));
805
- }
806
- const signedPoints = (x) => `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
807
- /** `login 4/5 · checkout 0/5`, with `(+80 pts)` on a key that has lift. */
808
- function judgeRatesLabel(rates) {
809
- return rates.map((r) => {
810
- const n = `${r.label} ${r.pass}/${r.counted}`;
811
- return r.lift ? `${n} (${signedPoints(r.lift.points)} pts)` : n;
812
- }).join(" · ");
813
- }
814
- /** Each key, or the lowest score plus how many judges there were. */
815
- function collapsedJudges(judgments) {
816
- const labels = uniqueJudgeLabels(judgments);
817
- const items = judgments.map((j) => ({
818
- verdict: j.verdict,
819
- score: j.score,
820
- threshold: j.threshold,
821
- label: labels.get(j.key) ?? judgeLabel(j.question),
822
- key: j.key
823
- }));
824
- if (items.length <= COLLAPSE_AFTER) return {
825
- shown: items,
826
- extra: 0
827
- };
828
- return {
829
- shown: [items.reduce((a, b) => a.score <= b.score ? a : b)],
830
- extra: items.length
831
- };
832
- }
833
- /** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
834
- function collapsedJudgeLabel(judgments) {
835
- if (!judgments.length) return null;
836
- const { shown, extra } = collapsedJudges(judgments);
837
- const parts = shown.map((j) => `${j.verdict} ${j.score.toFixed(2)}/${j.threshold.toFixed(2)}`);
838
- if (extra) parts.push(`${extra} judges`);
839
- return parts.join(" · ");
840
- }
841
- const JUDGE_ERROR = /^judge failed for /;
842
- /**
843
- * Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
844
- * failed judge call, which recorded no judgment) in call order, when they are exactly as many as
845
- * its judgments. `check` is that assertion's label, the key of its check; a judgment no label
846
- * reaches is named by its question.
847
- */
848
- function labeledJudgments(a) {
849
- const passes = (a.assertions ?? []).filter((x) => /(^|\.)toPass$/.test(x.matcher) && !JUDGE_ERROR.test(x.message ?? ""));
850
- const zip = passes.length === a.judgements.length;
851
- return a.judgements.map((judgment, i) => {
852
- const x = zip ? passes[i] : void 0;
853
- return {
854
- judgment,
855
- label: x?.labelled ? x.label : judgeText(judgment.question),
856
- labelled: x?.labelled === true,
857
- ...x ? { check: x.label } : {}
858
- };
859
- });
860
- }
861
- /** Sum of counted attempts' usage; null when none of them reported any. */
862
- function cellUsage(e, cellKey) {
863
- return sumUsage(e.attempts.filter((a) => a.cell === cellKey && counted(a)).map((a) => a.usage));
864
- }
865
- /**
866
- * Each `c.exec` check's `expect` label, by position: the attempt's `toSucceed` assertions in call
867
- * order, when they are exactly as many as its checks; undefined where no label reaches.
868
- */
869
- function checkLabels(a) {
870
- const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
871
- const zip = succeeds.length === a.checks.length;
872
- return a.checks.map((_, i) => {
873
- const x = zip ? succeeds[i] : void 0;
874
- return x?.labelled ? x.label : void 0;
875
- });
876
- }
877
- /**
878
- * Each `toSucceed` label's `c.exec` check, by position, when the attempt's `toSucceed` assertions
879
- * are exactly as many as its checks; empty otherwise.
880
- */
881
- function checksByLabel(a) {
882
- const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
883
- if (succeeds.length !== a.checks.length) return /* @__PURE__ */ new Map();
884
- return new Map(succeeds.map((x, i) => [x.label, a.checks[i]]));
885
- }
886
- /**
887
- * How a `c.exec` given `retries` ended: `passed on try 2 of 3`, `failed all 3 tries`, or null
888
- * when it had none or passed on its first try.
889
- */
890
- function triesNote(exitCode, tried, of) {
891
- if (of <= 1 || exitCode === 0 && tried <= 1) return null;
892
- if (exitCode === 0) return `passed on try ${tried} of ${of}`;
893
- return tried >= of ? `failed all ${of} tries` : `failed on try ${tried} of ${of}`;
894
- }
895
- /** {@link triesNote} for a recorded check. */
896
- const checkTries = (c) => c.retries && c.tries ? triesNote(c.exitCode, c.tries.length, c.retries + 1) : null;
897
- /** A check's command on one line: the first non-empty line, clipped; its log has the rest. */
898
- function commandLine(command, max = 80) {
899
- const lines = command.split("\n").map((l) => l.trim()).filter(Boolean);
900
- return clip(lines.length > 1 ? `${lines[0]} …` : lines[0] ?? "", max);
901
- }
902
553
  // src/view/model.ts
903
554
  /** Splits `file › group › … › name` back into its parts. */
904
555
  function splitEvalId(id) {
@@ -1006,28 +657,31 @@ const shortSha = (sha) => sha ? sha.slice(0, 7) : "no git";
1006
657
  /** Reasoning time, or `n/a` when the harness couldn't measure it: never `0s` for unknown. */
1007
658
  const reasoningLabel = (ms) => ms === null ? "n/a" : formatDuration(ms);
1008
659
  /**
1009
- * An attempt's effort, counted as the CLI's turn lines count it: tool calls, then file edits.
1010
- * Edit calls become one file edit per file they change, so they're counted apart, never summed.
1011
- * With the workspace diff's count, `files changed` follows: an edit made from the shell is in it,
1012
- * never in the file edits.
660
+ * An attempt's effort: tool calls, then the files the workspace diff changed, which counts a file
661
+ * the shell wrote, then the calls to the agent's edit tools, which doesn't.
1013
662
  */
1014
663
  function activityLabel(a, changed) {
1015
- const files = changed === void 0 ? "" : ` · ${plural(changed, "file")} changed`;
1016
- return `${plural(a.tools, "tool call")} · ${plural(a.fileEdits, "file edit")}${files}`;
664
+ return `${plural(a.tools, "tool call")} · ${plural(changed, "file")} changed · ${plural(a.fileEdits, "edit tool call")}`;
1017
665
  }
1018
666
  /**
1019
667
  * A run's outcome, which `reeval run`'s exit code is read from: an error outside any attempt, a
1020
668
  * failed gated cell or a budget stop fails the run, while failures in ungated cells (controls, by
1021
- * default) don't. An interrupted run is `skipped`, whatever ran.
669
+ * default) don't. An interrupted run is `skipped`, whatever ran; a resume its filters left
670
+ * `interrupted` exits on its outcome as if it were complete.
1022
671
  */
1023
672
  function runOutcome(run) {
1024
673
  if (run.status === "interrupted") return "skipped";
1025
674
  if (run.errors?.length) return "fail";
1026
- const outcomes = [...run.evals, ...run.families ?? []].map((e) => e.outcome);
675
+ const outcomes = run.evals.map((e) => e.outcome);
1027
676
  if (outcomes.includes("fail") || run.status === "budget") return "fail";
1028
677
  if (outcomes.includes("infra_error")) return "infra_error";
1029
678
  return "pass";
1030
679
  }
680
+ /**
681
+ * The eval hasn't run yet: its run stopped (interrupted, or out of budget) or a filtered resume
682
+ * left it out, and it's `skipped` without being a todo or declared skipped. A resume runs it.
683
+ */
684
+ const notRunYet = (run, e) => run.status !== "complete" && e.outcome === "skipped" && !e.todo && !e.skip;
1031
685
  function andList(items) {
1032
686
  if (items.length <= 1) return items[0] ?? "";
1033
687
  if (items.length === 2) return `${items[0]} and ${items[1]}`;
@@ -1325,7 +979,7 @@ function gatedPassAt1(cells) {
1325
979
  /** `reeval compare`'s classification, plus pass@1 and judge-score deltas from the records. */
1326
980
  function compareView(base, head, records = {}) {
1327
981
  const rows = compare(base, head).map((row) => {
1328
- const evalOf = (run) => row.kind === "family" ? void 0 : run.evals.find((e) => e.id === row.id);
982
+ const evalOf = (run) => run.evals.find((e) => e.id === row.id);
1329
983
  const b = evalOf(base);
1330
984
  const h = evalOf(head);
1331
985
  const baseKeys = new Set(b?.cells.map((c) => c.key));
@@ -1388,6 +1042,6 @@ const findEval = (record, slug) => record.evals.find((e) => e.slug === slug);
1388
1042
  const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId);
1389
1043
  /** The attempt to open by default: the first failure, else the first attempt. */
1390
1044
  const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
1391
- export { bothUnderATenth as $, runtimeLabel as A, collapsedJudges as B, isUngatedMiss as C, whyChanged as Ct, runLead as D, reasoningLabel as E, cellUsage as F, summaryJudgeRates as G, judgeLabel as H, checkLabels as I, DIRECTIONAL_MIN as J, triesNote as K, checkTries as L, splitEvalId as M, transcriptItems as N, runOutcome as O, cellJudgeRates as P, VERDICT_TEXT as Q, checksByLabel as R, isNotGatedPass as S, stackChanges as St, parseTranscript as T, judgeRatesLabel as U, commandLine as V, labeledJudgments as W, EFFORT_MIN_RUNS as X, EFFORT_ALPHA as Y, TOKENS_UNREPORTED as Z, formatUsage as _, cellsChanged as _t, buildIndex as a, effortChangeLabel as at, indexFiles as b, rowKey as bt, defaultAttempt as c, effortSample as ct, evalTree as d, filesChanged as dt, cellContrast as et, failedJudgeLabels as f, formatCount as ft, formatPercent as g, addedRow as gt, formatDuration as h, tokenValues as ht, attemptAnswerRows as i, directional as it, shortSha as j, runRows as k, evalHistory as l, effortSummary as lt, findEval as m, formatSeconds as mt, activityLabel as n, cellSamples as nt, cellRows as o, effortLabel as ot, findAttempt as p, formatRatio as pt, uniqueJudgeLabels as q, answerRows as r, contrastEffort as rt, compareView as s, effortOf as st, COMPARE_ORDER as t, cellEffort as tt, evalMarkOutcome as u, effortVerdict as ut, harnessLabel as v, compare as vt, languageLabel as w, isMatrix as x, sourceChanges as xt, hasUngatedCells as y, newRows as yt, collapsedJudgeLabel as z };
1045
+ export { runRows as A, stackChanges as B, isUngatedMiss as C, reasoningLabel as D, parseTranscript as E, addedRow as F, cellsChanged as I, compare as L, shortSha as M, splitEvalId as N, runLead as O, transcriptItems as P, newRows as R, isNotGatedPass as S, notRunYet as T, whyChanged as V, formatUsage as _, buildIndex as a, indexFiles as b, defaultAttempt as c, evalTree as d, failedJudgeLabels as f, formatPercent as g, formatDuration as h, attemptAnswerRows as i, runtimeLabel as j, runOutcome as k, evalHistory as l, findEval as m, activityLabel as n, cellRows as o, findAttempt as p, answerRows as r, compareView as s, COMPARE_ORDER as t, evalMarkOutcome as u, harnessLabel as v, languageLabel as w, isMatrix as x, hasUngatedCells as y, sourceChanges as z };
1392
1046
 
1393
- //# sourceMappingURL=model-DOiCiMCD.js.map
1047
+ //# sourceMappingURL=model-V3mMXp6x.js.map