reeval 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (380) hide show
  1. package/README.md +1 -1
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-CptK032x.js → adapter-C60z3wLq.js} +5 -4
  4. package/dist/adapter-C60z3wLq.js.map +1 -0
  5. package/dist/{agent-B63OzUX9.js → agent-BmsMXZXx.js} +30 -10
  6. package/dist/{agent-B63OzUX9.js.map → agent-BmsMXZXx.js.map} +1 -1
  7. package/dist/agents-DXuFxtGC.js +1151 -0
  8. package/dist/agents-DXuFxtGC.js.map +1 -0
  9. package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
  10. package/dist/app-files-DT5mJyV4.js.map +1 -0
  11. package/dist/assemble-CE4pHKIa.js +1 -0
  12. package/dist/{assemble-Me7Whp-y.js → assemble-DM2PNU4R.js} +152 -129
  13. package/dist/assemble-DM2PNU4R.js.map +1 -0
  14. package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
  15. package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
  16. package/dist/{attempt-host-Cik0SoIn.js → attempt-host-7VviAJrl.js} +6 -6
  17. package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-7VviAJrl.js.map} +1 -1
  18. package/dist/boat-BjLqFJIZ.js +108 -0
  19. package/dist/boat-BjLqFJIZ.js.map +1 -0
  20. package/dist/boat-DSNgZt-t.js +322 -0
  21. package/dist/boat-DSNgZt-t.js.map +1 -0
  22. package/dist/bridge/index.mjs +401 -79
  23. package/dist/capture-BzotD2hs.js +1 -0
  24. package/dist/{capture-C3RmjPph.js → capture-Ccf0Hmyx.js} +27 -25
  25. package/dist/capture-Ccf0Hmyx.js.map +1 -0
  26. package/dist/cases-Dy0mifal.js +1 -0
  27. package/dist/cases-z9E6DCd5.js +639 -0
  28. package/dist/cases-z9E6DCd5.js.map +1 -0
  29. package/dist/{cli-BBOlXKPG.js → cli-DTuLSlmW.js} +442 -707
  30. package/dist/cli-DTuLSlmW.js.map +1 -0
  31. package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
  32. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  33. package/dist/cli.js +2 -2
  34. package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
  35. package/dist/client-KM4C5r5_.js.map +1 -0
  36. package/dist/{command-DhiwsZl1.js → command-C2C7P5qo.js} +9 -9
  37. package/dist/{command-DhiwsZl1.js.map → command-C2C7P5qo.js.map} +1 -1
  38. package/dist/{command-CFQ9c1hk.js → command-D8oMB2fa.js} +16 -16
  39. package/dist/command-D8oMB2fa.js.map +1 -0
  40. package/dist/{command-fO0BFsTq.js → command-DSL7I-SI.js} +37 -37
  41. package/dist/command-DSL7I-SI.js.map +1 -0
  42. package/dist/{command-qGS802uX.js → command-Ff0T-1Hi.js} +21 -80
  43. package/dist/command-Ff0T-1Hi.js.map +1 -0
  44. package/dist/{commands-xYTMNTPv.js → commands-Bk9NG1s_.js} +1 -1
  45. package/dist/{commands-CC60TOT5.js → commands-xAOi_VEk.js} +186 -85
  46. package/dist/commands-xAOi_VEk.js.map +1 -0
  47. package/dist/{config-C2UtS9vy.js → config-SvK8pjEK.js} +261 -1708
  48. package/dist/config-SvK8pjEK.js.map +1 -0
  49. package/dist/{context-BajfiH0S.js → context-BdgCao9Q.js} +3 -2
  50. package/dist/context-BdgCao9Q.js.map +1 -0
  51. package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
  52. package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
  53. package/dist/cost-Cr26Alhq.js +1 -0
  54. package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-DEzan4IT.js} +3 -3
  55. package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-DEzan4IT.js.map} +1 -1
  56. package/dist/{db-C5SZ0510.js → db-BzOJ4Js4.js} +130 -84
  57. package/dist/db-BzOJ4Js4.js.map +1 -0
  58. package/dist/db-CQRZ3URX.js +2 -0
  59. package/dist/debug-CF4p4Lp_.js +1 -0
  60. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  61. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  62. package/dist/{default-effort-FEidNAIi.js → default-effort-BBDPbJ4D.js} +1 -1
  63. package/dist/{default-effort-BVmx7r7L.js → default-effort-BDKIxJyC.js} +4 -4
  64. package/dist/{default-effort-BVmx7r7L.js.map → default-effort-BDKIxJyC.js.map} +1 -1
  65. package/dist/{dist-ByYk_wAs.js → dist-Db7ifchA.js} +37 -8
  66. package/dist/dist-Db7ifchA.js.map +1 -0
  67. package/dist/{docker-BlSopHpZ.js → docker--mukta3L.js} +1 -1
  68. package/dist/{docker-B2F8-PGC.js → docker-DFrjoNs-.js} +3 -3
  69. package/dist/{docker-B2F8-PGC.js.map → docker-DFrjoNs-.js.map} +1 -1
  70. package/dist/{docker-Dto8ROCq.js → docker-DG_uFgn2.js} +109 -77
  71. package/dist/docker-DG_uFgn2.js.map +1 -0
  72. package/dist/{doctor-BcvfKpBX.js → doctor-DWK4ttex.js} +84 -27
  73. package/dist/doctor-DWK4ttex.js.map +1 -0
  74. package/dist/{environment-CC1RV21n.js → environment-cPMCOk-h.js} +79 -39
  75. package/dist/environment-cPMCOk-h.js.map +1 -0
  76. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  77. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  78. package/dist/{eval-source-CzxdgFBW.js → eval-source-C9b1h8L6.js} +10 -10
  79. package/dist/{eval-source-CzxdgFBW.js.map → eval-source-C9b1h8L6.js.map} +1 -1
  80. package/dist/{evaluate-NTNJ8T9H.js → evaluate-B12Tosvq.js} +377 -142
  81. package/dist/evaluate-B12Tosvq.js.map +1 -0
  82. package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
  83. package/dist/{files-aAzoGNCB.js → files-BOBdC0uI.js} +5 -5
  84. package/dist/{files-aAzoGNCB.js.map → files-BOBdC0uI.js.map} +1 -1
  85. package/dist/files-C1Rk9FYA.js +1 -0
  86. package/dist/files-DMQ9-Vv5.js +1 -0
  87. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  88. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  89. package/dist/files-Di0xDbCc.js +1 -0
  90. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  91. package/dist/files-DlQshsks.js.map +1 -0
  92. package/dist/{files-DsXULUi5.js → files-F5EG0WPT.js} +5 -5
  93. package/dist/{files-DsXULUi5.js.map → files-F5EG0WPT.js.map} +1 -1
  94. package/dist/files-G5jf8wBp.js +1 -0
  95. package/dist/{files-CZXrkPja.js → files-s28a6czq.js} +54 -13
  96. package/dist/{files-CZXrkPja.js.map → files-s28a6czq.js.map} +1 -1
  97. package/dist/files-tiNWe8Ih.js +64 -0
  98. package/dist/files-tiNWe8Ih.js.map +1 -0
  99. package/dist/gate-CgYdaprG.js +49 -0
  100. package/dist/gate-CgYdaprG.js.map +1 -0
  101. package/dist/gateway.d.ts +1 -1
  102. package/dist/{git-C1MB5bjr.js → git-BpoqEgDV.js} +2 -2
  103. package/dist/{git-C1MB5bjr.js.map → git-BpoqEgDV.js.map} +1 -1
  104. package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
  105. package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
  106. package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
  107. package/dist/harness-DXv2gnaT.js.map +1 -0
  108. package/dist/harness.d.ts +1 -1
  109. package/dist/harness.js +1 -1
  110. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  111. package/dist/ids-C_advdJA.js.map +1 -0
  112. package/dist/{images-CNayXCp2.js → images-D2jFze2v.js} +98 -10
  113. package/dist/images-D2jFze2v.js.map +1 -0
  114. package/dist/index.d.ts +23 -3
  115. package/dist/index.js +5 -5
  116. package/dist/{init-D5HtmyMI.js → init-Bi1Q3lRi.js} +7 -6
  117. package/dist/init-Bi1Q3lRi.js.map +1 -0
  118. package/dist/init-CUw65o36.js +1 -0
  119. package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
  120. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  121. package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
  122. package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
  123. package/dist/judge-COMXT1Hx.js +230 -0
  124. package/dist/judge-COMXT1Hx.js.map +1 -0
  125. package/dist/judge.d.ts +1 -1
  126. package/dist/judge.js +1 -1
  127. package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
  128. package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
  129. package/dist/kinds-Bh2I35id.js +698 -0
  130. package/dist/kinds-Bh2I35id.js.map +1 -0
  131. package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
  132. package/dist/layout-Cm85tQYk.js.map +1 -0
  133. package/dist/list-DuThsE28.js +1348 -0
  134. package/dist/list-DuThsE28.js.map +1 -0
  135. package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
  136. package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
  137. package/dist/{live-uRURFDMZ.js → live-DL0se4Zd.js} +7 -7
  138. package/dist/{live-uRURFDMZ.js.map → live-DL0se4Zd.js.map} +1 -1
  139. package/dist/live-DZLmj0iG.js +1 -0
  140. package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
  141. package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
  142. package/dist/{model-CXVVB4RK.js → model-CkNVAxob.js} +1 -1
  143. package/dist/model-Cv4GuTsZ.js +1 -0
  144. package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
  145. package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
  146. package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
  147. package/dist/model-V3mMXp6x.js.map +1 -0
  148. package/dist/{model-BxBWFOZn.js → model-cgt5uvu8.js} +5 -5
  149. package/dist/{model-BxBWFOZn.js.map → model-cgt5uvu8.js.map} +1 -1
  150. package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
  151. package/dist/node-D7pAU--0.js.map +1 -0
  152. package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
  153. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  154. package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
  155. package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  156. package/dist/options-DPUmk7v_.js +1 -0
  157. package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
  158. package/dist/pairs-BGb134Vu.js.map +1 -0
  159. package/dist/{patch-ug5HpMOj.js → patch-DAi1nCuH.js} +2 -2
  160. package/dist/{patch-ug5HpMOj.js.map → patch-DAi1nCuH.js.map} +1 -1
  161. package/dist/plain-BYUf_fUV.js +267 -0
  162. package/dist/plain-BYUf_fUV.js.map +1 -0
  163. package/dist/{plan-DqnzfDH1.js → plan-CsPilzpk.js} +110 -39
  164. package/dist/plan-CsPilzpk.js.map +1 -0
  165. package/dist/{plugins-B2KQ0y3P.js → plugins-BfnhazOv.js} +10 -267
  166. package/dist/plugins-BfnhazOv.js.map +1 -0
  167. package/dist/{preflight-DyzmvK7i.js → preflight-BixaE4BA.js} +16 -9
  168. package/dist/preflight-BixaE4BA.js.map +1 -0
  169. package/dist/preflight-Bn3ianye.js +1 -0
  170. package/dist/{preflight-DVXEvzFU.js → preflight-D1C-iH0U.js} +1 -1
  171. package/dist/preflight-GkpgrBxP.js +68 -0
  172. package/dist/preflight-GkpgrBxP.js.map +1 -0
  173. package/dist/{prepare-BcSRa7-3.js → prepare-C--rM_wO.js} +2 -2
  174. package/dist/{prepare-21WMQZyt.js → prepare-DQEDXoDp.js} +9 -9
  175. package/dist/{prepare-21WMQZyt.js.map → prepare-DQEDXoDp.js.map} +1 -1
  176. package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
  177. package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
  178. package/dist/{provenance-iRxjp4Pu.js → provenance-BXyYwUC8.js} +4 -3
  179. package/dist/provenance-BXyYwUC8.js.map +1 -0
  180. package/dist/provider-BnUoz4KB.js +1 -0
  181. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  182. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  183. package/dist/{providers-vXMXuwn-.js → providers-1p4HKYIp.js} +152 -9
  184. package/dist/providers-1p4HKYIp.js.map +1 -0
  185. package/dist/providers-B-ijCktf.js +1 -0
  186. package/dist/query-BYhyf0LR.js +1 -0
  187. package/dist/{query-BhaKQ07P.js → query-DhhU9ADI.js} +2 -2
  188. package/dist/{query-BhaKQ07P.js.map → query-DhhU9ADI.js.map} +1 -1
  189. package/dist/{questions-BzdLdV3C.js → questions-CMAq4pr8.js} +9 -9
  190. package/dist/{questions-BzdLdV3C.js.map → questions-CMAq4pr8.js.map} +1 -1
  191. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  192. package/dist/record-6IUzwP9K.js.map +1 -0
  193. package/dist/{remarks-CauU408t.js → remarks-BNABzemt.js} +115 -35
  194. package/dist/remarks-BNABzemt.js.map +1 -0
  195. package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
  196. package/dist/remarks-DPCONFxZ.js.map +1 -0
  197. package/dist/{remote-Dwy3fyo3.js → remote-U7fNt0UP.js} +6 -6
  198. package/dist/{remote-Dwy3fyo3.js.map → remote-U7fNt0UP.js.map} +1 -1
  199. package/dist/resolve-C14Pmtej.js +1 -0
  200. package/dist/{resolve-B1zMnPwK.js → resolve-Cx3a9UQ2.js} +10 -9
  201. package/dist/resolve-Cx3a9UQ2.js.map +1 -0
  202. package/dist/{results-CfC0id_u.js → results-CQKXmHym.js} +6 -6
  203. package/dist/{results-CfC0id_u.js.map → results-CQKXmHym.js.map} +1 -1
  204. package/dist/{resume-Dkhrpjow.js → resume-POpZa1ge.js} +169 -27
  205. package/dist/resume-POpZa1ge.js.map +1 -0
  206. package/dist/retry-CmG809Mp.js +105 -0
  207. package/dist/retry-CmG809Mp.js.map +1 -0
  208. package/dist/{runtime-DcvQcr36.js → runtime-Xm9oeC7o.js} +26 -7
  209. package/dist/runtime-Xm9oeC7o.js.map +1 -0
  210. package/dist/sandbox/boat.d.ts +26 -0
  211. package/dist/sandbox/boat.js +37 -0
  212. package/dist/sandbox/boat.js.map +1 -0
  213. package/dist/{sandbox-8zCrtuCf.js → sandbox-DrlL4yqb.js} +31 -3
  214. package/dist/sandbox-DrlL4yqb.js.map +1 -0
  215. package/dist/sandbox.d.ts +20 -2
  216. package/dist/sandbox.js +1 -1
  217. package/dist/{server-Dplq8g0r.js → server-C00zd6zl.js} +9 -9
  218. package/dist/{server-Dplq8g0r.js.map → server-C00zd6zl.js.map} +1 -1
  219. package/dist/server-C1UJcpUh.js +1 -0
  220. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  221. package/dist/shared-Cfi45Bvy.js.map +1 -0
  222. package/dist/slug-IHh3f770.js +136 -0
  223. package/dist/slug-IHh3f770.js.map +1 -0
  224. package/dist/{source-NWHqn5-o.js → source-DDgr48Gn.js} +2 -2
  225. package/dist/{source-NWHqn5-o.js.map → source-DDgr48Gn.js.map} +1 -1
  226. package/dist/source-DJHmQv_j.js +1 -0
  227. package/dist/src-1dBuXzs4.js +3 -0
  228. package/dist/{src-CUpoVVb8.js → src-BKpioayC.js} +2 -2
  229. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  230. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  231. package/dist/src-BY1cCaoJ.js +2 -0
  232. package/dist/src-BbZYaQ8b.js +2 -0
  233. package/dist/src-CDFH2vVN.js +2 -0
  234. package/dist/src-CMW5R9Q0.js +1 -0
  235. package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
  236. package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
  237. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  238. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  239. package/dist/{src-B9oE_mNS.js → src-CrV2WA2i.js} +2 -2
  240. package/dist/src-CsFaawcZ.js +2 -0
  241. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  242. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  243. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  244. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  245. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  246. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  247. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  248. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  249. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  250. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  251. package/dist/src-wKwznPgt.js +2 -0
  252. package/dist/{src-BQ6BiXoc.js → src-y0nOeRRE.js} +610 -72
  253. package/dist/src-y0nOeRRE.js.map +1 -0
  254. package/dist/{state-DDCGUhva.js → state-328JpnHA.js} +17 -17
  255. package/dist/{state-DDCGUhva.js.map → state-328JpnHA.js.map} +1 -1
  256. package/dist/state-Ca55Gk_O.js +1 -0
  257. package/dist/suite-CKBa6MKt.js +2 -0
  258. package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
  259. package/dist/suite-COglUhPd.js.map +1 -0
  260. package/dist/summary-DDy9DfyJ.js +148 -0
  261. package/dist/summary-DDy9DfyJ.js.map +1 -0
  262. package/dist/tart-cli-C-GsbVsQ.js +1 -0
  263. package/dist/tart-cli-CU0nFS-L.js +55 -0
  264. package/dist/tart-cli-CU0nFS-L.js.map +1 -0
  265. package/dist/tart-dWoJvWHc.js +586 -0
  266. package/dist/tart-dWoJvWHc.js.map +1 -0
  267. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  268. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  269. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  270. package/dist/tart-ref-DYZijjrU.js +55 -0
  271. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  272. package/dist/testing.d.ts +2 -2
  273. package/dist/testing.js +4 -4
  274. package/dist/trace-Dx2NBE1O.js +311 -0
  275. package/dist/trace-Dx2NBE1O.js.map +1 -0
  276. package/dist/traits-DKnYkqeK.js +66 -0
  277. package/dist/traits-DKnYkqeK.js.map +1 -0
  278. package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
  279. package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
  280. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  281. package/dist/types-C2mLpph3.js.map +1 -0
  282. package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
  283. package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
  284. package/dist/vercel-CGlSBv63.js.map +1 -0
  285. package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
  286. package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
  287. package/dist/vertex-O88TCwgY.js +1 -0
  288. package/dist/view/index.d.ts +101 -58
  289. package/dist/view/index.js +9 -8
  290. package/dist/views-B0Yt_Q1S.js +2 -0
  291. package/dist/{views-BISrsIjw.js → views-BROacjBv.js} +7 -6
  292. package/dist/views-BROacjBv.js.map +1 -0
  293. package/dist/{worker-entry-DJD_nHkS.js → worker-entry-Smb4bNDc.js} +11 -8
  294. package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-Smb4bNDc.js.map} +1 -1
  295. package/dist/workspace-C0dDP75l.js +1 -0
  296. package/dist/{workspace-BO1VhUR9.js → workspace-DNDTEk8v.js} +6 -6
  297. package/dist/{workspace-BO1VhUR9.js.map → workspace-DNDTEk8v.js.map} +1 -1
  298. package/package.json +24 -11
  299. package/skills/eval-rubric/SKILL.md +15 -7
  300. package/dist/adapter-CptK032x.js.map +0 -1
  301. package/dist/app-files-sgqwfjtn.js.map +0 -1
  302. package/dist/assemble-CxBHkavM.js +0 -1
  303. package/dist/assemble-Me7Whp-y.js.map +0 -1
  304. package/dist/capture-C3RmjPph.js.map +0 -1
  305. package/dist/capture-YPwxp_iK.js +0 -1
  306. package/dist/cli-BBOlXKPG.js.map +0 -1
  307. package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
  308. package/dist/client-tZ2WBDxf.js.map +0 -1
  309. package/dist/command-CFQ9c1hk.js.map +0 -1
  310. package/dist/command-fO0BFsTq.js.map +0 -1
  311. package/dist/command-qGS802uX.js.map +0 -1
  312. package/dist/commands-CC60TOT5.js.map +0 -1
  313. package/dist/config-C2UtS9vy.js.map +0 -1
  314. package/dist/context-BajfiH0S.js.map +0 -1
  315. package/dist/contrasts-BjOg7wyr.js +0 -101
  316. package/dist/contrasts-BjOg7wyr.js.map +0 -1
  317. package/dist/cost-BJU0MAji.js +0 -1
  318. package/dist/db-C5SZ0510.js.map +0 -1
  319. package/dist/db-puzLzX4k.js +0 -2
  320. package/dist/debug-BnXV7JbX.js +0 -1
  321. package/dist/dist-ByYk_wAs.js.map +0 -1
  322. package/dist/docker-Dto8ROCq.js.map +0 -1
  323. package/dist/doctor-BcvfKpBX.js.map +0 -1
  324. package/dist/environment-CC1RV21n.js.map +0 -1
  325. package/dist/evaluate-NTNJ8T9H.js.map +0 -1
  326. package/dist/files-COW5JBJM.js +0 -1
  327. package/dist/files-DFtuRQJx.js +0 -1
  328. package/dist/files-oOYNR7ZH.js +0 -1
  329. package/dist/files-p-uTdOjA.js.map +0 -1
  330. package/dist/gate-Cp_pQoEe.js.map +0 -1
  331. package/dist/harness-DYSC4zMh.js.map +0 -1
  332. package/dist/ids-DuK6xl4w.js.map +0 -1
  333. package/dist/images-CNayXCp2.js.map +0 -1
  334. package/dist/init-8o6W3_fn.js +0 -1
  335. package/dist/init-D5HtmyMI.js.map +0 -1
  336. package/dist/ink-list-Ca-5dunO.js.map +0 -1
  337. package/dist/judge-DYOF69Oe.js.map +0 -1
  338. package/dist/list-CM_YSJXq.js +0 -374
  339. package/dist/list-CM_YSJXq.js.map +0 -1
  340. package/dist/live-DQc8jr6o.js +0 -1
  341. package/dist/model-Ce9l9_rF.js.map +0 -1
  342. package/dist/model-Cxzo4cjw.js +0 -1
  343. package/dist/node-DhIRSrrK.js.map +0 -1
  344. package/dist/opencode-providers-Co7plCea.js.map +0 -1
  345. package/dist/plan-DqnzfDH1.js.map +0 -1
  346. package/dist/plugins-B2KQ0y3P.js.map +0 -1
  347. package/dist/preflight-DyzmvK7i.js.map +0 -1
  348. package/dist/provenance-iRxjp4Pu.js.map +0 -1
  349. package/dist/provider-DCdTHQpk.js +0 -1
  350. package/dist/providers-D0ob_tra.js +0 -1
  351. package/dist/providers-vXMXuwn-.js.map +0 -1
  352. package/dist/query-CvgUsvep.js +0 -1
  353. package/dist/record-DhzwkSXc.js.map +0 -1
  354. package/dist/remarks-CauU408t.js.map +0 -1
  355. package/dist/remarks-NMgPd30U.js.map +0 -1
  356. package/dist/resolve-B1zMnPwK.js.map +0 -1
  357. package/dist/resolve-BIIeVT12.js +0 -1
  358. package/dist/resume-Dkhrpjow.js.map +0 -1
  359. package/dist/runtime-DcvQcr36.js.map +0 -1
  360. package/dist/sandbox-8zCrtuCf.js.map +0 -1
  361. package/dist/server-0tuj8VSa.js +0 -1
  362. package/dist/shared-ogqPJ0uo.js.map +0 -1
  363. package/dist/source-llwLewOU.js +0 -1
  364. package/dist/src-BQ6BiXoc.js.map +0 -1
  365. package/dist/src-BXnv0ViU.js +0 -2
  366. package/dist/src-BiS8YNN_.js +0 -3
  367. package/dist/src-BkH4GOxu.js +0 -1
  368. package/dist/src-C-tH5ebd.js +0 -2
  369. package/dist/src-Cg1bFs9g.js +0 -2
  370. package/dist/src-DV_UorZv.js +0 -2
  371. package/dist/src-XxUpr2sa.js +0 -2
  372. package/dist/state-DZbAO95a.js +0 -1
  373. package/dist/suite-M8_7xio7.js.map +0 -1
  374. package/dist/suite-u2KJnGpY.js +0 -2
  375. package/dist/summary-Uzc9osfV.js.map +0 -1
  376. package/dist/types-C4OkYhZg.js.map +0 -1
  377. package/dist/vertex-h_peaZTN.js +0 -1
  378. package/dist/views-BISrsIjw.js.map +0 -1
  379. package/dist/views-Bfab8fM9.js +0 -2
  380. package/dist/workspace-B8EOCuxl.js +0 -1
@@ -1,10 +1,11 @@
1
- import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-Dwmi7L4X.js";
2
1
  import { a as plural } from "./text-B8-W3Cvn.js";
3
- import { a as rescore, h as sampleReasons, n as gateLabel } from "./gate-Cp_pQoEe.js";
4
- import { l as median, o as mannWhitney, p as quantile, u as minimumP } from "./stats-C18W4xPX.js";
5
- import { a as totalTokens, n as formatTokens, r as promptTokens } from "./usage-DJiDnh2D.js";
6
- import { m as uniqueJudgeLabels, n as cellUsage, t as cellJudgeRates } from "./judgments-m-mb0zv8.js";
7
- import { u as readableTranscriptLine } from "./record-DhzwkSXc.js";
2
+ import { n as formatTokens, r as promptTokens } from "./usage-DJiDnh2D.js";
3
+ import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-Dwmi7L4X.js";
4
+ import { l as sampleReasons } from "./pairs-BGb134Vu.js";
5
+ import { a as rescore, n as gateLabel } from "./gate-CgYdaprG.js";
6
+ import { T as cellEffort, w as cellContrast } from "./cases-z9E6DCd5.js";
7
+ import { m as uniqueJudgeLabels, n as cellUsage, t as cellJudgeRates } from "./judgments-CzgpsTqv.js";
8
+ import { u as readableTranscriptLine } from "./record-6IUzwP9K.js";
8
9
  // src/view/compare.ts
9
10
  const STACK_FIELDS = [
10
11
  {
@@ -224,7 +225,7 @@ function canceledSides(outcomes, cells) {
224
225
  const sides = ["base", "head"].filter((s) => canceledOn(outcomes[s], cells[s] ?? []));
225
226
  return sides.length ? { canceled: sides } : {};
226
227
  }
227
- /** An eval only head has. */
228
+ /** An eval only head has, with each of its cases. */
228
229
  const addedRow = (h) => ({
229
230
  id: h.id,
230
231
  base: null,
@@ -233,8 +234,41 @@ const addedRow = (h) => ({
233
234
  ...canceledSides({
234
235
  base: null,
235
236
  head: h.outcome
236
- }, { head: h.cells })
237
+ }, { head: h.cells }),
238
+ ...verdictOf(h),
239
+ ...h.cases ? { cases: caseEvals(h).map(addedRow) } : {},
240
+ ...casesGate(null, h)
241
+ });
242
+ /** An eval only base has, with each of its cases. */
243
+ const removedRow = (b) => ({
244
+ id: b.id,
245
+ base: b.outcome,
246
+ head: null,
247
+ status: "removed",
248
+ ...canceledSides({
249
+ base: b.outcome,
250
+ head: null
251
+ }, { base: b.cells }),
252
+ ...verdictOf(b),
253
+ ...b.cases ? { cases: caseEvals(b).map(removedRow) } : {},
254
+ ...casesGate(b, null)
237
255
  });
256
+ /** A case its eval's rate gates, on the side that decides: head's, or base's when only it ran. */
257
+ const verdictOf = (e) => e?.verdict ? { verdict: e.verdict } : {};
258
+ /** An eval's cases, each as an eval of its own keyed by the case key. */
259
+ const caseEvals = (e) => (e.cases ?? []).map((k) => ({
260
+ id: k.key,
261
+ fingerprint: k.fingerprint,
262
+ outcome: k.outcome,
263
+ ...k.verdict ? { verdict: k.verdict } : {},
264
+ sources: e.sources,
265
+ ...e.source ? { source: {
266
+ ...e.source,
267
+ ...k.hash ? { case: k.hash } : {}
268
+ } } : {},
269
+ ...e.revision !== void 0 ? { revision: e.revision } : {},
270
+ cells: k.cells
271
+ }));
238
272
  /**
239
273
  * When two runs of an eval differ only in which cells ran (a filtered run, a harness added), its
240
274
  * outcome on the cells both have, and the cells only one has. Null when no cell is shared or a
@@ -277,13 +311,34 @@ function compareCells(base, head) {
277
311
  }];
278
312
  });
279
313
  }
280
- /** A family with a gate: some pooled lift has a `minLift`. One without can't fail, so has no row. */
281
- const gated = (f) => !!f?.lifts.some((l) => l.minLift !== void 0);
314
+ /**
315
+ * An eval whose cases pool a gate: some lift has a `minLift`, or some cell's rate over the cases
316
+ * a `passRate`. Its outcome is then over the cases that ran, so a run of other cases can't be
317
+ * compared with it.
318
+ */
319
+ const pooledGate = (e) => !!e && ((e.lifts ?? []).some((l) => l.minLift !== void 0) || !!e.rates?.length);
282
320
  const liftSide = ({ variant: _, vs: __, ...side }) => side;
283
- /** Each variant's pooled lift on both sides, head's order first. */
284
- function familyLifts(b, h) {
321
+ /**
322
+ * When both runs pooled lifts over different variants: the variants only head or only base has,
323
+ * and those both have, over which the eval's outcome is then read.
324
+ */
325
+ function liftScope(b, h) {
326
+ const of = (e) => (e.lifts ?? []).map((l) => l.variant);
327
+ const [base, head] = [of(b), of(h)];
328
+ if (!base.length || !head.length) return null;
329
+ const added = head.filter((v) => !base.includes(v));
330
+ const removed = base.filter((v) => !head.includes(v));
331
+ if (!added.length && !removed.length) return null;
332
+ return {
333
+ added,
334
+ removed,
335
+ shared: head.filter((v) => base.includes(v))
336
+ };
337
+ }
338
+ /** Each variant's lift paired by case on both sides, head's order first. */
339
+ function casesLifts(b, h) {
285
340
  const out = /* @__PURE__ */ new Map();
286
- for (const [side, f] of [["head", h], ["base", b]]) for (const l of f?.lifts ?? []) {
341
+ for (const [side, e] of [["head", h], ["base", b]]) for (const l of e?.lifts ?? []) {
287
342
  const row = out.get(l.variant) ?? {
288
343
  key: l.variant,
289
344
  vs: l.vs,
@@ -295,131 +350,142 @@ function familyLifts(b, h) {
295
350
  }
296
351
  return [...out.values()];
297
352
  }
298
- const liftsOutcome = (lifts) => lifts.some((l) => l.outcome === "fail") ? "fail" : "pass";
299
- /** A gated family only head has. */
300
- const addedFamilyRow = (h) => ({
301
- id: h.id,
302
- kind: "family",
303
- base: null,
304
- head: h.outcome,
305
- status: "added",
306
- lifts: familyLifts(null, h)
307
- });
308
- /** The rows of a run with nothing to compare it with: every eval and gated family is new. */
309
- const newRows = (head) => [...head.evals.map(addedRow), ...(head.families ?? []).filter(gated).map(addedFamilyRow)];
353
+ /** Each gated cell's rate over the cases on both sides, head's order first. */
354
+ function casesRates(b, h) {
355
+ const out = /* @__PURE__ */ new Map();
356
+ for (const [side, e] of [["head", h], ["base", b]]) for (const { cell, ...rate } of e?.rates ?? []) {
357
+ const row = out.get(cell) ?? {
358
+ key: cell,
359
+ base: null,
360
+ head: null
361
+ };
362
+ row[side] = rate;
363
+ out.set(cell, row);
364
+ }
365
+ return [...out.values()];
366
+ }
367
+ /** The lifts and gated rates of an eval with `cases`, on both sides; unset when it has none. */
368
+ function casesGate(b, h) {
369
+ const lifts = casesLifts(b, h);
370
+ const rates = casesRates(b, h);
371
+ return {
372
+ ...lifts.length ? { lifts } : {},
373
+ ...rates.length ? { rates } : {}
374
+ };
375
+ }
376
+ /** The rows of a run with nothing to compare it with: every eval is new. */
377
+ const newRows = (head) => head.evals.map(addedRow);
378
+ /** Each eval's row, an eval with `cases` with a row for each of its cases. */
379
+ function compare(base, head) {
380
+ const graderChanged = stackChanges(base, head).some((c) => c.outsideFingerprint);
381
+ return evalRows(base.evals, head.evals, graderChanged);
382
+ }
383
+ /** A case's outcome as its eval's verdict reads it: a failure its eval's rate gates passes. */
384
+ const caseOutcome = (o, verdict) => verdict === "family" && o === "fail" ? "pass" : o;
310
385
  /**
311
- * A gated family's row. Its pooled outcome is comparable only when both runs ran the same cases
312
- * and every case is: a case that changed or hit an infrastructure error makes the family `drift`
313
- * or `infra` as it does the case, and a failing side with a canceled attempt in any case reads
314
- * `canceled`, since the pooled lift counts it. With different variants, it's over the lifts both
315
- * have, and with none in common it can't compare them.
386
+ * An eval with `cases` in both runs. Over the same cases, its outcome is the eval's; over
387
+ * different ones, the worst of the cases both ran, as their eval reads them, unless its cases
388
+ * pool a gate, which a run of other cases can't be compared with. A case that changed or hit an
389
+ * infrastructure error makes the eval `drift` or `infra` too, and a case that regressed outside
390
+ * a pooled gate makes it `regressed`.
316
391
  */
317
- function familyRow(id, b, h, ctx) {
318
- if (!b) return addedFamilyRow(h);
319
- const lifts = familyLifts(b, h ?? null);
320
- if (!h) return {
321
- id,
322
- kind: "family",
323
- base: b.outcome,
324
- head: null,
325
- status: "removed",
326
- lifts
392
+ function casesRow(b, h, graderChanged, row) {
393
+ const caseRows = evalRows(caseEvals(b), caseEvals(h), graderChanged);
394
+ const shared = caseRows.filter((r) => r.base !== null && r.head !== null);
395
+ const added = caseRows.filter((r) => r.base === null).map((r) => r.id);
396
+ const removed = caseRows.filter((r) => r.head === null).map((r) => r.id);
397
+ const out = {
398
+ ...row,
399
+ cases: caseRows,
400
+ ...casesGate(b, h)
327
401
  };
328
- const shared = lifts.flatMap((l) => l.base && l.head ? [{
329
- base: l.base,
330
- head: l.head
331
- }] : []);
332
- const scoped = shared.length > 0 && shared.length < lifts.length;
333
- const disjoint = !shared.length && b.lifts.length > 0 && h.lifts.length > 0;
334
- const row = {
335
- id,
336
- kind: "family",
337
- base: scoped ? liftsOutcome(shared.map((l) => l.base)) : b.outcome,
338
- head: scoped ? liftsOutcome(shared.map((l) => l.head)) : h.outcome,
339
- lifts,
340
- ...scoped || disjoint ? { cells: {
341
- added: lifts.filter((l) => !l.base).map((l) => l.key),
342
- removed: lifts.filter((l) => !l.head).map((l) => l.key)
343
- } } : {}
402
+ const changes = [.../* @__PURE__ */ new Set([...row.changes ?? [], ...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])])];
403
+ const failing = caseRows.filter((r) => r.base === null && caseOutcome(r.head, r.verdict) === "fail" && !r.canceled?.includes("head"));
404
+ const over = (side) => side === "head" ? [...shared, ...failing] : shared;
405
+ if (added.length || removed.length) {
406
+ if (pooledGate(b) || pooledGate(h) || !shared.length) changes.push(...added.map((k) => `case ${k} added`), ...removed.map((k) => `case ${k} removed`));
407
+ else {
408
+ const worst = (side) => outcomeOf(over(side).map((r) => ({
409
+ gate: "all",
410
+ outcome: caseOutcome(r[side], r.verdict) ?? "skipped"
411
+ })));
412
+ out.base = worst("base");
413
+ out.head = worst("head");
414
+ out.caseScope = {
415
+ added,
416
+ removed
417
+ };
418
+ changes.push(...failing.map((r) => `case ${r.id} added`));
419
+ }
420
+ }
421
+ const variants = liftScope(b, h);
422
+ if (variants) {
423
+ out.cells = {
424
+ added: variants.added,
425
+ removed: variants.removed
426
+ };
427
+ if (!variants.shared.length) return {
428
+ ...out,
429
+ ...changes.length ? { changes } : {},
430
+ status: "drift"
431
+ };
432
+ const worst = (side, e) => outcomeOf([
433
+ ...(shared.length ? over(side) : caseRows).map((r) => ({
434
+ gate: "all",
435
+ outcome: caseOutcome(r[side], r.verdict) ?? "skipped"
436
+ })),
437
+ ...(e.lifts ?? []).filter((l) => variants.shared.includes(l.variant)).map((l) => ({
438
+ gate: "all",
439
+ outcome: l.outcome
440
+ })),
441
+ ...(e.rates ?? []).filter((r) => !variants.added.includes(r.cell) && !variants.removed.includes(r.cell)).map((r) => ({
442
+ gate: "all",
443
+ outcome: r.outcome
444
+ }))
445
+ ]);
446
+ out.base = worst("base", b);
447
+ out.head = worst("head", h);
448
+ }
449
+ if (changes.length) out.changes = changes;
450
+ if (out.base === "infra_error" || out.head === "infra_error") return {
451
+ ...out,
452
+ status: "infra"
344
453
  };
345
- const prefix = id.slice(0, id.lastIndexOf(" › ") + 3);
346
- const name = (caseId) => caseId.startsWith(prefix) ? caseId.slice(prefix.length) : caseId;
347
- const before = new Set(b.caseIds);
348
- const after = new Set(h.caseIds);
349
- const caseRows = [.../* @__PURE__ */ new Set([...h.caseIds, ...b.caseIds])].flatMap((c) => ctx.rows.get(c) ?? []);
350
- const changes = [.../* @__PURE__ */ new Set([
351
- ...h.caseIds.filter((c) => !before.has(c)).map((c) => `case ${name(c)} added`),
352
- ...b.caseIds.filter((c) => !after.has(c)).map((c) => `case ${name(c)} removed`),
353
- ...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])
354
- ])];
355
- const canceled = ["base", "head"].filter((s) => row[s] === "fail" && ctx.canceledIn(s, s === "base" ? b.caseIds : h.caseIds));
356
- if (canceled.length) row.canceled = canceled;
357
454
  if (caseRows.some((r) => r.status === "infra")) return {
358
- ...row,
455
+ ...out,
359
456
  status: "infra"
360
457
  };
361
- if (changes.length) return {
362
- ...row,
363
- status: "drift",
364
- changes
365
- };
366
- if (disjoint || row.canceled || caseRows.some((r) => r.status === "drift")) return {
367
- ...row,
458
+ if (changes.some((c) => c !== "gate changed") || out.canceled || caseRows.some((r) => r.status === "drift" || r.canceled)) return {
459
+ ...out,
368
460
  status: "drift"
369
461
  };
370
- if (ctx.graderChanged && row.base !== row.head) return {
371
- ...row,
462
+ const from = out.gate?.rescored ?? out.base;
463
+ if (graderChanged && from !== out.head) return {
464
+ ...out,
372
465
  status: "drift"
373
466
  };
374
- if (row.base === "pass" && row.head === "fail") return {
375
- ...row,
467
+ const caseRegressed = caseRows.some((r) => r.status === "regressed" && r.verdict !== "family");
468
+ if (from === "pass" && out.head === "fail" || caseRegressed) return {
469
+ ...out,
376
470
  status: "regressed"
377
471
  };
378
- if (row.base === "fail" && row.head === "pass") return {
379
- ...row,
472
+ if (from === "fail" && out.head === "pass") return {
473
+ ...out,
380
474
  status: "fixed"
381
475
  };
382
476
  return {
383
- ...row,
384
- status: "same"
385
- };
386
- }
387
- /** Each eval's row, then each family's that either run gated. */
388
- function compare(base, head) {
389
- const graderChanged = stackChanges(base, head).some((c) => c.outsideFingerprint);
390
- const rows = evalRows(base, head, graderChanged);
391
- const before = new Map((base.families ?? []).map((f) => [f.id, f]));
392
- const after = new Map((head.families ?? []).map((f) => [f.id, f]));
393
- const families = [.../* @__PURE__ */ new Set([...after.keys(), ...before.keys()])].filter((id) => gated(before.get(id)) || gated(after.get(id)));
394
- const runs = {
395
- base,
396
- head
477
+ ...out,
478
+ status: out.base === out.head ? "same" : "gate"
397
479
  };
398
- const ctx = {
399
- rows: new Map(rows.map((r) => [r.id, r])),
400
- graderChanged,
401
- canceledIn: (side, caseIds) => runs[side].evals.some((e) => caseIds.includes(e.id) && e.cells.some((c) => (c.canceled ?? 0) > 0))
402
- };
403
- return [...rows, ...families.map((id) => familyRow(id, before.get(id), after.get(id), ctx))];
404
480
  }
405
- /** The key that tells a family's row from an eval's, whose ids can coincide. */
406
- const rowKey = (row) => row.kind === "family" ? `family\0${row.id}` : row.id;
407
- function evalRows(base, head, graderChanged) {
408
- return [.../* @__PURE__ */ new Set([...head.evals.map((e) => e.id), ...base.evals.map((e) => e.id)])].map((id) => {
409
- const b = base.evals.find((e) => e.id === id);
410
- const h = head.evals.find((e) => e.id === id);
481
+ function evalRows(baseEvals, headEvals, graderChanged) {
482
+ return [.../* @__PURE__ */ new Set([...headEvals.map((e) => e.id), ...baseEvals.map((e) => e.id)])].map((id) => {
483
+ const b = baseEvals.find((e) => e.id === id);
484
+ const h = headEvals.find((e) => e.id === id);
411
485
  if (!b) return addedRow(h);
412
- if (!h) return {
413
- id,
414
- base: b.outcome,
415
- head: null,
416
- status: "removed",
417
- ...canceledSides({
418
- base: b.outcome,
419
- head: null
420
- }, { base: b.cells })
421
- };
422
- const comparable = sameEval(b, h);
486
+ if (!h) return removedRow(b);
487
+ const withCases = !!(b.cases || h.cases);
488
+ const comparable = withCases || sameEval(b, h);
423
489
  const scoped = comparable ? null : sharedScope(b, h);
424
490
  const outcomes = scoped ? {
425
491
  base: scoped.base,
@@ -435,19 +501,22 @@ function evalRows(base, head, graderChanged) {
435
501
  ...canceledSides(outcomes, scoped?.over ?? {
436
502
  base: b.cells,
437
503
  head: h.cells
438
- })
504
+ }),
505
+ ...verdictOf(h)
439
506
  };
440
507
  const perCell = compareCells(b, h);
441
508
  if (perCell.length) row.perCell = perCell;
442
- if (row.base === "infra_error" || row.head === "infra_error") return {
443
- ...row,
444
- status: "infra"
445
- };
446
- if (!comparable && !scoped) return {
447
- ...row,
448
- status: "drift",
449
- changes: whyChanged(b, h)
450
- };
509
+ if (!withCases) {
510
+ if (row.base === "infra_error" || row.head === "infra_error") return {
511
+ ...row,
512
+ status: "infra"
513
+ };
514
+ if (!comparable && !scoped) return {
515
+ ...row,
516
+ status: "drift",
517
+ changes: whyChanged(b, h)
518
+ };
519
+ }
451
520
  const over = scoped?.over ?? {
452
521
  base: b.cells ?? [],
453
522
  head: h.cells ?? []
@@ -457,6 +526,7 @@ function evalRows(base, head, graderChanged) {
457
526
  changes: ["gate changed"],
458
527
  gate
459
528
  });
529
+ if (withCases) return casesRow(b, h, graderChanged, row);
460
530
  const from = gate?.rescored ?? row.base;
461
531
  if (row.canceled) return {
462
532
  ...row,
@@ -480,186 +550,6 @@ function evalRows(base, head, graderChanged) {
480
550
  };
481
551
  });
482
552
  }
483
- // src/view/effort.ts
484
- /** Below this two-sided p, an effort difference is an effect. */
485
- const EFFORT_ALPHA = .05;
486
- /** Some counted attempts reported no usage. Not a verdict; never printed as one. */
487
- const TOKENS_UNREPORTED = "some cells did not report usage";
488
- const effortSample = (a) => ({
489
- tools: a.activity.tools,
490
- agentMs: a.timings.agentMs,
491
- fileEdits: a.activity.fileEdits,
492
- filesChanged: filesChanged(a.changedFiles),
493
- ...a.usage ? { tokens: totalTokens(a.usage) } : {}
494
- });
495
- /** The files the workspace diff changed, a collapsed directory counting each of its files. */
496
- const filesChanged = (files) => files.reduce((n, f) => n + (f.count ?? 1), 0);
497
- const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
498
- const spread = (values) => ({
499
- median: median(values) ?? 0,
500
- q1: quantile(values, .25) ?? 0,
501
- q3: quantile(values, .75) ?? 0
502
- });
503
- /** Token totals from the attempts that reported usage; empty when none did. */
504
- const tokenValues = (samples) => samples.flatMap((s) => s.tokens === void 0 ? [] : [s.tokens]);
505
- function effortOf(samples) {
506
- if (samples.length === 0) return null;
507
- const tokens = tokenValues(samples);
508
- return {
509
- n: samples.length,
510
- tools: spread(samples.map((s) => s.tools)),
511
- agentMs: spread(samples.map((s) => s.agentMs)),
512
- fileEdits: median(samples.map((s) => s.fileEdits)) ?? 0,
513
- filesChanged: median(samples.map((s) => s.filesChanged)) ?? 0,
514
- ...tokens.length ? { tokens: spread(tokens) } : {}
515
- };
516
- }
517
- /**
518
- * The verdict of an exact rank test of `xs` (variant) against `ys` (control). Its direction is
519
- * the test's (which side ranks lower), not the medians', which can be equal or disagree.
520
- */
521
- function effortVerdict(xs, ys) {
522
- const test = mannWhitney(xs, ys);
523
- const p = test?.p ?? null;
524
- if (test === null || test.floor >= .05) return {
525
- p,
526
- verdict: "insufficient"
527
- };
528
- if (test.p >= .05) return {
529
- p,
530
- verdict: "noise"
531
- };
532
- return {
533
- p,
534
- verdict: test.sign < 0 ? "fewer" : "more"
535
- };
536
- }
537
- const ratio = (a, b) => b === 0 ? null : a / b;
538
- /** Every value on one side is below every value on the other. */
539
- const apart = (xs, ys) => Math.max(...xs) < Math.min(...ys) || Math.min(...xs) > Math.max(...ys);
540
- /**
541
- * The fewest attempts a side for "no overlap" to say anything: at 3 against 3 one ordering in
542
- * ten separates by chance; at 1 against 1 every pair of unequal values does.
543
- */
544
- const DIRECTIONAL_MIN = 3;
545
- /**
546
- * The fewest runs per cell whose effort test can reach `EFFORT_ALPHA` (4 against 4: p = 2/70).
547
- */
548
- const EFFORT_MIN_RUNS = (() => {
549
- let n = 1;
550
- while (minimumP(n, n) >= EFFORT_ALPHA) n++;
551
- return n;
552
- })();
553
- /**
554
- * The metrics whose test is `insufficient` but whose ranges don't overlap at
555
- * `DIRECTIONAL_MIN` a side or more: a direction, not an effect.
556
- */
557
- function directional(c) {
558
- if (Math.min(c.variant.n, c.control.n) < 3) return [];
559
- const out = [];
560
- if (c.separated && c.verdict === "insufficient") out.push("tools");
561
- if (c.timeSeparated && c.timeVerdict === "insufficient") out.push("agentS");
562
- if (c.tokensSeparated && c.tokensVerdict === "insufficient") out.push("tokens");
563
- return out;
564
- }
565
- function contrastEffort(variant, control) {
566
- const v = effortOf(variant);
567
- const c = effortOf(control);
568
- if (!v || !c) return null;
569
- const tools = effortVerdict(variant.map((s) => s.tools), control.map((s) => s.tools));
570
- const time = effortVerdict(variant.map((s) => s.agentMs), control.map((s) => s.agentMs));
571
- const vTok = tokenValues(variant);
572
- const cTok = tokenValues(control);
573
- const tokens = vTok.length && cTok.length ? effortVerdict(vTok, cTok) : null;
574
- const reported = vTok.length + cTok.length;
575
- const unreported = reported > 0 && reported < variant.length + control.length;
576
- return {
577
- variant: v,
578
- control: c,
579
- toolsRatio: ratio(v.tools.median, c.tools.median),
580
- agentMsRatio: ratio(v.agentMs.median, c.agentMs.median),
581
- agentMsDelta: v.agentMs.median - c.agentMs.median,
582
- toolsP: tools.p,
583
- agentMsP: time.p,
584
- separated: apart(variant.map((s) => s.tools), control.map((s) => s.tools)),
585
- timeSeparated: apart(variant.map((s) => s.agentMs), control.map((s) => s.agentMs)),
586
- tokensSeparated: tokens ? apart(vTok, cTok) : null,
587
- verdict: tools.verdict,
588
- timeVerdict: time.verdict,
589
- tokensRatio: tokens && v.tokens && c.tokens ? ratio(v.tokens.median, c.tokens.median) : null,
590
- tokensP: tokens?.p ?? null,
591
- tokensVerdict: tokens?.verdict ?? null,
592
- ...unreported ? { tokensNote: TOKENS_UNREPORTED } : {}
593
- };
594
- }
595
- /** The counted attempts of one cell of an eval, as effort samples. */
596
- const cellSamples = (e, cellKey) => e.attempts.filter((a) => a.cell === cellKey && counted(a)).map(effortSample);
597
- const cellEffort = (e, cellKey) => effortOf(cellSamples(e, cellKey));
598
- /** A variant cell's effort against its control's, in the same run; null for controls. */
599
- function cellContrast(e, cellKey) {
600
- const cell = e.cells.find((c) => c.key === cellKey);
601
- if (!cell?.lift) return null;
602
- return contrastEffort(cellSamples(e, cellKey), cellSamples(e, cell.lift.vs));
603
- }
604
- /** `4`, `4.5`: medians of counts are whole or halves. */
605
- const formatCount = (n) => Number.isInteger(n) ? String(n) : n.toFixed(1);
606
- const formatRatio = (r) => r === null ? "×–" : `×${r.toFixed(2)}`;
607
- /** Below this, agent time rounds under a tenth of a second and prints `<0.1s`. */
608
- const TENTH_MS = 50;
609
- /** Two agent times that both print `<0.1s` have no ratio worth printing. */
610
- const bothUnderATenth = (a, b) => a < TENTH_MS && b < TENTH_MS;
611
- /** Seconds for agent time: `<0.1s`, `0.4s`, `21s`, `1m 05s`. */
612
- function formatSeconds(ms) {
613
- if (ms < 9950) return ms < TENTH_MS ? "<0.1s" : `${(ms / 1e3).toFixed(1)}s`;
614
- const s = Math.round(ms / 1e3);
615
- return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${String(s % 60).padStart(2, "0")}s`;
616
- }
617
- const formatP = (p) => p === null ? "" : ` (p = ${p < .001 ? "<0.001" : p.toFixed(3)})`;
618
- const VERDICT_TEXT = {
619
- fewer: (what) => `fewer ${what}`,
620
- more: (what) => `more ${what}`,
621
- noise: () => "within noise",
622
- insufficient: () => "too few attempts to test"
623
- };
624
- const toolsVerdictText = (c) => c.verdict === "fewer" || c.verdict === "more" ? `${VERDICT_TEXT[c.verdict]("tool calls")}${formatP(c.toolsP)}` : VERDICT_TEXT[c.verdict]("");
625
- /** Token verdict after the tools one; empty when omitted or already covered as insufficient. */
626
- const tokensVerdictText = (c) => {
627
- if (c.tokensVerdict === null) return "";
628
- if (c.verdict === "insufficient" && c.tokensVerdict === "insufficient") return "";
629
- if (c.tokensVerdict === "fewer" || c.tokensVerdict === "more") return `; ${VERDICT_TEXT[c.tokensVerdict]("tokens")}${formatP(c.tokensP)}`;
630
- if (c.tokensVerdict === "noise") return "; tokens within noise";
631
- return `; ${VERDICT_TEXT[c.tokensVerdict]("")}`;
632
- };
633
- /**
634
- * `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
635
- * against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
636
- * when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
637
- * A metric too small to test whose ranges don't overlap says so: `(×0.37, no overlap)`. Agent
638
- * times both under a tenth of a second have no ratio: `agent <0.1s vs <0.1s`.
639
- */
640
- function effortLabel(c) {
641
- const [va, ca] = [c.variant.agentMs.median, c.control.agentMs.median];
642
- const tiny = bothUnderATenth(va, ca);
643
- const apartOn = new Set(directional(c).filter((m) => !(tiny && m === "agentS")));
644
- const r = (metric, value) => `(${formatRatio(value)}${apartOn.has(metric) ? ", no overlap" : ""})`;
645
- const parts = [`tools ${formatCount(c.variant.tools.median)} vs ${formatCount(c.control.tools.median)} ${r("tools", c.toolsRatio)}`, `agent ${formatSeconds(va)} vs ${formatSeconds(ca)}${tiny ? "" : ` ${r("agentS", c.agentMsRatio)}`}`];
646
- if (c.tokensVerdict !== null && c.variant.tokens && c.control.tokens) parts.push(`tokens ${formatTokens(c.variant.tokens.median)} vs ${formatTokens(c.control.tokens.median)} ${r("tokens", c.tokensRatio)}`);
647
- const n = apartOn.size ? ` (n = ${c.variant.n} vs ${c.control.n})` : "";
648
- return `${parts.join(", ")}: ${toolsVerdictText(c)}${tokensVerdictText(c)}${n}`;
649
- }
650
- /** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
651
- const effortSummary = (e) => e.tokens ? `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)} · tokens ${formatTokens(e.tokens.median)}` : `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)}`;
652
- /** The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`. */
653
- function effortChangeLabel(base, head) {
654
- const t = ratio(head.tools.median, base.tools.median);
655
- const a = ratio(head.agentMs.median, base.agentMs.median);
656
- const [ba, ha] = [base.agentMs.median, head.agentMs.median];
657
- const agent = `agent ${formatSeconds(ba)} → ${formatSeconds(ha)}${bothUnderATenth(ba, ha) ? "" : ` (${formatRatio(a)})`}`;
658
- const core = `tools ${formatCount(base.tools.median)} → ${formatCount(head.tools.median)} (${formatRatio(t)}) · ${agent}`;
659
- if (!base.tokens || !head.tokens) return core;
660
- const tok = ratio(head.tokens.median, base.tokens.median);
661
- return `${core} · tokens ${formatTokens(base.tokens.median)} → ${formatTokens(head.tokens.median)} (${formatRatio(tok)})`;
662
- }
663
553
  // src/view/model.ts
664
554
  /** Splits `file › group › … › name` back into its parts. */
665
555
  function splitEvalId(id) {
@@ -782,7 +672,7 @@ function activityLabel(a, changed) {
782
672
  function runOutcome(run) {
783
673
  if (run.status === "interrupted") return "skipped";
784
674
  if (run.errors?.length) return "fail";
785
- const outcomes = [...run.evals, ...run.families ?? []].map((e) => e.outcome);
675
+ const outcomes = run.evals.map((e) => e.outcome);
786
676
  if (outcomes.includes("fail") || run.status === "budget") return "fail";
787
677
  if (outcomes.includes("infra_error")) return "infra_error";
788
678
  return "pass";
@@ -1089,7 +979,7 @@ function gatedPassAt1(cells) {
1089
979
  /** `reeval compare`'s classification, plus pass@1 and judge-score deltas from the records. */
1090
980
  function compareView(base, head, records = {}) {
1091
981
  const rows = compare(base, head).map((row) => {
1092
- const evalOf = (run) => row.kind === "family" ? void 0 : run.evals.find((e) => e.id === row.id);
982
+ const evalOf = (run) => run.evals.find((e) => e.id === row.id);
1093
983
  const b = evalOf(base);
1094
984
  const h = evalOf(head);
1095
985
  const baseKeys = new Set(b?.cells.map((c) => c.key));
@@ -1152,6 +1042,6 @@ const findEval = (record, slug) => record.evals.find((e) => e.slug === slug);
1152
1042
  const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId);
1153
1043
  /** The attempt to open by default: the first failure, else the first attempt. */
1154
1044
  const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
1155
- export { formatCount as $, runRows as A, bothUnderATenth as B, isUngatedMiss as C, reasoningLabel as D, parseTranscript as E, DIRECTIONAL_MIN as F, directional as G, cellEffort as H, EFFORT_ALPHA as I, effortOf as J, effortChangeLabel as K, EFFORT_MIN_RUNS as L, shortSha as M, splitEvalId as N, runLead as O, transcriptItems as P, filesChanged as Q, TOKENS_UNREPORTED as R, isNotGatedPass as S, notRunYet as T, cellSamples as U, cellContrast as V, contrastEffort as W, effortSummary as X, effortSample as Y, effortVerdict as Z, formatUsage as _, buildIndex as a, compare as at, indexFiles as b, defaultAttempt as c, sourceChanges as ct, evalTree as d, formatRatio as et, failedJudgeLabels as f, formatPercent as g, formatDuration as h, attemptAnswerRows as i, cellsChanged as it, runtimeLabel as j, runOutcome as k, evalHistory as l, stackChanges as lt, findEval as m, activityLabel as n, tokenValues as nt, cellRows as o, newRows as ot, findAttempt as p, effortLabel as q, answerRows as r, addedRow as rt, compareView as s, rowKey as st, COMPARE_ORDER as t, formatSeconds as tt, evalMarkOutcome as u, whyChanged as ut, harnessLabel as v, languageLabel as w, isMatrix as x, hasUngatedCells as y, VERDICT_TEXT as z };
1045
+ export { runRows as A, stackChanges as B, isUngatedMiss as C, reasoningLabel as D, parseTranscript as E, addedRow as F, cellsChanged as I, compare as L, shortSha as M, splitEvalId as N, runLead as O, transcriptItems as P, newRows as R, isNotGatedPass as S, notRunYet as T, whyChanged as V, formatUsage as _, buildIndex as a, indexFiles as b, defaultAttempt as c, evalTree as d, failedJudgeLabels as f, formatPercent as g, formatDuration as h, attemptAnswerRows as i, runtimeLabel as j, runOutcome as k, evalHistory as l, findEval as m, activityLabel as n, cellRows as o, findAttempt as p, answerRows as r, compareView as s, COMPARE_ORDER as t, evalMarkOutcome as u, harnessLabel as v, languageLabel as w, isMatrix as x, hasUngatedCells as y, sourceChanges as z };
1156
1046
 
1157
- //# sourceMappingURL=model-Ce9l9_rF.js.map
1047
+ //# sourceMappingURL=model-V3mMXp6x.js.map