reeval 0.17.2 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/README.md +4 -2
  2. package/dist/{adapter-DyvDj4ZN.js → adapter-CptK032x.js} +2 -2
  3. package/dist/{adapter-DyvDj4ZN.js.map → adapter-CptK032x.js.map} +1 -1
  4. package/dist/{agent-v036GEgG.js → agent-B63OzUX9.js} +6 -6
  5. package/dist/{agent-v036GEgG.js.map → agent-B63OzUX9.js.map} +1 -1
  6. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  7. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  8. package/dist/{app-files-hpRT5lj_.js → app-files-sgqwfjtn.js} +4 -4
  9. package/dist/app-files-sgqwfjtn.js.map +1 -0
  10. package/dist/assemble-CxBHkavM.js +1 -0
  11. package/dist/{assemble-Dlh6Xv3U.js → assemble-Me7Whp-y.js} +58 -194
  12. package/dist/assemble-Me7Whp-y.js.map +1 -0
  13. package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
  14. package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
  15. package/dist/{attempt-host-DErH-DYV.js → attempt-host-Cik0SoIn.js} +17 -9
  16. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
  17. package/dist/{capture-Bdcx9Gap.js → capture-C3RmjPph.js} +392 -613
  18. package/dist/capture-C3RmjPph.js.map +1 -0
  19. package/dist/capture-YPwxp_iK.js +1 -0
  20. package/dist/{cli-CGd51thd.js → cli-BBOlXKPG.js} +1075 -457
  21. package/dist/cli-BBOlXKPG.js.map +1 -0
  22. package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
  23. package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
  26. package/dist/client-tZ2WBDxf.js.map +1 -0
  27. package/dist/{command-TLmY-rYp.js → command-CFQ9c1hk.js} +13 -13
  28. package/dist/{command-TLmY-rYp.js.map → command-CFQ9c1hk.js.map} +1 -1
  29. package/dist/{command-CtTdCF3B.js → command-DhiwsZl1.js} +9 -9
  30. package/dist/{command-CtTdCF3B.js.map → command-DhiwsZl1.js.map} +1 -1
  31. package/dist/{command-CyIb9BSC.js → command-fO0BFsTq.js} +102 -37
  32. package/dist/command-fO0BFsTq.js.map +1 -0
  33. package/dist/{command-CsfcwMyc.js → command-qGS802uX.js} +16 -14
  34. package/dist/command-qGS802uX.js.map +1 -0
  35. package/dist/{commands-C54S45d7.js → commands-CC60TOT5.js} +64 -18
  36. package/dist/commands-CC60TOT5.js.map +1 -0
  37. package/dist/{commands-jO9Jj35r.js → commands-xYTMNTPv.js} +1 -1
  38. package/dist/{config-ImY6AlRE.js → config-C2UtS9vy.js} +718 -76
  39. package/dist/config-C2UtS9vy.js.map +1 -0
  40. package/dist/{contrasts-DtSX8E_V.js → contrasts-BjOg7wyr.js} +3 -3
  41. package/dist/{contrasts-DtSX8E_V.js.map → contrasts-BjOg7wyr.js.map} +1 -1
  42. package/dist/cost-BJU0MAji.js +1 -0
  43. package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
  44. package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
  45. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BTK_apL-.js} +2 -2
  46. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
  47. package/dist/{db-sHGpaLo9.js → db-C5SZ0510.js} +72 -14
  48. package/dist/db-C5SZ0510.js.map +1 -0
  49. package/dist/db-puzLzX4k.js +2 -0
  50. package/dist/{default-effort-BZRa-SQP.js → default-effort-BVmx7r7L.js} +8 -5
  51. package/dist/default-effort-BVmx7r7L.js.map +1 -0
  52. package/dist/{default-effort-fpdbYVKm.js → default-effort-FEidNAIi.js} +1 -1
  53. package/dist/{docker-CWuSRj7a.js → docker-B2F8-PGC.js} +4 -4
  54. package/dist/{docker-CWuSRj7a.js.map → docker-B2F8-PGC.js.map} +1 -1
  55. package/dist/{docker-CIzHzwlC.js → docker-BlSopHpZ.js} +1 -1
  56. package/dist/{docker--2ytOHcu.js → docker-Dto8ROCq.js} +5 -5
  57. package/dist/{docker--2ytOHcu.js.map → docker-Dto8ROCq.js.map} +1 -1
  58. package/dist/{doctor-eE65YpEv.js → doctor-BcvfKpBX.js} +20 -20
  59. package/dist/{doctor-eE65YpEv.js.map → doctor-BcvfKpBX.js.map} +1 -1
  60. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  61. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  62. package/dist/{environment-Cy3FVLR5.js → environment-CC1RV21n.js} +3 -3
  63. package/dist/{environment-Cy3FVLR5.js.map → environment-CC1RV21n.js.map} +1 -1
  64. package/dist/{eval-source-BTsTRiBW.js → eval-source-CzxdgFBW.js} +18 -10
  65. package/dist/eval-source-CzxdgFBW.js.map +1 -0
  66. package/dist/evaluate-DToSB2bt.d.ts +101 -0
  67. package/dist/{evaluate-CFFdyYZX.js → evaluate-NTNJ8T9H.js} +129 -51
  68. package/dist/evaluate-NTNJ8T9H.js.map +1 -0
  69. package/dist/{files-tnMgu7-X.js → files-CZXrkPja.js} +5 -5
  70. package/dist/{files-tnMgu7-X.js.map → files-CZXrkPja.js.map} +1 -1
  71. package/dist/files-DFtuRQJx.js +1 -0
  72. package/dist/{files-CQU08uh4.js → files-DsXULUi5.js} +3 -3
  73. package/dist/{files-CQU08uh4.js.map → files-DsXULUi5.js.map} +1 -1
  74. package/dist/{files-CIX-pLl4.js → files-aAzoGNCB.js} +4 -4
  75. package/dist/{files-CIX-pLl4.js.map → files-aAzoGNCB.js.map} +1 -1
  76. package/dist/files-oOYNR7ZH.js +1 -0
  77. package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
  78. package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
  79. package/dist/gateway.d.ts +1 -1
  80. package/dist/{git-BWVkG4Yl.js → git-C1MB5bjr.js} +3 -3
  81. package/dist/{git-BWVkG4Yl.js.map → git-C1MB5bjr.js.map} +1 -1
  82. package/dist/harness-DGsRZDSo.js +105 -0
  83. package/dist/harness-DGsRZDSo.js.map +1 -0
  84. package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
  85. package/dist/harness-DYSC4zMh.js.map +1 -0
  86. package/dist/harness.d.ts +1 -1
  87. package/dist/harness.js +1 -1
  88. package/dist/{images-BVZHDf_z.js → images-CNayXCp2.js} +8 -8
  89. package/dist/{images-BVZHDf_z.js.map → images-CNayXCp2.js.map} +1 -1
  90. package/dist/index.d.ts +3 -3
  91. package/dist/index.js +5 -5
  92. package/dist/init-8o6W3_fn.js +1 -0
  93. package/dist/{init-BWnMbUnc.js → init-D5HtmyMI.js} +5 -5
  94. package/dist/{init-BWnMbUnc.js.map → init-D5HtmyMI.js.map} +1 -1
  95. package/dist/ink-list-Ca-5dunO.js +184 -0
  96. package/dist/ink-list-Ca-5dunO.js.map +1 -0
  97. package/dist/{invocations-BGRs1r_C.js → invocations-DrGXF9dT.js} +3 -3
  98. package/dist/{invocations-BGRs1r_C.js.map → invocations-DrGXF9dT.js.map} +1 -1
  99. package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
  100. package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
  101. package/dist/judge.d.ts +1 -1
  102. package/dist/judge.js +1 -1
  103. package/dist/judgments-m-mb0zv8.js +246 -0
  104. package/dist/judgments-m-mb0zv8.js.map +1 -0
  105. package/dist/list-CM_YSJXq.js +374 -0
  106. package/dist/list-CM_YSJXq.js.map +1 -0
  107. package/dist/{live-d_5a5iUv.js → live-B71o2TkX.js} +21 -3
  108. package/dist/live-B71o2TkX.js.map +1 -0
  109. package/dist/live-DQc8jr6o.js +1 -0
  110. package/dist/{live-ColieOai.js → live-uRURFDMZ.js} +7 -7
  111. package/dist/{live-ColieOai.js.map → live-uRURFDMZ.js.map} +1 -1
  112. package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
  113. package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
  114. package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
  115. package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
  116. package/dist/{model-DBpdRXtO.js → model-BxBWFOZn.js} +43 -19
  117. package/dist/{model-DBpdRXtO.js.map → model-BxBWFOZn.js.map} +1 -1
  118. package/dist/{model-DOXnWH65.js → model-CXVVB4RK.js} +1 -1
  119. package/dist/{model-DOiCiMCD.js → model-Ce9l9_rF.js} +19 -255
  120. package/dist/model-Ce9l9_rF.js.map +1 -0
  121. package/dist/model-Cxzo4cjw.js +1 -0
  122. package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
  123. package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
  124. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
  125. package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
  126. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
  127. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
  128. package/dist/{patch-DvLr8wCZ.js → patch-ug5HpMOj.js} +3 -3
  129. package/dist/{patch-DvLr8wCZ.js.map → patch-ug5HpMOj.js.map} +1 -1
  130. package/dist/{plan-CIfOFhth.js → plan-DqnzfDH1.js} +15 -14
  131. package/dist/plan-DqnzfDH1.js.map +1 -0
  132. package/dist/{plugins-BEs2Am3L.js → plugins-B2KQ0y3P.js} +4 -4
  133. package/dist/{plugins-BEs2Am3L.js.map → plugins-B2KQ0y3P.js.map} +1 -1
  134. package/dist/{preflight-BAIRsTpl.js → preflight-DVXEvzFU.js} +1 -1
  135. package/dist/{preflight-50Qw5e5K.js → preflight-DyzmvK7i.js} +3 -3
  136. package/dist/{preflight-50Qw5e5K.js.map → preflight-DyzmvK7i.js.map} +1 -1
  137. package/dist/{prepare-8XhzhLHX.js → prepare-21WMQZyt.js} +7 -7
  138. package/dist/{prepare-8XhzhLHX.js.map → prepare-21WMQZyt.js.map} +1 -1
  139. package/dist/{prepare-8nvfDFQx.js → prepare-BcSRa7-3.js} +2 -2
  140. package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
  141. package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
  142. package/dist/{provenance-Cc8fFE7Z.js → provenance-iRxjp4Pu.js} +4 -4
  143. package/dist/{provenance-Cc8fFE7Z.js.map → provenance-iRxjp4Pu.js.map} +1 -1
  144. package/dist/{providers-CP_bBtYm.js → providers-D0ob_tra.js} +1 -1
  145. package/dist/{providers-DZSTlC9R.js → providers-vXMXuwn-.js} +5 -5
  146. package/dist/{providers-DZSTlC9R.js.map → providers-vXMXuwn-.js.map} +1 -1
  147. package/dist/{query-DXpxaaTH.js → query-BhaKQ07P.js} +3 -3
  148. package/dist/{query-DXpxaaTH.js.map → query-BhaKQ07P.js.map} +1 -1
  149. package/dist/query-CvgUsvep.js +1 -0
  150. package/dist/{questions-C_Y-oXW6.js → questions-BzdLdV3C.js} +39 -9
  151. package/dist/questions-BzdLdV3C.js.map +1 -0
  152. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  153. package/dist/redact-BhjtnjrU.js.map +1 -0
  154. package/dist/remarks-CAvTSq16.js +1 -0
  155. package/dist/{remarks-E4rpi8Ym.js → remarks-CauU408t.js} +231 -25
  156. package/dist/remarks-CauU408t.js.map +1 -0
  157. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  158. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  159. package/dist/{remarks-DeZZT8PK.js → remarks-NMgPd30U.js} +4 -3
  160. package/dist/{remarks-DeZZT8PK.js.map → remarks-NMgPd30U.js.map} +1 -1
  161. package/dist/{remote-DZ00A8V_.js → remote-Dwy3fyo3.js} +5 -5
  162. package/dist/{remote-DZ00A8V_.js.map → remote-Dwy3fyo3.js.map} +1 -1
  163. package/dist/{resolve-B_JlQ0La.js → resolve-B1zMnPwK.js} +7 -7
  164. package/dist/{resolve-B_JlQ0La.js.map → resolve-B1zMnPwK.js.map} +1 -1
  165. package/dist/resolve-BIIeVT12.js +1 -0
  166. package/dist/{results-BMJHaQbi.js → results-CfC0id_u.js} +7 -7
  167. package/dist/{results-BMJHaQbi.js.map → results-CfC0id_u.js.map} +1 -1
  168. package/dist/{resume-CexsWuGK.js → resume-Dkhrpjow.js} +123 -8
  169. package/dist/resume-Dkhrpjow.js.map +1 -0
  170. package/dist/{runtime-CSZ1hjlN.js → runtime-DcvQcr36.js} +2 -2
  171. package/dist/{runtime-CSZ1hjlN.js.map → runtime-DcvQcr36.js.map} +1 -1
  172. package/dist/{sandbox-B5W8lLL1.js → sandbox-8zCrtuCf.js} +2 -2
  173. package/dist/{sandbox-B5W8lLL1.js.map → sandbox-8zCrtuCf.js.map} +1 -1
  174. package/dist/sandbox.d.ts +1 -1
  175. package/dist/sandbox.js +1 -1
  176. package/dist/server-0tuj8VSa.js +1 -0
  177. package/dist/{server-BfN6q2lH.js → server-Dplq8g0r.js} +7 -7
  178. package/dist/{server-BfN6q2lH.js.map → server-Dplq8g0r.js.map} +1 -1
  179. package/dist/{source-BD6Bg0Tr.js → source-NWHqn5-o.js} +2 -2
  180. package/dist/{source-BD6Bg0Tr.js.map → source-NWHqn5-o.js.map} +1 -1
  181. package/dist/source-llwLewOU.js +1 -0
  182. package/dist/{src-BC2WXULN.js → src-B9oE_mNS.js} +1 -1
  183. package/dist/{src-Cpy9MNS3.js → src-BQ6BiXoc.js} +19 -11
  184. package/dist/{src-Cpy9MNS3.js.map → src-BQ6BiXoc.js.map} +1 -1
  185. package/dist/{src-Dp-fWOfG.js → src-BQXdf-Gw.js} +2 -2
  186. package/dist/{src-Dp-fWOfG.js.map → src-BQXdf-Gw.js.map} +1 -1
  187. package/dist/{src-DurP--_D.js → src-BXnv0ViU.js} +1 -1
  188. package/dist/src-BiS8YNN_.js +3 -0
  189. package/dist/{src-DVaAKCsH.js → src-BkH4GOxu.js} +1 -1
  190. package/dist/{src-CM7xdJFA.js → src-C-tH5ebd.js} +1 -1
  191. package/dist/{src-Cx0HJIJ-.js → src-CUpoVVb8.js} +1 -1
  192. package/dist/{src-C9elednc.js → src-Cg1bFs9g.js} +1 -1
  193. package/dist/{src-BKDZO3ra.js → src-DV_UorZv.js} +1 -1
  194. package/dist/{src-CWEGknZo.js → src-XxUpr2sa.js} +1 -1
  195. package/dist/{state-CdSjXIfE.js → state-DDCGUhva.js} +5 -5
  196. package/dist/{state-CdSjXIfE.js.map → state-DDCGUhva.js.map} +1 -1
  197. package/dist/state-DZbAO95a.js +1 -0
  198. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  199. package/dist/stats-C18W4xPX.js.map +1 -0
  200. package/dist/suite-M8_7xio7.js +1294 -0
  201. package/dist/suite-M8_7xio7.js.map +1 -0
  202. package/dist/suite-u2KJnGpY.js +2 -0
  203. package/dist/{summary-zP3cCXDN.js → summary-Uzc9osfV.js} +47 -13
  204. package/dist/summary-Uzc9osfV.js.map +1 -0
  205. package/dist/testing.d.ts +47 -37
  206. package/dist/testing.js +4 -4
  207. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  208. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  209. package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
  210. package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
  211. package/dist/{types-BHR1lGPh.d.ts → types-DGvexkcp.d.ts} +71 -63
  212. package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
  213. package/dist/vertex-BLQMayc1.js.map +1 -0
  214. package/dist/vertex-h_peaZTN.js +1 -0
  215. package/dist/view/index.d.ts +140 -35
  216. package/dist/view/index.js +8 -7
  217. package/dist/{views-CKoN5NAx.js → views-BISrsIjw.js} +6 -6
  218. package/dist/{views-CKoN5NAx.js.map → views-BISrsIjw.js.map} +1 -1
  219. package/dist/views-Bfab8fM9.js +2 -0
  220. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-DJD_nHkS.js} +5 -5
  221. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
  222. package/dist/workspace-B8EOCuxl.js +1 -0
  223. package/dist/{workspace-Cho30qlM.js → workspace-BO1VhUR9.js} +5 -5
  224. package/dist/{workspace-Cho30qlM.js.map → workspace-BO1VhUR9.js.map} +1 -1
  225. package/package.json +3 -1
  226. package/skills/eval-rubric/SKILL.md +22 -3
  227. package/dist/app-files-hpRT5lj_.js.map +0 -1
  228. package/dist/assemble-DTXwCL3m.js +0 -1
  229. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  230. package/dist/capture-BT2fAmpZ.js +0 -1
  231. package/dist/capture-Bdcx9Gap.js.map +0 -1
  232. package/dist/cli-CGd51thd.js.map +0 -1
  233. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  234. package/dist/client-CF-DOjOU.js.map +0 -1
  235. package/dist/command-CsfcwMyc.js.map +0 -1
  236. package/dist/command-CyIb9BSC.js.map +0 -1
  237. package/dist/commands-C54S45d7.js.map +0 -1
  238. package/dist/config-ImY6AlRE.js.map +0 -1
  239. package/dist/cost-CMzY9VXv.js +0 -1
  240. package/dist/db-ZIk6S2CD.js +0 -2
  241. package/dist/db-sHGpaLo9.js.map +0 -1
  242. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  243. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  244. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  245. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  246. package/dist/files-B3d-8Fe0.js +0 -1
  247. package/dist/files-Do4Rp3yZ.js +0 -1
  248. package/dist/harness-Ch2P81sx.js.map +0 -1
  249. package/dist/init-s1jN-PCI.js +0 -1
  250. package/dist/live-DmJix4c8.js +0 -1
  251. package/dist/live-d_5a5iUv.js.map +0 -1
  252. package/dist/model-8uZBYqV5.js +0 -1
  253. package/dist/model-DOiCiMCD.js.map +0 -1
  254. package/dist/plan-CIfOFhth.js.map +0 -1
  255. package/dist/query-kFPuqBv1.js +0 -1
  256. package/dist/questions-C_Y-oXW6.js.map +0 -1
  257. package/dist/redact-BEEGc37e.js.map +0 -1
  258. package/dist/remarks-0EXMh6AD.js.map +0 -1
  259. package/dist/remarks-BL18Id09.js +0 -1
  260. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  261. package/dist/resolve-BJSay360.js +0 -1
  262. package/dist/resume-CexsWuGK.js.map +0 -1
  263. package/dist/server-SBzrf35T.js +0 -1
  264. package/dist/source-Dn1uqPon.js +0 -1
  265. package/dist/src-BxrHipV0.js +0 -3
  266. package/dist/state-Cicc9Oa5.js +0 -1
  267. package/dist/stats-D-Gqwqt0.js.map +0 -1
  268. package/dist/suite-BYJ2Q0S-.js +0 -2
  269. package/dist/suite-BtHPBFYL.js +0 -544
  270. package/dist/suite-BtHPBFYL.js.map +0 -1
  271. package/dist/summary-zP3cCXDN.js.map +0 -1
  272. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  273. package/dist/vertex-BaAz44YN.js +0 -1
  274. package/dist/views-CmyX255j.js +0 -2
  275. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -0,0 +1,1294 @@
1
+ import { n as clip } from "./text-B8-W3Cvn.js";
2
+ import { d as pairTallies } from "./gate-Cp_pQoEe.js";
3
+ import { _ as signedRank, a as liftInterval, c as meanInterval, d as normalCdf, f as pairedLiftInterval, h as runsToHold, l as median$1, m as round3, o as mannWhitney, r as containsZero, s as mcnemar, u as minimumP, y as wilson } from "./stats-C18W4xPX.js";
4
+ import { B as bothUnderATenth, H as cellEffort, I as EFFORT_ALPHA, S as isNotGatedPass, T as notRunYet, Y as effortSample, Z as effortVerdict, i as attemptAnswerRows, k as runOutcome } from "./model-Ce9l9_rF.js";
5
+ import { a as checksByLabel, d as labeledJudgments, i as checkTries } from "./judgments-m-mb0zv8.js";
6
+ // src/view/family.ts
7
+ const signed = (x) => `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
8
+ const pct = (x) => `${Math.round(x * 100)}%`;
9
+ const pValue = (p) => p < .001 ? "<0.001" : String(p);
10
+ /** `pass@1 2/3 · 67% (95% 21…94)`, or `pass@1 67% over 3 cases (95% 30…100)` at several runs. */
11
+ function familyRateLabel(rate) {
12
+ const { interval: i } = rate;
13
+ const range = `95% ${Math.round(i.lo * 100)}…${Math.round(i.hi * 100)}`;
14
+ return i.method === "wilson" ? `pass@1 ${rate.pass}/${rate.counted} · ${pct(rate.rate)} (${range})` : `pass@1 ${pct(rate.rate)} over ${rate.cases} cases (${range})`;
15
+ }
16
+ const VERDICT_NOTE = {
17
+ lift: "",
18
+ drop: "",
19
+ noise: ", within noise",
20
+ insufficient: ", insufficient"
21
+ };
22
+ /**
23
+ * `pass@1 88% vs 20% lift +68 pts (95% +40…+82, paired, 18 vs 1 discordant, p <0.001)`. At
24
+ * one attempt a case, the counts are the discordant pairs; at several, the paired cases.
25
+ */
26
+ function familyLiftLabel(lift) {
27
+ const { interval: i } = lift;
28
+ const range = `95% ${signed(i.lo)}…${signed(i.hi)}`;
29
+ const pairs = lift.discordant ? `${lift.discordant.variantOnly} vs ${lift.discordant.controlOnly} discordant` : `${lift.pairs} cases`;
30
+ const p = lift.p === null ? "" : `, p ${pValue(lift.p)}`;
31
+ return `${`pass@1 ${pct(lift.variantRate)} vs ${pct(lift.controlRate)}`} lift ${signed(lift.points)} pts (${range}, paired, ${pairs}${p})${VERDICT_NOTE[lift.verdict]}`;
32
+ }
33
+ const metricText = (name, m) => !m || m.ratio === null ? [] : [`${name} ×${m.ratio.toFixed(2)} per case (${m.verdict}${m.p === null ? "" : `, signed-rank p ${pValue(m.p)}`})`];
34
+ /**
35
+ * `tools ×0.41 per case (fewer, signed-rank p 0.002) · agent time ×0.80 per case (noise, …)`;
36
+ * a metric whose control median is 0 has no ratio and is left out.
37
+ */
38
+ function familyEffortLabel(lift) {
39
+ const e = lift.effort;
40
+ if (!e) return null;
41
+ const parts = [
42
+ ...metricText("tools", e.tools),
43
+ ...metricText("agent time", e.agentMs),
44
+ ...metricText("tokens", e.tokens)
45
+ ];
46
+ return parts.length ? parts.join(" · ") : null;
47
+ }
48
+ /** `25 cases · paired` with a compare, and `3 of 25 cases` when a filter left some out. */
49
+ function familyHeadLabel(family) {
50
+ const cases = family.ran === family.cases ? `${family.cases} cases` : `${family.ran} of ${family.cases} cases`;
51
+ return family.lifts.length ? `${cases} · paired` : cases;
52
+ }
53
+ /**
54
+ * A case's short label: its name less the template's literal text before the first placeholder
55
+ * and after the last, so `fixes $id` labels `fixes GH-101` as `GH-101`.
56
+ */
57
+ function caseLabel(template, name) {
58
+ const matches = [...template.matchAll(/%#|%s|\$[A-Za-z_][A-Za-z0-9_]*/g)];
59
+ const first = matches[0];
60
+ const last = matches.at(-1);
61
+ if (!first || !last) return name;
62
+ const prefix = template.slice(0, first.index);
63
+ const suffix = template.slice((last.index ?? 0) + last[0].length);
64
+ if (!name.startsWith(prefix) || !name.endsWith(suffix)) return name;
65
+ return name.slice(prefix.length, name.length - suffix.length) || name;
66
+ }
67
+ /** The family's cases in list order. */
68
+ const familyCases = (family, evals) => evals.filter((e) => e.family?.id === family.id).sort((a, b) => (a.family?.case ?? 0) - (b.family?.case ?? 0));
69
+ // src/view/observations.ts
70
+ /** A scalar's tally label; numbers in an array count by their text. */
71
+ const label = (v) => typeof v === "string" ? v : JSON.stringify(v);
72
+ const scalar = (v) => v === null || typeof v !== "object";
73
+ function median(sorted) {
74
+ const mid = Math.floor(sorted.length / 2);
75
+ return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
76
+ }
77
+ /**
78
+ * Each name and cell any attempt observed it on, by name in first-observed order and then by
79
+ * cell, tallied over that cell's attempts.
80
+ */
81
+ function observedTallies(attempts) {
82
+ const byName = /* @__PURE__ */ new Map();
83
+ for (const a of attempts) for (const { name, value } of a.observations ?? []) {
84
+ const cells = byName.get(name) ?? /* @__PURE__ */ new Map();
85
+ byName.set(name, cells);
86
+ let t = cells.get(a.cell);
87
+ if (!t) {
88
+ t = {
89
+ name,
90
+ cell: a.cell,
91
+ counts: /* @__PURE__ */ new Map(),
92
+ nums: [],
93
+ others: 0,
94
+ attempts: 0
95
+ };
96
+ cells.set(a.cell, t);
97
+ }
98
+ t.attempts++;
99
+ const count = (v) => t.counts.set(v, (t.counts.get(v) ?? 0) + 1);
100
+ if (typeof value === "number") t.nums.push(value);
101
+ else if (scalar(value)) count(label(value));
102
+ else if (Array.isArray(value) && value.some(scalar)) for (const item of new Set(value.filter(scalar).map(label))) count(item);
103
+ else t.others++;
104
+ }
105
+ return [...byName.values()].flatMap((cells) => [...cells.values()]).map((t) => {
106
+ const nums = t.nums.toSorted((x, y) => x - y);
107
+ return {
108
+ name: t.name,
109
+ cell: t.cell,
110
+ attempts: t.attempts,
111
+ counts: [...t.counts].map(([value, n]) => ({
112
+ value,
113
+ n
114
+ })).sort((x, y) => y.n - x.n),
115
+ numbers: nums.length ? {
116
+ median: median(nums),
117
+ min: nums[0],
118
+ max: nums.at(-1),
119
+ n: nums.length
120
+ } : null,
121
+ others: t.others
122
+ };
123
+ });
124
+ }
125
+ /** Tallied values shown before `+N more`. */
126
+ const SHOWN = 5;
127
+ const num = (n) => String(Math.round(n * 1e3) / 1e3);
128
+ /** On one line: a line break or a terminal escape in an observed string would break the table. */
129
+ const clipValue = (v) => {
130
+ const flat = v.replace(/[\s\p{Cc}]+/gu, " ").trim();
131
+ return flat.length > 40 ? `${flat.slice(0, 39)}…` : flat;
132
+ };
133
+ /** `drizzle 9 · prisma 3`, `median 4 · range 1–12 · 13 attempts`, as the run output shows a tally. */
134
+ function observedLine(t) {
135
+ const parts = t.counts.slice(0, SHOWN).map((c) => `${clipValue(c.value)} ${c.n}`);
136
+ const more = t.counts.length - SHOWN;
137
+ if (more > 0) parts.push(`+${more} more`);
138
+ if (t.numbers) {
139
+ const { median: m, min, max } = t.numbers;
140
+ parts.push(min === max ? `${num(m)}` : `median ${num(m)} · range ${num(min)}–${num(max)}`);
141
+ }
142
+ if (t.others) parts.push(`${t.others} ${t.others === 1 ? "value" : "values"} in reeval query attempt_values`);
143
+ const tallied = t.counts.reduce((sum, c) => sum + c.n, 0);
144
+ return !t.numbers && !t.others && tallied === t.attempts ? parts.join(" · ") : `${parts.join(" · ")} · ${t.attempts} ${t.attempts === 1 ? "attempt" : "attempts"}`;
145
+ }
146
+ const sameTally = (a, b) => a.attempts === b.attempts && a.others === b.others && JSON.stringify(a.counts) === JSON.stringify(b.counts) && JSON.stringify(a.numbers) === JSON.stringify(b.numbers);
147
+ /**
148
+ * How one name's tally on one cell moved from base to head, or null when it didn't:
149
+ * `prisma 6 → 1 · drizzle 2 → 8` (each value whose count changed, head's most first),
150
+ * `median 4 → 9`, or a side's whole line against `none` when only one side observed it.
151
+ */
152
+ function observedShift(base, head) {
153
+ if (!base && !head) return null;
154
+ if (!base || !head) return base ? `${observedLine(base)} → none` : `none → ${observedLine(head)}`;
155
+ if (sameTally(base, head)) return null;
156
+ const before = new Map(base.counts.map((c) => [c.value, c.n]));
157
+ const after = new Map(head.counts.map((c) => [c.value, c.n]));
158
+ const moved = [.../* @__PURE__ */ new Set([...after.keys(), ...before.keys()])].filter((v) => (before.get(v) ?? 0) !== (after.get(v) ?? 0));
159
+ const parts = moved.slice(0, SHOWN).map((v) => `${clipValue(v)} ${before.get(v) ?? 0} → ${after.get(v) ?? 0}`);
160
+ if (moved.length > SHOWN) parts.push(`+${moved.length - SHOWN} more`);
161
+ const [b, h] = [base.numbers, head.numbers];
162
+ if (b && h && b.median !== h.median) parts.push(`median ${num(b.median)} → ${num(h.median)}`);
163
+ else if (b && h && (b.min !== h.min || b.max !== h.max)) parts.push(`range ${num(b.min)}–${num(b.max)} → ${num(h.min)}–${num(h.max)}`);
164
+ if (base.attempts !== head.attempts) parts.push(`${base.attempts} → ${head.attempts} attempts`);
165
+ return parts.length ? parts.join(" · ") : null;
166
+ }
167
+ // src/history/families.ts
168
+ const isCounted$1 = (a) => a.outcome === "pass" || a.outcome === "fail";
169
+ function sideOf(entry, cell) {
170
+ const counted = entry.attempts.filter((a) => a.cell === cell && isCounted$1(a));
171
+ if (!counted.length) return null;
172
+ return {
173
+ pass: counted.filter((a) => a.outcome === "pass").length,
174
+ counted: counted.length,
175
+ samples: counted.map(effortSample)
176
+ };
177
+ }
178
+ const clamp = (x, lo, hi) => Math.min(hi, Math.max(lo, x));
179
+ const interval = (lo, hi, method, bound) => ({
180
+ lo: round3(clamp(lo, ...bound)),
181
+ hi: round3(clamp(hi, ...bound)),
182
+ level: .95,
183
+ method
184
+ });
185
+ function rateOf(cell, sides) {
186
+ const pass = sides.reduce((s, x) => s + x.pass, 0);
187
+ const counted = sides.reduce((s, x) => s + x.counted, 0);
188
+ const single = sides.every((x) => x.counted === 1);
189
+ const rates = sides.map((x) => x.pass / x.counted);
190
+ const t = meanInterval(rates);
191
+ const w = wilson(pass, counted);
192
+ return {
193
+ cell,
194
+ cases: sides.length,
195
+ pass,
196
+ counted,
197
+ rate: round3(t.mean),
198
+ interval: single ? interval(w.low, w.high, "wilson", [0, 1]) : interval(t.low, t.high, "t-cases", [0, 1])
199
+ };
200
+ }
201
+ function metric(pairs) {
202
+ const differences = pairs.map(([v, c]) => v - c);
203
+ const control = median$1(pairs.map(([, c]) => c)) ?? 0;
204
+ const variant = median$1(pairs.map(([v]) => v)) ?? 0;
205
+ const test = signedRank(differences);
206
+ const verdict = differences.filter((d) => d !== 0).length < 6 ? "insufficient" : test === null || test.p >= .05 ? "noise" : test.sign < 0 ? "fewer" : "more";
207
+ return {
208
+ pairs: pairs.length,
209
+ ratio: control === 0 ? null : round3(variant / control),
210
+ p: test === null ? null : round3(test.p),
211
+ verdict
212
+ };
213
+ }
214
+ /**
215
+ * Each case's variant and control medians of one effort measure, over the cases where every
216
+ * counted attempt on both sides has it: the pairs a family's paired effort test ranks.
217
+ */
218
+ const casePairs = (pairs, pick) => pairs.flatMap(([v, c]) => {
219
+ const vs = v.map(pick);
220
+ const cs = c.map(pick);
221
+ if (!vs.every((x) => x !== void 0) || !cs.every((x) => x !== void 0)) return [];
222
+ return [[median$1(vs) ?? 0, median$1(cs) ?? 0]];
223
+ });
224
+ function effortOf$1(sides) {
225
+ const pairs = sides.map(([v, c]) => [v.samples, c.samples]);
226
+ const per = (pick) => casePairs(pairs, pick);
227
+ const tokens = per((s) => s.tokens);
228
+ return {
229
+ tools: metric(per((s) => s.tools)),
230
+ agentMs: metric(per((s) => s.agentMs)),
231
+ ...tokens.length ? { tokens: metric(tokens) } : {}
232
+ };
233
+ }
234
+ function liftOf(variant, control, cases, minLift) {
235
+ const pairs = cases.flatMap((e) => {
236
+ const v = sideOf(e, variant);
237
+ const c = sideOf(e, control);
238
+ return v && c ? [[v, c]] : [];
239
+ });
240
+ if (!pairs.length) return null;
241
+ const single = pairs.every(([v, c]) => v.counted === 1 && c.counted === 1);
242
+ const vRates = pairs.map(([v]) => v.pass / v.counted);
243
+ const cRates = pairs.map(([, c]) => c.pass / c.counted);
244
+ const mean = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
245
+ let points;
246
+ let range;
247
+ let p;
248
+ let discordant;
249
+ let decided;
250
+ if (single) {
251
+ const counts = {
252
+ both: 0,
253
+ variantOnly: 0,
254
+ controlOnly: 0,
255
+ neither: 0
256
+ };
257
+ for (const [v, c] of pairs) if (v.pass && c.pass) counts.both++;
258
+ else if (v.pass) counts.variantOnly++;
259
+ else if (c.pass) counts.controlOnly++;
260
+ else counts.neither++;
261
+ const paired = pairedLiftInterval(counts);
262
+ points = paired.points;
263
+ range = interval(paired.low, paired.high, "newcombe-paired", [-1, 1]);
264
+ p = round3(mcnemar(counts.variantOnly, counts.controlOnly));
265
+ discordant = {
266
+ variantOnly: counts.variantOnly,
267
+ controlOnly: counts.controlOnly
268
+ };
269
+ decided = counts.variantOnly + counts.controlOnly >= 6;
270
+ } else {
271
+ const differences = pairs.map(([v, c]) => (v.pass * c.counted - c.pass * v.counted) / (v.counted * c.counted));
272
+ const t = meanInterval(differences);
273
+ points = t.mean;
274
+ range = interval(t.low, t.high, "t-cases", [-1, 1]);
275
+ const rank = signedRank(differences);
276
+ p = rank === null ? null : round3(rank.p);
277
+ decided = pairs.length >= 6;
278
+ }
279
+ const verdict = !decided ? "insufficient" : range.lo > 0 ? "lift" : range.hi < 0 ? "drop" : "noise";
280
+ return {
281
+ variant,
282
+ vs: control,
283
+ pairs: pairs.length,
284
+ variantRate: round3(mean(vRates)),
285
+ controlRate: round3(mean(cRates)),
286
+ ...discordant ? { discordant } : {},
287
+ points: round3(points),
288
+ interval: range,
289
+ p,
290
+ withinNoise: containsZero(range.lo, range.hi),
291
+ verdict,
292
+ ...minLift !== void 0 ? { minLift } : {},
293
+ outcome: minLift !== void 0 && range.lo < minLift ? "fail" : "pass",
294
+ effort: effortOf$1(pairs)
295
+ };
296
+ }
297
+ /**
298
+ * The run's pooled families, in the order their first case appears; `minLiftOf` gives a variant
299
+ * cell's `compare.minLift`.
300
+ */
301
+ function familiesOf(evals, minLiftOf) {
302
+ const byId = /* @__PURE__ */ new Map();
303
+ for (const e of evals) {
304
+ if (!e.family || e.todo) continue;
305
+ byId.set(e.family.id, [...byId.get(e.family.id) ?? [], e]);
306
+ }
307
+ return [...byId.values()].map((members) => {
308
+ const cases = [...members].sort((a, b) => (a.family?.case ?? 0) - (b.family?.case ?? 0));
309
+ const first = cases[0];
310
+ const family = first.family;
311
+ const cellKeys = [];
312
+ for (const e of cases) for (const c of e.cells) if (!cellKeys.includes(c.key)) cellKeys.push(c.key);
313
+ const rates = cellKeys.flatMap((key) => {
314
+ const sides = cases.flatMap((e) => sideOf(e, key) ?? []);
315
+ return sides.length ? [rateOf(key, sides)] : [];
316
+ });
317
+ const lifts = [];
318
+ for (const key of cellKeys) {
319
+ const holder = cases.find((e) => e.cells.some((c) => c.key === key && c.role === "variant"));
320
+ const variant = holder?.cells.find((c) => c.key === key);
321
+ if (!holder || !variant) continue;
322
+ const control = holder.cells.find((c) => c.role === "control" && c.harness === variant.harness && c.sandbox === variant.sandbox);
323
+ if (!control) continue;
324
+ const minLift = cases.flatMap((e) => e.cells.filter((c) => c.key === key)).map(minLiftOf).find((m) => m !== void 0);
325
+ const lift = liftOf(key, control.key, cases, minLift);
326
+ if (lift) lifts.push(lift);
327
+ }
328
+ return {
329
+ id: family.id,
330
+ file: first.file,
331
+ group: first.group,
332
+ name: family.name,
333
+ cases: family.cases,
334
+ ran: cases.length,
335
+ rates,
336
+ lifts,
337
+ outcome: lifts.some((l) => l.outcome === "fail") ? "fail" : "pass"
338
+ };
339
+ });
340
+ }
341
+ // src/view/measures.ts
342
+ const LABEL_CHARS = 60;
343
+ /**
344
+ * What a difference its test doesn't hold needs to lean, else it reads `same`, so run-to-run noise
345
+ * doesn't stand in for a difference: at least the measure's smallest effect (pass-rate points; a
346
+ * judge's 0–1 score; a question's decisive win share; effort's relative difference in medians),
347
+ * and its test's p below `LEAN_P`. A question needs `QUESTION_PAIRS_MIN` decisive pairs and a p
348
+ * below `QUESTION_LEAN_P` instead: questions are few by design, so a unanimous 4 of 4 (p 0.125)
349
+ * leans. A difference its test holds is better or worse at any size.
350
+ */
351
+ const RATE_MIN = .1;
352
+ const JUDGE_MIN = .05;
353
+ const QUESTION_MIN = .6;
354
+ const EFFORT_MIN = .1;
355
+ const LEAN_P = .1;
356
+ const QUESTION_LEAN_P = .2;
357
+ /**
358
+ * A lean below its measure's smallest effect, or at a p of `leanP` or more, is `same`; unless a
359
+ * sample this size can't reach `leanP` at all (`floor`, the test's smallest p at these sizes, ties
360
+ * aside), when it stays a lean on too few runs, since its p says nothing.
361
+ */
362
+ const screened = (verdict, size, min, p, floor, leanP = LEAN_P) => verdict === "leans" && (size < min - 1e-9 || p === null || p >= leanP && floor < leanP) ? "same" : verdict;
363
+ /** On a lean its test can't reach `LEAN_P` on, the sample it has and the fewest that can. */
364
+ const shortOf = (verdict, floor, now, reaches) => verdict === "leans" && floor >= .1 ? {
365
+ now,
366
+ at: smallestFrom(now + 1, reaches)
367
+ } : null;
368
+ /** `12× faster, but 3 runs a side can't confirm it; 4 is the least that can`: a lean's text. */
369
+ function leanText(m) {
370
+ if (!m.short) return m.text;
371
+ const { now, at } = m.short;
372
+ const has = m.unit === "cases" ? `${now} ${now === 1 ? "case" : "cases"}` : `${now} ${now === 1 ? "run" : "runs"} a side`;
373
+ return `${m.text}, but ${has} can't confirm it; ${at} is the least that can`;
374
+ }
375
+ /** Two-sided p of a pooled two-proportion z test: it screens a rate's lean, never its verdict. */
376
+ function proportionP(v, c) {
377
+ const pooled = (v.pass + c.pass) / (v.counted + c.counted);
378
+ const se = Math.sqrt(pooled * (1 - pooled) * (1 / v.counted + 1 / c.counted));
379
+ if (se === 0) return 1;
380
+ const z = Math.abs(v.pass / v.counted - c.pass / c.counted) / se;
381
+ return 2 * (1 - normalCdf(z));
382
+ }
383
+ const isCounted = (a) => a.outcome === "pass" || a.outcome === "fail";
384
+ const sign = (x) => x > 0 ? "+" : x < 0 ? "−" : "±";
385
+ const pts = (x) => `${sign(x)}${Math.round(Math.abs(x) * 100)} pts`;
386
+ const mean = (n) => n.reduce((a, b) => a + b, 0) / n.length;
387
+ /** `5×`, `1.8×`, or `20%` below 1.5×: how many times, or how much, one side's median is the other's. */
388
+ function factor(ratio, lower) {
389
+ if (ratio === null || ratio <= 0) return "";
390
+ const f = lower ? 1 / ratio : ratio;
391
+ if (f >= 1.5) return `${Number(f.toFixed(f >= 10 ? 0 : 1))}× `;
392
+ const pct = Math.round(Math.abs(1 - ratio) * 100);
393
+ return pct ? `${pct}% ` : "";
394
+ }
395
+ /** The z a two-sided normal p corresponds to. */
396
+ function zOf(p) {
397
+ let lo = 0;
398
+ let hi = 40;
399
+ for (let i = 0; i < 100; i++) {
400
+ const mid = (lo + hi) / 2;
401
+ if (2 * (1 - normalCdf(mid)) > p) lo = mid;
402
+ else hi = mid;
403
+ }
404
+ return (lo + hi) / 2;
405
+ }
406
+ const smallestFrom = (from, reaches) => {
407
+ let n = from;
408
+ while (!reaches(n)) n++;
409
+ return n;
410
+ };
411
+ const smallest = (reaches) => smallestFrom(1, reaches);
412
+ /** Whether a sample of `n` a side (or `n` cases) can reach `LEAN_P` on each test. */
413
+ const rankReaches = (n) => minimumP(n, n) < LEAN_P;
414
+ const signedReaches = (n) => 2 / 2 ** n < LEAN_P;
415
+ const rateFloor = (nv, nc) => proportionP({
416
+ pass: nv,
417
+ counted: nv
418
+ }, {
419
+ pass: 0,
420
+ counted: nc
421
+ });
422
+ /** A rate confirms only from `COMPARE_RUNS` a side, where lift's interval can hold. */
423
+ const rateReaches = (n) => n >= 4 && rateFloor(n, n) < .1;
424
+ /**
425
+ * The sample at which a rank test's lean would hold at `EFFORT_ALPHA`: at a fixed effect a z grows
426
+ * with the square root of the sample, so the observed one scaled up to the critical z, and never
427
+ * below the size whose smallest possible p reaches it (`floor`). A p of 1 on a difference (one run
428
+ * a side) gives no z to scale, so the floor is all it can say.
429
+ */
430
+ function scaled(p, now, floor) {
431
+ const z = zOf(p);
432
+ if (z === 0) return Math.max(now + 1, floor);
433
+ const n = Math.max(now + 1, floor, Math.ceil(now * (zOf(EFFORT_ALPHA) / z) ** 2));
434
+ return n > 200 ? null : n;
435
+ }
436
+ const RANK_FLOOR = smallest((n) => minimumP(n, n) < EFFORT_ALPHA);
437
+ const SIGNED_FLOOR = smallest((n) => 2 / 2 ** n < EFFORT_ALPHA);
438
+ /**
439
+ * The runs a side (a case, over `cases`) at which a question's decisive pairs would hold: one pair
440
+ * a run, the same share of them decisive and the same share of those for the leading side,
441
+ * through the exact sign test.
442
+ */
443
+ function signRuns(wins, cases) {
444
+ const decisive = wins.variant + wins.control;
445
+ const share = Math.max(wins.variant, wins.control) / decisive;
446
+ const runs = wins.voted / cases;
447
+ for (let n = Math.floor(runs) + 1; n <= 200; n++) {
448
+ const d = Math.round(decisive * n / runs);
449
+ const lead = Math.round(d * share);
450
+ if (mcnemar(lead, d - lead) < .05) return n;
451
+ }
452
+ return null;
453
+ }
454
+ /**
455
+ * A pass rate's verdict, lift's maths: `same` at equal rates; `better` or `worse` when the
456
+ * interval on the difference excludes 0 on `COMPARE_RUNS` or more a side; else `leans`, with the
457
+ * runs at which the observed rates would hold.
458
+ */
459
+ function rateMeasure(measure, label, v, c, stored) {
460
+ const vr = v.pass / v.counted;
461
+ const cr = c.pass / c.counted;
462
+ const computed = liftInterval({
463
+ pass: v.pass,
464
+ n: v.counted
465
+ }, {
466
+ pass: c.pass,
467
+ n: c.counted
468
+ });
469
+ const lo = stored?.lo ?? computed.low;
470
+ const hi = stored?.hi ?? computed.high;
471
+ const holds = !containsZero(lo, hi) && Math.min(v.counted, c.counted) >= 4;
472
+ const way = vr > cr ? "better" : "worse";
473
+ const p = proportionP(v, c);
474
+ const floor = rateFloor(v.counted, c.counted);
475
+ const verdict = screened(vr === cr ? "same" : holds ? way : "leans", Math.abs(vr - cr), RATE_MIN, p, floor);
476
+ const name = measure === "pass" ? "pass rate" : `"${clip(label ?? "", LABEL_CHARS)}"`;
477
+ const from = Math.max(v.counted, c.counted, 3) + 1;
478
+ return {
479
+ measure,
480
+ label,
481
+ variant: vr,
482
+ control: cr,
483
+ estimate: vr - cr,
484
+ lo,
485
+ hi,
486
+ p: null,
487
+ verdict,
488
+ leaning: verdict === "leans" ? way : null,
489
+ text: vr === cr ? `same ${name}` : `${name} ${pts(vr - cr)}`,
490
+ needed: verdict !== "leans" ? null : containsZero(lo, hi) ? runsToHold(vr, cr, from) : Math.max(from, 4),
491
+ unit: "runs",
492
+ short: shortOf(verdict, floor, Math.min(v.counted, c.counted), rateReaches)
493
+ };
494
+ }
495
+ /** A verdict from a test's p and the sign of its statistic (positive: the variant's is higher). */
496
+ const tested = (p, s, higherIsBetter) => {
497
+ if (s === 0) return "same";
498
+ return p < .05 ? s > 0 === higherIsBetter ? "better" : "worse" : "leans";
499
+ };
500
+ const leaningOf = (verdict, s, higherIsBetter) => verdict === "leans" ? s > 0 === higherIsBetter ? "better" : "worse" : null;
501
+ const EFFORT = [
502
+ {
503
+ measure: "tools",
504
+ pick: (s) => s.tools,
505
+ words: (fewer) => fewer ? "fewer tool calls" : "more tool calls",
506
+ same: "same tool calls"
507
+ },
508
+ {
509
+ measure: "agentMs",
510
+ pick: (s) => s.agentMs,
511
+ words: (fewer) => fewer ? "faster" : "slower",
512
+ same: "same agent time"
513
+ },
514
+ {
515
+ measure: "tokens",
516
+ pick: (s) => s.tokens,
517
+ words: (fewer) => fewer ? "fewer tokens" : "more tokens",
518
+ same: "same tokens"
519
+ }
520
+ ];
521
+ /** Values of one effort measure, left out where an attempt didn't report it (tokens). */
522
+ const valuesOf = (samples, pick) => samples.flatMap((s) => {
523
+ const x = pick(s);
524
+ return x === void 0 ? [] : [x];
525
+ });
526
+ function effortMeasures(input, runs) {
527
+ const out = [];
528
+ const vs = input.variant.map((a) => a.attempts.map(effortSample));
529
+ const cs = input.control.map((a) => a.attempts.map(effortSample));
530
+ for (const m of EFFORT) {
531
+ const v = valuesOf(vs.flat(), m.pick);
532
+ const c = valuesOf(cs.flat(), m.pick);
533
+ if (!v.length || !c.length) continue;
534
+ const vm = median$1(v) ?? 0;
535
+ const cm = median$1(c) ?? 0;
536
+ const ratio = cm === 0 ? null : vm / cm;
537
+ let p;
538
+ let s;
539
+ let floor;
540
+ let now;
541
+ let reaches;
542
+ let verdict;
543
+ let needed = null;
544
+ let unit = "runs";
545
+ if (input.family?.effort) {
546
+ const metric = m.measure === "tokens" ? input.family.effort.tokens : input.family.effort[m.measure];
547
+ if (!metric) continue;
548
+ const cases = casePairs(vs.map((x, i) => [x, cs[i] ?? []]), m.pick);
549
+ const test = signedRank(cases.map(([a, b]) => a - b));
550
+ s = test?.sign ?? 0;
551
+ p = test?.p ?? 1;
552
+ now = cases.length;
553
+ floor = Math.min(1, 2 / 2 ** now);
554
+ reaches = signedReaches;
555
+ verdict = metric.verdict === "fewer" ? "better" : metric.verdict === "more" ? "worse" : s === 0 ? "same" : "leans";
556
+ unit = "cases";
557
+ if (verdict === "leans") needed = scaled(p, cases.length, Math.max(SIGNED_FLOOR, 6));
558
+ } else {
559
+ const test = mannWhitney(v, c);
560
+ if (!test) continue;
561
+ const printed = effortVerdict(v, c).verdict;
562
+ s = test.sign;
563
+ p = test.p;
564
+ floor = minimumP(v.length, c.length);
565
+ now = Math.min(v.length, c.length);
566
+ reaches = rankReaches;
567
+ verdict = printed === "fewer" ? "better" : printed === "more" ? "worse" : s === 0 ? "same" : "leans";
568
+ if (verdict === "leans") needed = scaled(p, runs, RANK_FLOOR);
569
+ }
570
+ verdict = m.measure === "agentMs" && bothUnderATenth(vm, cm) ? "same" : screened(verdict, ratio === null ? vm === 0 ? 0 : Number.POSITIVE_INFINITY : Math.abs(ratio - 1), EFFORT_MIN, p, floor);
571
+ if (verdict !== "leans") needed = null;
572
+ const fewer = s < 0;
573
+ const agrees = (r) => r !== null && r < 1 === fewer && r !== 1;
574
+ const cMean = mean(c);
575
+ const meanRatio = cMean === 0 ? null : mean(v) / cMean;
576
+ const shown = agrees(ratio) ? ratio : agrees(meanRatio) ? meanRatio : null;
577
+ out.push({
578
+ measure: m.measure,
579
+ label: null,
580
+ variant: vm,
581
+ control: cm,
582
+ estimate: ratio,
583
+ lo: null,
584
+ hi: null,
585
+ p,
586
+ verdict,
587
+ leaning: leaningOf(verdict, s, false),
588
+ text: verdict === "same" ? m.same : `${shown === null ? "" : factor(shown, fewer)}${m.words(fewer)}`,
589
+ needed,
590
+ unit,
591
+ short: shortOf(verdict, floor, now, reaches)
592
+ });
593
+ }
594
+ return out;
595
+ }
596
+ function checkMeasures(input) {
597
+ const sum = (arms) => {
598
+ const by = /* @__PURE__ */ new Map();
599
+ for (const arm of arms) for (const k of arm.checks ?? []) {
600
+ const t = by.get(k.label) ?? {
601
+ pass: 0,
602
+ counted: 0
603
+ };
604
+ t.pass += k.pass;
605
+ t.counted += k.pass + k.fail;
606
+ by.set(k.label, t);
607
+ }
608
+ return by;
609
+ };
610
+ const v = sum(input.variant);
611
+ const c = sum(input.control);
612
+ return [...v].flatMap(([label, side]) => {
613
+ const other = c.get(label);
614
+ if (!other || !side.counted || !other.counted) return [];
615
+ return [rateMeasure("check", label, side, other)];
616
+ });
617
+ }
618
+ function judgeMeasures(input, runs) {
619
+ const scores = (arms) => {
620
+ const by = /* @__PURE__ */ new Map();
621
+ for (const a of arms.flatMap((x) => x.attempts)) for (const j of labeledJudgments(a)) {
622
+ const key = j.check ?? j.label;
623
+ by.set(key, [...by.get(key) ?? [], j.judgment.score]);
624
+ }
625
+ return by;
626
+ };
627
+ const v = scores(input.variant);
628
+ const c = scores(input.control);
629
+ return [...v].flatMap(([label, xs]) => {
630
+ const ys = c.get(label);
631
+ const test = ys && mannWhitney(xs, ys);
632
+ if (!ys || !test) return [];
633
+ const diff = mean(xs) - mean(ys);
634
+ const verdict = screened(tested(test.p, test.sign, true), Math.abs(diff), JUDGE_MIN, test.p, minimumP(xs.length, ys.length));
635
+ const name = `judge "${clip(label, LABEL_CHARS)}"`;
636
+ return [{
637
+ measure: "judge",
638
+ label,
639
+ variant: mean(xs),
640
+ control: mean(ys),
641
+ estimate: diff,
642
+ lo: null,
643
+ hi: null,
644
+ p: test.p,
645
+ verdict,
646
+ leaning: leaningOf(verdict, test.sign, true),
647
+ text: verdict === "same" ? `same ${name}` : `${name} ${sign(diff)}${Math.abs(diff).toFixed(2)}`,
648
+ needed: verdict === "leans" ? scaled(test.p, runs, RANK_FLOOR) : null,
649
+ unit: "runs",
650
+ short: shortOf(verdict, minimumP(xs.length, ys.length), Math.min(xs.length, ys.length), rankReaches)
651
+ }];
652
+ });
653
+ }
654
+ function questionMeasures(input, cases) {
655
+ return [...input.questions].flatMap(([question, wins]) => {
656
+ if (!wins.voted) return [];
657
+ const decisive = wins.variant + wins.control;
658
+ const p = mcnemar(wins.variant, wins.control);
659
+ const s = wins.variant - wins.control;
660
+ const verdict = screened(tested(p, s, true), decisive >= 4 ? Math.max(wins.variant, wins.control) / decisive : 0, QUESTION_MIN, p, 0, QUESTION_LEAN_P);
661
+ const name = `"${clip(question, LABEL_CHARS)}"`;
662
+ const lead = s > 0 ? "the variant" : "the baseline";
663
+ const votes = wins.variantVotes + wins.controlVotes;
664
+ const pairs = `${Math.max(wins.variant, wins.control)} of ${decisive} decisive ${decisive === 1 ? "pair" : "pairs"}`;
665
+ return [{
666
+ measure: "question",
667
+ label: question,
668
+ variant: wins.variant,
669
+ control: wins.control,
670
+ estimate: decisive ? wins.variant / decisive : null,
671
+ lo: null,
672
+ hi: null,
673
+ p: decisive ? p : null,
674
+ verdict,
675
+ leaning: leaningOf(verdict, s, true),
676
+ text: s === 0 ? `${name} picked ${decisive ? "each side as often" : "neither"}` : `${name} picked ${lead} in ${s > 0 ? wins.variantVotes : wins.controlVotes} of ${votes} votes (${pairs})`,
677
+ needed: verdict === "leans" ? signRuns(wins, cases) : null,
678
+ unit: "runs",
679
+ short: null
680
+ }];
681
+ });
682
+ }
683
+ function measuresOf(input) {
684
+ const runs = Math.max(1, ...[...input.variant, ...input.control].map((a) => a.attempts.length));
685
+ return [
686
+ input.pass,
687
+ ...effortMeasures(input, runs),
688
+ ...judgeMeasures(input, runs),
689
+ ...questionMeasures(input, input.variant.length),
690
+ ...checkMeasures(input)
691
+ ];
692
+ }
693
+ /**
694
+ * Each pairwise question's decisive pairs for the variant and for the control, summed over the
695
+ * entries. A pair decides only when both its orders pick the same side (or one is a tie): a split
696
+ * followed the position, not the work, and a tie says nothing.
697
+ */
698
+ function questionWins(pairs) {
699
+ const out = /* @__PURE__ */ new Map();
700
+ for (const { entry, variant, control } of pairs) {
701
+ const v = entry.cells.find((c) => c.key === variant);
702
+ const c = entry.cells.find((x) => x.key === control);
703
+ if (!v || !c || v.harness !== c.harness) continue;
704
+ for (const answer of entry.answers ?? []) {
705
+ for (const t of pairTallies(answer.votes)) {
706
+ if (t.cut !== "within" || t.held !== v.harness) continue;
707
+ const forward = t.a === v.environment && t.b === c.environment;
708
+ const back = t.a === c.environment && t.b === v.environment;
709
+ if (!forward && !back) continue;
710
+ const wins = out.get(answer.question) ?? {
711
+ variant: 0,
712
+ control: 0,
713
+ voted: 0,
714
+ variantVotes: 0,
715
+ controlVotes: 0
716
+ };
717
+ wins.variant += forward ? t.aWins : t.bWins;
718
+ wins.control += forward ? t.bWins : t.aWins;
719
+ wins.voted += t.pairs;
720
+ out.set(answer.question, wins);
721
+ }
722
+ for (const vote of answer.votes) {
723
+ const wins = out.get(answer.question);
724
+ if (!wins || vote.cut !== "within" || vote.harness !== v.harness) continue;
725
+ const sides = /* @__PURE__ */ new Set([vote.a, vote.b]);
726
+ if (!sides.has(v.environment) || !sides.has(c.environment)) continue;
727
+ if (vote.winner === v.environment) wins.variantVotes++;
728
+ else if (vote.winner === c.environment) wins.controlVotes++;
729
+ }
730
+ }
731
+ }
732
+ return out;
733
+ }
734
+ const armOf = (e, key) => ({
735
+ attempts: e.attempts.filter((a) => a.cell === key && isCounted(a)),
736
+ checks: e.cells.find((c) => c.key === key)?.checks
737
+ });
738
+ /** A variant cell against its control in one entry; null when either side counted nothing. */
739
+ function cellMeasures(entry, variant, control) {
740
+ const v = entry.cells.find((c) => c.key === variant);
741
+ const c = entry.cells.find((x) => x.key === control);
742
+ if (!v || !c || v.pass + v.fail === 0 || c.pass + c.fail === 0) return null;
743
+ const lift = v.lift?.vs === control ? v.lift : void 0;
744
+ return measuresOf({
745
+ variant: [armOf(entry, variant)],
746
+ control: [armOf(entry, control)],
747
+ pass: rateMeasure("pass", null, {
748
+ pass: v.pass,
749
+ counted: v.pass + v.fail
750
+ }, {
751
+ pass: c.pass,
752
+ counted: c.pass + c.fail
753
+ }, lift?.interval),
754
+ questions: questionWins([{
755
+ entry,
756
+ variant,
757
+ control
758
+ }])
759
+ });
760
+ }
761
+ /**
762
+ * A family's variant against its control over its cases: pass rate is the family's pooled lift
763
+ * (`lift`/`drop` hold, a difference within noise or `insufficient` leans), effort its paired
764
+ * verdict by case, and checks, judges and questions every case's attempts pooled.
765
+ */
766
+ function familyMeasures(cases, lift) {
767
+ const paired = cases.filter((e) => e.cells.some((c) => c.key === lift.variant && c.pass + c.fail > 0) && e.cells.some((c) => c.key === lift.vs && c.pass + c.fail > 0));
768
+ const same = lift.variantRate === lift.controlRate;
769
+ const way = lift.variantRate > lift.controlRate ? "better" : "worse";
770
+ const verdict = screened(same ? "same" : lift.verdict === "lift" ? "better" : lift.verdict === "drop" ? "worse" : "leans", Math.abs(lift.points), RATE_MIN, lift.p, 0);
771
+ return measuresOf({
772
+ variant: paired.map((e) => armOf(e, lift.variant)),
773
+ control: paired.map((e) => armOf(e, lift.vs)),
774
+ pass: {
775
+ measure: "pass",
776
+ label: null,
777
+ variant: lift.variantRate,
778
+ control: lift.controlRate,
779
+ estimate: lift.points,
780
+ lo: lift.interval.lo,
781
+ hi: lift.interval.hi,
782
+ p: lift.p,
783
+ verdict,
784
+ leaning: verdict === "leans" ? way : null,
785
+ text: same ? "same pass rate" : `pass rate ${pts(lift.points)}`,
786
+ needed: null,
787
+ unit: "cases",
788
+ short: null
789
+ },
790
+ questions: questionWins(paired.map((entry) => ({
791
+ entry,
792
+ variant: lift.variant,
793
+ control: lift.vs
794
+ }))),
795
+ family: lift
796
+ });
797
+ }
798
+ function readMeasures(measures) {
799
+ const rest = measures.filter((m) => m.measure !== "pass");
800
+ const apart = rest.filter((m) => m.verdict === "better" || m.verdict === "worse");
801
+ if (apart.length) return {
802
+ told: "apart",
803
+ measures: apart
804
+ };
805
+ const leans = rest.filter((m) => m.verdict === "leans");
806
+ if (!leans.length) return { told: "same" };
807
+ const confirmable = leans.filter((m) => m.needed !== null);
808
+ const unit = confirmable.some((m) => m.unit === "runs") ? "runs" : "cases";
809
+ const soonest = confirmable.filter((m) => m.unit === unit).sort((a, b) => (a.needed ?? 0) - (b.needed ?? 0))[0];
810
+ return {
811
+ told: "leans",
812
+ measures: leans,
813
+ needed: soonest?.needed ?? null,
814
+ unit: soonest?.unit ?? leans[0]?.unit ?? "runs"
815
+ };
816
+ }
817
+ /** `same pass rate · 5× faster · judge "tidy" +0.12`: pass rate, then the measures that hold. */
818
+ const measuresLine = (measures, leans = false) => measures.filter((m) => m.measure === "pass" || m.verdict === "better" || m.verdict === "worse" || leans && m.verdict === "leans").map((m) => m.verdict === "leans" && m.measure !== "pass" ? `looks ${m.text}` : m.text).join(" · ");
819
+ /**
820
+ * Every side passed every counted run (`pass`), or failed every one (`fail`): a cell, or each of a
821
+ * family's case cells. Null when they're mixed, or when any side counted nothing.
822
+ */
823
+ function boundOf(sides) {
824
+ if (!sides.length || sides.some((s) => s.pass + s.fail === 0)) return null;
825
+ if (sides.every((s) => s.fail === 0)) return "pass";
826
+ return sides.every((s) => s.pass === 0) ? "fail" : null;
827
+ }
828
+ /**
829
+ * How arms that both passed, or both failed, every run read: a floor is always a `tie` (a timeout
830
+ * counts as a fail, so effort differs exactly when one arm hangs); a ceiling is a `tie` only when
831
+ * every other measure is the same, a lean when some only lean, and null when one is better or
832
+ * worse, since then the comparison is the result.
833
+ */
834
+ function tieOf(bound, measures) {
835
+ if (bound === "fail") return "tie";
836
+ const reading = readMeasures(measures ?? []);
837
+ return reading.told === "same" ? "tie" : reading.told === "leans" ? reading : null;
838
+ }
839
+ // src/view/suite.ts
840
+ /** Summed per provider, in first-seen order. */
841
+ function tallyUpstreams(counts) {
842
+ const by = /* @__PURE__ */ new Map();
843
+ for (const { provider, calls } of counts) by.set(provider, (by.get(provider) ?? 0) + calls);
844
+ return [...by].map(([provider, calls]) => ({
845
+ provider,
846
+ calls
847
+ }));
848
+ }
849
+ /** A cell's attempts the run's interrupt canceled: recorded as fails, but not the agent's. */
850
+ const canceledOf = (cell) => cell.errors.filter((err) => err.kind === "aborted").reduce((n, err) => n + err.attempts, 0);
851
+ /** A cell's counted tries: its passes and fails, less the ones the interrupt canceled. */
852
+ const countedOf = (cell) => Math.max(0, cell.passed + cell.failed - canceledOf(cell));
853
+ /** The direction of `variant`'s pass rate against `control`'s; null when a side counted none. */
854
+ function effectOf(variant, control) {
855
+ if (variant.counted === 0 || control.counted === 0) return null;
856
+ const v = variant.pass * control.counted;
857
+ const c = control.pass * variant.counted;
858
+ return v > c ? "better" : v < c ? "worse" : "same";
859
+ }
860
+ /** A check's passes over the attempts that answered it; `notRun` and `error` aren't counted. */
861
+ const checkSide = (t) => ({
862
+ pass: t.pass,
863
+ counted: t.pass + t.fail
864
+ });
865
+ /** The check's direction on the variant against the baseline; null when a side never answered it. */
866
+ const checkEffect = (variant, control) => effectOf(checkSide(variant), checkSide(control));
867
+ /** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
868
+ const factSegment = (s) => s.replace(/%/g, "%25").replace(/\//g, "%2F").replace(/\+/g, "%2B");
869
+ const cellSegment = (c) => [
870
+ c.harness,
871
+ c.environment,
872
+ c.sandbox
873
+ ].map(factSegment).join("+");
874
+ const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
875
+ function evalAcross(cells) {
876
+ const harness = new Set(cells.map((c) => c.harness)).size > 1;
877
+ const change = cells.some((c) => c.role === "variant") || new Set(cells.map((c) => c.environment)).size > 1;
878
+ return harness && change ? "both" : harness ? "harness" : change ? "change" : "none";
879
+ }
880
+ /** The skill sources and files an environment placed; `local:` sources as their path. */
881
+ function contextOf(record, cell) {
882
+ const envs = record.environments.filter((e) => e.digest === cell.environmentDigest);
883
+ const env = envs.find((e) => e.name === cell.environment) ?? envs[0];
884
+ if (!env) return [];
885
+ return [...env.skills.map((s) => s.source.replace(/^local:/, "")), ...env.files.map((f) => f.path)];
886
+ }
887
+ /** The control a variant is measured against: its lift's, else the control on its harness and sandbox. */
888
+ function baselineOf(e, cell) {
889
+ if (cell.role !== "variant") return void 0;
890
+ const vs = cell.lift?.vs;
891
+ return vs ? e.cells.find((c) => c.key === vs) : e.cells.find((c) => c.role === "control" && c.harness === cell.harness && c.sandbox === cell.sandbox);
892
+ }
893
+ /** Every check label the attempts recorded, in first-seen order: the eval's declared checks. */
894
+ function declaredChecks(entries) {
895
+ return [...new Set(entries.flatMap((e) => e.attempts.flatMap((a) => (a.assertions ?? []).map((x) => x.label))))];
896
+ }
897
+ function checksOf(e, cell, baseline, base, declared) {
898
+ const attempts = e.attempts.filter((a) => a.cell === cell.key && counted(a));
899
+ return (cell.checks ?? declared.map((label) => ({
900
+ label,
901
+ pass: 0,
902
+ fail: 0,
903
+ error: 0,
904
+ notRun: 0
905
+ }))).map((k, order) => {
906
+ const ran = k.pass + k.fail + k.error;
907
+ const answered = attempts.flatMap((a) => a.assertions ?? []).filter((x) => x.label === k.label && x.message);
908
+ const shown = answered.find((x) => x.status === "fail") ?? answered.find((x) => x.status === "error");
909
+ const assertion = shown?.message ?? null;
910
+ const control = baseline ? baseline.checks?.find((x) => x.label === k.label) ?? {
911
+ pass: 0,
912
+ fail: 0,
913
+ error: 0,
914
+ notRun: 0
915
+ } : void 0;
916
+ const judged = attempts.flatMap((a) => labeledJudgments(a).filter((j) => j.check === k.label).map((j) => j.judgment));
917
+ const first = judged[0];
918
+ const notes = attempts.flatMap((a) => {
919
+ const check = checksByLabel(a).get(k.label);
920
+ const note = check && checkTries(check);
921
+ return note ? [note] : [];
922
+ });
923
+ const retried = attempts.length === 1 ? notes[0] : notes.length ? `retried in ${notes.length} of ${attempts.length} attempts` : void 0;
924
+ return {
925
+ path: `${base}/checks/${factSegment(k.label)}`,
926
+ label: k.label,
927
+ order,
928
+ outcome: ran === 0 ? { status: "not run" } : {
929
+ status: "ran",
930
+ passed: k.pass,
931
+ failed: k.fail,
932
+ errored: k.error,
933
+ notRun: k.notRun,
934
+ assertion,
935
+ at: shown?.at ?? null
936
+ },
937
+ effect: control ? checkEffect(k, control) : null,
938
+ judge: first ? {
939
+ question: first.question,
940
+ threshold: first.threshold,
941
+ scores: judged.map((j) => j.score)
942
+ } : null,
943
+ ...retried ? { retried } : {}
944
+ };
945
+ });
946
+ }
947
+ const informative = (line) => /[\p{L}\p{N}]/u.test(line);
948
+ /**
949
+ * An error's one line: its first line with any letter or digit in it, joined with the next one
950
+ * when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
951
+ */
952
+ function messageLine(text) {
953
+ const [first = "", next] = text.split("\n").map((l) => l.trim()).filter(informative);
954
+ return first.endsWith(":") && next ? `${first} ${next.replace(/^[|>]\s*/, "")}` : first;
955
+ }
956
+ /**
957
+ * The errors of a cell's failed and infra-error attempts that no failed check explains: every
958
+ * timeout, infra error and abort, and an assertion or judge error on an attempt whose recorded
959
+ * checks all passed (a throw in the eval's body), plus each turn's error, which is often why.
960
+ */
961
+ function errorsOf(e, cell) {
962
+ const out = [];
963
+ const add = (kind, text) => {
964
+ const message = messageLine(text);
965
+ const same = out.find((x) => x.kind === kind && x.message === message);
966
+ if (same) same.attempts++;
967
+ else out.push({
968
+ kind,
969
+ message,
970
+ attempts: 1
971
+ });
972
+ };
973
+ for (const a of e.attempts) {
974
+ if (a.cell !== cell.key) continue;
975
+ if (a.outcome !== "fail" && a.outcome !== "infra_error") continue;
976
+ for (const turn of a.turns ?? []) if (turn.error) add("turn", turn.error);
977
+ if (!a.error) continue;
978
+ if ((a.error.kind === "assertion" || a.error.kind === "judge") && (a.assertions ?? []).some((x) => x.status !== "pass")) continue;
979
+ add(a.error.kind, a.error.message);
980
+ }
981
+ return out;
982
+ }
983
+ function questionsOf(e) {
984
+ const pairs = (e.answers ?? []).map((a) => ({
985
+ kind: "pairs",
986
+ question: a.question,
987
+ votes: a.votes.length,
988
+ rates: a.rates.map((r) => ({
989
+ cut: r.cut,
990
+ name: r.name,
991
+ wins: r.wins,
992
+ n: r.n
993
+ })),
994
+ error: a.error ?? null
995
+ }));
996
+ const each = attemptAnswerRows(e).map((row) => {
997
+ const failed = row.answers.filter((a) => a.error !== null);
998
+ return {
999
+ kind: "attempts",
1000
+ question: row.question,
1001
+ asked: row.answers.length,
1002
+ choices: row.choices,
1003
+ answers: row.answers.filter((a) => a.error === null).map((a) => ({
1004
+ attempt: a.attempt,
1005
+ choice: a.choice,
1006
+ text: a.text
1007
+ })),
1008
+ failed: failed.length,
1009
+ error: failed[0]?.error ?? null
1010
+ };
1011
+ });
1012
+ return [...pairs, ...each];
1013
+ }
1014
+ function effortOf(e, cell) {
1015
+ const effort = cellEffort(e, cell.key);
1016
+ if (!effort || effort.n < 4) return null;
1017
+ return {
1018
+ tools: effort.tools.median,
1019
+ agentMs: effort.agentMs.median,
1020
+ tokens: effort.tokens?.median ?? null
1021
+ };
1022
+ }
1023
+ /** A variant's direction against its baseline; null without one, or with nothing counted on a side. */
1024
+ function cellEffect(cell, baseline) {
1025
+ if (cell.lift) return effectOf(cell.lift.variant, cell.lift.control);
1026
+ if (!baseline) return null;
1027
+ return effectOf({
1028
+ pass: cell.pass,
1029
+ counted: cell.pass + cell.fail
1030
+ }, {
1031
+ pass: baseline.pass,
1032
+ counted: baseline.pass + baseline.fail
1033
+ });
1034
+ }
1035
+ function cellsOf(record, e, parent, declared) {
1036
+ const hasVariants = e.cells.some((c) => c.role === "variant");
1037
+ return e.cells.map((cell, order) => {
1038
+ const path = `${parent}/${cellSegment(cell)}`;
1039
+ const baseline = baselineOf(e, cell);
1040
+ const attempts = e.attempts.filter((a) => a.cell === cell.key && counted(a));
1041
+ const ran = e.attempts.filter((a) => a.cell === cell.key);
1042
+ const upstreams = tallyUpstreams(ran.flatMap((a) => a.upstreams ?? []));
1043
+ const judgeUpstreams = tallyUpstreams(ran.flatMap((a) => a.judgements.flatMap((j) => j.upstream ? [{
1044
+ provider: j.upstream,
1045
+ calls: 1
1046
+ }] : [])));
1047
+ const lift = cell.lift;
1048
+ const interval = lift ? {
1049
+ low: lift.interval.lo,
1050
+ high: lift.interval.hi
1051
+ } : null;
1052
+ const snapshots = ran.flatMap((a) => {
1053
+ const path = (a.outcome === "fail" || a.outcome === "infra_error") && a.snapshots?.find((s) => s.name === "final")?.path;
1054
+ return path ? [`.reeval/runs/${record.runId}/${path}`] : [];
1055
+ });
1056
+ const pageErrors = ran.flatMap((a) => (a.files.artifacts ?? []).flatMap(({ name, capture }) => capture?.pageErrors && capture.page ? [{
1057
+ capture: name,
1058
+ errors: capture.pageErrors,
1059
+ file: `.reeval/runs/${record.runId}/${capture.page.path}`
1060
+ }] : []));
1061
+ return {
1062
+ path,
1063
+ key: cell.key,
1064
+ harness: cell.harness,
1065
+ environment: cell.environment,
1066
+ sandbox: cell.sandbox,
1067
+ variant: cell.variant ?? null,
1068
+ baseline: hasVariants && cell.role === "control",
1069
+ vs: baseline?.key ?? null,
1070
+ order,
1071
+ gated: cell.gate !== "none",
1072
+ outcome: cell.outcome,
1073
+ attempts: cell.runs,
1074
+ passed: cell.pass,
1075
+ failed: cell.fail,
1076
+ infraError: cell.infraError,
1077
+ skipped: cell.skipped,
1078
+ ...cell.bestOf !== void 0 ? {
1079
+ bestOf: cell.bestOf,
1080
+ notNeeded: ran.filter((a) => a.bestOf === "skipped").length
1081
+ } : {},
1082
+ fingerprint: cell.fingerprint,
1083
+ context: contextOf(record, cell),
1084
+ checks: checksOf(e, cell, baseline, path, declared),
1085
+ effect: cell.role === "variant" ? cellEffect(cell, baseline) : null,
1086
+ interval,
1087
+ minLift: lift?.minLift ?? null,
1088
+ effort: effortOf(e, cell),
1089
+ measures: cell.role === "variant" && baseline ? cellMeasures(e, cell.key, baseline.key) : null,
1090
+ errors: errorsOf(e, cell),
1091
+ ...snapshots.length ? { snapshots } : {},
1092
+ searches: attempts.map((a) => a.activity.webSearches),
1093
+ fetches: attempts.map((a) => a.activity.webFetches),
1094
+ ...upstreams.length ? { upstreams } : {},
1095
+ ...judgeUpstreams.length ? { judgeUpstreams } : {},
1096
+ ...pageErrors.length ? { pageErrors } : {}
1097
+ };
1098
+ });
1099
+ }
1100
+ function pooledEffect(lift) {
1101
+ if (lift.variantRate === lift.controlRate) return "same";
1102
+ return lift.variantRate > lift.controlRate ? "better" : "worse";
1103
+ }
1104
+ /** Fail beats infra error, which beats canceled, which beats pass; skipped only when every one was. */
1105
+ function worst(outcomes) {
1106
+ if (outcomes.includes("fail")) return "fail";
1107
+ if (outcomes.includes("infra_error")) return "infra_error";
1108
+ if (outcomes.includes("canceled")) return "canceled";
1109
+ if (outcomes.length && outcomes.every((o) => o === "skipped")) return "skipped";
1110
+ return "pass";
1111
+ }
1112
+ /** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
1113
+ const isStudy = (cells) => cells.length > 0 && cells.every((c) => !c.gated);
1114
+ /**
1115
+ * A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
1116
+ * gating it, so it reads `not gated`, never `pass`.
1117
+ */
1118
+ const isNotGatedResult = (e) => e.outcome === "pass" && (e.cases ? e.cases.length > 0 && e.cases.every((k) => isStudy(k.cells)) && (e.pooled ?? []).every((p) => p.minLift === null) : isStudy(e.cells ?? []));
1119
+ /** The record's outcome, or `canceled` when every gated attempt that failed was canceled. */
1120
+ function outcomeOf(e) {
1121
+ if (e.outcome !== "fail") return e.outcome;
1122
+ const gated = new Set(e.cells.filter((c) => c.gate !== "none").map((c) => c.key));
1123
+ const failed = e.attempts.filter((a) => a.outcome === "fail" && gated.has(a.cell));
1124
+ return failed.length && failed.every((a) => a.error?.kind === "aborted") ? "canceled" : "fail";
1125
+ }
1126
+ /** Each question any case asked, in first-asked order, with its votes or answers summed over the cases. */
1127
+ function familyQuestions(cases) {
1128
+ const out = /* @__PURE__ */ new Map();
1129
+ for (const q of cases.flatMap(questionsOf)) {
1130
+ const key = `${q.kind}\0${q.question}`;
1131
+ const have = out.get(key);
1132
+ if (!have) {
1133
+ out.set(key, q.kind === "pairs" ? {
1134
+ ...q,
1135
+ rates: q.rates.map((r) => ({ ...r }))
1136
+ } : {
1137
+ ...q,
1138
+ choices: q.choices?.map((c) => ({ ...c })) ?? null,
1139
+ answers: [...q.answers]
1140
+ });
1141
+ continue;
1142
+ }
1143
+ have.error ??= q.error;
1144
+ if (have.kind === "attempts" && q.kind === "attempts") {
1145
+ have.asked += q.asked;
1146
+ have.failed += q.failed;
1147
+ have.answers.push(...q.answers);
1148
+ for (const c of q.choices ?? []) {
1149
+ const same = have.choices?.find((x) => x.choice === c.choice);
1150
+ if (same) same.n += c.n;
1151
+ }
1152
+ continue;
1153
+ }
1154
+ if (have.kind !== "pairs" || q.kind !== "pairs") continue;
1155
+ have.votes += q.votes;
1156
+ for (const r of q.rates) {
1157
+ const same = have.rates.find((x) => x.cut === r.cut && x.name === r.name);
1158
+ if (!same) have.rates.push({ ...r });
1159
+ else {
1160
+ same.wins += r.wins;
1161
+ same.n += r.n;
1162
+ }
1163
+ }
1164
+ }
1165
+ return [...out.values()];
1166
+ }
1167
+ function familyResult(record, family, cases, order) {
1168
+ const path = factSegment(family.id);
1169
+ const sorted = [...cases].sort((a, b) => (a.family?.case ?? 0) - (b.family?.case ?? 0));
1170
+ const declared = declaredChecks(sorted);
1171
+ const cellOf = (key) => sorted.flatMap((c) => c.cells).find((c) => c.key === key);
1172
+ const results = sorted.map((c) => {
1173
+ const label = caseLabel(family.name, c.name);
1174
+ const casePath = `${path}/${factSegment(label)}`;
1175
+ return {
1176
+ path: casePath,
1177
+ id: c.id,
1178
+ label,
1179
+ name: c.name,
1180
+ order: c.family?.case ?? 0,
1181
+ outcome: outcomeOf(c),
1182
+ notRun: notRunYet(record, c),
1183
+ cells: cellsOf(record, c, casePath, declared)
1184
+ };
1185
+ });
1186
+ return {
1187
+ path,
1188
+ id: family.id,
1189
+ group: family.group,
1190
+ name: family.name,
1191
+ template: family.name,
1192
+ order,
1193
+ across: evalAcross(cases.flatMap((c) => c.cells)),
1194
+ family: true,
1195
+ todo: false,
1196
+ notRun: results.every((c) => c.notRun),
1197
+ outcome: worst([...family.outcome === "fail" ? ["fail"] : [], ...results.map((c) => c.outcome)]),
1198
+ cells: null,
1199
+ cases: results,
1200
+ pooled: family.lifts.map((lift) => {
1201
+ const cell = cellOf(lift.variant);
1202
+ return {
1203
+ path: `${path}/${cell ? cellSegment(cell) : factSegment(lift.variant)}`,
1204
+ key: lift.variant,
1205
+ vs: lift.vs,
1206
+ effect: pooledEffect(lift),
1207
+ pairs: lift.pairs,
1208
+ interval: {
1209
+ low: lift.interval.lo,
1210
+ high: lift.interval.hi
1211
+ },
1212
+ verdict: lift.verdict,
1213
+ minLift: lift.minLift ?? null,
1214
+ outcome: lift.outcome,
1215
+ measures: familyMeasures(sorted, lift)
1216
+ };
1217
+ }),
1218
+ questions: familyQuestions(sorted),
1219
+ observations: []
1220
+ };
1221
+ }
1222
+ function evalResult(record, e, order) {
1223
+ const path = factSegment(e.id);
1224
+ return {
1225
+ path,
1226
+ id: e.id,
1227
+ group: e.group,
1228
+ name: e.name,
1229
+ template: null,
1230
+ order,
1231
+ across: evalAcross(e.cells),
1232
+ family: false,
1233
+ todo: e.todo,
1234
+ notRun: notRunYet(record, e),
1235
+ outcome: outcomeOf(e),
1236
+ cells: cellsOf(record, e, path, declaredChecks([e])),
1237
+ cases: null,
1238
+ pooled: null,
1239
+ questions: questionsOf(e),
1240
+ observations: observedTallies(e.attempts)
1241
+ };
1242
+ }
1243
+ /** The facts of a run, from its record. Pure: the same record always gives the same result. */
1244
+ function suiteResult(record) {
1245
+ const families = new Map((record.families ?? []).map((f) => [f.id, f]));
1246
+ const byFile = /* @__PURE__ */ new Map();
1247
+ const seen = /* @__PURE__ */ new Set();
1248
+ const errors = record.errors ?? [];
1249
+ const tally = {
1250
+ errors: errors.length,
1251
+ pass: 0,
1252
+ notGated: 0,
1253
+ fail: 0,
1254
+ infraError: 0,
1255
+ canceled: 0,
1256
+ skipped: 0
1257
+ };
1258
+ const count = (e) => {
1259
+ const o = outcomeOf(e);
1260
+ if (o === "infra_error") tally.infraError++;
1261
+ else if (o === "pass" && isNotGatedPass(e)) tally.notGated++;
1262
+ else tally[o]++;
1263
+ };
1264
+ for (const e of record.evals) {
1265
+ const evals = byFile.get(e.file) ?? [];
1266
+ byFile.set(e.file, evals);
1267
+ count(e);
1268
+ const family = e.family && families.get(e.family.id);
1269
+ if (!family) {
1270
+ evals.push(evalResult(record, e, evals.length));
1271
+ continue;
1272
+ }
1273
+ if (seen.has(family.id)) continue;
1274
+ seen.add(family.id);
1275
+ if (family.outcome === "fail" && family.lifts.some((l) => l.outcome === "fail")) tally.fail++;
1276
+ const cases = record.evals.filter((x) => x.family?.id === family.id);
1277
+ evals.push(familyResult(record, family, cases, evals.length));
1278
+ }
1279
+ const files = [...byFile].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([path, evals]) => ({
1280
+ path,
1281
+ evals
1282
+ }));
1283
+ return {
1284
+ runId: record.runId,
1285
+ status: record.status,
1286
+ outcome: runOutcome(record),
1287
+ errors,
1288
+ tally,
1289
+ files
1290
+ };
1291
+ }
1292
+ export { familyEffortLabel as C, familyRateLabel as E, familyCases as S, familyLiftLabel as T, familiesOf as _, evalAcross as a, observedTallies as b, isStudy as c, boundOf as d, cellMeasures as f, tieOf as g, measuresLine as h, effectOf as i, messageLine as l, leanText as m, checkEffect as n, factSegment as o, familyMeasures as p, countedOf as r, isNotGatedResult as s, canceledOf as t, suiteResult as u, observedLine as v, familyHeadLabel as w, caseLabel as x, observedShift as y };
1293
+
1294
+ //# sourceMappingURL=suite-M8_7xio7.js.map