reeval 0.17.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
  4. package/dist/adapter-C8xPuXYq.js.map +1 -0
  5. package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
  6. package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
  7. package/dist/agents-C6zhkxp3.js +1151 -0
  8. package/dist/agents-C6zhkxp3.js.map +1 -0
  9. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  10. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  11. package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
  12. package/dist/app-files-DT5mJyV4.js.map +1 -0
  13. package/dist/assemble-UPCRrtOz.js +1 -0
  14. package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
  15. package/dist/assemble-t6h018An.js.map +1 -0
  16. package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
  17. package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
  18. package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
  19. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
  20. package/dist/boat-BjLqFJIZ.js +108 -0
  21. package/dist/boat-BjLqFJIZ.js.map +1 -0
  22. package/dist/boat-CJCtzfB1.js +322 -0
  23. package/dist/boat-CJCtzfB1.js.map +1 -0
  24. package/dist/bridge/index.mjs +300 -78
  25. package/dist/capture-DoxFUN3J.js +1 -0
  26. package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
  27. package/dist/capture-iBzFiC0j.js.map +1 -0
  28. package/dist/cases-Dy0mifal.js +1 -0
  29. package/dist/cases-z9E6DCd5.js +639 -0
  30. package/dist/cases-z9E6DCd5.js.map +1 -0
  31. package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
  32. package/dist/cli-20Fpvs5H.js.map +1 -0
  33. package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
  34. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
  37. package/dist/client-KM4C5r5_.js.map +1 -0
  38. package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
  39. package/dist/command-B5OyHpOu.js.map +1 -0
  40. package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
  41. package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
  42. package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
  43. package/dist/command-CRh-cb6Z.js.map +1 -0
  44. package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
  45. package/dist/command-aUJvENwy.js.map +1 -0
  46. package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
  47. package/dist/commands-CWy-WndG.js.map +1 -0
  48. package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
  49. package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
  50. package/dist/config-B9mB1kF5.js.map +1 -0
  51. package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
  52. package/dist/context-Bkv3ndjq.js.map +1 -0
  53. package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
  54. package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
  55. package/dist/cost-Cr26Alhq.js +1 -0
  56. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
  57. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
  58. package/dist/db-P2fXw4nx.js +2 -0
  59. package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
  60. package/dist/db-zgC1tvKB.js.map +1 -0
  61. package/dist/debug-CF4p4Lp_.js +1 -0
  62. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  63. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  64. package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
  65. package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
  66. package/dist/default-effort-DGhQmu95.js.map +1 -0
  67. package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
  68. package/dist/dist-CAFD1yfv.js.map +1 -0
  69. package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
  70. package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
  71. package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
  72. package/dist/docker-DAMnm3L9.js.map +1 -0
  73. package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
  74. package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
  75. package/dist/doctor-B2PohneB.js.map +1 -0
  76. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  77. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  78. package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
  79. package/dist/environment-DB8o3hGi.js.map +1 -0
  80. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  81. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  82. package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
  83. package/dist/eval-source-D3sbAzt-.js.map +1 -0
  84. package/dist/evaluate-COdzf7Hl.d.ts +92 -0
  85. package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
  86. package/dist/evaluate-CwKeN_fo.js.map +1 -0
  87. package/dist/files-B4KQUiM7.js +1 -0
  88. package/dist/files-BEwfjD8J.js +64 -0
  89. package/dist/files-BEwfjD8J.js.map +1 -0
  90. package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
  91. package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
  92. package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
  93. package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
  94. package/dist/files-C1Rk9FYA.js +1 -0
  95. package/dist/files-C3mCC8Mn.js +1 -0
  96. package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
  97. package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
  98. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  99. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  100. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  101. package/dist/files-DlQshsks.js.map +1 -0
  102. package/dist/files-DnlA1A7N.js +1 -0
  103. package/dist/gate-CgYdaprG.js +49 -0
  104. package/dist/gate-CgYdaprG.js.map +1 -0
  105. package/dist/gateway.d.ts +1 -1
  106. package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
  107. package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
  108. package/dist/harness-C2berMiM.js +105 -0
  109. package/dist/harness-C2berMiM.js.map +1 -0
  110. package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
  111. package/dist/harness-DXv2gnaT.js.map +1 -0
  112. package/dist/harness.d.ts +1 -1
  113. package/dist/harness.js +1 -1
  114. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  115. package/dist/ids-C_advdJA.js.map +1 -0
  116. package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
  117. package/dist/images-BNgcR77K.js.map +1 -0
  118. package/dist/index.d.ts +23 -3
  119. package/dist/index.js +5 -5
  120. package/dist/init-4RPeD-K-.js +1 -0
  121. package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
  122. package/dist/init-DbuZVRrW.js.map +1 -0
  123. package/dist/ink-list-BpfidQ1h.js +274 -0
  124. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  125. package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
  126. package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
  127. package/dist/judge-COMXT1Hx.js +230 -0
  128. package/dist/judge-COMXT1Hx.js.map +1 -0
  129. package/dist/judge.d.ts +1 -1
  130. package/dist/judge.js +1 -1
  131. package/dist/judgments-CzgpsTqv.js +246 -0
  132. package/dist/judgments-CzgpsTqv.js.map +1 -0
  133. package/dist/kinds-C1oY3qyC.js +697 -0
  134. package/dist/kinds-C1oY3qyC.js.map +1 -0
  135. package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
  136. package/dist/layout-Cm85tQYk.js.map +1 -0
  137. package/dist/list-DuThsE28.js +1348 -0
  138. package/dist/list-DuThsE28.js.map +1 -0
  139. package/dist/live-CJCK11nh.js +1 -0
  140. package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
  141. package/dist/live-CgYbO4Te.js.map +1 -0
  142. package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
  143. package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
  144. package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
  145. package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
  146. package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
  147. package/dist/model-Cv4GuTsZ.js +1 -0
  148. package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
  149. package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
  150. package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
  151. package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
  152. package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
  153. package/dist/model-V3mMXp6x.js.map +1 -0
  154. package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
  155. package/dist/node-D7pAU--0.js.map +1 -0
  156. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
  157. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  158. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
  159. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  160. package/dist/options-CHubslQf.js +1 -0
  161. package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
  162. package/dist/pairs-BGb134Vu.js.map +1 -0
  163. package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
  164. package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
  165. package/dist/plain-BYUf_fUV.js +267 -0
  166. package/dist/plain-BYUf_fUV.js.map +1 -0
  167. package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
  168. package/dist/plan-Cgq9NCEX.js.map +1 -0
  169. package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
  170. package/dist/plugins-BkmPxDiY.js.map +1 -0
  171. package/dist/preflight-4gWSy0Pe.js +1 -0
  172. package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
  173. package/dist/preflight-BElE_q0u.js.map +1 -0
  174. package/dist/preflight-CtlloeOQ.js +68 -0
  175. package/dist/preflight-CtlloeOQ.js.map +1 -0
  176. package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
  177. package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
  178. package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
  179. package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
  180. package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
  181. package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
  182. package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
  183. package/dist/provenance-zsc4--dP.js.map +1 -0
  184. package/dist/provider-BnUoz4KB.js +1 -0
  185. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  186. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  187. package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
  188. package/dist/providers-B8xYMeah.js.map +1 -0
  189. package/dist/providers-CVu8e0xS.js +1 -0
  190. package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
  191. package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
  192. package/dist/query-DIfkUcPh.js +1 -0
  193. package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
  194. package/dist/questions-CWlR9ffu.js.map +1 -0
  195. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  196. package/dist/record-6IUzwP9K.js.map +1 -0
  197. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  198. package/dist/redact-BhjtnjrU.js.map +1 -0
  199. package/dist/remarks-CAvTSq16.js +1 -0
  200. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  201. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  202. package/dist/remarks-DPCONFxZ.js +138 -0
  203. package/dist/remarks-DPCONFxZ.js.map +1 -0
  204. package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
  205. package/dist/remarks-vtkPmQb9.js.map +1 -0
  206. package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
  207. package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
  208. package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
  209. package/dist/resolve-Do0ZaYH6.js.map +1 -0
  210. package/dist/resolve-amw_2Odz.js +1 -0
  211. package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
  212. package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
  213. package/dist/resume-AW5oXzUc.js +691 -0
  214. package/dist/resume-AW5oXzUc.js.map +1 -0
  215. package/dist/retry-ChzISxmy.js +105 -0
  216. package/dist/retry-ChzISxmy.js.map +1 -0
  217. package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
  218. package/dist/runtime-BpW1f1Mx.js.map +1 -0
  219. package/dist/sandbox/boat.d.ts +26 -0
  220. package/dist/sandbox/boat.js +37 -0
  221. package/dist/sandbox/boat.js.map +1 -0
  222. package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
  223. package/dist/sandbox-kSgr7vo5.js.map +1 -0
  224. package/dist/sandbox.d.ts +20 -2
  225. package/dist/sandbox.js +1 -1
  226. package/dist/server-2_gUnVvf.js +1 -0
  227. package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
  228. package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
  229. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  230. package/dist/shared-Cfi45Bvy.js.map +1 -0
  231. package/dist/slug-IHh3f770.js +136 -0
  232. package/dist/slug-IHh3f770.js.map +1 -0
  233. package/dist/source-CY3etKzw.js +1 -0
  234. package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
  235. package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
  236. package/dist/src-6Zzyg4Qd.js +2 -0
  237. package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
  238. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  239. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  240. package/dist/src-BRMsk9nh.js +2 -0
  241. package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
  242. package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
  243. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  244. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  245. package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
  246. package/dist/src-Ck2r_EMT.js.map +1 -0
  247. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  248. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  249. package/dist/src-DB7N8TYJ.js +2 -0
  250. package/dist/src-DPDaAYwh.js +2 -0
  251. package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
  252. package/dist/src-DtodMT59.js +1 -0
  253. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  254. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  255. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  256. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  257. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  258. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  259. package/dist/src-T-OeuJLr.js +2 -0
  260. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  261. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  262. package/dist/src-zntEDlnx.js +3 -0
  263. package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
  264. package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
  265. package/dist/state-kimtIhBP.js +1 -0
  266. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  267. package/dist/stats-C18W4xPX.js.map +1 -0
  268. package/dist/suite-CKBa6MKt.js +2 -0
  269. package/dist/suite-COglUhPd.js +1272 -0
  270. package/dist/suite-COglUhPd.js.map +1 -0
  271. package/dist/summary-DU5zaibx.js +148 -0
  272. package/dist/summary-DU5zaibx.js.map +1 -0
  273. package/dist/tart-BTVXp8rz.js +373 -0
  274. package/dist/tart-BTVXp8rz.js.map +1 -0
  275. package/dist/tart-cli-BIC9HqPg.js +37 -0
  276. package/dist/tart-cli-BIC9HqPg.js.map +1 -0
  277. package/dist/tart-cli-BMrkmBjI.js +1 -0
  278. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  279. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  280. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  281. package/dist/tart-ref-DYZijjrU.js +55 -0
  282. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  283. package/dist/testing.d.ts +47 -37
  284. package/dist/testing.js +4 -4
  285. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  286. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  287. package/dist/trace-Dx2NBE1O.js +311 -0
  288. package/dist/trace-Dx2NBE1O.js.map +1 -0
  289. package/dist/traits-DKnYkqeK.js +66 -0
  290. package/dist/traits-DKnYkqeK.js.map +1 -0
  291. package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
  292. package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
  293. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  294. package/dist/types-C2mLpph3.js.map +1 -0
  295. package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
  296. package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
  297. package/dist/vercel-CGlSBv63.js.map +1 -0
  298. package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
  299. package/dist/vertex-Eb2wqahk.js.map +1 -0
  300. package/dist/vertex-O88TCwgY.js +1 -0
  301. package/dist/view/index.d.ts +238 -90
  302. package/dist/view/index.js +11 -9
  303. package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
  304. package/dist/views-DAIiLiV3.js.map +1 -0
  305. package/dist/views-DtRhj2Tn.js +2 -0
  306. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
  307. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
  308. package/dist/workspace-CcQRT31C.js +1 -0
  309. package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
  310. package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
  311. package/package.json +26 -11
  312. package/skills/eval-rubric/SKILL.md +31 -4
  313. package/dist/adapter-DyvDj4ZN.js.map +0 -1
  314. package/dist/app-files-hpRT5lj_.js.map +0 -1
  315. package/dist/assemble-DTXwCL3m.js +0 -1
  316. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  317. package/dist/capture-BT2fAmpZ.js +0 -1
  318. package/dist/capture-Bdcx9Gap.js.map +0 -1
  319. package/dist/cli-CGd51thd.js.map +0 -1
  320. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  321. package/dist/client-CF-DOjOU.js.map +0 -1
  322. package/dist/command-CsfcwMyc.js.map +0 -1
  323. package/dist/command-CyIb9BSC.js.map +0 -1
  324. package/dist/command-TLmY-rYp.js.map +0 -1
  325. package/dist/commands-C54S45d7.js.map +0 -1
  326. package/dist/config-ImY6AlRE.js.map +0 -1
  327. package/dist/context-BajfiH0S.js.map +0 -1
  328. package/dist/contrasts-DtSX8E_V.js +0 -101
  329. package/dist/contrasts-DtSX8E_V.js.map +0 -1
  330. package/dist/cost-CMzY9VXv.js +0 -1
  331. package/dist/db-ZIk6S2CD.js +0 -2
  332. package/dist/db-sHGpaLo9.js.map +0 -1
  333. package/dist/debug-BnXV7JbX.js +0 -1
  334. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  335. package/dist/dist-ByYk_wAs.js.map +0 -1
  336. package/dist/docker--2ytOHcu.js.map +0 -1
  337. package/dist/doctor-eE65YpEv.js.map +0 -1
  338. package/dist/environment-Cy3FVLR5.js.map +0 -1
  339. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  340. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  341. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  342. package/dist/files-B3d-8Fe0.js +0 -1
  343. package/dist/files-COW5JBJM.js +0 -1
  344. package/dist/files-Do4Rp3yZ.js +0 -1
  345. package/dist/files-p-uTdOjA.js.map +0 -1
  346. package/dist/gate-BoDKeuJu.js.map +0 -1
  347. package/dist/harness-Ch2P81sx.js.map +0 -1
  348. package/dist/ids-DuK6xl4w.js.map +0 -1
  349. package/dist/images-BVZHDf_z.js.map +0 -1
  350. package/dist/init-BWnMbUnc.js.map +0 -1
  351. package/dist/init-s1jN-PCI.js +0 -1
  352. package/dist/judge-ByEUlzOM.js.map +0 -1
  353. package/dist/live-DmJix4c8.js +0 -1
  354. package/dist/live-d_5a5iUv.js.map +0 -1
  355. package/dist/model-8uZBYqV5.js +0 -1
  356. package/dist/model-DOiCiMCD.js.map +0 -1
  357. package/dist/node-CwP1G178.js.map +0 -1
  358. package/dist/opencode-providers-yNKyXD49.js.map +0 -1
  359. package/dist/plan-CIfOFhth.js.map +0 -1
  360. package/dist/plugins-BEs2Am3L.js.map +0 -1
  361. package/dist/preflight-50Qw5e5K.js.map +0 -1
  362. package/dist/provenance-Cc8fFE7Z.js.map +0 -1
  363. package/dist/provider-DCdTHQpk.js +0 -1
  364. package/dist/providers-CP_bBtYm.js +0 -1
  365. package/dist/providers-DZSTlC9R.js.map +0 -1
  366. package/dist/query-kFPuqBv1.js +0 -1
  367. package/dist/questions-C_Y-oXW6.js.map +0 -1
  368. package/dist/record-DhzwkSXc.js.map +0 -1
  369. package/dist/redact-BEEGc37e.js.map +0 -1
  370. package/dist/remarks-0EXMh6AD.js.map +0 -1
  371. package/dist/remarks-BL18Id09.js +0 -1
  372. package/dist/remarks-DeZZT8PK.js +0 -116
  373. package/dist/remarks-DeZZT8PK.js.map +0 -1
  374. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  375. package/dist/resolve-BJSay360.js +0 -1
  376. package/dist/resolve-B_JlQ0La.js.map +0 -1
  377. package/dist/resume-CexsWuGK.js +0 -434
  378. package/dist/resume-CexsWuGK.js.map +0 -1
  379. package/dist/runtime-CSZ1hjlN.js.map +0 -1
  380. package/dist/sandbox-B5W8lLL1.js.map +0 -1
  381. package/dist/server-SBzrf35T.js +0 -1
  382. package/dist/shared-ogqPJ0uo.js.map +0 -1
  383. package/dist/source-Dn1uqPon.js +0 -1
  384. package/dist/src-BKDZO3ra.js +0 -2
  385. package/dist/src-BxrHipV0.js +0 -3
  386. package/dist/src-C9elednc.js +0 -2
  387. package/dist/src-CM7xdJFA.js +0 -2
  388. package/dist/src-CWEGknZo.js +0 -2
  389. package/dist/src-Cpy9MNS3.js.map +0 -1
  390. package/dist/src-DVaAKCsH.js +0 -1
  391. package/dist/src-DurP--_D.js +0 -2
  392. package/dist/state-Cicc9Oa5.js +0 -1
  393. package/dist/stats-D-Gqwqt0.js.map +0 -1
  394. package/dist/suite-BYJ2Q0S-.js +0 -2
  395. package/dist/suite-BtHPBFYL.js +0 -544
  396. package/dist/suite-BtHPBFYL.js.map +0 -1
  397. package/dist/summary-zP3cCXDN.js.map +0 -1
  398. package/dist/types-C4OkYhZg.js.map +0 -1
  399. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  400. package/dist/vertex-BaAz44YN.js +0 -1
  401. package/dist/views-CKoN5NAx.js.map +0 -1
  402. package/dist/views-CmyX255j.js +0 -2
  403. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -0,0 +1,1272 @@
1
+ import { n as clip } from "./text-B8-W3Cvn.js";
2
+ import { a as pairTallies } from "./pairs-BGb134Vu.js";
3
+ import { _ as signedRank, a as liftInterval, c as meanInterval, d as normalCdf, f as pairedLiftInterval, h as runsToHold, l as median$1, m as round3, o as mannWhitney, r as containsZero, s as mcnemar, u as minimumP, y as wilson } from "./stats-C18W4xPX.js";
4
+ import { C as bothUnderATenth, M as effortSample, P as effortVerdict, T as cellEffort, l as unfoldCases, u as caseLabel, y as EFFORT_ALPHA } from "./cases-z9E6DCd5.js";
5
+ import { T as notRunYet, i as attemptAnswerRows, k as runOutcome } from "./model-V3mMXp6x.js";
6
+ import { a as checksByLabel, d as labeledJudgments, i as checkTries } from "./judgments-CzgpsTqv.js";
7
+ // src/view/observations.ts
8
+ /** A scalar's tally label; numbers in an array count by their text. */
9
+ const label = (v) => typeof v === "string" ? v : JSON.stringify(v);
10
+ const scalar = (v) => v === null || typeof v !== "object";
11
+ function median(sorted) {
12
+ const mid = Math.floor(sorted.length / 2);
13
+ return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
14
+ }
15
+ /**
16
+ * Each name and cell any attempt observed it on, by name in first-observed order and then by
17
+ * cell, tallied over that cell's attempts.
18
+ */
19
+ function observedTallies(attempts) {
20
+ const byName = /* @__PURE__ */ new Map();
21
+ for (const a of attempts) for (const { name, value } of a.observations ?? []) {
22
+ const cells = byName.get(name) ?? /* @__PURE__ */ new Map();
23
+ byName.set(name, cells);
24
+ let t = cells.get(a.cell);
25
+ if (!t) {
26
+ t = {
27
+ name,
28
+ cell: a.cell,
29
+ counts: /* @__PURE__ */ new Map(),
30
+ nums: [],
31
+ others: 0,
32
+ attempts: 0
33
+ };
34
+ cells.set(a.cell, t);
35
+ }
36
+ t.attempts++;
37
+ const count = (v) => t.counts.set(v, (t.counts.get(v) ?? 0) + 1);
38
+ if (typeof value === "number") t.nums.push(value);
39
+ else if (scalar(value)) count(label(value));
40
+ else if (Array.isArray(value) && value.some(scalar)) for (const item of new Set(value.filter(scalar).map(label))) count(item);
41
+ else t.others++;
42
+ }
43
+ return [...byName.values()].flatMap((cells) => [...cells.values()]).map((t) => {
44
+ const nums = t.nums.toSorted((x, y) => x - y);
45
+ return {
46
+ name: t.name,
47
+ cell: t.cell,
48
+ attempts: t.attempts,
49
+ counts: [...t.counts].map(([value, n]) => ({
50
+ value,
51
+ n
52
+ })).sort((x, y) => y.n - x.n),
53
+ numbers: nums.length ? {
54
+ median: median(nums),
55
+ min: nums[0],
56
+ max: nums.at(-1),
57
+ n: nums.length
58
+ } : null,
59
+ others: t.others
60
+ };
61
+ });
62
+ }
63
+ /** Tallied values shown before `+N more`. */
64
+ const SHOWN = 5;
65
+ const num = (n) => String(Math.round(n * 1e3) / 1e3);
66
+ /** On one line: a line break or a terminal escape in an observed string would break the table. */
67
+ const clipValue = (v) => {
68
+ const flat = v.replace(/[\s\p{Cc}]+/gu, " ").trim();
69
+ return flat.length > 40 ? `${flat.slice(0, 39)}…` : flat;
70
+ };
71
+ /** `drizzle 9 · prisma 3`, `median 4 · range 1–12 · 13 attempts`, as the run output shows a tally. */
72
+ function observedLine(t) {
73
+ const parts = t.counts.slice(0, SHOWN).map((c) => `${clipValue(c.value)} ${c.n}`);
74
+ const more = t.counts.length - SHOWN;
75
+ if (more > 0) parts.push(`+${more} more`);
76
+ if (t.numbers) {
77
+ const { median: m, min, max } = t.numbers;
78
+ parts.push(min === max ? `${num(m)}` : `median ${num(m)} · range ${num(min)}–${num(max)}`);
79
+ }
80
+ if (t.others) parts.push(`${t.others} ${t.others === 1 ? "value" : "values"} in reeval query attempt_values`);
81
+ const tallied = t.counts.reduce((sum, c) => sum + c.n, 0);
82
+ return !t.numbers && !t.others && tallied === t.attempts ? parts.join(" · ") : `${parts.join(" · ")} · ${t.attempts} ${t.attempts === 1 ? "attempt" : "attempts"}`;
83
+ }
84
+ const sameTally = (a, b) => a.attempts === b.attempts && a.others === b.others && JSON.stringify(a.counts) === JSON.stringify(b.counts) && JSON.stringify(a.numbers) === JSON.stringify(b.numbers);
85
+ /**
86
+ * How one name's tally on one cell moved from base to head, or null when it didn't:
87
+ * `prisma 6 → 1 · drizzle 2 → 8` (each value whose count changed, head's most first),
88
+ * `median 4 → 9`, or a side's whole line against `none` when only one side observed it.
89
+ */
90
+ function observedShift(base, head) {
91
+ if (!base && !head) return null;
92
+ if (!base || !head) return base ? `${observedLine(base)} → none` : `none → ${observedLine(head)}`;
93
+ if (sameTally(base, head)) return null;
94
+ const before = new Map(base.counts.map((c) => [c.value, c.n]));
95
+ const after = new Map(head.counts.map((c) => [c.value, c.n]));
96
+ const moved = [.../* @__PURE__ */ new Set([...after.keys(), ...before.keys()])].filter((v) => (before.get(v) ?? 0) !== (after.get(v) ?? 0));
97
+ const parts = moved.slice(0, SHOWN).map((v) => `${clipValue(v)} ${before.get(v) ?? 0} → ${after.get(v) ?? 0}`);
98
+ if (moved.length > SHOWN) parts.push(`+${moved.length - SHOWN} more`);
99
+ const [b, h] = [base.numbers, head.numbers];
100
+ if (b && h && b.median !== h.median) parts.push(`median ${num(b.median)} → ${num(h.median)}`);
101
+ else if (b && h && (b.min !== h.min || b.max !== h.max)) parts.push(`range ${num(b.min)}–${num(b.max)} → ${num(h.min)}–${num(h.max)}`);
102
+ if (base.attempts !== head.attempts) parts.push(`${base.attempts} → ${head.attempts} attempts`);
103
+ return parts.length ? parts.join(" · ") : null;
104
+ }
105
+ // src/history/families.ts
106
+ const isCounted$1 = (a) => a.outcome === "pass" || a.outcome === "fail";
107
+ function sideOf(entry, cell) {
108
+ const counted = entry.attempts.filter((a) => a.cell === cell && isCounted$1(a));
109
+ if (!counted.length) return null;
110
+ return {
111
+ pass: counted.filter((a) => a.outcome === "pass").length,
112
+ counted: counted.length,
113
+ samples: counted.map(effortSample)
114
+ };
115
+ }
116
+ const clamp = (x, lo, hi) => Math.min(hi, Math.max(lo, x));
117
+ const interval = (lo, hi, method, bound) => ({
118
+ lo: round3(clamp(lo, ...bound)),
119
+ hi: round3(clamp(hi, ...bound)),
120
+ level: .95,
121
+ method
122
+ });
123
+ function rateOf(cell, sides, passRate) {
124
+ const pass = sides.reduce((s, x) => s + x.pass, 0);
125
+ const counted = sides.reduce((s, x) => s + x.counted, 0);
126
+ const single = sides.every((x) => x.counted === 1);
127
+ const rates = sides.map((x) => x.pass / x.counted);
128
+ const t = meanInterval(rates);
129
+ const w = wilson(pass, counted);
130
+ return {
131
+ cell,
132
+ cases: sides.length,
133
+ pass,
134
+ counted,
135
+ rate: round3(t.mean),
136
+ interval: single ? interval(w.low, w.high, "wilson", [0, 1]) : interval(t.low, t.high, "t-cases", [0, 1]),
137
+ ...passRate !== void 0 ? {
138
+ passRate,
139
+ outcome: t.mean >= passRate ? "pass" : "fail"
140
+ } : {}
141
+ };
142
+ }
143
+ /** A cell's `{ passRate }` gate, which its family gates over the cases. */
144
+ const pooledPassRate = (cell) => typeof cell.gate === "object" ? cell.gate.passRate : void 0;
145
+ function metric(pairs) {
146
+ const differences = pairs.map(([v, c]) => v - c);
147
+ const control = median$1(pairs.map(([, c]) => c)) ?? 0;
148
+ const variant = median$1(pairs.map(([v]) => v)) ?? 0;
149
+ const test = signedRank(differences);
150
+ const verdict = differences.filter((d) => d !== 0).length < 6 ? "insufficient" : test === null || test.p >= .05 ? "noise" : test.sign < 0 ? "fewer" : "more";
151
+ return {
152
+ pairs: pairs.length,
153
+ ratio: control === 0 ? null : round3(variant / control),
154
+ p: test === null ? null : round3(test.p),
155
+ verdict
156
+ };
157
+ }
158
+ /**
159
+ * Each case's variant and control medians of one effort measure, over the cases where every
160
+ * counted attempt on both sides has it: the pairs a family's paired effort test ranks.
161
+ */
162
+ const casePairs = (pairs, pick) => pairs.flatMap(([v, c]) => {
163
+ const vs = v.map(pick);
164
+ const cs = c.map(pick);
165
+ if (!vs.every((x) => x !== void 0) || !cs.every((x) => x !== void 0)) return [];
166
+ return [[median$1(vs) ?? 0, median$1(cs) ?? 0]];
167
+ });
168
+ function effortOf$1(sides) {
169
+ const pairs = sides.map(([v, c]) => [v.samples, c.samples]);
170
+ const per = (pick) => casePairs(pairs, pick);
171
+ const tokens = per((s) => s.tokens);
172
+ return {
173
+ tools: metric(per((s) => s.tools)),
174
+ agentMs: metric(per((s) => s.agentMs)),
175
+ ...tokens.length ? { tokens: metric(tokens) } : {}
176
+ };
177
+ }
178
+ function liftOf(variant, control, cases, minLift) {
179
+ const pairs = cases.flatMap((e) => {
180
+ const v = sideOf(e, variant);
181
+ const c = sideOf(e, control);
182
+ return v && c ? [[v, c]] : [];
183
+ });
184
+ if (!pairs.length) return null;
185
+ const single = pairs.every(([v, c]) => v.counted === 1 && c.counted === 1);
186
+ const vRates = pairs.map(([v]) => v.pass / v.counted);
187
+ const cRates = pairs.map(([, c]) => c.pass / c.counted);
188
+ const mean = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
189
+ let points;
190
+ let range;
191
+ let p;
192
+ let discordant;
193
+ let decided;
194
+ if (single) {
195
+ const counts = {
196
+ both: 0,
197
+ variantOnly: 0,
198
+ controlOnly: 0,
199
+ neither: 0
200
+ };
201
+ for (const [v, c] of pairs) if (v.pass && c.pass) counts.both++;
202
+ else if (v.pass) counts.variantOnly++;
203
+ else if (c.pass) counts.controlOnly++;
204
+ else counts.neither++;
205
+ const paired = pairedLiftInterval(counts);
206
+ points = paired.points;
207
+ range = interval(paired.low, paired.high, "newcombe-paired", [-1, 1]);
208
+ p = round3(mcnemar(counts.variantOnly, counts.controlOnly));
209
+ discordant = {
210
+ variantOnly: counts.variantOnly,
211
+ controlOnly: counts.controlOnly
212
+ };
213
+ decided = counts.variantOnly + counts.controlOnly >= 6;
214
+ } else {
215
+ const differences = pairs.map(([v, c]) => (v.pass * c.counted - c.pass * v.counted) / (v.counted * c.counted));
216
+ const t = meanInterval(differences);
217
+ points = t.mean;
218
+ range = interval(t.low, t.high, "t-cases", [-1, 1]);
219
+ const rank = signedRank(differences);
220
+ p = rank === null ? null : round3(rank.p);
221
+ decided = pairs.length >= 6;
222
+ }
223
+ const verdict = !decided ? "insufficient" : range.lo > 0 ? "lift" : range.hi < 0 ? "drop" : "noise";
224
+ return {
225
+ variant,
226
+ vs: control,
227
+ pairs: pairs.length,
228
+ variantRate: round3(mean(vRates)),
229
+ controlRate: round3(mean(cRates)),
230
+ ...discordant ? { discordant } : {},
231
+ points: round3(points),
232
+ interval: range,
233
+ p,
234
+ withinNoise: containsZero(range.lo, range.hi),
235
+ verdict,
236
+ ...minLift !== void 0 ? { minLift } : {},
237
+ outcome: minLift !== void 0 && range.lo < minLift ? "fail" : "pass",
238
+ effort: effortOf$1(pairs)
239
+ };
240
+ }
241
+ /**
242
+ * The run's pooled families, in the order their first case appears; `minLiftOf` gives a variant
243
+ * cell's `compare.minLift`, and `excludedOf` the pairs `where` left out of a family.
244
+ */
245
+ function familiesOf(evals, minLiftOf, excludedOf = () => void 0) {
246
+ const byId = /* @__PURE__ */ new Map();
247
+ for (const e of evals) {
248
+ if (!e.family || e.todo) continue;
249
+ byId.set(e.family.id, [...byId.get(e.family.id) ?? [], e]);
250
+ }
251
+ return [...byId.values()].map((members) => {
252
+ const cases = [...members].sort((a, b) => (a.family?.case ?? 0) - (b.family?.case ?? 0));
253
+ const first = cases[0];
254
+ const family = first.family;
255
+ const excluded = excludedOf(family.id);
256
+ const cellKeys = [];
257
+ for (const e of cases) for (const c of e.cells) if (!cellKeys.includes(c.key)) cellKeys.push(c.key);
258
+ const rates = cellKeys.flatMap((key) => {
259
+ const sides = cases.flatMap((e) => sideOf(e, key) ?? []);
260
+ const cell = cases.flatMap((e) => e.cells).find((c) => c.key === key);
261
+ return sides.length ? [rateOf(key, sides, cell && pooledPassRate(cell))] : [];
262
+ });
263
+ const lifts = [];
264
+ for (const key of cellKeys) {
265
+ const holder = cases.find((e) => e.cells.some((c) => c.key === key && c.role === "variant"));
266
+ const variant = holder?.cells.find((c) => c.key === key);
267
+ if (!holder || !variant) continue;
268
+ const control = holder.cells.find((c) => c.role === "control" && c.harness === variant.harness && c.sandbox === variant.sandbox);
269
+ if (!control) continue;
270
+ const minLift = cases.flatMap((e) => e.cells.filter((c) => c.key === key)).map(minLiftOf).find((m) => m !== void 0);
271
+ const lift = liftOf(key, control.key, cases, minLift);
272
+ if (lift) lifts.push(lift);
273
+ }
274
+ return {
275
+ id: family.id,
276
+ file: first.file,
277
+ group: first.group,
278
+ name: family.name,
279
+ cases: family.cases,
280
+ ran: cases.filter((e) => e.attempts.some((a) => a.outcome !== "skipped")).length,
281
+ rates,
282
+ lifts,
283
+ ...excluded?.length ? { excluded } : {},
284
+ outcome: [...rates, ...lifts].some((x) => x.outcome === "fail") ? "fail" : "pass"
285
+ };
286
+ });
287
+ }
288
+ // src/view/measures.ts
289
+ const LABEL_CHARS = 60;
290
+ /**
291
+ * What a difference its test doesn't hold needs to lean, else it reads `same`, so run-to-run noise
292
+ * doesn't stand in for a difference: at least the measure's smallest effect (pass-rate points; a
293
+ * judge's 0–1 score; a question's decisive win share; effort's relative difference in medians),
294
+ * and its test's p below `LEAN_P`. A question needs `QUESTION_PAIRS_MIN` decisive pairs and a p
295
+ * below `QUESTION_LEAN_P` instead: questions are few by design, so a unanimous 4 of 4 (p 0.125)
296
+ * leans. A difference its test holds is better or worse at any size.
297
+ */
298
+ const RATE_MIN = .1;
299
+ const JUDGE_MIN = .05;
300
+ const QUESTION_MIN = .6;
301
+ const EFFORT_MIN = .1;
302
+ const LEAN_P = .1;
303
+ const QUESTION_LEAN_P = .2;
304
+ /**
305
+ * A lean below its measure's smallest effect, or at a p of `leanP` or more, is `same`; unless a
306
+ * sample this size can't reach `leanP` at all (`floor`, the test's smallest p at these sizes, ties
307
+ * aside), when it stays a lean on too few runs, since its p says nothing.
308
+ */
309
+ const screened = (verdict, size, min, p, floor, leanP = LEAN_P) => verdict === "leans" && (size < min - 1e-9 || p === null || p >= leanP && floor < leanP) ? "same" : verdict;
310
+ /** On a lean its test can't reach `LEAN_P` on, the sample it has and the fewest that can. */
311
+ const shortOf = (verdict, floor, now, reaches) => verdict === "leans" && floor >= .1 ? {
312
+ now,
313
+ at: smallestFrom(now + 1, reaches)
314
+ } : null;
315
+ /** `12× faster, but 3 runs a side can't confirm it; 4 is the least that can`: a lean's text. */
316
+ function leanText(m) {
317
+ if (!m.short) return m.text;
318
+ const { now, at } = m.short;
319
+ const has = m.unit === "cases" ? `${now} ${now === 1 ? "case" : "cases"}` : `${now} ${now === 1 ? "run" : "runs"} a side`;
320
+ return `${m.text}, but ${has} can't confirm it; ${at} is the least that can`;
321
+ }
322
+ /**
323
+ * A measure as what the variant does, to follow its name or `It`: `runs 5× faster`, `uses 2.3×
324
+ * more tokens`, `scores 0.97 higher on judge "tidy"`, `passes "builds" 50 points more often`.
325
+ */
326
+ function measurePhrase(m) {
327
+ if (m.verdict === "same" || m.estimate === null) return m.text;
328
+ const label = m.label === null ? "" : `"${clip(m.label, LABEL_CHARS)}" `;
329
+ switch (m.measure) {
330
+ case "agentMs": return `runs ${m.text}`;
331
+ case "tools":
332
+ case "tokens": return `uses ${m.text}`;
333
+ case "judge": return `scores ${Math.abs(m.estimate).toFixed(2)} ${m.estimate > 0 ? "higher" : "lower"} on judge ${label.trimEnd()}`;
334
+ case "pass":
335
+ case "check": return `passes ${label}${Math.round(Math.abs(m.estimate) * 100)} points ${m.estimate > 0 ? "more" : "less"} often`;
336
+ default: return m.text;
337
+ }
338
+ }
339
+ /** Two-sided p of a pooled two-proportion z test: it screens a rate's lean, never its verdict. */
340
+ function proportionP(v, c) {
341
+ const pooled = (v.pass + c.pass) / (v.counted + c.counted);
342
+ const se = Math.sqrt(pooled * (1 - pooled) * (1 / v.counted + 1 / c.counted));
343
+ if (se === 0) return 1;
344
+ const z = Math.abs(v.pass / v.counted - c.pass / c.counted) / se;
345
+ return 2 * (1 - normalCdf(z));
346
+ }
347
+ const isCounted = (a) => a.outcome === "pass" || a.outcome === "fail";
348
+ const sign = (x) => x > 0 ? "+" : x < 0 ? "−" : "±";
349
+ const pts = (x) => `${sign(x)}${Math.round(Math.abs(x) * 100)} pts`;
350
+ const mean = (n) => n.reduce((a, b) => a + b, 0) / n.length;
351
+ /** `5×`, `1.8×`, or `20%` below 1.5×: how many times, or how much, one side's median is the other's. */
352
+ function factor(ratio, lower) {
353
+ if (ratio === null || ratio <= 0) return "";
354
+ const f = lower ? 1 / ratio : ratio;
355
+ if (f >= 1.5) return `${Number(f.toFixed(f >= 10 ? 0 : 1))}× `;
356
+ const pct = Math.round(Math.abs(1 - ratio) * 100);
357
+ return pct ? `${pct}% ` : "";
358
+ }
359
+ /** The z a two-sided normal p corresponds to. */
360
+ function zOf(p) {
361
+ let lo = 0;
362
+ let hi = 40;
363
+ for (let i = 0; i < 100; i++) {
364
+ const mid = (lo + hi) / 2;
365
+ if (2 * (1 - normalCdf(mid)) > p) lo = mid;
366
+ else hi = mid;
367
+ }
368
+ return (lo + hi) / 2;
369
+ }
370
+ const smallestFrom = (from, reaches) => {
371
+ let n = from;
372
+ while (!reaches(n)) n++;
373
+ return n;
374
+ };
375
+ const smallest = (reaches) => smallestFrom(1, reaches);
376
+ /** Whether a sample of `n` a side (or `n` cases) can reach `LEAN_P` on each test. */
377
+ const rankReaches = (n) => minimumP(n, n) < LEAN_P;
378
+ const signedReaches = (n) => 2 / 2 ** n < LEAN_P;
379
+ const rateFloor = (nv, nc) => proportionP({
380
+ pass: nv,
381
+ counted: nv
382
+ }, {
383
+ pass: 0,
384
+ counted: nc
385
+ });
386
+ /** A rate confirms only from `COMPARE_RUNS` a side, where lift's interval can hold. */
387
+ const rateReaches = (n) => n >= 4 && rateFloor(n, n) < .1;
388
+ /**
389
+ * The sample at which a rank test's lean would hold at `EFFORT_ALPHA`: at a fixed effect a z grows
390
+ * with the square root of the sample, so the observed one scaled up to the critical z, and never
391
+ * below the size whose smallest possible p reaches it (`floor`). A p of 1 on a difference (one run
392
+ * a side) gives no z to scale, so the floor is all it can say.
393
+ */
394
+ function scaled(p, now, floor) {
395
+ const z = zOf(p);
396
+ if (z === 0) return Math.max(now + 1, floor);
397
+ const n = Math.max(now + 1, floor, Math.ceil(now * (zOf(EFFORT_ALPHA) / z) ** 2));
398
+ return n > 200 ? null : n;
399
+ }
400
+ const RANK_FLOOR = smallest((n) => minimumP(n, n) < EFFORT_ALPHA);
401
+ const SIGNED_FLOOR = smallest((n) => 2 / 2 ** n < EFFORT_ALPHA);
402
+ /**
403
+ * The runs a side (a case, over `cases`) at which a question's decisive pairs would hold: one pair
404
+ * a run, the same share of them decisive and the same share of those for the leading side,
405
+ * through the exact sign test.
406
+ */
407
+ function signRuns(wins, cases) {
408
+ const decisive = wins.variant + wins.control;
409
+ const share = Math.max(wins.variant, wins.control) / decisive;
410
+ const runs = wins.voted / cases;
411
+ for (let n = Math.floor(runs) + 1; n <= 200; n++) {
412
+ const d = Math.round(decisive * n / runs);
413
+ const lead = Math.round(d * share);
414
+ if (mcnemar(lead, d - lead) < .05) return n;
415
+ }
416
+ return null;
417
+ }
418
+ /**
419
+ * A pass rate's verdict, lift's maths: `same` at equal rates; `better` or `worse` when the
420
+ * interval on the difference excludes 0 on `COMPARE_RUNS` or more a side; else `leans`, with the
421
+ * runs at which the observed rates would hold.
422
+ */
423
+ function rateMeasure(measure, label, v, c, stored) {
424
+ const vr = v.pass / v.counted;
425
+ const cr = c.pass / c.counted;
426
+ const computed = liftInterval({
427
+ pass: v.pass,
428
+ n: v.counted
429
+ }, {
430
+ pass: c.pass,
431
+ n: c.counted
432
+ });
433
+ const lo = stored?.lo ?? computed.low;
434
+ const hi = stored?.hi ?? computed.high;
435
+ const holds = !containsZero(lo, hi) && Math.min(v.counted, c.counted) >= 4;
436
+ const way = vr > cr ? "better" : "worse";
437
+ const p = proportionP(v, c);
438
+ const floor = rateFloor(v.counted, c.counted);
439
+ const verdict = screened(vr === cr ? "same" : holds ? way : "leans", Math.abs(vr - cr), RATE_MIN, p, floor);
440
+ const name = measure === "pass" ? "pass rate" : `"${clip(label ?? "", LABEL_CHARS)}"`;
441
+ const from = Math.max(v.counted, c.counted, 3) + 1;
442
+ return {
443
+ measure,
444
+ label,
445
+ variant: vr,
446
+ control: cr,
447
+ estimate: vr - cr,
448
+ lo,
449
+ hi,
450
+ p: null,
451
+ verdict,
452
+ leaning: verdict === "leans" ? way : null,
453
+ text: vr === cr ? `same ${name}` : `${name} ${pts(vr - cr)}`,
454
+ needed: verdict !== "leans" ? null : containsZero(lo, hi) ? runsToHold(vr, cr, from) : Math.max(from, 4),
455
+ unit: "runs",
456
+ short: shortOf(verdict, floor, Math.min(v.counted, c.counted), rateReaches)
457
+ };
458
+ }
459
+ /** A verdict from a test's p and the sign of its statistic (positive: the variant's is higher). */
460
+ const tested = (p, s, higherIsBetter) => {
461
+ if (s === 0) return "same";
462
+ return p < .05 ? s > 0 === higherIsBetter ? "better" : "worse" : "leans";
463
+ };
464
+ const leaningOf = (verdict, s, higherIsBetter) => verdict === "leans" ? s > 0 === higherIsBetter ? "better" : "worse" : null;
465
+ const EFFORT = [
466
+ {
467
+ measure: "tools",
468
+ pick: (s) => s.tools,
469
+ words: (fewer) => fewer ? "fewer tool calls" : "more tool calls",
470
+ same: "same tool calls"
471
+ },
472
+ {
473
+ measure: "agentMs",
474
+ pick: (s) => s.agentMs,
475
+ words: (fewer) => fewer ? "faster" : "slower",
476
+ same: "same agent time"
477
+ },
478
+ {
479
+ measure: "tokens",
480
+ pick: (s) => s.tokens,
481
+ words: (fewer) => fewer ? "fewer tokens" : "more tokens",
482
+ same: "same tokens"
483
+ }
484
+ ];
485
+ /** Values of one effort measure, left out where an attempt didn't report it (tokens). */
486
+ const valuesOf = (samples, pick) => samples.flatMap((s) => {
487
+ const x = pick(s);
488
+ return x === void 0 ? [] : [x];
489
+ });
490
+ function effortMeasures(input, runs) {
491
+ const out = [];
492
+ const vs = input.variant.map((a) => a.attempts.map(effortSample));
493
+ const cs = input.control.map((a) => a.attempts.map(effortSample));
494
+ for (const m of EFFORT) {
495
+ const v = valuesOf(vs.flat(), m.pick);
496
+ const c = valuesOf(cs.flat(), m.pick);
497
+ if (!v.length || !c.length) continue;
498
+ const vm = median$1(v) ?? 0;
499
+ const cm = median$1(c) ?? 0;
500
+ const ratio = cm === 0 ? null : vm / cm;
501
+ let p;
502
+ let s;
503
+ let floor;
504
+ let now;
505
+ let reaches;
506
+ let verdict;
507
+ let needed = null;
508
+ let unit = "runs";
509
+ if (input.family?.effort) {
510
+ const metric = m.measure === "tokens" ? input.family.effort.tokens : input.family.effort[m.measure];
511
+ if (!metric) continue;
512
+ const cases = casePairs(vs.map((x, i) => [x, cs[i] ?? []]), m.pick);
513
+ const test = signedRank(cases.map(([a, b]) => a - b));
514
+ s = test?.sign ?? 0;
515
+ p = test?.p ?? 1;
516
+ now = cases.length;
517
+ floor = Math.min(1, 2 / 2 ** now);
518
+ reaches = signedReaches;
519
+ verdict = metric.verdict === "fewer" ? "better" : metric.verdict === "more" ? "worse" : s === 0 ? "same" : "leans";
520
+ unit = "cases";
521
+ if (verdict === "leans") needed = scaled(p, cases.length, Math.max(SIGNED_FLOOR, 6));
522
+ } else {
523
+ const test = mannWhitney(v, c);
524
+ if (!test) continue;
525
+ const printed = effortVerdict(v, c).verdict;
526
+ s = test.sign;
527
+ p = test.p;
528
+ floor = minimumP(v.length, c.length);
529
+ now = Math.min(v.length, c.length);
530
+ reaches = rankReaches;
531
+ verdict = printed === "fewer" ? "better" : printed === "more" ? "worse" : s === 0 ? "same" : "leans";
532
+ if (verdict === "leans") needed = scaled(p, runs, RANK_FLOOR);
533
+ }
534
+ verdict = m.measure === "agentMs" && bothUnderATenth(vm, cm) ? "same" : screened(verdict, ratio === null ? vm === 0 ? 0 : Number.POSITIVE_INFINITY : Math.abs(ratio - 1), EFFORT_MIN, p, floor);
535
+ if (verdict !== "leans") needed = null;
536
+ const fewer = s < 0;
537
+ const agrees = (r) => r !== null && r < 1 === fewer && r !== 1;
538
+ const cMean = mean(c);
539
+ const meanRatio = cMean === 0 ? null : mean(v) / cMean;
540
+ const shown = agrees(ratio) ? ratio : agrees(meanRatio) ? meanRatio : null;
541
+ out.push({
542
+ measure: m.measure,
543
+ label: null,
544
+ variant: vm,
545
+ control: cm,
546
+ estimate: ratio,
547
+ lo: null,
548
+ hi: null,
549
+ p,
550
+ verdict,
551
+ leaning: leaningOf(verdict, s, false),
552
+ text: verdict === "same" ? m.same : `${shown === null ? "" : factor(shown, fewer)}${m.words(fewer)}`,
553
+ needed,
554
+ unit,
555
+ short: shortOf(verdict, floor, now, reaches)
556
+ });
557
+ }
558
+ return out;
559
+ }
560
+ function checkMeasures(input) {
561
+ const sum = (arms) => {
562
+ const by = /* @__PURE__ */ new Map();
563
+ for (const arm of arms) for (const k of arm.checks ?? []) {
564
+ const t = by.get(k.label) ?? {
565
+ pass: 0,
566
+ counted: 0
567
+ };
568
+ t.pass += k.pass;
569
+ t.counted += k.pass + k.fail;
570
+ by.set(k.label, t);
571
+ }
572
+ return by;
573
+ };
574
+ const v = sum(input.variant);
575
+ const c = sum(input.control);
576
+ return [...v].flatMap(([label, side]) => {
577
+ const other = c.get(label);
578
+ if (!other || !side.counted || !other.counted) return [];
579
+ return [rateMeasure("check", label, side, other)];
580
+ });
581
+ }
582
+ function judgeMeasures(input, runs) {
583
+ const scores = (arms) => {
584
+ const by = /* @__PURE__ */ new Map();
585
+ for (const a of arms.flatMap((x) => x.attempts)) for (const j of labeledJudgments(a)) {
586
+ const key = j.check ?? j.label;
587
+ by.set(key, [...by.get(key) ?? [], j.judgment.score]);
588
+ }
589
+ return by;
590
+ };
591
+ const v = scores(input.variant);
592
+ const c = scores(input.control);
593
+ return [...v].flatMap(([label, xs]) => {
594
+ const ys = c.get(label);
595
+ const test = ys && mannWhitney(xs, ys);
596
+ if (!ys || !test) return [];
597
+ const diff = mean(xs) - mean(ys);
598
+ const verdict = screened(tested(test.p, test.sign, true), Math.abs(diff), JUDGE_MIN, test.p, minimumP(xs.length, ys.length));
599
+ const name = `judge "${clip(label, LABEL_CHARS)}"`;
600
+ return [{
601
+ measure: "judge",
602
+ label,
603
+ variant: mean(xs),
604
+ control: mean(ys),
605
+ estimate: diff,
606
+ lo: null,
607
+ hi: null,
608
+ p: test.p,
609
+ verdict,
610
+ leaning: leaningOf(verdict, test.sign, true),
611
+ text: verdict === "same" ? `same ${name}` : `${name} ${sign(diff)}${Math.abs(diff).toFixed(2)}`,
612
+ needed: verdict === "leans" ? scaled(test.p, runs, RANK_FLOOR) : null,
613
+ unit: "runs",
614
+ short: shortOf(verdict, minimumP(xs.length, ys.length), Math.min(xs.length, ys.length), rankReaches)
615
+ }];
616
+ });
617
+ }
618
+ function questionMeasures(input, cases) {
619
+ return [...input.questions].flatMap(([question, wins]) => {
620
+ if (!wins.voted) return [];
621
+ const decisive = wins.variant + wins.control;
622
+ const p = mcnemar(wins.variant, wins.control);
623
+ const s = wins.variant - wins.control;
624
+ const verdict = screened(tested(p, s, true), decisive >= 4 ? Math.max(wins.variant, wins.control) / decisive : 0, QUESTION_MIN, p, 0, QUESTION_LEAN_P);
625
+ const name = `"${clip(question, LABEL_CHARS)}"`;
626
+ const lead = s > 0 ? "the variant" : "the control";
627
+ const votes = wins.variantVotes + wins.controlVotes;
628
+ const pairs = `${Math.max(wins.variant, wins.control)} of ${decisive} decisive ${decisive === 1 ? "pair" : "pairs"}`;
629
+ return [{
630
+ measure: "question",
631
+ label: question,
632
+ variant: wins.variant,
633
+ control: wins.control,
634
+ estimate: decisive ? wins.variant / decisive : null,
635
+ lo: null,
636
+ hi: null,
637
+ p: decisive ? p : null,
638
+ verdict,
639
+ leaning: leaningOf(verdict, s, true),
640
+ text: s === 0 ? `${name} picked ${decisive ? "each side as often" : "neither"}` : `${name} picked ${lead} in ${s > 0 ? wins.variantVotes : wins.controlVotes} of ${votes} votes (${pairs})`,
641
+ needed: verdict === "leans" ? signRuns(wins, cases) : null,
642
+ unit: "runs",
643
+ short: null
644
+ }];
645
+ });
646
+ }
647
+ function measuresOf(input) {
648
+ const runs = Math.max(1, ...[...input.variant, ...input.control].map((a) => a.attempts.length));
649
+ return [
650
+ input.pass,
651
+ ...effortMeasures(input, runs),
652
+ ...judgeMeasures(input, runs),
653
+ ...questionMeasures(input, input.variant.length),
654
+ ...checkMeasures(input)
655
+ ];
656
+ }
657
+ /**
658
+ * Each pairwise question's decisive pairs for the variant and for the control, summed over the
659
+ * entries. A pair decides only when both its orders pick the same side (or one is a tie): a split
660
+ * followed the position, not the work, and a tie says nothing.
661
+ */
662
+ function questionWins(pairs) {
663
+ const out = /* @__PURE__ */ new Map();
664
+ for (const { entry, variant, control } of pairs) {
665
+ const v = entry.cells.find((c) => c.key === variant);
666
+ const c = entry.cells.find((x) => x.key === control);
667
+ if (!v || !c || v.harness !== c.harness) continue;
668
+ for (const answer of entry.answers ?? []) {
669
+ for (const t of pairTallies(answer.votes)) {
670
+ if (t.cut !== "within" || t.held !== v.harness) continue;
671
+ const forward = t.a === v.environment && t.b === c.environment;
672
+ const back = t.a === c.environment && t.b === v.environment;
673
+ if (!forward && !back) continue;
674
+ const wins = out.get(answer.question) ?? {
675
+ variant: 0,
676
+ control: 0,
677
+ voted: 0,
678
+ variantVotes: 0,
679
+ controlVotes: 0
680
+ };
681
+ wins.variant += forward ? t.aWins : t.bWins;
682
+ wins.control += forward ? t.bWins : t.aWins;
683
+ wins.voted += t.pairs;
684
+ out.set(answer.question, wins);
685
+ }
686
+ for (const vote of answer.votes) {
687
+ const wins = out.get(answer.question);
688
+ if (!wins || vote.cut !== "within" || vote.harness !== v.harness) continue;
689
+ const sides = /* @__PURE__ */ new Set([vote.a, vote.b]);
690
+ if (!sides.has(v.environment) || !sides.has(c.environment)) continue;
691
+ if (vote.winner === v.environment) wins.variantVotes++;
692
+ else if (vote.winner === c.environment) wins.controlVotes++;
693
+ }
694
+ }
695
+ }
696
+ return out;
697
+ }
698
+ const armOf = (e, key) => ({
699
+ attempts: e.attempts.filter((a) => a.cell === key && isCounted(a)),
700
+ checks: e.cells.find((c) => c.key === key)?.checks
701
+ });
702
+ /** A variant cell against its control in one entry; null when either side counted nothing. */
703
+ function cellMeasures(entry, variant, control) {
704
+ const v = entry.cells.find((c) => c.key === variant);
705
+ const c = entry.cells.find((x) => x.key === control);
706
+ if (!v || !c || v.pass + v.fail === 0 || c.pass + c.fail === 0) return null;
707
+ const lift = v.lift?.vs === control ? v.lift : void 0;
708
+ return measuresOf({
709
+ variant: [armOf(entry, variant)],
710
+ control: [armOf(entry, control)],
711
+ pass: rateMeasure("pass", null, {
712
+ pass: v.pass,
713
+ counted: v.pass + v.fail
714
+ }, {
715
+ pass: c.pass,
716
+ counted: c.pass + c.fail
717
+ }, lift?.interval),
718
+ questions: questionWins([{
719
+ entry,
720
+ variant,
721
+ control
722
+ }])
723
+ });
724
+ }
725
+ /**
726
+ * A family's variant against its control over its cases: pass rate is the family's pooled lift
727
+ * (`lift`/`drop` hold, a difference within noise or `insufficient` leans), effort its paired
728
+ * verdict by case, and checks, judges and questions every case's attempts pooled.
729
+ */
730
+ function familyMeasures(cases, lift) {
731
+ const paired = cases.filter((e) => e.cells.some((c) => c.key === lift.variant && c.pass + c.fail > 0) && e.cells.some((c) => c.key === lift.vs && c.pass + c.fail > 0));
732
+ const same = lift.variantRate === lift.controlRate;
733
+ const way = lift.variantRate > lift.controlRate ? "better" : "worse";
734
+ const verdict = screened(same ? "same" : lift.verdict === "lift" ? "better" : lift.verdict === "drop" ? "worse" : "leans", Math.abs(lift.points), RATE_MIN, lift.p, 0);
735
+ return measuresOf({
736
+ variant: paired.map((e) => armOf(e, lift.variant)),
737
+ control: paired.map((e) => armOf(e, lift.vs)),
738
+ pass: {
739
+ measure: "pass",
740
+ label: null,
741
+ variant: lift.variantRate,
742
+ control: lift.controlRate,
743
+ estimate: lift.points,
744
+ lo: lift.interval.lo,
745
+ hi: lift.interval.hi,
746
+ p: lift.p,
747
+ verdict,
748
+ leaning: verdict === "leans" ? way : null,
749
+ text: same ? "same pass rate" : `pass rate ${pts(lift.points)}`,
750
+ needed: null,
751
+ unit: "cases",
752
+ short: null
753
+ },
754
+ questions: questionWins(paired.map((entry) => ({
755
+ entry,
756
+ variant: lift.variant,
757
+ control: lift.vs
758
+ }))),
759
+ family: lift
760
+ });
761
+ }
762
+ function readMeasures(measures) {
763
+ const rest = measures.filter((m) => m.measure !== "pass");
764
+ const apart = rest.filter((m) => m.verdict === "better" || m.verdict === "worse");
765
+ if (apart.length) return {
766
+ told: "apart",
767
+ measures: apart
768
+ };
769
+ const leans = rest.filter((m) => m.verdict === "leans");
770
+ if (!leans.length) return { told: "same" };
771
+ const confirmable = leans.filter((m) => m.needed !== null);
772
+ const unit = confirmable.some((m) => m.unit === "runs") ? "runs" : "cases";
773
+ const soonest = confirmable.filter((m) => m.unit === unit).sort((a, b) => (a.needed ?? 0) - (b.needed ?? 0))[0];
774
+ return {
775
+ told: "leans",
776
+ measures: leans,
777
+ needed: soonest?.needed ?? null,
778
+ unit: soonest?.unit ?? leans[0]?.unit ?? "runs"
779
+ };
780
+ }
781
+ /**
782
+ * `same pass rate · 5× faster · judge "tidy" +0.12`: pass rate, then the measures that hold, and with
783
+ * `leans` the ones that only lean, each marked `(leans)`.
784
+ */
785
+ const measuresLine = (measures, leans = false) => measures.filter((m) => m.measure === "pass" || m.verdict === "better" || m.verdict === "worse" || leans && m.verdict === "leans").map((m) => m.verdict === "leans" && m.measure !== "pass" ? `${m.text} (leans)` : m.text).join(" · ");
786
+ /**
787
+ * Every side passed every counted run (`pass`), or failed every one (`fail`): a cell, or each of a
788
+ * family's case cells. Null when they're mixed, or when any side counted nothing.
789
+ */
790
+ function boundOf(sides) {
791
+ if (!sides.length || sides.some((s) => s.pass + s.fail === 0)) return null;
792
+ if (sides.every((s) => s.fail === 0)) return "pass";
793
+ return sides.every((s) => s.pass === 0) ? "fail" : null;
794
+ }
795
+ /**
796
+ * How arms that both passed, or both failed, every run read: a floor is always a `tie` (a timeout
797
+ * counts as a fail, so effort differs exactly when one arm hangs); a ceiling is a `tie` only when
798
+ * every other measure is the same, a lean when some only lean, and null when one is better or
799
+ * worse, since then the comparison is the result.
800
+ */
801
+ function tieOf(bound, measures) {
802
+ if (bound === "fail") return "tie";
803
+ const reading = readMeasures(measures ?? []);
804
+ return reading.told === "same" ? "tie" : reading.told === "leans" ? reading : null;
805
+ }
806
+ // src/view/suite.ts
807
+ /** Summed per provider, in first-seen order. */
808
+ function tallyUpstreams(counts) {
809
+ const by = /* @__PURE__ */ new Map();
810
+ for (const { provider, calls } of counts) by.set(provider, (by.get(provider) ?? 0) + calls);
811
+ return [...by].map(([provider, calls]) => ({
812
+ provider,
813
+ calls
814
+ }));
815
+ }
816
+ /** A cell's attempts the run's interrupt canceled: recorded as fails, but not the agent's. */
817
+ const canceledOf = (cell) => cell.errors.filter((err) => err.kind === "aborted").reduce((n, err) => n + err.attempts, 0);
818
+ /** A cell's counted tries: its passes and fails, less the ones the interrupt canceled. */
819
+ const countedOf = (cell) => Math.max(0, cell.passed + cell.failed - canceledOf(cell));
820
+ /** The direction of `variant`'s pass rate against `control`'s; null when a side counted none. */
821
+ function effectOf(variant, control) {
822
+ if (variant.counted === 0 || control.counted === 0) return null;
823
+ const v = variant.pass * control.counted;
824
+ const c = control.pass * variant.counted;
825
+ return v > c ? "better" : v < c ? "worse" : "same";
826
+ }
827
+ /** A check's passes over the attempts that answered it; `notRun` and `error` aren't counted. */
828
+ const checkSide = (t) => ({
829
+ pass: t.pass,
830
+ counted: t.pass + t.fail
831
+ });
832
+ /** The check's direction on the variant against the baseline; null when a side never answered it. */
833
+ const checkEffect = (variant, control) => effectOf(checkSide(variant), checkSide(control));
834
+ /** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
835
+ const factSegment = (s) => s.replace(/%/g, "%25").replace(/\//g, "%2F").replace(/\+/g, "%2B");
836
+ const cellSegment = (c) => [
837
+ c.harness,
838
+ c.environment,
839
+ c.sandbox
840
+ ].map(factSegment).join("+");
841
+ const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
842
+ function evalAcross(cells) {
843
+ const harness = new Set(cells.map((c) => c.harness)).size > 1;
844
+ const change = cells.some((c) => c.role === "variant") || new Set(cells.map((c) => c.environment)).size > 1;
845
+ return harness && change ? "both" : harness ? "harness" : change ? "change" : "none";
846
+ }
847
+ /** The skill sources and files an environment placed; `local:` sources as their path. */
848
+ function contextOf(record, cell) {
849
+ const envs = record.environments.filter((e) => e.digest === cell.environmentDigest);
850
+ const env = envs.find((e) => e.name === cell.environment) ?? envs[0];
851
+ if (!env) return [];
852
+ return [...env.skills.map((s) => s.source.replace(/^local:/, "")), ...env.files.map((f) => f.path)];
853
+ }
854
+ /** The control a variant is measured against: its lift's, else the control on its harness and sandbox. */
855
+ function baselineOf(e, cell) {
856
+ if (cell.role !== "variant") return void 0;
857
+ const vs = cell.lift?.vs;
858
+ return vs ? e.cells.find((c) => c.key === vs) : e.cells.find((c) => c.role === "control" && c.harness === cell.harness && c.sandbox === cell.sandbox);
859
+ }
860
+ /** Every check label the attempts recorded, in first-seen order: the eval's declared checks. */
861
+ function declaredChecks(entries) {
862
+ return [...new Set(entries.flatMap((e) => e.attempts.flatMap((a) => (a.assertions ?? []).map((x) => x.label))))];
863
+ }
864
+ function checksOf(e, cell, baseline, base, declared) {
865
+ const attempts = e.attempts.filter((a) => a.cell === cell.key && counted(a));
866
+ return (cell.checks ?? declared.map((label) => ({
867
+ label,
868
+ pass: 0,
869
+ fail: 0,
870
+ error: 0,
871
+ notRun: 0
872
+ }))).map((k, order) => {
873
+ const ran = k.pass + k.fail + k.error;
874
+ const answered = attempts.flatMap((a) => a.assertions ?? []).filter((x) => x.label === k.label && x.message);
875
+ const shown = answered.find((x) => x.status === "fail") ?? answered.find((x) => x.status === "error");
876
+ const assertion = shown?.message ?? null;
877
+ const control = baseline ? baseline.checks?.find((x) => x.label === k.label) ?? {
878
+ pass: 0,
879
+ fail: 0,
880
+ error: 0,
881
+ notRun: 0
882
+ } : void 0;
883
+ const judged = attempts.flatMap((a) => labeledJudgments(a).filter((j) => j.check === k.label).map((j) => j.judgment));
884
+ const first = judged[0];
885
+ const notes = attempts.flatMap((a) => {
886
+ const check = checksByLabel(a).get(k.label);
887
+ const note = check && checkTries(check);
888
+ return note ? [note] : [];
889
+ });
890
+ const retried = attempts.length === 1 ? notes[0] : notes.length ? `retried in ${notes.length} of ${attempts.length} attempts` : void 0;
891
+ return {
892
+ path: `${base}/checks/${factSegment(k.label)}`,
893
+ label: k.label,
894
+ order,
895
+ outcome: ran === 0 ? { status: "not run" } : {
896
+ status: "ran",
897
+ passed: k.pass,
898
+ failed: k.fail,
899
+ errored: k.error,
900
+ notRun: k.notRun,
901
+ assertion,
902
+ at: shown?.at ?? null
903
+ },
904
+ effect: control ? checkEffect(k, control) : null,
905
+ judge: first ? {
906
+ question: first.question,
907
+ threshold: first.threshold,
908
+ scores: judged.map((j) => j.score)
909
+ } : null,
910
+ ...retried ? { retried } : {}
911
+ };
912
+ });
913
+ }
914
+ const informative = (line) => /[\p{L}\p{N}]/u.test(line);
915
+ /**
916
+ * An error's one line: its first line with any letter or digit in it, joined with the next one
917
+ * when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
918
+ */
919
+ function messageLine(text) {
920
+ const [first = "", next] = text.split("\n").map((l) => l.trim()).filter(informative);
921
+ return first.endsWith(":") && next ? `${first} ${next.replace(/^[|>]\s*/, "")}` : first;
922
+ }
923
+ /**
924
+ * The errors of a cell's failed and infra-error attempts that no failed check explains: every
925
+ * timeout, infra error and abort, and an assertion or judge error on an attempt whose recorded
926
+ * checks all passed (a throw in the eval's body), plus each turn's error, which is often why.
927
+ */
928
+ function errorsOf(e, cell) {
929
+ const out = [];
930
+ const add = (kind, text) => {
931
+ const message = messageLine(text);
932
+ const same = out.find((x) => x.kind === kind && x.message === message);
933
+ if (same) same.attempts++;
934
+ else out.push({
935
+ kind,
936
+ message,
937
+ attempts: 1
938
+ });
939
+ };
940
+ for (const a of e.attempts) {
941
+ if (a.cell !== cell.key) continue;
942
+ if (a.outcome !== "fail" && a.outcome !== "infra_error") continue;
943
+ for (const turn of a.turns ?? []) if (turn.error) add("turn", turn.error);
944
+ if (!a.error) continue;
945
+ if ((a.error.kind === "assertion" || a.error.kind === "judge") && (a.assertions ?? []).some((x) => x.status !== "pass")) continue;
946
+ add(a.error.kind, a.error.message);
947
+ }
948
+ return out;
949
+ }
950
+ function questionsOf(e) {
951
+ const pairs = (e.answers ?? []).map((a) => ({
952
+ kind: "pairs",
953
+ question: a.question,
954
+ votes: a.votes.length,
955
+ rates: a.rates.map((r) => ({
956
+ cut: r.cut,
957
+ name: r.name,
958
+ wins: r.wins,
959
+ n: r.n
960
+ })),
961
+ error: a.error ?? null
962
+ }));
963
+ const each = attemptAnswerRows(e).map((row) => {
964
+ const failed = row.answers.filter((a) => a.error !== null);
965
+ return {
966
+ kind: "attempts",
967
+ question: row.question,
968
+ asked: row.answers.length,
969
+ choices: row.choices,
970
+ answers: row.answers.filter((a) => a.error === null).map((a) => ({
971
+ attempt: a.attempt,
972
+ choice: a.choice,
973
+ text: a.text
974
+ })),
975
+ failed: failed.length,
976
+ error: failed[0]?.error ?? null
977
+ };
978
+ });
979
+ return [...pairs, ...each];
980
+ }
981
+ function effortOf(e, cell) {
982
+ const effort = cellEffort(e, cell.key);
983
+ if (!effort || effort.n < 4) return null;
984
+ return {
985
+ tools: effort.tools.median,
986
+ agentMs: effort.agentMs.median,
987
+ tokens: effort.tokens?.median ?? null
988
+ };
989
+ }
990
+ /** A variant's direction against its baseline; null without one, or with nothing counted on a side. */
991
+ function cellEffect(cell, baseline) {
992
+ if (cell.lift) return effectOf(cell.lift.variant, cell.lift.control);
993
+ if (!baseline) return null;
994
+ return effectOf({
995
+ pass: cell.pass,
996
+ counted: cell.pass + cell.fail
997
+ }, {
998
+ pass: baseline.pass,
999
+ counted: baseline.pass + baseline.fail
1000
+ });
1001
+ }
1002
+ function cellsOf(record, e, parent, declared) {
1003
+ const hasVariants = e.cells.some((c) => c.role === "variant");
1004
+ return e.cells.map((cell, order) => {
1005
+ const path = `${parent}/${cellSegment(cell)}`;
1006
+ const baseline = baselineOf(e, cell);
1007
+ const attempts = e.attempts.filter((a) => a.cell === cell.key && counted(a));
1008
+ const ran = e.attempts.filter((a) => a.cell === cell.key);
1009
+ const upstreams = tallyUpstreams(ran.flatMap((a) => a.upstreams ?? []));
1010
+ const judgeUpstreams = tallyUpstreams(ran.flatMap((a) => a.judgements.flatMap((j) => j.upstream ? [{
1011
+ provider: j.upstream,
1012
+ calls: 1
1013
+ }] : [])));
1014
+ const lift = cell.lift;
1015
+ const interval = lift ? {
1016
+ low: lift.interval.lo,
1017
+ high: lift.interval.hi
1018
+ } : null;
1019
+ const snapshots = ran.flatMap((a) => {
1020
+ const path = (a.outcome === "fail" || a.outcome === "infra_error") && a.snapshots?.find((s) => s.name === "final")?.path;
1021
+ return path ? [`.reeval/runs/${record.runId}/${path}`] : [];
1022
+ });
1023
+ const pageErrors = ran.flatMap((a) => (a.files.artifacts ?? []).flatMap(({ name, capture }) => capture?.pageErrors && capture.page ? [{
1024
+ capture: name,
1025
+ errors: capture.pageErrors,
1026
+ file: `.reeval/runs/${record.runId}/${capture.page.path}`
1027
+ }] : []));
1028
+ return {
1029
+ path,
1030
+ key: cell.key,
1031
+ harness: cell.harness,
1032
+ environment: cell.environment,
1033
+ sandbox: cell.sandbox,
1034
+ variant: cell.variant ?? null,
1035
+ baseline: hasVariants && cell.role === "control",
1036
+ vs: baseline?.key ?? null,
1037
+ order,
1038
+ gated: cell.gate !== "none",
1039
+ outcome: cell.outcome,
1040
+ attempts: cell.runs,
1041
+ passed: cell.pass,
1042
+ failed: cell.fail,
1043
+ infraError: cell.infraError,
1044
+ skipped: cell.skipped,
1045
+ ...cell.bestOf !== void 0 ? {
1046
+ bestOf: cell.bestOf,
1047
+ notNeeded: ran.filter((a) => a.bestOf === "skipped").length
1048
+ } : {},
1049
+ fingerprint: cell.fingerprint,
1050
+ context: contextOf(record, cell),
1051
+ checks: checksOf(e, cell, baseline, path, declared),
1052
+ effect: cell.role === "variant" ? cellEffect(cell, baseline) : null,
1053
+ interval,
1054
+ minLift: lift?.minLift ?? null,
1055
+ effort: effortOf(e, cell),
1056
+ measures: cell.role === "variant" && baseline ? cellMeasures(e, cell.key, baseline.key) : null,
1057
+ errors: errorsOf(e, cell),
1058
+ ...snapshots.length ? { snapshots } : {},
1059
+ searches: attempts.map((a) => a.activity.webSearches),
1060
+ fetches: attempts.map((a) => a.activity.webFetches),
1061
+ ...upstreams.length ? { upstreams } : {},
1062
+ ...judgeUpstreams.length ? { judgeUpstreams } : {},
1063
+ ...pageErrors.length ? { pageErrors } : {}
1064
+ };
1065
+ });
1066
+ }
1067
+ function pooledEffect(lift) {
1068
+ if (lift.variantRate === lift.controlRate) return "same";
1069
+ return lift.variantRate > lift.controlRate ? "better" : "worse";
1070
+ }
1071
+ /** Fail beats infra error, which beats canceled, which beats pass; skipped only when every one was. */
1072
+ function worst(outcomes) {
1073
+ if (outcomes.includes("fail")) return "fail";
1074
+ if (outcomes.includes("infra_error")) return "infra_error";
1075
+ if (outcomes.includes("canceled")) return "canceled";
1076
+ if (outcomes.length && outcomes.every((o) => o === "skipped")) return "skipped";
1077
+ return "pass";
1078
+ }
1079
+ /** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
1080
+ const isStudy = (cells) => cells.length > 0 && cells.every((c) => !c.gated);
1081
+ /**
1082
+ * A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
1083
+ * or `passRate` gating it, so it reads `ran`, never `pass`.
1084
+ */
1085
+ const isNotGatedResult = (e) => e.outcome === "pass" && (e.cases ? e.cases.length > 0 && e.cases.every((k) => isStudy(k.cells)) && (e.pooled ?? []).every((p) => p.minLift === null) && !e.rates?.length : isStudy(e.cells ?? []));
1086
+ /** The record's outcome, or `canceled` when every gated attempt that failed was canceled. */
1087
+ function outcomeOf(e) {
1088
+ if (e.outcome !== "fail") return e.outcome;
1089
+ const gated = new Set(e.cells.filter((c) => c.gate !== "none").map((c) => c.key));
1090
+ const failed = e.attempts.filter((a) => a.outcome === "fail" && gated.has(a.cell));
1091
+ return failed.length && failed.every((a) => a.error?.kind === "aborted") ? "canceled" : "fail";
1092
+ }
1093
+ /** A case's outcome as its family's verdict reads it: a failure its family's rate gates passes. */
1094
+ const caseVerdict = (e) => e.verdict === "family" && e.outcome === "fail" ? "pass" : outcomeOf(e);
1095
+ /** Each question any case asked, in first-asked order, with its votes or answers summed over the cases. */
1096
+ function familyQuestions(cases) {
1097
+ const out = /* @__PURE__ */ new Map();
1098
+ for (const q of cases.flatMap(questionsOf)) {
1099
+ const key = `${q.kind}\0${q.question}`;
1100
+ const have = out.get(key);
1101
+ if (!have) {
1102
+ out.set(key, q.kind === "pairs" ? {
1103
+ ...q,
1104
+ rates: q.rates.map((r) => ({ ...r }))
1105
+ } : {
1106
+ ...q,
1107
+ choices: q.choices?.map((c) => ({ ...c })) ?? null,
1108
+ answers: [...q.answers]
1109
+ });
1110
+ continue;
1111
+ }
1112
+ have.error ??= q.error;
1113
+ if (have.kind === "attempts" && q.kind === "attempts") {
1114
+ have.asked += q.asked;
1115
+ have.failed += q.failed;
1116
+ have.answers.push(...q.answers);
1117
+ for (const c of q.choices ?? []) {
1118
+ const same = have.choices?.find((x) => x.choice === c.choice);
1119
+ if (same) same.n += c.n;
1120
+ }
1121
+ continue;
1122
+ }
1123
+ if (have.kind !== "pairs" || q.kind !== "pairs") continue;
1124
+ have.votes += q.votes;
1125
+ for (const r of q.rates) {
1126
+ const same = have.rates.find((x) => x.cut === r.cut && x.name === r.name);
1127
+ if (!same) have.rates.push({ ...r });
1128
+ else {
1129
+ same.wins += r.wins;
1130
+ same.n += r.n;
1131
+ }
1132
+ }
1133
+ }
1134
+ return [...out.values()];
1135
+ }
1136
+ function familyResult(record, family, cases, order) {
1137
+ const path = factSegment(family.id);
1138
+ const sorted = [...cases].sort((a, b) => (a.family?.case ?? 0) - (b.family?.case ?? 0));
1139
+ const declared = declaredChecks(sorted);
1140
+ const cellOf = (key) => sorted.flatMap((c) => c.cells).find((c) => c.key === key);
1141
+ const results = sorted.map((c) => {
1142
+ const label = caseLabel(family.name, c.name);
1143
+ const casePath = `${path}/${factSegment(label)}`;
1144
+ return {
1145
+ path: casePath,
1146
+ id: c.id,
1147
+ label,
1148
+ name: c.name,
1149
+ order: c.family?.case ?? 0,
1150
+ outcome: outcomeOf(c),
1151
+ notRun: notRunYet(record, c),
1152
+ cells: cellsOf(record, c, casePath, declared)
1153
+ };
1154
+ });
1155
+ return {
1156
+ path,
1157
+ id: family.id,
1158
+ group: family.group,
1159
+ name: family.name,
1160
+ template: family.name,
1161
+ order,
1162
+ across: evalAcross(cases.flatMap((c) => c.cells)),
1163
+ family: true,
1164
+ todo: false,
1165
+ notRun: results.every((c) => c.notRun),
1166
+ outcome: worst([...family.outcome === "fail" ? ["fail"] : [], ...sorted.map(caseVerdict)]),
1167
+ cells: null,
1168
+ cases: results,
1169
+ pooled: family.lifts.map((lift) => {
1170
+ const cell = cellOf(lift.variant);
1171
+ return {
1172
+ path: `${path}/${cell ? cellSegment(cell) : factSegment(lift.variant)}`,
1173
+ key: lift.variant,
1174
+ vs: lift.vs,
1175
+ effect: pooledEffect(lift),
1176
+ pairs: lift.pairs,
1177
+ interval: {
1178
+ low: lift.interval.lo,
1179
+ high: lift.interval.hi
1180
+ },
1181
+ verdict: lift.verdict,
1182
+ minLift: lift.minLift ?? null,
1183
+ outcome: lift.outcome,
1184
+ measures: familyMeasures(sorted, lift)
1185
+ };
1186
+ }),
1187
+ rates: family.rates.flatMap((r) => r.passRate !== void 0 && r.outcome !== void 0 ? [{
1188
+ key: r.cell,
1189
+ rate: r.rate,
1190
+ passRate: r.passRate,
1191
+ outcome: r.outcome
1192
+ }] : []),
1193
+ questions: familyQuestions(sorted),
1194
+ observations: observedTallies(sorted.flatMap((k) => k.attempts))
1195
+ };
1196
+ }
1197
+ function evalResult(record, e, order) {
1198
+ const path = factSegment(e.id);
1199
+ return {
1200
+ path,
1201
+ id: e.id,
1202
+ group: e.group,
1203
+ name: e.name,
1204
+ template: null,
1205
+ order,
1206
+ across: evalAcross(e.cells),
1207
+ family: false,
1208
+ todo: e.todo,
1209
+ notRun: notRunYet(record, e),
1210
+ outcome: outcomeOf(e),
1211
+ cells: cellsOf(record, e, path, declaredChecks([e])),
1212
+ cases: null,
1213
+ pooled: null,
1214
+ rates: null,
1215
+ questions: questionsOf(e),
1216
+ observations: observedTallies(e.attempts)
1217
+ };
1218
+ }
1219
+ /** The facts of a run, from its record. Pure: the same record always gives the same result. */
1220
+ function suiteResult(folded) {
1221
+ const record = unfoldCases(folded);
1222
+ const families = new Map((record.families ?? []).map((f) => [f.id, f]));
1223
+ const byFile = /* @__PURE__ */ new Map();
1224
+ const seen = /* @__PURE__ */ new Set();
1225
+ const errors = record.errors ?? [];
1226
+ const tally = {
1227
+ errors: errors.length,
1228
+ pass: 0,
1229
+ notGated: 0,
1230
+ fail: 0,
1231
+ infraError: 0,
1232
+ canceled: 0,
1233
+ skipped: 0
1234
+ };
1235
+ const count = (result) => {
1236
+ const o = result.outcome;
1237
+ if (o === "infra_error") tally.infraError++;
1238
+ else if (o === "pass" && isNotGatedResult(result)) tally.notGated++;
1239
+ else tally[o]++;
1240
+ };
1241
+ for (const e of record.evals) {
1242
+ const evals = byFile.get(e.file) ?? [];
1243
+ byFile.set(e.file, evals);
1244
+ const family = e.family && families.get(e.family.id);
1245
+ if (!family) {
1246
+ const result = evalResult(record, e, evals.length);
1247
+ count(result);
1248
+ evals.push(result);
1249
+ continue;
1250
+ }
1251
+ if (seen.has(family.id)) continue;
1252
+ seen.add(family.id);
1253
+ const result = familyResult(record, family, record.evals.filter((x) => x.family?.id === family.id), evals.length);
1254
+ count(result);
1255
+ evals.push(result);
1256
+ }
1257
+ const files = [...byFile].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([path, evals]) => ({
1258
+ path,
1259
+ evals
1260
+ }));
1261
+ return {
1262
+ runId: record.runId,
1263
+ status: record.status,
1264
+ outcome: runOutcome(folded),
1265
+ errors,
1266
+ tally,
1267
+ files
1268
+ };
1269
+ }
1270
+ export { observedTallies as S, tieOf as _, evalAcross as a, observedLine as b, isStudy as c, boundOf as d, cellMeasures as f, measuresLine as g, measurePhrase as h, effectOf as i, messageLine as l, leanText as m, checkEffect as n, factSegment as o, familyMeasures as p, countedOf as r, isNotGatedResult as s, canceledOf as t, suiteResult as u, familiesOf as v, observedShift as x, pooledPassRate as y };
1271
+
1272
+ //# sourceMappingURL=suite-COglUhPd.js.map