reeval 0.24.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (560) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +16 -16
  3. package/THIRD_PARTY_NOTICES.md +0 -2068
  4. package/assets/package.json +1 -1
  5. package/assets/pnpm-lock.yaml +65 -50
  6. package/dist/{adapter-dHdepWKb.js → adapter-BHMbIeH9.js} +7 -7
  7. package/dist/{adapter-dHdepWKb.js.map → adapter-BHMbIeH9.js.map} +1 -1
  8. package/dist/{agent-CZpiwA_r.js → agent-D67VCd6Q.js} +8 -8
  9. package/dist/{agent-CZpiwA_r.js.map → agent-D67VCd6Q.js.map} +1 -1
  10. package/dist/{agents-CEapsHV-.js → agents-Dnv1MmgY.js} +40 -59
  11. package/dist/agents-Dnv1MmgY.js.map +1 -0
  12. package/dist/{analysis-BnG_1x_O.js → analysis-8vvMnUJB.js} +4 -5
  13. package/dist/analysis-8vvMnUJB.js.map +1 -0
  14. package/dist/api-ZeGnedB3.js +141 -0
  15. package/dist/api-ZeGnedB3.js.map +1 -0
  16. package/dist/assemble-BlA7FWRn.js +1 -0
  17. package/dist/{assemble-Dw1bF13I.js → assemble-CuvlsOaE.js} +117 -66
  18. package/dist/assemble-CuvlsOaE.js.map +1 -0
  19. package/dist/{attach-DvYiMu6w.js → attach-6av4DmO3.js} +24 -6
  20. package/dist/attach-6av4DmO3.js.map +1 -0
  21. package/dist/{attempt-DxR1agFo.js → attempt-D5WH0yFa.js} +3 -3
  22. package/dist/{attempt-DxR1agFo.js.map → attempt-D5WH0yFa.js.map} +1 -1
  23. package/dist/{attempt-host-DgeJFSZw.js → attempt-host-Be_-uDKJ.js} +37 -14
  24. package/dist/attempt-host-Be_-uDKJ.js.map +1 -0
  25. package/dist/boat-BCNNEJni.js +298 -0
  26. package/dist/boat-BCNNEJni.js.map +1 -0
  27. package/dist/boat-CwBW2sbp.js +1 -0
  28. package/dist/{boat-CUNgSc75.js → boat-CygRcaM0.js} +115 -196
  29. package/dist/boat-CygRcaM0.js.map +1 -0
  30. package/dist/{boat-images-BuFigiEb.js → boat-images-asGLQXD5.js} +19 -23
  31. package/dist/boat-images-asGLQXD5.js.map +1 -0
  32. package/dist/{bundle-yEgI3RsI.js → bundle-B7P-Awnm.js} +105 -17
  33. package/dist/bundle-B7P-Awnm.js.map +1 -0
  34. package/dist/{capture-DdEKVrwk.js → capture-BwLa-4s-.js} +20 -19
  35. package/dist/{capture-DdEKVrwk.js.map → capture-BwLa-4s-.js.map} +1 -1
  36. package/dist/capture-CG-J-1-j.js +1 -0
  37. package/dist/cases-Bsp-G-ZL.js +1 -0
  38. package/dist/{cases-B7QkvMGE.js → cases-C6vnV-gn.js} +60 -42
  39. package/dist/cases-C6vnV-gn.js.map +1 -0
  40. package/dist/classify-BIJQ2Xqo.js +163 -0
  41. package/dist/classify-BIJQ2Xqo.js.map +1 -0
  42. package/dist/{cli-Z7aw8k25.js → cli-BoCPYLUQ.js} +305 -124
  43. package/dist/cli-BoCPYLUQ.js.map +1 -0
  44. package/dist/{cli-spec-3zvLC9YW.js → cli-spec-DgEWHOt3.js} +164 -50
  45. package/dist/cli-spec-DgEWHOt3.js.map +1 -0
  46. package/dist/cli.js +2 -2
  47. package/dist/client-Dj65gXG3.js +105 -0
  48. package/dist/client-Dj65gXG3.js.map +1 -0
  49. package/dist/{cloud-owVBaSRt.js → cloud-TuNUlZ0L.js} +3 -3
  50. package/dist/{cloud-owVBaSRt.js.map → cloud-TuNUlZ0L.js.map} +1 -1
  51. package/dist/{cloud-files-BjinrTTe.js → cloud-files-CBovrfZA.js} +8 -8
  52. package/dist/{cloud-files-BjinrTTe.js.map → cloud-files-CBovrfZA.js.map} +1 -1
  53. package/dist/{cloud-DuHcNQVV.js → cloud-igGIS4Bm.js} +43 -20
  54. package/dist/cloud-igGIS4Bm.js.map +1 -0
  55. package/dist/cloud-project-Ch7jjlwh.js +110 -0
  56. package/dist/cloud-project-Ch7jjlwh.js.map +1 -0
  57. package/dist/cloud-project-DdUVB4EE.js +1 -0
  58. package/dist/cloud-push-DvnqOgRd.js +817 -0
  59. package/dist/cloud-push-DvnqOgRd.js.map +1 -0
  60. package/dist/cloud-push-MXKreNyq.js +1 -0
  61. package/dist/{cloud-run-DqMkSGYB.js → cloud-run-CbGF8Lwt.js} +285 -48
  62. package/dist/cloud-run-CbGF8Lwt.js.map +1 -0
  63. package/dist/cloud-run-CghE2dG6.js +1 -0
  64. package/dist/{cloud-secrets-Bak83tik.js → cloud-secrets-BWoV4ddM.js} +43 -35
  65. package/dist/cloud-secrets-BWoV4ddM.js.map +1 -0
  66. package/dist/cloud-secrets-DlPRfS2l.js +1 -0
  67. package/dist/command-Bps8kBML.js +141 -0
  68. package/dist/command-Bps8kBML.js.map +1 -0
  69. package/dist/{command-CtmSJzPC.js → command-ByUmSm2D.js} +29 -23
  70. package/dist/command-ByUmSm2D.js.map +1 -0
  71. package/dist/{command-CeygdoVe.js → command-CQk7FCI1.js} +92 -26
  72. package/dist/command-CQk7FCI1.js.map +1 -0
  73. package/dist/{command-CoHN1ibe.js → command-CxuGky7A.js} +44 -20
  74. package/dist/command-CxuGky7A.js.map +1 -0
  75. package/dist/{command-efTGfUYb.js → command-pxfK2bkV.js} +26 -16
  76. package/dist/command-pxfK2bkV.js.map +1 -0
  77. package/dist/commands-Cl3Y_UXg.js +1 -0
  78. package/dist/{commands-DR-NolV2.js → commands-DGWNNyRD.js} +54 -38
  79. package/dist/commands-DGWNNyRD.js.map +1 -0
  80. package/dist/{config-Bn2OhCN_.js → config-mU7wFvQq.js} +46 -32
  81. package/dist/config-mU7wFvQq.js.map +1 -0
  82. package/dist/core-BXYzOEyQ.js +2058 -0
  83. package/dist/core-BXYzOEyQ.js.map +1 -0
  84. package/dist/{slug-IHh3f770.js → cost-DQ0ayy6h.js} +37 -2
  85. package/dist/cost-DQ0ayy6h.js.map +1 -0
  86. package/dist/{cursor-catalog-BYIDSsLn.js → cursor-catalog-EOFft6LJ.js} +4 -4
  87. package/dist/{cursor-catalog-BYIDSsLn.js.map → cursor-catalog-EOFft6LJ.js.map} +1 -1
  88. package/dist/{daytona-CWs9j9mt.js → daytona-DtCjv8rH.js} +22 -17
  89. package/dist/daytona-DtCjv8rH.js.map +1 -0
  90. package/dist/daytona-image-Xy5o53Ji.js +1 -0
  91. package/dist/{daytona-image-DmGlFAyq.js → daytona-image-w4uTRZ0z.js} +6 -6
  92. package/dist/{daytona-image-DmGlFAyq.js.map → daytona-image-w4uTRZ0z.js.map} +1 -1
  93. package/dist/db-BcplCdzu.js +1 -0
  94. package/dist/{db-D6yQib7M.js → db-yigMuHM2.js} +31 -119
  95. package/dist/db-yigMuHM2.js.map +1 -0
  96. package/dist/debug-CGg7KHKq.js +1 -0
  97. package/dist/{debug-CIXzB4wK.js → debug-T29p6HOX.js} +2 -10
  98. package/dist/debug-T29p6HOX.js.map +1 -0
  99. package/dist/{default-effort-Ck6ViSaI.js → default-effort-B1b8t81b.js} +1 -1
  100. package/dist/{default-effort-Bpz5ULfC.js → default-effort-BDYXBzu5.js} +3 -3
  101. package/dist/{default-effort-Bpz5ULfC.js.map → default-effort-BDYXBzu5.js.map} +1 -1
  102. package/dist/{dist-CZtW7eI5.js → dist-DCG7vZGQ.js} +3 -3
  103. package/dist/{dist-CZtW7eI5.js.map → dist-DCG7vZGQ.js.map} +1 -1
  104. package/dist/{docker-Dgw4aaRk.js → docker-DcnX2WpM.js} +4 -4
  105. package/dist/{docker-Dgw4aaRk.js.map → docker-DcnX2WpM.js.map} +1 -1
  106. package/dist/{docker-Bbi4kmFj.js → docker-XCyDUd7_.js} +1 -1
  107. package/dist/{doctor-BfS6jswW.js → doctor-DTKa8rrq.js} +27 -27
  108. package/dist/{doctor-BfS6jswW.js.map → doctor-DTKa8rrq.js.map} +1 -1
  109. package/dist/duration-Bz2H7pXb.js +34 -0
  110. package/dist/duration-Bz2H7pXb.js.map +1 -0
  111. package/dist/{environment-BY8lVFFK.js → environment-BCBg6c0p.js} +38 -12
  112. package/dist/environment-BCBg6c0p.js.map +1 -0
  113. package/dist/errors-BjzEWESY.js +93 -0
  114. package/dist/errors-BjzEWESY.js.map +1 -0
  115. package/dist/{errors-P8oHhNzy.js → errors-Cda7UzRA.js} +2 -2
  116. package/dist/{errors-P8oHhNzy.js.map → errors-Cda7UzRA.js.map} +1 -1
  117. package/dist/{eval-source-BB8In3xo.js → eval-source-CSPyKuV6.js} +6 -6
  118. package/dist/{eval-source-BB8In3xo.js.map → eval-source-CSPyKuV6.js.map} +1 -1
  119. package/dist/{evaluate-B9_nukrc.d.ts → evaluate-Nnh0iPuo.d.ts} +1 -1
  120. package/dist/{evaluate-B8I9AKgw.js → evaluate-wgW1lGQj.js} +280 -60
  121. package/dist/evaluate-wgW1lGQj.js.map +1 -0
  122. package/dist/{events-D97ZzAeq.js → events-DG_Cq5Pn.js} +154 -33
  123. package/dist/events-DG_Cq5Pn.js.map +1 -0
  124. package/dist/events-RaK1pDmT.js +1 -0
  125. package/dist/{fake-bootstrap-4DFtzib0.js → fake-bootstrap-uCyHhnMO.js} +2 -2
  126. package/dist/{fake-bootstrap-4DFtzib0.js.map → fake-bootstrap-uCyHhnMO.js.map} +1 -1
  127. package/dist/files-BnsRehge.js +1 -0
  128. package/dist/{files-B0DbptiO.js → files-Bv6ipFjc.js} +4 -4
  129. package/dist/{files-B0DbptiO.js.map → files-Bv6ipFjc.js.map} +1 -1
  130. package/dist/{files-GBTUBAek.js → files-CcnSILye.js} +3 -3
  131. package/dist/{files-GBTUBAek.js.map → files-CcnSILye.js.map} +1 -1
  132. package/dist/{files-BesexM88.js → files-CjPm3yyj.js} +173 -41
  133. package/dist/files-CjPm3yyj.js.map +1 -0
  134. package/dist/files-CkmCV5rH.js +1 -0
  135. package/dist/{files-DE13BqBC.js → files-ClUbQPj9.js} +2 -2
  136. package/dist/{files-DE13BqBC.js.map → files-ClUbQPj9.js.map} +1 -1
  137. package/dist/{files-B4K1k__s.js → files-CqfVnRXw.js} +5 -5
  138. package/dist/{files-B4K1k__s.js.map → files-CqfVnRXw.js.map} +1 -1
  139. package/dist/files-D4Hb9HZS.js +1 -0
  140. package/dist/files-DCcRkJN4.js +1 -0
  141. package/dist/{files-TwSVeWwM.js → files-GAocj_Z1.js} +27 -41
  142. package/dist/files-GAocj_Z1.js.map +1 -0
  143. package/dist/gateway.d.ts +1 -1
  144. package/dist/{git-DatgUcnf.js → git-CRPr6I2T.js} +3 -3
  145. package/dist/{git-DatgUcnf.js.map → git-CRPr6I2T.js.map} +1 -1
  146. package/dist/{harness-FcTyyPco.js → harness-Dcd3ffkY.js} +3 -2
  147. package/dist/harness-Dcd3ffkY.js.map +1 -0
  148. package/dist/{harness-EZNoeUBf.js → harness-DvqNGwl6.js} +2 -2
  149. package/dist/{harness-EZNoeUBf.js.map → harness-DvqNGwl6.js.map} +1 -1
  150. package/dist/{harness-install-HgR73tpF.js → harness-install-B0Y2GjnO.js} +4 -4
  151. package/dist/{harness-install-HgR73tpF.js.map → harness-install-B0Y2GjnO.js.map} +1 -1
  152. package/dist/harness.d.ts +1 -1
  153. package/dist/harness.js +1 -1
  154. package/dist/http-DtlSzw4A.js +96 -0
  155. package/dist/http-DtlSzw4A.js.map +1 -0
  156. package/dist/{ids-DqEvrBFZ.js → ids-CtxRn13a.js} +4 -3
  157. package/dist/ids-CtxRn13a.js.map +1 -0
  158. package/dist/{images-SRE4f3SM.js → images-JspTw1vt.js} +13 -13
  159. package/dist/{images-SRE4f3SM.js.map → images-JspTw1vt.js.map} +1 -1
  160. package/dist/index.d.ts +7 -2
  161. package/dist/index.js +4 -5
  162. package/dist/init-BHIWfE2L.js +1 -0
  163. package/dist/{init-D5F9MYPC.js → init-wmMbauok.js} +6 -6
  164. package/dist/{init-D5F9MYPC.js.map → init-wmMbauok.js.map} +1 -1
  165. package/dist/{ink-list-CLq6YVB8.js → ink-list-B5Gul5iZ.js} +7 -6
  166. package/dist/ink-list-B5Gul5iZ.js.map +1 -0
  167. package/dist/{inputs-B_FsOD2N.js → inputs-BWTtz1-v.js} +6 -6
  168. package/dist/{inputs-B_FsOD2N.js.map → inputs-BWTtz1-v.js.map} +1 -1
  169. package/dist/inputs-load-C8HEtAej.js +1 -0
  170. package/dist/{inputs-load-B4MFkp0Q.js → inputs-load-DQRC3dUA.js} +3 -3
  171. package/dist/{inputs-load-B4MFkp0Q.js.map → inputs-load-DQRC3dUA.js.map} +1 -1
  172. package/dist/internal/vm-api/boat.d.ts +46 -0
  173. package/dist/internal/vm-api/boat.js +1 -0
  174. package/dist/internal/vm-api/daytona.d.ts +26 -0
  175. package/dist/internal/vm-api/daytona.js +220 -0
  176. package/dist/internal/vm-api/daytona.js.map +1 -0
  177. package/dist/internal/vm-api.d.ts +86 -0
  178. package/dist/internal/vm-api.js +3 -0
  179. package/dist/internal/vm.d.ts +1 -1
  180. package/dist/internal/vm.js +1 -1
  181. package/dist/{invocations-BxQqe5GU.js → invocations-CSA9OKFn.js} +5 -5
  182. package/dist/{invocations-BxQqe5GU.js.map → invocations-CSA9OKFn.js.map} +1 -1
  183. package/dist/issue-ipG9Yy1h.js +1 -0
  184. package/dist/{issue-iErT9KkK.js → issue-lKqtL3iX.js} +4 -4
  185. package/dist/{issue-iErT9KkK.js.map → issue-lKqtL3iX.js.map} +1 -1
  186. package/dist/{jiti-CtahaHaY.js → jiti-aNfS_lSj.js} +2 -2
  187. package/dist/{jiti-CtahaHaY.js.map → jiti-aNfS_lSj.js.map} +1 -1
  188. package/dist/{judge-MOKPsr6R.js → judge-etHC4K02.js} +5 -5
  189. package/dist/{judge-MOKPsr6R.js.map → judge-etHC4K02.js.map} +1 -1
  190. package/dist/judge.d.ts +1 -1
  191. package/dist/judge.js +1 -1
  192. package/dist/{judgments-Dur4D2pj.js → judgments-CVCUWT-g.js} +7 -4
  193. package/dist/{judgments-Dur4D2pj.js.map → judgments-CVCUWT-g.js.map} +1 -1
  194. package/dist/{kinds-Dohijzi-.js → kinds-BQX5ZxpK.js} +37 -11
  195. package/dist/{kinds-Dohijzi-.js.map → kinds-BQX5ZxpK.js.map} +1 -1
  196. package/dist/labels-Dtyf5-pd.js +251 -0
  197. package/dist/labels-Dtyf5-pd.js.map +1 -0
  198. package/dist/{layout-BhhJAEgf.js → layout-Co4tPHWI.js} +3 -3
  199. package/dist/{layout-BhhJAEgf.js.map → layout-Co4tPHWI.js.map} +1 -1
  200. package/dist/link-Cqs2DhxR.js +127 -0
  201. package/dist/link-Cqs2DhxR.js.map +1 -0
  202. package/dist/{list-ClDb8qiX.js → list-CjcNH-5e.js} +10 -7
  203. package/dist/list-CjcNH-5e.js.map +1 -0
  204. package/dist/{live-QFILMmql.js → live-BRF0ogR6.js} +3 -5
  205. package/dist/live-BRF0ogR6.js.map +1 -0
  206. package/dist/live-DTpM9b4D.js +56 -0
  207. package/dist/live-DTpM9b4D.js.map +1 -0
  208. package/dist/{machine-service-xFhakTs5.js → machine-service-BetvBWHz.js} +146 -13
  209. package/dist/machine-service-BetvBWHz.js.map +1 -0
  210. package/dist/{machine-service-DpJScNtA.js → machine-service-XFymGCHj.js} +1 -1
  211. package/dist/{machines-Cm-rU7jp.js → machines-CkDTbMZ6.js} +25 -25
  212. package/dist/{machines-Cm-rU7jp.js.map → machines-CkDTbMZ6.js.map} +1 -1
  213. package/dist/markdown-COts-jxf.js +653 -0
  214. package/dist/markdown-COts-jxf.js.map +1 -0
  215. package/dist/{model-Rmv-L9Jb.js → model-BYHXnYLe.js} +1 -1
  216. package/dist/model-CrsA2Evf.js +1 -0
  217. package/dist/{model-CcvSrrq9.js → model-DNC53cPL.js} +4 -4
  218. package/dist/{model-CcvSrrq9.js.map → model-DNC53cPL.js.map} +1 -1
  219. package/dist/{model-D9WMMEWD.js → model-F4UH_YG6.js} +4 -4
  220. package/dist/{model-D9WMMEWD.js.map → model-F4UH_YG6.js.map} +1 -1
  221. package/dist/{model-mTaaEkeQ.js → model-xemZQ7DK.js} +251 -65
  222. package/dist/model-xemZQ7DK.js.map +1 -0
  223. package/dist/{node-Bl2CtSjB.js → node-gPT36n_n.js} +19 -8
  224. package/dist/{node-Bl2CtSjB.js.map → node-gPT36n_n.js.map} +1 -1
  225. package/dist/options-CAZfxE0h.js +1 -0
  226. package/dist/otlp-BxPnO99e.js +1 -0
  227. package/dist/otlp-DbNz-occ.js +25 -0
  228. package/dist/otlp-DbNz-occ.js.map +1 -0
  229. package/dist/outcome-1LBk2e3w.js +38 -0
  230. package/dist/outcome-1LBk2e3w.js.map +1 -0
  231. package/dist/{patch-DhJDHRFs.js → patch-Dxzv8msl.js} +3 -3
  232. package/dist/{patch-DhJDHRFs.js.map → patch-Dxzv8msl.js.map} +1 -1
  233. package/dist/{patch-C7yBztZj.js → patch-uwo6t4Zy.js} +26 -2
  234. package/dist/patch-uwo6t4Zy.js.map +1 -0
  235. package/dist/{pi-providers-HygmL1y-.js → pi-providers-BjcQv2A1.js} +23 -9
  236. package/dist/{pi-providers-HygmL1y-.js.map → pi-providers-BjcQv2A1.js.map} +1 -1
  237. package/dist/{plain-Ctn8oUdp.js → plain-DXUNwqpO.js} +2 -2
  238. package/dist/{plain-Ctn8oUdp.js.map → plain-DXUNwqpO.js.map} +1 -1
  239. package/dist/{plan-CC0bKbm9.js → plan-DSJX7Oyq.js} +9 -9
  240. package/dist/{plan-CC0bKbm9.js.map → plan-DSJX7Oyq.js.map} +1 -1
  241. package/dist/{plugins-PwEtW8GH.js → plugins-DVEOMsYw.js} +7 -7
  242. package/dist/{plugins-PwEtW8GH.js.map → plugins-DVEOMsYw.js.map} +1 -1
  243. package/dist/{preflight-BHUEfA4m.js → preflight-B3oLZttl.js} +4 -4
  244. package/dist/{preflight-BHUEfA4m.js.map → preflight-B3oLZttl.js.map} +1 -1
  245. package/dist/preflight-CLYRuQ3r.js +1 -0
  246. package/dist/{preflight-DEV_vrdB.js → preflight-DXMD6jou.js} +1 -1
  247. package/dist/{preflight-BzvLYQ70.js → preflight-dUe0QiPK.js} +9 -9
  248. package/dist/{preflight-BzvLYQ70.js.map → preflight-dUe0QiPK.js.map} +1 -1
  249. package/dist/{prepare-D7hPrh7Q.js → prepare-BTaX7vcC.js} +8 -8
  250. package/dist/{prepare-D7hPrh7Q.js.map → prepare-BTaX7vcC.js.map} +1 -1
  251. package/dist/{prepare-DS42k409.js → prepare-QdiYPOwR.js} +2 -2
  252. package/dist/prices-CfnkJOth.js +1 -0
  253. package/dist/{prices-DXJsP2Ha.js → prices-CoLymMRd.js} +3 -3
  254. package/dist/{prices-DXJsP2Ha.js.map → prices-CoLymMRd.js.map} +1 -1
  255. package/dist/{process-RPlEhpHz.js → process-C_nh1H7X.js} +9 -3
  256. package/dist/process-C_nh1H7X.js.map +1 -0
  257. package/dist/provenance-Cylt_YxT.d.ts +222 -0
  258. package/dist/{provenance-DgQJ-dq7.js → provenance-Di42xQIS.js} +5 -5
  259. package/dist/{provenance-DgQJ-dq7.js.map → provenance-Di42xQIS.js.map} +1 -1
  260. package/dist/provider-B24hZX95.js +1 -0
  261. package/dist/{provider-ieWOn4YS.js → provider-Cn7lmECy.js} +2 -2
  262. package/dist/{provider-ieWOn4YS.js.map → provider-Cn7lmECy.js.map} +1 -1
  263. package/dist/{provider-stop-CxzWW7Ft.js → provider-stop-DjBMfZW5.js} +2 -2
  264. package/dist/{provider-stop-CxzWW7Ft.js.map → provider-stop-DjBMfZW5.js.map} +1 -1
  265. package/dist/{providers-N4HDMeeo.js → providers-Bm5NLaRh.js} +1 -1
  266. package/dist/{providers-B5Vr0hKH.js → providers-CTTfYbm0.js} +28 -28
  267. package/dist/{providers-B5Vr0hKH.js.map → providers-CTTfYbm0.js.map} +1 -1
  268. package/dist/query-CTMdWXbx.js +1 -0
  269. package/dist/{query-CmTjYbqh.js → query-TzNtTvwf.js} +2 -2
  270. package/dist/{query-CmTjYbqh.js.map → query-TzNtTvwf.js.map} +1 -1
  271. package/dist/{questions-DnqkJga0.js → questions-BqisU7A0.js} +19 -13
  272. package/dist/questions-BqisU7A0.js.map +1 -0
  273. package/dist/{recipe-B0NeT_Sn.js → recipe-DzjXne35.js} +4 -4
  274. package/dist/{recipe-B0NeT_Sn.js.map → recipe-DzjXne35.js.map} +1 -1
  275. package/dist/{record-2liAelrB.js → record-Cj3yWR0E.js} +43 -12
  276. package/dist/record-Cj3yWR0E.js.map +1 -0
  277. package/dist/{redact-DMueYS-v.js → redact-CfTHYWLv.js} +30 -1
  278. package/dist/redact-CfTHYWLv.js.map +1 -0
  279. package/dist/{remarks-D5qOZzcX.js → remarks-BPKGorBW.js} +4 -2
  280. package/dist/remarks-BPKGorBW.js.map +1 -0
  281. package/dist/{remarks-D-9f1y1B.js → remarks-CYXYiB_O.js} +21 -11
  282. package/dist/{remarks-D-9f1y1B.js.map → remarks-CYXYiB_O.js.map} +1 -1
  283. package/dist/{remote-B7BIRUTK.js → remote-D4UEa7Rs.js} +10 -8
  284. package/dist/{remote-B7BIRUTK.js.map → remote-D4UEa7Rs.js.map} +1 -1
  285. package/dist/renderer/ascii.d.ts +6 -0
  286. package/dist/renderer/ascii.js +7 -0
  287. package/dist/renderer/ascii.js.map +1 -0
  288. package/dist/renderer/markdown.d.ts +31 -0
  289. package/dist/renderer/markdown.js +1 -0
  290. package/dist/report-BSi-R8hC.js +941 -0
  291. package/dist/report-BSi-R8hC.js.map +1 -0
  292. package/dist/report-inputs-5XwZBIRK.js +113 -0
  293. package/dist/report-inputs-5XwZBIRK.js.map +1 -0
  294. package/dist/{resolve-B60w6DcB.js → resolve-BEltmYhD.js} +6 -5
  295. package/dist/resolve-BEltmYhD.js.map +1 -0
  296. package/dist/resolve-C7A5Q2Jj.js +1 -0
  297. package/dist/{resolve-t9sOkGGS.js → resolve-nRzGt4Ff.js} +30 -13
  298. package/dist/resolve-nRzGt4Ff.js.map +1 -0
  299. package/dist/{results-Df910ONm.js → results-DXBaxfIM.js} +11 -7
  300. package/dist/{results-Df910ONm.js.map → results-DXBaxfIM.js.map} +1 -1
  301. package/dist/{resume-4Q9q8AcE.js → resume-DZPjjToJ.js} +46 -21
  302. package/dist/resume-DZPjjToJ.js.map +1 -0
  303. package/dist/{retry-C15f5yyy.js → retry-rLdXCinf.js} +5 -5
  304. package/dist/{retry-C15f5yyy.js.map → retry-rLdXCinf.js.map} +1 -1
  305. package/dist/{root-Cm93-6ix.js → root-Co0jhOT5.js} +2 -2
  306. package/dist/{root-Cm93-6ix.js.map → root-Co0jhOT5.js.map} +1 -1
  307. package/dist/run-D_o_8Fkv.js +128 -0
  308. package/dist/run-D_o_8Fkv.js.map +1 -0
  309. package/dist/{client-kUwtq-vt.js → runtime-DsRjnydE.js} +525 -105
  310. package/dist/runtime-DsRjnydE.js.map +1 -0
  311. package/dist/sandbox/boat.d.ts +1 -1
  312. package/dist/sandbox/boat.js +2 -2
  313. package/dist/sandbox/daytona.d.ts +1 -1
  314. package/dist/sandbox/daytona.js +1 -1
  315. package/dist/{sandbox-BffNervX.js → sandbox-B8gDRs5E.js} +3 -3
  316. package/dist/{sandbox-BffNervX.js.map → sandbox-B8gDRs5E.js.map} +1 -1
  317. package/dist/sandbox.d.ts +1 -1
  318. package/dist/sandbox.js +1 -1
  319. package/dist/{secrets-file-Dtd3h3kY.js → secrets-file-B8rUJV2L.js} +4 -4
  320. package/dist/{secrets-file-Dtd3h3kY.js.map → secrets-file-B8rUJV2L.js.map} +1 -1
  321. package/dist/server-eUWbjZoi.js +1 -0
  322. package/dist/server-mJzPXBMM.js +204 -0
  323. package/dist/server-mJzPXBMM.js.map +1 -0
  324. package/dist/{services-BG8xuU82.js → services-CHh83UDC.js} +4 -2
  325. package/dist/services-CHh83UDC.js.map +1 -0
  326. package/dist/{shared-DtZwIAfy.js → shared-bXaqVxk-.js} +5 -34
  327. package/dist/shared-bXaqVxk-.js.map +1 -0
  328. package/dist/skill-dirs-B7o83G5I.js +63 -0
  329. package/dist/skill-dirs-B7o83G5I.js.map +1 -0
  330. package/dist/source-C6JgdJdJ.js +1 -0
  331. package/dist/{source-zoRWRMql.js → source-DxgCge_u.js} +33 -18
  332. package/dist/source-DxgCge_u.js.map +1 -0
  333. package/dist/spans-B4MwgEyy.js +1 -0
  334. package/dist/spans-Dq3fo6HM.js +6 -0
  335. package/dist/spans-Dq3fo6HM.js.map +1 -0
  336. package/dist/spans-tf3biLpT.js +72 -0
  337. package/dist/spans-tf3biLpT.js.map +1 -0
  338. package/dist/src-B2psni1B.js +1 -0
  339. package/dist/{src-DiS13D_q.js → src-BI35cTig.js} +2 -2
  340. package/dist/src-BLePHsjw.js +2 -0
  341. package/dist/{src-Bj3WJs3t.js → src-CftydE9Q.js} +2 -2
  342. package/dist/{src-CDoAEO4u.js → src-ClMhIgi3.js} +2 -2
  343. package/dist/{src-CDoAEO4u.js.map → src-ClMhIgi3.js.map} +1 -1
  344. package/dist/{src-CUrcaZ91.js → src-CnOj4_n4.js} +1 -1
  345. package/dist/src-Cs3nmbrr.js +2 -0
  346. package/dist/{src-TMKittwH.js → src-D0GezGkJ.js} +2 -2
  347. package/dist/{src-OC89dAjO.js → src-D1Q6ElaF.js} +182 -36
  348. package/dist/src-D1Q6ElaF.js.map +1 -0
  349. package/dist/{src-CFBg2ecq.js → src-D_9qqp-Y.js} +2 -2
  350. package/dist/{src-DwcTn4JY.js → src-DhZuOcgq.js} +5 -5
  351. package/dist/{src-DwcTn4JY.js.map → src-DhZuOcgq.js.map} +1 -1
  352. package/dist/{src-CGBwMZgM.js → src-Dm2YIeXn.js} +8 -8
  353. package/dist/{src-CGBwMZgM.js.map → src-Dm2YIeXn.js.map} +1 -1
  354. package/dist/{src-CcbV6VpV.js → src-Dpzfe-vl.js} +3 -3
  355. package/dist/{src-c3ZQu7Uu.js → src-NYxoDS3b.js} +5 -5
  356. package/dist/{src-c3ZQu7Uu.js.map → src-NYxoDS3b.js.map} +1 -1
  357. package/dist/{src-C3OhA5pw.js → src-Y4AiZxLt.js} +5 -5
  358. package/dist/{src-C3OhA5pw.js.map → src-Y4AiZxLt.js.map} +1 -1
  359. package/dist/{src-BQWAxhKD.js → src-YpmRtJmP.js} +3 -3
  360. package/dist/{src-BQWAxhKD.js.map → src-YpmRtJmP.js.map} +1 -1
  361. package/dist/{src-Bb3k9Kn_.js → src-nIPDQ297.js} +66 -11
  362. package/dist/{src-Bb3k9Kn_.js.map → src-nIPDQ297.js.map} +1 -1
  363. package/dist/ssh-pool-B9bnSQIj.js +1 -0
  364. package/dist/{ssh-pool-DwboLN9O.js → ssh-pool-Gv8gAsgZ.js} +19 -16
  365. package/dist/ssh-pool-Gv8gAsgZ.js.map +1 -0
  366. package/dist/{start-BwMyDneA.js → start-YOYgbaNU.js} +45 -13
  367. package/dist/start-YOYgbaNU.js.map +1 -0
  368. package/dist/state-Bvsi2WgZ.js +1 -0
  369. package/dist/{state-CXTaT0t8.js → state-CJlYQtdl.js} +21 -38
  370. package/dist/state-CJlYQtdl.js.map +1 -0
  371. package/dist/store/local.d.ts +1 -1
  372. package/dist/store/s3.d.ts +1 -1
  373. package/dist/studio-share-4T6GYO7l.js +209 -0
  374. package/dist/studio-share-4T6GYO7l.js.map +1 -0
  375. package/dist/suite-fO-u8jlS.js +2 -0
  376. package/dist/{suite-C4XSg6Ls.js → suite-oH7uiCRq.js} +107 -106
  377. package/dist/suite-oH7uiCRq.js.map +1 -0
  378. package/dist/summary-CTMWfnOF.js +75 -0
  379. package/dist/summary-CTMWfnOF.js.map +1 -0
  380. package/dist/{sync-bnI6WWaL.js → sync-BNduS38K.js} +6 -6
  381. package/dist/{sync-bnI6WWaL.js.map → sync-BNduS38K.js.map} +1 -1
  382. package/dist/{tart-CSP5Q0qV.js → tart-B6Uz4S3I.js} +43 -13
  383. package/dist/tart-B6Uz4S3I.js.map +1 -0
  384. package/dist/{tart-DmVuVsmX.js → tart-DNrumWZu.js} +1 -1
  385. package/dist/{tart-CzViW3WZ.js → tart-Duqp33Zh.js} +2 -2
  386. package/dist/{tart-CzViW3WZ.js.map → tart-Duqp33Zh.js.map} +1 -1
  387. package/dist/{tart-bake-d9UvIfe3.js → tart-bake-B4QyNb7f.js} +1 -1
  388. package/dist/{tart-bake-BCKRfyKm.js → tart-bake-s99wBzxO.js} +9 -9
  389. package/dist/{tart-bake-BCKRfyKm.js.map → tart-bake-s99wBzxO.js.map} +1 -1
  390. package/dist/{tart-cli-CqSpUYGV.js → tart-cli-B0xVYBti.js} +6 -4
  391. package/dist/tart-cli-B0xVYBti.js.map +1 -0
  392. package/dist/{tart-cli-C4d5FXG1.js → tart-cli-DfteJCN9.js} +1 -1
  393. package/dist/{tart-hosts-Du7SZM4w.js → tart-hosts-B3Ze8aMp.js} +1 -1
  394. package/dist/{tart-hosts-G0EUDt07.js → tart-hosts-DfVUcZqA.js} +6 -6
  395. package/dist/{tart-hosts-G0EUDt07.js.map → tart-hosts-DfVUcZqA.js.map} +1 -1
  396. package/dist/{tart-images-De0-ib7R.js → tart-images-yzIyb0bQ.js} +8 -3
  397. package/dist/tart-images-yzIyb0bQ.js.map +1 -0
  398. package/dist/telemetry-bBhK3DSs.js +399 -0
  399. package/dist/telemetry-bBhK3DSs.js.map +1 -0
  400. package/dist/testing.d.ts +11 -2
  401. package/dist/testing.js +4 -4
  402. package/dist/{tool-class-B3ZFTSOw.js → tool-class-CB-m4PiF.js} +2 -2
  403. package/dist/{tool-class-B3ZFTSOw.js.map → tool-class-CB-m4PiF.js.map} +1 -1
  404. package/dist/{trace-CfppUzD9.js → trace-CJQLlAbB.js} +2 -2
  405. package/dist/{trace-CfppUzD9.js.map → trace-CJQLlAbB.js.map} +1 -1
  406. package/dist/{traits-Bj1PM5F7.js → traits-DPIffsN8.js} +2 -2
  407. package/dist/{traits-Bj1PM5F7.js.map → traits-DPIffsN8.js.map} +1 -1
  408. package/dist/{transcript-CW02A4s9.js → transcript-TWiaAuoL.js} +2 -2
  409. package/dist/{transcript-CW02A4s9.js.map → transcript-TWiaAuoL.js.map} +1 -1
  410. package/dist/{resources-DqdwihhT.js → turn-error-Dmv_Muj7.js} +15 -2
  411. package/dist/turn-error-Dmv_Muj7.js.map +1 -0
  412. package/dist/{types-D4WEnzhY.d.ts → types-BZaItiui.d.ts} +159 -13
  413. package/dist/types-Buut5yGe.js +29 -0
  414. package/dist/{types-CbcnJxCy.js.map → types-Buut5yGe.js.map} +1 -1
  415. package/dist/types-C48zuZbT.d.ts +156 -0
  416. package/dist/types-CYz4dqAd.d.ts +1366 -0
  417. package/dist/{ui-DfvPNmPP.js → ui-BIYUGm8D.js} +21 -17
  418. package/dist/ui-BIYUGm8D.js.map +1 -0
  419. package/dist/{vertex-BLxYasfw.js → vertex-DQQGhUn7.js} +2 -2
  420. package/dist/{vertex-BLxYasfw.js.map → vertex-DQQGhUn7.js.map} +1 -1
  421. package/dist/vertex-DtWe0Xf6.js +1 -0
  422. package/dist/view/index.d.ts +142 -809
  423. package/dist/view/index.js +17 -12
  424. package/dist/view/report/index.d.ts +217 -0
  425. package/dist/view/report/index.js +4 -0
  426. package/dist/views-DOYgIWRs.js +2 -0
  427. package/dist/{views-ChoSQxfF.js → views-DPsEJBBP.js} +6 -6
  428. package/dist/{views-ChoSQxfF.js.map → views-DPsEJBBP.js.map} +1 -1
  429. package/dist/{vm-container-DOtIi_rd.js → vm-container-rHUmgBRf.js} +13 -11
  430. package/dist/vm-container-rHUmgBRf.js.map +1 -0
  431. package/dist/{vm-images-CHutQVml.js → vm-images-BC2ZjZl5.js} +19 -21
  432. package/dist/vm-images-BC2ZjZl5.js.map +1 -0
  433. package/dist/wait-CixM6W0G.js +65 -0
  434. package/dist/wait-CixM6W0G.js.map +1 -0
  435. package/dist/{worker-entry-CKBfEt0h.js → worker-entry-CN1eVLzz.js} +7 -7
  436. package/dist/{worker-entry-CKBfEt0h.js.map → worker-entry-CN1eVLzz.js.map} +1 -1
  437. package/dist/{workspace-Cip2dVwU.js → workspace-BS-FUp5I.js} +9 -9
  438. package/dist/{workspace-Cip2dVwU.js.map → workspace-BS-FUp5I.js.map} +1 -1
  439. package/dist/workspace-D8HqjfsQ.js +1 -0
  440. package/package.json +38 -1
  441. package/skills/reeval/references/cli.md +1 -1
  442. package/skills/reeval/references/config.md +1 -1
  443. package/skills/reeval/references/environments.md +1 -1
  444. package/dist/agents-CEapsHV-.js.map +0 -1
  445. package/dist/analysis-BnG_1x_O.js.map +0 -1
  446. package/dist/app-files-Dsr3U1Q7.js +0 -10
  447. package/dist/app-files-Dsr3U1Q7.js.map +0 -1
  448. package/dist/assemble-DUtbByvj.js +0 -1
  449. package/dist/assemble-Dw1bF13I.js.map +0 -1
  450. package/dist/attach-DvYiMu6w.js.map +0 -1
  451. package/dist/attempt-host-DgeJFSZw.js.map +0 -1
  452. package/dist/boat-BBPOc_bs.js +0 -1
  453. package/dist/boat-CUNgSc75.js.map +0 -1
  454. package/dist/boat-images-BuFigiEb.js.map +0 -1
  455. package/dist/bundle-yEgI3RsI.js.map +0 -1
  456. package/dist/capture-DxxB3d7p.js +0 -1
  457. package/dist/cases-B7QkvMGE.js.map +0 -1
  458. package/dist/cases-CA7gxxdr.js +0 -1
  459. package/dist/cli-Z7aw8k25.js.map +0 -1
  460. package/dist/cli-spec-3zvLC9YW.js.map +0 -1
  461. package/dist/client-kUwtq-vt.js.map +0 -1
  462. package/dist/cloud-DuHcNQVV.js.map +0 -1
  463. package/dist/cloud-run-C08P_Uap.js +0 -1
  464. package/dist/cloud-run-DqMkSGYB.js.map +0 -1
  465. package/dist/cloud-secrets-Bak83tik.js.map +0 -1
  466. package/dist/command-Bs6Knfm4.js +0 -298
  467. package/dist/command-Bs6Knfm4.js.map +0 -1
  468. package/dist/command-CeygdoVe.js.map +0 -1
  469. package/dist/command-CoHN1ibe.js.map +0 -1
  470. package/dist/command-CtmSJzPC.js.map +0 -1
  471. package/dist/command-efTGfUYb.js.map +0 -1
  472. package/dist/commands-B_cAs8nG.js +0 -1
  473. package/dist/commands-DR-NolV2.js.map +0 -1
  474. package/dist/config-Bn2OhCN_.js.map +0 -1
  475. package/dist/context-Ba7PNMMe.js +0 -120
  476. package/dist/context-Ba7PNMMe.js.map +0 -1
  477. package/dist/daytona-CWs9j9mt.js.map +0 -1
  478. package/dist/daytona-image-BjVpi3KS.js +0 -1
  479. package/dist/db-Bjq6qr6c.js +0 -2
  480. package/dist/db-D6yQib7M.js.map +0 -1
  481. package/dist/debug-CF4p4Lp_.js +0 -1
  482. package/dist/debug-CIXzB4wK.js.map +0 -1
  483. package/dist/environment-BY8lVFFK.js.map +0 -1
  484. package/dist/evaluate-B8I9AKgw.js.map +0 -1
  485. package/dist/events-D97ZzAeq.js.map +0 -1
  486. package/dist/files-5HsFIpxw.js +0 -1
  487. package/dist/files-BNg3wgw_.js +0 -1
  488. package/dist/files-BesexM88.js.map +0 -1
  489. package/dist/files-DNgQwgA0.js +0 -1
  490. package/dist/files-DvDUSrWL.js +0 -1
  491. package/dist/files-TwSVeWwM.js.map +0 -1
  492. package/dist/gate-CgYdaprG.js +0 -49
  493. package/dist/gate-CgYdaprG.js.map +0 -1
  494. package/dist/harness-FcTyyPco.js.map +0 -1
  495. package/dist/ids-DqEvrBFZ.js.map +0 -1
  496. package/dist/init-CS5kEaVg.js +0 -1
  497. package/dist/ink-list-CLq6YVB8.js.map +0 -1
  498. package/dist/inputs-load-D4IsEjq3.js +0 -1
  499. package/dist/issue-DmSorsTF.js +0 -1
  500. package/dist/list-ClDb8qiX.js.map +0 -1
  501. package/dist/live-C7iGaDCA.js +0 -120
  502. package/dist/live-C7iGaDCA.js.map +0 -1
  503. package/dist/live-C7kN5v2P.js +0 -1
  504. package/dist/live-QFILMmql.js.map +0 -1
  505. package/dist/machine-service-xFhakTs5.js.map +0 -1
  506. package/dist/model-Colu5mEp.js +0 -1
  507. package/dist/model-mTaaEkeQ.js.map +0 -1
  508. package/dist/options-CmRy82gl.js +0 -1
  509. package/dist/pairs-BGb134Vu.js +0 -230
  510. package/dist/pairs-BGb134Vu.js.map +0 -1
  511. package/dist/patch-C7yBztZj.js.map +0 -1
  512. package/dist/paths-Dwmi7L4X.js +0 -75
  513. package/dist/paths-Dwmi7L4X.js.map +0 -1
  514. package/dist/preflight-xa8_eGOW.js +0 -1
  515. package/dist/prices-C_mbZa5a.js +0 -1
  516. package/dist/process-RPlEhpHz.js.map +0 -1
  517. package/dist/provider-DHT20Gl8.js +0 -1
  518. package/dist/query-BXV6pRyI.js +0 -1
  519. package/dist/questions-DnqkJga0.js.map +0 -1
  520. package/dist/record-2liAelrB.js.map +0 -1
  521. package/dist/redact-DMueYS-v.js.map +0 -1
  522. package/dist/remarks-D5qOZzcX.js.map +0 -1
  523. package/dist/resolve-B60w6DcB.js.map +0 -1
  524. package/dist/resolve-CCLiJX4i.js +0 -1
  525. package/dist/resolve-t9sOkGGS.js.map +0 -1
  526. package/dist/resources-DqdwihhT.js.map +0 -1
  527. package/dist/resume-4Q9q8AcE.js.map +0 -1
  528. package/dist/runtime-CIsljVBy.js +0 -457
  529. package/dist/runtime-CIsljVBy.js.map +0 -1
  530. package/dist/server-CeYUmuuu.js +0 -1
  531. package/dist/server-fSNuT3Z4.js +0 -284
  532. package/dist/server-fSNuT3Z4.js.map +0 -1
  533. package/dist/services-BG8xuU82.js.map +0 -1
  534. package/dist/shared-DtZwIAfy.js.map +0 -1
  535. package/dist/slug-IHh3f770.js.map +0 -1
  536. package/dist/source-n7nUVm9p.js +0 -1
  537. package/dist/source-zoRWRMql.js.map +0 -1
  538. package/dist/src-9V3C6A7C.js +0 -2
  539. package/dist/src-CB5cZhaN.js +0 -2
  540. package/dist/src-DB0vml0O.js +0 -1
  541. package/dist/src-OC89dAjO.js.map +0 -1
  542. package/dist/ssh-pool-ChpT-fDU.js +0 -1
  543. package/dist/ssh-pool-DwboLN9O.js.map +0 -1
  544. package/dist/start-BwMyDneA.js.map +0 -1
  545. package/dist/state-CXTaT0t8.js.map +0 -1
  546. package/dist/state-CltRWlCu.js +0 -1
  547. package/dist/suite-C4XSg6Ls.js.map +0 -1
  548. package/dist/suite-CIsyPqPA.js +0 -2
  549. package/dist/summary-DeeQJAR5.js +0 -169
  550. package/dist/summary-DeeQJAR5.js.map +0 -1
  551. package/dist/tart-CSP5Q0qV.js.map +0 -1
  552. package/dist/tart-cli-CqSpUYGV.js.map +0 -1
  553. package/dist/tart-images-De0-ib7R.js.map +0 -1
  554. package/dist/types-CbcnJxCy.js +0 -15
  555. package/dist/ui-DfvPNmPP.js.map +0 -1
  556. package/dist/vertex-DZDUpp0W.js +0 -1
  557. package/dist/views-DU0PM0UW.js +0 -2
  558. package/dist/vm-container-DOtIi_rd.js.map +0 -1
  559. package/dist/vm-images-CHutQVml.js.map +0 -1
  560. package/dist/workspace-BKcNJAPb.js +0 -1
@@ -0,0 +1,2058 @@
1
+ import { r as parseDuration } from "./duration-Bz2H7pXb.js";
2
+ import { o as totalTokens } from "./usage-CwIRPPZ9.js";
3
+ import { r as runCost } from "./cost-DQ0ayy6h.js";
4
+ import { A as runtimeLabel, S as languageLabel, _ as harnessLabel, i as attemptAnswerRows, x as isUngatedMiss } from "./model-xemZQ7DK.js";
5
+ import { I as filesChanged, L as formatCount, N as effortSample, r as caseName, v as contrastsOf, z as formatSeconds } from "./cases-C6vnV-gn.js";
6
+ import { a as liftInterval, r as containsZero } from "./stats-C18W4xPX.js";
7
+ import { d as labeledJudgments } from "./judgments-CVCUWT-g.js";
8
+ import { r as runResourcesLine, t as turnErrorKind } from "./turn-error-Dmv_Muj7.js";
9
+ import { l as messageLine, o as factSegment } from "./suite-oH7uiCRq.js";
10
+ import { A as tokensText, D as rateOf, E as ptsInterval, O as spreadOf, T as ptsDelta, _ as countFigure, b as durationFigure, f as attemptAnchor, g as selectedEvals, h as idOf, i as PASS_RATE_OUTCOME_LABEL, j as usdFigure, k as tokensFigure, l as measurePhrase, m as attemptIds, u as passRateOutcomeFromMatchers, v as countOf, w as pctDelta, y as diffLinesText } from "./labels-Dtyf5-pd.js";
11
+ // src/view/sha256.ts
12
+ const K = new Uint32Array([
13
+ 1116352408,
14
+ 1899447441,
15
+ 3049323471,
16
+ 3921009573,
17
+ 961987163,
18
+ 1508970993,
19
+ 2453635748,
20
+ 2870763221,
21
+ 3624381080,
22
+ 310598401,
23
+ 607225278,
24
+ 1426881987,
25
+ 1925078388,
26
+ 2162078206,
27
+ 2614888103,
28
+ 3248222580,
29
+ 3835390401,
30
+ 4022224774,
31
+ 264347078,
32
+ 604807628,
33
+ 770255983,
34
+ 1249150122,
35
+ 1555081692,
36
+ 1996064986,
37
+ 2554220882,
38
+ 2821834349,
39
+ 2952996808,
40
+ 3210313671,
41
+ 3336571891,
42
+ 3584528711,
43
+ 113926993,
44
+ 338241895,
45
+ 666307205,
46
+ 773529912,
47
+ 1294757372,
48
+ 1396182291,
49
+ 1695183700,
50
+ 1986661051,
51
+ 2177026350,
52
+ 2456956037,
53
+ 2730485921,
54
+ 2820302411,
55
+ 3259730800,
56
+ 3345764771,
57
+ 3516065817,
58
+ 3600352804,
59
+ 4094571909,
60
+ 275423344,
61
+ 430227734,
62
+ 506948616,
63
+ 659060556,
64
+ 883997877,
65
+ 958139571,
66
+ 1322822218,
67
+ 1537002063,
68
+ 1747873779,
69
+ 1955562222,
70
+ 2024104815,
71
+ 2227730452,
72
+ 2361852424,
73
+ 2428436474,
74
+ 2756734187,
75
+ 3204031479,
76
+ 3329325298
77
+ ]);
78
+ const rotr = (x, n) => x >>> n | x << 32 - n;
79
+ /** The lowercase hex SHA-256 of `input`; a string is hashed as UTF-8. */
80
+ function sha256Hex(input) {
81
+ const data = typeof input === "string" ? new TextEncoder().encode(input) : input;
82
+ const bytes = new Uint8Array(Math.ceil((data.length + 9) / 64) * 64);
83
+ bytes.set(data);
84
+ bytes[data.length] = 128;
85
+ const view = new DataView(bytes.buffer);
86
+ const bits = data.length * 8;
87
+ view.setUint32(bytes.length - 8, Math.floor(bits / 2 ** 32));
88
+ view.setUint32(bytes.length - 4, bits >>> 0);
89
+ const h = [
90
+ 1779033703,
91
+ 3144134277,
92
+ 1013904242,
93
+ 2773480762,
94
+ 1359893119,
95
+ 2600822924,
96
+ 528734635,
97
+ 1541459225
98
+ ];
99
+ const w = /* @__PURE__ */ new Uint32Array(64);
100
+ const word = (i) => w[i] ?? 0;
101
+ for (let off = 0; off < bytes.length; off += 64) {
102
+ for (let i = 0; i < 16; i++) w[i] = view.getUint32(off + i * 4);
103
+ for (let i = 16; i < 64; i++) {
104
+ const x = word(i - 15);
105
+ const y = word(i - 2);
106
+ const s0 = rotr(x, 7) ^ rotr(x, 18) ^ x >>> 3;
107
+ const s1 = rotr(y, 17) ^ rotr(y, 19) ^ y >>> 10;
108
+ w[i] = word(i - 16) + s0 + word(i - 7) + s1;
109
+ }
110
+ let [a = 0, b = 0, c = 0, d = 0, e = 0, f = 0, g = 0, k = 0] = h;
111
+ for (let i = 0; i < 64; i++) {
112
+ const ch = e & f ^ ~e & g;
113
+ const t1 = k + (rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25)) + ch + (K[i] ?? 0) + word(i) >>> 0;
114
+ const t2 = (rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22)) + (a & b ^ a & c ^ b & c) >>> 0;
115
+ k = g;
116
+ g = f;
117
+ f = e;
118
+ e = d + t1 >>> 0;
119
+ d = c;
120
+ c = b;
121
+ b = a;
122
+ a = t1 + t2 >>> 0;
123
+ }
124
+ const next = [
125
+ a,
126
+ b,
127
+ c,
128
+ d,
129
+ e,
130
+ f,
131
+ g,
132
+ k
133
+ ];
134
+ for (let i = 0; i < 8; i++) h[i] = (h[i] ?? 0) + (next[i] ?? 0) >>> 0;
135
+ }
136
+ let hex = "";
137
+ for (const v of h) hex += v.toString(16).padStart(8, "0");
138
+ return hex;
139
+ }
140
+ // src/view/scope.ts
141
+ /** JSON with every object's keys sorted and `undefined` members left out, so equal data hashes equal. */
142
+ function canonical(value) {
143
+ if (Array.isArray(value)) return `[${value.map((v) => canonical(v ?? null)).join(",")}]`;
144
+ if (value && typeof value === "object") return `{${Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`;
145
+ return JSON.stringify(value);
146
+ }
147
+ /** A duration as milliseconds, so `'10m'` and `600000` hash equal; a value it can't read, as written. */
148
+ function durationMs(value) {
149
+ if (value === void 0) return void 0;
150
+ try {
151
+ return parseDuration(value);
152
+ } catch {
153
+ return value;
154
+ }
155
+ }
156
+ /** In the hashed input; bumped by any change to what `configKey` hashes or how. */
157
+ const CONFIG_KEY_VERSION = 1;
158
+ const sorted = (list) => list ? [...list].sort() : void 0;
159
+ function profileKey(profile) {
160
+ const { defaults, limit } = profile;
161
+ return {
162
+ name: profile.name,
163
+ defaults: {
164
+ ...defaults,
165
+ harness: sorted(defaults.harness),
166
+ env: sorted(defaults.env),
167
+ timeout: durationMs(defaults.timeout)
168
+ },
169
+ limit: {
170
+ ...limit,
171
+ harness: sorted(limit.harness),
172
+ traits: limit.traits ? Object.fromEntries(Object.entries(limit.traits).map(([k, v]) => [k, typeof v === "string" ? v : sorted(v)])) : void 0
173
+ }
174
+ };
175
+ }
176
+ /** The run's scope, from its record alone; a record without `invocation` reads as one run with none. */
177
+ function runScope(record) {
178
+ const invocation = record.invocation;
179
+ const filters = invocation?.filters ?? [];
180
+ const tags = invocation?.tags ?? [];
181
+ const traits = invocation?.traits ?? [];
182
+ const { config } = record;
183
+ const key = {
184
+ v: CONFIG_KEY_VERSION,
185
+ profile: record.profile ? profileKey(record.profile) : null,
186
+ invocation: {
187
+ runs: invocation?.runs,
188
+ filters: sorted(filters),
189
+ tags: sorted(tags),
190
+ traits: sorted(traits)
191
+ },
192
+ config: {
193
+ environment: config.environment,
194
+ runs: config.runs,
195
+ bestOf: config.bestOf,
196
+ judgeModel: config.judgeModel,
197
+ threshold: config.threshold,
198
+ timeoutMs: config.timeoutMs
199
+ },
200
+ fingerprintScope: sorted(record.stack.fingerprintScope)
201
+ };
202
+ return {
203
+ profile: record.profile?.name ?? null,
204
+ subset: filters.length > 0 || tags.length > 0 || traits.length > 0 || invocation?.runs !== void 0,
205
+ configKey: sha256Hex(canonical(key))
206
+ };
207
+ }
208
+ // src/view/report/links.ts
209
+ /** Prefix every relative artifact link: `linkBase` as a folder, `./` by default. */
210
+ function withBase(linkBase, path) {
211
+ const base = !linkBase ? linkBase ?? "./" : linkBase.endsWith("/") ? linkBase : `${linkBase}/`;
212
+ if (path.startsWith("#")) return path;
213
+ if (path.startsWith("../")) return path;
214
+ return `${base}${path.replace(/^\.\//, "")}`;
215
+ }
216
+ /** The record's link: `href`, or `../<runId>.json`, the record beside the run's folder. */
217
+ function recordHref(runId, href) {
218
+ return href ?? `../${runId}.json`;
219
+ }
220
+ function kept(ref) {
221
+ return ref?.path ?? null;
222
+ }
223
+ /**
224
+ * Citation target for an attempt: transcript, else diff, else first capture, else its anchor in
225
+ * the attempt index (`attemptAnchor`).
226
+ * `FileRef.path` values are already relative to the run directory.
227
+ * When `used` is set, a path already claimed by another attempt falls back to the anchor
228
+ * so baseline and variant cites stay distinct.
229
+ */
230
+ function attemptHref(_evalEntry, attempt, linkBase, used) {
231
+ const files = attempt.files;
232
+ const transcript = kept(files.transcript);
233
+ const preferred = transcript ? withBase(linkBase, transcript) : (() => {
234
+ const diff = kept(files.diff);
235
+ if (diff) return withBase(linkBase, diff);
236
+ const capture = files.artifacts?.[0];
237
+ if (capture) return withBase(linkBase, capture.path);
238
+ return attemptAnchor(attempt.key);
239
+ })();
240
+ if (!used) return preferred;
241
+ if (!used.has(preferred) || preferred.startsWith("#")) {
242
+ used.add(preferred);
243
+ return preferred;
244
+ }
245
+ const fallback = attemptAnchor(attempt.key);
246
+ used.add(fallback);
247
+ return fallback;
248
+ }
249
+ function attemptRef(ids, evalEntry, attempt, linkBase, used, hrefByKey) {
250
+ const id = idOf(ids, attempt.key);
251
+ if (!id) return null;
252
+ const assigned = hrefByKey?.get(attempt.key);
253
+ if (assigned) return {
254
+ id,
255
+ key: attempt.key,
256
+ href: assigned
257
+ };
258
+ return {
259
+ id,
260
+ key: attempt.key,
261
+ href: attemptHref(evalEntry, attempt, linkBase, used)
262
+ };
263
+ }
264
+ /** Capture PNG link, or null when not kept (caller prints `name (not kept)`). */
265
+ function captureHref(path, linkBase) {
266
+ if (!path) return null;
267
+ return withBase(linkBase, path);
268
+ }
269
+ /** Question input JSON link; omitted when not kept. Paths are run-relative. */
270
+ function questionHref(path, linkBase) {
271
+ if (!path) return null;
272
+ return withBase(linkBase, path);
273
+ }
274
+ /** Worktree patch link, or the "not saved" note when absent. */
275
+ function worktreePatchLink(patch, note, linkBase) {
276
+ if (patch) return {
277
+ href: withBase(linkBase, patch.path),
278
+ text: "worktree.patch"
279
+ };
280
+ return {
281
+ href: null,
282
+ text: `not saved: ${note ?? "unknown"}`
283
+ };
284
+ }
285
+ /** GFM heading anchor: lowercase, punctuation stripped, spaces to hyphens. */
286
+ function gfmAnchor(heading) {
287
+ return heading.toLowerCase().replace(/[^\p{L}\p{N}\s-]/gu, "").trim().replace(/\s+/g, "-");
288
+ }
289
+ function evalAnchorHref(anchor) {
290
+ return `#${anchor}`;
291
+ }
292
+ // src/view/report/policy.ts
293
+ /**
294
+ * Report layout and delta defaults. Flip one field to change every surface that
295
+ * renders a `RunReport`; the shape of `RunReport` itself does not change.
296
+ */
297
+ const REPORT_POLICY = {
298
+ /**
299
+ * Measure vs baseline: raw median, then a signed integer `%` delta in
300
+ * parentheses. Absolute value below 2 is `±0%`. Rates take `pts` instead.
301
+ */
302
+ deltas: "pct",
303
+ /**
304
+ * When baseline and the best variant both pass every applicable assertion,
305
+ * the headline is `tied-ceiling` on the largest decisive measure delta
306
+ * (absolute value at least 10%), not a `±0 pts` lift.
307
+ */
308
+ tiedCeiling: true,
309
+ /**
310
+ * Two or more variants: one N-arm summary table, then a harness-by-arm effort
311
+ * pivot, then pairwise contrasts.
312
+ */
313
+ variantLayout: "n-arm"
314
+ };
315
+ // src/view/report/provenance.ts
316
+ const short = (hash) => hash.replace(/^sha256:/, "").slice(0, 12);
317
+ /**
318
+ * The state a run executed in, as label/value pairs. Pure and browser-safe.
319
+ * The `Context` row is split out into `ProvenanceState.context` so Markdown can
320
+ * put long file lists under `<details>`.
321
+ */
322
+ function stateRows(run) {
323
+ const { git, stack: env } = run;
324
+ const rows = [];
325
+ rows.push({
326
+ field: "Executed at",
327
+ value: git ? `${git.sha.slice(0, 12)}${git.branch ? ` on ${git.branch}` : ""}${git.dirty ? ", dirty" : ", clean"} (the commit that adds this record is later)` : "not a git repository"
328
+ });
329
+ if (git?.dirty) {
330
+ const patch = worktreePatchLink(git.worktreePatch, git.worktreePatchNote);
331
+ rows.push({
332
+ field: "Uncommitted",
333
+ value: git.worktreePatch ? `worktree.patch, ${git.worktreePatch.bytes} bytes, sha256 ${short(git.worktreePatch.sha256)}` : `${patch.text}${git.diffHash ? `; sha256 ${short(git.diffHash)}` : ""}`
334
+ });
335
+ }
336
+ rows.push({
337
+ field: "Lockfiles",
338
+ value: run.dependencies.lockfiles.map((l) => `${l.path} ${short(l.sha256)}`).join(", ") || "none found"
339
+ });
340
+ rows.push({
341
+ field: "Images",
342
+ value: env.images.length ? env.images.map((i) => `${i.ref} · id ${short(i.id)} · ${i.repoDigests[0] ?? "no registry digest (built locally)"} · CLI ${i.agentCli ?? "n/a"}${i.dockerfile ? ` · dockerfile ${i.dockerfile.path} ${short(i.dockerfile.sha256)}` : ""}${i.install?.length ? ` · install ${i.install.length} on ${i.base}` : ""}`).join("; ") : "none (fake sandbox)"
343
+ });
344
+ rows.push({
345
+ field: "Harnesses",
346
+ value: env.harnesses.map(harnessLabel).join(", ") || "none"
347
+ });
348
+ const resources = runResourcesLine(run.evals.flatMap((e) => e.attempts.map((a) => a.resources)));
349
+ if (resources) rows.push({
350
+ field: "Resources",
351
+ value: resources
352
+ });
353
+ rows.push({
354
+ field: "Environments",
355
+ value: run.environments.map((e) => `${e.name} ${short(e.digest)}${e.skills.length ? ` · skills ${e.skills.map((s) => `${s.name}${s.commit ? `@${s.commit.slice(0, 7)}` : ""}`).join(", ")}` : ""}${e.mcp.length ? ` · mcp ${e.mcp.join(", ")}` : ""}${e.secrets.length ? ` · secrets ${e.secrets.join(", ")}` : ""}`).join("; ") || "none"
356
+ });
357
+ rows.push({
358
+ field: "Judge",
359
+ value: `${env.judge.model ?? "none by default"}${env.judge.resolvedModels.length ? ` (served ${env.judge.resolvedModels.join(", ")})` : ""} · threshold ${env.judge.threshold} · version ${env.judge.version}`
360
+ });
361
+ rows.push({
362
+ field: "Packages",
363
+ value: Object.entries(env.packages).map(([name, v]) => `${name} ${v}`).join(" · ")
364
+ });
365
+ const language = languageLabel(env);
366
+ rows.push({
367
+ field: "reeval",
368
+ value: `${env.reeval} · ${runtimeLabel(env)}${language ? ` · ${language}` : ""} · ${env.os}/${env.arch}`
369
+ });
370
+ rows.push({
371
+ field: "Runner",
372
+ value: run.runner.kind === "ci" ? `CI (${run.runner.provider})${run.runner.runUrl ? ` ${run.runner.runUrl}` : ""}` : `local · ${run.author}`
373
+ });
374
+ return rows;
375
+ }
376
+ /**
377
+ * One-line warnings for states that stop an exact rerun. Derived when shown,
378
+ * never stored. Pure and browser-safe.
379
+ */
380
+ function warnings(run) {
381
+ const out = [];
382
+ for (const w of run.plan.sandboxWarnings) out.push(w.message);
383
+ for (const image of run.stack.images.filter((i) => i.repoDigests.length === 0)) out.push(`Image ${image.ref} isn't pushed, so an exact rerun needs this machine's image (id ${image.id.replace(/^sha256:/, "").slice(0, 12)}).`);
384
+ if (!run.git) out.push("Not a git repository, so the code under test isn't pinned.");
385
+ else if (run.git.dirty && !run.git.worktreePatch) out.push(`Uncommitted changes weren't saved (${run.git.worktreePatchNote ?? "no patch"}), so this exact state can't be recreated.`);
386
+ return out;
387
+ }
388
+ /** Provenance block for a run report. */
389
+ function provenanceState(run, recordLink) {
390
+ const contextFiles = run.stack.context.map((c) => ({
391
+ path: c.path,
392
+ sha256: c.sha256
393
+ }));
394
+ return {
395
+ rows: stateRows(run),
396
+ context: {
397
+ count: contextFiles.length,
398
+ files: contextFiles
399
+ },
400
+ artifacts: run.artifacts,
401
+ record: recordHref(run.runId, recordLink),
402
+ warnings: warnings(run)
403
+ };
404
+ }
405
+ // src/view/report/build.ts
406
+ const MEASURE_KEYS = [
407
+ "tokens",
408
+ "toolCalls",
409
+ "turns",
410
+ "costUsd",
411
+ "agentMs",
412
+ "filesChanged",
413
+ "linesAdded",
414
+ "linesRemoved"
415
+ ];
416
+ const LOCKFILE_NAMES = /* @__PURE__ */ new Set([
417
+ "pnpm-lock.yaml",
418
+ "package-lock.json",
419
+ "npm-shrinkwrap.json",
420
+ "yarn.lock",
421
+ "bun.lock",
422
+ "bun.lockb",
423
+ "pubspec.lock",
424
+ "Cargo.lock",
425
+ "go.sum",
426
+ "poetry.lock",
427
+ "uv.lock",
428
+ "Pipfile.lock",
429
+ "Gemfile.lock",
430
+ "composer.lock",
431
+ "Podfile.lock",
432
+ "gradle.lockfile"
433
+ ]);
434
+ const isLockfilePath = (path) => LOCKFILE_NAMES.has(path.split("/").pop() ?? path);
435
+ const countedOutcome = (o) => o === "pass" || o === "fail";
436
+ const cellOf = (e, key) => e.cells.find((c) => c.key === key);
437
+ function assignAttemptHrefs(ctx) {
438
+ for (const e of ctx.evals) for (const a of e.attempts) {
439
+ if (a.seq === null) continue;
440
+ if (!ctx.ids.get(a.key)) continue;
441
+ const href = attemptHref(e, a, ctx.linkBase, ctx.hrefs);
442
+ ctx.hrefByKey.set(a.key, href);
443
+ }
444
+ }
445
+ const repoBasename = (remote) => {
446
+ if (!remote) return null;
447
+ const cleaned = remote.replace(/\.git$/, "");
448
+ const slash = cleaned.lastIndexOf("/");
449
+ const colon = cleaned.lastIndexOf(":");
450
+ const at = Math.max(slash, colon);
451
+ return at === -1 ? cleaned : cleaned.slice(at + 1) || null;
452
+ };
453
+ function emptyMeasures() {
454
+ return {
455
+ tokens: null,
456
+ toolCalls: null,
457
+ turns: null,
458
+ costUsd: null,
459
+ agentMs: null,
460
+ filesChanged: null,
461
+ linesAdded: null,
462
+ linesRemoved: null
463
+ };
464
+ }
465
+ function emptyMeasureVsBaseline() {
466
+ const blank = () => ({
467
+ value: null,
468
+ delta: null,
469
+ lower: null
470
+ });
471
+ return {
472
+ tokens: blank(),
473
+ toolCalls: blank(),
474
+ turns: blank(),
475
+ costUsd: blank(),
476
+ agentMs: blank(),
477
+ filesChanged: blank(),
478
+ linesAdded: blank(),
479
+ linesRemoved: blank()
480
+ };
481
+ }
482
+ /** Diff line totals for an attempt; lockfiles excluded. Null counts when not recorded. */
483
+ function attemptDiffStats(a) {
484
+ const files = filesChanged(a.changedFiles);
485
+ if (a.changedFiles.filter((f) => {
486
+ const row = f;
487
+ return !isLockfilePath(f.path) && !row.lockfile && (row.added !== void 0 || row.removed !== void 0);
488
+ }).length === 0) return {
489
+ files,
490
+ added: null,
491
+ removed: null
492
+ };
493
+ let added = 0;
494
+ let removed = 0;
495
+ for (const f of a.changedFiles) {
496
+ const row = f;
497
+ if (isLockfilePath(f.path) || row.lockfile) continue;
498
+ added += row.added ?? 0;
499
+ removed += row.removed ?? 0;
500
+ }
501
+ return {
502
+ files,
503
+ added,
504
+ removed
505
+ };
506
+ }
507
+ function measureValues(attempts) {
508
+ const out = {
509
+ tokens: [],
510
+ toolCalls: [],
511
+ turns: [],
512
+ costUsd: [],
513
+ agentMs: [],
514
+ filesChanged: [],
515
+ linesAdded: [],
516
+ linesRemoved: []
517
+ };
518
+ for (const a of attempts) {
519
+ if (!countedOutcome(a.outcome)) continue;
520
+ const sample = effortSample(a);
521
+ if (sample.tokens !== void 0) out.tokens.push(sample.tokens);
522
+ out.toolCalls.push(sample.tools);
523
+ out.turns.push(a.turns.length);
524
+ if (sample.usd !== void 0) out.costUsd.push(sample.usd);
525
+ out.agentMs.push(sample.agentMs);
526
+ out.filesChanged.push(sample.filesChanged);
527
+ const diff = attemptDiffStats(a);
528
+ if (diff.added !== null) out.linesAdded.push(diff.added);
529
+ if (diff.removed !== null) out.linesRemoved.push(diff.removed);
530
+ }
531
+ return out;
532
+ }
533
+ function measuresOf(attempts) {
534
+ const values = measureValues(attempts);
535
+ const fmt = {
536
+ tokens: tokensText,
537
+ toolCalls: formatCount,
538
+ turns: formatCount,
539
+ costUsd: (n) => usdFigure(n).text,
540
+ agentMs: formatSeconds,
541
+ filesChanged: formatCount,
542
+ linesAdded: formatCount,
543
+ linesRemoved: formatCount
544
+ };
545
+ const cells = emptyMeasures();
546
+ for (const key of MEASURE_KEYS) cells[key] = spreadOf(values[key], fmt[key]);
547
+ return cells;
548
+ }
549
+ function measureVs(value, baseline, lower) {
550
+ if (!value) return {
551
+ value: null,
552
+ delta: null,
553
+ lower: null
554
+ };
555
+ return {
556
+ value,
557
+ delta: baseline && baseline.median.value !== null && value.median.value !== null ? pctDelta(value.median.value, baseline.median.value) : null,
558
+ lower: lower ? countOf(lower.k, lower.n) : null
559
+ };
560
+ }
561
+ function deriveTally(evals) {
562
+ let passed = 0;
563
+ let failed = 0;
564
+ let ungatedPassed = 0;
565
+ let ungatedFailed = 0;
566
+ let infraError = 0;
567
+ let reclassified = 0;
568
+ let skipped = 0;
569
+ for (const e of evals) for (const a of e.attempts) {
570
+ const ungated = cellOf(e, a.cell)?.gate === "none";
571
+ if (a.outcome === "skipped") {
572
+ skipped++;
573
+ continue;
574
+ }
575
+ if (a.outcome === "infra_error") {
576
+ infraError++;
577
+ if (wasReclassified(a)) reclassified++;
578
+ continue;
579
+ }
580
+ if (a.outcome === "pass") {
581
+ if (ungated) ungatedPassed++;
582
+ else passed++;
583
+ continue;
584
+ }
585
+ if (a.outcome === "fail") {
586
+ if (ungated || isUngatedMiss(e, a)) ungatedFailed++;
587
+ else failed++;
588
+ }
589
+ }
590
+ return {
591
+ counted: passed + failed,
592
+ passed,
593
+ failed,
594
+ ungated: {
595
+ passed: ungatedPassed,
596
+ failed: ungatedFailed
597
+ },
598
+ infraError,
599
+ reclassified,
600
+ skipped
601
+ };
602
+ }
603
+ /** Older records: a turn error on a pass/fail attempt is treated as reclassified. */
604
+ function wasReclassified(a) {
605
+ if (a.turnStatusRead === true) return false;
606
+ if (a.error && "turn" in a.error && a.error.turn) return true;
607
+ return a.turns.some((t) => t.status === "error") && countedOutcome(a.outcome);
608
+ }
609
+ function turnErrorMessage(a) {
610
+ const turn = a.turns.find((t) => t.status === "error" && t.error);
611
+ if (!turn?.error) return null;
612
+ return `${turnErrorKind(turn.error)}: ${messageLine(turn.error)}`;
613
+ }
614
+ function runTitle(record, evals, title) {
615
+ if (title) return title;
616
+ if (evals.length === 1) return evals[0].name;
617
+ const groups = evals.map((e) => e.group[0]).filter(Boolean);
618
+ if (groups.length === evals.length && new Set(groups).size === 1) return groups[0];
619
+ return `Run ${record.runId}`;
620
+ }
621
+ function runsPerCell(evals) {
622
+ const runs = new Set(evals.flatMap((e) => e.cells.map((c) => c.runs)));
623
+ return runs.size === 1 ? [...runs][0] ?? null : null;
624
+ }
625
+ function buildHeader(ctx) {
626
+ const { record, evals } = ctx;
627
+ const tally = deriveTally(evals);
628
+ const usageAttempts = evals.flatMap((e) => e.attempts).filter((a) => a.usage);
629
+ let total = null;
630
+ let cacheRead = null;
631
+ if (usageAttempts.length) {
632
+ total = 0;
633
+ cacheRead = 0;
634
+ for (const a of usageAttempts) {
635
+ if (!a.usage) continue;
636
+ total += totalTokens(a.usage);
637
+ cacheRead += a.usage.cacheRead;
638
+ }
639
+ }
640
+ const cost = runCost(record.cost);
641
+ const agent = record.cost?.agentUsd;
642
+ const judge = record.cost?.judgeUsd;
643
+ let basis = null;
644
+ if (agent || judge) {
645
+ const bases = [agent?.basis, judge?.basis].filter(Boolean);
646
+ basis = bases.every((b) => b === "reported") ? "reported" : bases.every((b) => b === "estimated") ? "estimated" : "mixed";
647
+ }
648
+ return {
649
+ title: runTitle(record, evals),
650
+ status: record.status,
651
+ startedAt: record.startedAt,
652
+ author: record.author,
653
+ reeval: record.stack.reeval,
654
+ profile: record.profile?.name ?? null,
655
+ scope: runScope(record),
656
+ runsPerCell: runsPerCell(evals),
657
+ wall: durationFigure(record.durationMs),
658
+ tally,
659
+ tokens: total === null ? null : {
660
+ total: tokensFigure(total),
661
+ cacheRead: tokensFigure(cacheRead)
662
+ },
663
+ cost: basis ? {
664
+ agent: agent ? usdFigure(agent.usd) : null,
665
+ judge: judge ? usdFigure(judge.usd) : null,
666
+ total: agent && judge ? usdFigure(agent.usd + judge.usd) : null,
667
+ basis
668
+ } : cost ? {
669
+ agent: null,
670
+ judge: null,
671
+ total: usdFigure(cost.usd),
672
+ basis: cost.basis === "reported" ? "reported" : "estimated"
673
+ } : null,
674
+ git: record.git ? {
675
+ sha: record.git.sha,
676
+ branch: record.git.branch,
677
+ dirty: record.git.dirty,
678
+ repo: repoBasename(record.git.remote)
679
+ } : null
680
+ };
681
+ }
682
+ function uniqueEvalNames(evals) {
683
+ const counts = /* @__PURE__ */ new Map();
684
+ for (const e of evals) counts.set(e.name, (counts.get(e.name) ?? 0) + 1);
685
+ const out = /* @__PURE__ */ new Map();
686
+ for (const e of evals) out.set(e.key, (counts.get(e.name) ?? 0) > 1 ? `${e.name} (${e.file})` : e.name);
687
+ return out;
688
+ }
689
+ function shapesOf(e, prior) {
690
+ const shapes = [];
691
+ if (prior) shapes.push("regression");
692
+ if (e.compare) shapes.push("ab");
693
+ if (e.attempts.some((a) => (a.assertions ?? []).some((x) => /toHaveCalledSkill|toHaveFetched|toHaveCalledMcp|toHaveReadSkill/i.test(x.matcher)))) shapes.push("discoverability");
694
+ if (!e.compare && !prior) shapes.push("task");
695
+ return shapes.length ? shapes : ["task"];
696
+ }
697
+ function designOf(e) {
698
+ const harnesses = new Set(e.cells.map((c) => c.harness)).size;
699
+ const arms = new Set(e.cells.map((c) => c.environment)).size;
700
+ const prompts = new Set(e.attempts.flatMap((a) => a.turns[0]?.prompt ? [a.turns[0].prompt] : [])).size;
701
+ const attempts = e.attempts.filter((a) => a.seq !== null).length;
702
+ const parts = [];
703
+ if (prompts > 1) parts.push(`${prompts} prompts`);
704
+ parts.push(`${harnesses} coding harness${harnesses === 1 ? "" : "es"}`);
705
+ if (arms > 1) parts.push(`${arms} arms`);
706
+ parts.push(`${attempts} attempt${attempts === 1 ? "" : "s"}`);
707
+ return parts.join(", ");
708
+ }
709
+ function assertionLabels(e) {
710
+ const seen = /* @__PURE__ */ new Map();
711
+ for (const a of e.attempts) for (const x of a.assertions ?? []) {
712
+ if (seen.has(x.label)) continue;
713
+ seen.set(x.label, {
714
+ label: x.label,
715
+ matcher: x.matcher,
716
+ at: x.at ?? null
717
+ });
718
+ }
719
+ return [...seen.values()];
720
+ }
721
+ function checkCellOf(attempts, label) {
722
+ let passed = 0;
723
+ let failed = 0;
724
+ let errored = 0;
725
+ let notApplicable = 0;
726
+ let preSatisfied = 0;
727
+ let notRun = 0;
728
+ let reached = 0;
729
+ let reason = null;
730
+ for (const a of attempts) {
731
+ if (!countedOutcome(a.outcome) && a.outcome !== "infra_error") continue;
732
+ const hit = (a.assertions ?? []).find((x) => x.label === label);
733
+ if (!hit) {
734
+ notRun++;
735
+ continue;
736
+ }
737
+ reached++;
738
+ const status = hit.status;
739
+ if (status === "not_applicable") {
740
+ notApplicable++;
741
+ if (!reason && hit.reason) reason = hit.reason;
742
+ continue;
743
+ }
744
+ if (hit.preSatisfied) preSatisfied++;
745
+ if (status === "pass") passed++;
746
+ else if (status === "fail") failed++;
747
+ else if (status === "error") errored++;
748
+ }
749
+ const answered = passed + failed;
750
+ if (reached === 0 || answered === 0 && notApplicable > 0 && errored === 0) {
751
+ const na = reason ? `n/a (${reason})` : "n/a";
752
+ return {
753
+ status: reached === 0 ? "not-run" : "na",
754
+ passed: 0,
755
+ answered: 0,
756
+ notApplicable,
757
+ preSatisfied,
758
+ text: reached === 0 ? "n/a" : na,
759
+ reason: reached === 0 ? null : reason
760
+ };
761
+ }
762
+ if (errored && answered === 0) return {
763
+ status: "error",
764
+ passed,
765
+ answered,
766
+ notApplicable,
767
+ preSatisfied,
768
+ text: `${errored} error${errored === 1 ? "" : "s"}`,
769
+ reason: null
770
+ };
771
+ const status = answered === 0 ? "na" : failed === 0 && errored === 0 ? "pass" : passed === 0 ? "fail" : "mixed";
772
+ const base = answered === 0 ? reason ? `n/a (${reason})` : "n/a" : `${passed} of ${answered}`;
773
+ const text = preSatisfied > 0 ? `${base}, ${preSatisfied} pre-satisfied` : base;
774
+ return {
775
+ status,
776
+ passed,
777
+ answered,
778
+ notApplicable,
779
+ preSatisfied,
780
+ text,
781
+ reason: status === "na" ? reason : null
782
+ };
783
+ }
784
+ function passRateForAttempts(attempts) {
785
+ const counted = attempts.filter((a) => countedOutcome(a.outcome));
786
+ const passed = counted.filter((a) => a.outcome === "pass").length;
787
+ return rateOf(passed, counted.length);
788
+ }
789
+ function armLabel(cell, digestClash) {
790
+ if (cell.role === "control") return cell.environment;
791
+ const name = cell.variant ?? cell.environment;
792
+ if (!digestClash || !cell.environmentDigest) return name;
793
+ return `${name} (${cell.environmentDigest.replace(/^sha256:/, "").slice(0, 7)})`;
794
+ }
795
+ function armsOf(e) {
796
+ const byEnv = /* @__PURE__ */ new Map();
797
+ for (const c of e.cells) {
798
+ const list = byEnv.get(c.environment) ?? [];
799
+ list.push(c);
800
+ byEnv.set(c.environment, list);
801
+ }
802
+ const digestClash = [...byEnv.values()].some((cells) => {
803
+ return new Set(cells.map((c) => c.environmentDigest ?? c.environment)).size > 1;
804
+ });
805
+ const controls = e.cells.filter((c) => c.role === "control");
806
+ const variants = e.cells.filter((c) => c.role === "variant");
807
+ const order = [];
808
+ const seen = /* @__PURE__ */ new Set();
809
+ for (const c of [...controls, ...variants]) {
810
+ const key = c.environmentDigest ?? c.environment;
811
+ if (seen.has(key)) continue;
812
+ seen.add(key);
813
+ order.push(c);
814
+ }
815
+ return order.map((c) => {
816
+ const attempts = e.attempts.filter((a) => cellOf(e, a.cell)?.environment === c.environment && countedOutcome(a.outcome));
817
+ const prompts = [...new Set(attempts.flatMap((a) => {
818
+ const p = a.turns[0]?.prompt;
819
+ return p ? [p] : [];
820
+ }))];
821
+ return {
822
+ id: c.environmentDigest ?? c.environment,
823
+ label: armLabel(c, digestClash),
824
+ environment: c.environment,
825
+ environmentDigest: c.environmentDigest,
826
+ role: c.role === "control" ? "baseline" : "variant",
827
+ change: c.role === "variant" ? c.variant ?? c.environment : null,
828
+ prompts: prompts.map((p) => {
829
+ let h = 0;
830
+ for (let i = 0; i < p.length; i++) h = h * 31 + p.charCodeAt(i) | 0;
831
+ return `p${(h >>> 0).toString(16).padStart(8, "0")}`;
832
+ })
833
+ };
834
+ });
835
+ }
836
+ function fairnessOf(nA, nB, interval) {
837
+ if (Math.min(nA, nB) < 4) return "too-few-runs";
838
+ if (interval && containsZero(interval.lo, interval.hi)) return "within-noise";
839
+ return "clear";
840
+ }
841
+ function floorSentence(fairness, n) {
842
+ if (fairness === "too-few-runs") return `${n} per side is below 4, so this cannot come back helped or made worse.`;
843
+ if (fairness === "within-noise") return "The interval spans 0: within noise.";
844
+ return "";
845
+ }
846
+ function headlinePrimaryText(primary) {
847
+ switch (primary.kind) {
848
+ case "pass-rate": return `${primary.rate.text} ${PASS_RATE_OUTCOME_LABEL[primary.outcome]}`;
849
+ case "lift": return `${primary.variant.label} vs ${primary.baseline.label}: ${primary.points.text} pass rate (${primary.variantRate.text} vs ${primary.baselineRate.text})`;
850
+ case "tied-ceiling": return `${primary.variant.label} vs ${primary.baseline.label}: tied at ${primary.rate.text} pass rate, median ${measurePhrase(primary.lead.measure)} ${primary.lead.delta.text} (${primary.lead.variant.text} vs ${primary.lead.baseline.text})`;
851
+ case "choice": return `${primary.question}: ${primary.options.map((o) => `${o.label} ${o.count}`).join(", ")}`;
852
+ case "measure": return `Median ${primary.label}: ${primary.spread.median.text}`;
853
+ case "judge": return `${primary.label}: median ${primary.spread.median.text} on ${primary.scale[0]} to ${primary.scale[1]}`;
854
+ case "text": return primary.question;
855
+ case "regression": {
856
+ const parts = [primary.worse ? `${primary.worse} worse` : null, primary.fixed ? `${primary.fixed} fixed` : null].filter(Boolean);
857
+ const counts = parts.length ? parts.join(", ") : "no gated flips";
858
+ if (!primary.lead) return counts;
859
+ return `${counts}: ${primary.lead.signal}, ${primary.lead.prior} to ${primary.lead.current}`;
860
+ }
861
+ }
862
+ }
863
+ function chooseHeadline(e, arms, prior, regression) {
864
+ const harnesses = new Set(e.cells.map((c) => c.harness)).size;
865
+ const assertions = assertionLabels(e);
866
+ const compare = Boolean(e.compare);
867
+ let kicker = null;
868
+ if (regression && prior) {
869
+ const p = regression.prior;
870
+ kicker = `Compared with ${[
871
+ p.startedAt.slice(0, 10),
872
+ p.gitSha ? p.gitSha.slice(0, 7) : null,
873
+ p.runId
874
+ ].filter(Boolean).join(", ")}`;
875
+ } else if (compare && arms.length >= 2) {
876
+ const baseline = arms.find((a) => a.role === "baseline");
877
+ const variant = arms.find((a) => a.role === "variant");
878
+ if (baseline && variant) kicker = `${variant.label} against the ${baseline.label} baseline`;
879
+ }
880
+ if (regression) {
881
+ const worse = regression.rows.filter((r) => r.status === "regressed").length;
882
+ const fixed = regression.rows.filter((r) => r.status === "fixed").length;
883
+ const lead = regression.rows.find((r) => r.gate && (r.status === "regressed" || r.status === "fixed")) ?? regression.rows.find((r) => r.status === "regressed" || r.status === "fixed") ?? regression.rows[0] ?? null;
884
+ const sampleNs = (prior?.rows ?? []).flatMap((row) => (row.perCell ?? []).map((c) => Math.min(c.base.pass + c.base.fail, c.head.pass + c.head.fail)));
885
+ const minN = sampleNs.length ? Math.min(...sampleNs) : 4;
886
+ const fairness = minN < 4 ? "too-few-runs" : "clear";
887
+ const footParts = [
888
+ `${worse} worse, ${fixed} fixed since ${regression.prior.runId}.`,
889
+ regression.prior.gitSha ? `Prior ${regression.prior.startedAt.slice(0, 10)} at ${regression.prior.gitSha.slice(0, 7)}.` : `Prior ${regression.prior.startedAt.slice(0, 10)}.`,
890
+ fairness === "too-few-runs" ? floorSentence("too-few-runs", minN) : null
891
+ ];
892
+ return {
893
+ primary: {
894
+ kind: "regression",
895
+ worse,
896
+ fixed,
897
+ lead
898
+ },
899
+ fairness,
900
+ foot: footParts.filter(Boolean).join(" "),
901
+ kicker
902
+ };
903
+ }
904
+ if (compare && arms.length >= 2) {
905
+ const baseline = arms.find((a) => a.role === "baseline");
906
+ const variants = arms.filter((a) => a.role === "variant");
907
+ const attemptsFor = (arm) => e.attempts.filter((a) => {
908
+ const cell = cellOf(e, a.cell);
909
+ return cell && (cell.environmentDigest ?? cell.environment) === arm.id;
910
+ });
911
+ const rateFor = (arm) => passRateForAttempts(attemptsFor(arm));
912
+ const baselineRate = rateFor(baseline);
913
+ const lifts = variants.map((v) => {
914
+ const rate = rateFor(v);
915
+ const nV = rate.n;
916
+ const nB = baselineRate.n;
917
+ const { points, low, high } = liftInterval({
918
+ pass: rate.k,
919
+ n: nV
920
+ }, {
921
+ pass: baselineRate.k,
922
+ n: nB
923
+ });
924
+ return {
925
+ variant: {
926
+ id: v.id,
927
+ label: v.label
928
+ },
929
+ points: ptsDelta(points),
930
+ interval: ptsInterval(low, high),
931
+ rate,
932
+ rawPoints: points,
933
+ lo: low,
934
+ hi: high,
935
+ n: Math.min(nV, nB)
936
+ };
937
+ });
938
+ lifts.sort((a, b) => Math.abs(b.rawPoints) - Math.abs(a.rawPoints));
939
+ const best = lifts[0];
940
+ const others = lifts.slice(1).map(({ variant, points, interval, rate }) => ({
941
+ variant,
942
+ points,
943
+ interval,
944
+ rate
945
+ }));
946
+ const fairness = fairnessOf(best.rate.n, baselineRate.n, {
947
+ lo: best.lo,
948
+ hi: best.hi
949
+ });
950
+ if (REPORT_POLICY.tiedCeiling && baselineRate.rate === 1 && best.rate.rate === 1 && assertions.length > 0) {
951
+ const baseAttempts = attemptsFor(baseline).filter((a) => countedOutcome(a.outcome));
952
+ const varAttempts = attemptsFor(variants.find((v) => v.id === best.variant.id)).filter((a) => countedOutcome(a.outcome));
953
+ const baseMeasures = measuresOf(baseAttempts);
954
+ const varMeasures = measuresOf(varAttempts);
955
+ let lead = null;
956
+ for (const key of MEASURE_KEYS) {
957
+ const b = baseMeasures[key]?.median;
958
+ const v = varMeasures[key]?.median;
959
+ if (!b || !v || b.value === null || v.value === null) continue;
960
+ const delta = pctDelta(v.value, b.value);
961
+ if (delta.value === null) continue;
962
+ const abs = Math.abs(delta.value);
963
+ if (abs < .1) continue;
964
+ if (!lead || abs > lead.abs) lead = {
965
+ measure: key,
966
+ baseline: b,
967
+ variant: v,
968
+ delta,
969
+ abs
970
+ };
971
+ }
972
+ if (lead) {
973
+ const n = baselineRate.n + best.rate.n;
974
+ return {
975
+ primary: {
976
+ kind: "tied-ceiling",
977
+ baseline: {
978
+ id: baseline.id,
979
+ label: baseline.label
980
+ },
981
+ variant: best.variant,
982
+ rate: baselineRate,
983
+ assertions: assertions.length,
984
+ lead: {
985
+ measure: lead.measure,
986
+ baseline: lead.baseline,
987
+ variant: lead.variant,
988
+ delta: lead.delta
989
+ }
990
+ },
991
+ fairness,
992
+ foot: `Every attempt passed all ${assertions.length} applicable assertions (${n} of ${n}).`,
993
+ kicker
994
+ };
995
+ }
996
+ }
997
+ const floor = floorSentence(fairness, best.n);
998
+ return {
999
+ primary: {
1000
+ kind: "lift",
1001
+ baseline: {
1002
+ id: baseline.id,
1003
+ label: baseline.label
1004
+ },
1005
+ variant: best.variant,
1006
+ points: best.points,
1007
+ interval: best.interval,
1008
+ baselineRate,
1009
+ variantRate: best.rate,
1010
+ others
1011
+ },
1012
+ fairness,
1013
+ foot: [`n = ${best.rate.n} vs ${baselineRate.n} counted attempts. 95% CI ${best.interval.text}.`, floor].filter(Boolean).join(" "),
1014
+ kicker
1015
+ };
1016
+ }
1017
+ if (assertions.length > 0) {
1018
+ const gated = e.attempts.filter((a) => {
1019
+ const cell = cellOf(e, a.cell);
1020
+ return cell && cell.gate !== "none" && countedOutcome(a.outcome);
1021
+ });
1022
+ const pool = gated.length ? gated : e.attempts.filter((a) => countedOutcome(a.outcome));
1023
+ let passedAll = 0;
1024
+ for (const a of pool) {
1025
+ const applicable = (a.assertions ?? []).filter((x) => x.status !== "not_applicable");
1026
+ if (applicable.length === 0) continue;
1027
+ if (applicable.every((x) => x.status === "pass")) passedAll++;
1028
+ }
1029
+ const rate = rateOf(passedAll, pool.length);
1030
+ const outcome = passRateOutcomeFromMatchers(assertions.map((a) => a.matcher)) ?? "passed";
1031
+ const outcomeWord = PASS_RATE_OUTCOME_LABEL[outcome];
1032
+ return {
1033
+ primary: {
1034
+ kind: "pass-rate",
1035
+ rate,
1036
+ assertions: assertions.length,
1037
+ harnesses: Math.max(1, harnesses),
1038
+ outcome
1039
+ },
1040
+ fairness: pool.length < 4 ? "too-few-runs" : "clear",
1041
+ foot: `${rate.k} of ${rate.n} counted attempts ${outcomeWord} on all ${assertions.length} assertions. ${harnesses} coding harness${harnesses === 1 ? "" : "es"}.`,
1042
+ kicker
1043
+ };
1044
+ }
1045
+ const answers = attemptAnswerRows(e);
1046
+ const choiceRow = answers.find((r) => r.choices && r.choices.length >= 2);
1047
+ if (choiceRow?.choices) {
1048
+ const options = choiceRow.choices.map((c) => ({
1049
+ label: c.choice,
1050
+ count: c.n
1051
+ }));
1052
+ const noPick = choiceRow.answers.filter((a) => !a.choice).length;
1053
+ return {
1054
+ primary: {
1055
+ kind: "choice",
1056
+ question: choiceRow.question,
1057
+ options,
1058
+ attempts: choiceRow.answers.length,
1059
+ noPick
1060
+ },
1061
+ fairness: "clear",
1062
+ foot: `${choiceRow.answers.length} attempts, ${noPick} without a pick.`,
1063
+ kicker
1064
+ };
1065
+ }
1066
+ const judgeRows = e.attempts.flatMap((a) => labeledJudgments(a));
1067
+ if (judgeRows.length) {
1068
+ const first = judgeRows[0];
1069
+ const scores = judgeRows.filter((j) => j.label === first.label).map((j) => j.judgment.score);
1070
+ const spread = spreadOf(scores, (n) => n.toFixed(2));
1071
+ const threshold = (first.check ?? null) === null ? null : first.judgment.threshold ?? null;
1072
+ const atOrAbove = threshold === null ? null : countOf(scores.filter((s) => s >= threshold).length, scores.length);
1073
+ return {
1074
+ primary: {
1075
+ kind: "judge",
1076
+ label: first.label,
1077
+ question: first.judgment.question,
1078
+ scale: [0, 1],
1079
+ threshold,
1080
+ spread,
1081
+ atOrAbove
1082
+ },
1083
+ fairness: scores.length < 4 ? "too-few-runs" : "clear",
1084
+ foot: atOrAbove ? `${atOrAbove.text} at or above the ${threshold} threshold.` : "Diagnostic judgment, no threshold.",
1085
+ kicker
1086
+ };
1087
+ }
1088
+ const observeNumeric = e.attempts.flatMap((a) => (a.observations ?? []).flatMap((o) => typeof o.value === "number" ? [{
1089
+ name: o.name,
1090
+ value: o.value
1091
+ }] : []));
1092
+ if (observeNumeric.length) {
1093
+ const name = observeNumeric[0].name;
1094
+ const values = observeNumeric.filter((o) => o.name === name).map((o) => o.value);
1095
+ const spread = spreadOf(values, formatCount);
1096
+ return {
1097
+ primary: {
1098
+ kind: "measure",
1099
+ label: name,
1100
+ unit: "count",
1101
+ spread
1102
+ },
1103
+ fairness: values.length < 4 ? "too-few-runs" : "clear",
1104
+ foot: `${spread.n} attempts, interquartile range ${spread.q1.text} to ${spread.q3.text}.`,
1105
+ kicker
1106
+ };
1107
+ }
1108
+ const textRow = answers.find((r) => !r.choices);
1109
+ if (textRow) {
1110
+ const answered = textRow.answers.filter((a) => a.text.trim());
1111
+ const leadAnswer = answered[0];
1112
+ let lead = null;
1113
+ if (leadAnswer) {
1114
+ const attempt = e.attempts.find((a) => a.id === leadAnswer.attempt);
1115
+ if (attempt) lead = {
1116
+ text: leadAnswer.text.slice(0, 200),
1117
+ attempt: {
1118
+ id: "A?",
1119
+ key: attempt.key,
1120
+ href: "#"
1121
+ },
1122
+ harness: cellOf(e, attempt.cell)?.harness ?? attempt.cell,
1123
+ arm: cellOf(e, attempt.cell)?.environment ?? null,
1124
+ case: attempt.case ?? null,
1125
+ input: leadAnswer.input
1126
+ };
1127
+ }
1128
+ const leadRole = answered.length > 1 && lead ? "representative" : answered.length === 1 && lead ? "sole" : null;
1129
+ return {
1130
+ primary: {
1131
+ kind: "text",
1132
+ question: textRow.question,
1133
+ answered: countOf(answered.length, textRow.answers.length),
1134
+ lead,
1135
+ leadRole
1136
+ },
1137
+ fairness: "clear",
1138
+ foot: leadRole === "representative" ? `${answered.length} of ${textRow.answers.length} attempts answered. Showing 1 representative answer.` : `${answered.length} of ${textRow.answers.length} attempts answered.`,
1139
+ kicker
1140
+ };
1141
+ }
1142
+ const tokenValues = e.attempts.filter((a) => countedOutcome(a.outcome) && a.usage).map((a) => totalTokens(a.usage));
1143
+ const spread = spreadOf(tokenValues, tokensText) ?? {
1144
+ n: 0,
1145
+ median: tokensFigure(null),
1146
+ q1: tokensFigure(null),
1147
+ q3: tokensFigure(null)
1148
+ };
1149
+ return {
1150
+ primary: {
1151
+ kind: "measure",
1152
+ label: "tokens",
1153
+ unit: "tokens",
1154
+ spread
1155
+ },
1156
+ fairness: tokenValues.length < 4 ? "too-few-runs" : "clear",
1157
+ foot: spread.n === 0 ? "No attempt reported usage." : `${spread.n} attempts, interquartile range ${spread.q1.text} to ${spread.q3.text}.`,
1158
+ kicker
1159
+ };
1160
+ }
1161
+ function buildTaskResults(e) {
1162
+ const labels = assertionLabels(e);
1163
+ if (!labels.length) return null;
1164
+ const harnesses = [...new Set(e.cells.map((c) => c.harness))];
1165
+ const columns = harnesses;
1166
+ const assertions = labels.map((row) => {
1167
+ const cells = {};
1168
+ for (const h of columns) cells[h] = checkCellOf(e.attempts.filter((a) => cellOf(e, a.cell)?.harness === h), row.label);
1169
+ const total = checkCellOf(e.attempts.filter((a) => countedOutcome(a.outcome) || a.outcome === "infra_error"), row.label);
1170
+ return {
1171
+ label: row.label,
1172
+ matcher: row.matcher,
1173
+ at: row.at,
1174
+ cells,
1175
+ total
1176
+ };
1177
+ });
1178
+ const axes = ["Coding harness"];
1179
+ const breakdown = harnesses.map((h) => {
1180
+ const attempts = e.attempts.filter((a) => cellOf(e, a.cell)?.harness === h);
1181
+ return {
1182
+ keys: [h],
1183
+ rate: passRateForAttempts(attempts),
1184
+ measures: measuresOf(attempts)
1185
+ };
1186
+ });
1187
+ return {
1188
+ kind: "task",
1189
+ columns,
1190
+ assertions,
1191
+ axes,
1192
+ breakdown,
1193
+ median: breakdown.length > 1 ? {
1194
+ keys: ["Median"],
1195
+ rate: passRateForAttempts(e.attempts),
1196
+ measures: measuresOf(e.attempts.filter((a) => countedOutcome(a.outcome)))
1197
+ } : null
1198
+ };
1199
+ }
1200
+ function buildVariantComparison(e, arms) {
1201
+ const baseline = arms.find((a) => a.role === "baseline");
1202
+ const attemptsFor = (arm) => e.attempts.filter((a) => {
1203
+ const cell = cellOf(e, a.cell);
1204
+ return cell && (cell.environmentDigest ?? cell.environment) === arm.id;
1205
+ });
1206
+ const baseAttempts = baseline ? attemptsFor(baseline).filter((a) => countedOutcome(a.outcome)) : [];
1207
+ const baseMeasures = measuresOf(baseAttempts);
1208
+ const baseRate = passRateForAttempts(baseAttempts);
1209
+ const summary = arms.map((arm) => {
1210
+ const attempts = attemptsFor(arm).filter((a) => countedOutcome(a.outcome));
1211
+ const rate = passRateForAttempts(attempts);
1212
+ const measures = measuresOf(attempts);
1213
+ const vs = emptyMeasureVsBaseline();
1214
+ for (const key of MEASURE_KEYS) vs[key] = measureVs(measures[key], arm.role === "baseline" ? null : baseMeasures[key], null);
1215
+ if (arm.role === "baseline") return {
1216
+ arm: arm.label,
1217
+ rate,
1218
+ lift: null,
1219
+ interval: null,
1220
+ verdict: "baseline",
1221
+ verdictScope: "pass-rate",
1222
+ measures: vs
1223
+ };
1224
+ const { points, low, high } = liftInterval({
1225
+ pass: rate.k,
1226
+ n: rate.n
1227
+ }, {
1228
+ pass: baseRate.k,
1229
+ n: baseRate.n
1230
+ });
1231
+ const fairness = fairnessOf(rate.n, baseRate.n, {
1232
+ lo: low,
1233
+ hi: high
1234
+ });
1235
+ const verdict = fairness === "too-few-runs" ? "too-few-runs" : fairness === "within-noise" ? "within-noise" : points > 0 ? "helped" : points < 0 ? "made-worse" : "within-noise";
1236
+ return {
1237
+ arm: arm.label,
1238
+ rate,
1239
+ lift: ptsDelta(points),
1240
+ interval: ptsInterval(low, high),
1241
+ verdict,
1242
+ verdictScope: "pass-rate",
1243
+ measures: vs
1244
+ };
1245
+ });
1246
+ const labels = assertionLabels(e);
1247
+ arms.map((a) => a.label);
1248
+ let uniform = labels.length > 0;
1249
+ const assertionsByArm = [];
1250
+ for (const row of labels) {
1251
+ const cells = {};
1252
+ for (const arm of arms) {
1253
+ cells[arm.label] = checkCellOf(attemptsFor(arm), row.label);
1254
+ const cell = cells[arm.label];
1255
+ if (cell.status !== "pass" && cell.status !== "na") uniform = false;
1256
+ }
1257
+ const total = checkCellOf(e.attempts, row.label);
1258
+ assertionsByArm.push({
1259
+ label: row.label,
1260
+ matcher: row.matcher,
1261
+ at: row.at,
1262
+ cells,
1263
+ total
1264
+ });
1265
+ }
1266
+ const uniformLead = uniform ? `Every arm passed every applicable assertion (${labels.length} checks).` : null;
1267
+ const contrasts = contrastsOf(e).flatMap((c) => {
1268
+ const aCell = cellOf(e, c.a);
1269
+ const bCell = cellOf(e, c.b);
1270
+ if (!aCell || !bCell || !c.lift) return [];
1271
+ const fairness = fairnessOf(c.lift.variant.counted, c.lift.control.counted, c.lift.interval);
1272
+ const verdict = fairness === "too-few-runs" ? "too-few-runs" : fairness === "within-noise" ? "within-noise" : c.lift.points > 0 ? "helped" : c.lift.points < 0 ? "made-worse" : "within-noise";
1273
+ return [{
1274
+ from: aCell.variant ?? aCell.environment,
1275
+ to: bCell.variant ?? bCell.environment,
1276
+ measure: "pass-rate",
1277
+ delta: ptsDelta(c.lift.points),
1278
+ interval: ptsInterval(c.lift.interval.lo, c.lift.interval.hi),
1279
+ verdict,
1280
+ verdictScope: "pass-rate"
1281
+ }];
1282
+ });
1283
+ const effortByHarness = [...new Set(e.cells.map((c) => c.harness))].flatMap((h) => arms.map((arm) => {
1284
+ const attempts = e.attempts.filter((a) => {
1285
+ const cell = cellOf(e, a.cell);
1286
+ return cell?.harness === h && cell && (cell.environmentDigest ?? cell.environment) === arm.id && countedOutcome(a.outcome);
1287
+ });
1288
+ const measures = measuresOf(attempts);
1289
+ const vs = emptyMeasureVsBaseline();
1290
+ const baseM = measuresOf(baseline ? e.attempts.filter((a) => {
1291
+ const cell = cellOf(e, a.cell);
1292
+ return cell?.harness === h && cell && (cell.environmentDigest ?? cell.environment) === baseline.id && countedOutcome(a.outcome);
1293
+ }) : []);
1294
+ for (const key of MEASURE_KEYS) vs[key] = measureVs(measures[key], arm.role === "baseline" ? null : baseM[key], null);
1295
+ return {
1296
+ harness: h,
1297
+ arm: arm.label,
1298
+ rate: passRateForAttempts(attempts),
1299
+ measures: vs
1300
+ };
1301
+ }));
1302
+ return {
1303
+ kind: "comparison",
1304
+ arms,
1305
+ uniformLead,
1306
+ summary,
1307
+ assertionsByArm: uniformLead ? [] : assertionsByArm,
1308
+ behavior: [],
1309
+ contrasts,
1310
+ effortByHarness
1311
+ };
1312
+ }
1313
+ function compareSideText(row, side) {
1314
+ const cells = row.perCell;
1315
+ if (cells?.length) {
1316
+ const pass = cells.reduce((n, c) => n + c[side].pass, 0);
1317
+ const n = pass + cells.reduce((n, c) => n + c[side].fail, 0);
1318
+ if (n > 0) return `${pass} of ${n}`;
1319
+ }
1320
+ return row[side] ?? "n/a";
1321
+ }
1322
+ function compareSideDelta(row) {
1323
+ const cells = row.perCell;
1324
+ if (!cells?.length) return null;
1325
+ const basePass = cells.reduce((n, c) => n + c.base.pass, 0);
1326
+ const baseN = cells.reduce((n, c) => n + c.base.pass + c.base.fail, 0);
1327
+ const headPass = cells.reduce((n, c) => n + c.head.pass, 0);
1328
+ const headN = cells.reduce((n, c) => n + c.head.pass + c.head.fail, 0);
1329
+ if (!baseN || !headN) return null;
1330
+ return ptsDelta(headPass / headN - basePass / baseN);
1331
+ }
1332
+ function buildRegression(prior) {
1333
+ const rows = [];
1334
+ const unchanged = [];
1335
+ for (const row of prior.rows) {
1336
+ const change = {
1337
+ signal: row.id,
1338
+ kind: "outcome",
1339
+ harness: null,
1340
+ prior: compareSideText(row, "base"),
1341
+ current: compareSideText(row, "head"),
1342
+ delta: compareSideDelta(row),
1343
+ status: row.status,
1344
+ gate: Boolean(row.gate)
1345
+ };
1346
+ if (row.status === "same") unchanged.push(change);
1347
+ else rows.push(change);
1348
+ }
1349
+ const whatChanged = [...prior.changes.map((c) => `${c.label}: ${c.base} to ${c.head}`), ...prior.rows.flatMap((r) => (r.changes ?? []).map((c) => `${r.id}: ${c}`))];
1350
+ const priorRec = prior.record;
1351
+ return {
1352
+ kind: "regression",
1353
+ baseRunId: priorRec.runId,
1354
+ prior: {
1355
+ runId: priorRec.runId,
1356
+ startedAt: priorRec.startedAt,
1357
+ gitSha: priorRec.git?.sha ?? null,
1358
+ branch: priorRec.git?.branch ?? null,
1359
+ author: priorRec.author
1360
+ },
1361
+ whatChanged,
1362
+ rows,
1363
+ unchanged
1364
+ };
1365
+ }
1366
+ function buildChoices(e) {
1367
+ return attemptAnswerRows(e).filter((r) => r.choices && r.choices.length >= 2).map((r) => {
1368
+ const options = [...r.choices ?? []].map((c) => ({
1369
+ label: c.choice,
1370
+ count: c.n
1371
+ })).sort((a, b) => b.count - a.count || a.label.localeCompare(b.label));
1372
+ const noPick = r.answers.filter((a) => !a.choice).length;
1373
+ const byKey = /* @__PURE__ */ new Map();
1374
+ for (const ans of r.answers) {
1375
+ const attempt = e.attempts.find((a) => a.id === ans.attempt);
1376
+ if (!attempt) continue;
1377
+ const cell = cellOf(e, attempt.cell);
1378
+ const harness = cell?.harness ?? attempt.cell;
1379
+ const arm = cell?.environment ?? null;
1380
+ const key = `${arm ?? ""}\0${harness}`;
1381
+ const row = byKey.get(key) ?? {
1382
+ arm,
1383
+ prompt: null,
1384
+ harness,
1385
+ counts: Object.fromEntries(options.map((o) => [o.label, 0])),
1386
+ attempts: 0
1387
+ };
1388
+ row.attempts++;
1389
+ if (ans.choice && row.counts[ans.choice] !== void 0) row.counts[ans.choice]++;
1390
+ byKey.set(key, row);
1391
+ }
1392
+ const rows = [...byKey.values()].map((row) => ({
1393
+ ...row,
1394
+ text: options.map((o) => `${o.label} ${row.counts[o.label] ?? 0}`).join(", ")
1395
+ }));
1396
+ return {
1397
+ question: r.question,
1398
+ options,
1399
+ noPick,
1400
+ rows,
1401
+ passByPick: options.map((o) => {
1402
+ const picked = r.answers.filter((a) => a.choice === o.label);
1403
+ const passed = picked.filter((a) => a.outcome === "pass").length;
1404
+ return {
1405
+ option: o.label,
1406
+ rate: rateOf(passed, picked.length)
1407
+ };
1408
+ })
1409
+ };
1410
+ });
1411
+ }
1412
+ const SKILL_SOURCES = [
1413
+ "placed",
1414
+ "skill-dir",
1415
+ "elsewhere"
1416
+ ];
1417
+ /**
1418
+ * The skill an attempt's first passing `toHaveCalledSkill` (or `poll.`, `resolves.` … but not a
1419
+ * `not.`) checked, read from its default label: the matcher as recorded, the skill, then any ` #n`
1420
+ * that keeps the label unique. Undefined when every such call had a label of its own, which
1421
+ * doesn't name the skill.
1422
+ */
1423
+ function checkedSkill(a) {
1424
+ for (const x of a.assertions ?? []) {
1425
+ const parts = x.matcher.split(".");
1426
+ if (parts.at(-1) !== "toHaveCalledSkill" || parts.includes("not")) continue;
1427
+ if (x.status !== "pass" || x.labelled || !x.label.startsWith(`${x.matcher} `)) continue;
1428
+ const name = x.label.slice(x.matcher.length + 1).replace(/(?: #\d+)+$/, "");
1429
+ if (name) return name;
1430
+ }
1431
+ }
1432
+ /**
1433
+ * How many of `attempts` (each found the skill) used the skill it checked from each source, by its
1434
+ * first use in their facts: one source an attempt, so the counts sum to at most the attempts.
1435
+ */
1436
+ function skillSources(attempts, facts) {
1437
+ if (!facts) return [];
1438
+ const counts = /* @__PURE__ */ new Map();
1439
+ for (const a of attempts) {
1440
+ const name = checkedSkill(a);
1441
+ if (name === void 0) continue;
1442
+ const first = facts.get(a.key)?.skills.filter((s) => s.skill === name || s.skill.endsWith(`:${name}`)).sort((x, y) => x.firstLine - y.firstLine)[0];
1443
+ if (first) counts.set(first.source, (counts.get(first.source) ?? 0) + 1);
1444
+ }
1445
+ return SKILL_SOURCES.flatMap((source) => {
1446
+ const count = counts.get(source) ?? 0;
1447
+ return count > 0 ? [{
1448
+ source,
1449
+ count
1450
+ }] : [];
1451
+ });
1452
+ }
1453
+ function buildDiscovery(e, facts) {
1454
+ if (e.compare) return [];
1455
+ const matchers = [
1456
+ {
1457
+ what: "skill",
1458
+ matcher: "toHaveCalledSkill"
1459
+ },
1460
+ {
1461
+ what: "MCP tool",
1462
+ matcher: "toHaveCalledMcp"
1463
+ },
1464
+ {
1465
+ what: "web fetch",
1466
+ matcher: "toHaveFetched"
1467
+ }
1468
+ ];
1469
+ const out = [];
1470
+ for (const m of matchers) {
1471
+ const attempts = e.attempts.filter((a) => countedOutcome(a.outcome));
1472
+ const foundAttempts = attempts.filter((a) => (a.assertions ?? []).some((x) => x.matcher.includes(m.matcher) && x.status === "pass"));
1473
+ if (!foundAttempts.length && !attempts.some((a) => (a.assertions ?? []).some((x) => x.matcher.includes(m.matcher)))) continue;
1474
+ const applicable = attempts.filter((a) => (a.assertions ?? []).some((x) => x.matcher.includes(m.matcher)));
1475
+ if (!applicable.length) continue;
1476
+ out.push({
1477
+ what: m.what,
1478
+ matcher: m.matcher,
1479
+ found: countOf(foundAttempts.length, applicable.length),
1480
+ rows: [...new Set(e.cells.map((c) => c.harness))].map((h) => {
1481
+ const pool = applicable.filter((a) => cellOf(e, a.cell)?.harness === h);
1482
+ const found = pool.filter((a) => (a.assertions ?? []).some((x) => x.matcher.includes(m.matcher) && x.status === "pass"));
1483
+ return {
1484
+ keys: [h],
1485
+ found: countOf(found.length, pool.length)
1486
+ };
1487
+ }),
1488
+ sources: m.what === "skill" ? skillSources(foundAttempts, facts) : []
1489
+ });
1490
+ }
1491
+ return out;
1492
+ }
1493
+ function buildJudgments(e) {
1494
+ const scores = e.attempts.flatMap((a) => {
1495
+ const cell = cellOf(e, a.cell);
1496
+ return labeledJudgments(a).map((j) => ({
1497
+ ...j,
1498
+ keys: [cell?.harness ?? a.cell, cell?.environment ?? ""].filter(Boolean)
1499
+ }));
1500
+ });
1501
+ if (!scores.length && !(e.answers ?? []).length) return null;
1502
+ const byLabel = /* @__PURE__ */ new Map();
1503
+ for (const s of scores) {
1504
+ const list = byLabel.get(s.label) ?? [];
1505
+ list.push(s);
1506
+ byLabel.set(s.label, list);
1507
+ }
1508
+ const hasArms = e.cells.some((c) => Boolean(c.environment));
1509
+ const judgeRows = [...byLabel.entries()].map(([label, rows]) => {
1510
+ const values = rows.map((r) => r.judgment.score);
1511
+ const spread = spreadOf(values, (n) => n.toFixed(2));
1512
+ const assertion = rows[0].check ?? null;
1513
+ const threshold = assertion === null ? null : rows[0].judgment.threshold ?? null;
1514
+ return {
1515
+ label,
1516
+ question: rows[0].judgment.question,
1517
+ assertion,
1518
+ threshold,
1519
+ keys: hasArms ? ["Coding harness", "Arm"] : ["Coding harness"],
1520
+ spread,
1521
+ atOrAbove: threshold === null ? null : countOf(values.filter((v) => v >= threshold).length, values.length)
1522
+ };
1523
+ });
1524
+ const pairwise = [];
1525
+ for (const ans of e.answers ?? []) {
1526
+ const ties = ans.votes.filter((v) => v.winner === "tie").length;
1527
+ for (const cut of ["within", "across"]) {
1528
+ const rates = ans.rates.filter((r) => r.cut === cut);
1529
+ if (rates.length < 2) {
1530
+ for (const r of rates) pairwise.push({
1531
+ question: ans.question,
1532
+ cut,
1533
+ a: r.name,
1534
+ b: "rest",
1535
+ aWins: r.wins,
1536
+ bWins: Math.max(0, r.n - r.wins),
1537
+ ties,
1538
+ pairs: r.n,
1539
+ reasons: []
1540
+ });
1541
+ continue;
1542
+ }
1543
+ const [a, b] = rates;
1544
+ if (!a || !b) continue;
1545
+ pairwise.push({
1546
+ question: ans.question,
1547
+ cut,
1548
+ a: a.name,
1549
+ b: b.name,
1550
+ aWins: a.wins,
1551
+ bWins: b.wins,
1552
+ ties,
1553
+ pairs: Math.max(a.n, b.n),
1554
+ reasons: []
1555
+ });
1556
+ }
1557
+ }
1558
+ return {
1559
+ axes: hasArms ? ["Coding harness", "Arm"] : ["Coding harness"],
1560
+ scores: judgeRows,
1561
+ pairwise
1562
+ };
1563
+ }
1564
+ function buildQualitative(e, ids, linkBase, hrefs, hrefByKey) {
1565
+ return attemptAnswerRows(e).filter((r) => !r.choices).map((r) => {
1566
+ const entries = [];
1567
+ for (const ans of r.answers) {
1568
+ if (!ans.text.trim()) continue;
1569
+ const attempt = e.attempts.find((a) => a.id === ans.attempt);
1570
+ if (!attempt) continue;
1571
+ const ref = attemptRef(ids, e, attempt, linkBase, hrefs, hrefByKey);
1572
+ if (!ref) continue;
1573
+ const cell = cellOf(e, attempt.cell);
1574
+ entries.push({
1575
+ text: ans.text.slice(0, 400),
1576
+ attempt: ref,
1577
+ harness: cell?.harness ?? attempt.cell,
1578
+ arm: cell?.environment ?? null,
1579
+ case: attempt.case ? caseName(e, {
1580
+ key: attempt.case,
1581
+ name: void 0
1582
+ }) : null,
1583
+ input: ans.input ? withBase(linkBase, ans.input) : null
1584
+ });
1585
+ }
1586
+ return {
1587
+ question: r.question,
1588
+ answered: countOf(entries.length, r.answers.length),
1589
+ themes: null,
1590
+ entries
1591
+ };
1592
+ });
1593
+ }
1594
+ function buildCaptures(e, ids, linkBase, hrefs, hrefByKey, pages) {
1595
+ const rows = /* @__PURE__ */ new Map();
1596
+ for (const a of e.attempts) {
1597
+ const arts = a.files.artifacts ?? [];
1598
+ if (!arts.length) continue;
1599
+ const ref = attemptRef(ids, e, a, linkBase, hrefs, hrefByKey);
1600
+ if (!ref) continue;
1601
+ const cell = cellOf(e, a.cell);
1602
+ const key = `${cell?.harness ?? ""}\0${cell?.environment ?? ""}\0${a.case ?? ""}`;
1603
+ const row = rows.get(key) ?? {
1604
+ harness: cell?.harness ?? a.cell,
1605
+ arm: cell?.environment ?? null,
1606
+ case: a.case ?? null,
1607
+ shots: []
1608
+ };
1609
+ for (const art of arts) {
1610
+ const pagePath = art.capture?.page?.path;
1611
+ const page = pagePath ? pages?.get(pagePath) : void 0;
1612
+ row.shots.push({
1613
+ name: art.name,
1614
+ href: captureHref(art.path, linkBase),
1615
+ meta: page?.finalUrl ?? (art.capture?.instructions?.length ? art.capture.instructions.join("; ") : art.capture?.path ?? null),
1616
+ attempt: ref,
1617
+ status: art.capture?.status ?? page?.status ?? null,
1618
+ pageErrors: art.capture?.pageErrors ?? page?.errors.length ?? null,
1619
+ consoleErrors: art.capture?.consoleErrors ?? page?.console.filter((c) => c.level === "error").length ?? null
1620
+ });
1621
+ }
1622
+ rows.set(key, row);
1623
+ }
1624
+ if (!rows.size) return null;
1625
+ return { rows: [...rows.values()] };
1626
+ }
1627
+ function buildFailures(e, ids, linkBase, hrefs, hrefByKey, remarks) {
1628
+ const counted = e.attempts.filter((a) => countedOutcome(a.outcome) || a.outcome === "infra_error");
1629
+ const clusters = /* @__PURE__ */ new Map();
1630
+ const add = (kind, label, at, attempt) => {
1631
+ const ref = attemptRef(ids, e, attempt, linkBase, hrefs, hrefByKey);
1632
+ if (!ref) return;
1633
+ const key = `${kind}\0${label}\0${at ?? ""}`;
1634
+ const existing = clusters.get(key);
1635
+ if (existing) {
1636
+ existing.attempts.push(ref);
1637
+ existing.count = countOf(existing.attempts.length, counted.length);
1638
+ return;
1639
+ }
1640
+ clusters.set(key, {
1641
+ kind,
1642
+ label,
1643
+ at,
1644
+ count: countOf(1, counted.length),
1645
+ attempts: [ref]
1646
+ });
1647
+ };
1648
+ for (const a of e.attempts) {
1649
+ for (const x of a.assertions ?? []) if (x.status === "fail") add("assertion", x.label, x.at ?? null, a);
1650
+ for (const t of a.turns) if (t.status === "error" && t.error) add("turn-error", messageLine(t.error), null, a);
1651
+ if (a.outcome === "infra_error" && a.error) add("infra-error", messageLine(a.error.message), null, a);
1652
+ else if (a.error && a.outcome === "fail") add("error", messageLine(a.error.message), null, a);
1653
+ }
1654
+ const evalPath = factSegment(e.id);
1655
+ const evalRemarks = remarks?.remarks.filter((r) => r.severity === "concern" && (r.evalKey === e.key || r.evalId === e.id || r.evalId === e.key || r.evalId === evalPath)).map((r) => ({
1656
+ kind: r.kind,
1657
+ severity: r.severity,
1658
+ text: r.text,
1659
+ fix: r.fix
1660
+ })) ?? [];
1661
+ if (!clusters.size && !evalRemarks.length) return null;
1662
+ return {
1663
+ clusters: [...clusters.values()].sort((a, b) => b.count.k - a.count.k),
1664
+ remarks: evalRemarks
1665
+ };
1666
+ }
1667
+ function fillTextLeadCites(primary, e, ids, linkBase, hrefs, hrefByKey) {
1668
+ if (primary.kind !== "text" || !primary.lead) return primary;
1669
+ const first = attemptAnswerRows(e).find((r) => r.question === primary.question)?.answers.find((a) => a.text.trim());
1670
+ const attempt = first && e.attempts.find((a) => a.id === first.attempt);
1671
+ const ref = attempt && attemptRef(ids, e, attempt, linkBase, hrefs, hrefByKey);
1672
+ if (!ref) return {
1673
+ ...primary,
1674
+ lead: null,
1675
+ leadRole: null
1676
+ };
1677
+ return {
1678
+ ...primary,
1679
+ lead: {
1680
+ ...primary.lead,
1681
+ attempt: ref
1682
+ }
1683
+ };
1684
+ }
1685
+ function buildEvalSection(ctx, e, displayName) {
1686
+ const arms = armsOf(e);
1687
+ const shapes = shapesOf(e, Boolean(ctx.prior));
1688
+ const design = designOf(e);
1689
+ const regression = ctx.prior ? buildRegression(ctx.prior) : null;
1690
+ const chosen = chooseHeadline(e, arms, ctx.prior, regression);
1691
+ const primaryHeadline = fillTextLeadCites(chosen.primary, e, ctx.ids, ctx.linkBase, ctx.hrefs, ctx.hrefByKey);
1692
+ const headline = {
1693
+ task: displayName,
1694
+ kicker: chosen.kicker,
1695
+ fairness: chosen.fairness,
1696
+ primary: primaryHeadline,
1697
+ foot: chosen.foot
1698
+ };
1699
+ let primary = null;
1700
+ if (regression) primary = regression;
1701
+ else if (e.compare) primary = buildVariantComparison(e, arms);
1702
+ else primary = buildTaskResults(e);
1703
+ const insights = [];
1704
+ if (ctx.analysis) {
1705
+ const body = ctx.analysis.analysis;
1706
+ for (const claim of [
1707
+ ...body.whatHappened,
1708
+ ...body.results,
1709
+ ...body.why
1710
+ ]) insights.push({
1711
+ role: "finding",
1712
+ text: claim.text,
1713
+ confidence: claim.confidence,
1714
+ cites: [],
1715
+ action: null
1716
+ });
1717
+ for (const anomaly of body.anomalies) insights.push({
1718
+ role: "caveat",
1719
+ text: anomaly.text,
1720
+ confidence: "medium",
1721
+ cites: [],
1722
+ action: null
1723
+ });
1724
+ }
1725
+ return {
1726
+ id: e.key,
1727
+ anchor: gfmAnchor(displayName),
1728
+ name: displayName,
1729
+ file: e.file,
1730
+ shapes,
1731
+ design,
1732
+ headline,
1733
+ insights,
1734
+ primary,
1735
+ choices: buildChoices(e),
1736
+ discovery: buildDiscovery(e, ctx.facts),
1737
+ judgments: buildJudgments(e),
1738
+ qualitative: buildQualitative(e, ctx.ids, ctx.linkBase, ctx.hrefs, ctx.hrefByKey),
1739
+ captures: buildCaptures(e, ctx.ids, ctx.linkBase, ctx.hrefs, ctx.hrefByKey, ctx.pages),
1740
+ failures: buildFailures(e, ctx.ids, ctx.linkBase, ctx.hrefs, ctx.hrefByKey, ctx.remarks)
1741
+ };
1742
+ }
1743
+ function buildIntegrity(ctx) {
1744
+ const issues = [];
1745
+ const { record, evals, ids, linkBase } = ctx;
1746
+ if (record.status !== "complete") issues.push({
1747
+ kind: "run-incomplete",
1748
+ severity: "error",
1749
+ text: `Run stopped early (${record.status}); pass rates and medians cover only the attempts that finished.`,
1750
+ evalId: null,
1751
+ cites: []
1752
+ });
1753
+ if (record.git?.dirty) issues.push({
1754
+ kind: "dirty-worktree",
1755
+ severity: "warn",
1756
+ text: "The worktree was dirty; an exact rerun needs the saved worktree patch.",
1757
+ evalId: null,
1758
+ cites: []
1759
+ });
1760
+ for (const e of evals) {
1761
+ const cites = [];
1762
+ for (const a of e.attempts) {
1763
+ if (!a.turns.some((t) => t.status === "error")) continue;
1764
+ const ref = attemptRef(ids, e, a, linkBase, ctx.hrefs, ctx.hrefByKey);
1765
+ if (ref) cites.push(ref);
1766
+ }
1767
+ if (cites.length) issues.push({
1768
+ kind: "turn-error",
1769
+ severity: "error",
1770
+ text: `${cites.length} attempt${cites.length === 1 ? "" : "s"} had a mid-turn error; those outcomes may not reflect a finished agent turn.`,
1771
+ evalId: e.key,
1772
+ cites
1773
+ });
1774
+ const envNames = /* @__PURE__ */ new Map();
1775
+ for (const c of e.cells) {
1776
+ const set = envNames.get(c.environment) ?? /* @__PURE__ */ new Set();
1777
+ set.add(c.environmentDigest ?? c.environment);
1778
+ envNames.set(c.environment, set);
1779
+ }
1780
+ for (const [name, digests] of envNames) if (digests.size > 1) issues.push({
1781
+ kind: "duplicate-environment",
1782
+ severity: "warn",
1783
+ text: `Environment name "${name}" maps to ${digests.size} digests; arm labels include the digest prefix.`,
1784
+ evalId: e.key,
1785
+ cites: []
1786
+ });
1787
+ if (e.compare) {
1788
+ for (const c of e.cells.filter((x) => x.role === "control")) for (const check of c.checks ?? []) if (check.pass + check.fail > 0 && check.pass === 0) {
1789
+ if (e.cells.find((v) => v.role === "variant" && v.harness === c.harness && v.sandbox === c.sandbox && (v.checks ?? []).some((k) => k.label === check.label && k.pass === k.pass + k.fail && k.pass > 0))) issues.push({
1790
+ kind: "baseline-never-passes",
1791
+ severity: "warn",
1792
+ text: `Check "${check.label}" never passes on the baseline and always passes on a variant; it may not apply to the baseline.`,
1793
+ evalId: e.key,
1794
+ cites: []
1795
+ });
1796
+ }
1797
+ }
1798
+ for (const a of e.attempts) if (!a.usage && countedOutcome(a.outcome)) {}
1799
+ const unreported = e.attempts.filter((a) => countedOutcome(a.outcome) && !a.usage);
1800
+ if (unreported.length) issues.push({
1801
+ kind: "unreported-usage",
1802
+ severity: "warn",
1803
+ text: `${unreported.length} counted attempt${unreported.length === 1 ? "" : "s"} reported no token usage; token medians omit them.`,
1804
+ evalId: e.key,
1805
+ cites: unreported.flatMap((a) => {
1806
+ const ref = attemptRef(ids, e, a, linkBase, ctx.hrefs, ctx.hrefByKey);
1807
+ return ref ? [ref] : [];
1808
+ })
1809
+ });
1810
+ const preHits = e.attempts.flatMap((a) => (a.assertions ?? []).filter((x) => x.preSatisfied).map((x) => ({
1811
+ attempt: a,
1812
+ label: x.label
1813
+ })));
1814
+ if (preHits.length) {
1815
+ const labels = [...new Set(preHits.map((h) => h.label))];
1816
+ issues.push({
1817
+ kind: "pre-satisfied",
1818
+ severity: "warn",
1819
+ text: `${preHits.length} pre-satisfied check${preHits.length === 1 ? "" : "s"} (${labels.join(", ")}); existing workspace state may satisfy them without agent work.`,
1820
+ evalId: e.key,
1821
+ cites: preHits.flatMap((h) => {
1822
+ const ref = attemptRef(ids, e, h.attempt, linkBase, ctx.hrefs, ctx.hrefByKey);
1823
+ return ref ? [ref] : [];
1824
+ })
1825
+ });
1826
+ }
1827
+ }
1828
+ if (ctx.prior) {
1829
+ const sampleNs = ctx.prior.rows.flatMap((row) => (row.perCell ?? []).map((c) => Math.min(c.base.pass + c.base.fail, c.head.pass + c.head.fail)));
1830
+ if (ctx.prior.rows.some((row) => (row.status === "regressed" || row.status === "fixed") && Boolean(row.gate) && (row.perCell ?? []).some((c) => Math.min(c.base.pass + c.base.fail, c.head.pass + c.head.fail) < 4)) || sampleNs.length && Math.min(...sampleNs) < 4) {
1831
+ const minN = sampleNs.length ? Math.min(...sampleNs) : 0;
1832
+ issues.push({
1833
+ kind: "fairness-floor",
1834
+ severity: "warn",
1835
+ text: `A gated regression claim rests on fewer than 4 counted attempts per side${minN ? ` (as low as ${minN})` : ""}; treat worse/fixed as provisional.`,
1836
+ evalId: null,
1837
+ cites: []
1838
+ });
1839
+ }
1840
+ }
1841
+ return issues;
1842
+ }
1843
+ function buildRoster(evals, record) {
1844
+ if (evals.length < 2) return [];
1845
+ const harnesses = record.stack.harnesses;
1846
+ if (harnesses.length < 2) return [];
1847
+ return harnesses.map((h) => {
1848
+ const attempts = evals.flatMap((e) => e.attempts.filter((a) => cellOf(e, a.cell)?.harness === h.id && a.seq !== null));
1849
+ const gateways = [...new Set(evals.flatMap((e) => e.cells.filter((c) => c.harness === h.id).map((c) => c.gateway).filter(Boolean)))];
1850
+ return {
1851
+ harness: h.id,
1852
+ model: h.model ?? null,
1853
+ effort: h.effort ?? null,
1854
+ servedModels: h.resolvedModels ?? [],
1855
+ gateway: gateways[0] ?? null,
1856
+ attempts: attempts.length
1857
+ };
1858
+ });
1859
+ }
1860
+ function buildRollup(sections, analysis) {
1861
+ if (sections.length < 2 && !analysis?.analysis.tldr) return null;
1862
+ const lead = analysis?.analysis.tldr ? {
1863
+ role: "finding",
1864
+ text: analysis.analysis.tldr,
1865
+ confidence: analysis.analysis.confidence.overall,
1866
+ cites: [],
1867
+ action: null
1868
+ } : null;
1869
+ const findings = analysis ? [...analysis.analysis.whatHappened, ...analysis.analysis.results].map((c) => ({
1870
+ role: "finding",
1871
+ text: c.text,
1872
+ confidence: c.confidence,
1873
+ cites: [],
1874
+ action: null
1875
+ })) : [];
1876
+ return {
1877
+ lead,
1878
+ rows: sections.map((s) => ({
1879
+ evalId: s.id,
1880
+ name: s.name,
1881
+ anchor: s.anchor,
1882
+ shapes: s.shapes,
1883
+ design: s.design,
1884
+ primary: headlinePrimaryText(s.headline.primary),
1885
+ medianTokens: s.headline.primary.kind === "measure" && s.headline.primary.unit === "tokens" ? s.headline.primary.spread.median : tokensFigure(s.primary && "breakdown" in s.primary ? s.primary.median?.measures.tokens?.median.value ?? s.primary.breakdown[0]?.measures.tokens?.median.value ?? null : null)
1886
+ })),
1887
+ findings
1888
+ };
1889
+ }
1890
+ function buildActions(analysis) {
1891
+ if (!analysis) return [];
1892
+ return analysis.analysis.actions.map((a) => ({
1893
+ role: "action",
1894
+ text: a.title,
1895
+ confidence: analysis.analysis.confidence.overall,
1896
+ cites: [],
1897
+ action: {
1898
+ kind: a.kind,
1899
+ target: a.target,
1900
+ change: a.change,
1901
+ verify: a.verify
1902
+ }
1903
+ }));
1904
+ }
1905
+ function buildAttemptIndex(ctx) {
1906
+ const { evals, ids, linkBase, record, hrefByKey } = ctx;
1907
+ const rows = [];
1908
+ for (const e of evals) for (const a of e.attempts) {
1909
+ if (a.seq === null) continue;
1910
+ const id = ids.get(a.key);
1911
+ if (!id) continue;
1912
+ const cell = cellOf(e, a.cell);
1913
+ const href = hrefByKey.get(a.key) ?? attemptHref(e, a, linkBase, ctx.hrefs);
1914
+ const diffStats = attemptDiffStats(a);
1915
+ const reclassified = a.outcome === "infra_error" && (a.turnStatusRead === false || a.turnStatusRead === void 0 && a.turns.some((t) => t.status === "error"));
1916
+ const canceled = a.outcome === "fail" && a.error?.kind === "aborted" && record.status === "interrupted";
1917
+ rows.push({
1918
+ id,
1919
+ key: a.key,
1920
+ evalId: e.key,
1921
+ harness: cell?.harness ?? a.cell,
1922
+ arm: cell?.environment ?? null,
1923
+ case: a.case ?? null,
1924
+ outcome: canceled ? "canceled" : a.outcome,
1925
+ reclassified,
1926
+ turnError: turnErrorMessage(a),
1927
+ tokens: tokensFigure(a.usage ? totalTokens(a.usage) : null),
1928
+ agentMs: durationFigure(a.timings.agentMs),
1929
+ toolCalls: countFigure(a.activity.tools),
1930
+ ...ctx.facts?.has(a.key) ? { facts: ctx.facts.get(a.key) } : {},
1931
+ diff: diffStats.files === 0 && diffStats.added === null ? null : {
1932
+ files: diffStats.files,
1933
+ added: diffStats.added,
1934
+ removed: diffStats.removed,
1935
+ text: diffStats.added === null || diffStats.removed === null ? `${diffStats.files} files (lines not recorded)` : `${diffStats.files} files, ${diffLinesText(diffStats.added, diffStats.removed)}`
1936
+ },
1937
+ links: {
1938
+ transcript: a.files.transcript ? withBase(linkBase, a.files.transcript.path) : null,
1939
+ diff: a.files.diff ? withBase(linkBase, a.files.diff.path) : null,
1940
+ captures: (a.files.artifacts ?? []).map((art) => ({
1941
+ name: art.name,
1942
+ href: withBase(linkBase, art.path)
1943
+ })),
1944
+ questions: (a.answers ?? []).map((q) => q.input?.path).filter((p) => Boolean(p)).map((p) => withBase(linkBase, p))
1945
+ },
1946
+ href
1947
+ });
1948
+ }
1949
+ return rows.sort((a, b) => Number(a.id.slice(1)) - Number(b.id.slice(1)));
1950
+ }
1951
+ /** The only reader of a `RunRecord` for a report. Pure and deterministic. */
1952
+ function buildRunReport(record, options = {}) {
1953
+ const evals = selectedEvals(record, options.evals);
1954
+ if (evals.length === 0) throw new Error("buildRunReport: no evals selected");
1955
+ const names = uniqueEvalNames(evals);
1956
+ const ctx = {
1957
+ record,
1958
+ evals,
1959
+ ids: attemptIds(record, new Set(evals.map((e) => e.key))),
1960
+ linkBase: options.links?.base,
1961
+ remarks: options.remarks ?? null,
1962
+ analysis: options.analysis ?? null,
1963
+ pages: options.pages,
1964
+ facts: options.facts,
1965
+ prior: options.prior ?? null,
1966
+ hrefs: /* @__PURE__ */ new Set(),
1967
+ hrefByKey: /* @__PURE__ */ new Map()
1968
+ };
1969
+ assignAttemptHrefs(ctx);
1970
+ const header = buildHeader(ctx);
1971
+ if (options.title) header.title = options.title;
1972
+ else header.title = runTitle(record, evals);
1973
+ const sections = evals.map((e) => buildEvalSection(ctx, e, names.get(e.key) ?? e.name));
1974
+ const single = sections.length === 1;
1975
+ const rollup = buildRollup(sections, ctx.analysis);
1976
+ const roster = buildRoster(evals, record);
1977
+ return {
1978
+ reportVersion: 1,
1979
+ runId: record.runId,
1980
+ single,
1981
+ header,
1982
+ integrity: buildIntegrity(ctx),
1983
+ rollup: single && !ctx.analysis?.analysis.tldr ? null : rollup,
1984
+ roster,
1985
+ evals: sections,
1986
+ actions: buildActions(ctx.analysis),
1987
+ provenance: provenanceState(record, options.links?.record),
1988
+ attempts: buildAttemptIndex(ctx)
1989
+ };
1990
+ }
1991
+ // src/view/report/split.ts
1992
+ /** The newest `reportVersion` this reeval writes and reads. */
1993
+ const REPORT_VERSION = 1;
1994
+ /** The attempt index's order: by the number in each row's display id, `A1..AN`. */
1995
+ const inIdOrder = (a, b) => Number(a.id.slice(1)) - Number(b.id.slice(1));
1996
+ /**
1997
+ * `report` as its index and one detail per eval, which `joinRunReport` puts back together exactly.
1998
+ * Each eval's `id` is its own, each attempt index row names its eval (`AttemptIndexRow.evalId`,
1999
+ * the section's `id`) and the rows are in their display ids' order, as `buildRunReport` writes
2000
+ * them; a report that isn't throws.
2001
+ */
2002
+ function splitRunReport(report) {
2003
+ const { evals, attempts, ...top } = report;
2004
+ const rows = /* @__PURE__ */ new Map();
2005
+ for (const e of evals) {
2006
+ if (rows.has(e.id)) throw new Error(`Eval ${e.id} is in the report twice`);
2007
+ rows.set(e.id, []);
2008
+ }
2009
+ attempts.forEach((row, i) => {
2010
+ const of = rows.get(row.evalId);
2011
+ if (!of) throw new Error(`Attempt ${row.key} names eval ${row.evalId}, which the report lacks`);
2012
+ const prev = attempts[i - 1];
2013
+ if (prev && inIdOrder(prev, row) >= 0) throw new Error(`Attempt ${row.id} is out of the index's order`);
2014
+ of.push(row);
2015
+ });
2016
+ const details = evals.map((e) => ({
2017
+ reportVersion: report.reportVersion,
2018
+ runId: report.runId,
2019
+ eval: e,
2020
+ attempts: rows.get(e.id) ?? []
2021
+ }));
2022
+ return {
2023
+ index: {
2024
+ ...top,
2025
+ evals: details.map(({ eval: e, attempts }) => ({
2026
+ id: e.id,
2027
+ anchor: e.anchor,
2028
+ name: e.name,
2029
+ file: e.file,
2030
+ shapes: e.shapes,
2031
+ design: e.design,
2032
+ headline: e.headline,
2033
+ attempts: attempts.length
2034
+ }))
2035
+ },
2036
+ details
2037
+ };
2038
+ }
2039
+ /** The `RunReport` that `splitRunReport` split into `index` and `details`, keys in its order. */
2040
+ function joinRunReport(index, details) {
2041
+ const byId = new Map(details.map((d) => [d.eval.id, d]));
2042
+ const detailOf = (id) => {
2043
+ const d = byId.get(id);
2044
+ if (!d) throw new Error(`No detail for eval ${id}`);
2045
+ return d;
2046
+ };
2047
+ const { evals, actions, provenance, ...head } = index;
2048
+ return {
2049
+ ...head,
2050
+ evals: evals.map((e) => detailOf(e.id).eval),
2051
+ actions,
2052
+ provenance,
2053
+ attempts: evals.flatMap((e) => detailOf(e.id).attempts).sort(inIdOrder)
2054
+ };
2055
+ }
2056
+ export { worktreePatchLink as _, provenanceState as a, REPORT_POLICY as c, captureHref as d, evalAnchorHref as f, withBase as g, recordHref as h, buildRunReport as i, attemptHref as l, questionHref as m, joinRunReport as n, stateRows as o, gfmAnchor as p, splitRunReport as r, warnings as s, REPORT_VERSION as t, attemptRef as u, runScope as v };
2057
+
2058
+ //# sourceMappingURL=core-BXYzOEyQ.js.map