reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -1,8 +1,56 @@
1
1
  import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-BmR-7tx3.js";
2
- import { n as clip } from "./text-CrVy5ioL.js";
3
- import { a as totalTokens, i as sumUsage, n as formatTokens, r as promptTokens } from "./usage-CleNKkly.js";
4
- import { l as readableTranscriptLine } from "./record-kk9M1hj-.js";
5
- import { s as sampleReasons } from "./pairs-DP6XOJby.js";
2
+ import { a as plural, n as clip } from "./text-DTigcA64.js";
3
+ import { l as sampleReasons } from "./pairs-puYi-zj2.js";
4
+ import { c as minimumP, d as round3, i as mannWhitney, n as containsZero, r as liftInterval, s as median, u as quantile } from "./stats-hfpuYsAM.js";
5
+ import { a as totalTokens, i as sumUsage, n as formatTokens, r as promptTokens } from "./usage-BNY1ABrU.js";
6
+ import { l as readableTranscriptLine } from "./record-DZBCjuwm.js";
7
+ //#region src/view/gate.ts
8
+ /** `all`, `none` or `passRate 0.5`; `{ passRate: 1 }` is `all`, which scores the same. */
9
+ const gateLabel = (gate) => typeof gate === "string" ? gate : gate.passRate === 1 ? "all" : `passRate ${gate.passRate}`;
10
+ /** The gate's part of a fingerprint: none on `'all'`, so such a cell keeps its fingerprint. */
11
+ const gatePart = (gate) => {
12
+ const label = gateLabel(gate);
13
+ return label === "all" ? {} : { gate: label };
14
+ };
15
+ /** Any failure fails; then infra errors; then all-skipped; else pass. */
16
+ function worstOutcome(outcomes) {
17
+ if (outcomes.includes("fail")) return "fail";
18
+ if (outcomes.includes("infra_error")) return "infra_error";
19
+ if (outcomes.length > 0 && outcomes.every((o) => o === "skipped")) return "skipped";
20
+ return outcomes.length ? "pass" : "skipped";
21
+ }
22
+ const passRateOf = (outcomes) => {
23
+ const pass = outcomes.filter((o) => o === "pass").length;
24
+ const counted = pass + outcomes.filter((o) => o === "fail").length;
25
+ return counted ? pass / counted : null;
26
+ };
27
+ /**
28
+ * A cell's outcome by its gate. `'all'` matches `{ passRate: 1 }`: every counted attempt must
29
+ * pass, and it is an infra error only when nothing counted. `'none'` takes the worst attempt.
30
+ */
31
+ function cellOutcome(outcomes, gate) {
32
+ if (gate === "none") return worstOutcome(outcomes);
33
+ const required = gate === "all" ? 1 : gate.passRate;
34
+ const rate = passRateOf(outcomes);
35
+ if (rate === null) return worstOutcome(outcomes);
36
+ return rate >= required ? "pass" : "fail";
37
+ }
38
+ /** The cell's attempts, less the fails the run's interrupt canceled, which the agent didn't fail. */
39
+ const outcomesOf = (c) => [
40
+ ...Array(c.pass).fill("pass"),
41
+ ...Array(c.fail - (c.canceled ?? 0)).fill("fail"),
42
+ ...Array(c.infraError).fill("infra_error"),
43
+ ...Array(c.skipped).fill("skipped")
44
+ ];
45
+ /**
46
+ * A cell's recorded attempts scored under another gate. A cell its own gate passes but that
47
+ * failed anyway fell short of its `minLift`, which no gate changes, so it still fails.
48
+ */
49
+ function rescore(c, gate) {
50
+ const outcomes = outcomesOf(c);
51
+ return c.outcome === "fail" && cellOutcome(outcomes, c.gate) === "pass" ? "fail" : cellOutcome(outcomes, gate);
52
+ }
53
+ //#endregion
6
54
  //#region src/view/compare.ts
7
55
  const STACK_FIELDS = [
8
56
  {
@@ -125,11 +173,32 @@ const PART_LABELS = {
125
173
  image: "image",
126
174
  agentCli: "agent CLI",
127
175
  judge: "judge",
128
- context: "agent context"
176
+ context: "agent context",
177
+ gateway: "gateway"
129
178
  };
130
- /** Why two runs of an eval aren't comparable: the source files and fingerprint parts that differ. */
179
+ const showRevision = (r) => r === void 0 ? "unset" : JSON.stringify(r);
180
+ const SOURCE_LABELS = {
181
+ body: "body",
182
+ options: "options",
183
+ case: "case data"
184
+ };
185
+ /** What in the eval's own source differs: its body, options, case data and revision. */
186
+ function sourceChanged(base, head) {
187
+ const out = [];
188
+ if (base.source && head.source) {
189
+ for (const key of Object.keys(SOURCE_LABELS)) if (base.source[key] !== head.source[key]) out.push(`${SOURCE_LABELS[key]} changed`);
190
+ } else if (base.source || head.source) out.push("fingerprint scheme changed");
191
+ else out.push(...sourceChanges(base.sources, head.sources));
192
+ if (base.revision !== head.revision) out.push(`revision ${showRevision(base.revision)} → ${showRevision(head.revision)}`);
193
+ return out;
194
+ }
195
+ /**
196
+ * Why two runs of an eval aren't comparable: what in its source differs (`body`, `options`,
197
+ * `case data`, `revision 2 → 3`; a record from before `source`, its files), then the fingerprint
198
+ * parts that differ. Each reason but the revision ends in ` changed`.
199
+ */
131
200
  function whyChanged(base, head) {
132
- const out = sourceChanges(base.sources, head.sources);
201
+ const out = sourceChanged(base, head);
133
202
  const keys = [...new Set([...base.cells, ...head.cells].map((c) => c.key))];
134
203
  const parts = /* @__PURE__ */ new Set();
135
204
  for (const key of keys) {
@@ -140,9 +209,48 @@ function whyChanged(base, head) {
140
209
  continue;
141
210
  }
142
211
  for (const part of Object.keys(PART_LABELS)) if (b.fingerprintParts[part] !== h.fingerprintParts[part]) parts.add(PART_LABELS[part]);
212
+ if (gateLabel(b.gate) !== gateLabel(h.gate)) parts.add("gate");
143
213
  }
144
214
  return [...out, ...[...parts].map((p) => `${p} changed`)];
145
215
  }
216
+ const withoutGate = ({ gate: _, ...parts }) => JSON.stringify(parts);
217
+ /**
218
+ * Whether two runs of a cell are comparable: the same fingerprint, the gate aside. A record from
219
+ * before the gate was a part has none, so its parts are compared without it.
220
+ */
221
+ const sameCell = (b, h) => b.fingerprint === h.fingerprint || (b.fingerprintParts.gate !== void 0 || h.fingerprintParts.gate !== void 0) && withoutGate(b.fingerprintParts) === withoutGate(h.fingerprintParts);
222
+ /** Whether two runs of an eval ran the same cells, each comparable. */
223
+ function sameEval(b, h) {
224
+ if (b.fingerprint === h.fingerprint) return true;
225
+ if (b.cells.length !== h.cells.length) return false;
226
+ const before = new Map(b.cells.map((c) => [c.key, c]));
227
+ return h.cells.every((c) => {
228
+ const was = before.get(c.key);
229
+ return was !== void 0 && sameCell(was, c);
230
+ });
231
+ }
232
+ /**
233
+ * When the gate of a cell both sides ran changed: each side's gates, and base's outcome over
234
+ * those cells with each one's attempts scored under head's gate.
235
+ */
236
+ function gateChange(base, head) {
237
+ const before = new Map(base.map((c) => [c.key, c]));
238
+ const pairs = head.flatMap((h) => {
239
+ const b = before.get(h.key);
240
+ return b ? [[b, h]] : [];
241
+ });
242
+ const changed = pairs.filter(([b, h]) => gateLabel(b.gate) !== gateLabel(h.gate));
243
+ if (!changed.length) return void 0;
244
+ const labels = (side) => [...new Set(changed.map((p) => gateLabel(p[side].gate)))].join(", ");
245
+ return {
246
+ base: labels(0),
247
+ head: labels(1),
248
+ rescored: outcomeOf(pairs.map(([b, h]) => ({
249
+ gate: h.gate,
250
+ outcome: rescore(b, h.gate)
251
+ })))
252
+ };
253
+ }
146
254
  /** An eval's outcome from some of its cells, as the run rolls it up: ungated failures pass. */
147
255
  function outcomeOf(cells) {
148
256
  const outcomes = cells.map((c) => c.gate === "none" && c.outcome === "fail" ? "pass" : c.outcome);
@@ -151,16 +259,16 @@ function outcomeOf(cells) {
151
259
  return outcomes.every((o) => o === "skipped") ? "skipped" : "pass";
152
260
  }
153
261
  /**
154
- * Whether a failing eval failed only on attempts the run's interrupt cancelled: every failed
155
- * attempt in a gated cell was cut short, so the agent didn't fail it (`reeval show`'s `cancelled`).
262
+ * Whether a failing eval failed only on attempts the run's interrupt canceled: every failed
263
+ * attempt in a gated cell was cut short, so the agent didn't fail it (`reeval show`'s `canceled`).
156
264
  */
157
- const cancelledOn = (outcome, cells) => {
265
+ const canceledOn = (outcome, cells) => {
158
266
  const gated = cells.filter((c) => c.gate !== "none");
159
- return outcome === "fail" && gated.some((c) => c.fail > 0) && gated.every((c) => c.fail === (c.cancelled ?? 0));
267
+ return outcome === "fail" && gated.some((c) => c.fail > 0) && gated.every((c) => c.fail === (c.canceled ?? 0));
160
268
  };
161
- function cancelledSides(outcomes, cells) {
162
- const sides = ["base", "head"].filter((s) => cancelledOn(outcomes[s], cells[s] ?? []));
163
- return sides.length ? { cancelled: sides } : {};
269
+ function canceledSides(outcomes, cells) {
270
+ const sides = ["base", "head"].filter((s) => canceledOn(outcomes[s], cells[s] ?? []));
271
+ return sides.length ? { canceled: sides } : {};
164
272
  }
165
273
  /** An eval only head has. */
166
274
  const addedRow = (h) => ({
@@ -168,7 +276,7 @@ const addedRow = (h) => ({
168
276
  base: null,
169
277
  head: h.outcome,
170
278
  status: "added",
171
- ...cancelledSides({
279
+ ...canceledSides({
172
280
  base: null,
173
281
  head: h.outcome
174
282
  }, { head: h.cells })
@@ -181,7 +289,7 @@ const addedRow = (h) => ({
181
289
  function sharedScope(b, h) {
182
290
  const before = new Map(b.cells.map((c) => [c.key, c]));
183
291
  const shared = h.cells.filter((c) => before.has(c.key));
184
- if (!shared.length || shared.some((c) => before.get(c.key)?.fingerprint !== c.fingerprint)) return null;
292
+ if (!shared.length || shared.some((c) => !sameCell(before.get(c.key) ?? c, c))) return null;
185
293
  const keys = new Set(h.cells.map((c) => c.key));
186
294
  const baseCells = shared.map((c) => before.get(c.key) ?? c);
187
295
  return {
@@ -215,8 +323,134 @@ function compareCells(base, head) {
215
323
  }];
216
324
  });
217
325
  }
326
+ /** A family with a gate: some pooled lift has a `minLift`. One without can't fail, so has no row. */
327
+ const gated = (f) => !!f?.lifts.some((l) => l.minLift !== void 0);
328
+ const liftSide = ({ variant: _, vs: __, ...side }) => side;
329
+ /** Each variant's pooled lift on both sides, head's order first. */
330
+ function familyLifts(b, h) {
331
+ const out = /* @__PURE__ */ new Map();
332
+ for (const [side, f] of [["head", h], ["base", b]]) for (const l of f?.lifts ?? []) {
333
+ const row = out.get(l.variant) ?? {
334
+ key: l.variant,
335
+ vs: l.vs,
336
+ base: null,
337
+ head: null
338
+ };
339
+ row[side] = liftSide(l);
340
+ out.set(l.variant, row);
341
+ }
342
+ return [...out.values()];
343
+ }
344
+ const liftsOutcome = (lifts) => lifts.some((l) => l.outcome === "fail") ? "fail" : "pass";
345
+ /** A gated family only head has. */
346
+ const addedFamilyRow = (h) => ({
347
+ id: h.id,
348
+ kind: "family",
349
+ base: null,
350
+ head: h.outcome,
351
+ status: "added",
352
+ lifts: familyLifts(null, h)
353
+ });
354
+ /** The rows of a run with nothing to compare it with: every eval and gated family is new. */
355
+ const newRows = (head) => [...head.evals.map(addedRow), ...(head.families ?? []).filter(gated).map(addedFamilyRow)];
356
+ /**
357
+ * A gated family's row. Its pooled outcome is comparable only when both runs ran the same cases
358
+ * and every case is: a case that changed or hit an infrastructure error makes the family `drift`
359
+ * or `infra` as it does the case, and a failing side with a canceled attempt in any case reads
360
+ * `canceled`, since the pooled lift counts it. With different variants, it's over the lifts both
361
+ * have, and with none in common it can't compare them.
362
+ */
363
+ function familyRow(id, b, h, ctx) {
364
+ if (!b) return addedFamilyRow(h);
365
+ const lifts = familyLifts(b, h ?? null);
366
+ if (!h) return {
367
+ id,
368
+ kind: "family",
369
+ base: b.outcome,
370
+ head: null,
371
+ status: "removed",
372
+ lifts
373
+ };
374
+ const shared = lifts.flatMap((l) => l.base && l.head ? [{
375
+ base: l.base,
376
+ head: l.head
377
+ }] : []);
378
+ const scoped = shared.length > 0 && shared.length < lifts.length;
379
+ const disjoint = !shared.length && b.lifts.length > 0 && h.lifts.length > 0;
380
+ const row = {
381
+ id,
382
+ kind: "family",
383
+ base: scoped ? liftsOutcome(shared.map((l) => l.base)) : b.outcome,
384
+ head: scoped ? liftsOutcome(shared.map((l) => l.head)) : h.outcome,
385
+ lifts,
386
+ ...scoped || disjoint ? { cells: {
387
+ added: lifts.filter((l) => !l.base).map((l) => l.key),
388
+ removed: lifts.filter((l) => !l.head).map((l) => l.key)
389
+ } } : {}
390
+ };
391
+ const prefix = id.slice(0, id.lastIndexOf(" › ") + 3);
392
+ const name = (caseId) => caseId.startsWith(prefix) ? caseId.slice(prefix.length) : caseId;
393
+ const before = new Set(b.caseIds);
394
+ const after = new Set(h.caseIds);
395
+ const caseRows = [.../* @__PURE__ */ new Set([...h.caseIds, ...b.caseIds])].flatMap((c) => ctx.rows.get(c) ?? []);
396
+ const changes = [.../* @__PURE__ */ new Set([
397
+ ...h.caseIds.filter((c) => !before.has(c)).map((c) => `case ${name(c)} added`),
398
+ ...b.caseIds.filter((c) => !after.has(c)).map((c) => `case ${name(c)} removed`),
399
+ ...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])
400
+ ])];
401
+ const canceled = ["base", "head"].filter((s) => row[s] === "fail" && ctx.canceledIn(s, s === "base" ? b.caseIds : h.caseIds));
402
+ if (canceled.length) row.canceled = canceled;
403
+ if (caseRows.some((r) => r.status === "infra")) return {
404
+ ...row,
405
+ status: "infra"
406
+ };
407
+ if (changes.length) return {
408
+ ...row,
409
+ status: "drift",
410
+ changes
411
+ };
412
+ if (disjoint || row.canceled || caseRows.some((r) => r.status === "drift")) return {
413
+ ...row,
414
+ status: "drift"
415
+ };
416
+ if (ctx.graderChanged && row.base !== row.head) return {
417
+ ...row,
418
+ status: "drift"
419
+ };
420
+ if (row.base === "pass" && row.head === "fail") return {
421
+ ...row,
422
+ status: "regressed"
423
+ };
424
+ if (row.base === "fail" && row.head === "pass") return {
425
+ ...row,
426
+ status: "fixed"
427
+ };
428
+ return {
429
+ ...row,
430
+ status: "same"
431
+ };
432
+ }
433
+ /** Each eval's row, then each family's that either run gated. */
218
434
  function compare(base, head) {
219
435
  const graderChanged = stackChanges(base, head).some((c) => c.outsideFingerprint);
436
+ const rows = evalRows(base, head, graderChanged);
437
+ const before = new Map((base.families ?? []).map((f) => [f.id, f]));
438
+ const after = new Map((head.families ?? []).map((f) => [f.id, f]));
439
+ const families = [.../* @__PURE__ */ new Set([...after.keys(), ...before.keys()])].filter((id) => gated(before.get(id)) || gated(after.get(id)));
440
+ const runs = {
441
+ base,
442
+ head
443
+ };
444
+ const ctx = {
445
+ rows: new Map(rows.map((r) => [r.id, r])),
446
+ graderChanged,
447
+ canceledIn: (side, caseIds) => runs[side].evals.some((e) => caseIds.includes(e.id) && e.cells.some((c) => (c.canceled ?? 0) > 0))
448
+ };
449
+ return [...rows, ...families.map((id) => familyRow(id, before.get(id), after.get(id), ctx))];
450
+ }
451
+ /** The key that tells a family's row from an eval's, whose ids can coincide. */
452
+ const rowKey = (row) => row.kind === "family" ? `family\0${row.id}` : row.id;
453
+ function evalRows(base, head, graderChanged) {
220
454
  return [.../* @__PURE__ */ new Set([...head.evals.map((e) => e.id), ...base.evals.map((e) => e.id)])].map((id) => {
221
455
  const b = base.evals.find((e) => e.id === id);
222
456
  const h = head.evals.find((e) => e.id === id);
@@ -226,12 +460,13 @@ function compare(base, head) {
226
460
  base: b.outcome,
227
461
  head: null,
228
462
  status: "removed",
229
- ...cancelledSides({
463
+ ...canceledSides({
230
464
  base: b.outcome,
231
465
  head: null
232
466
  }, { base: b.cells })
233
467
  };
234
- const scoped = b.fingerprint === h.fingerprint ? null : sharedScope(b, h);
468
+ const comparable = sameEval(b, h);
469
+ const scoped = comparable ? null : sharedScope(b, h);
235
470
  const outcomes = scoped ? {
236
471
  base: scoped.base,
237
472
  head: scoped.head
@@ -243,7 +478,7 @@ function compare(base, head) {
243
478
  id,
244
479
  ...outcomes,
245
480
  ...scoped ? { cells: scoped.cells } : {},
246
- ...cancelledSides(outcomes, scoped?.over ?? {
481
+ ...canceledSides(outcomes, scoped?.over ?? {
247
482
  base: b.cells,
248
483
  head: h.cells
249
484
  })
@@ -254,346 +489,44 @@ function compare(base, head) {
254
489
  ...row,
255
490
  status: "infra"
256
491
  };
257
- if (b.fingerprint !== h.fingerprint && !scoped) return {
492
+ if (!comparable && !scoped) return {
258
493
  ...row,
259
494
  status: "drift",
260
495
  changes: whyChanged(b, h)
261
496
  };
262
- if (row.cancelled) return {
497
+ const over = scoped?.over ?? {
498
+ base: b.cells ?? [],
499
+ head: h.cells ?? []
500
+ };
501
+ const gate = gateChange(over.base, over.head);
502
+ if (gate) Object.assign(row, {
503
+ changes: ["gate changed"],
504
+ gate
505
+ });
506
+ const from = gate?.rescored ?? row.base;
507
+ if (row.canceled) return {
263
508
  ...row,
264
509
  status: "drift"
265
510
  };
266
- if (graderChanged && row.base !== row.head) return {
511
+ if (graderChanged && from !== row.head) return {
267
512
  ...row,
268
513
  status: "drift"
269
514
  };
270
- if (row.base === "pass" && row.head === "fail") return {
515
+ if (from === "pass" && row.head === "fail") return {
271
516
  ...row,
272
517
  status: "regressed"
273
518
  };
274
- if (row.base === "fail" && row.head === "pass") return {
519
+ if (from === "fail" && row.head === "pass") return {
275
520
  ...row,
276
521
  status: "fixed"
277
522
  };
278
523
  return {
279
524
  ...row,
280
- status: "same"
525
+ status: row.base === row.head ? "same" : "gate"
281
526
  };
282
527
  });
283
528
  }
284
529
  //#endregion
285
- //#region src/stats.ts
286
- /** Two-sided 95%. */
287
- const Z = 1.959964;
288
- /**
289
- * The Wilson score interval for `pass` successes out of `n`. Unlike the normal approximation it
290
- * stays inside [0, 1] and is sensible at 0/n and n/n. With no attempts it's [0, 1].
291
- */
292
- function wilson(pass, n) {
293
- if (n === 0) return {
294
- low: 0,
295
- high: 1
296
- };
297
- const p = pass / n;
298
- const z2 = Z * Z;
299
- const denominator = 1 + z2 / n;
300
- const center = (p + z2 / (2 * n)) / denominator;
301
- const half = Z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n)) / denominator;
302
- return {
303
- low: Math.max(0, center - half),
304
- high: Math.min(1, center + half)
305
- };
306
- }
307
- /**
308
- * The difference of two pass rates (variant minus control), with Newcombe's hybrid score
309
- * interval: each rate's Wilson interval combined, so it inherits Wilson's behaviour at 0 and 1.
310
- */
311
- function liftInterval(variant, control) {
312
- const p1 = variant.n ? variant.pass / variant.n : 0;
313
- const p2 = control.n ? control.pass / control.n : 0;
314
- const a = wilson(variant.pass, variant.n);
315
- const b = wilson(control.pass, control.n);
316
- const points = p1 - p2;
317
- return {
318
- points,
319
- low: points - Math.sqrt((p1 - a.low) ** 2 + (b.high - p2) ** 2),
320
- high: points + Math.sqrt((a.high - p1) ** 2 + (p2 - b.low) ** 2)
321
- };
322
- }
323
- /** A 95% interval is within noise when it contains 0; touching 0 does not exclude it. */
324
- const containsZero = (low, high) => low <= 0 && high >= 0;
325
- /** Rounded to three places, as the record stores rates and points. */
326
- const round3 = (x) => Math.round(x * 1e3) / 1e3;
327
- /** Linear-interpolated quantile (`q` from 0 to 1) of unsorted numbers; null for none. */
328
- function quantile(values, q) {
329
- if (values.length === 0) return null;
330
- const sorted = [...values].sort((a, b) => a - b);
331
- const at = (sorted.length - 1) * q;
332
- const lo = Math.floor(at);
333
- const hi = Math.ceil(at);
334
- const a = sorted[lo];
335
- return a + (sorted[hi] - a) * (at - lo);
336
- }
337
- const median = (values) => quantile(values, .5);
338
- /** Above this many ways to split the pooled values, the normal approximation stands in. */
339
- const EXACT_LIMIT = 2e5;
340
- const choose = (n, k) => {
341
- let r = 1;
342
- for (let i = 1; i <= k; i++) r = r * (n - k + i) / i;
343
- return Math.round(r);
344
- };
345
- /**
346
- * Two-sided Mann–Whitney (Wilcoxon rank-sum) p-value for `xs` against `ys`: exact, by every split
347
- * of the pooled values, with mid-ranks for ties; the normal approximation with a tie correction
348
- * above `EXACT_LIMIT` splits. `sign` is the sign of `xs`'s rank sum minus its expectation: below 0,
349
- * `xs` ranks lower. `floor` is the smallest p any split of these pooled values could give, so ties
350
- * count: `[2, 2, 2, 3]` against `[2, 2, 3, 3]` can never go below 10/70. Null when either side is
351
- * empty.
352
- */
353
- function mannWhitney(xs, ys) {
354
- const n1 = xs.length;
355
- const n2 = ys.length;
356
- if (n1 === 0 || n2 === 0) return null;
357
- const pooled = [...xs, ...ys].map((v, i) => ({
358
- v,
359
- i
360
- })).sort((a, b) => a.v - b.v);
361
- const ranks = new Array(pooled.length);
362
- for (let i = 0; i < pooled.length;) {
363
- let j = i;
364
- while (j + 1 < pooled.length && pooled[j + 1]?.v === pooled[i]?.v) j++;
365
- for (let k = i; k <= j; k++) ranks[pooled[k]?.i] = (i + j) / 2 + 1;
366
- i = j + 1;
367
- }
368
- const n = n1 + n2;
369
- const expected = n1 * (n + 1) / 2;
370
- const deviation = ranks.slice(0, n1).reduce((a, b) => a + b, 0) - expected;
371
- const sign = Math.sign(deviation);
372
- const observed = Math.abs(deviation);
373
- if (choose(n, n1) > EXACT_LIMIT) {
374
- const counts = /* @__PURE__ */ new Map();
375
- for (const { v } of pooled) counts.set(v, (counts.get(v) ?? 0) + 1);
376
- const ties = [...counts.values()].reduce((a, t) => a + t ** 3 - t, 0);
377
- const variance = n1 * n2 / 12 * (n + 1 - ties / (n * (n - 1)));
378
- if (variance <= 0) return {
379
- p: 1,
380
- sign,
381
- floor: 1
382
- };
383
- const pOf = (d) => Math.min(1, 2 * (1 - normalCdf(Math.max(0, d - .5) / Math.sqrt(variance))));
384
- const sorted = pooled.map(({ i }) => ranks[i]);
385
- const sum = (rs) => rs.reduce((a, b) => a + b, 0);
386
- const widest = Math.max(Math.abs(sum(sorted.slice(0, n1)) - expected), Math.abs(sum(sorted.slice(n - n1)) - expected));
387
- return {
388
- p: pOf(observed),
389
- sign,
390
- floor: pOf(widest)
391
- };
392
- }
393
- let extreme = 0;
394
- let widest = 0;
395
- let atWidest = 0;
396
- let total = 0;
397
- const pick = (start, left, sum) => {
398
- if (left === 0) {
399
- total++;
400
- const d = Math.abs(sum - expected);
401
- if (d >= observed - 1e-9) extreme++;
402
- if (d > widest + 1e-9) {
403
- widest = d;
404
- atWidest = 1;
405
- } else if (d >= widest - 1e-9) atWidest++;
406
- return;
407
- }
408
- for (let i = start; i <= n - left; i++) pick(i + 1, left - 1, sum + ranks[i]);
409
- };
410
- pick(0, n1, 0);
411
- return {
412
- p: extreme / total,
413
- sign,
414
- floor: atWidest / total
415
- };
416
- }
417
- /**
418
- * The smallest two-sided p an exact rank test can reach with these group sizes and no ties: for
419
- * planning `runs` before there's data. On data, `mannWhitney`'s `floor` counts the ties.
420
- */
421
- const minimumP = (n1, n2) => n1 === 0 || n2 === 0 ? 1 : Math.min(1, 2 / choose(n1 + n2, n1));
422
- /**
423
- * Counted attempts a side before a pass-rate lift can be called: below it, every view says
424
- * `insufficient` (`can't tell yet`, `unclear`), whatever the interval. A product floor that keeps
425
- * `reeval run`, `reeval query`, the judge-key lift and `reeval analyze` agreeing, not one that
426
- * makes the stated 95% exact: at 4 a side, a 4/4 against 0/4 split happens by chance about 7% of
427
- * the time when nothing changed.
428
- */
429
- const EFFECT_MIN_COUNTED = 4;
430
- /**
431
- * The difference of two paired pass rates (variant minus control) with Newcombe's score interval
432
- * for paired proportions (his method 10): each marginal rate's Wilson interval, combined with the
433
- * correlation between the sides, so concordant pairs narrow the interval.
434
- */
435
- function pairedLiftInterval(c) {
436
- const { both: a, variantOnly: b, controlOnly: k, neither: d } = c;
437
- const n = a + b + k + d;
438
- if (n === 0) return {
439
- points: 0,
440
- low: -1,
441
- high: 1
442
- };
443
- const p1 = (a + b) / n;
444
- const p2 = (a + k) / n;
445
- const v = wilson(a + b, n);
446
- const w = wilson(a + k, n);
447
- const denominator = (a + b) * (k + d) * (a + k) * (b + d);
448
- const numerator = a * d - b * k > 0 ? Math.max(a * d - b * k - n / 2, 0) : a * d - b * k;
449
- const phi = denominator === 0 ? 0 : numerator / Math.sqrt(denominator);
450
- const points = (b - k) / n;
451
- const dl = Math.sqrt(Math.max(0, (p1 - v.low) ** 2 - 2 * phi * (p1 - v.low) * (w.high - p2) + (w.high - p2) ** 2));
452
- const du = Math.sqrt(Math.max(0, (v.high - p1) ** 2 - 2 * phi * (v.high - p1) * (p2 - w.low) + (p2 - w.low) ** 2));
453
- return {
454
- points,
455
- low: Math.max(-1, points - dl),
456
- high: Math.min(1, points + du)
457
- };
458
- }
459
- /** McNemar's exact test: the two-sided binomial p of the discordant pairs' split at ½. */
460
- function mcnemar(variantOnly, controlOnly) {
461
- const n = variantOnly + controlOnly;
462
- if (n === 0) return 1;
463
- const low = Math.min(variantOnly, controlOnly);
464
- let logTerm = -n * Math.LN2;
465
- let logTail = logTerm;
466
- for (let i = 0; i < low; i++) {
467
- logTerm += Math.log((n - i) / (i + 1));
468
- const top = Math.max(logTail, logTerm);
469
- logTail = top + Math.log1p(Math.exp(Math.min(logTail, logTerm) - top));
470
- }
471
- return Math.min(1, 2 * Math.exp(logTail));
472
- }
473
- /** Two-sided 95% Student t quantiles for 1 to 30 degrees of freedom. */
474
- const T975 = [
475
- 12.706,
476
- 4.303,
477
- 3.182,
478
- 2.776,
479
- 2.571,
480
- 2.447,
481
- 2.365,
482
- 2.306,
483
- 2.262,
484
- 2.228,
485
- 2.201,
486
- 2.179,
487
- 2.16,
488
- 2.145,
489
- 2.131,
490
- 2.12,
491
- 2.11,
492
- 2.101,
493
- 2.093,
494
- 2.086,
495
- 2.08,
496
- 2.074,
497
- 2.069,
498
- 2.064,
499
- 2.06,
500
- 2.056,
501
- 2.052,
502
- 2.048,
503
- 2.045,
504
- 2.042
505
- ];
506
- /** The two-sided 95% t quantile; above 30 degrees of freedom, the Cornish–Fisher expansion. */
507
- function t975(df) {
508
- if (df < 1) return Number.POSITIVE_INFINITY;
509
- const tabled = T975[df - 1];
510
- if (tabled !== void 0) return tabled;
511
- return Z + (Z ** 3 + Z) / (4 * df) + (5 * Z ** 5 + 16 * Z ** 3 + 3 * Z) / (96 * df * df);
512
- }
513
- /**
514
- * The mean of `values` with a 95% t interval over them: with each value one case, attempts
515
- * within a case are never counted as independent. A single value gives an unbounded interval.
516
- */
517
- function meanInterval(values) {
518
- const n = values.length;
519
- if (n === 0) return {
520
- mean: 0,
521
- low: Number.NEGATIVE_INFINITY,
522
- high: Number.POSITIVE_INFINITY
523
- };
524
- const mean = values.reduce((s, v) => s + v, 0) / n;
525
- if (n === 1) return {
526
- mean,
527
- low: Number.NEGATIVE_INFINITY,
528
- high: Number.POSITIVE_INFINITY
529
- };
530
- const variance = values.reduce((s, v) => s + (v - mean) ** 2, 0) / (n - 1);
531
- const half = t975(n - 1) * Math.sqrt(variance / n);
532
- return {
533
- mean,
534
- low: mean - half,
535
- high: mean + half
536
- };
537
- }
538
- /**
539
- * Two-sided Wilcoxon signed-rank p for paired differences: zeros dropped, mid-ranks for ties,
540
- * exact over every sign assignment up to 20 non-zero differences, else the normal approximation.
541
- * `sign` is the sign of the positive differences' rank sum minus its expectation: below 0, the
542
- * negative differences outrank the positive ones. Null when every difference is zero. Ties are
543
- * exact equality, so form each difference of two rates with one division (`(a·d − c·b) / (b·d)`),
544
- * never as `a/b − c/d`.
545
- */
546
- function signedRank(differences) {
547
- const d = differences.filter((x) => x !== 0);
548
- const n = d.length;
549
- if (n === 0) return null;
550
- const sorted = d.map((v, i) => ({
551
- a: Math.abs(v),
552
- i
553
- })).sort((x, y) => x.a - y.a);
554
- const ranks = new Array(n);
555
- for (let i = 0; i < n;) {
556
- let j = i;
557
- while (j + 1 < n && sorted[j + 1]?.a === sorted[i]?.a) j++;
558
- for (let k = i; k <= j; k++) ranks[sorted[k]?.i] = (i + j) / 2 + 1;
559
- i = j + 1;
560
- }
561
- const total = n * (n + 1) / 2;
562
- const plus = d.reduce((s, v, i) => v > 0 ? s + ranks[i] : s, 0);
563
- const sign = Math.sign(plus - total / 2);
564
- const observed = Math.abs(plus - total / 2);
565
- if (n > 20) {
566
- const counts = /* @__PURE__ */ new Map();
567
- for (const { a } of sorted) counts.set(a, (counts.get(a) ?? 0) + 1);
568
- const ties = [...counts.values()].reduce((s, t) => s + t ** 3 - t, 0);
569
- const variance = n * (n + 1) * (2 * n + 1) / 24 - ties / 48;
570
- if (variance <= 0) return {
571
- p: 1,
572
- sign
573
- };
574
- const z = Math.max(0, observed - .5) / Math.sqrt(variance);
575
- return {
576
- p: Math.min(1, 2 * (1 - normalCdf(z))),
577
- sign
578
- };
579
- }
580
- let extreme = 0;
581
- for (let mask = 0; mask < 2 ** n; mask++) {
582
- let sum = 0;
583
- for (let i = 0; i < n; i++) if (mask & 1 << i) sum += ranks[i];
584
- if (Math.abs(sum - total / 2) >= observed - 1e-9) extreme++;
585
- }
586
- return {
587
- p: extreme / 2 ** n,
588
- sign
589
- };
590
- }
591
- function normalCdf(z) {
592
- const t = 1 / (1 + .3275911 * Math.abs(z) / Math.SQRT2);
593
- const erf = 1 - t * (.254829592 + t * (-.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))) * Math.exp(-(z * z) / 2);
594
- return z >= 0 ? (1 + erf) / 2 : (1 - erf) / 2;
595
- }
596
- //#endregion
597
530
  //#region src/view/effort.ts
598
531
  /** Below this two-sided p, an effort difference is an effect. */
599
532
  const EFFORT_ALPHA = .05;
@@ -771,7 +704,7 @@ function effortChangeLabel(base, head) {
771
704
  return `${core} · tokens ${formatTokens(base.tokens.median)} → ${formatTokens(head.tokens.median)} (${formatRatio(tok)})`;
772
705
  }
773
706
  //#endregion
774
- //#region src/view/judgements.ts
707
+ //#region src/view/judgments.ts
775
708
  /** Show each key until this many; then the minimum score plus a count. */
776
709
  const COLLAPSE_AFTER = 2;
777
710
  const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
@@ -844,7 +777,7 @@ function keyTallies(e, cellKey) {
844
777
  const expectLabels = /* @__PURE__ */ new Map();
845
778
  for (const a of e.attempts) {
846
779
  if (a.cell !== cellKey || !counted(a)) continue;
847
- for (const { judgement: j, label, labelled } of labelledJudgements(a)) {
780
+ for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
848
781
  const row = byKey.get(j.key) ?? {
849
782
  label: "",
850
783
  labelled: false,
@@ -855,7 +788,7 @@ function keyTallies(e, cellKey) {
855
788
  row.counted++;
856
789
  if (j.verdict === "pass") row.pass++;
857
790
  byKey.set(j.key, row);
858
- if (labelled && !expectLabels.has(j.key)) expectLabels.set(j.key, label);
791
+ if (labeled && !expectLabels.has(j.key)) expectLabels.set(j.key, label);
859
792
  }
860
793
  }
861
794
  const labels = uniqueJudgeLabels([...byKey].map(([key, row]) => ({
@@ -908,7 +841,7 @@ function cellJudgeRates(e, cellKey) {
908
841
  }));
909
842
  }
910
843
  /**
911
- * The judge rates a cell's summary line adds to its checks: a labelled judge whose label is one
844
+ * The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
912
845
  * of the cell's checks against its control is already there.
913
846
  */
914
847
  function summaryJudgeRates(e, cell) {
@@ -924,9 +857,9 @@ function judgeRatesLabel(rates) {
924
857
  }).join(" · ");
925
858
  }
926
859
  /** Each key, or the lowest score plus how many judges there were. */
927
- function collapsedJudges(judgements) {
928
- const labels = uniqueJudgeLabels(judgements);
929
- const items = judgements.map((j) => ({
860
+ function collapsedJudges(judgments) {
861
+ const labels = uniqueJudgeLabels(judgments);
862
+ const items = judgments.map((j) => ({
930
863
  verdict: j.verdict,
931
864
  score: j.score,
932
865
  threshold: j.threshold,
@@ -943,28 +876,28 @@ function collapsedJudges(judgements) {
943
876
  };
944
877
  }
945
878
  /** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
946
- function collapsedJudgeLabel(judgements) {
947
- if (!judgements.length) return null;
948
- const { shown, extra } = collapsedJudges(judgements);
879
+ function collapsedJudgeLabel(judgments) {
880
+ if (!judgments.length) return null;
881
+ const { shown, extra } = collapsedJudges(judgments);
949
882
  const parts = shown.map((j) => `${j.verdict} ${j.score.toFixed(2)}/${j.threshold.toFixed(2)}`);
950
883
  if (extra) parts.push(`${extra} judges`);
951
884
  return parts.join(" · ");
952
885
  }
953
886
  const JUDGE_ERROR = /^judge failed for /;
954
887
  /**
955
- * Each judgement's `expect` label, by position: the attempt's `toPass` assertions (less those on a
956
- * failed judge call, which recorded no judgement) in call order, when they are exactly as many as
957
- * its judgements. `check` is that assertion's label, the key of its check; a judgement no label
888
+ * Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
889
+ * failed judge call, which recorded no judgment) in call order, when they are exactly as many as
890
+ * its judgments. `check` is that assertion's label, the key of its check; a judgment no label
958
891
  * reaches is named by its question.
959
892
  */
960
- function labelledJudgements(a) {
893
+ function labeledJudgments(a) {
961
894
  const passes = (a.assertions ?? []).filter((x) => /(^|\.)toPass$/.test(x.matcher) && !JUDGE_ERROR.test(x.message ?? ""));
962
895
  const zip = passes.length === a.judgements.length;
963
- return a.judgements.map((judgement, i) => {
896
+ return a.judgements.map((judgment, i) => {
964
897
  const x = zip ? passes[i] : void 0;
965
898
  return {
966
- judgement,
967
- label: x?.labelled ? x.label : judgeText(judgement.question),
899
+ judgment,
900
+ label: x?.labelled ? x.label : judgeText(judgment.question),
968
901
  labelled: x?.labelled === true,
969
902
  ...x ? { check: x.label } : {}
970
903
  };
@@ -986,6 +919,26 @@ function checkLabels(a) {
986
919
  return x?.labelled ? x.label : void 0;
987
920
  });
988
921
  }
922
+ /**
923
+ * Each `toSucceed` label's `c.exec` check, by position, when the attempt's `toSucceed` assertions
924
+ * are exactly as many as its checks; empty otherwise.
925
+ */
926
+ function checksByLabel(a) {
927
+ const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
928
+ if (succeeds.length !== a.checks.length) return /* @__PURE__ */ new Map();
929
+ return new Map(succeeds.map((x, i) => [x.label, a.checks[i]]));
930
+ }
931
+ /**
932
+ * How a `c.exec` given `retries` ended: `passed on try 2 of 3`, `failed all 3 tries`, or null
933
+ * when it had none or passed on its first try.
934
+ */
935
+ function triesNote(exitCode, tried, of) {
936
+ if (of <= 1 || exitCode === 0 && tried <= 1) return null;
937
+ if (exitCode === 0) return `passed on try ${tried} of ${of}`;
938
+ return tried >= of ? `failed all ${of} tries` : `failed on try ${tried} of ${of}`;
939
+ }
940
+ /** {@link triesNote} for a recorded check. */
941
+ const checkTries = (c) => c.retries && c.tries ? triesNote(c.exitCode, c.tries.length, c.retries + 1) : null;
989
942
  /** A check's command on one line: the first non-empty line, clipped; its log has the rest. */
990
943
  function commandLine(command, max = 80) {
991
944
  const lines = command.split("\n").map((l) => l.trim()).filter(Boolean);
@@ -1101,7 +1054,6 @@ const shortSha = (sha) => sha ? sha.slice(0, 7) : "no git";
1101
1054
  * Edit calls become one file edit per file they change, so they're counted apart, never summed.
1102
1055
  */
1103
1056
  function activityLabel(a) {
1104
- const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
1105
1057
  return `${plural(a.tools, "tool call")} · ${plural(a.fileEdits, "file edit")}`;
1106
1058
  }
1107
1059
  /**
@@ -1298,7 +1250,7 @@ function cellRows(e) {
1298
1250
  attempts: e.attempts.filter((a) => a.cell === cell.key)
1299
1251
  }));
1300
1252
  }
1301
- /** One row per compare question, with each cut's win rates in the record's order. */
1253
+ /** One row per pairwise question, with each cut's win rates in the record's order. */
1302
1254
  function answerRows(e) {
1303
1255
  return (e.answers ?? []).map((a) => {
1304
1256
  const rates = (cut) => a.rates.filter((r) => r.cut === cut).map((r) => ({
@@ -1321,6 +1273,33 @@ function answerRows(e) {
1321
1273
  };
1322
1274
  });
1323
1275
  }
1276
+ /** One row per question asked of each attempt on its own, in first-asked order. */
1277
+ function attemptAnswerRows(e) {
1278
+ const rows = /* @__PURE__ */ new Map();
1279
+ for (const a of e.attempts) for (const q of a.answers ?? []) {
1280
+ const row = rows.get(q.question) ?? {
1281
+ question: q.question,
1282
+ model: q.resolvedModel ?? q.model,
1283
+ choices: q.choices ? q.choices.map((choice) => ({
1284
+ choice,
1285
+ n: 0
1286
+ })) : null,
1287
+ answers: []
1288
+ };
1289
+ rows.set(q.question, row);
1290
+ const picked = q.error === void 0 && row.choices?.find((c) => c.choice === q.choice);
1291
+ if (picked) picked.n++;
1292
+ row.answers.push({
1293
+ attempt: a.id,
1294
+ outcome: a.outcome,
1295
+ choice: q.choice ?? null,
1296
+ text: q.text,
1297
+ error: q.error ?? null,
1298
+ input: q.input?.path ?? null
1299
+ });
1300
+ }
1301
+ return [...rows.values()];
1302
+ }
1324
1303
  /** Whether an eval needs the cells table: several cells, or several runs of one. */
1325
1304
  const isMatrix = (e) => e.cells.length > 1 || e.cells.some((c) => c.runs > 1);
1326
1305
  /** One eval across runs, oldest first. */
@@ -1349,6 +1328,7 @@ const COMPARE_ORDER = [
1349
1328
  "regressed",
1350
1329
  "fixed",
1351
1330
  "drift",
1331
+ "gate",
1352
1332
  "infra",
1353
1333
  "added",
1354
1334
  "removed",
@@ -1360,6 +1340,8 @@ const COMPARE_ORDER = [
1360
1340
  * the eval line labels it `gated`: `2/4 · pass@1 100% gated`.
1361
1341
  */
1362
1342
  const hasUngatedCells = (cells) => cells.some((c) => c.gate === "none");
1343
+ /** A study's pass: every cell `gate: 'none'`, so nothing it saw passed or failed it. */
1344
+ const isNotGatedPass = (e) => e.outcome === "pass" && e.cells.length > 0 && e.cells.every((c) => c.gate === "none");
1363
1345
  /**
1364
1346
  * An expected miss: a failed attempt in an ungated cell (a control, by default). It's reported
1365
1347
  * apart from failures, since it fails neither its eval nor the run.
@@ -1384,8 +1366,9 @@ function gatedPassAt1(cells) {
1384
1366
  /** `reeval compare`'s classification, plus pass@1 and judge-score deltas from the records. */
1385
1367
  function compareView(base, head, records = {}) {
1386
1368
  const rows = compare(base, head).map((row) => {
1387
- const b = base.evals.find((e) => e.id === row.id);
1388
- const h = head.evals.find((e) => e.id === row.id);
1369
+ const evalOf = (run) => row.kind === "family" ? void 0 : run.evals.find((e) => e.id === row.id);
1370
+ const b = evalOf(base);
1371
+ const h = evalOf(head);
1389
1372
  const baseKeys = new Set(b?.cells.map((c) => c.key));
1390
1373
  const shared = row.cells && new Set(h?.cells.filter((c) => baseKeys.has(c.key)).map((c) => c.key));
1391
1374
  const passAt1 = (e) => {
@@ -1397,8 +1380,8 @@ function compareView(base, head, records = {}) {
1397
1380
  slug: h?.slug ?? b?.slug ?? null,
1398
1381
  basePassAt1: passAt1(b),
1399
1382
  headPassAt1: passAt1(h),
1400
- baseScore: meanScore(records.base?.evals.find((e) => e.id === row.id)),
1401
- headScore: meanScore(records.head?.evals.find((e) => e.id === row.id)),
1383
+ baseScore: b ? meanScore(records.base?.evals.find((e) => e.id === row.id)) : null,
1384
+ headScore: h ? meanScore(records.head?.evals.find((e) => e.id === row.id)) : null,
1402
1385
  perCell: row.perCell ?? []
1403
1386
  };
1404
1387
  });
@@ -1442,31 +1425,11 @@ function transcriptItems(lines) {
1442
1425
  };
1443
1426
  });
1444
1427
  }
1445
- const HEADER = /^diff --git a\/(.+?) b\/(.+)$/m;
1446
- /** Splits a git patch into one entry per file, with line counts. */
1447
- function splitPatch(patch) {
1448
- if (!patch.trim()) return [];
1449
- return patch.split(/(?=^diff --git )/m).filter((part) => part.startsWith("diff --git ")).map((text) => {
1450
- const header = HEADER.exec(text);
1451
- const path = header?.[2] ?? header?.[1] ?? "unknown";
1452
- let added = 0;
1453
- let removed = 0;
1454
- for (const line of text.split("\n")) if (line.startsWith("+") && !line.startsWith("+++ ")) added++;
1455
- else if (line.startsWith("-") && !line.startsWith("--- ")) removed++;
1456
- return {
1457
- path,
1458
- status: /^new file mode/m.test(text) ? "added" : /^deleted file mode/m.test(text) ? "deleted" : /^rename from/m.test(text) ? "renamed" : /^(Binary files|GIT binary patch)/m.test(text) ? "binary" : "modified",
1459
- added,
1460
- removed,
1461
- text
1462
- };
1463
- });
1464
- }
1465
1428
  const findEval = (record, slug) => record.evals.find((e) => e.slug === slug);
1466
1429
  const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId);
1467
1430
  /** The attempt to open by default: the first failure, else the first attempt. */
1468
1431
  const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
1469
1432
  //#endregion
1470
- export { directional as $, splitPatch as A, labelledJudgements as B, parseTranscript as C, sourceChanges as Ct, runtimeLabel as D, runRows as E, collapsedJudgeLabel as F, EFFORT_MIN_RUNS as G, uniqueJudgeLabels as H, collapsedJudges as I, bothUnderATenth as J, TOKENS_UNREPORTED as K, commandLine as L, cellJudgeRates as M, cellUsage as N, shortSha as O, checkLabels as P, contrastEffort as Q, judgeLabel as R, languageLabel as S, compare as St, runOutcome as T, whyChanged as Tt, DIRECTIONAL_MIN as U, summaryJudgeRates as V, EFFORT_ALPHA as W, cellEffort as X, cellContrast as Y, cellSamples as Z, harnessLabel as _, round3 as _t, cellRows as a, effortVerdict as at, isMatrix as b, addedRow as bt, evalHistory as c, formatSeconds as ct, failedJudgeLabels as d, containsZero as dt, effortChangeLabel as et, findAttempt as f, liftInterval as ft, formatUsage as g, pairedLiftInterval as gt, formatPercent as h, median as ht, buildIndex as i, effortSummary as it, transcriptItems as j, splitEvalId as k, evalMarkOutcome as l, tokenValues as lt, formatDuration as m, meanInterval as mt, activityLabel as n, effortOf as nt, compareView as o, formatCount as ot, findEval as p, mcnemar as pt, VERDICT_TEXT as q, answerRows as r, effortSample as rt, defaultAttempt as s, formatRatio as st, COMPARE_ORDER as t, effortLabel as tt, evalTree as u, EFFECT_MIN_COUNTED as ut, hasUngatedCells as v, signedRank as vt, runLead as w, stackChanges as wt, isUngatedMiss as x, cellsChanged as xt, indexFiles as y, wilson as yt, judgeRatesLabel as z };
1433
+ export { cellContrast as $, shortSha as A, commandLine as B, isUngatedMiss as C, gateLabel as Ct, runOutcome as D, runLead as E, worstOutcome as Et, checkLabels as F, triesNote as G, judgeRatesLabel as H, checkTries as I, EFFORT_ALPHA as J, uniqueJudgeLabels as K, checksByLabel as L, transcriptItems as M, cellJudgeRates as N, runRows as O, cellUsage as P, bothUnderATenth as Q, collapsedJudgeLabel as R, isNotGatedPass as S, cellOutcome as St, parseTranscript as T, passRateOf as Tt, labeledJudgments as U, judgeLabel as V, summaryJudgeRates as W, TOKENS_UNREPORTED as X, EFFORT_MIN_RUNS as Y, VERDICT_TEXT as Z, formatUsage as _, newRows as _t, buildIndex as a, effortLabel as at, indexFiles as b, stackChanges as bt, defaultAttempt as c, effortSummary as ct, evalTree as d, formatRatio as dt, cellEffort as et, failedJudgeLabels as f, formatSeconds as ft, formatPercent as g, compare as gt, formatDuration as h, cellsChanged as ht, attemptAnswerRows as i, effortChangeLabel as it, splitEvalId as j, runtimeLabel as k, evalHistory as l, effortVerdict as lt, findEval as m, addedRow as mt, activityLabel as n, contrastEffort as nt, cellRows as o, effortOf as ot, findAttempt as p, tokenValues as pt, DIRECTIONAL_MIN as q, answerRows as r, directional as rt, compareView as s, effortSample as st, COMPARE_ORDER as t, cellSamples as tt, evalMarkOutcome as u, formatCount as ut, harnessLabel as v, rowKey as vt, languageLabel as w, gatePart as wt, isMatrix as x, whyChanged as xt, hasUngatedCells as y, sourceChanges as yt, collapsedJudges as z };
1471
1434
 
1472
- //# sourceMappingURL=model-UgHDMhI1.js.map
1435
+ //# sourceMappingURL=model-CTnyvh7H.js.map