reeval 0.17.2 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/README.md +4 -2
  2. package/dist/{adapter-DyvDj4ZN.js → adapter-CptK032x.js} +2 -2
  3. package/dist/{adapter-DyvDj4ZN.js.map → adapter-CptK032x.js.map} +1 -1
  4. package/dist/{agent-v036GEgG.js → agent-B63OzUX9.js} +6 -6
  5. package/dist/{agent-v036GEgG.js.map → agent-B63OzUX9.js.map} +1 -1
  6. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  7. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  8. package/dist/{app-files-hpRT5lj_.js → app-files-sgqwfjtn.js} +4 -4
  9. package/dist/app-files-sgqwfjtn.js.map +1 -0
  10. package/dist/assemble-CxBHkavM.js +1 -0
  11. package/dist/{assemble-Dlh6Xv3U.js → assemble-Me7Whp-y.js} +58 -194
  12. package/dist/assemble-Me7Whp-y.js.map +1 -0
  13. package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
  14. package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
  15. package/dist/{attempt-host-DErH-DYV.js → attempt-host-Cik0SoIn.js} +17 -9
  16. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
  17. package/dist/{capture-Bdcx9Gap.js → capture-C3RmjPph.js} +392 -613
  18. package/dist/capture-C3RmjPph.js.map +1 -0
  19. package/dist/capture-YPwxp_iK.js +1 -0
  20. package/dist/{cli-CGd51thd.js → cli-BBOlXKPG.js} +1075 -457
  21. package/dist/cli-BBOlXKPG.js.map +1 -0
  22. package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
  23. package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
  26. package/dist/client-tZ2WBDxf.js.map +1 -0
  27. package/dist/{command-TLmY-rYp.js → command-CFQ9c1hk.js} +13 -13
  28. package/dist/{command-TLmY-rYp.js.map → command-CFQ9c1hk.js.map} +1 -1
  29. package/dist/{command-CtTdCF3B.js → command-DhiwsZl1.js} +9 -9
  30. package/dist/{command-CtTdCF3B.js.map → command-DhiwsZl1.js.map} +1 -1
  31. package/dist/{command-CyIb9BSC.js → command-fO0BFsTq.js} +102 -37
  32. package/dist/command-fO0BFsTq.js.map +1 -0
  33. package/dist/{command-CsfcwMyc.js → command-qGS802uX.js} +16 -14
  34. package/dist/command-qGS802uX.js.map +1 -0
  35. package/dist/{commands-C54S45d7.js → commands-CC60TOT5.js} +64 -18
  36. package/dist/commands-CC60TOT5.js.map +1 -0
  37. package/dist/{commands-jO9Jj35r.js → commands-xYTMNTPv.js} +1 -1
  38. package/dist/{config-ImY6AlRE.js → config-C2UtS9vy.js} +718 -76
  39. package/dist/config-C2UtS9vy.js.map +1 -0
  40. package/dist/{contrasts-DtSX8E_V.js → contrasts-BjOg7wyr.js} +3 -3
  41. package/dist/{contrasts-DtSX8E_V.js.map → contrasts-BjOg7wyr.js.map} +1 -1
  42. package/dist/cost-BJU0MAji.js +1 -0
  43. package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
  44. package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
  45. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BTK_apL-.js} +2 -2
  46. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
  47. package/dist/{db-sHGpaLo9.js → db-C5SZ0510.js} +72 -14
  48. package/dist/db-C5SZ0510.js.map +1 -0
  49. package/dist/db-puzLzX4k.js +2 -0
  50. package/dist/{default-effort-BZRa-SQP.js → default-effort-BVmx7r7L.js} +8 -5
  51. package/dist/default-effort-BVmx7r7L.js.map +1 -0
  52. package/dist/{default-effort-fpdbYVKm.js → default-effort-FEidNAIi.js} +1 -1
  53. package/dist/{docker-CWuSRj7a.js → docker-B2F8-PGC.js} +4 -4
  54. package/dist/{docker-CWuSRj7a.js.map → docker-B2F8-PGC.js.map} +1 -1
  55. package/dist/{docker-CIzHzwlC.js → docker-BlSopHpZ.js} +1 -1
  56. package/dist/{docker--2ytOHcu.js → docker-Dto8ROCq.js} +5 -5
  57. package/dist/{docker--2ytOHcu.js.map → docker-Dto8ROCq.js.map} +1 -1
  58. package/dist/{doctor-eE65YpEv.js → doctor-BcvfKpBX.js} +20 -20
  59. package/dist/{doctor-eE65YpEv.js.map → doctor-BcvfKpBX.js.map} +1 -1
  60. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  61. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  62. package/dist/{environment-Cy3FVLR5.js → environment-CC1RV21n.js} +3 -3
  63. package/dist/{environment-Cy3FVLR5.js.map → environment-CC1RV21n.js.map} +1 -1
  64. package/dist/{eval-source-BTsTRiBW.js → eval-source-CzxdgFBW.js} +18 -10
  65. package/dist/eval-source-CzxdgFBW.js.map +1 -0
  66. package/dist/evaluate-DToSB2bt.d.ts +101 -0
  67. package/dist/{evaluate-CFFdyYZX.js → evaluate-NTNJ8T9H.js} +129 -51
  68. package/dist/evaluate-NTNJ8T9H.js.map +1 -0
  69. package/dist/{files-tnMgu7-X.js → files-CZXrkPja.js} +5 -5
  70. package/dist/{files-tnMgu7-X.js.map → files-CZXrkPja.js.map} +1 -1
  71. package/dist/files-DFtuRQJx.js +1 -0
  72. package/dist/{files-CQU08uh4.js → files-DsXULUi5.js} +3 -3
  73. package/dist/{files-CQU08uh4.js.map → files-DsXULUi5.js.map} +1 -1
  74. package/dist/{files-CIX-pLl4.js → files-aAzoGNCB.js} +4 -4
  75. package/dist/{files-CIX-pLl4.js.map → files-aAzoGNCB.js.map} +1 -1
  76. package/dist/files-oOYNR7ZH.js +1 -0
  77. package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
  78. package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
  79. package/dist/gateway.d.ts +1 -1
  80. package/dist/{git-BWVkG4Yl.js → git-C1MB5bjr.js} +3 -3
  81. package/dist/{git-BWVkG4Yl.js.map → git-C1MB5bjr.js.map} +1 -1
  82. package/dist/harness-DGsRZDSo.js +105 -0
  83. package/dist/harness-DGsRZDSo.js.map +1 -0
  84. package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
  85. package/dist/harness-DYSC4zMh.js.map +1 -0
  86. package/dist/harness.d.ts +1 -1
  87. package/dist/harness.js +1 -1
  88. package/dist/{images-BVZHDf_z.js → images-CNayXCp2.js} +8 -8
  89. package/dist/{images-BVZHDf_z.js.map → images-CNayXCp2.js.map} +1 -1
  90. package/dist/index.d.ts +3 -3
  91. package/dist/index.js +5 -5
  92. package/dist/init-8o6W3_fn.js +1 -0
  93. package/dist/{init-BWnMbUnc.js → init-D5HtmyMI.js} +5 -5
  94. package/dist/{init-BWnMbUnc.js.map → init-D5HtmyMI.js.map} +1 -1
  95. package/dist/ink-list-Ca-5dunO.js +184 -0
  96. package/dist/ink-list-Ca-5dunO.js.map +1 -0
  97. package/dist/{invocations-BGRs1r_C.js → invocations-DrGXF9dT.js} +3 -3
  98. package/dist/{invocations-BGRs1r_C.js.map → invocations-DrGXF9dT.js.map} +1 -1
  99. package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
  100. package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
  101. package/dist/judge.d.ts +1 -1
  102. package/dist/judge.js +1 -1
  103. package/dist/judgments-m-mb0zv8.js +246 -0
  104. package/dist/judgments-m-mb0zv8.js.map +1 -0
  105. package/dist/list-CM_YSJXq.js +374 -0
  106. package/dist/list-CM_YSJXq.js.map +1 -0
  107. package/dist/{live-d_5a5iUv.js → live-B71o2TkX.js} +21 -3
  108. package/dist/live-B71o2TkX.js.map +1 -0
  109. package/dist/live-DQc8jr6o.js +1 -0
  110. package/dist/{live-ColieOai.js → live-uRURFDMZ.js} +7 -7
  111. package/dist/{live-ColieOai.js.map → live-uRURFDMZ.js.map} +1 -1
  112. package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
  113. package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
  114. package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
  115. package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
  116. package/dist/{model-DBpdRXtO.js → model-BxBWFOZn.js} +43 -19
  117. package/dist/{model-DBpdRXtO.js.map → model-BxBWFOZn.js.map} +1 -1
  118. package/dist/{model-DOXnWH65.js → model-CXVVB4RK.js} +1 -1
  119. package/dist/{model-DOiCiMCD.js → model-Ce9l9_rF.js} +19 -255
  120. package/dist/model-Ce9l9_rF.js.map +1 -0
  121. package/dist/model-Cxzo4cjw.js +1 -0
  122. package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
  123. package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
  124. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
  125. package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
  126. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
  127. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
  128. package/dist/{patch-DvLr8wCZ.js → patch-ug5HpMOj.js} +3 -3
  129. package/dist/{patch-DvLr8wCZ.js.map → patch-ug5HpMOj.js.map} +1 -1
  130. package/dist/{plan-CIfOFhth.js → plan-DqnzfDH1.js} +15 -14
  131. package/dist/plan-DqnzfDH1.js.map +1 -0
  132. package/dist/{plugins-BEs2Am3L.js → plugins-B2KQ0y3P.js} +4 -4
  133. package/dist/{plugins-BEs2Am3L.js.map → plugins-B2KQ0y3P.js.map} +1 -1
  134. package/dist/{preflight-BAIRsTpl.js → preflight-DVXEvzFU.js} +1 -1
  135. package/dist/{preflight-50Qw5e5K.js → preflight-DyzmvK7i.js} +3 -3
  136. package/dist/{preflight-50Qw5e5K.js.map → preflight-DyzmvK7i.js.map} +1 -1
  137. package/dist/{prepare-8XhzhLHX.js → prepare-21WMQZyt.js} +7 -7
  138. package/dist/{prepare-8XhzhLHX.js.map → prepare-21WMQZyt.js.map} +1 -1
  139. package/dist/{prepare-8nvfDFQx.js → prepare-BcSRa7-3.js} +2 -2
  140. package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
  141. package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
  142. package/dist/{provenance-Cc8fFE7Z.js → provenance-iRxjp4Pu.js} +4 -4
  143. package/dist/{provenance-Cc8fFE7Z.js.map → provenance-iRxjp4Pu.js.map} +1 -1
  144. package/dist/{providers-CP_bBtYm.js → providers-D0ob_tra.js} +1 -1
  145. package/dist/{providers-DZSTlC9R.js → providers-vXMXuwn-.js} +5 -5
  146. package/dist/{providers-DZSTlC9R.js.map → providers-vXMXuwn-.js.map} +1 -1
  147. package/dist/{query-DXpxaaTH.js → query-BhaKQ07P.js} +3 -3
  148. package/dist/{query-DXpxaaTH.js.map → query-BhaKQ07P.js.map} +1 -1
  149. package/dist/query-CvgUsvep.js +1 -0
  150. package/dist/{questions-C_Y-oXW6.js → questions-BzdLdV3C.js} +39 -9
  151. package/dist/questions-BzdLdV3C.js.map +1 -0
  152. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  153. package/dist/redact-BhjtnjrU.js.map +1 -0
  154. package/dist/remarks-CAvTSq16.js +1 -0
  155. package/dist/{remarks-E4rpi8Ym.js → remarks-CauU408t.js} +231 -25
  156. package/dist/remarks-CauU408t.js.map +1 -0
  157. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  158. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  159. package/dist/{remarks-DeZZT8PK.js → remarks-NMgPd30U.js} +4 -3
  160. package/dist/{remarks-DeZZT8PK.js.map → remarks-NMgPd30U.js.map} +1 -1
  161. package/dist/{remote-DZ00A8V_.js → remote-Dwy3fyo3.js} +5 -5
  162. package/dist/{remote-DZ00A8V_.js.map → remote-Dwy3fyo3.js.map} +1 -1
  163. package/dist/{resolve-B_JlQ0La.js → resolve-B1zMnPwK.js} +7 -7
  164. package/dist/{resolve-B_JlQ0La.js.map → resolve-B1zMnPwK.js.map} +1 -1
  165. package/dist/resolve-BIIeVT12.js +1 -0
  166. package/dist/{results-BMJHaQbi.js → results-CfC0id_u.js} +7 -7
  167. package/dist/{results-BMJHaQbi.js.map → results-CfC0id_u.js.map} +1 -1
  168. package/dist/{resume-CexsWuGK.js → resume-Dkhrpjow.js} +123 -8
  169. package/dist/resume-Dkhrpjow.js.map +1 -0
  170. package/dist/{runtime-CSZ1hjlN.js → runtime-DcvQcr36.js} +2 -2
  171. package/dist/{runtime-CSZ1hjlN.js.map → runtime-DcvQcr36.js.map} +1 -1
  172. package/dist/{sandbox-B5W8lLL1.js → sandbox-8zCrtuCf.js} +2 -2
  173. package/dist/{sandbox-B5W8lLL1.js.map → sandbox-8zCrtuCf.js.map} +1 -1
  174. package/dist/sandbox.d.ts +1 -1
  175. package/dist/sandbox.js +1 -1
  176. package/dist/server-0tuj8VSa.js +1 -0
  177. package/dist/{server-BfN6q2lH.js → server-Dplq8g0r.js} +7 -7
  178. package/dist/{server-BfN6q2lH.js.map → server-Dplq8g0r.js.map} +1 -1
  179. package/dist/{source-BD6Bg0Tr.js → source-NWHqn5-o.js} +2 -2
  180. package/dist/{source-BD6Bg0Tr.js.map → source-NWHqn5-o.js.map} +1 -1
  181. package/dist/source-llwLewOU.js +1 -0
  182. package/dist/{src-BC2WXULN.js → src-B9oE_mNS.js} +1 -1
  183. package/dist/{src-Cpy9MNS3.js → src-BQ6BiXoc.js} +19 -11
  184. package/dist/{src-Cpy9MNS3.js.map → src-BQ6BiXoc.js.map} +1 -1
  185. package/dist/{src-Dp-fWOfG.js → src-BQXdf-Gw.js} +2 -2
  186. package/dist/{src-Dp-fWOfG.js.map → src-BQXdf-Gw.js.map} +1 -1
  187. package/dist/{src-DurP--_D.js → src-BXnv0ViU.js} +1 -1
  188. package/dist/src-BiS8YNN_.js +3 -0
  189. package/dist/{src-DVaAKCsH.js → src-BkH4GOxu.js} +1 -1
  190. package/dist/{src-CM7xdJFA.js → src-C-tH5ebd.js} +1 -1
  191. package/dist/{src-Cx0HJIJ-.js → src-CUpoVVb8.js} +1 -1
  192. package/dist/{src-C9elednc.js → src-Cg1bFs9g.js} +1 -1
  193. package/dist/{src-BKDZO3ra.js → src-DV_UorZv.js} +1 -1
  194. package/dist/{src-CWEGknZo.js → src-XxUpr2sa.js} +1 -1
  195. package/dist/{state-CdSjXIfE.js → state-DDCGUhva.js} +5 -5
  196. package/dist/{state-CdSjXIfE.js.map → state-DDCGUhva.js.map} +1 -1
  197. package/dist/state-DZbAO95a.js +1 -0
  198. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  199. package/dist/stats-C18W4xPX.js.map +1 -0
  200. package/dist/suite-M8_7xio7.js +1294 -0
  201. package/dist/suite-M8_7xio7.js.map +1 -0
  202. package/dist/suite-u2KJnGpY.js +2 -0
  203. package/dist/{summary-zP3cCXDN.js → summary-Uzc9osfV.js} +47 -13
  204. package/dist/summary-Uzc9osfV.js.map +1 -0
  205. package/dist/testing.d.ts +47 -37
  206. package/dist/testing.js +4 -4
  207. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  208. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  209. package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
  210. package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
  211. package/dist/{types-BHR1lGPh.d.ts → types-DGvexkcp.d.ts} +71 -63
  212. package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
  213. package/dist/vertex-BLQMayc1.js.map +1 -0
  214. package/dist/vertex-h_peaZTN.js +1 -0
  215. package/dist/view/index.d.ts +140 -35
  216. package/dist/view/index.js +8 -7
  217. package/dist/{views-CKoN5NAx.js → views-BISrsIjw.js} +6 -6
  218. package/dist/{views-CKoN5NAx.js.map → views-BISrsIjw.js.map} +1 -1
  219. package/dist/views-Bfab8fM9.js +2 -0
  220. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-DJD_nHkS.js} +5 -5
  221. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
  222. package/dist/workspace-B8EOCuxl.js +1 -0
  223. package/dist/{workspace-Cho30qlM.js → workspace-BO1VhUR9.js} +5 -5
  224. package/dist/{workspace-Cho30qlM.js.map → workspace-BO1VhUR9.js.map} +1 -1
  225. package/package.json +3 -1
  226. package/skills/eval-rubric/SKILL.md +22 -3
  227. package/dist/app-files-hpRT5lj_.js.map +0 -1
  228. package/dist/assemble-DTXwCL3m.js +0 -1
  229. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  230. package/dist/capture-BT2fAmpZ.js +0 -1
  231. package/dist/capture-Bdcx9Gap.js.map +0 -1
  232. package/dist/cli-CGd51thd.js.map +0 -1
  233. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  234. package/dist/client-CF-DOjOU.js.map +0 -1
  235. package/dist/command-CsfcwMyc.js.map +0 -1
  236. package/dist/command-CyIb9BSC.js.map +0 -1
  237. package/dist/commands-C54S45d7.js.map +0 -1
  238. package/dist/config-ImY6AlRE.js.map +0 -1
  239. package/dist/cost-CMzY9VXv.js +0 -1
  240. package/dist/db-ZIk6S2CD.js +0 -2
  241. package/dist/db-sHGpaLo9.js.map +0 -1
  242. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  243. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  244. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  245. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  246. package/dist/files-B3d-8Fe0.js +0 -1
  247. package/dist/files-Do4Rp3yZ.js +0 -1
  248. package/dist/harness-Ch2P81sx.js.map +0 -1
  249. package/dist/init-s1jN-PCI.js +0 -1
  250. package/dist/live-DmJix4c8.js +0 -1
  251. package/dist/live-d_5a5iUv.js.map +0 -1
  252. package/dist/model-8uZBYqV5.js +0 -1
  253. package/dist/model-DOiCiMCD.js.map +0 -1
  254. package/dist/plan-CIfOFhth.js.map +0 -1
  255. package/dist/query-kFPuqBv1.js +0 -1
  256. package/dist/questions-C_Y-oXW6.js.map +0 -1
  257. package/dist/redact-BEEGc37e.js.map +0 -1
  258. package/dist/remarks-0EXMh6AD.js.map +0 -1
  259. package/dist/remarks-BL18Id09.js +0 -1
  260. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  261. package/dist/resolve-BJSay360.js +0 -1
  262. package/dist/resume-CexsWuGK.js.map +0 -1
  263. package/dist/server-SBzrf35T.js +0 -1
  264. package/dist/source-Dn1uqPon.js +0 -1
  265. package/dist/src-BxrHipV0.js +0 -3
  266. package/dist/state-Cicc9Oa5.js +0 -1
  267. package/dist/stats-D-Gqwqt0.js.map +0 -1
  268. package/dist/suite-BYJ2Q0S-.js +0 -2
  269. package/dist/suite-BtHPBFYL.js +0 -544
  270. package/dist/suite-BtHPBFYL.js.map +0 -1
  271. package/dist/summary-zP3cCXDN.js.map +0 -1
  272. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  273. package/dist/vertex-BaAz44YN.js +0 -1
  274. package/dist/views-CmyX255j.js +0 -2
  275. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -1,61 +1,3 @@
1
- /** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
2
- type DeclaredMode = 'run' | 'skip' | 'only';
3
- /** A `group`: options its evals and groups inherit, and its children in declaration order. */
4
- type DeclaredGroup = {
5
- type: 'group';
6
- /** The group's name, as passed to `group`. */
7
- name: string;
8
- mode: DeclaredMode;
9
- /** The group's own options, before the cascade. */
10
- options: EvalOptionsData;
11
- children: DeclaredNode[];
12
- };
13
- /** An `evaluate`: one eval, planned into a test per attempt. */
14
- type DeclaredEval = {
15
- type: 'eval';
16
- /** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
17
- name: string;
18
- mode: DeclaredMode;
19
- /** The eval's own options, before the cascade. */
20
- options: EvalOptionsData;
21
- /** Names the eval's body to whoever runs it. */
22
- body: string;
23
- source: DeclaredSource;
24
- /** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
25
- family?: DeclaredFamily;
26
- };
27
- /**
28
- * An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
29
- * text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
30
- * whitespace, non-ASCII characters as themselves).
31
- */
32
- type DeclaredSource = {
33
- /** sha256 of the body function's source text. */
34
- body: string;
35
- /** sha256 of the case's data; set on each case of an `evaluate.each`. */
36
- case?: string;
37
- };
38
- /** Where a case of a pooled `evaluate.each` sits in its list. */
39
- type DeclaredFamily = {
40
- /** The name template passed to `evaluate.each`. */
41
- name: string;
42
- /** The case's index, from 0. */
43
- case: number;
44
- /** How many cases the list has. */
45
- cases: number;
46
- };
47
- /** `evaluate.todo` or `group.todo`. */
48
- type DeclaredTodo = {
49
- type: 'todo';
50
- /** The name passed to `.todo`. */
51
- name: string;
52
- };
53
- /** One node of the tree: a group, an eval or a todo. */
54
- type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
55
- /** An eval file's top-level groups, evals and todos, in declaration order. */
56
- type DeclarationTree = {
57
- children: DeclaredNode[];
58
- };
59
1
  /** Below this two-sided p, an effort difference is an effect. */
60
2
  declare const EFFORT_ALPHA = 0.05;
61
3
  /** Some counted attempts reported no usage. Not a verdict; never printed as one. */
@@ -516,6 +458,11 @@ type AttemptRecord = {
516
458
  * in the attempt's directory, `<n>` the question's place in the eval's `questions`.
517
459
  */
518
460
  answers?: AttemptAnswer[];
461
+ /**
462
+ * Each `c.observe(name, value)` the attempt made, one per name in first-call order with its last
463
+ * value; unset without any. They annotate: `outcome` never depends on them.
464
+ */
465
+ observations?: Observation[];
519
466
  };
520
467
  type EvalEntry = {
521
468
  /** `file › group › … › name`, as shown. */
@@ -555,6 +502,12 @@ type EvalEntry = {
555
502
  outcome: Outcome;
556
503
  /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
557
504
  todo: boolean;
505
+ /**
506
+ * Declared not to run: `evaluate.skip`, inside a `group.skip`, or left out by an `.only` in its
507
+ * file. Its outcome is `skipped`. Unset otherwise, so in a run that isn't `complete` a skipped
508
+ * eval with neither `todo` nor `skip` hasn't run yet. Unset on records from before it.
509
+ */
510
+ skip?: true;
558
511
  /** pass@1, pass / (pass + fail), over the gated cells' attempts; infra errors and skips excluded. */
559
512
  passRate: number | null;
560
513
  cells: CellRecord[];
@@ -817,6 +770,22 @@ type Cost = {
817
770
  usage: TokenUsage | null;
818
771
  source: 'reported' | 'unreported';
819
772
  };
773
+ /**
774
+ * The `reeval run` settings that shaped a run's plan, as it was first started. A resume keeps
775
+ * them, so it finishes the attempts the run planned.
776
+ */
777
+ type RunInvocation = {
778
+ /** `--runs`; unset when each cell ran its own `runs`. */
779
+ runs?: number;
780
+ /** The profile the run was sized with; unset when it had none. */
781
+ profile?: string;
782
+ /** `--concurrency` or `REEVAL_CONCURRENCY`; unset when the config or the machine sized it. */
783
+ concurrency?: number;
784
+ /** The filter arguments. */
785
+ filters: string[];
786
+ /** `--tag`, without the profile's own tags. */
787
+ tags: string[];
788
+ };
820
789
  type RunRecord = {
821
790
  kind: 'reeval-run';
822
791
  schemaVersion: typeof SCHEMA_VERSION;
@@ -863,6 +832,8 @@ type RunRecord = {
863
832
  harness?: string[];
864
833
  };
865
834
  } | null;
835
+ /** How `reeval run` was started; `--resume` plans with it again. Unset in older records. */
836
+ invocation?: RunInvocation;
866
837
  /**
867
838
  * What the run planned before it started, and each level whose sandbox ran as root or with
868
839
  * capabilities added back.
@@ -1519,6 +1490,8 @@ type JudgeInputs = {
1519
1490
  transcript?: boolean | 'tools' | 'full';
1520
1491
  /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
1521
1492
  checks?: boolean | string[];
1493
+ /** `c.observe()` values so far, by name; a list shows only those names. */
1494
+ observations?: boolean | string[];
1522
1495
  /**
1523
1496
  * Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
1524
1497
  * capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
@@ -1553,6 +1526,8 @@ type ResolvedJudgeStateOptions = {
1553
1526
  transcript: boolean | 'tools' | 'full';
1554
1527
  turns: 'last' | 'all';
1555
1528
  checks: boolean | string[];
1529
+ /** Set only when the call turned it on, so never `false` as written. */
1530
+ observations?: boolean | string[];
1556
1531
  };
1557
1532
  type Verdict = {
1558
1533
  verdict: 'pass' | 'fail';
@@ -1628,6 +1603,12 @@ type AttemptContext = {
1628
1603
  * made to it afterwards.
1629
1604
  */
1630
1605
  writeFile(path: string, contents: string | Uint8Array): Promise<void>;
1606
+ /**
1607
+ * Records a fact about the attempt under `name`, such as what the agent installed. It never
1608
+ * affects the outcome. Calling it again with a name replaces its value. The value is JSON, at
1609
+ * most 4 KB, and is masked like the rest of the record.
1610
+ */
1611
+ observe(name: string, value: JSONValue): void;
1631
1612
  /** Every turn so far, oldest first. */
1632
1613
  readonly turns: readonly Turn[];
1633
1614
  /** Aborted on timeout or Ctrl-C. */
@@ -2612,6 +2593,8 @@ type QuestionInputs = {
2612
2593
  transcript?: boolean | 'tools' | 'full';
2613
2594
  /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
2614
2595
  checks?: boolean | string[];
2596
+ /** `c.observe()` values, by name; a list shows only those names. */
2597
+ observations?: boolean | string[];
2615
2598
  /** Each attempt's own captures, by name; `true` sends all of them. */
2616
2599
  artifacts?: true | string[];
2617
2600
  };
@@ -2624,6 +2607,8 @@ type ResolvedQuestionStateOptions = {
2624
2607
  transcript: boolean | 'tools' | 'full';
2625
2608
  turns: 'last' | 'all';
2626
2609
  checks: boolean | string[];
2610
+ /** Set only when the question turned it on, so never `false` as written. */
2611
+ observations?: boolean | string[];
2627
2612
  };
2628
2613
  /** Defaults for every question below it. What a question is shown is its own, never a default. */
2629
2614
  type QuestionOptions = {
@@ -2794,7 +2779,14 @@ type Revision = number | string;
2794
2779
  * An eval's own source as hashes: its body and case data as declared, and its options and its
2795
2780
  * enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
2796
2781
  */
2797
- type EvalSource = DeclaredSource & {
2782
+ type EvalSource = {
2783
+ /** sha256 of the body function's source text. */
2784
+ body: string;
2785
+ /**
2786
+ * sha256 of the case's data as canonical JSON, with secrets in its strings masked as the record
2787
+ * masks them; set on each case of an `evaluate.each`.
2788
+ */
2789
+ case?: string;
2798
2790
  /** sha256 of the canonical options. */
2799
2791
  options: string;
2800
2792
  };
@@ -2985,6 +2977,14 @@ type BlindSide = BlindTurn & {
2985
2977
  command: string;
2986
2978
  notRun: true;
2987
2979
  }>;
2980
+ /** Its `c.observe()` values; a name the question lists that it never observed is `missing`. */
2981
+ observations?: Array<{
2982
+ name: string;
2983
+ value: JSONValue;
2984
+ } | {
2985
+ name: string;
2986
+ missing: true;
2987
+ }>;
2988
2988
  /** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
2989
2989
  artifacts?: Array<{
2990
2990
  label: string;
@@ -3006,7 +3006,7 @@ type AnalysisConfig = {
3006
3006
  instructions?: string;
3007
3007
  /** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
3008
3008
  maxInputTokens?: number;
3009
- /** The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap. */
3009
+ /** The most tokens the model may answer with, reasoning included (default 32000). reeval has no prices, so tokens are the cap. */
3010
3010
  maxOutputTokens?: number;
3011
3011
  };
3012
3012
  type ReevalConfig = {
@@ -3415,6 +3415,14 @@ type AssertionRecord = {
3415
3415
  /** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
3416
3416
  durationMs?: number;
3417
3417
  };
3418
+ /** A `c.observe(name, value)` call: the last value given for the name. */
3419
+ type Observation = {
3420
+ name: string;
3421
+ /** Masked like the rest of the record. */
3422
+ value: JSONValue;
3423
+ /** How many turns the attempt had sent when the value was given: 0 before the first. */
3424
+ turn: number;
3425
+ };
3418
3426
  /**
3419
3427
  * A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
3420
3428
  * folder), or why it wasn't.
@@ -3505,7 +3513,7 @@ type SandboxWarning = {
3505
3513
  file: string | null;
3506
3514
  /** The capabilities added back, for `capabilities`. */
3507
3515
  capabilities?: string[];
3508
- /** The line reeval prints, after `WARNING: ` (root) or `warning: ` (capabilities). */
3516
+ /** The line reeval prints after `⚠ Warning:`, in red for root and orange for capabilities. */
3509
3517
  message: string;
3510
3518
  };
3511
3519
  /** Where a cell's harness resolved in the `gateway` list, and the entries before it that couldn't serve it. */
@@ -3516,5 +3524,5 @@ type ResolvedGateway = {
3516
3524
  why: string;
3517
3525
  }>;
3518
3526
  };
3519
- export { FakeAnswer as $, formatSeconds as $n, TranscriptEvent as $t, CompareCut as A, EFFORT_MIN_RUNS as An, PairFactor as At, EgressPolicy as B, cellEffort as Bn, QuestionInputs as Bt, CaptureWindow as C, Lift as Cn, JudgeOptions as Ct, ClaudeCodeHarnessDescriptor as D, CellEffort as Dn, OpenCodeHarnessDescriptor as Dt, ClaudeCodeEffort as E, UpstreamCount as En, McpServer as Et, Diff as F, Spread as Fn, PluginSource as Ft, EnvironmentInput as G, effortLabel as Gn, SandboxWarning as Gt, EnvVars as H, contrastEffort as Hn, ReevalConfig as Ht, DirectGatewayDescriptor as I, TOKENS_UNREPORTED as In, Profile as It, EvalFn as J, effortSummary as Jn, ServeHandle as Jt, EnvironmentSpec as K, effortOf as Kn, Secret as Kt, DirectoryOptions as L, VERDICT_TEXT as Ln, ProfileDefaults as Lt, ConfigHelpers as M, EffortMetric as Mn, PiHarnessDescriptor as Mt, ConfigJudgeOptions as N, EffortSample as Nn, PinnableKey as Nt, CodexHarnessDescriptor as O, DIRECTIONAL_MIN as On, OpenRouterGatewayDescriptor as Ot, CursorHarnessDescriptor as P, EffortVerdict as Pn, PluginPart as Pt, FakeAnalysisModel as Q, formatRatio as Qn, TokenUsage as Qt, DockerSandboxDescriptor as R, bothUnderATenth as Rn, ProfileLimit as Rt, CaptureWait as S, JudgmentRecord as Sn, JudgeModel as St, CellRole as T, StackSummary as Tn, JudgeSubject as Tt, Environment as U, directional as Un, ResolvedGateway as Ut, EgressRecord as V, cellSamples as Vn, QuestionOptions as Vt, EnvironmentHelpers as W, effortChangeLabel as Wn, SandboxDescriptor as Wt, ExecResult as X, filesChanged as Xn, SkillSource as Xt, EvalOptions as Y, effortVerdict as Yn, SetupStep as Yt, ExecRetry as Z, formatCount as Zn, StatusRange as Zt, CapturePage as _, FamilyRate as _n, JSONValue as _t, AttemptContext as a, VertexGatewayDescriptor as an, FxHarnessDescriptor as at, CaptureUseAction as b, HistoryCell as bn, JudgeFailure as bt, BlindPair as c, Wait as cn, GooseHarnessDescriptor as ct, CaptureCall as d, AttemptRecord as dn, HarnessData as dt, Turn as en, tokenValues as er, FakeHarnessDescriptor as et, CaptureConsoleMessage as f, CellRecord as fn, HarnessDescriptor as ft, CaptureOptions as g, FamilyLiftSummary as gn, HostFile as gt, CaptureKey as h, FamilyLift as hn, HistoryConfig as ht, AttemptAnswer as i, Verdict as in, FakeVote as it, CompareOptions as j, EffortContrast as jn, PairVote as jt, CompareAnswer as k, EFFORT_ALPHA as kn, Outcome as kt, Budget as l, WebEventType as ln, GrokBuildHarnessDescriptor as lt, CaptureInstruction as m, EvalEntry as mn, HarnessOption as mt, AnalysisConfig as n, TurnStatus as nn, DeclaredNode as nr, FakeSandboxDescriptor as nt, BackgroundHandle as o, VertexPublisher as on, GatewayDescriptor as ot, CaptureFailedRequest as p, CheckTally as pn, HarnessEdit as pt, ErrorKind as q, effortSample as qn, SecretTemplate as qt, AntigravityHarnessDescriptor as r, VercelGatewayDescriptor as rn, FakeScriptStep as rt, BlindAttempt as s, Viewport as sn, GatewayName as st, AgentHarnessDescriptor as t, TurnData as tn, DeclarationTree as tr, FakeJudgeModel as tt, Capture as u, ArtifactRef as un, GroupOptions as ut, CapturePageFile as v, FamilyRecord as vn, JudgeArtifactRef as vt, Cell as w, RunRecord as wn, JudgeState as wt, CaptureUseScript as x, HistoryEntry as xn, JudgeInputs as xt, CaptureUse as y, FileRef as yn, JudgeCallOptions as yt, Duration as z, cellContrast as zn, Question as zt };
3520
- //# sourceMappingURL=types-BHR1lGPh.d.ts.map
3527
+ export { FakeAnalysisModel as $, effortVerdict as $n, StatusRange as $t, CompareCut as A, UpstreamCount as An, OpenRouterGatewayDescriptor as At, EgressPolicy as B, TOKENS_UNREPORTED as Bn, ProfileLimit as Bt, CaptureWindow as C, HistoryCell as Cn, JudgeModel as Ct, ClaudeCodeHarnessDescriptor as D, RunInvocation as Dn, McpServer as Dt, ClaudeCodeEffort as E, Lift as En, JudgeSubject as Et, Diff as F, EffortContrast as Fn, PinnableKey as Ft, EnvironmentInput as G, cellSamples as Gn, ResolvedGateway as Gt, EnvVars as H, bothUnderATenth as Hn, QuestionInputs as Ht, DirectGatewayDescriptor as I, EffortMetric as In, PluginPart as It, EvalFn as J, effortChangeLabel as Jn, Secret as Jt, EnvironmentSpec as K, contrastEffort as Kn, SandboxDescriptor as Kt, DirectoryOptions as L, EffortSample as Ln, PluginSource as Lt, ConfigHelpers as M, DIRECTIONAL_MIN as Mn, PairFactor as Mt, ConfigJudgeOptions as N, EFFORT_ALPHA as Nn, PairVote as Nt, CodexHarnessDescriptor as O, RunRecord as On, Observation as Ot, CursorHarnessDescriptor as P, EFFORT_MIN_RUNS as Pn, PiHarnessDescriptor as Pt, ExecRetry as Q, effortSummary as Qn, SkillSource as Qt, DockerSandboxDescriptor as R, EffortVerdict as Rn, Profile as Rt, CaptureWait as S, FileRef as Sn, JudgeInputs as St, CellRole as T, JudgmentRecord as Tn, JudgeState as Tt, Environment as U, cellContrast as Un, QuestionOptions as Ut, EgressRecord as V, VERDICT_TEXT as Vn, Question as Vt, EnvironmentHelpers as W, cellEffort as Wn, ReevalConfig as Wt, EvalOptionsData as X, effortOf as Xn, ServeHandle as Xt, EvalOptions as Y, effortLabel as Yn, SecretTemplate as Yt, ExecResult as Z, effortSample as Zn, SetupStep as Zt, CapturePage as _, FamilyLift as _n, HostFile as _t, AttemptContext as a, VercelGatewayDescriptor as an, FakeVote as at, CaptureUseAction as b, FamilyRecord as bn, JudgeCallOptions as bt, BlindPair as c, VertexPublisher as cn, GatewayName as ct, CaptureCall as d, WebEventType as dn, GroupOptions as dt, TokenUsage as en, filesChanged as er, FakeAnswer as et, CaptureConsoleMessage as f, ArtifactRef as fn, HarnessData as ft, CaptureOptions as g, EvalEntry as gn, HistoryConfig as gt, CaptureKey as h, CheckTally as hn, HarnessOption as ht, AttemptAnswer as i, TurnStatus as in, tokenValues as ir, FakeScriptStep as it, CompareOptions as j, CellEffort as jn, Outcome as jt, CompareAnswer as k, StackSummary as kn, OpenCodeHarnessDescriptor as kt, Budget as l, Viewport as ln, GooseHarnessDescriptor as lt, CaptureInstruction as m, CellRecord as mn, HarnessEdit as mt, AnalysisConfig as n, Turn as nn, formatRatio as nr, FakeJudgeModel as nt, BackgroundHandle as o, Verdict as on, FxHarnessDescriptor as ot, CaptureFailedRequest as p, AttemptRecord as pn, HarnessDescriptor as pt, ErrorKind as q, directional as qn, SandboxWarning as qt, AntigravityHarnessDescriptor as r, TurnData as rn, formatSeconds as rr, FakeSandboxDescriptor as rt, BlindAttempt as s, VertexGatewayDescriptor as sn, GatewayDescriptor as st, AgentHarnessDescriptor as t, TranscriptEvent as tn, formatCount as tr, FakeHarnessDescriptor as tt, Capture as u, Wait as un, GrokBuildHarnessDescriptor as ut, CapturePageFile as v, FamilyLiftSummary as vn, JSONValue as vt, Cell as w, HistoryEntry as wn, JudgeOptions as wt, CaptureUseScript as x, FamilyVerdict as xn, JudgeFailure as xt, CaptureUse as y, FamilyRate as yn, JudgeArtifactRef as yt, Duration as z, Spread as zn, ProfileDefaults as zt };
3528
+ //# sourceMappingURL=types-DGvexkcp.d.ts.map
@@ -1,4 +1,5 @@
1
1
  import { n as UsageError } from "./errors-ttMTnIYX.js";
2
+ import { a as claudeCodeVertexAliasRefusal } from "./harness-DYSC4zMh.js";
2
3
  import { generateKeyPairSync } from "node:crypto";
3
4
  // src/gateway/token.ts
4
5
  /** How long before a token expires it's replaced. */
@@ -397,10 +398,12 @@ function vertexModelProbe(model, settings) {
397
398
  /**
398
399
  * Why Vertex AI can't serve a routed harness's model, or undefined when it can: fx's Chat
399
400
  * Completions endpoint and OpenCode's `google-vertex` provider serve Google's models, and Claude
400
- * only over Anthropic's own API there.
401
+ * only over Anthropic's own API there; Claude Code's `sonnet` and `opusplan` aliases name a
402
+ * retiring model there.
401
403
  * A bare fx id is OpenAI's everywhere else, so only a bare `gemini-…` one is taken as Google's.
402
404
  */
403
405
  function vertexModelRefusalFor(h) {
406
+ if (h.kind === "claude-code") return claudeCodeVertexAliasRefusal(h.model);
404
407
  const provider = /^([^/]+)\//.exec(h.model)?.[1];
405
408
  if (h.kind === "opencode") return provider === "google" || provider === "google-vertex" ? void 0 : `OpenCode reaches Vertex AI only on its google-vertex provider, which serves Google's models, not ${JSON.stringify(h.model)}: use a Gemini model, such as google/gemini-3.8-flash`;
406
409
  if (h.kind !== "fx") return void 0;
@@ -453,4 +456,4 @@ async function vertexDecisionModel(model, settings, source = vertexTokens()) {
453
456
  }
454
457
  export { brokerToken as S, vertexRegion as _, VERTEX_ROUTES as a, vertexTokens as b, openCodeVertexId as c, vertexHarnessModel as d, vertexHost as f, vertexModelRefusalFor as g, vertexModelRefusal as h, VERTEX_REFUSALS as i, vertexClaudeId as l, vertexModelProbe as m, VERTEX_MODEL_PROVIDERS as n, adcTokenSource as o, vertexLanguageModel as p, VERTEX_PROJECT_ENV as r, fxVertexId as s, SANDBOX_ACCESS_TOKEN as t, vertexDecisionModel as u, vertexSettings as v, vertexXaiId as x, vertexSettingsIn as y };
455
458
 
456
- //# sourceMappingURL=vertex-6uSpZ5L8.js.map
459
+ //# sourceMappingURL=vertex-BLQMayc1.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vertex-BLQMayc1.js","names":[],"sources":["../src/gateway/token.ts","../src/gateway/vertex.ts"],"mappings":";;;;;AAkBA,MAAa,mBAAmB;;AAEhC,MAAa,iBAAiB;;AAE9B,MAAa,WAAW;;;;;AAMxB,SAAgB,kBAAkB,QAAqB,MAAM,KAAK,KAAkB;CAClF,IAAI;CACJ,IAAI;CACJ,aAAa;EACX,IAAI,WAAW,QAAQ,YAAA,MAA+B,IAAI,GAAG,OAAO,QAAQ,QAAQ,OAAO;EAC3F,YAAY,OAAO,CAAC,CACjB,MAAM,UAAU;GACf,UAAU;GACV,OAAO;EACT,CAAC,CAAC,CACD,cAAc;GACb,UAAU,KAAA;EACZ,CAAC;EACH,OAAO;CACT;AACF;AAIA,MAAM,cAAwB,KAAK,OAAO;CACxC,MAAM,QAAQ,WAAW,KAAK,EAAE;CAChC,MAAM,MAAM;CACZ,aAAa,aAAa,KAAK;AACjC;;;;;;AAYA,eAAsB,YACpB,UACA,SACA,UAKI,CAAC,GACgB;CACrB,MAAM,EAAE,MAAM,KAAK,KAAK,WAAW,eAAe;CAClD,MAAM,MAAM,QAAQ,2BAA2B,KAAK,OAAO;CAC3D,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,+CAA+C;CACzE,IAAI,UAAU;CACd,IAAI,eAAe,CAAC;CACpB,MAAM,QAAQ,YAAY;EACxB,MAAM,QAAQ,MAAM,SAAS,OAAO;EACpC,IAAI,CAAC,SAAS,MAAM,IAAI,SAAS,gBAAgB,MAAM,KAAK,CAAC;EAC7D,OAAO;CACT;CACA,MAAM,QAAQ,UAAuB;EACnC,IAAI,SAAS;EACb,MAAM,OAAO,KAAK,IAAI,MAAM,YAAY,mBAAmB,IAAI,GAAG,cAAc;EAChF,SAAS,eAAe,KAAK,QAAQ,GAAG,IAAI;CAC9C;CACA,MAAM,UAAU,YAAY;EAC1B,IAAI;GACF,MAAM,QAAQ,MAAM,MAAM;GAC1B,QAAQ,YAAY,KAAK;GACzB,KAAK,KAAK;EACZ,SAAS,OAAO;GACd,QAAQ,UAAU,KAAK;GACvB,IAAI,CAAC,SAAS,SAAS,eAAe,KAAK,QAAQ,GAAG,QAAQ;EAChE;CACF;CACA,KAAK,MAAM,MAAM,CAAC;CAClB,aAAa;EACX,UAAU;EACV,OAAO;CACT;AACF;;;ACtFA,MAAa,qBAAqB;;AAUlC,SAAgB,eACd,QAA+C,CAAC,GAChD,MAAyB,QAAQ,KACjB;CAChB,MAAM,UAAU,MAAM,WAAW,IAAA;CACjC,IAAI,CAAC,SACH,MAAM,IAAI,WACR,+DAA+D,oBACjE;CAEF,OAAO;EAAE;EAAS,UAAU,MAAM,YAAY;EAAU,SAAS,MAAM,WAAW,CAAC;CAAE;AACvF;;AAGA,SAAgB,iBACd,UACA,MAAyB,QAAQ,KACjB;CAChB,MAAM,QAAQ,SAAS,MAAM,MAAM,EAAE,SAAS,QAAQ;CACtD,OAAO,eAAe,OAAO,SAAS,WAAW,QAAQ,CAAC,GAAG,GAAG;AAClE;;AAGA,MAAa,gBAAgB,UAA0B,cACrD,SAAS,QAAQ,cAAc,SAAS;;AAG1C,MAAa,cAAc,WACzB,WAAW,WAAW,8BAA8B,GAAG,OAAO;AAEhE,MAAM,iBAAiB;;AAEvB,MAAM,sBAAsB;;AAG5B,SAAgB,eAAe,MAAM,KAAK,KAAkB;CAC1D,IAAI;CACJ,OAAO,YAAY;EACjB,WAAW,OAAO,sBAAsB,CAAC,MAAM,EAAE,iBAC/C,IAAI,WAAW,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,UAAU,CACzD;EACA,MAAM,OAAO,MAAM;EACnB,MAAM,EAAE,UAAU,MAAM,KAAK,eAAe;EAC5C,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,sDAAsD;EAClF,MAAM,SAAU,KAA2D,aACvE;EACJ,OAAO;GAAE;GAAO,WAAW,OAAO,WAAW,WAAW,SAAS,IAAI,IAAI;EAAoB;CAC/F;AACF;AAEA,IAAI;;AAEJ,MAAa,qBAAkC;CAC7C,WAAW,kBAAkB,eAAe,CAAC;CAC7C,OAAO;AACT;;;;;;AAsCA,MAAa,kBAAkB,UAC7B,MAAM,QAAQ,gBAAgB,EAAE,CAAC,CAAC,QAAQ,aAAa,KAAK;AAE9D,MAAM,eAAe,SAAiB,KAAK,QAAQ,uBAAuB,MAAM;;AAGhF,SAAS,kBACP,UACA,WACA,SACA,WAA8B,CAAC,IAAI,GACpB;CACf,MAAM,SAAS,aAAa,UAAU,SAAS;CAC/C,OAAO;EACL,MAAM,WAAW,MAAM;EACvB,MAAM,EACJ,OAAO,QAAQ,SAAS,KAAK,GAAG,EAAE,aAAa,YAAY,SAAS,OAAO,EAAE,aAAa,YAAY,MAAM,EAAE,cAAc,UAAU,qBAAqB,QAAQ,KAAK,GAAG,EAAE,cAC/K;EACA,QAAQ,CAAC,MAAM;CACjB;AACF;;AAGA,MAAM,YACH,eACA,UACC,UAAU,KAAK,WAAW;CACxB;CACA,WAAW,EAAE,SAAS,EAAE,eAAe,UAAU,QAAQ,EAAE;AAC7D,EAAE;;;;;;AAON,SAAS,WAAW,UAA0B,SAAS,aAAa,GAAgB;CAClF,MAAM,SAAS,aAAa,UAAU,WAAW;CACjD,MAAM,YAAY,CAAC,kBAAkB,UAAU,aAAa,CAAC,oBAAoB,YAAY,CAAC,CAAC;CAC/F,OAAO;EACL,YAAY;EACZ;EACA,OAAO,CAAC,WAAW,MAAM,CAAC;EAC1B,cAAc,EAAE,MAAM,WAAW;GAC/B,IAAI,SAAS,WAAW,MAAM,GAAG,OAAO,KAAA;GACxC,MAAM,QAAQ,6CAA6C,KAAK,IAAI,CAAC,GAAG;GACxE,OAAO,SAAS,UAAU,iBAAiB,QAAQ,KAAA;EACrD;EACA,SAAS;EACT,UAAU;GACR,wBAAwB;GACxB,8BAA8B;GAC9B,6BAA6B,SAAS;GACtC,iBAAiB;EACnB;EACA,OAAO;GAAE;GAAQ,iBAAiB,SAAS,SAAS;EAAE;CACxD;AACF;;AAGA,MAAM,iBAAgC;CACpC,MAAM;CACN,MAAM,EAAE,OAAO,SAAS;CACxB,QAAQ,CAAC,MAAM;AACjB;;AAGA,MAAa,uBAAuB;AAEpC,IAAI;;AAEJ,MAAM,0BAA0B;CAC9B,eAAe,oBAAoB,OAAO;EACxC,eAAe;EACf,oBAAoB;GAAE,MAAM;GAAS,QAAQ;EAAM;EACnD,mBAAmB;GAAE,MAAM;GAAQ,QAAQ;EAAM;CACnD,CAAC,CAAC,CAAC;CACH,OAAO;AACT;AAEA,MAAM,kBAAkB,YAAoB,kBAAkB,QAAQ;;AAGtE,SAAS,WAAW,UAAsC;CACxD,MAAM,UAAU,KAAK,UAAU;EAC7B,MAAM;EACN,YAAY,SAAS;EACrB,gBAAgB;EAChB,aAAa,kBAAkB;EAC/B,cAAc,eAAe,SAAS,OAAO;EAC7C,WAAW;EACX,WAAW,WAAW,eAAe,KAAK;CAC5C,CAAC;CACD,OAAO;EACL,SAAS,SAAS;EAClB,UAAU,aAAa,UAAU,QAAQ;EACzC,MAAM;EACN;CACF;AACF;AAEA,MAAM,QAAQ,SAAkB,KAAK,UAAU,IAAI;;AAGnD,MAAM,gBAA+B;CACnC,OAAO;CACP,QAAQ;CACR,MAAM,KAAK;EAAE,cAAc;EAAsB,YAAY;EAAM,YAAY;CAAS,CAAC;AAC3F;;;;;;AAOA,SAAS,OACP,aAAgC,CAAC,GACjC,uBAA+E,CAAC,GAChF,WAAsC,UAAU,MAAM,QAAQ,aAAa,EAAE,GAC7E;CACA,QAAQ,UAA0B,SAAS,aAAa,MAAmB;EACzE,MAAM,SAAS,aAAa,UAAU,QAAQ;EAC9C,MAAM,YAAY,CAChB,kBACE,UACA,UACA,CAAC,yBAAyB,iBAAiB,GAC3C,CAAC,MAAM,SAAS,CAClB,CACF;EACA,OAAO;GACL,YAAY;GACZ;GACA,OAAO;IAAC,WAAW,MAAM;IAAG,eAAe;IAAM,GAAG;GAAU;GAC9D,cAAc,EAAE,MAAM,WACpB,SAAS,WAAW,MAAM,IACtB,0CAA0C,KAAK,IAAI,CAAC,GAAG,KACvD,KAAA;GACN;GACA,OAAO;IAAE;IAAQ,iBAAiB,SAAS,SAAS;GAAE;GACtD,KAAK,WAAW,QAAQ;GACxB,gBAAgB,CAAC,eAAe,GAAG,eAAe,QAAQ,CAAC;EAC7D;CACF;AACF;;AAGA,MAAM,uBAAuB,aAA8C,CACzE;CACE,OAAO;EAAE,MAAM;EAAsB,MAAM,EAAE,OAAO,sBAAsB;EAAG,QAAQ,CAAC,KAAK;CAAE;CAC7F,QAAQ;CACR,MAAM,KAAK;EAAE,IAAI;EAAK,OAAO,eAAe,SAAS,OAAO;EAAG,gBAAgB;CAAK,CAAC;AACvF,CACF;;AAGA,MAAa,eAAe,UAAmB,MAAM,SAAS,GAAG,IAAI,QAAQ,OAAO;;;;;;AAOpF,SAAS,UAAU,UAA0B,SAAS,aAAa,GAAgB;CACjF,MAAM,SAAS,aAAa,UAAU,KAAK;CAC3C,MAAM,YAA6B,CACjC;EACE,MAAM,WAAW,MAAM;EACvB,MAAM,EAEJ,OAAO,iBAAiB,YAAY,mBAAmB,SAAS,OAAO,CAAC,EAAE,aAAa,YAAY,MAAM,EAAE,gDAC7G;EACA,QAAQ,CAAC,MAAM;CACjB,CACF;CACA,OAAO;EACL,YAAY;EACZ;EACA,gBAAgB;EAChB,OAAO,CAAC,WAAW,MAAM,CAAC;EAC1B,cAAc,EAAE,MAAM,oBACpB,SAAS,WAAW,MAAM,IAAI,gBAAgB,KAAA;EAChD,SAAS;EACT,YAAY;GAAE,SAAS,SAAS;GAAS;EAAO;EAChD,OAAO;GAAE;GAAQ,iBAAiB,SAAS,SAAS;EAAE;CACxD;AACF;;AAGA,MAAa,oBAAoB,UAC/B,iBAAiB,MAAM,QAAQ,+BAA+B,EAAE;;AAGlE,MAAa,cAAc,UAAkB,UAAU,MAAM,QAAQ,aAAa,EAAE;;;;;;;AAQpF,SAAS,GAAG,UAA0B,SAAS,aAAa,GAAgB;CAC1E,MAAM,SAAS,aAAa,UAAU,QAAQ;CAC9C,MAAM,OAAO,WAAW,MAAM;CAC9B,MAAM,YAA6B,CACjC;EACE;EACA,MAAM,EACJ,OAAO,6BAA6B,YAAY,SAAS,OAAO,EAAE,aAAa,YAAY,MAAM,EAAE,gDACrG;EACA,QAAQ,CAAC,MAAM;CACjB,CACF;CACA,OAAO;EACL,YAAY;EACZ;EACA,gBAAgB;EAChB,OAAO,CAAC,IAAI;EACZ,cAAc,YACZ,QAAQ,SAAS,OAAO,QAAQ,eAAe,QAAQ,aAAa,EAAE,IAAI,KAAA;EAC5E,SAAS;EACT,UAAU;GAAE,mBAAmB,SAAS;GAAS,oBAAoB;EAAO;EAC5E,OAAO;GAAE;GAAQ,iBAAiB,SAAS,SAAS;EAAE;CACxD;AACF;;AAGA,MAAa,gBAET;CACF,eAAe;CAEf,aAAa,OAAO,CAAC,oBAAoB,GAAG,sBAAsB,UAAU,KAAK;CAEjF,OAAO,OAAO;CAEd,IAAI,OAAO;CAGX,UAAU,OAAO,CAAC,YAAY,SAAS,CAAC,GAAG,gBAAgB;CAC3D,cAAc;CACd;AACF;;;;;AAMA,MAAM,4BAAyE;CAC7E,eAAe;CACf,aAAa;CACb,OAAO;CACP,IAAI;CACJ,UAAU;CACV,IAAI;AACN;;;;;;AAOA,SAAgB,mBAAmB,MAAiB,OAAmC;CACrF,MAAM,YAAY,0BAA0B;CAC5C,IAAI,CAAC,WAAW,OAAO,KAAA;CACvB,MAAM,KAAK,MAAM,QAAQ,yCAAyC,EAAE;CACpE,OAAO,GAAG,UAAU,GAAG,SAAS,gBAAgB,GAAG,QAAQ,yBAAyB,EAAE,IAAI;AAC5F;;;;;;AAOA,SAAgB,iBACd,OACA,UAC4D;CAC5D,MAAM,YAAY,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;CACnD,MAAM,SAAS,aAAa,UAAU,SAAS;CAC/C,MAAM,OAAO,WAAW,WAAW,MAAM,EAAE,eAAe,mBAAmB,SAAS,OAAO,EAAE,aAAa,OAAO,cAAc,UAAU;CAC3I,MAAM,KAAK,MAAM,MAAM,UAAU,SAAS,CAAC;CAC3C,IAAI,cAAc,aAAa;EAC7B,MAAM,SAAS,eAAe,EAAE;EAChC,OAAO;GACL,IAAI;GACJ;GACA,KAAK,GAAG,KAAK;GACb,MAAM;IAAE,OAAO;IAAQ,UAAU,CAAC;KAAE,MAAM;KAAQ,SAAS;IAAO,CAAC;GAAE;EACvE;CACF;CACA,OAAO;EACL;EACA;EACA,KAAK,GAAG,KAAK,GAAG,GAAG;EACnB,MAAM,EAAE,UAAU,CAAC;GAAE,MAAM;GAAQ,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC;EAAE,CAAC,EAAE;CAClE;AACF;;;;;;;;AASA,SAAgB,sBAAsB,GAA+C;CACnF,IAAI,EAAE,SAAS,eAAe,OAAO,6BAA6B,EAAE,KAAK;CACzE,MAAM,WAAW,aAAa,KAAK,EAAE,KAAK,CAAC,GAAG;CAC9C,IAAI,EAAE,SAAS,YACb,OAAO,aAAa,YAAY,aAAa,kBACzC,KAAA,IACA,oGAAoG,KAAK,UAAU,EAAE,KAAK,EAAE;CAElI,IAAI,EAAE,SAAS,MAAM,OAAO,KAAA;CAC5B,IAAI,aAAa,YAAa,CAAC,YAAY,WAAW,KAAK,EAAE,KAAK,GAAI,OAAO,KAAA;CAC7E,OAAO,iGAAiG,KAAK,UAAU,EAAE,KAAK,EAAE;AAClI;;AAGA,MAAa,kBAAsD;CACjE,QAAQ;CACR,OAAO;AACT;;AAGA,MAAa,yBAAyB,CAAC,UAAU,WAAW;;AAG5D,MAAa,sBAAsB,UAChC,uBAA6C,SAAS,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC,CAAC,IACrF,KAAA,IACA,qBAAqB,uBAAuB,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,OAAO,EAAE;;;;;AAMrF,eAAsB,oBACpB,OACA,UACA,SAAsB,aAAa,GACnC;CACA,MAAM,UAAU,mBAAmB,KAAK;CACxC,IAAI,SAAS,MAAM,IAAI,MAAM,GAAG,QAAQ,SAAS,MAAM,GAAG;CAC1D,MAAM,WAAW,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;CAClD,MAAM,KAAK,MAAM,MAAM,SAAS,SAAS,CAAC;CAC1C,MAAM,oBAAoB,EACxB,YAAY,EAAE,gBAAgB,aAAa,EAAE,QAAQ,MAAM,OAAO,EAAA,CAAG,MAAM,GAAG,EAChF;CAGA,IAAI,aAAa,aAAa;EAC5B,MAAM,EAAE,0BAA0B,MAAM,OAAO;EAC/C,MAAM,WAAW,aAAa,UAAU,WAAW;EACnD,OAAO,sBAAsB;GAAE,SAAS,SAAS;GAAS;GAAU;EAAkB,CAAC,CAAC,CACtF,eAAe,EAAE,CACnB;CACF;CACA,MAAM,EAAE,iBAAiB,MAAM,OAAO;CACtC,MAAM,WAAW,aAAa,UAAU,QAAQ;CAChD,OAAO,aAAa;EAAE,SAAS,SAAS;EAAS;EAAU;CAAkB,CAAC,CAAC,CAAC,EAAE;AACpF;;AAGA,eAAsB,oBACpB,OACA,UACA,SAAsB,aAAa,GACnC;CACA,MAAM,EAAE,uCAAuC,MAAM,OACnD;CAEF,OAAO,IAAI,mCAAmC;EAC5C,OAAO,MAAM,oBAAoB,OAAO,UAAU,MAAM;EACxD,UAAU;CACZ,CAAC;AACH"}
@@ -0,0 +1 @@
1
+ export { o as adcTokenSource } from "./vertex-BLQMayc1.js";
@@ -1,8 +1,8 @@
1
- import { $n as formatSeconds, $t as TranscriptEvent, An as EFFORT_MIN_RUNS, Bn as cellEffort, Cn as Lift, Dn as CellEffort, En as UpstreamCount, Fn as Spread, Gn as effortLabel, Gt as SandboxWarning, Hn as contrastEffort, In as TOKENS_UNREPORTED, Jn as effortSummary, Kn as effortOf, Ln as VERDICT_TEXT, Mn as EffortMetric, Nn as EffortSample, On as DIRECTIONAL_MIN, Pn as EffortVerdict, Qn as formatRatio, Qt as TokenUsage, Rn as bothUnderATenth, Sn as JudgmentRecord, Tn as StackSummary, Un as directional, Vn as cellSamples, Wn as effortChangeLabel, Xn as filesChanged, Yn as effortVerdict, Zn as formatCount, _ as CapturePage, _n as FamilyRate, bn as HistoryCell, dn as AttemptRecord, er as tokenValues, f as CaptureConsoleMessage, fn as CellRecord, gn as FamilyLiftSummary, hn as FamilyLift, jn as EffortContrast, kn as EFFORT_ALPHA, kt as Outcome, mn as EvalEntry, nn as TurnStatus, p as CaptureFailedRequest, pn as CheckTally, q as ErrorKind, qn as effortSample, un as ArtifactRef, v as CapturePageFile, vn as FamilyRecord, wn as RunRecord, xn as HistoryEntry, yn as FileRef, zn as cellContrast } from "../types-BHR1lGPh.js";
1
+ import { $n as effortVerdict, An as UpstreamCount, Bn as TOKENS_UNREPORTED, Cn as HistoryCell, Dn as RunInvocation, En as Lift, Fn as EffortContrast, Gn as cellSamples, Hn as bothUnderATenth, In as EffortMetric, Jn as effortChangeLabel, Kn as contrastEffort, Ln as EffortSample, Mn as DIRECTIONAL_MIN, Nn as EFFORT_ALPHA, On as RunRecord, Pn as EFFORT_MIN_RUNS, Qn as effortSummary, Rn as EffortVerdict, Sn as FileRef, Tn as JudgmentRecord, Un as cellContrast, Vn as VERDICT_TEXT, Wn as cellEffort, Xn as effortOf, Yn as effortLabel, Zn as effortSample, _ as CapturePage, _n as FamilyLift, bn as FamilyRecord, en as TokenUsage, er as filesChanged, f as CaptureConsoleMessage, fn as ArtifactRef, gn as EvalEntry, hn as CheckTally, in as TurnStatus, ir as tokenValues, jn as CellEffort, jt as Outcome, kn as StackSummary, mn as CellRecord, nr as formatRatio, p as CaptureFailedRequest, pn as AttemptRecord, q as ErrorKind, qn as directional, qt as SandboxWarning, rr as formatSeconds, tn as TranscriptEvent, tr as formatCount, v as CapturePageFile, vn as FamilyLiftSummary, wn as HistoryEntry, xn as FamilyVerdict, yn as FamilyRate, zn as Spread } from "../types-DGvexkcp.js";
2
2
  /** Bumped when the system prompt or output schema changes: an older analysis is superseded. */
3
- export declare const ANALYSIS_PROMPT_VERSION = "analysis-v12";
3
+ export declare const ANALYSIS_PROMPT_VERSION = "analysis-v13";
4
4
  /** Bumped when the digest's shape or selection rules change. */
5
- export declare const DIGEST_VERSION = 13;
5
+ export declare const DIGEST_VERSION = 15;
6
6
  export type Confidence = 'high' | 'medium' | 'low';
7
7
  export type AnalysisClaim = {
8
8
  text: string;
@@ -393,10 +393,10 @@ export type JudgeKeyRate = {
393
393
  question: string;
394
394
  pass: number;
395
395
  counted: number;
396
- /** Compact lift vs the paired control, only when both sides have enough counted. */
396
+ /** Compact lift vs the paired control, when both sides counted an attempt. */
397
397
  lift: Pick<Lift, 'points' | 'control' | 'variant' | 'withinNoise'> | null;
398
398
  };
399
- /** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides qualify. */
399
+ /** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides counted an attempt. */
400
400
  export declare function cellJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey: string): JudgeKeyRate[];
401
401
  /**
402
402
  * The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
@@ -500,6 +500,8 @@ export type LiveProgress = {
500
500
  planned: number;
501
501
  counts: LiveCounts;
502
502
  attempts: LiveAttempt[];
503
+ /** How the run was started, for a `--resume` after a quit that wrote no record. */
504
+ invocation?: RunInvocation;
503
505
  };
504
506
  /** A live run as the studio lists it: `stopped` when its process is gone or went quiet. */
505
507
  export type LiveRun = Omit<LiveProgress, 'schemaVersion' | 'pid' | 'host' | 'heartbeatAt'> & {
@@ -513,6 +515,8 @@ export declare const liveTranscriptPath: (runId: string, attemptKey: string) =>
513
515
  export declare const FINISHED: readonly LiveStatus[];
514
516
  export declare const isFinished: (status: LiveStatus) => boolean;
515
517
  export declare function liveCounts(attempts: readonly Pick<LiveAttempt, 'status'>[]): LiveCounts;
518
+ /** A saved invocation, or undefined when it isn't one; its numbers are whole and at least 1. */
519
+ export declare function readableInvocation(v: unknown): RunInvocation | undefined;
516
520
  /**
517
521
  * A progress file as the studio reads it, or null when it isn't one it can read: half-written,
518
522
  * corrupt, or from a newer reeval. Counts are recomputed from the attempts, never trusted.
@@ -585,16 +589,15 @@ export declare const shortSha: (sha: string | null | undefined) => string;
585
589
  /** Reasoning time, or `n/a` when the harness couldn't measure it: never `0s` for unknown. */
586
590
  export declare const reasoningLabel: (ms: number | null) => string;
587
591
  /**
588
- * An attempt's effort, counted as the CLI's turn lines count it: tool calls, then file edits.
589
- * Edit calls become one file edit per file they change, so they're counted apart, never summed.
590
- * With the workspace diff's count, `files changed` follows: an edit made from the shell is in it,
591
- * never in the file edits.
592
+ * An attempt's effort: tool calls, then the files the workspace diff changed, which counts a file
593
+ * the shell wrote, then the calls to the agent's edit tools, which doesn't.
592
594
  */
593
- export declare function activityLabel(a: Pick<AttemptRecord['activity'], 'tools' | 'fileEdits'>, changed?: number): string;
595
+ export declare function activityLabel(a: Pick<AttemptRecord['activity'], 'tools' | 'fileEdits'>, changed: number): string;
594
596
  /**
595
597
  * A run's outcome, which `reeval run`'s exit code is read from: an error outside any attempt, a
596
598
  * failed gated cell or a budget stop fails the run, while failures in ungated cells (controls, by
597
- * default) don't. An interrupted run is `skipped`, whatever ran.
599
+ * default) don't. An interrupted run is `skipped`, whatever ran; a resume its filters left
600
+ * `interrupted` exits on its outcome as if it were complete.
598
601
  */
599
602
  export declare function runOutcome(run: {
600
603
  status: RunRecord['status'];
@@ -606,6 +609,11 @@ export declare function runOutcome(run: {
606
609
  }>;
607
610
  errors?: readonly string[];
608
611
  }): Outcome;
612
+ /**
613
+ * The eval hasn't run yet: its run stopped (interrupted, or out of budget) or a filtered resume
614
+ * left it out, and it's `skipped` without being a todo or declared skipped. A resume runs it.
615
+ */
616
+ export declare const notRunYet: (run: Pick<RunRecord, 'status'>, e: Pick<EvalEntry, 'outcome' | 'todo' | 'skip'>) => boolean;
609
617
  /** Distinct judge labels that failed on a counted attempt, in first-seen order. */
610
618
  export declare function failedJudgeLabels(record: Pick<RunRecord, 'evals'>): string[];
611
619
  /**
@@ -852,6 +860,44 @@ export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string)
852
860
  export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
853
861
  /** The attempt to open by default: the first failure, else the first attempt. */
854
862
  export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
863
+ /** One observed name over one cell's attempts: its values tallied, or its numbers summarized. Never gates. */
864
+ export type ObservedTally = {
865
+ name: string;
866
+ /** The cell's key. */
867
+ cell: string;
868
+ /** Attempts that observed it. */
869
+ attempts: number;
870
+ /**
871
+ * Each string, boolean or null value, and each such item of an array value once per attempt,
872
+ * by how many attempts gave it, most first (ties in first-seen order); empty when none.
873
+ */
874
+ counts: Array<{
875
+ value: string;
876
+ n: number;
877
+ }>;
878
+ /** Over the number values; null when none. */
879
+ numbers: {
880
+ median: number;
881
+ min: number;
882
+ max: number;
883
+ n: number;
884
+ } | null;
885
+ /** Values neither tallied nor summarized: objects, and arrays with no scalar item. */
886
+ others: number;
887
+ };
888
+ /**
889
+ * Each name and cell any attempt observed it on, by name in first-observed order and then by
890
+ * cell, tallied over that cell's attempts.
891
+ */
892
+ export declare function observedTallies(attempts: ReadonlyArray<Pick<AttemptRecord, 'cell' | 'observations'>>): ObservedTally[];
893
+ /** `drizzle 9 · prisma 3`, `median 4 · range 1–12 · 13 attempts`, as the run output shows a tally. */
894
+ export declare function observedLine(t: ObservedTally): string;
895
+ /**
896
+ * How one name's tally on one cell moved from base to head, or null when it didn't:
897
+ * `prisma 6 → 1 · drizzle 2 → 8` (each value whose count changed, head's most first),
898
+ * `median 4 → 9`, or a side's whole line against `none` when only one side observed it.
899
+ */
900
+ export declare function observedShift(base: ObservedTally | undefined, head: ObservedTally | undefined): string | null;
855
901
  export type PatchFile = {
856
902
  path: string;
857
903
  status: 'added' | 'deleted' | 'modified' | 'renamed' | 'binary';
@@ -937,7 +983,7 @@ export type Remark = {
937
983
  */
938
984
  id: string;
939
985
  evalId: string;
940
- kind: 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'slow-mcp-start' | 'proxy-only-egress' | 'setup-process-secret' | 'snapshot-too-large';
986
+ kind: 'too-few-tries' | 'cant-tell-apart' | 'check-never-fails' | 'flaky-check' | 'slow-mcp-start' | 'proxy-only-egress' | 'setup-process-secret' | 'snapshot-too-large';
941
987
  /** Fixed by kind (`REMARK_KINDS`). */
942
988
  severity: 'concern' | 'suggestion';
943
989
  /** One plain sentence: what's wrong. */
@@ -982,19 +1028,67 @@ export declare const readableRemarks: (doc: RunRemarks, what?: string) => RunRem
982
1028
  /** A capture's `<name>.capture.json` as the studio sees it; one from a newer reeval throws. */
983
1029
  export declare const readableCapturePage: (page: CapturePageFile, what?: string) => CapturePageFile;
984
1030
  /**
985
- * Counted attempts a side before a pass-rate lift can be called: below it, every view says
986
- * `insufficient` (`can't tell yet`, `unclear`), whatever the interval. A product floor that keeps
987
- * `reeval run`, `reeval query`, the judge-key lift and `reeval analyze` agreeing, not one that
988
- * makes the stated 95% exact: at 4 a side, a 4/4 against 0/4 split happens by chance about 7% of
989
- * the time when nothing changed.
1031
+ * The runs a side a compare defaults to and the plan suggests. It doesn't gate the effect: a lift
1032
+ * always reads as its observed direction, and whether its interval excludes 0 decides whether it
1033
+ * holds. At 4 a side, a 4/4 against 0/4 split happens by chance about 7% of the time when nothing
1034
+ * changed.
990
1035
  */
991
- export declare const EFFECT_MIN_COUNTED = 4;
1036
+ export declare const COMPARE_RUNS = 4;
1037
+ type MeasureName = 'pass' | 'check' | 'judge' | 'question' | 'tools' | 'agentMs' | 'tokens';
992
1038
  /**
993
- * A variant against its baseline: `better` or `worse` when the stored 95% interval on the
994
- * difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
995
- * including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
1039
+ * `better` and `worse` are the measure's own test holding in that direction (for tool calls,
1040
+ * agent time and tokens, fewer is better); `leans` is a difference the test doesn't hold that
1041
+ * passes the lean screen (`screened`); `same` is every other difference, or none.
996
1042
  */
997
- export type Effect = 'better' | 'worse' | 'same' | 'unclear';
1043
+ type MeasureVerdict = 'better' | 'worse' | 'same' | 'leans';
1044
+ type Measure = {
1045
+ measure: MeasureName;
1046
+ /** The check's label, the judge's check, the question; null for the rest. */
1047
+ label: string | null;
1048
+ /**
1049
+ * Each side's value: a pass rate (0–1), a mean judge score, a question's decisive pairs won, or
1050
+ * a median count, milliseconds or tokens.
1051
+ */
1052
+ variant: number;
1053
+ control: number;
1054
+ /**
1055
+ * The difference: pass-rate points (-1–1) for pass rate and checks, the mean score's difference
1056
+ * for a judge, the variant's share of decisive pairs for a question, and variant over control
1057
+ * for effort (null when the control's is 0).
1058
+ */
1059
+ estimate: number | null;
1060
+ /** The 95% interval on pass-rate points; null for the other measures. */
1061
+ lo: number | null;
1062
+ hi: number | null;
1063
+ /** The test's two-sided p; null where the measure's verdict is an interval's (pass rate, checks). */
1064
+ p: number | null;
1065
+ verdict: MeasureVerdict;
1066
+ /** On a lean, which way it leans. */
1067
+ leaning: 'better' | 'worse' | null;
1068
+ /** `5× faster`, `same pass rate`, `judge "tidy" +0.12`. */
1069
+ text: string;
1070
+ /**
1071
+ * On a lean, the runs a side (a case, on a family) or the cases (`unit`) at which a lean this
1072
+ * size would hold; null past `RUNS_TO_HOLD_MAX`, or on any other verdict.
1073
+ */
1074
+ needed: number | null;
1075
+ unit: 'runs' | 'cases';
1076
+ /**
1077
+ * On a lean whose test can't reach its lean p at this sample at all (two runs a side, or three,
1078
+ * for a rank test): the runs a side or cases (`unit`) it has (`now`) and the fewest at which it
1079
+ * can (`at`). Null otherwise.
1080
+ */
1081
+ short: {
1082
+ now: number;
1083
+ at: number;
1084
+ } | null;
1085
+ };
1086
+ /**
1087
+ * A variant against its baseline: the direction of their observed pass rates, at any number of
1088
+ * runs. Whether it holds is the interval's to say (`CellResult.interval`), and the
1089
+ * `too-few-tries` remark says how many runs would settle it.
1090
+ */
1091
+ export type Effect = 'better' | 'worse' | 'same';
998
1092
  /**
999
1093
  * What an eval's cells compare: `change` when it has variants (or several environments),
1000
1094
  * `harness` when it has several harnesses, `both`, or `none`.
@@ -1021,7 +1115,8 @@ export type SuiteResult = {
1021
1115
  status: RunRecord['status'];
1022
1116
  /**
1023
1117
  * The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
1024
- * an attempt, and `skipped` on an interrupted run.
1118
+ * an attempt, and `skipped` on an interrupted run. `reeval run --json` gives a resume that its
1119
+ * filters left `interrupted` the outcome of the attempts it ran, as its exit code.
1025
1120
  */
1026
1121
  outcome: Outcome;
1027
1122
  /**
@@ -1075,6 +1170,11 @@ export type EvalResult = {
1075
1170
  family: boolean;
1076
1171
  /** A placeholder (`evaluate.todo`) that never ran. */
1077
1172
  todo: boolean;
1173
+ /**
1174
+ * Hasn't run yet: skipped, but not a todo or declared skipped, in a run that isn't complete. On
1175
+ * a family, every case.
1176
+ */
1177
+ notRun: boolean;
1078
1178
  /** On a family: `fail` when a case failed or its pooled gate did. */
1079
1179
  outcome: ResultOutcome;
1080
1180
  cells: CellResult[] | null;
@@ -1086,6 +1186,8 @@ export type EvalResult = {
1086
1186
  * each question asked of any case, in first-asked order, summed over the cases that answered it.
1087
1187
  */
1088
1188
  questions: Array<QuestionResult | AttemptQuestionResult>;
1189
+ /** Each name its attempts observed with `c.observe`, tallied per cell; empty on a family. */
1190
+ observations: ObservedTally[];
1089
1191
  };
1090
1192
  /** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
1091
1193
  export type QuestionResult = {
@@ -1133,6 +1235,8 @@ export type CaseResult = {
1133
1235
  /** Its place in the case list. */
1134
1236
  order: number;
1135
1237
  outcome: ResultOutcome;
1238
+ /** Hasn't run yet, as `EvalResult.notRun`. */
1239
+ notRun: boolean;
1136
1240
  cells: CellResult[];
1137
1241
  };
1138
1242
  export type CellResult = {
@@ -1172,16 +1276,16 @@ export type CellResult = {
1172
1276
  * attempt (every one an infra error or skipped) lists each as `not run`.
1173
1277
  */
1174
1278
  checks: CheckResult[];
1175
- /** On a variant, against its baseline; null on a control. */
1279
+ /** On a variant, the direction of its pass rate against its baseline's; null on a control or with nothing counted on a side. */
1176
1280
  effect: Effect | null;
1177
1281
  /** The stored 95% interval on the variant's difference from its baseline. Not for display. */
1178
1282
  interval: Interval | null;
1179
1283
  /** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
1180
1284
  minLift: number | null;
1181
- /** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
1285
+ /** Medians over the counted attempts; null below `COMPARE_RUNS` of them. */
1182
1286
  effort: EffortCounters | null;
1183
- /** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
1184
- effortDiffers: boolean;
1287
+ /** On a variant, every measure against its baseline (`cellMeasures`); null on a control. */
1288
+ measures: Measure[] | null;
1185
1289
  /** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
1186
1290
  errors: CellError[];
1187
1291
  /**
@@ -1220,7 +1324,7 @@ export type CheckResult = {
1220
1324
  order: number;
1221
1325
  outcome: CheckOutcome;
1222
1326
  /**
1223
- * On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
1327
+ * On a variant, this check against the baseline's (null when either side never answered it);
1224
1328
  * null on a control or without one.
1225
1329
  */
1226
1330
  effect: Effect | null;
@@ -1284,21 +1388,22 @@ export type PooledArm = {
1284
1388
  /** Cases with counted attempts on both sides. */
1285
1389
  pairs: number;
1286
1390
  interval: Interval;
1391
+ /** The pooled lift's verdict: `insufficient` below `PAIRED_MIN` discordant pairs or cases. */
1392
+ verdict: FamilyVerdict;
1287
1393
  /** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
1288
1394
  minLift: number | null;
1289
1395
  outcome: 'pass' | 'fail';
1396
+ /** Every measure over the cases (`familyMeasures`). */
1397
+ measures: Measure[];
1290
1398
  };
1291
1399
  type Side = {
1292
1400
  pass: number;
1293
1401
  counted: number;
1294
1402
  };
1295
- /**
1296
- * The effect of `variant` against `control`, from the interval on their difference. A side
1297
- * with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
1298
- */
1299
- export declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
1300
- /** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
1301
- export declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
1403
+ /** The direction of `variant`'s pass rate against `control`'s; null when a side counted none. */
1404
+ export declare function effectOf(variant: Side, control: Side): Effect | null;
1405
+ /** The check's direction on the variant against the baseline; null when a side never answered it. */
1406
+ export declare const checkEffect: (variant: CheckTally, control: CheckTally) => Effect | null;
1302
1407
  /** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
1303
1408
  export declare const factSegment: (s: string) => string;
1304
1409
  export declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;