reeval 0.14.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/README.md +3 -3
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
  9. package/dist/app-files-ChbvRx3P.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
  11. package/dist/assemble-SKQGW_QF.js.map +1 -0
  12. package/dist/assemble-XBWuK3Ed.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
  16. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
  18. package/dist/capture-BM_3100g.js.map +1 -0
  19. package/dist/capture-DSK8O5bA.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
  21. package/dist/cli-PdYrcoKM.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
  23. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
  26. package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
  27. package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
  28. package/dist/command-4pJYVCT4.js.map +1 -0
  29. package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
  30. package/dist/command-9xm5ImqI.js.map +1 -0
  31. package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
  32. package/dist/command-BqBxLgTc.js.map +1 -0
  33. package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
  34. package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
  35. package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
  36. package/dist/commands-BU6YJCz9.js.map +1 -0
  37. package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
  38. package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
  39. package/dist/config-k6m7XbAz.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-CpsifVln.js +1530 -0
  46. package/dist/db-CpsifVln.js.map +1 -0
  47. package/dist/db-DkD-F-X-.js +2 -0
  48. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  49. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  50. package/dist/debug-DeM5pswK.js +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
  52. package/dist/docker-51WvhN_X.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
  54. package/dist/docker-B4-rx4dA.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
  57. package/dist/doctor-BuQnfz4e.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
  61. package/dist/environment-BOFgzFDg.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  63. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  64. package/dist/eval-source-Dje3rERU.js +145 -0
  65. package/dist/eval-source-Dje3rERU.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
  67. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  68. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  69. package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
  70. package/dist/files-Bi461Be2.js.map +1 -0
  71. package/dist/files-C2h2Cw4F.js +1 -0
  72. package/dist/files-Clipx4OW.js +37 -0
  73. package/dist/files-Clipx4OW.js.map +1 -0
  74. package/dist/files-DmUrRmmL.js +1 -0
  75. package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
  76. package/dist/files-H24pz0hL.js.map +1 -0
  77. package/dist/gateway.d.ts +22 -9
  78. package/dist/gateway.js +30 -8
  79. package/dist/gateway.js.map +1 -1
  80. package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
  81. package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
  82. package/dist/harness-3NVyTSGK.js +392 -0
  83. package/dist/harness-3NVyTSGK.js.map +1 -0
  84. package/dist/harness.d.ts +12 -8
  85. package/dist/harness.js +1 -1
  86. package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
  87. package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
  88. package/dist/index.d.ts +51 -10
  89. package/dist/index.js +5 -5
  90. package/dist/init-BDEOopK7.js +1 -0
  91. package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
  92. package/dist/init-cOk3D44b.js.map +1 -0
  93. package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
  94. package/dist/invocations-DW-lTk2f.js.map +1 -0
  95. package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
  96. package/dist/judge-CKdgEXN7.js.map +1 -0
  97. package/dist/judge.d.ts +5 -7
  98. package/dist/judge.js +4 -8
  99. package/dist/judge.js.map +1 -1
  100. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  101. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  102. package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
  103. package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
  104. package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
  105. package/dist/model-Bq3G0OG1.js.map +1 -0
  106. package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
  107. package/dist/model-CTnyvh7H.js.map +1 -0
  108. package/dist/model-D-SWrLZG.js +1 -0
  109. package/dist/model-g3XnzUv0.js +1 -0
  110. package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
  111. package/dist/model-vRiUycWh.js.map +1 -0
  112. package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
  113. package/dist/node-BVbbnFtv.js.map +1 -0
  114. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  115. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  116. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  117. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  118. package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
  119. package/dist/pairs-puYi-zj2.js.map +1 -0
  120. package/dist/patch-CquGAGX5.js +76 -0
  121. package/dist/patch-CquGAGX5.js.map +1 -0
  122. package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
  123. package/dist/patch-DKJiC5Sq.js.map +1 -0
  124. package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
  125. package/dist/plan-DdWkM6SW.js.map +1 -0
  126. package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
  127. package/dist/plugins-D-CP6mMd.js.map +1 -0
  128. package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
  129. package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
  130. package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
  131. package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
  132. package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
  133. package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
  134. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  135. package/dist/prices-QaS-kjSQ.js.map +1 -0
  136. package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
  137. package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
  138. package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
  139. package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
  140. package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
  141. package/dist/query-Clm2mNSm.js +1 -0
  142. package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
  143. package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
  144. package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
  145. package/dist/questions-BRhU8MzB.js.map +1 -0
  146. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  147. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  148. package/dist/redact-DVIaahLv.js +658 -0
  149. package/dist/redact-DVIaahLv.js.map +1 -0
  150. package/dist/remarks-BwJ8pcTQ.js +1 -0
  151. package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
  152. package/dist/remarks-C7i2LMRb.js.map +1 -0
  153. package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
  154. package/dist/remarks-D_3qhBWU.js.map +1 -0
  155. package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
  156. package/dist/remarks-dA8LyoPc.js.map +1 -0
  157. package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
  158. package/dist/remote-DFIzcx_6.js.map +1 -0
  159. package/dist/resolve--rQZ8u6Z.js +1 -0
  160. package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
  161. package/dist/resolve-DWSJpKHX.js.map +1 -0
  162. package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
  163. package/dist/results-BC3pcG_Z.js.map +1 -0
  164. package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
  165. package/dist/resume-j-rcIryi.js.map +1 -0
  166. package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
  167. package/dist/runtime-d56Gg3J5.js.map +1 -0
  168. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
  169. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
  170. package/dist/sandbox.d.ts +1 -1
  171. package/dist/sandbox.js +1 -1
  172. package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
  173. package/dist/server-BOoH10J8.js.map +1 -0
  174. package/dist/server-DpuVAbrO.js +1 -0
  175. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  176. package/dist/shared-B_lvGqWz.js.map +1 -0
  177. package/dist/source-BFd6f0-R.js +1 -0
  178. package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
  179. package/dist/source-wD04-rRG.js.map +1 -0
  180. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  181. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  182. package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
  183. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  184. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  185. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  186. package/dist/src-BO8AYqJ5.js.map +1 -0
  187. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  188. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  189. package/dist/src-BYPMM1x7.js +2 -0
  190. package/dist/src-Bp4aN3rA.js +2 -0
  191. package/dist/src-C7upnZSd.js +2 -0
  192. package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
  193. package/dist/src-D91Hd6hT.js.map +1 -0
  194. package/dist/src-DH-9IO7f.js +1 -0
  195. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  196. package/dist/src-DSZyg0Uq.js.map +1 -0
  197. package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
  198. package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
  199. package/dist/src-HSxBazcg.js +3 -0
  200. package/dist/src-LD1BAJFa.js +2 -0
  201. package/dist/src-dlX-Lmxg.js +2 -0
  202. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  203. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  204. package/dist/src-kYeaJJrl.js +2 -0
  205. package/dist/state-B5PLm5qY.js +1 -0
  206. package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
  207. package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
  208. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  209. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  210. package/dist/suite-22Kt9Grx.js +2 -0
  211. package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
  212. package/dist/suite-Co8cwelb.js.map +1 -0
  213. package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
  214. package/dist/summary-DLFHVoux.js.map +1 -0
  215. package/dist/testing.d.ts +417 -142
  216. package/dist/testing.js +12 -10
  217. package/dist/testing.js.map +1 -1
  218. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  219. package/dist/text-DTigcA64.js.map +1 -0
  220. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  221. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  222. package/dist/types-C5vI3g1C.js.map +1 -1
  223. package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
  224. package/dist/vertex-Bv7vhMgs.js +1 -0
  225. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  226. package/dist/vertex-DypYjx71.js +456 -0
  227. package/dist/vertex-DypYjx71.js.map +1 -0
  228. package/dist/view/index.d.ts +536 -121
  229. package/dist/view/index.js +9 -9
  230. package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
  231. package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
  232. package/dist/views-BOf8qEL8.js +2 -0
  233. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
  234. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  235. package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
  236. package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
  237. package/dist/workspace-DDHzkMjm.js +1 -0
  238. package/docker/seccomp-default.json +1 -0
  239. package/package.json +5 -1
  240. package/skills/eval-rubric/SKILL.md +35 -104
  241. package/dist/adapter-BloGVZ_a.js.map +0 -1
  242. package/dist/agent-m-_t-IuW.js.map +0 -1
  243. package/dist/app-files-C-jwhRDt.js.map +0 -1
  244. package/dist/assemble-BDEpelpi.js.map +0 -1
  245. package/dist/assemble-C5bfncHU.js +0 -1
  246. package/dist/capture-CVRwRTnb.js.map +0 -1
  247. package/dist/capture-CherbjzB.js +0 -1
  248. package/dist/cli-C_4gaJ50.js.map +0 -1
  249. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  250. package/dist/command-BWAuwQNk.js +0 -91
  251. package/dist/command-BWAuwQNk.js.map +0 -1
  252. package/dist/command-BXLUfw4h.js.map +0 -1
  253. package/dist/command-Be6BSE2L.js +0 -335
  254. package/dist/command-Be6BSE2L.js.map +0 -1
  255. package/dist/command-Bxdy62dW.js.map +0 -1
  256. package/dist/commands-jzigbNFb.js.map +0 -1
  257. package/dist/config-BrdHA9bJ.js.map +0 -1
  258. package/dist/cost-D4elx933.js.map +0 -1
  259. package/dist/cost-DmnAod9x.js +0 -1
  260. package/dist/db-CAfksinX.js +0 -2
  261. package/dist/db-DO0p7xgP.js +0 -619
  262. package/dist/db-DO0p7xgP.js.map +0 -1
  263. package/dist/debug-D7bEv_UV.js +0 -1
  264. package/dist/docker-BnRlTpZA.js.map +0 -1
  265. package/dist/docker-D01slDF3.js.map +0 -1
  266. package/dist/doctor-DQ2p0euG.js.map +0 -1
  267. package/dist/env-BQqj93gu.js.map +0 -1
  268. package/dist/environment-CO0P0soF.js.map +0 -1
  269. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  270. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  271. package/dist/facts-DzOUTreB.js +0 -906
  272. package/dist/facts-DzOUTreB.js.map +0 -1
  273. package/dist/files-B-CJeeEh.js.map +0 -1
  274. package/dist/files-Bp5j5BqI.js +0 -1
  275. package/dist/files-C2sjmXhk.js +0 -1
  276. package/dist/files-C_gQNVB-.js +0 -65
  277. package/dist/files-C_gQNVB-.js.map +0 -1
  278. package/dist/files-kUCrswI6.js.map +0 -1
  279. package/dist/generate-DZ7vp69F.js +0 -1647
  280. package/dist/generate-DZ7vp69F.js.map +0 -1
  281. package/dist/harness-COVXSwJ6.js +0 -354
  282. package/dist/harness-COVXSwJ6.js.map +0 -1
  283. package/dist/header-BLPP6aCU.js.map +0 -1
  284. package/dist/init-RgCQXZhb.js +0 -1
  285. package/dist/init-SRalM5xF.js.map +0 -1
  286. package/dist/insights-TyAvz9aV.js +0 -68
  287. package/dist/insights-TyAvz9aV.js.map +0 -1
  288. package/dist/invocations-DzyKSJBa.js.map +0 -1
  289. package/dist/judge-yLvjC5A3.js.map +0 -1
  290. package/dist/match-Bz0nSdzN.js.map +0 -1
  291. package/dist/model-1ZLr9Dcv.js.map +0 -1
  292. package/dist/model-B45lDUV1.js.map +0 -1
  293. package/dist/model-BTZq8HIZ.js +0 -1
  294. package/dist/model-CgnOpss4.js.map +0 -1
  295. package/dist/model-Ch8kX87u.js +0 -1
  296. package/dist/node-BSq5qHnQ.js.map +0 -1
  297. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  298. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  299. package/dist/pairs-CHfZNRT6.js.map +0 -1
  300. package/dist/patch-DWf-I_Q_.js.map +0 -1
  301. package/dist/plan-BIwUzlB8.js.map +0 -1
  302. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  303. package/dist/prices-BcLEXvdY.js.map +0 -1
  304. package/dist/query-Cvbwl804.js +0 -1
  305. package/dist/questions-bAudIc04.js.map +0 -1
  306. package/dist/redact-CRGdfV2s.js +0 -240
  307. package/dist/redact-CRGdfV2s.js.map +0 -1
  308. package/dist/remarks-CXwSnSjR.js +0 -278
  309. package/dist/remarks-CXwSnSjR.js.map +0 -1
  310. package/dist/remarks-CyAu2DiB.js +0 -1
  311. package/dist/remarks-mujJv1Ey.js.map +0 -1
  312. package/dist/remote-CKcEKja4.js.map +0 -1
  313. package/dist/resolve-B5-uqmJ6.js +0 -1
  314. package/dist/resolve-CBoKzQMP.js.map +0 -1
  315. package/dist/results-CMgZEANy.js.map +0 -1
  316. package/dist/resume-B9SOSnK9.js.map +0 -1
  317. package/dist/rules-VhftPOOT.js.map +0 -1
  318. package/dist/run-CwIEK16q.js +0 -88
  319. package/dist/run-CwIEK16q.js.map +0 -1
  320. package/dist/runtime-BgQRyCLl.js.map +0 -1
  321. package/dist/server-CHRSgKRp.js +0 -1
  322. package/dist/server-TGW4WyAz.js.map +0 -1
  323. package/dist/shared-BDUzStad.js.map +0 -1
  324. package/dist/source-B1WvBIQt.js.map +0 -1
  325. package/dist/source-DAzztZwU.js +0 -1
  326. package/dist/src-1jHbFaGW.js +0 -1
  327. package/dist/src-8zoX_qrb.js.map +0 -1
  328. package/dist/src-C_9wgowC.js +0 -2
  329. package/dist/src-ChhtI6BP.js.map +0 -1
  330. package/dist/src-D005Cbqz.js +0 -2
  331. package/dist/src-DIfRbp5a.js +0 -3
  332. package/dist/src-Dg8121Xy.js +0 -2
  333. package/dist/src-ElBqEN-S.js +0 -2
  334. package/dist/src-NSdQ9zI6.js.map +0 -1
  335. package/dist/src-YHGaUrGE.js +0 -2
  336. package/dist/src-mJVxw1Jm.js +0 -2
  337. package/dist/state-vznMh2wl.js +0 -1
  338. package/dist/suite-BsWGHzlG.js +0 -2
  339. package/dist/suite-WgT7m6Vs.js.map +0 -1
  340. package/dist/summary-DIGEVr08.js.map +0 -1
  341. package/dist/text-CrVy5ioL.js.map +0 -1
  342. package/dist/transcript-BTxJpW4R.js.map +0 -1
  343. package/dist/usage-DF3_p5pE.js +0 -1
  344. package/dist/validate-BhwqrVsm.js.map +0 -1
  345. package/dist/views-BW_C5eVH.js +0 -2
  346. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  347. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,16 +1,23 @@
1
- import { l as sha256 } from "./ids-Bp6ut7ZD.js";
2
- import { $ as DEFAULT_QUESTION_STATE, tt as questionShows } from "./config-BrdHA9bJ.js";
3
- import { n as clip } from "./text-CrVy5ioL.js";
4
- import { c as sampleReasons, d as splitsLabel, i as pairTallies, n as headToHeadLines } from "./pairs-CHfZNRT6.js";
1
+ import { n as UsageError } from "./errors-B6BSMs03.js";
2
+ import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
3
+ import { at as questionShows, rt as DEFAULT_QUESTION_STATE } from "./config-k6m7XbAz.js";
4
+ import { n as clip, r as fenced } from "./text-DTigcA64.js";
5
+ import { a as pairTallies, f as splitsLabel, l as sampleReasons, r as headToHeadLines } from "./pairs-puYi-zj2.js";
6
+ import "./stats-hfpuYsAM.js";
7
+ import { l as resolveAnalysisModel, n as callAnalysisModel, t as analysisModelName, u as SYSTEM_PROMPT } from "./model-Bq3G0OG1.js";
5
8
  import { n as formatTokens } from "./usage-BNY1ABrU.js";
6
- import "./analysis-C2-aEbr7.js";
7
- import { $ as bothUnderATenth, W as labelledJudgements, _t as compare } from "./model-CgnOpss4.js";
8
- import "./stats-DSy5HfOv.js";
9
- import { g as familyRateLabel, h as familyLiftLabel, p as familyEffortLabel } from "./suite-WgT7m6Vs.js";
10
- import { t as runHeader } from "./header-BLPP6aCU.js";
11
- import { i as checkCommand, o as commandSummary } from "./cli-spec-CpnbATtI.js";
12
- import { r as contrastsOf } from "./contrasts-DRxMQQTC.js";
13
- import { i as TOOL_CLASSES } from "./facts-DzOUTreB.js";
9
+ import { t as ANALYSIS_PROMPT_VERSION } from "./analysis-C2-aEbr7.js";
10
+ import { Q as bothUnderATenth, U as labeledJudgments, gt as compare } from "./model-CTnyvh7H.js";
11
+ import { g as familyRateLabel, h as familyLiftLabel, p as familyEffortLabel } from "./suite-Co8cwelb.js";
12
+ import { r as runHeader } from "./remarks-D_3qhBWU.js";
13
+ import { i as checkCommand, o as commandSummary } from "./cli-spec-D0qyrPZi.js";
14
+ import { o as parseRecord, p as scratchDir } from "./files-Bi461Be2.js";
15
+ import { a as pickBaseline, o as resolveRun, t as RefError } from "./views-BFO1BLKu.js";
16
+ import { c as lines, n as allRuns, t as CommandError, u as reachability } from "./commands-BU6YJCz9.js";
17
+ import { r as contrastsOf } from "./contrasts-DLZF4dGF.js";
18
+ import { l as TOOL_CLASSES, s as openHistoryDb } from "./db-CpsifVln.js";
19
+ import { t as author } from "./provenance-BizwUn3h.js";
20
+ import { a as readCached, c as writeCached, o as saveAnalysis } from "./files-H24pz0hL.js";
14
21
  import { basename, join } from "node:path";
15
22
  import { readFile } from "node:fs/promises";
16
23
  //#region src/analysis/digest.ts
@@ -20,8 +27,9 @@ const RULES = [
20
27
  `Effort (tool calls, agent time, and tokens when both sides have usage samples) is compared with an exact Mann-Whitney test: "fewer"/"more" at p < 0.05; "insufficient" when no split of the attempts' values can reach it (2 vs 2 cannot, and ties raise the floor) — never because a harness dropped usage. Attempts without usage are left out of the token sample only. If either side then has no token sample, the token metric is omitted (no line, not insufficient). A Goose cell does not blank tokens on Cursor cells in the same study. Pooled across evals, tokens are an effect only when at least 2/3 of the strata that reported usage agree. Effort's separated (given from 3 vs 3) says, per metric, that the two sides' ranges do not overlap: every attempt on one side beat every attempt on the other. insufficient with separated true is a directional signal: report the ratio, the n and "no overlap, not significant" as directional, never as an effect. separated false is no direction at all.`,
21
28
  "A family fact is one pooled evaluate.each: its cases are paired samples of one suite, so its lift (Newcombe paired interval and McNemar at one attempt a case, a t interval over cases otherwise) is the result for those cases; their own pass-rate facts carry no lift. insufficient (fewer than 6 discordant pairs or 6 cases) is not measured. A family's minLift gates the pooled lower bound.",
22
29
  "Each factor claim has a tier: controlled (two cells in one run that differ in one factor), pooled (the same contrast over comparable runs, and across evals), observed (a change between runs, which is weaker; \"confounded\" when several parts changed), not_measured (one level only).",
23
- `A cell with mixed outcomes is flaky only with at least 6 counted attempts over comparable runs; below that it is "insufficient", not flaky.`,
30
+ `A cell with mixed outcomes is flaky only with at least 6 counted attempts over comparable runs; below that it is "mixed": it passed and failed, too few times to call it flaky.`,
24
31
  "Drift marked noise changed only an agent CLI's own runtime state (e.g. ~/.cursor/cli-config.json): not a real change, and those runs stay comparable.",
32
+ "Drift on the gate is a policy change: the gate scores the same attempts, so the runs stay comparable. A compare row with status gate flipped only because its gate did (gate.rescored, base's attempts under head's gate, matches head): never call it a regression or a fix.",
25
33
  "An eval where every cell passes at least 95% is saturated: pass@1 cannot tell cells apart there, effort can.",
26
34
  "Judge verdicts carry no recorded reason; an explanation of a failed verdict is your reading of the cited output, and must say so.",
27
35
  "A criteria fact lists each check (expect label, judges included) per cell: rows only for labels whose rate differs between cells, or is mixed within one; neverMet labels failed on every counted attempt of every cell, alwaysMet passed on all of them. A neverMet or alwaysMet label is the same in every arm: it can't explain a difference between arms.",
@@ -30,7 +38,7 @@ const RULES = [
30
38
  "A context-use fact counts each attempt's tool calls by class (context, dependency, web, verify, workspace) and lists which placed files each arm opened: how much an arm leaned on what its environment declared versus digging in installed packages. The classes are a heuristic over tool names and paths; cite them as counts, not as proof of intent.",
31
39
  "A contrast fact (kind contrast) is one pair of arms on one harness, a against b, when an eval has two variants or more: pass rate and lift of a over b, the checks that tell them apart (discriminates or mixed; sameOnBoth counts the rest) and effort. It answers how two variants differ from each other, not only from the control; gating stays each variant against its control.",
32
40
  "A check (kind checks) whose verdict is both-pass or both-fail passes, or fails, on both the variant and its control: it is no evidence for or against the variant. Only discriminates rows say which requirement separated them.",
33
- "Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check. Each pair is voted in both orders; splits counts pairs whose two orders picked opposite sides (the position decided, not the work), and rates count such a pair as one win each. With more than two arms, read headToHead (a–b wins per pair, splits apart), not the pooled rates. shows lists what each side was shown: a question that shows output or transcript can favour the better-written account rather than the better work, and one that shows artifacts rests on screenshots."
41
+ "Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check. Each pair is voted in both orders; splits counts pairs whose two orders picked opposite sides (the position decided, not the work), and rates count such a pair as one win each. With more than two arms, read headToHead (a–b wins per pair, splits apart), not the pooled rates. shows lists what each side was shown: a question that shows output or transcript can favor the better-written account rather than the better work, and one that shows artifacts rests on screenshots."
34
42
  ];
35
43
  const pts = (x) => x === null || x === void 0 ? null : `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
36
44
  const liftText = (points, lo, hi) => `${pts(points)} pts (95% ${pts(lo)}…${pts(hi)})`;
@@ -46,10 +54,12 @@ function digestLiftVerdict(variantCounted, controlCounted, withinNoise) {
46
54
  function evalChangedLift() {
47
55
  return "not comparable: the eval changed";
48
56
  }
57
+ /** What the eval's own source is keyed on: `source` and `revision`, else (older records) its files. */
49
58
  function sourcesKey(e) {
59
+ if (e.source) return JSON.stringify([e.source, e.revision ?? null]);
50
60
  return JSON.stringify(e.sources.map((s) => [s.path, s.sha256]));
51
61
  }
52
- /** The eval's sources, or the sources part of its fingerprint, differ across the two runs. */
62
+ /** The eval's own source, or the sources part of its fingerprint, differs across the two runs. */
53
63
  function evalSourcesChanged(base, head) {
54
64
  if (sourcesKey(base) !== sourcesKey(head)) return true;
55
65
  const prev = new Map(base.cells.map((c) => [c.key, c.fingerprintParts.sources]));
@@ -129,10 +139,10 @@ function criteriaOf(e, tag) {
129
139
  const scores = /* @__PURE__ */ new Map();
130
140
  for (const a of e.attempts) {
131
141
  if (!counted(a)) continue;
132
- for (const j of labelledJudgements(a)) {
142
+ for (const j of labeledJudgments(a)) {
133
143
  if (j.check === void 0) continue;
134
144
  const id = `${a.cell}\0${j.check}`;
135
- scores.set(id, [...scores.get(id) ?? [], j.judgement.score]);
145
+ scores.set(id, [...scores.get(id) ?? [], j.judgment.score]);
136
146
  }
137
147
  }
138
148
  const rows = [];
@@ -191,7 +201,7 @@ async function buildDigestKeepingDropped(input) {
191
201
  const any = a.files.transcript ?? a.files.judgeInput ?? a.checks.find((c) => c.log)?.log;
192
202
  const assertions = a.assertions ?? [];
193
203
  const failed = failedLabels(a);
194
- const judges = labelledJudgements(a);
204
+ const judges = labeledJudgments(a);
195
205
  attempts[id] = {
196
206
  run: short.get(r.record.runId),
197
207
  runId: r.record.runId,
@@ -205,7 +215,7 @@ async function buildDigestKeepingDropped(input) {
205
215
  agentS: Math.round(a.timings.agentMs / 1e3),
206
216
  ...assertions.length ? { checks: `${assertions.filter((x) => x.status === "pass").length}/${assertions.length}` } : {},
207
217
  ...failed.length ? { failed } : {},
208
- ...judges.length ? { judges: Object.fromEntries(judges.map((j) => [clip(j.label, LABEL_CHARS), round2(j.judgement.score)])) } : {},
218
+ ...judges.length ? { judges: Object.fromEntries(judges.map((j) => [clip(j.label, LABEL_CHARS), round2(j.judgment.score)])) } : {},
209
219
  execs: a.checks.length,
210
220
  artifacts: any ? `${r.record.runId}/${any.path.split("/").slice(0, 3).join("/")}` : null
211
221
  };
@@ -280,36 +290,39 @@ async function buildDigestKeepingDropped(input) {
280
290
  fact("effort", {
281
291
  run: short.get(runId),
282
292
  eval: tagOf(first.eval_key),
283
- cells: rows.map((row) => ({
284
- cell: cellTag(runId, row.eval_key, row.cell_key),
285
- n: row.n,
286
- medianToolCalls: row.tools_median,
287
- medianAgentS: row.agent_s_median,
288
- ...row.tokens_median != null ? { medianTokens: row.tokens_median } : {},
289
- ...row.workspace_calls_median != null ? { medianCallsByClass: Object.fromEntries(TOOL_CLASSES.flatMap((k) => {
293
+ cells: rows.map((row) => {
294
+ const byClass = TOOL_CLASSES.flatMap((k) => {
290
295
  const m = row[`${k}_calls_median`];
291
296
  return m ? [[k, m]] : [];
292
- })) } : {},
293
- ...row.control_key ? { vsControl: {
294
- n: `${row.n} vs ${row.control_n}`,
295
- toolCalls: `${row.tools_median} vs ${row.control_tools_median} (${ratioText(row.tools_ratio)})`,
296
- agentS: bothUnderATenth(Number(row.agent_s_median) * 1e3, Number(row.control_agent_s_median) * 1e3) ? "<0.1 vs <0.1" : `${row.agent_s_median} vs ${row.control_agent_s_median} (${ratioText(row.agent_s_ratio)})`,
297
- p: row.tools_p,
298
- verdict: row.verdict,
299
- timeVerdict: row.time_verdict,
300
- ...row.tokens_verdict ? {
301
- tokens: `${formatTokens(row.tokens_median)} vs ${formatTokens(row.control_tokens_median)} (${ratioText(row.tokens_ratio)})`,
302
- tokensP: row.tokens_p,
303
- tokensVerdict: row.tokens_verdict
304
- } : {},
305
- ...Math.min(Number(row.n), Number(row.control_n)) >= 3 ? { separated: {
306
- tools: row.separated === 1,
307
- agentS: row.time_separated === 1,
308
- ...row.tokens_separated != null ? { tokens: row.tokens_separated === 1 } : {}
309
- } } : {},
310
- ...row.tokens_note ? { unreportedUsage: row.tokens_note } : {}
311
- } } : {}
312
- }))
297
+ });
298
+ return {
299
+ cell: cellTag(runId, row.eval_key, row.cell_key),
300
+ n: row.n,
301
+ medianToolCalls: row.tools_median,
302
+ medianAgentS: row.agent_s_median,
303
+ ...row.tokens_median != null ? { medianTokens: row.tokens_median } : {},
304
+ ...byClass.length ? { medianCallsByClass: Object.fromEntries(byClass) } : {},
305
+ ...row.control_key ? { vsControl: {
306
+ n: `${row.n} vs ${row.control_n}`,
307
+ toolCalls: `${row.tools_median} vs ${row.control_tools_median} (${ratioText(row.tools_ratio)})`,
308
+ agentS: bothUnderATenth(Number(row.agent_s_median) * 1e3, Number(row.control_agent_s_median) * 1e3) ? "<0.1 vs <0.1" : `${row.agent_s_median} vs ${row.control_agent_s_median} (${ratioText(row.agent_s_ratio)})`,
309
+ p: row.tools_p,
310
+ verdict: row.verdict,
311
+ timeVerdict: row.time_verdict,
312
+ ...row.tokens_verdict ? {
313
+ tokens: `${formatTokens(row.tokens_median)} vs ${formatTokens(row.control_tokens_median)} (${ratioText(row.tokens_ratio)})`,
314
+ tokensP: row.tokens_p,
315
+ tokensVerdict: row.tokens_verdict
316
+ } : {},
317
+ ...Math.min(Number(row.n), Number(row.control_n)) >= 3 ? { separated: {
318
+ tools: row.separated === 1,
319
+ agentS: row.time_separated === 1,
320
+ ...row.tokens_separated != null ? { tokens: row.tokens_separated === 1 } : {}
321
+ } } : {},
322
+ ...row.tokens_note ? { unreportedUsage: row.tokens_note } : {}
323
+ } } : {}
324
+ };
325
+ })
313
326
  });
314
327
  }
315
328
  const changedEvalKeys = /* @__PURE__ */ new Set();
@@ -508,7 +521,7 @@ async function buildDigestKeepingDropped(input) {
508
521
  }) : failures.slice(0, 8);
509
522
  for (const { r, e, a } of failureFacts) {
510
523
  const failed = failedLabels(a);
511
- const judges = labelledJudgements(a).filter((j) => j.judgement.verdict === "fail");
524
+ const judges = labeledJudgments(a).filter((j) => j.judgment.verdict === "fail");
512
525
  fact("failure", {
513
526
  attempt: idOf.get(`${r.record.runId}\0${a.key}`),
514
527
  outcome: a.outcome,
@@ -518,9 +531,9 @@ async function buildDigestKeepingDropped(input) {
518
531
  ...failed.length ? { failed } : {},
519
532
  ...judges.length ? { judges: judges.map((j) => ({
520
533
  label: clip(j.label, LABEL_CHARS),
521
- question: clip(j.judgement.question, 300),
522
- score: j.judgement.score,
523
- threshold: j.judgement.threshold
534
+ question: clip(j.judgment.question, 300),
535
+ score: j.judgment.score,
536
+ threshold: j.judgment.threshold
524
537
  })) } : {},
525
538
  failingExecs: a.checks.map((c, i) => ({
526
539
  exec: i + 1,
@@ -561,16 +574,20 @@ async function buildDigestKeepingDropped(input) {
561
574
  cells: d.cells,
562
575
  changed: d.changed,
563
576
  ...d.changed_paths ? { paths: String(d.changed_paths).split(", ") } : {},
564
- verdict: d.noise ? "noise: agent runtime state only" : `real change: ${d.real_changed}`
577
+ verdict: d.noise ? "noise: agent runtime state only" : d.real_changed === "gate" ? "policy change: the gate scores the same attempts differently" : `real change: ${d.real_changed}`
565
578
  });
579
+ const [baseHeader, headHeader] = [runHeader(base.record, true), runHeader(head.record, true)];
580
+ const inScope = (id) => scopeEvals.some((e) => e.id === id);
581
+ const familyCases = new Map([...baseHeader.families ?? [], ...headHeader.families ?? []].map((f) => [f.id, f.caseIds]));
566
582
  fact("compare", {
567
583
  base: "r1",
568
584
  head: "r2",
569
- rows: compare(runHeader(base.record, true), runHeader(head.record, true)).filter((row) => scopeEvals.some((e) => e.id === row.id)).map((row) => ({
570
- eval: tagOf(scopeEvals.find((e) => e.id === row.id)?.key ?? row.id),
585
+ rows: compare(baseHeader, headHeader).filter((row) => row.kind === "family" ? (familyCases.get(row.id) ?? []).some(inScope) : inScope(row.id)).map((row) => ({
586
+ ...row.kind === "family" ? { family: row.id } : { eval: tagOf(scopeEvals.find((e) => e.id === row.id)?.key ?? row.id) },
571
587
  status: row.status,
572
588
  ...row.changes?.length ? { changes: row.changes } : {},
573
- ...row.cancelled ? { cancelled: row.cancelled } : {}
589
+ ...row.gate ? { gate: row.gate } : {},
590
+ ...row.canceled ? { canceled: row.canceled } : {}
574
591
  }))
575
592
  });
576
593
  }
@@ -732,8 +749,8 @@ async function buildDigestKeepingDropped(input) {
732
749
  for (const [i, { r, e, a }] of [...perCell.values()].slice(0, 6).entries()) {
733
750
  const id = idOf.get(`${r.record.runId}\0${a.key}`);
734
751
  const lines = await lastText(r.record.runId, a);
735
- const judges = labelledJudgements(a).filter((j) => j.judgement.verdict === "fail");
736
- for (const { label, judgement: j } of judges) lines.push(`judge ${clip(label, LABEL_CHARS)}: ${j.question}`, `judge score: ${j.score} (threshold ${j.threshold})`);
752
+ const judges = labeledJudgments(a).filter((j) => j.judgment.verdict === "fail");
753
+ for (const { label, judgment: j } of judges) lines.push(`judge ${clip(label, LABEL_CHARS)}: ${j.question}`, `judge score: ${j.score} (threshold ${j.threshold})`);
737
754
  for (const x of a.assertions ?? []) {
738
755
  if (x.status === "pass" || /(^|\.)toPass$/.test(x.matcher)) continue;
739
756
  lines.push(`check ${x.status}: ${clip(x.label, LABEL_CHARS)}: ${clip(x.message ?? "", 300)}`);
@@ -828,6 +845,77 @@ function citable(d) {
828
845
  };
829
846
  }
830
847
  //#endregion
848
+ //#region src/analysis/render.ts
849
+ const cite = (cs) => cs.length ? ` ${cs.map((c) => `[${c}]`).join(" ")}` : "";
850
+ const unverified = (numbers) => numbers?.length ? `unverified: ${numbers.join(", ")}` : "";
851
+ const note = (c) => [c.confidence === "high" ? "" : `${c.confidence} confidence`, unverified(c.unverifiedNumbers)].filter(Boolean).join("; ");
852
+ function footer(doc) {
853
+ const v = doc.validation;
854
+ const tokens = doc.usage.inputTokens !== null ? `${doc.usage.inputTokens} in / ${doc.usage.outputTokens ?? "?"} out tokens · ` : "";
855
+ return `${doc.resolvedModel ?? doc.model} · ${doc.promptVersion} · digest v${doc.inputs.digestVersion} · ${tokens}${Math.round(doc.latencyMs / 1e3)}s · ${v.citations} citations checked, ${v.invalid.length} invalid, ${v.dropped} dropped · ${v.commands} commands checked, ${v.invalidCommands.length} rejected`;
856
+ }
857
+ /** The analysis as Markdown (`--md`, and the `.md` twin of a saved analysis). */
858
+ function renderMarkdown(doc) {
859
+ const a = doc.analysis;
860
+ const md = [
861
+ `## reeval analysis · ${doc.scope.head}${doc.scope.base ? ` vs ${doc.scope.base}` : ""}`,
862
+ "",
863
+ `**TL;DR** (${a.confidence.overall}). ${a.tldr}`,
864
+ ""
865
+ ];
866
+ for (const [title, items] of [
867
+ ["What happened", a.whatHappened],
868
+ ["Results", a.results],
869
+ ["Why", a.why]
870
+ ]) {
871
+ if (!items.length) continue;
872
+ md.push(`### ${title}`, "");
873
+ for (const c of items) md.push(`- ${c.text}${cite(c.cites)}${note(c) ? ` *(${note(c)})*` : ""}`);
874
+ md.push("");
875
+ }
876
+ if (a.anomalies.length) {
877
+ md.push("### Anomalies", "");
878
+ for (const x of a.anomalies) md.push(`- **${x.kind}**: ${x.text}${cite(x.cites)}${unverified(x.unverifiedNumbers) ? ` *(${unverified(x.unverifiedNumbers)})*` : ""}`);
879
+ md.push("");
880
+ }
881
+ md.push("### Suggested actions", "");
882
+ if (!a.actions.length) md.push("None: nothing here is worth acting on.");
883
+ a.actions.forEach((x, i) => {
884
+ md.push(`${i + 1}. **${x.title}** (${x.kind} · impact ${x.impact} · effort ${x.effort}) `, ` Target: \`${x.target}\`. ${x.change} `, ` Expect: ${x.expectedEffect} `, x.verify ? ` Verify: \`${x.verify}\`${cite(x.cites)}` : ` Verify: none (the suggested command isn't reeval's CLI: ${x.verifyProblem ?? "unknown"})${cite(x.cites)}`);
885
+ });
886
+ md.push("", `**Confidence: ${a.confidence.overall}.** ${a.confidence.why}`, "", `<sub>${footer(doc)}</sub>`, "");
887
+ return md.join("\n");
888
+ }
889
+ /** The analysis for a terminal: the same sections, plain. */
890
+ function renderText(doc) {
891
+ const a = doc.analysis;
892
+ const lines = [
893
+ `analysis ${doc.scope.head}${doc.scope.base ? ` vs ${doc.scope.base}` : ""}`,
894
+ "",
895
+ `TL;DR (${a.confidence.overall}) ${a.tldr}`
896
+ ];
897
+ for (const [title, items] of [
898
+ ["What happened", a.whatHappened],
899
+ ["Results", a.results],
900
+ ["Why", a.why]
901
+ ]) {
902
+ if (!items.length) continue;
903
+ lines.push("", title);
904
+ for (const c of items) lines.push(` - ${c.text}${cite(c.cites)}${note(c) ? ` (${note(c)})` : ""}`);
905
+ }
906
+ if (a.anomalies.length) {
907
+ lines.push("", "Anomalies");
908
+ for (const x of a.anomalies) lines.push(` - ${x.kind}: ${x.text}${cite(x.cites)}${unverified(x.unverifiedNumbers) ? ` (${unverified(x.unverifiedNumbers)})` : ""}`);
909
+ }
910
+ lines.push("", "Suggested actions");
911
+ if (!a.actions.length) lines.push(" none: nothing here is worth acting on");
912
+ a.actions.forEach((x, i) => {
913
+ lines.push(` ${i + 1}. ${x.title} (${x.kind} · impact ${x.impact} · effort ${x.effort})${cite(x.cites)}`, ` target: ${x.target}. ${x.change}`, ` expect: ${x.expectedEffect}`, x.verify ? ` verify: ${x.verify}` : ` verify: none (${x.verifyProblem ?? "rejected"})`);
914
+ });
915
+ lines.push("", `Confidence: ${a.confidence.overall}. ${a.confidence.why}`, "", footer(doc));
916
+ return lines.join("\n");
917
+ }
918
+ //#endregion
831
919
  //#region src/analysis/validate.ts
832
920
  const EXEC = /^(.+):exec(\d+)$/;
833
921
  const LINE = /^(E\d+):L(\d+)$/;
@@ -1061,6 +1149,246 @@ function validateAnalysis(output, digest, tables, evals) {
1061
1149
  };
1062
1150
  }
1063
1151
  //#endregion
1064
- export { digestChars as a, buildDigestKeepingDropped as i, validateAnalysis as n, digestHash as o, RULES as r, unverifiedNumbers as t };
1152
+ //#region src/analysis/command.ts
1153
+ async function load(config, entry) {
1154
+ const dir = entry.recorded ? config.history.dir : scratchDir(config.root);
1155
+ const bytes = await readFile(join(dir, `${entry.runId}.json`));
1156
+ return {
1157
+ record: parseRecord(bytes, entry.runId),
1158
+ dir,
1159
+ bytes
1160
+ };
1161
+ }
1162
+ /** The model's input: the project's own instructions, which set the question, then the digest as JSON. */
1163
+ const promptOf = (d) => `${d.instructions ? `PROJECT INSTRUCTIONS (the question this analysis answers): ${d.instructions}\n\n` : ""}DIGEST:\n${JSON.stringify(d)}`;
1164
+ /** `--context`: the same digest and rules, for the user's own coding agent. */
1165
+ function contextMarkdown(d, head, base) {
1166
+ return [
1167
+ `# reeval analysis context: ${head}${base ? ` vs ${base}` : ""}`,
1168
+ "",
1169
+ "Analyze this reeval run for the engineer who wrote the evals. The digest below holds facts reeval computed (F…), evidence excerpts (E…, transcript lines as L<n>) and the attempts table. Cite those ids for every claim, keep to the verdicts in the facts, and suggest at most 5 concrete actions, each verified with a reeval command from the CLI list.",
1170
+ "",
1171
+ ...d.frame === "study" ? ["This run is a study: every cell is ungated (gate: 'none'), so it measures how the arms differ. Failing attempts are data, not a failed gate.", ""] : [],
1172
+ ...d.instructions ? [
1173
+ "## Project instructions (the question this analysis answers)",
1174
+ "",
1175
+ d.instructions,
1176
+ ""
1177
+ ] : [],
1178
+ "## How reeval decided each verdict",
1179
+ "",
1180
+ ...RULES.map((r) => `- ${r}`),
1181
+ "",
1182
+ "## Instructions reeval gives its own analysis model",
1183
+ "",
1184
+ SYSTEM_PROMPT,
1185
+ "",
1186
+ "## reeval CLI (verify commands must be one of these)",
1187
+ "",
1188
+ ...commandSummary().map((c) => `- \`${c}\``),
1189
+ "",
1190
+ "## To dig further",
1191
+ "",
1192
+ "- `reeval query effort`, `reeval query factor_effects`, `reeval query skill_reads`, `reeval query flaky_cells`, `reeval query drift`",
1193
+ `- \`reeval query "SELECT line, tool, target FROM tool_calls WHERE attempt_key = '<key>'"\`: an attempt's tool calls`,
1194
+ "- `reeval show <run> --json`: the whole record; transcripts are under the run's artifact folder",
1195
+ "",
1196
+ "## Digest",
1197
+ "",
1198
+ ...fenced([JSON.stringify(d, null, 1)], "json"),
1199
+ ""
1200
+ ].join("\n");
1201
+ }
1202
+ function citedSummary(d, cites) {
1203
+ const out = {};
1204
+ for (const c of cites) {
1205
+ const fact = d.facts.find((f) => f.id === c);
1206
+ const evidence = d.evidence.find((e) => e.id === c || c.startsWith(`${e.id}:`));
1207
+ const attempt = d.attempts[c.replace(/:exec\d+$/, "")];
1208
+ if (fact) {
1209
+ const { id: _id, ...rest } = fact;
1210
+ out[c] = clip(JSON.stringify(rest), 400);
1211
+ } else if (evidence) {
1212
+ const line = /:L(\d+)$/.exec(c);
1213
+ out[c] = line ? clip(evidence.lines.find((l) => l.startsWith(`L${line[1]} `)) ?? evidence.ref, 400) : `${evidence.ref}: ${evidence.why}`;
1214
+ } else if (attempt) out[c] = `${attempt.runId} · ${attempt.attempt} · ${attempt.outcome} · ${attempt.tools} tool calls${attempt.checks ? ` · checks ${attempt.checks}` : ""}`;
1215
+ }
1216
+ return out;
1217
+ }
1218
+ /** The run `reeval analyze` compares `head` with when `--vs` is omitted: its nearest recorded ancestor. */
1219
+ async function defaultBase(config, entries, head) {
1220
+ const reach = await reachability(config.root, [head.git?.sha]);
1221
+ const from = head.git && (!reach || reach.has(head.git.sha)) ? head.git.sha : "HEAD";
1222
+ const ancestry = await lines(config.root, [
1223
+ "rev-list",
1224
+ "--max-count=5000",
1225
+ from
1226
+ ]);
1227
+ return pickBaseline(entries.filter((e) => e.recorded && e.runId !== head.runId), head, ancestry);
1228
+ }
1229
+ /** Reads the runs and builds the digest and its cache key; no model call. */
1230
+ async function prepareAnalysis(config, head, base, evalFilter) {
1231
+ const headRun = await load(config, head);
1232
+ const baseRun = base ? await load(config, base) : null;
1233
+ const db = await openHistoryDb(config.root, config.history.dir);
1234
+ let built;
1235
+ let tables;
1236
+ try {
1237
+ built = await buildDigestKeepingDropped({
1238
+ head: headRun,
1239
+ base: baseRun,
1240
+ evalFilter,
1241
+ db,
1242
+ maxChars: config.analysis.maxInputTokens * 4,
1243
+ ...config.analysis.instructions ? { instructions: config.analysis.instructions } : {}
1244
+ });
1245
+ tables = db.schema().map((o) => o.name);
1246
+ } finally {
1247
+ db.close();
1248
+ }
1249
+ const modelName = (() => {
1250
+ try {
1251
+ const m = analysisModelName(config);
1252
+ return typeof m === "string" ? m : "fake";
1253
+ } catch {
1254
+ return "none";
1255
+ }
1256
+ })();
1257
+ const scope = {
1258
+ kind: "run",
1259
+ head: head.runId,
1260
+ base: base?.runId ?? null,
1261
+ evals: evalFilter
1262
+ };
1263
+ const key = sha256(JSON.stringify({
1264
+ scope,
1265
+ hash: digestHash(built.digest),
1266
+ prompt: ANALYSIS_PROMPT_VERSION,
1267
+ model: modelName
1268
+ })).replace(/^sha256:/, "");
1269
+ return {
1270
+ head,
1271
+ base,
1272
+ headRun,
1273
+ baseRun,
1274
+ ...built,
1275
+ tables,
1276
+ scope,
1277
+ modelName,
1278
+ key
1279
+ };
1280
+ }
1281
+ const docBase = async (config, p) => ({
1282
+ kind: "reeval-analysis",
1283
+ schemaVersion: 1,
1284
+ scope: p.scope,
1285
+ anchor: p.head.runId,
1286
+ inputs: {
1287
+ records: [p.baseRun, p.headRun].flatMap((r) => r ? [{
1288
+ runId: r.record.runId,
1289
+ sha256: sha256(r.bytes)
1290
+ }] : []),
1291
+ digestVersion: p.digest.digestVersion,
1292
+ digestHash: digestHash(p.digest)
1293
+ },
1294
+ promptVersion: ANALYSIS_PROMPT_VERSION,
1295
+ author: await author(config.root)
1296
+ });
1297
+ /** One structured call on the prepared digest (and what tools read), validated: the document the cache holds. */
1298
+ async function analyzeDigest(config, p, model, reads) {
1299
+ const seen = reads?.evidence.length ? {
1300
+ ...p.digest,
1301
+ evidence: [...p.digest.evidence, ...reads.evidence]
1302
+ } : p.digest;
1303
+ const answer = await callAnalysisModel(model, seen, `${promptOf(p.digest)}${reads?.text ?? ""}`, config.analysis.maxOutputTokens);
1304
+ const evals = p.headRun.record.evals.map((e) => ({
1305
+ file: e.file,
1306
+ name: [...e.group, e.name].join(" "),
1307
+ id: e.id,
1308
+ tags: e.tags,
1309
+ cells: e.cells.map((c) => c.key)
1310
+ }));
1311
+ const { body, validation } = validateAnalysis(answer.output, seen, p.tables, evals);
1312
+ const cites = [
1313
+ ...body.whatHappened,
1314
+ ...body.results,
1315
+ ...body.why,
1316
+ ...body.anomalies,
1317
+ ...body.actions
1318
+ ].flatMap((x) => x.cites);
1319
+ return {
1320
+ ...await docBase(config, p),
1321
+ model: model.name,
1322
+ resolvedModel: answer.resolvedModel,
1323
+ route: model.route,
1324
+ usage: answer.usage,
1325
+ latencyMs: answer.latencyMs,
1326
+ createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1327
+ validation,
1328
+ analysis: body,
1329
+ attempts: Object.fromEntries(Object.entries(p.digest.attempts).map(([id, a]) => [id, {
1330
+ runId: a.runId,
1331
+ evalSlug: a.evalSlug,
1332
+ attempt: a.attempt
1333
+ }])),
1334
+ cited: citedSummary(seen, [...new Set(cites)])
1335
+ };
1336
+ }
1337
+ async function analyzeCommand(ctx, args) {
1338
+ const { config, flags } = ctx;
1339
+ if (flags.json && flags.md) throw new UsageError("reeval analyze takes --json or --md, not both");
1340
+ if (flags.context && flags.save) throw new UsageError("reeval analyze --context makes no analysis to save; drop --save");
1341
+ const entries = await allRuns(config);
1342
+ let evalFilter = flags.eval ?? null;
1343
+ let head;
1344
+ try {
1345
+ head = resolveRun(entries, args[0] ?? "latest");
1346
+ } catch (error) {
1347
+ const needle = args[0]?.toLowerCase();
1348
+ const latest = entries.at(-1);
1349
+ if (!(error instanceof RefError) || !needle || !latest?.evals.some((e) => e.id.toLowerCase().includes(needle))) throw error;
1350
+ evalFilter ??= args[0];
1351
+ head = latest;
1352
+ }
1353
+ const base = flags.vs === "none" ? null : flags.vs ? resolveRun(entries, flags.vs) : await defaultBase(config, entries, head);
1354
+ if (base?.runId === head.runId) throw new UsageError("reeval analyze --vs names the run itself");
1355
+ if (flags.save && !head.recorded) throw new CommandError(`${head.runId} isn't recorded, so there's no history folder to save into. Run reeval record ${head.runId} first, then reeval analyze ${head.runId} --save.`);
1356
+ const model = flags.context || flags["dry-run"] ? null : resolveAnalysisModel(config);
1357
+ const prepared = await prepareAnalysis(config, head, base, evalFilter);
1358
+ const { digest, key, modelName } = prepared;
1359
+ if (evalFilter && digest.setup.evals.length === 0) throw new UsageError(`No eval in ${head.runId} matches "${evalFilter}"`);
1360
+ if (flags.context) {
1361
+ ctx.out(contextMarkdown(digest, head.runId, base?.runId ?? null));
1362
+ return 0;
1363
+ }
1364
+ const cached = flags["no-cache"] ? null : await readCached(config.root, key);
1365
+ if (flags["dry-run"]) {
1366
+ const chars = digestChars(digest);
1367
+ ctx.out([
1368
+ `digest ${head.runId}${base ? ` vs ${base.runId}` : ""}: ${digest.facts.length} facts, ${digest.evidence.length} evidence items${digest.dropped.length ? ` (${digest.dropped.length} dropped to fit)` : ""}, ${Object.keys(digest.attempts).length} attempts`,
1369
+ `size ${chars} characters ≈ ${Math.ceil(chars / 4)} tokens (cap ${config.analysis.maxInputTokens} in, ${config.analysis.maxOutputTokens} out)`,
1370
+ `model ${modelName} · ${ANALYSIS_PROMPT_VERSION} · cache ${cached ? "hit" : "miss"} (.reeval/analysis/${key}.json)`
1371
+ ].join("\n"));
1372
+ return 0;
1373
+ }
1374
+ let doc;
1375
+ if (cached) {
1376
+ doc = cached;
1377
+ ctx.err(`cached analysis from ${cached.createdAt} (.reeval/analysis/${key}.json); --no-cache calls the model again`);
1378
+ } else {
1379
+ const resolved = model ?? resolveAnalysisModel(config);
1380
+ ctx.err(`analyzing ${head.runId}${base ? ` vs ${base.runId}` : ""} with ${resolved.name}…`);
1381
+ doc = await analyzeDigest(config, prepared, resolved);
1382
+ await writeCached(config.root, key, doc);
1383
+ }
1384
+ if (flags.save) {
1385
+ const saved = await saveAnalysis(config.history.dir, doc, key, renderMarkdown(doc));
1386
+ ctx.err(`saved ${saved.map((p) => fromCwd(p)).join(" and ")}. Commit them with git.`);
1387
+ }
1388
+ ctx.out(flags.json ? JSON.stringify(doc, null, 2) : flags.md ? renderMarkdown(doc) : renderText(doc));
1389
+ return 0;
1390
+ }
1391
+ //#endregion
1392
+ export { analyzeCommand };
1065
1393
 
1066
- //# sourceMappingURL=validate-BhwqrVsm.js.map
1394
+ //# sourceMappingURL=command-9xm5ImqI.js.map