reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -1,15 +1,23 @@
1
- import { l as sha256 } from "./ids-Bp6ut7ZD.js";
2
- import { r as questionShows, t as DEFAULT_QUESTION_STATE } from "./state-CeR6oz0k.js";
3
- import { n as clip } from "./text-CrVy5ioL.js";
4
- import { n as formatTokens } from "./usage-CleNKkly.js";
5
- import "./analysis-B0nOfAzA.js";
6
- import { B as labelledJudgements, J as bothUnderATenth, St as compare } from "./model-UgHDMhI1.js";
7
- import { d as familyEffortLabel, m as familyRateLabel, p as familyLiftLabel } from "./suite-DmuF_ApO.js";
8
- import { t as runHeader } from "./header-BVzbRnNv.js";
9
- import { n as headToHeadLines, r as pairTallies, s as sampleReasons, u as splitsLabel } from "./pairs-DP6XOJby.js";
10
- import { i as checkCommand, o as commandSummary } from "./cli-spec-BXIBUlaK.js";
11
- import { r as contrastsOf } from "./contrasts-B4nOhn0Z.js";
12
- import { i as TOOL_CLASSES } from "./facts-Cfx1H70y.js";
1
+ import { n as UsageError } from "./errors-B6BSMs03.js";
2
+ import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
3
+ import { at as questionShows, rt as DEFAULT_QUESTION_STATE } from "./config-k6m7XbAz.js";
4
+ import { n as clip, r as fenced } from "./text-DTigcA64.js";
5
+ import { a as pairTallies, f as splitsLabel, l as sampleReasons, r as headToHeadLines } from "./pairs-puYi-zj2.js";
6
+ import "./stats-hfpuYsAM.js";
7
+ import { l as resolveAnalysisModel, n as callAnalysisModel, t as analysisModelName, u as SYSTEM_PROMPT } from "./model-Bq3G0OG1.js";
8
+ import { n as formatTokens } from "./usage-BNY1ABrU.js";
9
+ import { t as ANALYSIS_PROMPT_VERSION } from "./analysis-C2-aEbr7.js";
10
+ import { Q as bothUnderATenth, U as labeledJudgments, gt as compare } from "./model-CTnyvh7H.js";
11
+ import { g as familyRateLabel, h as familyLiftLabel, p as familyEffortLabel } from "./suite-Co8cwelb.js";
12
+ import { r as runHeader } from "./remarks-D_3qhBWU.js";
13
+ import { i as checkCommand, o as commandSummary } from "./cli-spec-D0qyrPZi.js";
14
+ import { o as parseRecord, p as scratchDir } from "./files-Bi461Be2.js";
15
+ import { a as pickBaseline, o as resolveRun, t as RefError } from "./views-BFO1BLKu.js";
16
+ import { c as lines, n as allRuns, t as CommandError, u as reachability } from "./commands-BU6YJCz9.js";
17
+ import { r as contrastsOf } from "./contrasts-DLZF4dGF.js";
18
+ import { l as TOOL_CLASSES, s as openHistoryDb } from "./db-CpsifVln.js";
19
+ import { t as author } from "./provenance-BizwUn3h.js";
20
+ import { a as readCached, c as writeCached, o as saveAnalysis } from "./files-H24pz0hL.js";
13
21
  import { basename, join } from "node:path";
14
22
  import { readFile } from "node:fs/promises";
15
23
  //#region src/analysis/digest.ts
@@ -19,8 +27,9 @@ const RULES = [
19
27
  `Effort (tool calls, agent time, and tokens when both sides have usage samples) is compared with an exact Mann-Whitney test: "fewer"/"more" at p < 0.05; "insufficient" when no split of the attempts' values can reach it (2 vs 2 cannot, and ties raise the floor) — never because a harness dropped usage. Attempts without usage are left out of the token sample only. If either side then has no token sample, the token metric is omitted (no line, not insufficient). A Goose cell does not blank tokens on Cursor cells in the same study. Pooled across evals, tokens are an effect only when at least 2/3 of the strata that reported usage agree. Effort's separated (given from 3 vs 3) says, per metric, that the two sides' ranges do not overlap: every attempt on one side beat every attempt on the other. insufficient with separated true is a directional signal: report the ratio, the n and "no overlap, not significant" as directional, never as an effect. separated false is no direction at all.`,
20
28
  "A family fact is one pooled evaluate.each: its cases are paired samples of one suite, so its lift (Newcombe paired interval and McNemar at one attempt a case, a t interval over cases otherwise) is the result for those cases; their own pass-rate facts carry no lift. insufficient (fewer than 6 discordant pairs or 6 cases) is not measured. A family's minLift gates the pooled lower bound.",
21
29
  "Each factor claim has a tier: controlled (two cells in one run that differ in one factor), pooled (the same contrast over comparable runs, and across evals), observed (a change between runs, which is weaker; \"confounded\" when several parts changed), not_measured (one level only).",
22
- `A cell with mixed outcomes is flaky only with at least 6 counted attempts over comparable runs; below that it is "insufficient", not flaky.`,
30
+ `A cell with mixed outcomes is flaky only with at least 6 counted attempts over comparable runs; below that it is "mixed": it passed and failed, too few times to call it flaky.`,
23
31
  "Drift marked noise changed only an agent CLI's own runtime state (e.g. ~/.cursor/cli-config.json): not a real change, and those runs stay comparable.",
32
+ "Drift on the gate is a policy change: the gate scores the same attempts, so the runs stay comparable. A compare row with status gate flipped only because its gate did (gate.rescored, base's attempts under head's gate, matches head): never call it a regression or a fix.",
24
33
  "An eval where every cell passes at least 95% is saturated: pass@1 cannot tell cells apart there, effort can.",
25
34
  "Judge verdicts carry no recorded reason; an explanation of a failed verdict is your reading of the cited output, and must say so.",
26
35
  "A criteria fact lists each check (expect label, judges included) per cell: rows only for labels whose rate differs between cells, or is mixed within one; neverMet labels failed on every counted attempt of every cell, alwaysMet passed on all of them. A neverMet or alwaysMet label is the same in every arm: it can't explain a difference between arms.",
@@ -29,7 +38,7 @@ const RULES = [
29
38
  "A context-use fact counts each attempt's tool calls by class (context, dependency, web, verify, workspace) and lists which placed files each arm opened: how much an arm leaned on what its environment declared versus digging in installed packages. The classes are a heuristic over tool names and paths; cite them as counts, not as proof of intent.",
30
39
  "A contrast fact (kind contrast) is one pair of arms on one harness, a against b, when an eval has two variants or more: pass rate and lift of a over b, the checks that tell them apart (discriminates or mixed; sameOnBoth counts the rest) and effort. It answers how two variants differ from each other, not only from the control; gating stays each variant against its control.",
31
40
  "A check (kind checks) whose verdict is both-pass or both-fail passes, or fails, on both the variant and its control: it is no evidence for or against the variant. Only discriminates rows say which requirement separated them.",
32
- "Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check. Each pair is voted in both orders; splits counts pairs whose two orders picked opposite sides (the position decided, not the work), and rates count such a pair as one win each. With more than two arms, read headToHead (a–b wins per pair, splits apart), not the pooled rates. shows lists what each side was shown: a question that shows output or transcript can favour the better-written account rather than the better work, and one that shows artifacts rests on screenshots."
41
+ "Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check. Each pair is voted in both orders; splits counts pairs whose two orders picked opposite sides (the position decided, not the work), and rates count such a pair as one win each. With more than two arms, read headToHead (a–b wins per pair, splits apart), not the pooled rates. shows lists what each side was shown: a question that shows output or transcript can favor the better-written account rather than the better work, and one that shows artifacts rests on screenshots."
33
42
  ];
34
43
  const pts = (x) => x === null || x === void 0 ? null : `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
35
44
  const liftText = (points, lo, hi) => `${pts(points)} pts (95% ${pts(lo)}…${pts(hi)})`;
@@ -45,10 +54,12 @@ function digestLiftVerdict(variantCounted, controlCounted, withinNoise) {
45
54
  function evalChangedLift() {
46
55
  return "not comparable: the eval changed";
47
56
  }
57
+ /** What the eval's own source is keyed on: `source` and `revision`, else (older records) its files. */
48
58
  function sourcesKey(e) {
59
+ if (e.source) return JSON.stringify([e.source, e.revision ?? null]);
49
60
  return JSON.stringify(e.sources.map((s) => [s.path, s.sha256]));
50
61
  }
51
- /** The eval's sources, or the sources part of its fingerprint, differ across the two runs. */
62
+ /** The eval's own source, or the sources part of its fingerprint, differs across the two runs. */
52
63
  function evalSourcesChanged(base, head) {
53
64
  if (sourcesKey(base) !== sourcesKey(head)) return true;
54
65
  const prev = new Map(base.cells.map((c) => [c.key, c.fingerprintParts.sources]));
@@ -128,10 +139,10 @@ function criteriaOf(e, tag) {
128
139
  const scores = /* @__PURE__ */ new Map();
129
140
  for (const a of e.attempts) {
130
141
  if (!counted(a)) continue;
131
- for (const j of labelledJudgements(a)) {
142
+ for (const j of labeledJudgments(a)) {
132
143
  if (j.check === void 0) continue;
133
144
  const id = `${a.cell}\0${j.check}`;
134
- scores.set(id, [...scores.get(id) ?? [], j.judgement.score]);
145
+ scores.set(id, [...scores.get(id) ?? [], j.judgment.score]);
135
146
  }
136
147
  }
137
148
  const rows = [];
@@ -190,7 +201,7 @@ async function buildDigestKeepingDropped(input) {
190
201
  const any = a.files.transcript ?? a.files.judgeInput ?? a.checks.find((c) => c.log)?.log;
191
202
  const assertions = a.assertions ?? [];
192
203
  const failed = failedLabels(a);
193
- const judges = labelledJudgements(a);
204
+ const judges = labeledJudgments(a);
194
205
  attempts[id] = {
195
206
  run: short.get(r.record.runId),
196
207
  runId: r.record.runId,
@@ -204,7 +215,7 @@ async function buildDigestKeepingDropped(input) {
204
215
  agentS: Math.round(a.timings.agentMs / 1e3),
205
216
  ...assertions.length ? { checks: `${assertions.filter((x) => x.status === "pass").length}/${assertions.length}` } : {},
206
217
  ...failed.length ? { failed } : {},
207
- ...judges.length ? { judges: Object.fromEntries(judges.map((j) => [clip(j.label, LABEL_CHARS), round2(j.judgement.score)])) } : {},
218
+ ...judges.length ? { judges: Object.fromEntries(judges.map((j) => [clip(j.label, LABEL_CHARS), round2(j.judgment.score)])) } : {},
208
219
  execs: a.checks.length,
209
220
  artifacts: any ? `${r.record.runId}/${any.path.split("/").slice(0, 3).join("/")}` : null
210
221
  };
@@ -279,36 +290,39 @@ async function buildDigestKeepingDropped(input) {
279
290
  fact("effort", {
280
291
  run: short.get(runId),
281
292
  eval: tagOf(first.eval_key),
282
- cells: rows.map((row) => ({
283
- cell: cellTag(runId, row.eval_key, row.cell_key),
284
- n: row.n,
285
- medianToolCalls: row.tools_median,
286
- medianAgentS: row.agent_s_median,
287
- ...row.tokens_median != null ? { medianTokens: row.tokens_median } : {},
288
- ...row.workspace_calls_median != null ? { medianCallsByClass: Object.fromEntries(TOOL_CLASSES.flatMap((k) => {
293
+ cells: rows.map((row) => {
294
+ const byClass = TOOL_CLASSES.flatMap((k) => {
289
295
  const m = row[`${k}_calls_median`];
290
296
  return m ? [[k, m]] : [];
291
- })) } : {},
292
- ...row.control_key ? { vsControl: {
293
- n: `${row.n} vs ${row.control_n}`,
294
- toolCalls: `${row.tools_median} vs ${row.control_tools_median} (${ratioText(row.tools_ratio)})`,
295
- agentS: bothUnderATenth(Number(row.agent_s_median) * 1e3, Number(row.control_agent_s_median) * 1e3) ? "<0.1 vs <0.1" : `${row.agent_s_median} vs ${row.control_agent_s_median} (${ratioText(row.agent_s_ratio)})`,
296
- p: row.tools_p,
297
- verdict: row.verdict,
298
- timeVerdict: row.time_verdict,
299
- ...row.tokens_verdict ? {
300
- tokens: `${formatTokens(row.tokens_median)} vs ${formatTokens(row.control_tokens_median)} (${ratioText(row.tokens_ratio)})`,
301
- tokensP: row.tokens_p,
302
- tokensVerdict: row.tokens_verdict
303
- } : {},
304
- ...Math.min(Number(row.n), Number(row.control_n)) >= 3 ? { separated: {
305
- tools: row.separated === 1,
306
- agentS: row.time_separated === 1,
307
- ...row.tokens_separated != null ? { tokens: row.tokens_separated === 1 } : {}
308
- } } : {},
309
- ...row.tokens_note ? { unreportedUsage: row.tokens_note } : {}
310
- } } : {}
311
- }))
297
+ });
298
+ return {
299
+ cell: cellTag(runId, row.eval_key, row.cell_key),
300
+ n: row.n,
301
+ medianToolCalls: row.tools_median,
302
+ medianAgentS: row.agent_s_median,
303
+ ...row.tokens_median != null ? { medianTokens: row.tokens_median } : {},
304
+ ...byClass.length ? { medianCallsByClass: Object.fromEntries(byClass) } : {},
305
+ ...row.control_key ? { vsControl: {
306
+ n: `${row.n} vs ${row.control_n}`,
307
+ toolCalls: `${row.tools_median} vs ${row.control_tools_median} (${ratioText(row.tools_ratio)})`,
308
+ agentS: bothUnderATenth(Number(row.agent_s_median) * 1e3, Number(row.control_agent_s_median) * 1e3) ? "<0.1 vs <0.1" : `${row.agent_s_median} vs ${row.control_agent_s_median} (${ratioText(row.agent_s_ratio)})`,
309
+ p: row.tools_p,
310
+ verdict: row.verdict,
311
+ timeVerdict: row.time_verdict,
312
+ ...row.tokens_verdict ? {
313
+ tokens: `${formatTokens(row.tokens_median)} vs ${formatTokens(row.control_tokens_median)} (${ratioText(row.tokens_ratio)})`,
314
+ tokensP: row.tokens_p,
315
+ tokensVerdict: row.tokens_verdict
316
+ } : {},
317
+ ...Math.min(Number(row.n), Number(row.control_n)) >= 3 ? { separated: {
318
+ tools: row.separated === 1,
319
+ agentS: row.time_separated === 1,
320
+ ...row.tokens_separated != null ? { tokens: row.tokens_separated === 1 } : {}
321
+ } } : {},
322
+ ...row.tokens_note ? { unreportedUsage: row.tokens_note } : {}
323
+ } } : {}
324
+ };
325
+ })
312
326
  });
313
327
  }
314
328
  const changedEvalKeys = /* @__PURE__ */ new Set();
@@ -507,7 +521,7 @@ async function buildDigestKeepingDropped(input) {
507
521
  }) : failures.slice(0, 8);
508
522
  for (const { r, e, a } of failureFacts) {
509
523
  const failed = failedLabels(a);
510
- const judges = labelledJudgements(a).filter((j) => j.judgement.verdict === "fail");
524
+ const judges = labeledJudgments(a).filter((j) => j.judgment.verdict === "fail");
511
525
  fact("failure", {
512
526
  attempt: idOf.get(`${r.record.runId}\0${a.key}`),
513
527
  outcome: a.outcome,
@@ -517,9 +531,9 @@ async function buildDigestKeepingDropped(input) {
517
531
  ...failed.length ? { failed } : {},
518
532
  ...judges.length ? { judges: judges.map((j) => ({
519
533
  label: clip(j.label, LABEL_CHARS),
520
- question: clip(j.judgement.question, 300),
521
- score: j.judgement.score,
522
- threshold: j.judgement.threshold
534
+ question: clip(j.judgment.question, 300),
535
+ score: j.judgment.score,
536
+ threshold: j.judgment.threshold
523
537
  })) } : {},
524
538
  failingExecs: a.checks.map((c, i) => ({
525
539
  exec: i + 1,
@@ -560,16 +574,20 @@ async function buildDigestKeepingDropped(input) {
560
574
  cells: d.cells,
561
575
  changed: d.changed,
562
576
  ...d.changed_paths ? { paths: String(d.changed_paths).split(", ") } : {},
563
- verdict: d.noise ? "noise: agent runtime state only" : `real change: ${d.real_changed}`
577
+ verdict: d.noise ? "noise: agent runtime state only" : d.real_changed === "gate" ? "policy change: the gate scores the same attempts differently" : `real change: ${d.real_changed}`
564
578
  });
579
+ const [baseHeader, headHeader] = [runHeader(base.record, true), runHeader(head.record, true)];
580
+ const inScope = (id) => scopeEvals.some((e) => e.id === id);
581
+ const familyCases = new Map([...baseHeader.families ?? [], ...headHeader.families ?? []].map((f) => [f.id, f.caseIds]));
565
582
  fact("compare", {
566
583
  base: "r1",
567
584
  head: "r2",
568
- rows: compare(runHeader(base.record, true), runHeader(head.record, true)).filter((row) => scopeEvals.some((e) => e.id === row.id)).map((row) => ({
569
- eval: tagOf(scopeEvals.find((e) => e.id === row.id)?.key ?? row.id),
585
+ rows: compare(baseHeader, headHeader).filter((row) => row.kind === "family" ? (familyCases.get(row.id) ?? []).some(inScope) : inScope(row.id)).map((row) => ({
586
+ ...row.kind === "family" ? { family: row.id } : { eval: tagOf(scopeEvals.find((e) => e.id === row.id)?.key ?? row.id) },
570
587
  status: row.status,
571
588
  ...row.changes?.length ? { changes: row.changes } : {},
572
- ...row.cancelled ? { cancelled: row.cancelled } : {}
589
+ ...row.gate ? { gate: row.gate } : {},
590
+ ...row.canceled ? { canceled: row.canceled } : {}
573
591
  }))
574
592
  });
575
593
  }
@@ -602,6 +620,37 @@ async function buildDigestKeepingDropped(input) {
602
620
  });
603
621
  }
604
622
  }
623
+ for (const r of runs) for (const e of r.record.evals) {
624
+ if (!evalKeys.has(e.key)) continue;
625
+ const byQuestion = /* @__PURE__ */ new Map();
626
+ for (const a of e.attempts) for (const q of a.answers ?? []) byQuestion.set(q.question, [...byQuestion.get(q.question) ?? [], {
627
+ attempt: idOf.get(`${r.record.runId}\0${a.key}`) ?? a.id,
628
+ answer: q
629
+ }]);
630
+ for (const [question, asked] of byQuestion) {
631
+ const first = asked[0]?.answer;
632
+ const answered = asked.filter((x) => x.answer.error === void 0);
633
+ const failed = asked.find((x) => x.answer.error !== void 0)?.answer.error;
634
+ fact("attempt-question", {
635
+ run: short.get(r.record.runId),
636
+ eval: tagOf(e.key),
637
+ question: clip(question, 120),
638
+ model: first.resolvedModel ?? first.model,
639
+ shows: questionShows(first.stateOptions, first.artifacts),
640
+ asked: asked.length,
641
+ ...first.choices ? { choices: first.choices.map((choice) => ({
642
+ choice,
643
+ n: answered.filter((x) => x.answer.choice === choice).length
644
+ })) } : {},
645
+ answers: answered.slice(0, 5).map((x) => ({
646
+ attempt: x.attempt,
647
+ ...x.answer.choice !== void 0 ? { choice: x.answer.choice } : {},
648
+ text: clip(x.answer.text, 300)
649
+ })),
650
+ ...failed ? { error: clip(failed, 200) } : {}
651
+ });
652
+ }
653
+ }
605
654
  for (const r of runs) for (const e of r.record.evals) {
606
655
  if (!evalKeys.has(e.key)) continue;
607
656
  const tag = (key) => cellTag(r.record.runId, e.key, key);
@@ -700,8 +749,8 @@ async function buildDigestKeepingDropped(input) {
700
749
  for (const [i, { r, e, a }] of [...perCell.values()].slice(0, 6).entries()) {
701
750
  const id = idOf.get(`${r.record.runId}\0${a.key}`);
702
751
  const lines = await lastText(r.record.runId, a);
703
- const judges = labelledJudgements(a).filter((j) => j.judgement.verdict === "fail");
704
- for (const { label, judgement: j } of judges) lines.push(`judge ${clip(label, LABEL_CHARS)}: ${j.question}`, `judge score: ${j.score} (threshold ${j.threshold})`);
752
+ const judges = labeledJudgments(a).filter((j) => j.judgment.verdict === "fail");
753
+ for (const { label, judgment: j } of judges) lines.push(`judge ${clip(label, LABEL_CHARS)}: ${j.question}`, `judge score: ${j.score} (threshold ${j.threshold})`);
705
754
  for (const x of a.assertions ?? []) {
706
755
  if (x.status === "pass" || /(^|\.)toPass$/.test(x.matcher)) continue;
707
756
  lines.push(`check ${x.status}: ${clip(x.label, LABEL_CHARS)}: ${clip(x.message ?? "", 300)}`);
@@ -732,7 +781,7 @@ async function buildDigestKeepingDropped(input) {
732
781
  }
733
782
  }
734
783
  const digest = {
735
- digestVersion: 11,
784
+ digestVersion: 12,
736
785
  scope: {
737
786
  kind: "run",
738
787
  head: short.get(head.record.runId),
@@ -796,6 +845,77 @@ function citable(d) {
796
845
  };
797
846
  }
798
847
  //#endregion
848
+ //#region src/analysis/render.ts
849
+ const cite = (cs) => cs.length ? ` ${cs.map((c) => `[${c}]`).join(" ")}` : "";
850
+ const unverified = (numbers) => numbers?.length ? `unverified: ${numbers.join(", ")}` : "";
851
+ const note = (c) => [c.confidence === "high" ? "" : `${c.confidence} confidence`, unverified(c.unverifiedNumbers)].filter(Boolean).join("; ");
852
+ function footer(doc) {
853
+ const v = doc.validation;
854
+ const tokens = doc.usage.inputTokens !== null ? `${doc.usage.inputTokens} in / ${doc.usage.outputTokens ?? "?"} out tokens · ` : "";
855
+ return `${doc.resolvedModel ?? doc.model} · ${doc.promptVersion} · digest v${doc.inputs.digestVersion} · ${tokens}${Math.round(doc.latencyMs / 1e3)}s · ${v.citations} citations checked, ${v.invalid.length} invalid, ${v.dropped} dropped · ${v.commands} commands checked, ${v.invalidCommands.length} rejected`;
856
+ }
857
+ /** The analysis as Markdown (`--md`, and the `.md` twin of a saved analysis). */
858
+ function renderMarkdown(doc) {
859
+ const a = doc.analysis;
860
+ const md = [
861
+ `## reeval analysis · ${doc.scope.head}${doc.scope.base ? ` vs ${doc.scope.base}` : ""}`,
862
+ "",
863
+ `**TL;DR** (${a.confidence.overall}). ${a.tldr}`,
864
+ ""
865
+ ];
866
+ for (const [title, items] of [
867
+ ["What happened", a.whatHappened],
868
+ ["Results", a.results],
869
+ ["Why", a.why]
870
+ ]) {
871
+ if (!items.length) continue;
872
+ md.push(`### ${title}`, "");
873
+ for (const c of items) md.push(`- ${c.text}${cite(c.cites)}${note(c) ? ` *(${note(c)})*` : ""}`);
874
+ md.push("");
875
+ }
876
+ if (a.anomalies.length) {
877
+ md.push("### Anomalies", "");
878
+ for (const x of a.anomalies) md.push(`- **${x.kind}**: ${x.text}${cite(x.cites)}${unverified(x.unverifiedNumbers) ? ` *(${unverified(x.unverifiedNumbers)})*` : ""}`);
879
+ md.push("");
880
+ }
881
+ md.push("### Suggested actions", "");
882
+ if (!a.actions.length) md.push("None: nothing here is worth acting on.");
883
+ a.actions.forEach((x, i) => {
884
+ md.push(`${i + 1}. **${x.title}** (${x.kind} · impact ${x.impact} · effort ${x.effort}) `, ` Target: \`${x.target}\`. ${x.change} `, ` Expect: ${x.expectedEffect} `, x.verify ? ` Verify: \`${x.verify}\`${cite(x.cites)}` : ` Verify: none (the suggested command isn't reeval's CLI: ${x.verifyProblem ?? "unknown"})${cite(x.cites)}`);
885
+ });
886
+ md.push("", `**Confidence: ${a.confidence.overall}.** ${a.confidence.why}`, "", `<sub>${footer(doc)}</sub>`, "");
887
+ return md.join("\n");
888
+ }
889
+ /** The analysis for a terminal: the same sections, plain. */
890
+ function renderText(doc) {
891
+ const a = doc.analysis;
892
+ const lines = [
893
+ `analysis ${doc.scope.head}${doc.scope.base ? ` vs ${doc.scope.base}` : ""}`,
894
+ "",
895
+ `TL;DR (${a.confidence.overall}) ${a.tldr}`
896
+ ];
897
+ for (const [title, items] of [
898
+ ["What happened", a.whatHappened],
899
+ ["Results", a.results],
900
+ ["Why", a.why]
901
+ ]) {
902
+ if (!items.length) continue;
903
+ lines.push("", title);
904
+ for (const c of items) lines.push(` - ${c.text}${cite(c.cites)}${note(c) ? ` (${note(c)})` : ""}`);
905
+ }
906
+ if (a.anomalies.length) {
907
+ lines.push("", "Anomalies");
908
+ for (const x of a.anomalies) lines.push(` - ${x.kind}: ${x.text}${cite(x.cites)}${unverified(x.unverifiedNumbers) ? ` (${unverified(x.unverifiedNumbers)})` : ""}`);
909
+ }
910
+ lines.push("", "Suggested actions");
911
+ if (!a.actions.length) lines.push(" none: nothing here is worth acting on");
912
+ a.actions.forEach((x, i) => {
913
+ lines.push(` ${i + 1}. ${x.title} (${x.kind} · impact ${x.impact} · effort ${x.effort})${cite(x.cites)}`, ` target: ${x.target}. ${x.change}`, ` expect: ${x.expectedEffect}`, x.verify ? ` verify: ${x.verify}` : ` verify: none (${x.verifyProblem ?? "rejected"})`);
914
+ });
915
+ lines.push("", `Confidence: ${a.confidence.overall}. ${a.confidence.why}`, "", footer(doc));
916
+ return lines.join("\n");
917
+ }
918
+ //#endregion
799
919
  //#region src/analysis/validate.ts
800
920
  const EXEC = /^(.+):exec(\d+)$/;
801
921
  const LINE = /^(E\d+):L(\d+)$/;
@@ -1029,6 +1149,246 @@ function validateAnalysis(output, digest, tables, evals) {
1029
1149
  };
1030
1150
  }
1031
1151
  //#endregion
1032
- export { digestChars as a, buildDigestKeepingDropped as i, validateAnalysis as n, digestHash as o, RULES as r, unverifiedNumbers as t };
1152
+ //#region src/analysis/command.ts
1153
+ async function load(config, entry) {
1154
+ const dir = entry.recorded ? config.history.dir : scratchDir(config.root);
1155
+ const bytes = await readFile(join(dir, `${entry.runId}.json`));
1156
+ return {
1157
+ record: parseRecord(bytes, entry.runId),
1158
+ dir,
1159
+ bytes
1160
+ };
1161
+ }
1162
+ /** The model's input: the project's own instructions, which set the question, then the digest as JSON. */
1163
+ const promptOf = (d) => `${d.instructions ? `PROJECT INSTRUCTIONS (the question this analysis answers): ${d.instructions}\n\n` : ""}DIGEST:\n${JSON.stringify(d)}`;
1164
+ /** `--context`: the same digest and rules, for the user's own coding agent. */
1165
+ function contextMarkdown(d, head, base) {
1166
+ return [
1167
+ `# reeval analysis context: ${head}${base ? ` vs ${base}` : ""}`,
1168
+ "",
1169
+ "Analyze this reeval run for the engineer who wrote the evals. The digest below holds facts reeval computed (F…), evidence excerpts (E…, transcript lines as L<n>) and the attempts table. Cite those ids for every claim, keep to the verdicts in the facts, and suggest at most 5 concrete actions, each verified with a reeval command from the CLI list.",
1170
+ "",
1171
+ ...d.frame === "study" ? ["This run is a study: every cell is ungated (gate: 'none'), so it measures how the arms differ. Failing attempts are data, not a failed gate.", ""] : [],
1172
+ ...d.instructions ? [
1173
+ "## Project instructions (the question this analysis answers)",
1174
+ "",
1175
+ d.instructions,
1176
+ ""
1177
+ ] : [],
1178
+ "## How reeval decided each verdict",
1179
+ "",
1180
+ ...RULES.map((r) => `- ${r}`),
1181
+ "",
1182
+ "## Instructions reeval gives its own analysis model",
1183
+ "",
1184
+ SYSTEM_PROMPT,
1185
+ "",
1186
+ "## reeval CLI (verify commands must be one of these)",
1187
+ "",
1188
+ ...commandSummary().map((c) => `- \`${c}\``),
1189
+ "",
1190
+ "## To dig further",
1191
+ "",
1192
+ "- `reeval query effort`, `reeval query factor_effects`, `reeval query skill_reads`, `reeval query flaky_cells`, `reeval query drift`",
1193
+ `- \`reeval query "SELECT line, tool, target FROM tool_calls WHERE attempt_key = '<key>'"\`: an attempt's tool calls`,
1194
+ "- `reeval show <run> --json`: the whole record; transcripts are under the run's artifact folder",
1195
+ "",
1196
+ "## Digest",
1197
+ "",
1198
+ ...fenced([JSON.stringify(d, null, 1)], "json"),
1199
+ ""
1200
+ ].join("\n");
1201
+ }
1202
+ function citedSummary(d, cites) {
1203
+ const out = {};
1204
+ for (const c of cites) {
1205
+ const fact = d.facts.find((f) => f.id === c);
1206
+ const evidence = d.evidence.find((e) => e.id === c || c.startsWith(`${e.id}:`));
1207
+ const attempt = d.attempts[c.replace(/:exec\d+$/, "")];
1208
+ if (fact) {
1209
+ const { id: _id, ...rest } = fact;
1210
+ out[c] = clip(JSON.stringify(rest), 400);
1211
+ } else if (evidence) {
1212
+ const line = /:L(\d+)$/.exec(c);
1213
+ out[c] = line ? clip(evidence.lines.find((l) => l.startsWith(`L${line[1]} `)) ?? evidence.ref, 400) : `${evidence.ref}: ${evidence.why}`;
1214
+ } else if (attempt) out[c] = `${attempt.runId} · ${attempt.attempt} · ${attempt.outcome} · ${attempt.tools} tool calls${attempt.checks ? ` · checks ${attempt.checks}` : ""}`;
1215
+ }
1216
+ return out;
1217
+ }
1218
+ /** The run `reeval analyze` compares `head` with when `--vs` is omitted: its nearest recorded ancestor. */
1219
+ async function defaultBase(config, entries, head) {
1220
+ const reach = await reachability(config.root, [head.git?.sha]);
1221
+ const from = head.git && (!reach || reach.has(head.git.sha)) ? head.git.sha : "HEAD";
1222
+ const ancestry = await lines(config.root, [
1223
+ "rev-list",
1224
+ "--max-count=5000",
1225
+ from
1226
+ ]);
1227
+ return pickBaseline(entries.filter((e) => e.recorded && e.runId !== head.runId), head, ancestry);
1228
+ }
1229
+ /** Reads the runs and builds the digest and its cache key; no model call. */
1230
+ async function prepareAnalysis(config, head, base, evalFilter) {
1231
+ const headRun = await load(config, head);
1232
+ const baseRun = base ? await load(config, base) : null;
1233
+ const db = await openHistoryDb(config.root, config.history.dir);
1234
+ let built;
1235
+ let tables;
1236
+ try {
1237
+ built = await buildDigestKeepingDropped({
1238
+ head: headRun,
1239
+ base: baseRun,
1240
+ evalFilter,
1241
+ db,
1242
+ maxChars: config.analysis.maxInputTokens * 4,
1243
+ ...config.analysis.instructions ? { instructions: config.analysis.instructions } : {}
1244
+ });
1245
+ tables = db.schema().map((o) => o.name);
1246
+ } finally {
1247
+ db.close();
1248
+ }
1249
+ const modelName = (() => {
1250
+ try {
1251
+ const m = analysisModelName(config);
1252
+ return typeof m === "string" ? m : "fake";
1253
+ } catch {
1254
+ return "none";
1255
+ }
1256
+ })();
1257
+ const scope = {
1258
+ kind: "run",
1259
+ head: head.runId,
1260
+ base: base?.runId ?? null,
1261
+ evals: evalFilter
1262
+ };
1263
+ const key = sha256(JSON.stringify({
1264
+ scope,
1265
+ hash: digestHash(built.digest),
1266
+ prompt: ANALYSIS_PROMPT_VERSION,
1267
+ model: modelName
1268
+ })).replace(/^sha256:/, "");
1269
+ return {
1270
+ head,
1271
+ base,
1272
+ headRun,
1273
+ baseRun,
1274
+ ...built,
1275
+ tables,
1276
+ scope,
1277
+ modelName,
1278
+ key
1279
+ };
1280
+ }
1281
+ const docBase = async (config, p) => ({
1282
+ kind: "reeval-analysis",
1283
+ schemaVersion: 1,
1284
+ scope: p.scope,
1285
+ anchor: p.head.runId,
1286
+ inputs: {
1287
+ records: [p.baseRun, p.headRun].flatMap((r) => r ? [{
1288
+ runId: r.record.runId,
1289
+ sha256: sha256(r.bytes)
1290
+ }] : []),
1291
+ digestVersion: p.digest.digestVersion,
1292
+ digestHash: digestHash(p.digest)
1293
+ },
1294
+ promptVersion: ANALYSIS_PROMPT_VERSION,
1295
+ author: await author(config.root)
1296
+ });
1297
+ /** One structured call on the prepared digest (and what tools read), validated: the document the cache holds. */
1298
+ async function analyzeDigest(config, p, model, reads) {
1299
+ const seen = reads?.evidence.length ? {
1300
+ ...p.digest,
1301
+ evidence: [...p.digest.evidence, ...reads.evidence]
1302
+ } : p.digest;
1303
+ const answer = await callAnalysisModel(model, seen, `${promptOf(p.digest)}${reads?.text ?? ""}`, config.analysis.maxOutputTokens);
1304
+ const evals = p.headRun.record.evals.map((e) => ({
1305
+ file: e.file,
1306
+ name: [...e.group, e.name].join(" "),
1307
+ id: e.id,
1308
+ tags: e.tags,
1309
+ cells: e.cells.map((c) => c.key)
1310
+ }));
1311
+ const { body, validation } = validateAnalysis(answer.output, seen, p.tables, evals);
1312
+ const cites = [
1313
+ ...body.whatHappened,
1314
+ ...body.results,
1315
+ ...body.why,
1316
+ ...body.anomalies,
1317
+ ...body.actions
1318
+ ].flatMap((x) => x.cites);
1319
+ return {
1320
+ ...await docBase(config, p),
1321
+ model: model.name,
1322
+ resolvedModel: answer.resolvedModel,
1323
+ route: model.route,
1324
+ usage: answer.usage,
1325
+ latencyMs: answer.latencyMs,
1326
+ createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1327
+ validation,
1328
+ analysis: body,
1329
+ attempts: Object.fromEntries(Object.entries(p.digest.attempts).map(([id, a]) => [id, {
1330
+ runId: a.runId,
1331
+ evalSlug: a.evalSlug,
1332
+ attempt: a.attempt
1333
+ }])),
1334
+ cited: citedSummary(seen, [...new Set(cites)])
1335
+ };
1336
+ }
1337
+ async function analyzeCommand(ctx, args) {
1338
+ const { config, flags } = ctx;
1339
+ if (flags.json && flags.md) throw new UsageError("reeval analyze takes --json or --md, not both");
1340
+ if (flags.context && flags.save) throw new UsageError("reeval analyze --context makes no analysis to save; drop --save");
1341
+ const entries = await allRuns(config);
1342
+ let evalFilter = flags.eval ?? null;
1343
+ let head;
1344
+ try {
1345
+ head = resolveRun(entries, args[0] ?? "latest");
1346
+ } catch (error) {
1347
+ const needle = args[0]?.toLowerCase();
1348
+ const latest = entries.at(-1);
1349
+ if (!(error instanceof RefError) || !needle || !latest?.evals.some((e) => e.id.toLowerCase().includes(needle))) throw error;
1350
+ evalFilter ??= args[0];
1351
+ head = latest;
1352
+ }
1353
+ const base = flags.vs === "none" ? null : flags.vs ? resolveRun(entries, flags.vs) : await defaultBase(config, entries, head);
1354
+ if (base?.runId === head.runId) throw new UsageError("reeval analyze --vs names the run itself");
1355
+ if (flags.save && !head.recorded) throw new CommandError(`${head.runId} isn't recorded, so there's no history folder to save into. Run reeval record ${head.runId} first, then reeval analyze ${head.runId} --save.`);
1356
+ const model = flags.context || flags["dry-run"] ? null : resolveAnalysisModel(config);
1357
+ const prepared = await prepareAnalysis(config, head, base, evalFilter);
1358
+ const { digest, key, modelName } = prepared;
1359
+ if (evalFilter && digest.setup.evals.length === 0) throw new UsageError(`No eval in ${head.runId} matches "${evalFilter}"`);
1360
+ if (flags.context) {
1361
+ ctx.out(contextMarkdown(digest, head.runId, base?.runId ?? null));
1362
+ return 0;
1363
+ }
1364
+ const cached = flags["no-cache"] ? null : await readCached(config.root, key);
1365
+ if (flags["dry-run"]) {
1366
+ const chars = digestChars(digest);
1367
+ ctx.out([
1368
+ `digest ${head.runId}${base ? ` vs ${base.runId}` : ""}: ${digest.facts.length} facts, ${digest.evidence.length} evidence items${digest.dropped.length ? ` (${digest.dropped.length} dropped to fit)` : ""}, ${Object.keys(digest.attempts).length} attempts`,
1369
+ `size ${chars} characters ≈ ${Math.ceil(chars / 4)} tokens (cap ${config.analysis.maxInputTokens} in, ${config.analysis.maxOutputTokens} out)`,
1370
+ `model ${modelName} · ${ANALYSIS_PROMPT_VERSION} · cache ${cached ? "hit" : "miss"} (.reeval/analysis/${key}.json)`
1371
+ ].join("\n"));
1372
+ return 0;
1373
+ }
1374
+ let doc;
1375
+ if (cached) {
1376
+ doc = cached;
1377
+ ctx.err(`cached analysis from ${cached.createdAt} (.reeval/analysis/${key}.json); --no-cache calls the model again`);
1378
+ } else {
1379
+ const resolved = model ?? resolveAnalysisModel(config);
1380
+ ctx.err(`analyzing ${head.runId}${base ? ` vs ${base.runId}` : ""} with ${resolved.name}…`);
1381
+ doc = await analyzeDigest(config, prepared, resolved);
1382
+ await writeCached(config.root, key, doc);
1383
+ }
1384
+ if (flags.save) {
1385
+ const saved = await saveAnalysis(config.history.dir, doc, key, renderMarkdown(doc));
1386
+ ctx.err(`saved ${saved.map((p) => fromCwd(p)).join(" and ")}. Commit them with git.`);
1387
+ }
1388
+ ctx.out(flags.json ? JSON.stringify(doc, null, 2) : flags.md ? renderMarkdown(doc) : renderText(doc));
1389
+ return 0;
1390
+ }
1391
+ //#endregion
1392
+ export { analyzeCommand };
1033
1393
 
1034
- //# sourceMappingURL=validate-DWjkshl1.js.map
1394
+ //# sourceMappingURL=command-9xm5ImqI.js.map