reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -1,67 +1,22 @@
1
- import { l as sha256 } from "./ids-Bp6ut7ZD.js";
2
- import { a as oneLine, i as formatDuration, n as clip, r as fenced } from "./text-CrVy5ioL.js";
3
- import { n as formatTokens } from "./usage-CleNKkly.js";
4
- import { B as labelledJudgements, D as runtimeLabel, L as commandLine, P as checkLabels, S as languageLabel, _ as harnessLabel, ct as formatSeconds, l as evalMarkOutcome, ot as formatCount, x as isUngatedMiss } from "./model-UgHDMhI1.js";
5
- import { c as suiteResult, r as countedOf, t as cancelledOf } from "./suite-DmuF_ApO.js";
6
- import { a as fromModel } from "./insights-B2Xmhz9o.js";
7
- import { o as warnings } from "./state-C0i_ujQZ.js";
1
+ import { a as plural, i as formatDuration, n as clip, r as fenced } from "./text-DTigcA64.js";
2
+ import { n as formatTokens } from "./usage-BNY1ABrU.js";
3
+ import { B as commandLine, C as isUngatedMiss, F as checkLabels, I as checkTries, S as isNotGatedPass, U as labeledJudgments, ft as formatSeconds, k as runtimeLabel, u as evalMarkOutcome, ut as formatCount, v as harnessLabel, w as languageLabel } from "./model-CTnyvh7H.js";
4
+ import { c as isStudy, r as countedOf, s as isNotGatedResult, t as canceledOf, u as suiteResult } from "./suite-Co8cwelb.js";
5
+ import { o as warnings } from "./state-DrJ687ut.js";
8
6
  import { t as colors } from "./colors-QV9dFVhN.js";
9
- import { r as remarkLines } from "./remarks-1521zwfW.js";
10
- //#region src/insights/hash.ts
11
- /** JSON with every object's keys sorted. */
12
- function canonicalJson(value) {
13
- return JSON.stringify(value, (_key, v) => v && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
14
- }
15
- const factsHash = (suite) => `sha256:${sha256(canonicalJson(suite))}`;
16
- //#endregion
7
+ import { r as remarkLines } from "./remarks-dA8LyoPc.js";
17
8
  //#region src/ui/layout.ts
18
- /** The block for an eval: a pure function of its job and its shape. */
19
- function layout(job, across, isFamily, isCompare) {
9
+ /** The block for an eval: a pure function of its shape. */
10
+ function layout(across, isFamily, isCompare) {
20
11
  if (isCompare) return {
21
12
  kind: "regressions",
22
13
  family: false
23
14
  };
24
15
  return {
25
- kind: job === "choose" && across === "none" ? "choose" : across === "change" ? "measure" : across === "harness" ? "harnesses" : across === "both" ? "both" : "results",
16
+ kind: across === "change" ? "measure" : across === "harness" ? "harnesses" : across === "both" ? "both" : "results",
26
17
  family: isFamily
27
18
  };
28
19
  }
29
- const moved = (c) => c.effect === "better" || c.effect === "worse";
30
- /** A cell the agent failed, or that errored; not one whose only fails the interrupt cancelled. */
31
- const failedCell = (c) => c.outcome === "fail" && c.failed > cancelledOf(c) || c.outcome === "infra_error";
32
- /**
33
- * The insight slots an eval's layout opens: which rows take a sentence (a failing case; an arm
34
- * that failed or moved; a failing harness), whether it has findings (a choose or finish eval that
35
- * measures a change), and whether it has named tools (a choose eval). An `other` eval takes only
36
- * its answer, unless it compares harnesses, which takes a sentence on any job. What the run's
37
- * interrupt cancelled opens nothing: the agent didn't fail it, and its effects aren't measured.
38
- */
39
- function slotsOf(job, e) {
40
- const known = job !== "other";
41
- const measured = e.outcome !== "cancelled";
42
- const harnesses = e.across === "harness" || e.across === "both";
43
- const rows = [];
44
- if (known || harnesses) {
45
- if (e.cases) {
46
- for (const k of e.cases) if (k.outcome !== "pass" && k.outcome !== "cancelled") rows.push(k.path);
47
- for (const p of e.pooled ?? []) {
48
- const pooledMoved = measured && (p.effect === "better" || p.effect === "worse");
49
- if (p.outcome === "fail" || pooledMoved) rows.push(p.path);
50
- }
51
- } else if (e.across === "harness") {
52
- for (const c of e.cells ?? []) if (failedCell(c)) rows.push(c.path);
53
- } else if (e.across === "change" || e.across === "both") {
54
- for (const c of e.cells ?? []) if (failedCell(c) || measured && moved(c)) rows.push(c.path);
55
- }
56
- }
57
- return {
58
- rows,
59
- findings: known && e.across === "change" && measured,
60
- namedTools: job === "choose"
61
- };
62
- }
63
- /** The last line of a run's output while no analysis model writes its answers. */
64
- const NO_MODEL_HINT = "Set analysis.model in reeval.config.ts for answers to what each eval asks.";
65
20
  /** The narrowest a row's label column is, so short labels line up under the name. */
66
21
  const LABEL = 17;
67
22
  const ASSERTION_MAX = 160;
@@ -72,7 +27,7 @@ const VERDICT = {
72
27
  pass: "pass",
73
28
  fail: "fail",
74
29
  infra_error: "infra error",
75
- cancelled: "cancelled",
30
+ canceled: "canceled",
76
31
  skipped: "skipped"
77
32
  };
78
33
  const EFFECT = {
@@ -108,26 +63,23 @@ function table(items, width = LABEL) {
108
63
  }).join(" ")}`.trimEnd();
109
64
  });
110
65
  }
111
- const paintOutcome = (c, o) => o === "pass" ? c.green : o === "fail" ? c.red : o === "infra_error" || o === "cancelled" ? c.yellow : c.dim;
112
- /** A study (every cell `gate: 'none'`) passes whatever it saw, so it never reads `pass`. */
66
+ const paintOutcome = (c, o) => o === "pass" ? c.green : o === "fail" ? c.red : o === "infra_error" || o === "canceled" ? c.yellow : c.dim;
113
67
  function verdict(c, e) {
114
68
  if (e.todo) return t("todo", c.dim);
115
- const cells = e.cells ?? [];
116
- if (e.outcome === "pass" && cells.length && cells.every((x) => !x.gated)) return t("not gated", c.dim);
69
+ if (e.outcome === "pass" && isStudy(e.cells ?? [])) return t("not gated", c.dim);
117
70
  return t(VERDICT[e.outcome], paintOutcome(c, e.outcome));
118
71
  }
119
- const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
120
72
  /**
121
73
  * `3 of 4`, with the attempts that didn't count named: `2 of 3 · 1 infra error · 1 skipped`. A
122
- * cancelled attempt is recorded as a fail, but the agent didn't fail it, so it's named apart.
74
+ * canceled attempt is recorded as a fail, but the agent didn't fail it, so it's named apart.
123
75
  */
124
76
  function cellCount(cell) {
125
- const cancelled = cancelledOf(cell);
77
+ const canceled = canceledOf(cell);
126
78
  const counted = countedOf(cell);
127
79
  const parts = [
128
80
  counted > 0 ? `${cell.passed} of ${counted}` : "",
129
81
  cell.infraError ? plural(cell.infraError, "infra error") : "",
130
- cancelled ? `${cancelled} cancelled` : "",
82
+ canceled ? `${canceled} canceled` : "",
131
83
  cell.skipped ? `${cell.skipped} skipped` : ""
132
84
  ].filter(Boolean);
133
85
  if (!parts.length) return "not run";
@@ -154,7 +106,7 @@ const ERROR_KIND = {
154
106
  assertion: "error",
155
107
  judge: "judge",
156
108
  timeout: "timeout",
157
- aborted: "cancelled",
109
+ aborted: "canceled",
158
110
  infra: "infra error",
159
111
  turn: "turn error"
160
112
  };
@@ -175,22 +127,32 @@ function evidence(c, cell, indent, more = true) {
175
127
  indent: indent + 2,
176
128
  free: t(assertion, c.dim)
177
129
  });
130
+ if (k.retried) items.push({
131
+ indent: indent + 2,
132
+ free: t(k.retried, c.dim)
133
+ });
178
134
  }
179
135
  for (const err of cell.errors) items.push({
180
136
  indent,
181
137
  cells: [errorKind(err), t(errorText(err), c.dim)]
182
138
  });
183
- const passed = cell.checks.filter((k) => !failing(k)).length;
139
+ items.push(...retriedItems(c, cell, indent));
140
+ const passed = cell.checks.filter((k) => !failing(k) && !k.retried).length;
184
141
  if (more && passed && items.length) items.push({
185
142
  indent,
186
143
  free: t(`+${passed} more passed`, c.dim)
187
144
  });
188
145
  return items;
189
146
  }
147
+ /** Passing checks that needed their `c.exec` retries: `npm test passed on try 2 of 3`. */
148
+ const retriedItems = (c, cell, indent) => cell.checks.filter((k) => !failing(k) && k.retried).map((k) => ({
149
+ indent,
150
+ cells: [checkLabel(k.label), t(k.retried ?? "", c.dim)]
151
+ }));
190
152
  /** A control with no environment of its own is the baseline the variants change. */
191
153
  const armOf = (cell) => cell.variant ?? (cell.environment === "-" ? "baseline" : cell.environment);
192
154
  /** Each cell's label: the parts that vary between them (harness, arm, sandbox). */
193
- function labeller(cells, parts) {
155
+ function labeler(cells, parts) {
194
156
  const vary = parts.filter((p) => {
195
157
  const of = (cell) => p === "harness" ? cell.harness : p === "arm" ? armOf(cell) : cell.sandbox;
196
158
  return new Set(cells.map(of)).size > 1;
@@ -269,73 +231,9 @@ function effortTable(c, cells, label, indent, failed, width) {
269
231
  ]
270
232
  }))], width);
271
233
  }
272
- const NO_SLOTS = {
273
- row: () => null,
274
- named: () => [],
275
- subject: null
276
- };
277
- const triesOf = (cells) => cells.reduce((n, x) => n + countedOf(x), 0);
278
- /** `named`, then a row per tool with the tries that named it. */
279
- function namedItems(c, slots, cells, indent) {
280
- const named = slots.named(cells);
281
- if (!named.length) return [];
282
- const of = triesOf(cells);
283
- return [{
284
- indent,
285
- free: t("named", c.dim)
286
- }, ...named.map((n) => ({
287
- indent: indent + 2,
288
- cells: [n.tool, `${n.tries} of ${of}`]
289
- }))];
290
- }
291
- /** The tries that named the subject, over the cells' tries: `0 of 4`; empty with no subject. */
292
- function subjectTries(slots, cells) {
293
- if (!slots.subject) return "";
294
- const subject = slots.subject.toLowerCase();
295
- return `${slots.named(cells).find((x) => x.tool.toLowerCase() === subject)?.tries ?? 0} of ${triesOf(cells)}`;
296
- }
297
- /** Web searches or fetches over the tries, when every try reported a number. */
298
- function webItems(cells, indent) {
299
- const row = (name, counts) => counts.length && counts.every((x) => x !== null) ? [{
300
- indent,
301
- cells: [name, String(counts.reduce((a, b) => (a ?? 0) + (b ?? 0), 0))]
302
- }] : [];
303
- return [...row("searches", cells.flatMap((x) => x.searches)), ...row("fetches", cells.flatMap((x) => x.fetches))];
304
- }
305
- /** A choose case that missed: its failed checks, the tools named, its searches, then `+N more passed`. */
306
- function chooseDetail(c, cells, indent, slots) {
307
- const checks = cells.flatMap((cell) => evidence(c, cell, indent, false));
308
- const passed = cells.flatMap((x) => x.checks.filter((k) => !failing(k))).length;
309
- return [
310
- ...checks,
311
- ...namedItems(c, slots, cells, indent),
312
- ...webItems(cells, indent),
313
- ...passed ? [{
314
- indent,
315
- free: t(`+${passed} more passed`, c.dim)
316
- }] : []
317
- ];
318
- }
319
- /** Choose a tool, on a single eval: the subject's count, and on a miss the detail. */
320
- function chooseBody(c, cells, indent, slots) {
321
- const items = slots.subject ? [{
322
- indent,
323
- cells: [slots.subject, subjectTries(slots, cells)]
324
- }] : [];
325
- if (!cells.every(clean)) items.push(...chooseDetail(c, cells, indent, slots));
326
- return table(items);
327
- }
328
- /** A row's sentence, as a line under it. */
329
- const rowLine = (slots, path, indent) => {
330
- const text = slots.row(path);
331
- return text ? [{
332
- indent,
333
- free: text
334
- }] : [];
335
- };
336
234
  /** Measure a change: a row per arm with its effect, the check matrix and the effort table. */
337
235
  function measureBody(c, cells, indent, o) {
338
- const label = labeller(cells, ["arm", "sandbox"]);
236
+ const label = labeler(cells, ["arm", "sandbox"]);
339
237
  const arms = [{
340
238
  indent,
341
239
  cells: ["arm", t("pass", c.dim)]
@@ -350,7 +248,6 @@ function measureBody(c, cells, indent, o) {
350
248
  ...effect.text ? [effect] : []
351
249
  ]
352
250
  });
353
- arms.push(...rowLine(o.slots, cell.path, indent + 2), ...namedItems(c, o.slots, [cell], indent + 2));
354
251
  const stats = o.verbose && o.effects ? statsLine(cell, cells) : null;
355
252
  if (stats) arms.push({
356
253
  indent: indent + 2,
@@ -371,8 +268,8 @@ function measureBody(c, cells, indent, o) {
371
268
  ];
372
269
  }
373
270
  /** A row per cell (harness, or sandbox), the ones that didn't pass outright with their evidence. */
374
- function cellRows(c, cells, heading, indent, slots) {
375
- const label = labeller(cells, heading === "harness" ? ["harness", "sandbox"] : ["sandbox"]);
271
+ function cellRows(c, cells, heading, indent) {
272
+ const label = labeler(cells, heading === "harness" ? ["harness", "sandbox"] : ["sandbox"]);
376
273
  const items = [{
377
274
  indent,
378
275
  cells: [heading, t("pass", c.dim)]
@@ -382,14 +279,13 @@ function cellRows(c, cells, heading, indent, slots) {
382
279
  indent,
383
280
  cells: [label(cell), cellCount(cell)]
384
281
  });
385
- items.push(...rowLine(slots, cell.path, indent + 2));
386
- if (!clean(cell)) items.push(...namedItems(c, slots, [cell], indent + 2), ...evidence(c, cell, indent + 2));
282
+ if (!clean(cell)) items.push(...evidence(c, cell, indent + 2));
387
283
  }
388
284
  return table(items);
389
285
  }
390
286
  /** Compare harnesses with arms: a heading per harness, its arms as Measure a change under it. */
391
287
  function bothBody(c, cells, indent, o) {
392
- const harnessOf = labeller(cells, ["harness", "sandbox"]);
288
+ const harnessOf = labeler(cells, ["harness", "sandbox"]);
393
289
  const groups = /* @__PURE__ */ new Map();
394
290
  for (const cell of cells) groups.set(harnessOf(cell), [...groups.get(harnessOf(cell)) ?? [], cell]);
395
291
  return [...groups].flatMap(([harness, group]) => [`${" ".repeat(indent)}${harness}`, ...measureBody(c, group, indent + 2, o)]);
@@ -397,15 +293,14 @@ function bothBody(c, cells, indent, o) {
397
293
  /** The rows under an eval's head (or a family case's), by what its cells compare. */
398
294
  function cellsBody(c, kind, cells, indent, o) {
399
295
  if (kind === "measure") return measureBody(c, cells, indent, o);
400
- if (kind === "harnesses") return cellRows(c, cells, "harness", indent, o.slots);
296
+ if (kind === "harnesses") return cellRows(c, cells, "harness", indent);
401
297
  if (kind === "both") return bothBody(c, cells, indent, o);
402
- if (kind === "choose") return chooseBody(c, cells, indent, o.slots);
403
- if (cells.length > 1) return cellRows(c, cells, "sandbox", indent, o.slots);
298
+ if (cells.length > 1) return cellRows(c, cells, "sandbox", indent);
404
299
  return resultsBody(c, cells[0], indent);
405
300
  }
406
301
  /** A single eval with one cell: its tries, and on anything but a clean pass, the evidence. */
407
302
  function resultsBody(c, cell, indent) {
408
- if (clean(cell)) return [];
303
+ if (clean(cell)) return table(retriedItems(c, cell, indent));
409
304
  const items = [];
410
305
  if (cell.attempts > 1) items.push({
411
306
  indent,
@@ -413,8 +308,34 @@ function resultsBody(c, cell, indent) {
413
308
  });
414
309
  return table([...items, ...evidence(c, cell, indent)]);
415
310
  }
311
+ /** Free-text answers shown per question; the rest are in the record. */
312
+ const ANSWERS_SHOWN = 3;
313
+ function attemptQuestionLines(c, q, indent) {
314
+ const pad = " ".repeat(indent);
315
+ const head = `${pad}${c.dim("question")} ${clip(q.question, 100)}`;
316
+ const failed = q.error === null ? [] : [`${pad} ${c.yellow(`${q.failed} of ${q.asked} not answered: ${clip(q.error, 200)}`)}`];
317
+ if (q.choices) return [
318
+ head,
319
+ `${pad} ${q.choices.map((t) => `${t.choice} ${t.n}`).join(" · ")}`,
320
+ ...failed
321
+ ];
322
+ const shown = q.answers.slice(0, ANSWERS_SHOWN);
323
+ const rows = table(shown.map((a) => ({
324
+ indent: indent + 2,
325
+ cells: [a.attempt, clip(a.text.replace(/\s+/g, " "), 100)]
326
+ })));
327
+ const more = q.answers.length - shown.length;
328
+ const rest = more ? [`${pad} ${c.dim(`${more} more: reeval query attempt_answers`)}`] : [];
329
+ return [
330
+ head,
331
+ ...rows,
332
+ ...rest,
333
+ ...failed
334
+ ];
335
+ }
416
336
  function questionLines(c, questions, indent) {
417
337
  return questions.flatMap((q) => {
338
+ if (q.kind === "attempts") return attemptQuestionLines(c, q, indent);
418
339
  const pad = " ".repeat(indent);
419
340
  const head = `${pad}${c.dim("question")} ${clip(q.question, 100)}`;
420
341
  if (q.votes === 0 && q.error === null) return [head, `${pad} ${c.dim("no pairs to judge: tries that failed their checks are left out")}`];
@@ -457,10 +378,9 @@ function pooledStats(p, label) {
457
378
  /**
458
379
  * A family: its template, then a row per case. With one cell a case is a row with its count;
459
380
  * with more, the case is outer and its cells inner, laid out as a single eval's are, and a
460
- * pooled `all cases` row sums each cell's counts with its variant's pooled effect. A choose
461
- * family's case row carries the subject's count instead, and a missed case the tools named.
381
+ * pooled `all cases` row sums each cell's counts with its variant's pooled effect.
462
382
  */
463
- function familyBlock(c, e, kind, verbose, slots) {
383
+ function familyBlock(c, e, kind, verbose) {
464
384
  const cases = e.cases ?? [];
465
385
  const columns = [...new Map(cases.flatMap((k) => k.cells).map((cell) => [cell.key, cell])).values()];
466
386
  const pooled = e.pooled ?? [];
@@ -470,7 +390,7 @@ function familyBlock(c, e, kind, verbose, slots) {
470
390
  todo: false,
471
391
  cells: k.cells
472
392
  });
473
- const pooledWord = t(VERDICT[pooledOutcome], paintOutcome(c, pooledOutcome));
393
+ const pooledWord = isNotGatedResult(e) ? t("not gated", c.dim) : t(VERDICT[pooledOutcome], paintOutcome(c, pooledOutcome));
474
394
  const words = [...cases.map(caseWord), ...pooled.length ? [pooledWord] : []];
475
395
  const wordWidth = Math.max(...words.map((w) => w.text.length));
476
396
  const lead = (word, name) => ({
@@ -479,38 +399,6 @@ function familyBlock(c, e, kind, verbose, slots) {
479
399
  });
480
400
  const head = `${" ".repeat(17)}${e.template ?? e.name}`;
481
401
  const questions = questionLines(c, e.questions, 2);
482
- if (kind === "choose") {
483
- const subject = slots.subject ? [t(slots.subject, c.dim)] : [];
484
- const items = [{
485
- indent: 2,
486
- cells: [
487
- "case",
488
- t("pass", c.dim),
489
- ...subject
490
- ]
491
- }];
492
- for (const k of cases) {
493
- const passed = k.cells.reduce((n, x) => n + x.passed, 0);
494
- const tries = triesOf(k.cells);
495
- const count = k.cells.length === 1 && k.cells[0] ? cellCount(k.cells[0]) : tries ? `${passed} of ${tries}` : "not run";
496
- const named = slots.subject && tries ? [subjectTries(slots, k.cells)] : [];
497
- items.push({
498
- indent: 2,
499
- cells: [
500
- lead(caseWord(k), k.label),
501
- count,
502
- ...named
503
- ]
504
- });
505
- items.push(...rowLine(slots, k.path, 4));
506
- if (!k.cells.every(clean)) items.push(...chooseDetail(c, k.cells, 4, slots));
507
- }
508
- return [
509
- head,
510
- ...table(items),
511
- ...questions
512
- ];
513
- }
514
402
  if (columns.length <= 1) {
515
403
  const items = [];
516
404
  for (const k of cases) {
@@ -519,7 +407,6 @@ function familyBlock(c, e, kind, verbose, slots) {
519
407
  indent: 2,
520
408
  cells: [lead(caseWord(k), k.label), cell ? cellCount(cell) : ""]
521
409
  });
522
- items.push(...rowLine(slots, k.path, 4));
523
410
  if (cell && !clean(cell)) items.push(...evidence(c, cell, 4));
524
411
  }
525
412
  return [
@@ -531,18 +418,15 @@ function familyBlock(c, e, kind, verbose, slots) {
531
418
  const out = [head];
532
419
  for (const k of cases) {
533
420
  out.push(` ${show(lead(caseWord(k), k.label))}`);
534
- const sentence = slots.row(k.path);
535
- if (sentence) out.push(` ${sentence}`);
536
421
  const failed = k.outcome !== "pass";
537
422
  out.push(...cellsBody(c, kind, k.cells, 4, {
538
423
  failed,
539
424
  verbose,
540
- effects: false,
541
- slots
425
+ effects: false
542
426
  }));
543
427
  }
544
428
  if (pooled.length) {
545
- const label = labeller(columns, [
429
+ const label = labeler(columns, [
546
430
  "harness",
547
431
  "arm",
548
432
  "sandbox"
@@ -563,7 +447,6 @@ function familyBlock(c, e, kind, verbose, slots) {
563
447
  ...effect.text ? [effect] : []
564
448
  ]
565
449
  });
566
- if (p) items.push(...rowLine(slots, p.path, 6));
567
450
  if (p && verbose) items.push({
568
451
  indent: 6,
569
452
  free: t(pooledStats(p, label(col)), c.dim)
@@ -573,43 +456,40 @@ function familyBlock(c, e, kind, verbose, slots) {
573
456
  }
574
457
  return [...out, ...questions];
575
458
  }
576
- /** The slots an eval's insight fills, or none without one. */
577
- function slotsFor(insight) {
578
- if (!insight) return NO_SLOTS;
579
- const rows = new Map((insight.rows ?? []).map((r) => [r.row, oneLine(r.text)]));
580
- const choose = insight.intent.job === "choose";
581
- const named = (cells) => {
582
- if (!choose) return [];
583
- const counts = /* @__PURE__ */ new Map();
584
- for (const n of insight.namedTools ?? []) {
585
- if (!cells.some((cell) => n.attempt.startsWith(`${cell.path}#`))) continue;
586
- for (const tool of new Set(n.tools.map(oneLine))) counts.set(tool, (counts.get(tool) ?? 0) + 1);
587
- }
588
- return [...counts].map(([tool, tries]) => ({
589
- tool,
590
- tries
591
- })).sort((a, b) => b.tries - a.tries || (a.tool < b.tool ? -1 : 1));
592
- };
593
- return {
594
- row: (path) => rows.get(path) ?? null,
595
- named,
596
- subject: choose && insight.intent.subject.tool ? oneLine(insight.intent.subject.tool) : null
597
- };
459
+ /** `Anthropic 12 · Amazon Bedrock 3`: each upstream provider and how many calls it served, summed. */
460
+ function upstreamList(counts) {
461
+ const total = /* @__PURE__ */ new Map();
462
+ for (const u of counts) total.set(u.provider, (total.get(u.provider) ?? 0) + u.calls);
463
+ return [...total].map(([provider, calls]) => `${provider} ${calls}`).join(" · ");
464
+ }
465
+ /**
466
+ * Which upstream providers a gateway said served each harness's calls and its judge's, a line per
467
+ * harness, so a run OpenRouter spread over several shows it. None where the gateway named none.
468
+ */
469
+ function upstreamLines(c, cells, indent) {
470
+ const by = /* @__PURE__ */ new Map();
471
+ for (const cell of cells) {
472
+ if (!cell.upstreams && !cell.judgeUpstreams) continue;
473
+ const seen = by.get(cell.harness) ?? {
474
+ agent: [],
475
+ judge: []
476
+ };
477
+ seen.agent.push(...cell.upstreams ?? []);
478
+ seen.judge.push(...cell.judgeUpstreams ?? []);
479
+ by.set(cell.harness, seen);
480
+ }
481
+ return [...by].map(([harness, { agent, judge }]) => {
482
+ const parts = [...agent.length ? [`served by ${upstreamList(agent)}`] : [], ...judge.length ? [`judge served by ${upstreamList(judge)}`] : []];
483
+ const label = by.size > 1 ? `${harness} ` : "";
484
+ return `${" ".repeat(indent)}${c.dim(`${label}${parts.join("; ")}`)}`;
485
+ });
598
486
  }
599
- /** One eval's block, laid out as `layout()` says for its job, with its insight's sentences. */
600
- function evalBlock(c, e, verbose = false, insight = null) {
601
- const shape = layout(insight?.intent.job ?? "other", e.across, e.family, false);
602
- const slots = slotsFor(insight);
603
- const answer = insight?.answer ? [` ${oneLine(insight.answer.text)}`] : [];
604
- const findings = (insight?.findings ?? []).map((f) => ` ${oneLine(f.text)}`);
487
+ /** One eval's block, laid out as `layout()` says for its shape. */
488
+ function evalBlock(c, e, verbose = false) {
489
+ const shape = layout(e.across, e.family, false);
605
490
  if (shape.family) {
606
- const [head = "", ...rest] = familyBlock(c, e, shape.kind, verbose, slots);
607
- return [
608
- head,
609
- ...answer,
610
- ...rest,
611
- ...findings
612
- ];
491
+ const served = upstreamLines(c, (e.cases ?? []).flatMap((k) => k.cells), 2);
492
+ return [...familyBlock(c, e, shape.kind, verbose), ...served];
613
493
  }
614
494
  const v = verdict(c, e);
615
495
  const name = `${e.group.length ? `${e.group.join(" › ")} › ` : ""}${e.name}`;
@@ -617,40 +497,37 @@ function evalBlock(c, e, verbose = false, insight = null) {
617
497
  const cells = e.cells ?? [];
618
498
  if (e.todo || e.outcome === "skipped" || !cells.length) return [head];
619
499
  const failed = e.outcome !== "pass";
620
- const body = cellsBody(c, shape.kind, cells, 2, {
621
- failed,
622
- verbose,
623
- effects: true,
624
- slots
625
- });
626
500
  return [
627
501
  head,
628
- ...answer,
629
- ...body,
630
- ...findings,
502
+ ...cellsBody(c, shape.kind, cells, 2, {
503
+ failed,
504
+ verbose,
505
+ effects: true
506
+ }),
507
+ ...upstreamLines(c, cells, 2),
631
508
  ...questionLines(c, e.questions, 2)
632
509
  ];
633
510
  }
634
511
  /**
635
- * `5 fail · 2 pass · 1 cancelled`: errors outside any eval and fails first, every case counting
636
- * one, zeros left out.
512
+ * `5 fail · 2 pass · 1 not gated · 1 canceled`: errors outside any eval and fails first, every
513
+ * case counting one, zeros left out.
637
514
  */
638
515
  function tallyLine(c, tally) {
639
516
  const parts = [
640
517
  tally.errors ? c.red(plural(tally.errors, "run error")) : "",
641
518
  tally.fail ? c.red(`${tally.fail} fail`) : "",
642
519
  tally.pass ? c.green(`${tally.pass} pass`) : "",
520
+ tally.notGated ? c.dim(`${tally.notGated} not gated`) : "",
643
521
  tally.infraError ? c.yellow(plural(tally.infraError, "infra error")) : "",
644
- tally.cancelled ? c.yellow(`${tally.cancelled} cancelled`) : "",
522
+ tally.canceled ? c.yellow(`${tally.canceled} canceled`) : "",
645
523
  tally.skipped ? c.dim(`${tally.skipped} skipped`) : ""
646
524
  ].filter(Boolean);
647
525
  return parts.length ? parts.join(c.dim(" · ")) : "no evals ran";
648
526
  }
649
527
  /** Every eval's block in file-path order, then source order; a blank line around each longer one. */
650
- function renderBlocks(suite, opts, insights = null) {
528
+ function renderBlocks(suite, opts) {
651
529
  const c = colors(opts.color);
652
- const byEval = new Map((insights?.evals ?? []).map((x) => [x.evalId, x]));
653
- const blocks = suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).map((e) => evalBlock(c, e, opts.verbose, byEval.get(e.path) ?? null)));
530
+ const blocks = suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).map((e) => evalBlock(c, e, opts.verbose)));
654
531
  const out = [];
655
532
  blocks.forEach((block, i) => {
656
533
  const prev = blocks[i - 1];
@@ -659,51 +536,20 @@ function renderBlocks(suite, opts, insights = null) {
659
536
  });
660
537
  return out;
661
538
  }
662
- /** Insights for these facts: null when there are none, they failed, or they were written for other facts. */
663
- const freshInsights = (suite, insights) => insights && !insights.error && insights.factsHash === factsHash(suite) ? insights : null;
664
- /** Why a run's insights didn't print, or with `-v` what wrote them. */
665
- function insightNotes(c, suite, insights, fresh, verbose) {
666
- if (!insights || !fromModel(insights)) return [];
667
- if (insights.error) return [c.yellow(`insights failed (${clip(insights.error, 200)})`)];
668
- if (!fresh) return [c.dim(`insights left out: insights.json was written for other facts than this run's; delete it and run reeval analyze ${suite.runId} --insights`)];
669
- const skipped = insights.budget?.skipped.length ?? 0;
670
- const failed = insights.failed ?? [];
671
- const stopped = insights.stopped;
672
- const finish = `reeval analyze ${suite.runId} --insights to finish`;
673
- const budget = [
674
- ...skipped ? [c.yellow(`insights skipped for ${plural(skipped, "eval")}: analysis.maxTokens (${formatTokens(insights.budget?.maxTokens ?? 0)}) reached — raise it in reeval.config.ts, or ${finish}`)] : [],
675
- ...failed[0] ? [c.yellow(`insights failed for ${plural(failed.length, "eval")} (${clip(failed[0].error, 160)}), so they print their facts: ${finish}`)] : [],
676
- ...stopped ? [c.yellow(`insights stopped: ${plural(stopped.skipped.length, "eval")} skipped (${stopped.signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"}): ${finish}`)] : []
677
- ];
678
- if (!verbose) return budget;
679
- const { dropped, regenerated } = insights.validation;
680
- return [...budget, c.dim([
681
- `insights by ${insights.model} (${insights.promptVersion})`,
682
- `${plural(dropped, "sentence")} removed`,
683
- ...regenerated ? [`${regenerated} asked again`] : [],
684
- `${formatTokens(insights.usage.inputTokens)} in, ${formatTokens(insights.usage.outputTokens)} out${insights.usage.reasoningTokens ? ` (${formatTokens(insights.usage.reasoningTokens)} reasoning)` : ""}`
685
- ].join(" · "))];
686
- }
687
539
  /**
688
- * A run's result: its errors outside any eval, its blocks with their insights, the headline,
689
- * then the tally and, with no analysis model, the hint, then the remarks. Insights written for
690
- * other facts are left out with a note; remarks come from the record and history, not a model,
691
- * so they print either way.
540
+ * A run's result: its errors outside any eval, its blocks, then the tally and the remarks, which
541
+ * come from the record and the history.
692
542
  */
693
- function render(suite, insights, opts) {
543
+ function render(suite, remarks, opts) {
694
544
  const c = colors(opts.color);
695
- const fresh = freshInsights(suite, insights);
696
- const blocks = renderBlocks(suite, opts, fresh);
545
+ const blocks = renderBlocks(suite, opts);
697
546
  return [
698
547
  ...suite.errors.flatMap((error) => [c.red(`error: ${error}`), ""]),
699
548
  ...blocks,
700
549
  ...blocks.length ? [""] : [],
701
- ...fresh?.headline ? [oneLine(fresh.headline.text)] : [],
702
- ...insightNotes(c, suite, insights, fresh, opts.verbose),
703
550
  ...opts.notes ?? [],
704
551
  tallyLine(c, suite.tally),
705
- ...opts.hint ? [NO_MODEL_HINT] : [],
706
- ...remarkLines(c, suite, insights, opts.verbose)
552
+ ...remarkLines(c, suite, remarks, opts.verbose)
707
553
  ];
708
554
  }
709
555
  //#endregion
@@ -742,7 +588,7 @@ function stateRows(run) {
742
588
  return rows;
743
589
  }
744
590
  /** Markdown for people and agents landing on a run cold. Links are relative to the artifact dir. */
745
- function renderSummary(run, opts) {
591
+ function renderSummary(run, opts = {}) {
746
592
  const lines = [
747
593
  `# reeval run ${run.runId}`,
748
594
  "",
@@ -753,10 +599,9 @@ function renderSummary(run, opts) {
753
599
  ...run.plan.sandboxWarnings.map((w) => `> **Warning:** ${w.message}\n`),
754
600
  "## Results",
755
601
  "",
756
- ...fenced(render(suiteResult(run), opts.insights ?? null, {
602
+ ...fenced(render(suiteResult(run), opts.remarks ?? null, {
757
603
  color: false,
758
- verbose: false,
759
- hint: opts.hint
604
+ verbose: false
760
605
  }), "text"),
761
606
  "",
762
607
  "## State",
@@ -773,7 +618,8 @@ function renderSummary(run, opts) {
773
618
  const rank = (o) => o === "fail" || o === "infra_error" ? 0 : o === "pass" ? 1 : 2;
774
619
  for (const e of [...run.evals].sort((x, y) => rank(x.outcome) - rank(y.outcome))) {
775
620
  if (e.outcome === "skipped") continue;
776
- lines.push("", `### ${OUTCOME_MARK[evalMarkOutcome(e)]} ${e.id}`, "");
621
+ const mark = isNotGatedPass(e) ? "○" : OUTCOME_MARK[evalMarkOutcome(e)];
622
+ lines.push("", `### ${mark} ${e.id}${isNotGatedPass(e) ? " · not gated" : ""}`, "");
777
623
  const multi = e.attempts.length > 1;
778
624
  const inputs = (e.answers ?? []).filter((a) => a.input);
779
625
  for (const a of inputs) lines.push(`- Question: ${cell(`"${clip(a.question, 80)}"`)} · [what the model saw](${a.input?.path})`);
@@ -808,14 +654,19 @@ function attemptDetail(lines, a) {
808
654
  const act = a.activity;
809
655
  const web = (n, what) => n === null ? `${what} not reported` : `${n} ${what}`;
810
656
  lines.push(`- Agent: ${act.fileEdits} file edits · ${act.tools} other tool calls · ${web(act.webSearches, "web searches")} · ${web(act.webFetches, "fetches")} · ${formatDuration(act.reasoningMs)} reasoning${act.stalledTools ? ` · ${act.stalledTools} stalled tool${act.stalledTools === 1 ? "" : "s"} (≥2m)${act.maxToolMs ? `, longest ${formatDuration(act.maxToolMs)}` : ""}` : ""}`);
811
- for (const { judgement: j, label, labelled } of labelledJudgements(a)) {
657
+ for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
812
658
  const retried = j.retries?.length ? `, after ${j.retries.length} ${j.retries.length === 1 ? "retry" : "retries"}` : "";
813
- lines.push(`- Judge${labelled ? ` ${cell(label)}` : ""} **${j.verdict}** ${j.score.toFixed(2)} (threshold ${j.threshold}, ${j.model}${retried}): ${j.question}`);
659
+ lines.push(`- Judge${labeled ? ` ${cell(label)}` : ""} **${j.verdict}** ${j.score.toFixed(2)} (threshold ${j.threshold}, ${j.model}${retried}): ${j.question}`);
660
+ }
661
+ for (const q of a.answers ?? []) {
662
+ const said = q.error ? `failed: ${clip(cell(q.error), 200)}` : `${q.choice !== void 0 ? `**${cell(q.choice)}** ` : ""}${clip(cell(q.text), 300)}`;
663
+ lines.push(`- Question ${cell(`"${clip(q.question, 80)}"`)}: ${said}${q.input ? ` ([what the model saw](${q.input.path}))` : ""}`);
814
664
  }
815
665
  const labels = checkLabels(a);
816
666
  a.checks.forEach((c, i) => {
817
667
  const label = labels[i];
818
- lines.push(`- Check${label ? ` ${cell(label)}` : ""} exit ${c.exitCode}: \`${cell(commandLine(c.command))}\`${c.log ? ` ([log](${c.log.path}))` : ""}`);
668
+ const tries = checkTries(c);
669
+ lines.push(`- Check${label ? ` ${cell(label)}` : ""} exit ${c.exitCode}${tries ? ` (${tries})` : ""}: \`${cell(commandLine(c.command))}\`${c.log ? ` ([log](${c.log.path}))` : ""}`);
819
670
  });
820
671
  if (a.changedFiles.length) lines.push(`- Changed: ${a.changedFiles.map((f) => f.count ? `\`${f.path}\` (${f.count} files)` : `${f.status} \`${f.path}\``).join(", ")}`);
821
672
  const last = a.turns.at(-1)?.text;
@@ -832,6 +683,6 @@ function attemptDetail(lines, a) {
832
683
  if (links.length) lines.push(`- Files: ${links.join(" · ")}`);
833
684
  }
834
685
  //#endregion
835
- export { t as _, assertionOf as a, errorKind as c, labeller as d, plural as f, slotsOf as g, show as h, stateRows as i, errorText as l, renderBlocks as m, renderSummary as n, checkCount as o, render as p, runTally as r, checkLabel as s, OUTCOME_MARK as t, evalBlock as u, table as v, factsHash as y };
686
+ export { assertionOf as a, errorKind as c, labeler as d, render as f, table as g, t as h, stateRows as i, errorText as l, show as m, renderSummary as n, checkCount as o, renderBlocks as p, runTally as r, checkLabel as s, OUTCOME_MARK as t, evalBlock as u };
836
687
 
837
- //# sourceMappingURL=summary-BFMHMTz2.js.map
688
+ //# sourceMappingURL=summary-DLFHVoux.js.map