reeval 0.14.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/README.md +3 -3
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
  9. package/dist/app-files-ChbvRx3P.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
  11. package/dist/assemble-SKQGW_QF.js.map +1 -0
  12. package/dist/assemble-XBWuK3Ed.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
  16. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
  18. package/dist/capture-BM_3100g.js.map +1 -0
  19. package/dist/capture-DSK8O5bA.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
  21. package/dist/cli-PdYrcoKM.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
  23. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
  26. package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
  27. package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
  28. package/dist/command-4pJYVCT4.js.map +1 -0
  29. package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
  30. package/dist/command-9xm5ImqI.js.map +1 -0
  31. package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
  32. package/dist/command-BqBxLgTc.js.map +1 -0
  33. package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
  34. package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
  35. package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
  36. package/dist/commands-BU6YJCz9.js.map +1 -0
  37. package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
  38. package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
  39. package/dist/config-k6m7XbAz.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-CpsifVln.js +1530 -0
  46. package/dist/db-CpsifVln.js.map +1 -0
  47. package/dist/db-DkD-F-X-.js +2 -0
  48. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  49. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  50. package/dist/debug-DeM5pswK.js +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
  52. package/dist/docker-51WvhN_X.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
  54. package/dist/docker-B4-rx4dA.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
  57. package/dist/doctor-BuQnfz4e.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
  61. package/dist/environment-BOFgzFDg.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  63. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  64. package/dist/eval-source-Dje3rERU.js +145 -0
  65. package/dist/eval-source-Dje3rERU.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
  67. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  68. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  69. package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
  70. package/dist/files-Bi461Be2.js.map +1 -0
  71. package/dist/files-C2h2Cw4F.js +1 -0
  72. package/dist/files-Clipx4OW.js +37 -0
  73. package/dist/files-Clipx4OW.js.map +1 -0
  74. package/dist/files-DmUrRmmL.js +1 -0
  75. package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
  76. package/dist/files-H24pz0hL.js.map +1 -0
  77. package/dist/gateway.d.ts +22 -9
  78. package/dist/gateway.js +30 -8
  79. package/dist/gateway.js.map +1 -1
  80. package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
  81. package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
  82. package/dist/harness-3NVyTSGK.js +392 -0
  83. package/dist/harness-3NVyTSGK.js.map +1 -0
  84. package/dist/harness.d.ts +12 -8
  85. package/dist/harness.js +1 -1
  86. package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
  87. package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
  88. package/dist/index.d.ts +51 -10
  89. package/dist/index.js +5 -5
  90. package/dist/init-BDEOopK7.js +1 -0
  91. package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
  92. package/dist/init-cOk3D44b.js.map +1 -0
  93. package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
  94. package/dist/invocations-DW-lTk2f.js.map +1 -0
  95. package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
  96. package/dist/judge-CKdgEXN7.js.map +1 -0
  97. package/dist/judge.d.ts +5 -7
  98. package/dist/judge.js +4 -8
  99. package/dist/judge.js.map +1 -1
  100. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  101. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  102. package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
  103. package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
  104. package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
  105. package/dist/model-Bq3G0OG1.js.map +1 -0
  106. package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
  107. package/dist/model-CTnyvh7H.js.map +1 -0
  108. package/dist/model-D-SWrLZG.js +1 -0
  109. package/dist/model-g3XnzUv0.js +1 -0
  110. package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
  111. package/dist/model-vRiUycWh.js.map +1 -0
  112. package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
  113. package/dist/node-BVbbnFtv.js.map +1 -0
  114. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  115. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  116. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  117. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  118. package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
  119. package/dist/pairs-puYi-zj2.js.map +1 -0
  120. package/dist/patch-CquGAGX5.js +76 -0
  121. package/dist/patch-CquGAGX5.js.map +1 -0
  122. package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
  123. package/dist/patch-DKJiC5Sq.js.map +1 -0
  124. package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
  125. package/dist/plan-DdWkM6SW.js.map +1 -0
  126. package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
  127. package/dist/plugins-D-CP6mMd.js.map +1 -0
  128. package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
  129. package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
  130. package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
  131. package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
  132. package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
  133. package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
  134. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  135. package/dist/prices-QaS-kjSQ.js.map +1 -0
  136. package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
  137. package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
  138. package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
  139. package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
  140. package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
  141. package/dist/query-Clm2mNSm.js +1 -0
  142. package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
  143. package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
  144. package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
  145. package/dist/questions-BRhU8MzB.js.map +1 -0
  146. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  147. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  148. package/dist/redact-DVIaahLv.js +658 -0
  149. package/dist/redact-DVIaahLv.js.map +1 -0
  150. package/dist/remarks-BwJ8pcTQ.js +1 -0
  151. package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
  152. package/dist/remarks-C7i2LMRb.js.map +1 -0
  153. package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
  154. package/dist/remarks-D_3qhBWU.js.map +1 -0
  155. package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
  156. package/dist/remarks-dA8LyoPc.js.map +1 -0
  157. package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
  158. package/dist/remote-DFIzcx_6.js.map +1 -0
  159. package/dist/resolve--rQZ8u6Z.js +1 -0
  160. package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
  161. package/dist/resolve-DWSJpKHX.js.map +1 -0
  162. package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
  163. package/dist/results-BC3pcG_Z.js.map +1 -0
  164. package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
  165. package/dist/resume-j-rcIryi.js.map +1 -0
  166. package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
  167. package/dist/runtime-d56Gg3J5.js.map +1 -0
  168. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
  169. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
  170. package/dist/sandbox.d.ts +1 -1
  171. package/dist/sandbox.js +1 -1
  172. package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
  173. package/dist/server-BOoH10J8.js.map +1 -0
  174. package/dist/server-DpuVAbrO.js +1 -0
  175. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  176. package/dist/shared-B_lvGqWz.js.map +1 -0
  177. package/dist/source-BFd6f0-R.js +1 -0
  178. package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
  179. package/dist/source-wD04-rRG.js.map +1 -0
  180. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  181. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  182. package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
  183. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  184. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  185. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  186. package/dist/src-BO8AYqJ5.js.map +1 -0
  187. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  188. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  189. package/dist/src-BYPMM1x7.js +2 -0
  190. package/dist/src-Bp4aN3rA.js +2 -0
  191. package/dist/src-C7upnZSd.js +2 -0
  192. package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
  193. package/dist/src-D91Hd6hT.js.map +1 -0
  194. package/dist/src-DH-9IO7f.js +1 -0
  195. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  196. package/dist/src-DSZyg0Uq.js.map +1 -0
  197. package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
  198. package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
  199. package/dist/src-HSxBazcg.js +3 -0
  200. package/dist/src-LD1BAJFa.js +2 -0
  201. package/dist/src-dlX-Lmxg.js +2 -0
  202. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  203. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  204. package/dist/src-kYeaJJrl.js +2 -0
  205. package/dist/state-B5PLm5qY.js +1 -0
  206. package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
  207. package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
  208. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  209. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  210. package/dist/suite-22Kt9Grx.js +2 -0
  211. package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
  212. package/dist/suite-Co8cwelb.js.map +1 -0
  213. package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
  214. package/dist/summary-DLFHVoux.js.map +1 -0
  215. package/dist/testing.d.ts +417 -142
  216. package/dist/testing.js +12 -10
  217. package/dist/testing.js.map +1 -1
  218. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  219. package/dist/text-DTigcA64.js.map +1 -0
  220. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  221. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  222. package/dist/types-C5vI3g1C.js.map +1 -1
  223. package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
  224. package/dist/vertex-Bv7vhMgs.js +1 -0
  225. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  226. package/dist/vertex-DypYjx71.js +456 -0
  227. package/dist/vertex-DypYjx71.js.map +1 -0
  228. package/dist/view/index.d.ts +536 -121
  229. package/dist/view/index.js +9 -9
  230. package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
  231. package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
  232. package/dist/views-BOf8qEL8.js +2 -0
  233. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
  234. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  235. package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
  236. package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
  237. package/dist/workspace-DDHzkMjm.js +1 -0
  238. package/docker/seccomp-default.json +1 -0
  239. package/package.json +5 -1
  240. package/skills/eval-rubric/SKILL.md +35 -104
  241. package/dist/adapter-BloGVZ_a.js.map +0 -1
  242. package/dist/agent-m-_t-IuW.js.map +0 -1
  243. package/dist/app-files-C-jwhRDt.js.map +0 -1
  244. package/dist/assemble-BDEpelpi.js.map +0 -1
  245. package/dist/assemble-C5bfncHU.js +0 -1
  246. package/dist/capture-CVRwRTnb.js.map +0 -1
  247. package/dist/capture-CherbjzB.js +0 -1
  248. package/dist/cli-C_4gaJ50.js.map +0 -1
  249. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  250. package/dist/command-BWAuwQNk.js +0 -91
  251. package/dist/command-BWAuwQNk.js.map +0 -1
  252. package/dist/command-BXLUfw4h.js.map +0 -1
  253. package/dist/command-Be6BSE2L.js +0 -335
  254. package/dist/command-Be6BSE2L.js.map +0 -1
  255. package/dist/command-Bxdy62dW.js.map +0 -1
  256. package/dist/commands-jzigbNFb.js.map +0 -1
  257. package/dist/config-BrdHA9bJ.js.map +0 -1
  258. package/dist/cost-D4elx933.js.map +0 -1
  259. package/dist/cost-DmnAod9x.js +0 -1
  260. package/dist/db-CAfksinX.js +0 -2
  261. package/dist/db-DO0p7xgP.js +0 -619
  262. package/dist/db-DO0p7xgP.js.map +0 -1
  263. package/dist/debug-D7bEv_UV.js +0 -1
  264. package/dist/docker-BnRlTpZA.js.map +0 -1
  265. package/dist/docker-D01slDF3.js.map +0 -1
  266. package/dist/doctor-DQ2p0euG.js.map +0 -1
  267. package/dist/env-BQqj93gu.js.map +0 -1
  268. package/dist/environment-CO0P0soF.js.map +0 -1
  269. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  270. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  271. package/dist/facts-DzOUTreB.js +0 -906
  272. package/dist/facts-DzOUTreB.js.map +0 -1
  273. package/dist/files-B-CJeeEh.js.map +0 -1
  274. package/dist/files-Bp5j5BqI.js +0 -1
  275. package/dist/files-C2sjmXhk.js +0 -1
  276. package/dist/files-C_gQNVB-.js +0 -65
  277. package/dist/files-C_gQNVB-.js.map +0 -1
  278. package/dist/files-kUCrswI6.js.map +0 -1
  279. package/dist/generate-DZ7vp69F.js +0 -1647
  280. package/dist/generate-DZ7vp69F.js.map +0 -1
  281. package/dist/harness-COVXSwJ6.js +0 -354
  282. package/dist/harness-COVXSwJ6.js.map +0 -1
  283. package/dist/header-BLPP6aCU.js.map +0 -1
  284. package/dist/init-RgCQXZhb.js +0 -1
  285. package/dist/init-SRalM5xF.js.map +0 -1
  286. package/dist/insights-TyAvz9aV.js +0 -68
  287. package/dist/insights-TyAvz9aV.js.map +0 -1
  288. package/dist/invocations-DzyKSJBa.js.map +0 -1
  289. package/dist/judge-yLvjC5A3.js.map +0 -1
  290. package/dist/match-Bz0nSdzN.js.map +0 -1
  291. package/dist/model-1ZLr9Dcv.js.map +0 -1
  292. package/dist/model-B45lDUV1.js.map +0 -1
  293. package/dist/model-BTZq8HIZ.js +0 -1
  294. package/dist/model-CgnOpss4.js.map +0 -1
  295. package/dist/model-Ch8kX87u.js +0 -1
  296. package/dist/node-BSq5qHnQ.js.map +0 -1
  297. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  298. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  299. package/dist/pairs-CHfZNRT6.js.map +0 -1
  300. package/dist/patch-DWf-I_Q_.js.map +0 -1
  301. package/dist/plan-BIwUzlB8.js.map +0 -1
  302. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  303. package/dist/prices-BcLEXvdY.js.map +0 -1
  304. package/dist/query-Cvbwl804.js +0 -1
  305. package/dist/questions-bAudIc04.js.map +0 -1
  306. package/dist/redact-CRGdfV2s.js +0 -240
  307. package/dist/redact-CRGdfV2s.js.map +0 -1
  308. package/dist/remarks-CXwSnSjR.js +0 -278
  309. package/dist/remarks-CXwSnSjR.js.map +0 -1
  310. package/dist/remarks-CyAu2DiB.js +0 -1
  311. package/dist/remarks-mujJv1Ey.js.map +0 -1
  312. package/dist/remote-CKcEKja4.js.map +0 -1
  313. package/dist/resolve-B5-uqmJ6.js +0 -1
  314. package/dist/resolve-CBoKzQMP.js.map +0 -1
  315. package/dist/results-CMgZEANy.js.map +0 -1
  316. package/dist/resume-B9SOSnK9.js.map +0 -1
  317. package/dist/rules-VhftPOOT.js.map +0 -1
  318. package/dist/run-CwIEK16q.js +0 -88
  319. package/dist/run-CwIEK16q.js.map +0 -1
  320. package/dist/runtime-BgQRyCLl.js.map +0 -1
  321. package/dist/server-CHRSgKRp.js +0 -1
  322. package/dist/server-TGW4WyAz.js.map +0 -1
  323. package/dist/shared-BDUzStad.js.map +0 -1
  324. package/dist/source-B1WvBIQt.js.map +0 -1
  325. package/dist/source-DAzztZwU.js +0 -1
  326. package/dist/src-1jHbFaGW.js +0 -1
  327. package/dist/src-8zoX_qrb.js.map +0 -1
  328. package/dist/src-C_9wgowC.js +0 -2
  329. package/dist/src-ChhtI6BP.js.map +0 -1
  330. package/dist/src-D005Cbqz.js +0 -2
  331. package/dist/src-DIfRbp5a.js +0 -3
  332. package/dist/src-Dg8121Xy.js +0 -2
  333. package/dist/src-ElBqEN-S.js +0 -2
  334. package/dist/src-NSdQ9zI6.js.map +0 -1
  335. package/dist/src-YHGaUrGE.js +0 -2
  336. package/dist/src-mJVxw1Jm.js +0 -2
  337. package/dist/state-vznMh2wl.js +0 -1
  338. package/dist/suite-BsWGHzlG.js +0 -2
  339. package/dist/suite-WgT7m6Vs.js.map +0 -1
  340. package/dist/summary-DIGEVr08.js.map +0 -1
  341. package/dist/text-CrVy5ioL.js.map +0 -1
  342. package/dist/transcript-BTxJpW4R.js.map +0 -1
  343. package/dist/usage-DF3_p5pE.js +0 -1
  344. package/dist/validate-BhwqrVsm.js.map +0 -1
  345. package/dist/views-BW_C5eVH.js +0 -2
  346. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  347. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,3 +1,63 @@
1
+ //#region src/declaration.d.ts
2
+ /** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
3
+ type DeclaredMode = 'run' | 'skip' | 'only';
4
+ /** A `group`: options its evals and groups inherit, and its children in declaration order. */
5
+ type DeclaredGroup = {
6
+ type: 'group';
7
+ /** The group's name, as passed to `group`. */
8
+ name: string;
9
+ mode: DeclaredMode;
10
+ /** The group's own options, before the cascade. */
11
+ options: EvalOptionsData;
12
+ children: DeclaredNode[];
13
+ };
14
+ /** An `evaluate`: one eval, planned into a test per attempt. */
15
+ type DeclaredEval = {
16
+ type: 'eval';
17
+ /** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
18
+ name: string;
19
+ mode: DeclaredMode;
20
+ /** The eval's own options, before the cascade. */
21
+ options: EvalOptionsData;
22
+ /** Names the eval's body to whoever runs it. */
23
+ body: string;
24
+ source: DeclaredSource;
25
+ /** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
26
+ family?: DeclaredFamily;
27
+ };
28
+ /**
29
+ * An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
30
+ * text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
31
+ * whitespace, non-ASCII characters as themselves).
32
+ */
33
+ type DeclaredSource = {
34
+ /** sha256 of the body function's source text. */
35
+ body: string;
36
+ /** sha256 of the case's data; set on each case of an `evaluate.each`. */
37
+ case?: string;
38
+ };
39
+ /** Where a case of a pooled `evaluate.each` sits in its list. */
40
+ type DeclaredFamily = {
41
+ /** The name template passed to `evaluate.each`. */
42
+ name: string;
43
+ /** The case's index, from 0. */
44
+ case: number;
45
+ /** How many cases the list has. */
46
+ cases: number;
47
+ };
48
+ /** `evaluate.todo` or `group.todo`. */
49
+ type DeclaredTodo = {
50
+ type: 'todo';
51
+ /** The name passed to `.todo`. */
52
+ name: string;
53
+ };
54
+ /** One node of the tree: a group, an eval or a todo. */
55
+ type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
56
+ /** An eval file's top-level groups, evals and todos, in declaration order. */
57
+ type DeclarationTree = {
58
+ children: DeclaredNode[];
59
+ };
60
+ //#endregion
1
61
  //#region src/view/effort.d.ts
2
62
  /** Below this two-sided p, an effort difference is an effect. */
3
63
  declare const EFFORT_ALPHA = 0.05;
@@ -192,7 +252,7 @@ type JudgeRetry = {
192
252
  error: string;
193
253
  delayMs: number;
194
254
  };
195
- type JudgementRecord = {
255
+ type JudgmentRecord = {
196
256
  question: string;
197
257
  verdict: 'pass' | 'fail';
198
258
  score: number;
@@ -202,9 +262,15 @@ type JudgementRecord = {
202
262
  resolvedModel?: string;
203
263
  /** The gateway from the config's `gateway` list the call went through; unset without one. */
204
264
  gateway?: string;
265
+ /** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
266
+ upstream?: string;
267
+ /** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
268
+ costUsd?: number;
205
269
  /**
206
270
  * `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
207
- * verdict can be followed across runs (and an edited question starts a new series).
271
+ * verdict can be followed across runs (and an edited question starts a new series). `<index>`
272
+ * is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
273
+ * whichever finishes first.
208
274
  */
209
275
  key: string;
210
276
  /** Transient provider errors retried before this verdict, with the backoff before each retry. */
@@ -216,7 +282,7 @@ type AttemptFiles = {
216
282
  /** Each environment setup step's command, exit code, duration and output. */
217
283
  setupLog?: FileRef;
218
284
  transcript?: FileRef;
219
- /** Unnormalised harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
285
+ /** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
220
286
  rawTranscript?: FileRef;
221
287
  diff?: FileRef;
222
288
  judgeInput?: FileRef;
@@ -227,6 +293,11 @@ type ContextFile = {
227
293
  path: string;
228
294
  sha256: string;
229
295
  };
296
+ /** An upstream provider behind a gateway, and how many of the attempt's calls it served. */
297
+ type UpstreamCount = {
298
+ provider: string;
299
+ calls: number;
300
+ };
230
301
  type AttemptRecord = {
231
302
  /** `<eval key> # <cell key>#<n>`: unique in the run. */
232
303
  key: string;
@@ -265,6 +336,17 @@ type AttemptRecord = {
265
336
  * gateway's, so never part of `servedModels`. Absent when there's none.
266
337
  */
267
338
  askedModels?: string[];
339
+ /**
340
+ * The upstream providers a gateway said served the harness's own API calls (OpenRouter's
341
+ * `provider`), each with how many it served, in the order first seen; more than one means the
342
+ * gateway spread or failed the calls over between them. Absent when the gateway names none.
343
+ */
344
+ upstreams?: UpstreamCount[];
345
+ /**
346
+ * What a gateway said the harness's own API calls cost, in US dollars, summed (OpenRouter's
347
+ * `usage.cost`); absent when it names none.
348
+ */
349
+ reportedCostUsd?: number;
268
350
  /**
269
351
  * The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
270
352
  * null when the factory left it unset. Fast is not this.
@@ -325,9 +407,18 @@ type AttemptRecord = {
325
407
  /** Longest tool `durationMs` in the transcript; null when none reported a duration. */
326
408
  maxToolMs: number | null;
327
409
  };
328
- judgements: JudgementRecord[];
410
+ judgements: JudgmentRecord[];
329
411
  /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
330
412
  startRetries?: StartRetry[];
413
+ /**
414
+ * Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
415
+ * without `hosts` in their environment, so they hold its real value while the agent works and
416
+ * the agent, as the same user, can read it from them. Found by the secret's variable name, so a
417
+ * copy under another name or in a command's arguments isn't. At most 20, each
418
+ * `{ command, secrets }`: its command line, masked and cut at 200 characters, and those secrets'
419
+ * names. Absent when there were none.
420
+ */
421
+ setupProcesses?: SetupProcess[];
331
422
  /**
332
423
  * Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
333
424
  * `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
@@ -369,11 +460,21 @@ type EvalEntry = {
369
460
  name: string;
370
461
  /** Results are comparable only when fingerprints match. */
371
462
  fingerprint: string;
372
- /** The eval file and every local file it imports, directly or not, by content hash. */
463
+ /**
464
+ * The eval file, by content hash: where the eval lives. Records from before `source` also list
465
+ * the local files it imported, and were fingerprinted over them.
466
+ */
373
467
  sources: Array<{
374
468
  path: string;
375
469
  sha256: string;
376
470
  }>;
471
+ /**
472
+ * Hashes of what the `sources` fingerprint part covers but `revision`: the eval's body, its
473
+ * case data and its options with its groups'. Unset on records from before it.
474
+ */
475
+ source?: EvalSource;
476
+ /** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
477
+ revision?: Revision;
377
478
  tags: string[];
378
479
  /** The baseline environment, when the eval uses `compare`. */
379
480
  compare?: EvalCompareMeta;
@@ -535,7 +636,7 @@ type Lift = {
535
636
  };
536
637
  /** Each input of a cell fingerprint, hashed on its own, so a change can be named. */
537
638
  type FingerprintParts = {
538
- /** Over `EvalEntry.sources`. */
639
+ /** Over `EvalEntry.source` and `EvalEntry.revision`; on records from before them, over `EvalEntry.sources`. */
539
640
  sources: string;
540
641
  /** Over the harness descriptor, less concurrency caps. */
541
642
  harness: string;
@@ -548,13 +649,21 @@ type FingerprintParts = {
548
649
  context: string;
549
650
  /** The gateway its model calls went through; unset on `direct`, each provider's own key. */
550
651
  gateway?: string;
652
+ /**
653
+ * The cell's gate (`none`, `passRate 0.5`); unset on the default `all`. Compare reads a change
654
+ * in it as a policy change, not a reason the runs can't be compared.
655
+ */
656
+ gate?: string;
551
657
  };
552
658
  /** One harness and environment of an eval, and how its attempts went. */
553
659
  type CellRecord = Cell & {
554
660
  /** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
555
661
  fingerprint: string;
556
662
  fingerprintParts: FingerprintParts;
557
- /** The environment's digest: the join key into `RunRecord.environments`; null for none. */
663
+ /**
664
+ * The environment's digest; null for none. With `environment`, the key into
665
+ * `RunRecord.environments`: environments with the same contents share a digest.
666
+ */
558
667
  environmentDigest: string | null;
559
668
  harnessKind: string;
560
669
  /**
@@ -698,6 +807,7 @@ type RunRecord = {
698
807
  config: {
699
808
  sandbox: string;
700
809
  environment: string | null;
810
+ /** The config's `runs`; 1 when it sets none, though a compare's cells then run 4 each. */
701
811
  runs: number;
702
812
  judgeModel: string | null;
703
813
  threshold: number;
@@ -824,8 +934,15 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
824
934
  };
825
935
  /** Median tool calls, agent time and tokens over the counted attempts; null when none counted. */
826
936
  effort: CellEffort | null;
827
- /** Of `fail`, the attempts the run's interrupt cancelled (`error.kind: 'aborted'`); unset when none. */
828
- cancelled?: number;
937
+ /** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
938
+ canceled?: number;
939
+ };
940
+ /** A family's pooled lift as the history listing keeps it: enough to compare its gate. */
941
+ type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
942
+ lo: number;
943
+ hi: number;
944
+ /** Unset when the compare has no `minLift`, so the lift can't fail. */
945
+ minLift?: number;
829
946
  };
830
947
  /**
831
948
  * One line of the derived history listing: cached locally by the CLI, and the run header in the
@@ -860,636 +977,24 @@ type HistoryEntry = {
860
977
  slug: string;
861
978
  fingerprint: string;
862
979
  sources: EvalEntry['sources'];
980
+ source?: EvalEntry['source'];
981
+ revision?: EvalEntry['revision'];
863
982
  outcome: Outcome;
864
983
  passRate: number | null;
865
984
  cells: HistoryCell[];
866
985
  }>;
867
986
  /** The run's pooled families; unset when none ran. */
868
- families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'>>;
987
+ families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'> & {
988
+ /** The eval ids of its cases in this run. */
989
+ caseIds: string[];
990
+ lifts: FamilyLiftSummary[];
991
+ }>;
869
992
  /** The run's errors outside any attempt; unset when there were none. */
870
993
  errors?: string[];
871
994
  };
872
995
  //#endregion
873
- //#region src/view/suite.d.ts
874
- /**
875
- * A variant against its baseline: `better` or `worse` when the stored 95% interval on the
876
- * difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
877
- * including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
878
- */
879
- type Effect = 'better' | 'worse' | 'same' | 'unclear';
880
- /**
881
- * What an eval's cells compare: `change` when it has variants (or several environments),
882
- * `harness` when it has several harnesses, `both`, or `none`.
883
- */
884
- type Across = 'none' | 'change' | 'harness' | 'both';
885
- /**
886
- * One finished run's facts. Every node has a `path`, a stable id to cite it by, and `paths`
887
- * lists them all in the order the nodes appear.
888
- *
889
- * A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
890
- *
891
- * - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
892
- * (`evals/a.eval.ts › fixes $id`);
893
- * - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
894
- * - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
895
- * - `<cell>/checks/<label>`: a check, by its assertion label;
896
- * - `<cell>/effort`: the cell's effort, only when it was tested;
897
- * - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
898
- *
899
- * A case segment never has a raw `+` and a cell segment always has two, so the two can't be
900
- * confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
901
- * fact in every run that has it.
902
- */
903
- type SuiteResult = {
904
- runId: string;
905
- status: RunRecord['status'];
906
- /**
907
- * The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
908
- * an attempt, and `skipped` on an interrupted run.
909
- */
910
- outcome: Outcome;
911
- /**
912
- * Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
913
- * awaited, a worker that died. Any one fails the run.
914
- */
915
- errors: string[];
916
- /**
917
- * Outcomes counted per block: each family case is one, each eval with no cases is one, and a
918
- * family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
919
- * `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
920
- */
921
- tally: SuiteTally;
922
- /** Sorted by path. */
923
- files: ResultFile[];
924
- /** Every node's `path`, in the order they appear. */
925
- paths: string[];
926
- };
927
- type SuiteTally = {
928
- errors: number;
929
- pass: number;
930
- notGated: number;
931
- fail: number;
932
- infraError: number;
933
- cancelled: number;
934
- skipped: number;
935
- };
936
- /**
937
- * An eval's or a family case's outcome: the record's, except `cancelled` where its only failed
938
- * gated attempts are ones the run's interrupt cancelled, since the agent didn't fail those.
939
- */
940
- type ResultOutcome = Outcome | 'cancelled';
941
- type ResultFile = {
942
- path: string;
943
- evals: EvalResult[];
944
- };
945
- type EvalResult = {
946
- path: string;
947
- /** The eval's id, or the family's. */
948
- id: string;
949
- group: string[];
950
- /** The eval's name; on a family, its template. */
951
- name: string;
952
- /** A family's name template (`fixes $id`); null on a single eval. */
953
- template: string | null;
954
- /** Source order in the file. */
955
- order: number;
956
- across: Across;
957
- /**
958
- * A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
959
- * unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
960
- */
961
- family: boolean;
962
- /** A placeholder (`evaluate.todo`) that never ran. */
963
- todo: boolean;
964
- /** On a family: `fail` when a case failed or its pooled gate did. */
965
- outcome: ResultOutcome;
966
- cells: CellResult[] | null;
967
- cases: CaseResult[] | null;
968
- /** A family's variants against their baseline, pooled over the cases; null on a single eval. */
969
- pooled: PooledArm[] | null;
970
- /**
971
- * Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
972
- * each question asked of any case, in first-asked order, summed over the cases that answered it.
973
- */
974
- questions: Array<QuestionResult | AttemptQuestionResult>;
975
- };
976
- /** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
977
- type QuestionResult = {
978
- kind: 'pairs';
979
- question: string;
980
- /** Pairs voted, each counted once per order it was asked in. */
981
- votes: number;
982
- rates: Array<{
983
- cut: 'within' | 'across';
984
- name: string;
985
- wins: number;
986
- n: number;
987
- }>;
988
- /** Why some or all of its votes are missing: the model call failed. */
989
- error: string | null;
990
- };
991
- /** A question asked of each attempt on its own: what each one answered. Never gates. */
992
- type AttemptQuestionResult = {
993
- kind: 'attempts';
994
- question: string;
995
- /** Attempts it was asked of, failed calls included. */
996
- asked: number;
997
- /** How many attempts picked each choice, in the question's order; null on a free-text question. */
998
- choices: Array<{
999
- choice: string;
1000
- n: number;
1001
- }> | null;
1002
- /** Each attempt's answer, in attempt order; failed calls left out. */
1003
- answers: Array<{
1004
- attempt: string;
1005
- choice: string | null;
1006
- text: string;
1007
- }>;
1008
- /** Calls that failed, and the first one's error. */
1009
- failed: number;
1010
- error: string | null;
1011
- };
1012
- type CaseResult = {
1013
- path: string;
1014
- /** The case's own eval id. */
1015
- id: string;
1016
- /** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
1017
- label: string;
1018
- name: string;
1019
- /** Its place in the case list. */
1020
- order: number;
1021
- outcome: ResultOutcome;
1022
- cells: CellResult[];
1023
- };
1024
- type CellResult = {
1025
- path: string;
1026
- key: string;
1027
- harness: string;
1028
- /** `-` when no environment is set. */
1029
- environment: string;
1030
- sandbox: string;
1031
- /** The variant's name; null on a control. */
1032
- variant: string | null;
1033
- /** The control of an eval that has variants: the arm the variants are measured against. */
1034
- baseline: boolean;
1035
- /** On a variant, its baseline cell's key; else null. */
1036
- vs: string | null;
1037
- /** The cell's place in the eval's cells. */
1038
- order: number;
1039
- /** Its failures fail the eval (its gate isn't `'none'`). */
1040
- gated: boolean;
1041
- outcome: Outcome;
1042
- /** Every attempt: `passed + failed + infraError + skipped`. */
1043
- attempts: number;
1044
- passed: number;
1045
- failed: number;
1046
- /** Infrastructure errors, which are neither passes nor fails. */
1047
- infraError: number;
1048
- skipped: number;
1049
- fingerprint: string;
1050
- /** The skill sources and files the cell's environment placed, as written. */
1051
- context: string[];
1052
- /**
1053
- * Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
1054
- * attempt (every one an infra error or skipped) lists each as `not run`.
1055
- */
1056
- checks: CheckResult[];
1057
- /** On a variant, against its baseline; null on a control. */
1058
- effect: Effect | null;
1059
- /** The stored 95% interval on the variant's difference from its baseline. Not for display. */
1060
- interval: Interval | null;
1061
- /** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
1062
- minLift: number | null;
1063
- /** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
1064
- effort: EffortCounters | null;
1065
- /** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
1066
- effortDiffers: boolean;
1067
- /** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
1068
- errors: CellError[];
1069
- /** Web searches and fetches per counted attempt; null where the harness can't report them. */
1070
- searches: Array<number | null>;
1071
- fetches: Array<number | null>;
1072
- };
1073
- /** A cell's attempts the run's interrupt cancelled: recorded as fails, but not the agent's. */
1074
- declare const cancelledOf: (cell: CellResult) => number;
1075
- /** A cell's counted tries: its passes and fails, less the ones the interrupt cancelled. */
1076
- declare const countedOf: (cell: CellResult) => number;
1077
- type CheckResult = {
1078
- path: string;
1079
- label: string;
1080
- /** First-seen order across the eval's attempts. */
1081
- order: number;
1082
- outcome: CheckOutcome;
1083
- /**
1084
- * On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
1085
- * null on a control or without one.
1086
- */
1087
- effect: Effect | null;
1088
- /** Set when the check is a judge verdict. */
1089
- judge: JudgeResult | null;
1090
- /**
1091
- * A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
1092
- * it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
1093
- * when none did.
1094
- */
1095
- retried?: string;
1096
- };
1097
- /**
1098
- * `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
1099
- * none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
1100
- * reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
1101
- * first failure's message.
1102
- */
1103
- type CheckOutcome = {
1104
- status: 'not run';
1105
- } | {
1106
- status: 'ran';
1107
- passed: number;
1108
- failed: number;
1109
- errored: number;
1110
- notRun: number;
1111
- assertion: string | null;
1112
- };
1113
- /**
1114
- * One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
1115
- * an error), its message's first line, and how often.
1116
- */
1117
- type CellError = {
1118
- kind: ErrorKind | 'turn';
1119
- message: string;
1120
- attempts: number;
1121
- };
1122
- type JudgeResult = {
1123
- question: string;
1124
- threshold: number;
1125
- /** One per counted attempt that judged it. */
1126
- scores: number[];
1127
- };
1128
- type Interval = {
1129
- low: number;
1130
- high: number;
1131
- };
1132
- /** A null counter is one the harness didn't record; a recorded zero stays zero. */
1133
- type EffortCounters = {
1134
- tools: number | null;
1135
- agentMs: number | null;
1136
- tokens: number | null;
1137
- };
1138
- type PooledArm = {
1139
- path: string;
1140
- /** The variant cell's key, the same in every case. */
1141
- key: string;
1142
- vs: string;
1143
- effect: Effect;
1144
- /** Cases with counted attempts on both sides. */
1145
- pairs: number;
1146
- interval: Interval;
1147
- /** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
1148
- minLift: number | null;
1149
- outcome: 'pass' | 'fail';
1150
- };
1151
- type Side = {
1152
- pass: number;
1153
- counted: number;
1154
- };
1155
- /**
1156
- * The effect of `variant` against `control`, from the interval on their difference. A side
1157
- * with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
1158
- */
1159
- declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
1160
- /** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
1161
- declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
1162
- /** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
1163
- declare const factSegment: (s: string) => string;
1164
- declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
1165
- /**
1166
- * An error's one line: its first line with any letter or digit in it, joined with the next one
1167
- * when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
1168
- */
1169
- declare function messageLine(text: string): string;
1170
- /** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
1171
- declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
1172
- /**
1173
- * A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
1174
- * gating it, so it reads `not gated`, never `pass`.
1175
- */
1176
- declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
1177
- /** The facts of a run, from its record. Pure: the same record always gives the same result. */
1178
- declare function suiteResult(record: RunRecord): SuiteResult;
1179
- //#endregion
1180
- //#region src/view/insights.d.ts
1181
- /** `<history>/<run>/insights.json`, beside the record. */
1182
- declare const INSIGHTS_FILE = "insights.json";
1183
- /** `RunInsights.model` when no analysis model wrote the file: it holds deterministic remarks only. */
1184
- declare const NO_MODEL = "none";
1185
- /** Whether an analysis model wrote these insights, rather than reeval alone for its remarks. */
1186
- declare const fromModel: (insights: RunInsights) => boolean;
1187
- /** What one attempt tests. Only an analysis model can tell; without one every eval is `other`. */
1188
- type Job = 'choose' | 'finish' | 'other';
1189
- /**
1190
- * A run's insights, `<history>/<run>/insights.json`. Written create-only; the record is untouched.
1191
- * They're for the facts they were written from: a `factsHash` that isn't the run's marks them stale.
1192
- */
1193
- type RunInsights = {
1194
- kind: 'reeval-insights';
1195
- schemaVersion: 1;
1196
- runId: string;
1197
- /**
1198
- * The analysis model, as configured (`google/gemini-3.8-flash`, or `fake`); `NO_MODEL` when
1199
- * reeval wrote the file without one, to hold the run's deterministic remarks only.
1200
- */
1201
- model: string;
1202
- /** The prompts' version: insights from another version were asked differently. */
1203
- promptVersion: string;
1204
- /** sha256 of the run's `SuiteResult` as canonical JSON: the facts the model saw. */
1205
- factsHash: string;
1206
- createdAt: string;
1207
- /**
1208
- * Over every call; `outputTokens` includes `reasoningTokens`, the model's thinking, when it
1209
- * reports them. `costUsd` is null when reeval doesn't know the model's price.
1210
- */
1211
- usage: {
1212
- inputTokens: number;
1213
- outputTokens: number;
1214
- reasoningTokens?: number;
1215
- costUsd: number | null;
1216
- };
1217
- /**
1218
- * `analysis.maxTokens`, and the evals (`SuiteResult` paths) it ran out before: they print
1219
- * without insights. Absent when no budget applied.
1220
- */
1221
- budget?: {
1222
- maxTokens: number;
1223
- skipped: string[];
1224
- };
1225
- /**
1226
- * Evals whose calls failed twice (an error, a reply cut off at the output cap, or a call past its
1227
- * timeout), with the last error: they print without insights. Absent when none failed.
1228
- */
1229
- failed?: Array<{
1230
- evalId: string;
1231
- error: string;
1232
- }>;
1233
- /**
1234
- * Set when the run was stopped (Ctrl-C, or SIGTERM) before every eval's insights had finished:
1235
- * the evals that hadn't, which print without insights. Absent otherwise, even when the stop came
1236
- * during the headline call, which leaves `headline` null.
1237
- */
1238
- stopped?: {
1239
- signal: 'SIGINT' | 'SIGTERM';
1240
- skipped: string[];
1241
- };
1242
- /** One sentence for the whole run, citing facts; null when the model's was dropped. */
1243
- headline: Sentence | null;
1244
- /** One per eval (a family is one), in the order the output prints them. */
1245
- evals: EvalInsight[];
1246
- /**
1247
- * Remarks about the evals' own quality, ranked: every one this run raised, including those an
1248
- * earlier run printed first (`firstSeenRun`), which don't print again.
1249
- */
1250
- remarks: Remark[];
1251
- /**
1252
- * What the checks dropped, and how many sentences were asked for again, so the output can say
1253
- * so. `drops` is each dropped sentence: its slot (`<eval> › answer`, `› row <path>`, `headline`),
1254
- * its text and why.
1255
- */
1256
- validation: {
1257
- dropped: number;
1258
- regenerated: number;
1259
- drops?: Array<{
1260
- slot: string;
1261
- text: string;
1262
- reason: string;
1263
- }>;
1264
- };
1265
- /** Set when insights failed; the facts still print. */
1266
- error?: string;
1267
- };
1268
- type EvalInsight = {
1269
- /** The eval's `SuiteResult` path; for a family, the family's. */
1270
- evalId: string;
1271
- intent: {
1272
- /** What one attempt is testing. Can't be read from the cell shape. */
1273
- job: Job;
1274
- /** What the cells compare. Deterministic, copied from the facts. */
1275
- across: Across;
1276
- /** The eval's question in plain words: "Do agents pick Genkit for a production AI feature?" */
1277
- question: string;
1278
- /** The tool a choose eval asks about, and the change a measure eval makes. */
1279
- subject: {
1280
- tool?: string;
1281
- change?: string;
1282
- };
1283
- /** How the eval name relates to what the eval checks. */
1284
- name: 'states-intent' | 'no-intent' | 'contradicts';
1285
- /**
1286
- * sha256 of the eval's id, sources, first attempt's prompts, judge questions, check labels,
1287
- * the model and the prompt version: the classification's cache key.
1288
- */
1289
- key: string;
1290
- };
1291
- /** The answer to `question`, at most two sentences; null when it was dropped. */
1292
- answer: Sentence | null;
1293
- /** Up to three supporting points: what moved, what broke, what the agent did. */
1294
- findings: Sentence[];
1295
- /**
1296
- * At most one sentence per row (a case, an arm or a harness), keyed by the row's fact path.
1297
- * Only rows the eval's layout opens a slot on are kept.
1298
- */
1299
- rows?: Array<Sentence & {
1300
- row: string;
1301
- }>;
1302
- /**
1303
- * Per attempt (`<cell path>#<n>`), the tools the reply recommended or used, each found
1304
- * verbatim in that attempt's final message. Only on `choose` evals.
1305
- */
1306
- namedTools?: Array<{
1307
- attempt: string;
1308
- tools: string[];
1309
- }>;
1310
- };
1311
- /**
1312
- * A plain sentence citing the facts it states: `SuiteResult` paths
1313
- * (`evals%2Fbilling.eval.ts › bills in pence/fake+-+fake/checks/amount`) or a transcript line
1314
- * (`transcript:<cell path>#<n>#L<line>`). A sentence citing nothing is dropped.
1315
- */
1316
- type Sentence = {
1317
- text: string;
1318
- cites: string[];
1319
- };
1320
- /**
1321
- * Each remark kind's origin and severity: an analysis model raises the `model` kinds, through the
1322
- * gates (a known kind, a quote verbatim on the cited line); reeval's rules raise the others. The
1323
- * severity is the kind's, never the model's.
1324
- */
1325
- declare const REMARK_KINDS: Record<Remark['kind'], {
1326
- origin: Remark['origin'];
1327
- severity: 'concern' | 'suggestion';
1328
- }>;
1329
- /** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
1330
- declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "proxy-only-egress" | "slow-mcp-start" | "too-few-tries")[];
1331
- /** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
1332
- type Remark = {
1333
- /**
1334
- * sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
1335
- * per kind (and server) per eval version.
1336
- */
1337
- id: string;
1338
- evalId: string;
1339
- /** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
1340
- kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start' | 'proxy-only-egress';
1341
- /** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
1342
- severity: 'concern' | 'suggestion';
1343
- /** One plain sentence: what's wrong. */
1344
- text: string;
1345
- /**
1346
- * Where the problem is: `source:<file>#L<n>` with `quote`, found verbatim on that line of the
1347
- * file at the run's source hash. Every model remark has one.
1348
- */
1349
- source?: {
1350
- at: string;
1351
- quote: string;
1352
- };
1353
- /** Facts from this run (or history facts) showing it mattered now; a model's, as it cited them. */
1354
- evidence: string[];
1355
- /**
1356
- * An edit at a source location (`source:<file>#L<n>`) in the eval. A model remark's `edit` is
1357
- * empty when the model gave none, and its `at` is the quoted line when it named no line.
1358
- */
1359
- fix: {
1360
- at: string;
1361
- edit: string;
1362
- };
1363
- origin: 'model' | 'deterministic';
1364
- /**
1365
- * The first run that printed this `id`, or this run when none has: it prints only in that run,
1366
- * except `slow-mcp-start`, which is about the environment, not the eval's source, and prints in
1367
- * every run it happens.
1368
- */
1369
- firstSeenRun: string;
1370
- /**
1371
- * Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
1372
- * when the cap or a run without `-v` left it out, so a later run prints it. Missing: printed in
1373
- * `firstSeenRun` only.
1374
- */
1375
- printed?: boolean;
1376
- /** Set on later runs once the cited span changed: the remark was acted on. */
1377
- actedOn?: {
1378
- run: string;
1379
- };
1380
- };
1381
- //#endregion
1382
- //#region src/insights/input.d.ts
1383
- /** One fact the model may cite, by its path. */
1384
- type InsightFact = {
1385
- path: string;
1386
- kind: string;
1387
- } & Record<string, unknown>;
1388
- type InsightAttempt = {
1389
- /** `<cell path>#<n>`. */
1390
- id: string;
1391
- cell: string;
1392
- outcome: string;
1393
- /** The agent's final message, clipped. */
1394
- lastText: string;
1395
- /** A failed attempt's transcript lines, `L<n> …`, each citable as `transcript:<id>#L<n>`. */
1396
- lines: string[];
1397
- };
1398
- /** Where an insight sentence can go, as the eval's layout opens them. */
1399
- type InsightSlots = {
1400
- /** Row paths that take one sentence each. */
1401
- rows: string[];
1402
- findings: boolean;
1403
- namedTools: boolean;
1404
- };
1405
- /** One eval as the classifier and the insight model see it. */
1406
- type InsightInput = {
1407
- call: 'classify' | 'insights';
1408
- eval: {
1409
- path: string;
1410
- /** The eval's name, or a family's template: the author's statement of what it asks. */
1411
- name: string;
1412
- group: string[];
1413
- file: string;
1414
- across: Across;
1415
- family: boolean;
1416
- verdict: string;
1417
- };
1418
- /** The first attempt's prompts, in order. */
1419
- prompts: string[];
1420
- judgeQuestions: string[];
1421
- checks: string[];
1422
- /** Set on the insights call: the classification. */
1423
- intent?: {
1424
- job: Job;
1425
- question: string;
1426
- subject: {
1427
- tool?: string;
1428
- change?: string;
1429
- };
1430
- };
1431
- /** Set on the insights call. */
1432
- slots?: InsightSlots;
1433
- facts: InsightFact[];
1434
- attempts: InsightAttempt[];
1435
- /**
1436
- * Set on the insights call: the eval's declaration in its file, each line `L<n> …` as the file
1437
- * numbers it, which a remark's `source` and `fix` cite.
1438
- */
1439
- source?: {
1440
- file: string;
1441
- lines: string[];
1442
- };
1443
- /** What was left out to fit the eval's size cap (`trim.ts`), when anything was. */
1444
- trimmed?: string[];
1445
- /** On a regeneration: why the earlier sentences were refused. */
1446
- retry?: string[];
1447
- };
1448
- /** The headline call's input: each eval's verdict and answer, and the run's tally with its total. */
1449
- type HeadlineInput = {
1450
- call: 'headline';
1451
- /**
1452
- * Counts results, not evals: each case of a family counts one, as the tally line does, and
1453
- * `total` is their sum, so no count is the model's arithmetic; `errors` (outside any eval) aren't
1454
- * results and stay out of it. The eval count is `evals.length`.
1455
- */
1456
- tally: SuiteResult['tally'] & {
1457
- total: number;
1458
- unit: 'results';
1459
- };
1460
- evals: Array<{
1461
- path: string;
1462
- name: string;
1463
- verdict: string;
1464
- answer: string | null;
1465
- }>;
1466
- retry?: string[];
1467
- };
1468
- /** A fact as the model sees it: its short id (`F1`), which is what it cites. */
1469
- type ModelFact = InsightFact & {
1470
- id: string;
1471
- };
1472
- /** An attempt as the model sees it: `A1`, its cell's fact id, and its lines citable as `A1#L<n>`. */
1473
- type ModelAttempt = Omit<InsightAttempt, 'id' | 'cell'> & {
1474
- id: string;
1475
- cell: string;
1476
- };
1477
- /** One eval's call as the model sees it: facts, attempts and row slots by short id. */
1478
- type ModelInsightInput = Omit<InsightInput, 'facts' | 'attempts' | 'slots'> & {
1479
- facts: ModelFact[];
1480
- attempts: ModelAttempt[];
1481
- slots?: Omit<InsightSlots, 'rows'> & {
1482
- rows: string[];
1483
- };
1484
- };
1485
- /** The headline call as the model sees it: each eval by its short id. */
1486
- type ModelHeadlineInput = Omit<HeadlineInput, 'evals'> & {
1487
- evals: Array<HeadlineInput['evals'][number] & {
1488
- id: string;
1489
- }>;
1490
- };
1491
- //#endregion
1492
996
  //#region src/types.d.ts
997
+ /** More than zero and at most 596h: `90s`, `15m`, `1h`, or a number of milliseconds. */
1493
998
  type Duration = number | `${number}${'ms' | 's' | 'm' | 'h'}`;
1494
999
  /** When an event started: wall clock, and ms since its turn started. */
1495
1000
  type EventTime = {
@@ -1500,13 +1005,17 @@ type FileEditEvent = {
1500
1005
  type: 'file-edit';
1501
1006
  toolCallId: string;
1502
1007
  path: string;
1503
- kind: 'create' | 'modify' | 'delete';
1008
+ /**
1009
+ * `write` is a whole-file write whose result doesn't say whether the file existed before
1010
+ * (Antigravity's, fx's, Goose's, Grok Build's and pi's write tools, for example).
1011
+ */
1012
+ kind: 'create' | 'modify' | 'write' | 'delete';
1504
1013
  /** Line counts, or null when the harness didn't send the content (e.g. deletes). */
1505
1014
  added: number | null;
1506
1015
  removed: number | null;
1507
1016
  };
1508
1017
  /**
1509
- * Normalised agent events. Each tool call is one event, merged with its result; `durationMs`
1018
+ * Normalized agent events. Each tool call is one event, merged with its result; `durationMs`
1510
1019
  * is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
1511
1020
  * when the harness never reported the end. Long shell or edit calls set `stalled: true`.
1512
1021
  */
@@ -1683,6 +1192,7 @@ type Capture = {
1683
1192
  * default `name` is derived from the path. `wait` and `instructions` are call-only.
1684
1193
  */
1685
1194
  type CaptureCall = {
1195
+ /** The `artifacts/<name>.png` file name: 1 to 100 of `a-z`, `0-9`, `_` and `-`, starting with a letter or digit. Unique within the attempt. */
1686
1196
  name?: string;
1687
1197
  path?: string;
1688
1198
  viewport?: Viewport;
@@ -1884,6 +1394,10 @@ type Verdict = {
1884
1394
  resolvedModel?: string;
1885
1395
  /** The gateway from the config's `gateway` list the call went through; unset without one. */
1886
1396
  gateway?: GatewayName;
1397
+ /** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
1398
+ upstream?: string;
1399
+ /** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
1400
+ costUsd?: number;
1887
1401
  /** Captures this call sent the judge as file parts, when any. */
1888
1402
  artifacts?: JudgeArtifactRef[];
1889
1403
  };
@@ -1891,7 +1405,7 @@ type Verdict = {
1891
1405
  * What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished,
1892
1406
  * so the judges and checks after it still run.
1893
1407
  */
1894
- type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
1408
+ type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel' | 'upstream' | 'costUsd'> & {
1895
1409
  verdict: 'error';
1896
1410
  score: null;
1897
1411
  error: string;
@@ -1966,7 +1480,7 @@ type FakeHarnessDescriptor = {
1966
1480
  files?: Record<string, string>;
1967
1481
  /** The fake agent's final message on every turn; default `Done.`. */
1968
1482
  text?: string;
1969
- /** Simulated turn length, honours the abort signal. */
1483
+ /** Simulated turn length, honors the abort signal. */
1970
1484
  delayMs?: number;
1971
1485
  /** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
1972
1486
  ignoreAbort?: boolean;
@@ -2243,7 +1757,7 @@ type CodexHarnessDescriptor = {
2243
1757
  };
2244
1758
  /**
2245
1759
  * fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
2246
- * OpenAI's API by the model id, and reports each turn's tokens.
1760
+ * OpenAI's API by the model id, or on OpenRouter or Vertex AI, and reports each turn's tokens.
2247
1761
  */
2248
1762
  type FxHarnessDescriptor = {
2249
1763
  kind: 'fx';
@@ -2255,10 +1769,18 @@ type FxHarnessDescriptor = {
2255
1769
  /**
2256
1770
  * A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
2257
1771
  * `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
2258
- * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`.
1772
+ * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`. On `openrouter()`, either
1773
+ * runs on OpenRouter's Chat Completions API with `OPENROUTER_API_KEY`, a bare id as `openai/<id>`.
1774
+ * On `vertex()`, a Gemini model id, `google/<id>` or a bare `gemini-…` id, run on Vertex AI's
1775
+ * Chat Completions endpoint, e.g. `google/gemini-3.8-flash`.
2259
1776
  */
2260
1777
  model: string;
2261
- /** Cell id; defaults to `fx:<model>`. */
1778
+ /**
1779
+ * Reasoning effort, sent as `reasoning_effort` on each model call; only on `vertex()`, since fx
1780
+ * sends none itself. Values are what the model takes there, e.g. `low`, `medium` or `high`.
1781
+ */
1782
+ effort?: string;
1783
+ /** Cell id; defaults to `fx:<model>`, or `fx:<model>[effort=<value>]` when `effort` is set. */
2262
1784
  id?: string;
2263
1785
  /** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
2264
1786
  maxConcurrent?: number;
@@ -2298,12 +1820,21 @@ type GrokBuildHarnessDescriptor = {
2298
1820
  };
2299
1821
  /**
2300
1822
  * A gateway the config lists by name: `direct` is each provider's own key, as without `gateway`;
2301
- * `vercel` is Vercel AI Gateway.
1823
+ * `vercel` is Vercel AI Gateway; `openrouter` is OpenRouter; `vertex` is
1824
+ * Google Cloud's Vertex AI.
1825
+ */
1826
+ type GatewayName = 'direct' | 'vercel' | 'openrouter' | 'vertex';
1827
+ /**
1828
+ * Each provider's own key: made by `direct()` in `reeval/gateway`. Every `gateway` list ends with
1829
+ * it unless it's listed elsewhere, where it's tried in that place.
2302
1830
  */
2303
- type GatewayName = 'direct' | 'vercel';
2304
- /** Each provider's own key, as without `gateway`: made by `direct()` in `reeval/gateway`. */
2305
1831
  type DirectGatewayDescriptor = {
2306
1832
  kind: 'direct';
1833
+ /**
1834
+ * `false` turns off the fallback to each provider's own key, so a cell no other gateway in the
1835
+ * list serves is refused at plan time. Default `true`.
1836
+ */
1837
+ enabled?: boolean;
2307
1838
  };
2308
1839
  /**
2309
1840
  * Vercel AI Gateway, with `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`): made by `vercel()` in
@@ -2312,8 +1843,37 @@ type DirectGatewayDescriptor = {
2312
1843
  type VercelGatewayDescriptor = {
2313
1844
  kind: 'vercel';
2314
1845
  };
1846
+ /**
1847
+ * OpenRouter, with `OPENROUTER_API_KEY`: made by `openrouter()` in `reeval/gateway`. It serves
1848
+ * Claude Code, Codex, Goose, OpenCode, fx, Grok Build and pi.
1849
+ */
1850
+ type OpenRouterGatewayDescriptor = {
1851
+ kind: 'openrouter';
1852
+ /**
1853
+ * Let OpenRouter fail over to another upstream provider when the one it picked can't serve a
1854
+ * call. Off by default, so reeval's own calls (the judge, analysis, compare questions and
1855
+ * captures) send `allow_fallbacks: false`. The harnesses' calls take OpenRouter's own routing
1856
+ * either way.
1857
+ */
1858
+ fallbacks?: boolean;
1859
+ };
1860
+ /** A model publisher on Vertex AI, which may serve its models in a region of its own. */
1861
+ type VertexPublisher = 'anthropic' | 'google' | 'xai';
1862
+ /**
1863
+ * Google Cloud's Vertex AI, on Application Default Credentials: made by `vertex()` in
1864
+ * `reeval/gateway`.
1865
+ */
1866
+ type VertexGatewayDescriptor = {
1867
+ kind: 'vertex';
1868
+ /** The Google Cloud project the calls are billed to; defaults to `GOOGLE_CLOUD_PROJECT`. */
1869
+ project?: string;
1870
+ /** The region the models are called in, e.g. `us-east5`; defaults to `global`. */
1871
+ location?: string;
1872
+ /** A region per publisher, where it differs from `location`, e.g. `{ anthropic: 'us-east5' }`. */
1873
+ regions?: Partial<Record<VertexPublisher, string>>;
1874
+ };
2315
1875
  /** One entry of the config's `gateway` list. */
2316
- type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor;
1876
+ type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor | OpenRouterGatewayDescriptor | VertexGatewayDescriptor;
2317
1877
  /** A real coding agent, driven through its AI SDK harness adapter. */
2318
1878
  type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
2319
1879
  /** The agent an attempt runs: a real harness, or the fake one. */
@@ -2892,7 +2452,7 @@ type QuestionOptions = {
2892
2452
  type PairFactor = 'environment' | 'harness';
2893
2453
  /** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
2894
2454
  type CompareCut = 'within' | 'across';
2895
- /** One judgement of one pair of attempts, in one order. */
2455
+ /** One judgment of one pair of attempts, in one order. */
2896
2456
  type PairVote = {
2897
2457
  question: string;
2898
2458
  cut: CompareCut;
@@ -2986,7 +2546,11 @@ type EvalOptionFields = {
2986
2546
  * Not an environment: it does not invent skills, files or a name.
2987
2547
  */
2988
2548
  sandbox?: SandboxDescriptor;
2989
- /** Attempts per cell. */
2549
+ /**
2550
+ * Attempts per cell. Unset at every level, a compare with variants runs 4 a side, the fewest
2551
+ * whose lift can reach a verdict, and anything else runs once, a pooled `evaluate.each` case
2552
+ * included.
2553
+ */
2990
2554
  runs?: number;
2991
2555
  /** How each cell's attempts decide its outcome; default `'all'`. */
2992
2556
  gate?: GatePolicy;
@@ -3014,9 +2578,25 @@ type EvalOptionFields = {
3014
2578
  timeout?: Duration;
3015
2579
  /** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
3016
2580
  tags?: string[];
2581
+ /**
2582
+ * The eval's version, part of its fingerprint: bump it when a change the eval file doesn't show
2583
+ * (a helper, a fixture, data it reads) changes what it measures. A group's applies to the evals
2584
+ * inside it that set none.
2585
+ */
2586
+ revision?: Revision;
3017
2587
  /** Keys a profile's `limit` may not narrow for this group or eval. */
3018
2588
  pin?: PinnableKey[];
3019
2589
  };
2590
+ /** An eval's `revision`: a whole number or a non-empty string, compared exactly. */
2591
+ type Revision = number | string;
2592
+ /**
2593
+ * An eval's own source as hashes: its body and case data as declared, and its options and its
2594
+ * enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
2595
+ */
2596
+ type EvalSource = DeclaredSource & {
2597
+ /** sha256 of the canonical options. */
2598
+ options: string;
2599
+ };
3020
2600
  /**
3021
2601
  * `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
3022
2602
  * A compare with a `baseline` names every arm, so the cascaded `environment` is not applied, and
@@ -3065,7 +2645,7 @@ type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
3065
2645
  type HistoryConfig = {
3066
2646
  /** Committed history folder (default: `.history` in the directory the evals live in). */
3067
2647
  dir?: string;
3068
- /** Artifacts recorded next to each run: every eval's, failing evals' only, or none. */
2648
+ /** Artifacts recorded next to each run: every attempt's, failed attempts' only, or none. */
3069
2649
  artifacts?: ArtifactLevel;
3070
2650
  };
3071
2651
  /** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
@@ -3148,74 +2728,6 @@ type FakeAnalysisModel = {
3148
2728
  answer?: FakeAnswer | ((attempt: BlindAttempt) => FakeAnswer) | {
3149
2729
  error: string;
3150
2730
  };
3151
- /**
3152
- * What insights get back, in `reeval run` and `reeval analyze --insights`: fixed insights, a
3153
- * function of each call's input (`input.call` is `classify` or `insights`, once each per eval,
3154
- * or `headline` once; it may return a promise), or `{ error }` to fail every call. It counts a
3155
- * token per 4 characters of each call's input and answer, so `analysis.maxTokens` applies.
3156
- */
3157
- insights?: FakeInsights | ((input: FakeInsightsInput) => FakeInsights | Promise<FakeInsights>) | {
3158
- error: string;
3159
- };
3160
- };
3161
- /** What a fake insights function is given: one eval's scoped facts, or the run's evals for the headline. */
3162
- type FakeInsightsInput = ModelInsightInput | ModelHeadlineInput;
3163
- /**
3164
- * A fake insight answer. Each call takes its part: `intent` the classification, `answer`,
3165
- * `findings`, `rows`, `namedTools` and `remarks` the eval's, `headline` the run's. Whatever's
3166
- * left out is empty, and the intent defaults to `job: 'other'`.
3167
- */
3168
- type FakeInsights = {
3169
- intent?: {
3170
- job: 'choose' | 'finish' | 'other';
3171
- question?: string;
3172
- subject?: {
3173
- tool?: string;
3174
- change?: string;
3175
- };
3176
- name?: 'states-intent' | 'no-intent' | 'contradicts';
3177
- /** `low` falls back to `job: 'other'`, as a real classification does. */
3178
- confidence?: 'high' | 'medium' | 'low';
3179
- };
3180
- answer?: {
3181
- text: string;
3182
- cites: string[];
3183
- } | null;
3184
- findings?: Array<{
3185
- text: string;
3186
- cites: string[];
3187
- }>;
3188
- rows?: Array<{
3189
- row: string;
3190
- text: string;
3191
- cites: string[];
3192
- }>;
3193
- namedTools?: Array<{
3194
- attempt: string;
3195
- tools: string[];
3196
- }>;
3197
- /**
3198
- * The eval's remarks, as the model's JSON would have them: `source.at` and `fix.at` are
3199
- * `L<n>` lines of the eval's source, `evidence` fact ids (`F1`) or attempt lines (`A1#L3`).
3200
- * They go through the same gates as a model's, so one missing a part is dropped.
3201
- */
3202
- remarks?: Array<{
3203
- kind: string;
3204
- text: string;
3205
- source?: {
3206
- at: string;
3207
- quote: string;
3208
- } | null;
3209
- evidence?: string[];
3210
- fix?: {
3211
- at: string;
3212
- edit: string;
3213
- } | null;
3214
- }>;
3215
- headline?: {
3216
- text: string;
3217
- cites: string[];
3218
- } | null;
3219
2731
  };
3220
2732
  /** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
3221
2733
  type FakeVote = 'A' | 'B' | 'tie';
@@ -3269,7 +2781,7 @@ type BlindSide = BlindTurn & {
3269
2781
  command: string;
3270
2782
  notRun: true;
3271
2783
  }>;
3272
- /** Its captures, in the order sent, labelled by position; a name it lacks is `missing`. */
2784
+ /** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
3273
2785
  artifacts?: Array<{
3274
2786
  label: string;
3275
2787
  mediaType: 'image/png';
@@ -3279,24 +2791,13 @@ type BlindSide = BlindTurn & {
3279
2791
  missing: true;
3280
2792
  }>;
3281
2793
  };
3282
- /** Insights and `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
2794
+ /** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
3283
2795
  type AnalysisConfig = {
3284
2796
  /**
3285
- * `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze`
3286
- * and questions fall back to the judge's when it's a language model.
2797
+ * `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Unset, `reeval analyze` and questions
2798
+ * fall back to the judge's when it's a language model.
3287
2799
  */
3288
2800
  model?: string | FakeAnalysisModel;
3289
- /**
3290
- * `reeval run` classifies each eval and explains its result with `model` once the attempts
3291
- * finish (default true when `model` is set; `--no-insights` turns it off for one run).
3292
- */
3293
- insights?: boolean;
3294
- /**
3295
- * Tokens (in and out) that one run's insights may use, over every classification, insight and
3296
- * headline call (default 250000). Past it, the evals left print without insights; the run's
3297
- * outcome and exit code never change.
3298
- */
3299
- maxTokens?: number;
3300
2801
  /** Added to the prompt, e.g. "Fewer tool calls count as a win. The skill under test is reeval-good." */
3301
2802
  instructions?: string;
3302
2803
  /** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
@@ -3316,7 +2817,7 @@ type ReevalConfig = {
3316
2817
  sandbox?: SandboxDescriptor;
3317
2818
  /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
3318
2819
  environment?: Environment;
3319
- /** Attempts per cell; default 1. */
2820
+ /** Attempts per cell; unset, a compare with variants runs 4 a side and anything else once. */
3320
2821
  runs?: number;
3321
2822
  /** How each cell's attempts decide its outcome; default `'all'`. */
3322
2823
  gate?: GatePolicy;
@@ -3350,7 +2851,7 @@ type ReevalConfig = {
3350
2851
  ci?: string;
3351
2852
  local?: string;
3352
2853
  };
3353
- /** `unbrokered: 'error'` refuses Docker secrets without `hosts` instead of warning (default `'warn'`). */
2854
+ /** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers written as plain strings, instead of warning (default `'warn'`). */
3354
2855
  secrets?: {
3355
2856
  unbrokered?: 'warn' | 'error';
3356
2857
  };
@@ -3362,11 +2863,12 @@ type ReevalConfig = {
3362
2863
  keep?: number;
3363
2864
  traceBufferMb?: number;
3364
2865
  };
3365
- /** Insights during `reeval run`, `reeval analyze`, and questions without a model. */
2866
+ /** `reeval analyze`, and questions without a model of their own. */
3366
2867
  analysis?: AnalysisConfig;
3367
2868
  /**
3368
2869
  * The gateways model calls may go through, in the order you prefer them: each harness takes
3369
- * the first that can serve it. Default `[direct()]`, each provider's own key.
2870
+ * the first that can serve it. `direct()`, each provider's own key, ends the list unless it's listed
2871
+ * in another place or turned off with `direct({ enabled: false })`. Default `[direct()]`.
3370
2872
  */
3371
2873
  gateway?: GatewayDescriptor[];
3372
2874
  };
@@ -3520,8 +3022,16 @@ type EnvironmentRecord = {
3520
3022
  sha256: string;
3521
3023
  }>;
3522
3024
  vars: string[];
3025
+ /**
3026
+ * Every secret, by the variable it's set as in the sandbox; an MCP header's (`secret.fromEnv`,
3027
+ * `secret.template`, a plugin's `${VAR}`) by each host variable it reads.
3028
+ */
3523
3029
  secrets: string[];
3524
- /** Secrets the sandbox only had placeholders for, swapped in by the egress gateway. */
3030
+ /**
3031
+ * Secrets the sandbox only had placeholders for, swapped in by the egress gateway, named as in
3032
+ * `secrets`. A variable it also got as its real value somewhere (an `http://` header, an env
3033
+ * secret without `hosts`) isn't listed.
3034
+ */
3525
3035
  brokered: string[];
3526
3036
  /** Optional secrets whose host variable was unset, so they weren't injected. */
3527
3037
  unset: string[];
@@ -3611,6 +3121,16 @@ type StartRetry = {
3611
3121
  /** How long it waited before giving up on that start. */
3612
3122
  waitedMs: number;
3613
3123
  };
3124
+ /**
3125
+ * A process the setup steps left running in a Docker sandbox with a `to: 'checks'` secret without
3126
+ * `hosts` in its environment: it holds the real value while the agent works.
3127
+ */
3128
+ type SetupProcess = {
3129
+ /** Its command line, cut at 200 characters. */
3130
+ command: string;
3131
+ /** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
3132
+ secrets: string[];
3133
+ };
3614
3134
  /** One `expect` or `expect.soft` matcher call an attempt made. */
3615
3135
  type AssertionRecord = {
3616
3136
  /**
@@ -3665,5 +3185,5 @@ type ResolvedGateway = {
3665
3185
  }>;
3666
3186
  };
3667
3187
  //#endregion
3668
- export { FakeJudgeModel as $, RunRecord as $n, Verdict as $t, CursorHarnessDescriptor as A, SuiteTally as An, PluginPart as At, EnvironmentHelpers as B, suiteResult as Bn, SandboxDescriptor as Bt, ClaudeCodeHarnessDescriptor as C, Interval as Cn, effortVerdict as Cr, McpServer as Ct, CompareOptions as D, ResultFile as Dn, tokenValues as Dr, PairVote as Dt, CompareCut as E, QuestionResult as En, formatSeconds as Er, PairFactor as Et, Duration as F, evalAcross as Fn, Question as Ft, EvalOptionsData as G, FamilyLift as Gn, SetupStep as Gt, EnvironmentSpec as H, AttemptRecord as Hn, Secret as Ht, EgressPolicy as I, factSegment as In, QuestionOptions as It, FakeAnalysisModel as J, FileRef as Jn, TranscriptEvent as Jt, ExecResult as K, FamilyRate as Kn, SkillSource as Kt, EgressRecord as L, isNotGatedResult as Ln, QuestionStateOptions as Lt, DirectGatewayDescriptor as M, checkEffect as Mn, Profile as Mt, DirectoryOptions as N, countedOf as Nn, ProfileDefaults as Nt, ConfigHelpers as O, ResultOutcome as On, PiHarnessDescriptor as Ot, DockerSandboxDescriptor as P, effectOf as Pn, ProfileLimit as Pt, FakeInsightsInput as Q, Lift as Qn, VercelGatewayDescriptor as Qt, EnvVars as R, isStudy as Rn, ReevalConfig as Rt, ClaudeCodeEffort as S, EvalResult as Sn, effortSummary as Sr, JudgeStateOptions as St, CompareAnswer as T, PooledArm as Tn, formatRatio as Tr, Outcome as Tt, EvalFn as U, CellRecord as Un, SecretTemplate as Ut, EnvironmentInput as V, ArtifactRef as Vn, SandboxWarning as Vt, EvalOptions as W, EvalEntry as Wn, ServeHandle as Wt, FakeHarnessDescriptor as X, HistoryEntry as Xn, TurnData as Xt, FakeAnswer as Y, HistoryCell as Yn, Turn as Yt, FakeInsights as Z, JudgementRecord as Zn, TurnStatus as Zt, CaptureUseScript as _, CellResult as _n, directional as _r, JudgeCallOptions as _t, AttemptAnswer as a, Job as an, EffortContrast as ar, GatewayName as at, Cell as b, Effect as bn, effortOf as br, JudgeOptions as bt, BlindAttempt as c, REMARK_KINDS as cn, EffortVerdict as cr, GroupOptions as ct, Capture as d, Sentence as dn, VERDICT_TEXT as dr, HarnessEdit as dt, Viewport as en, StackSummary as er, FakeSandboxDescriptor as et, CaptureCall as f, fromModel as fn, bothUnderATenth as fr, HarnessOption as ft, CaptureUseAction as g, CellError as gn, contrastEffort as gr, JudgeArtifactRef as gt, CaptureUse as h, CaseResult as hn, cellSamples as hr, JSONValue as ht, AssertionRecord as i, INSIGHTS_FILE as in, EFFORT_MIN_RUNS as ir, GatewayDescriptor as it, Diff as j, cancelledOf as jn, PluginSource as jt, ConfigJudgeOptions as k, SuiteResult as kn, PinnableKey as kt, BlindPair as l, Remark as ln, Spread as lr, HarnessData as lt, CaptureOptions as m, AttemptQuestionResult as mn, cellEffort as mr, HostFile as mt, AnalysisConfig as n, WebEventType as nn, DIRECTIONAL_MIN as nr, FakeVote as nt, AttemptContext as o, MODEL_REMARK_KINDS as on, EffortMetric as or, GooseHarnessDescriptor as ot, CaptureInstruction as p, Across as pn, cellContrast as pr, HistoryConfig as pt, ExecRetry as q, FamilyRecord as qn, TokenUsage as qt, AntigravityHarnessDescriptor as r, EvalInsight as rn, EFFORT_ALPHA as rr, FxHarnessDescriptor as rt, BackgroundHandle as s, NO_MODEL as sn, EffortSample as sr, GrokBuildHarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, CellEffort as tr, FakeScriptStep as tt, Budget as u, RunInsights as un, TOKENS_UNREPORTED as ur, HarnessDescriptor as ut, CaptureWait as v, CheckOutcome as vn, effortChangeLabel as vr, JudgeFailure as vt, CodexHarnessDescriptor as w, JudgeResult as wn, formatCount as wr, OpenCodeHarnessDescriptor as wt, CellRole as x, EffortCounters as xn, effortSample as xr, JudgeState as xt, CaptureWindow as y, CheckResult as yn, effortLabel as yr, JudgeModel as yt, Environment as z, messageLine as zn, ResolvedGateway as zt };
3669
- //# sourceMappingURL=types-DXBbP9Mk.d.ts.map
3188
+ export { FakeVote as $, VertexPublisher as $t, Diff as A, VERDICT_TEXT as An, Profile as At, EnvironmentInput as B, effortSample as Bn, Secret as Bt, CodexHarnessDescriptor as C, EFFORT_MIN_RUNS as Cn, Outcome as Ct, ConfigHelpers as D, EffortVerdict as Dn, PinnableKey as Dt, CompareOptions as E, EffortSample as En, PiHarnessDescriptor as Et, EgressPolicy as F, contrastEffort as Fn, QuestionStateOptions as Ft, ExecResult as G, formatSeconds as Gn, TokenUsage as Gt, ErrorKind as H, effortVerdict as Hn, ServeHandle as Ht, EgressRecord as I, directional as In, ReevalConfig as It, FakeAnswer as J, DeclaredNode as Jn, TurnData as Jt, ExecRetry as K, tokenValues as Kn, TranscriptEvent as Kt, EnvVars as L, effortChangeLabel as Ln, ResolvedGateway as Lt, DirectoryOptions as M, cellContrast as Mn, ProfileLimit as Mt, DockerSandboxDescriptor as N, cellEffort as Nn, Question as Nt, ConfigJudgeOptions as O, Spread as On, PluginPart as Ot, Duration as P, cellSamples as Pn, QuestionOptions as Pt, FakeScriptStep as Q, VertexGatewayDescriptor as Qt, Environment as R, effortLabel as Rn, SandboxDescriptor as Rt, ClaudeCodeHarnessDescriptor as S, EFFORT_ALPHA as Sn, OpenRouterGatewayDescriptor as St, CompareCut as T, EffortMetric as Tn, PairVote as Tt, EvalFn as U, formatCount as Un, SetupStep as Ut, EnvironmentSpec as V, effortSummary as Vn, SecretTemplate as Vt, EvalOptions as W, formatRatio as Wn, SkillSource as Wt, FakeJudgeModel as X, VercelGatewayDescriptor as Xt, FakeHarnessDescriptor as Y, TurnStatus as Yt, FakeSandboxDescriptor as Z, Verdict as Zt, CaptureWait as _, RunRecord as _n, JudgeOptions as _t, AttemptContext as a, CellRecord as an, GroupOptions as at, CellRole as b, CellEffort as bn, McpServer as bt, BlindPair as c, FamilyLift as cn, HarnessEdit as ct, CaptureCall as d, FamilyRecord as dn, HostFile as dt, Viewport as en, FxHarnessDescriptor as et, CaptureInstruction as f, FileRef as fn, JSONValue as ft, CaptureUseScript as g, Lift as gn, JudgeModel as gt, CaptureUseAction as h, JudgmentRecord as hn, JudgeFailure as ht, AttemptAnswer as i, AttemptRecord as in, GrokBuildHarnessDescriptor as it, DirectGatewayDescriptor as j, bothUnderATenth as jn, ProfileDefaults as jt, CursorHarnessDescriptor as k, TOKENS_UNREPORTED as kn, PluginSource as kt, Budget as l, FamilyLiftSummary as ln, HarnessOption as lt, CaptureUse as m, HistoryEntry as mn, JudgeCallOptions as mt, AnalysisConfig as n, WebEventType as nn, GatewayName as nt, BackgroundHandle as o, CheckTally as on, HarnessData as ot, CaptureOptions as p, HistoryCell as pn, JudgeArtifactRef as pt, FakeAnalysisModel as q, DeclarationTree as qn, Turn as qt, AntigravityHarnessDescriptor as r, ArtifactRef as rn, GooseHarnessDescriptor as rt, BlindAttempt as s, EvalEntry as sn, HarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, GatewayDescriptor as tt, Capture as u, FamilyRate as un, HistoryConfig as ut, CaptureWindow as v, StackSummary as vn, JudgeState as vt, CompareAnswer as w, EffortContrast as wn, PairFactor as wt, ClaudeCodeEffort as x, DIRECTIONAL_MIN as xn, OpenCodeHarnessDescriptor as xt, Cell as y, UpstreamCount as yn, JudgeStateOptions as yt, EnvironmentHelpers as z, effortOf as zn, SandboxWarning as zt };
3189
+ //# sourceMappingURL=types-coMWHaU2.d.ts.map