reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -1,3 +1,63 @@
1
+ //#region src/declaration.d.ts
2
+ /** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
3
+ type DeclaredMode = 'run' | 'skip' | 'only';
4
+ /** A `group`: options its evals and groups inherit, and its children in declaration order. */
5
+ type DeclaredGroup = {
6
+ type: 'group';
7
+ /** The group's name, as passed to `group`. */
8
+ name: string;
9
+ mode: DeclaredMode;
10
+ /** The group's own options, before the cascade. */
11
+ options: EvalOptionsData;
12
+ children: DeclaredNode[];
13
+ };
14
+ /** An `evaluate`: one eval, planned into a test per attempt. */
15
+ type DeclaredEval = {
16
+ type: 'eval';
17
+ /** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
18
+ name: string;
19
+ mode: DeclaredMode;
20
+ /** The eval's own options, before the cascade. */
21
+ options: EvalOptionsData;
22
+ /** Names the eval's body to whoever runs it. */
23
+ body: string;
24
+ source: DeclaredSource;
25
+ /** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
26
+ family?: DeclaredFamily;
27
+ };
28
+ /**
29
+ * An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
30
+ * text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
31
+ * whitespace, non-ASCII characters as themselves).
32
+ */
33
+ type DeclaredSource = {
34
+ /** sha256 of the body function's source text. */
35
+ body: string;
36
+ /** sha256 of the case's data; set on each case of an `evaluate.each`. */
37
+ case?: string;
38
+ };
39
+ /** Where a case of a pooled `evaluate.each` sits in its list. */
40
+ type DeclaredFamily = {
41
+ /** The name template passed to `evaluate.each`. */
42
+ name: string;
43
+ /** The case's index, from 0. */
44
+ case: number;
45
+ /** How many cases the list has. */
46
+ cases: number;
47
+ };
48
+ /** `evaluate.todo` or `group.todo`. */
49
+ type DeclaredTodo = {
50
+ type: 'todo';
51
+ /** The name passed to `.todo`. */
52
+ name: string;
53
+ };
54
+ /** One node of the tree: a group, an eval or a todo. */
55
+ type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
56
+ /** An eval file's top-level groups, evals and todos, in declaration order. */
57
+ type DeclarationTree = {
58
+ children: DeclaredNode[];
59
+ };
60
+ //#endregion
1
61
  //#region src/view/effort.d.ts
2
62
  /** Below this two-sided p, an effort difference is an effect. */
3
63
  declare const EFFORT_ALPHA = 0.05;
@@ -192,7 +252,7 @@ type JudgeRetry = {
192
252
  error: string;
193
253
  delayMs: number;
194
254
  };
195
- type JudgementRecord = {
255
+ type JudgmentRecord = {
196
256
  question: string;
197
257
  verdict: 'pass' | 'fail';
198
258
  score: number;
@@ -200,9 +260,17 @@ type JudgementRecord = {
200
260
  model: string;
201
261
  /** The model id the provider says served the call, when it reports one. */
202
262
  resolvedModel?: string;
263
+ /** The gateway from the config's `gateway` list the call went through; unset without one. */
264
+ gateway?: string;
265
+ /** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
266
+ upstream?: string;
267
+ /** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
268
+ costUsd?: number;
203
269
  /**
204
270
  * `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
205
- * verdict can be followed across runs (and an edited question starts a new series).
271
+ * verdict can be followed across runs (and an edited question starts a new series). `<index>`
272
+ * is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
273
+ * whichever finishes first.
206
274
  */
207
275
  key: string;
208
276
  /** Transient provider errors retried before this verdict, with the backoff before each retry. */
@@ -214,7 +282,7 @@ type AttemptFiles = {
214
282
  /** Each environment setup step's command, exit code, duration and output. */
215
283
  setupLog?: FileRef;
216
284
  transcript?: FileRef;
217
- /** Unnormalised harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
285
+ /** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
218
286
  rawTranscript?: FileRef;
219
287
  diff?: FileRef;
220
288
  judgeInput?: FileRef;
@@ -225,6 +293,11 @@ type ContextFile = {
225
293
  path: string;
226
294
  sha256: string;
227
295
  };
296
+ /** An upstream provider behind a gateway, and how many of the attempt's calls it served. */
297
+ type UpstreamCount = {
298
+ provider: string;
299
+ calls: number;
300
+ };
228
301
  type AttemptRecord = {
229
302
  /** `<eval key> # <cell key>#<n>`: unique in the run. */
230
303
  key: string;
@@ -257,6 +330,23 @@ type AttemptRecord = {
257
330
  * recorded them.
258
331
  */
259
332
  servedModels: string[] | null;
333
+ /**
334
+ * The models the harness's successful API calls asked for, when nothing named the one that
335
+ * served them (fx on Vercel AI Gateway with a Gemini model): the agent's word, not the
336
+ * gateway's, so never part of `servedModels`. Absent when there's none.
337
+ */
338
+ askedModels?: string[];
339
+ /**
340
+ * The upstream providers a gateway said served the harness's own API calls (OpenRouter's
341
+ * `provider`), each with how many it served, in the order first seen; more than one means the
342
+ * gateway spread or failed the calls over between them. Absent when the gateway names none.
343
+ */
344
+ upstreams?: UpstreamCount[];
345
+ /**
346
+ * What a gateway said the harness's own API calls cost, in US dollars, summed (OpenRouter's
347
+ * `usage.cost`); absent when it names none.
348
+ */
349
+ reportedCostUsd?: number;
260
350
  /**
261
351
  * The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
262
352
  * null when the factory left it unset. Fast is not this.
@@ -317,14 +407,31 @@ type AttemptRecord = {
317
407
  /** Longest tool `durationMs` in the transcript; null when none reported a duration. */
318
408
  maxToolMs: number | null;
319
409
  };
320
- judgements: JudgementRecord[];
410
+ judgements: JudgmentRecord[];
321
411
  /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
322
412
  startRetries?: StartRetry[];
413
+ /**
414
+ * Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
415
+ * without `hosts` in their environment, so they hold its real value while the agent works and
416
+ * the agent, as the same user, can read it from them. Found by the secret's variable name, so a
417
+ * copy under another name or in a command's arguments isn't. At most 20, each
418
+ * `{ command, secrets }`: its command line, masked and cut at 200 characters, and those secrets'
419
+ * names. Absent when there were none.
420
+ */
421
+ setupProcesses?: SetupProcess[];
422
+ /**
423
+ * Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
424
+ * `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
425
+ */
323
426
  checks: Array<{
324
427
  command: string;
325
428
  exitCode: number;
326
429
  durationMs: number;
327
430
  log?: FileRef;
431
+ /** The `retries` it was given, when more than 0: it could run `retries + 1` times. */
432
+ retries?: number;
433
+ /** Every try, in order, when it was given `retries`; its log has each try's output. */
434
+ tries?: ExecTry[];
328
435
  }>;
329
436
  /** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
330
437
  assertions?: AssertionRecord[];
@@ -333,6 +440,13 @@ type AttemptRecord = {
333
440
  count?: number;
334
441
  }>;
335
442
  files: AttemptFiles;
443
+ /**
444
+ * The eval's questions asked of this attempt on its own (those without `across`), in question
445
+ * order; unset without any. They annotate: `outcome` never depends on them. Asked once the
446
+ * attempt ends, when it ran to a pass or a fail. Each answer's `input` is `questions/<n>.json`
447
+ * in the attempt's directory, `<n>` the question's place in the eval's `questions`.
448
+ */
449
+ answers?: AttemptAnswer[];
336
450
  };
337
451
  type EvalEntry = {
338
452
  /** `file › group › … › name`, as shown. */
@@ -346,11 +460,21 @@ type EvalEntry = {
346
460
  name: string;
347
461
  /** Results are comparable only when fingerprints match. */
348
462
  fingerprint: string;
349
- /** The eval file and every local file it imports, directly or not, by content hash. */
463
+ /**
464
+ * The eval file, by content hash: where the eval lives. Records from before `source` also list
465
+ * the local files it imported, and were fingerprinted over them.
466
+ */
350
467
  sources: Array<{
351
468
  path: string;
352
469
  sha256: string;
353
470
  }>;
471
+ /**
472
+ * Hashes of what the `sources` fingerprint part covers but `revision`: the eval's body, its
473
+ * case data and its options with its groups'. Unset on records from before it.
474
+ */
475
+ source?: EvalSource;
476
+ /** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
477
+ revision?: Revision;
354
478
  tags: string[];
355
479
  /** The baseline environment, when the eval uses `compare`. */
356
480
  compare?: EvalCompareMeta;
@@ -367,8 +491,8 @@ type EvalEntry = {
367
491
  cells: CellRecord[];
368
492
  attempts: AttemptRecord[];
369
493
  /**
370
- * The compare's questions, answered after its attempts by pairwise votes, one per question;
371
- * unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
494
+ * The eval's pairwise questions (those with `across`), answered after its attempts by votes,
495
+ * one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
372
496
  * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
373
497
  * (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
374
498
  * `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
@@ -512,7 +636,7 @@ type Lift = {
512
636
  };
513
637
  /** Each input of a cell fingerprint, hashed on its own, so a change can be named. */
514
638
  type FingerprintParts = {
515
- /** Over `EvalEntry.sources`. */
639
+ /** Over `EvalEntry.source` and `EvalEntry.revision`; on records from before them, over `EvalEntry.sources`. */
516
640
  sources: string;
517
641
  /** Over the harness descriptor, less concurrency caps. */
518
642
  harness: string;
@@ -523,15 +647,30 @@ type FingerprintParts = {
523
647
  judge: string;
524
648
  /** Over the attempt's context files. */
525
649
  context: string;
650
+ /** The gateway its model calls went through; unset on `direct`, each provider's own key. */
651
+ gateway?: string;
652
+ /**
653
+ * The cell's gate (`none`, `passRate 0.5`); unset on the default `all`. Compare reads a change
654
+ * in it as a policy change, not a reason the runs can't be compared.
655
+ */
656
+ gate?: string;
526
657
  };
527
658
  /** One harness and environment of an eval, and how its attempts went. */
528
659
  type CellRecord = Cell & {
529
660
  /** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
530
661
  fingerprint: string;
531
662
  fingerprintParts: FingerprintParts;
532
- /** The environment's digest: the join key into `RunRecord.environments`; null for none. */
663
+ /**
664
+ * The environment's digest; null for none. With `environment`, the key into
665
+ * `RunRecord.environments`: environments with the same contents share a digest.
666
+ */
533
667
  environmentDigest: string | null;
534
668
  harnessKind: string;
669
+ /**
670
+ * The gateway in the config's `gateway` list its harness resolved to (`direct` is each
671
+ * provider's own key); unset when no config, profile or pin named one.
672
+ */
673
+ gateway?: string;
535
674
  /** The configured harness model, when it has one. */
536
675
  model: string | null;
537
676
  /**
@@ -668,6 +807,7 @@ type RunRecord = {
668
807
  config: {
669
808
  sandbox: string;
670
809
  environment: string | null;
810
+ /** The config's `runs`; 1 when it sets none, though a compare's cells then run 4 each. */
671
811
  runs: number;
672
812
  judgeModel: string | null;
673
813
  threshold: number;
@@ -794,8 +934,15 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
794
934
  };
795
935
  /** Median tool calls, agent time and tokens over the counted attempts; null when none counted. */
796
936
  effort: CellEffort | null;
797
- /** Of `fail`, the attempts the run's interrupt cancelled (`error.kind: 'aborted'`); unset when none. */
798
- cancelled?: number;
937
+ /** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
938
+ canceled?: number;
939
+ };
940
+ /** A family's pooled lift as the history listing keeps it: enough to compare its gate. */
941
+ type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
942
+ lo: number;
943
+ hi: number;
944
+ /** Unset when the compare has no `minLift`, so the lift can't fail. */
945
+ minLift?: number;
799
946
  };
800
947
  /**
801
948
  * One line of the derived history listing: cached locally by the CLI, and the run header in the
@@ -830,599 +977,24 @@ type HistoryEntry = {
830
977
  slug: string;
831
978
  fingerprint: string;
832
979
  sources: EvalEntry['sources'];
980
+ source?: EvalEntry['source'];
981
+ revision?: EvalEntry['revision'];
833
982
  outcome: Outcome;
834
983
  passRate: number | null;
835
984
  cells: HistoryCell[];
836
985
  }>;
837
986
  /** The run's pooled families; unset when none ran. */
838
- families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'>>;
987
+ families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'> & {
988
+ /** The eval ids of its cases in this run. */
989
+ caseIds: string[];
990
+ lifts: FamilyLiftSummary[];
991
+ }>;
839
992
  /** The run's errors outside any attempt; unset when there were none. */
840
993
  errors?: string[];
841
994
  };
842
995
  //#endregion
843
- //#region src/view/suite.d.ts
844
- /**
845
- * A variant against its baseline: `better` or `worse` when the stored 95% interval on the
846
- * difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
847
- * including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
848
- */
849
- type Effect = 'better' | 'worse' | 'same' | 'unclear';
850
- /**
851
- * What an eval's cells compare: `change` when it has variants (or several environments),
852
- * `harness` when it has several harnesses, `both`, or `none`.
853
- */
854
- type Across = 'none' | 'change' | 'harness' | 'both';
855
- /**
856
- * One finished run's facts. Every node has a `path`, a stable id to cite it by, and `paths`
857
- * lists them all in the order the nodes appear.
858
- *
859
- * A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
860
- *
861
- * - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
862
- * (`evals/a.eval.ts › fixes $id`);
863
- * - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
864
- * - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
865
- * - `<cell>/checks/<label>`: a check, by its assertion label;
866
- * - `<cell>/effort`: the cell's effort, only when it was tested;
867
- * - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
868
- *
869
- * A case segment never has a raw `+` and a cell segment always has two, so the two can't be
870
- * confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
871
- * fact in every run that has it.
872
- */
873
- type SuiteResult = {
874
- runId: string;
875
- status: RunRecord['status'];
876
- /**
877
- * The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
878
- * an attempt, and `skipped` on an interrupted run.
879
- */
880
- outcome: Outcome;
881
- /**
882
- * Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
883
- * awaited, a worker that died. Any one fails the run.
884
- */
885
- errors: string[];
886
- /**
887
- * Outcomes counted per block: each family case is one, each eval with no cases is one, and a
888
- * family whose pooled gate failed is one more fail. `errors` counts the run's `errors`.
889
- */
890
- tally: SuiteTally;
891
- /** Sorted by path. */
892
- files: ResultFile[];
893
- /** Every node's `path`, in the order they appear. */
894
- paths: string[];
895
- };
896
- type SuiteTally = {
897
- errors: number;
898
- pass: number;
899
- fail: number;
900
- infraError: number;
901
- cancelled: number;
902
- skipped: number;
903
- };
904
- /**
905
- * An eval's or a family case's outcome: the record's, except `cancelled` where its only failed
906
- * gated attempts are ones the run's interrupt cancelled, since the agent didn't fail those.
907
- */
908
- type ResultOutcome = Outcome | 'cancelled';
909
- type ResultFile = {
910
- path: string;
911
- evals: EvalResult[];
912
- };
913
- type EvalResult = {
914
- path: string;
915
- /** The eval's id, or the family's. */
916
- id: string;
917
- group: string[];
918
- /** The eval's name; on a family, its template. */
919
- name: string;
920
- /** A family's name template (`fixes $id`); null on a single eval. */
921
- template: string | null;
922
- /** Source order in the file. */
923
- order: number;
924
- across: Across;
925
- /**
926
- * A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
927
- * unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
928
- */
929
- family: boolean;
930
- /** A placeholder (`evaluate.todo`) that never ran. */
931
- todo: boolean;
932
- /** On a family: `fail` when a case failed or its pooled gate did. */
933
- outcome: ResultOutcome;
934
- cells: CellResult[] | null;
935
- cases: CaseResult[] | null;
936
- /** A family's variants against their baseline, pooled over the cases; null on a single eval. */
937
- pooled: PooledArm[] | null;
938
- /**
939
- * Its compare questions' answers. On a family, each question asked of any case, in first-asked
940
- * order, with its votes and wins summed over the cases that answered it.
941
- */
942
- questions: QuestionResult[];
943
- };
944
- /** A compare question: each side's wins over the pairs the model decided. Never gates. */
945
- type QuestionResult = {
946
- question: string;
947
- /** Pairs voted, each counted once per order it was asked in. */
948
- votes: number;
949
- rates: Array<{
950
- cut: 'within' | 'across';
951
- name: string;
952
- wins: number;
953
- n: number;
954
- }>;
955
- /** Why some or all of its votes are missing: the model call failed. */
956
- error: string | null;
957
- };
958
- type CaseResult = {
959
- path: string;
960
- /** The case's own eval id. */
961
- id: string;
962
- /** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
963
- label: string;
964
- name: string;
965
- /** Its place in the case list. */
966
- order: number;
967
- outcome: ResultOutcome;
968
- cells: CellResult[];
969
- };
970
- type CellResult = {
971
- path: string;
972
- key: string;
973
- harness: string;
974
- /** `-` when no environment is set. */
975
- environment: string;
976
- sandbox: string;
977
- /** The variant's name; null on a control. */
978
- variant: string | null;
979
- /** The control of an eval that has variants: the arm the variants are measured against. */
980
- baseline: boolean;
981
- /** On a variant, its baseline cell's key; else null. */
982
- vs: string | null;
983
- /** The cell's place in the eval's cells. */
984
- order: number;
985
- /** Its failures fail the eval (its gate isn't `'none'`). */
986
- gated: boolean;
987
- outcome: Outcome;
988
- /** Every attempt: `passed + failed + infraError + skipped`. */
989
- attempts: number;
990
- passed: number;
991
- failed: number;
992
- /** Infrastructure errors, which are neither passes nor fails. */
993
- infraError: number;
994
- skipped: number;
995
- fingerprint: string;
996
- /** The skill sources and files the cell's environment placed, as written. */
997
- context: string[];
998
- /**
999
- * Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
1000
- * attempt (every one an infra error or skipped) lists each as `not run`.
1001
- */
1002
- checks: CheckResult[];
1003
- /** On a variant, against its baseline; null on a control. */
1004
- effect: Effect | null;
1005
- /** The stored 95% interval on the variant's difference from its baseline. Not for display. */
1006
- interval: Interval | null;
1007
- /** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
1008
- minLift: number | null;
1009
- /** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
1010
- effort: EffortCounters | null;
1011
- /** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
1012
- effortDiffers: boolean;
1013
- /** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
1014
- errors: CellError[];
1015
- /** Web searches and fetches per counted attempt; null where the harness can't report them. */
1016
- searches: Array<number | null>;
1017
- fetches: Array<number | null>;
1018
- };
1019
- /** A cell's attempts the run's interrupt cancelled: recorded as fails, but not the agent's. */
1020
- declare const cancelledOf: (cell: CellResult) => number;
1021
- /** A cell's counted tries: its passes and fails, less the ones the interrupt cancelled. */
1022
- declare const countedOf: (cell: CellResult) => number;
1023
- type CheckResult = {
1024
- path: string;
1025
- label: string;
1026
- /** First-seen order across the eval's attempts. */
1027
- order: number;
1028
- outcome: CheckOutcome;
1029
- /**
1030
- * On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
1031
- * null on a control or without one.
1032
- */
1033
- effect: Effect | null;
1034
- /** Set when the check is a judge verdict. */
1035
- judge: JudgeResult | null;
1036
- };
1037
- /**
1038
- * `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
1039
- * none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
1040
- * reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
1041
- * first failure's message.
1042
- */
1043
- type CheckOutcome = {
1044
- status: 'not run';
1045
- } | {
1046
- status: 'ran';
1047
- passed: number;
1048
- failed: number;
1049
- errored: number;
1050
- notRun: number;
1051
- assertion: string | null;
1052
- };
1053
- /**
1054
- * One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
1055
- * an error), its message's first line, and how often.
1056
- */
1057
- type CellError = {
1058
- kind: ErrorKind | 'turn';
1059
- message: string;
1060
- attempts: number;
1061
- };
1062
- type JudgeResult = {
1063
- question: string;
1064
- threshold: number;
1065
- /** One per counted attempt that judged it. */
1066
- scores: number[];
1067
- };
1068
- type Interval = {
1069
- low: number;
1070
- high: number;
1071
- };
1072
- /** A null counter is one the harness didn't record; a recorded zero stays zero. */
1073
- type EffortCounters = {
1074
- tools: number | null;
1075
- agentMs: number | null;
1076
- tokens: number | null;
1077
- };
1078
- type PooledArm = {
1079
- path: string;
1080
- /** The variant cell's key, the same in every case. */
1081
- key: string;
1082
- vs: string;
1083
- effect: Effect;
1084
- /** Cases with counted attempts on both sides. */
1085
- pairs: number;
1086
- interval: Interval;
1087
- /** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
1088
- minLift: number | null;
1089
- outcome: 'pass' | 'fail';
1090
- };
1091
- type Side = {
1092
- pass: number;
1093
- counted: number;
1094
- };
1095
- /**
1096
- * The effect of `variant` against `control`, from the interval on their difference. A side
1097
- * with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
1098
- */
1099
- declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
1100
- /** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
1101
- declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
1102
- /** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
1103
- declare const factSegment: (s: string) => string;
1104
- declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
1105
- /**
1106
- * An error's one line: its first line with any letter or digit in it, joined with the next one
1107
- * when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
1108
- */
1109
- declare function messageLine(text: string): string;
1110
- /** The facts of a run, from its record. Pure: the same record always gives the same result. */
1111
- declare function suiteResult(record: RunRecord): SuiteResult;
1112
- //#endregion
1113
- //#region src/view/insights.d.ts
1114
- /** `<history>/<run>/insights.json`, beside the record. */
1115
- declare const INSIGHTS_FILE = "insights.json";
1116
- /** `RunInsights.model` when no analysis model wrote the file: it holds deterministic remarks only. */
1117
- declare const NO_MODEL = "none";
1118
- /** Whether an analysis model wrote these insights, rather than reeval alone for its remarks. */
1119
- declare const fromModel: (insights: RunInsights) => boolean;
1120
- /** What one attempt tests. Only an analysis model can tell; without one every eval is `other`. */
1121
- type Job = 'choose' | 'finish' | 'other';
1122
- /**
1123
- * A run's insights, `<history>/<run>/insights.json`. Written create-only; the record is untouched.
1124
- * They're for the facts they were written from: a `factsHash` that isn't the run's marks them stale.
1125
- */
1126
- type RunInsights = {
1127
- kind: 'reeval-insights';
1128
- schemaVersion: 1;
1129
- runId: string;
1130
- /**
1131
- * The analysis model, as configured (`google/gemini-3.8-flash`, or `fake`); `NO_MODEL` when
1132
- * reeval wrote the file without one, to hold the run's deterministic remarks only.
1133
- */
1134
- model: string;
1135
- /** The prompts' version: insights from another version were asked differently. */
1136
- promptVersion: string;
1137
- /** sha256 of the run's `SuiteResult` as canonical JSON: the facts the model saw. */
1138
- factsHash: string;
1139
- createdAt: string;
1140
- /**
1141
- * Over every call; `outputTokens` includes `reasoningTokens`, the model's thinking, when it
1142
- * reports them. `costUsd` is null when reeval doesn't know the model's price.
1143
- */
1144
- usage: {
1145
- inputTokens: number;
1146
- outputTokens: number;
1147
- reasoningTokens?: number;
1148
- costUsd: number | null;
1149
- };
1150
- /**
1151
- * `analysis.maxTokens`, and the evals (`SuiteResult` paths) it ran out before: they print
1152
- * without insights. Absent when no budget applied.
1153
- */
1154
- budget?: {
1155
- maxTokens: number;
1156
- skipped: string[];
1157
- };
1158
- /**
1159
- * Evals whose calls failed twice (an error, a reply cut off at the output cap, or a call past its
1160
- * timeout), with the last error: they print without insights. Absent when none failed.
1161
- */
1162
- failed?: Array<{
1163
- evalId: string;
1164
- error: string;
1165
- }>;
1166
- /**
1167
- * Set when the run was stopped (Ctrl-C, or SIGTERM) before every eval's insights had finished:
1168
- * the evals that hadn't, which print without insights. Absent otherwise, even when the stop came
1169
- * during the headline call, which leaves `headline` null.
1170
- */
1171
- stopped?: {
1172
- signal: 'SIGINT' | 'SIGTERM';
1173
- skipped: string[];
1174
- };
1175
- /** One sentence for the whole run, citing facts; null when the model's was dropped. */
1176
- headline: Sentence | null;
1177
- /** One per eval (a family is one), in the order the output prints them. */
1178
- evals: EvalInsight[];
1179
- /**
1180
- * Remarks about the evals' own quality, ranked: every one this run raised, including those an
1181
- * earlier run printed first (`firstSeenRun`), which don't print again.
1182
- */
1183
- remarks: Remark[];
1184
- /**
1185
- * What the checks dropped, and how many sentences were asked for again, so the output can say
1186
- * so. `drops` is each dropped sentence: its slot (`<eval> › answer`, `› row <path>`, `headline`),
1187
- * its text and why.
1188
- */
1189
- validation: {
1190
- dropped: number;
1191
- regenerated: number;
1192
- drops?: Array<{
1193
- slot: string;
1194
- text: string;
1195
- reason: string;
1196
- }>;
1197
- };
1198
- /** Set when insights failed; the facts still print. */
1199
- error?: string;
1200
- };
1201
- type EvalInsight = {
1202
- /** The eval's `SuiteResult` path; for a family, the family's. */
1203
- evalId: string;
1204
- intent: {
1205
- /** What one attempt is testing. Can't be read from the cell shape. */
1206
- job: Job;
1207
- /** What the cells compare. Deterministic, copied from the facts. */
1208
- across: Across;
1209
- /** The eval's question in plain words: "Do agents pick Genkit for a production AI feature?" */
1210
- question: string;
1211
- /** The tool a choose eval asks about, and the change a measure eval makes. */
1212
- subject: {
1213
- tool?: string;
1214
- change?: string;
1215
- };
1216
- /** How the eval name relates to what the eval checks. */
1217
- name: 'states-intent' | 'no-intent' | 'contradicts';
1218
- /**
1219
- * sha256 of the eval's id, sources, first attempt's prompts, judge questions, check labels,
1220
- * the model and the prompt version: the classification's cache key.
1221
- */
1222
- key: string;
1223
- };
1224
- /** The answer to `question`, at most two sentences; null when it was dropped. */
1225
- answer: Sentence | null;
1226
- /** Up to three supporting points: what moved, what broke, what the agent did. */
1227
- findings: Sentence[];
1228
- /**
1229
- * At most one sentence per row (a case, an arm or a harness), keyed by the row's fact path.
1230
- * Only rows the eval's layout opens a slot on are kept.
1231
- */
1232
- rows?: Array<Sentence & {
1233
- row: string;
1234
- }>;
1235
- /**
1236
- * Per attempt (`<cell path>#<n>`), the tools the reply recommended or used, each found
1237
- * verbatim in that attempt's final message. Only on `choose` evals.
1238
- */
1239
- namedTools?: Array<{
1240
- attempt: string;
1241
- tools: string[];
1242
- }>;
1243
- };
1244
- /**
1245
- * A plain sentence citing the facts it states: `SuiteResult` paths
1246
- * (`evals%2Fbilling.eval.ts › bills in pence/fake+-+fake/checks/amount`) or a transcript line
1247
- * (`transcript:<cell path>#<n>#L<line>`). A sentence citing nothing is dropped.
1248
- */
1249
- type Sentence = {
1250
- text: string;
1251
- cites: string[];
1252
- };
1253
- /**
1254
- * Each remark kind's origin and severity: an analysis model raises the `model` kinds, through the
1255
- * gates (a known kind, a quote verbatim on the cited line); reeval's rules raise the others. The
1256
- * severity is the kind's, never the model's.
1257
- */
1258
- declare const REMARK_KINDS: Record<Remark['kind'], {
1259
- origin: Remark['origin'];
1260
- severity: 'concern' | 'suggestion';
1261
- }>;
1262
- /** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
1263
- declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "slow-mcp-start" | "too-few-tries")[];
1264
- /** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
1265
- type Remark = {
1266
- /**
1267
- * sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
1268
- * per kind (and server) per eval version.
1269
- */
1270
- id: string;
1271
- evalId: string;
1272
- /** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
1273
- kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start';
1274
- /** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
1275
- severity: 'concern' | 'suggestion';
1276
- /** One plain sentence: what's wrong. */
1277
- text: string;
1278
- /**
1279
- * Where the problem is: `source:<file>#L<n>` with `quote`, found verbatim on that line of the
1280
- * file at the run's source hash. Every model remark has one.
1281
- */
1282
- source?: {
1283
- at: string;
1284
- quote: string;
1285
- };
1286
- /** Facts from this run (or history facts) showing it mattered now; a model's, as it cited them. */
1287
- evidence: string[];
1288
- /**
1289
- * An edit at a source location (`source:<file>#L<n>`) in the eval. A model remark's `edit` is
1290
- * empty when the model gave none, and its `at` is the quoted line when it named no line.
1291
- */
1292
- fix: {
1293
- at: string;
1294
- edit: string;
1295
- };
1296
- origin: 'model' | 'deterministic';
1297
- /**
1298
- * The first run that printed this `id`, or this run when none has: it prints only in that run,
1299
- * except `slow-mcp-start`, which is about the environment, not the eval's source, and prints in
1300
- * every run it happens.
1301
- */
1302
- firstSeenRun: string;
1303
- /**
1304
- * Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
1305
- * when the cap or a run without `-v` left it out, so a later run prints it. Missing: printed in
1306
- * `firstSeenRun` only.
1307
- */
1308
- printed?: boolean;
1309
- /** Set on later runs once the cited span changed: the remark was acted on. */
1310
- actedOn?: {
1311
- run: string;
1312
- };
1313
- };
1314
- //#endregion
1315
- //#region src/insights/input.d.ts
1316
- /** One fact the model may cite, by its path. */
1317
- type InsightFact = {
1318
- path: string;
1319
- kind: string;
1320
- } & Record<string, unknown>;
1321
- type InsightAttempt = {
1322
- /** `<cell path>#<n>`. */
1323
- id: string;
1324
- cell: string;
1325
- outcome: string;
1326
- /** The agent's final message, clipped. */
1327
- lastText: string;
1328
- /** A failed attempt's transcript lines, `L<n> …`, each citable as `transcript:<id>#L<n>`. */
1329
- lines: string[];
1330
- };
1331
- /** Where an insight sentence can go, as the eval's layout opens them. */
1332
- type InsightSlots = {
1333
- /** Row paths that take one sentence each. */
1334
- rows: string[];
1335
- findings: boolean;
1336
- namedTools: boolean;
1337
- };
1338
- /** One eval as the classifier and the insight model see it. */
1339
- type InsightInput = {
1340
- call: 'classify' | 'insights';
1341
- eval: {
1342
- path: string;
1343
- /** The eval's name, or a family's template: the author's statement of what it asks. */
1344
- name: string;
1345
- group: string[];
1346
- file: string;
1347
- across: Across;
1348
- family: boolean;
1349
- verdict: string;
1350
- };
1351
- /** The first attempt's prompts, in order. */
1352
- prompts: string[];
1353
- judgeQuestions: string[];
1354
- checks: string[];
1355
- /** Set on the insights call: the classification. */
1356
- intent?: {
1357
- job: Job;
1358
- question: string;
1359
- subject: {
1360
- tool?: string;
1361
- change?: string;
1362
- };
1363
- };
1364
- /** Set on the insights call. */
1365
- slots?: InsightSlots;
1366
- facts: InsightFact[];
1367
- attempts: InsightAttempt[];
1368
- /**
1369
- * Set on the insights call: the eval's declaration in its file, each line `L<n> …` as the file
1370
- * numbers it, which a remark's `source` and `fix` cite.
1371
- */
1372
- source?: {
1373
- file: string;
1374
- lines: string[];
1375
- };
1376
- /** What was left out to fit the eval's size cap (`trim.ts`), when anything was. */
1377
- trimmed?: string[];
1378
- /** On a regeneration: why the earlier sentences were refused. */
1379
- retry?: string[];
1380
- };
1381
- /** The headline call's input: each eval's verdict and answer, and the run's tally with its total. */
1382
- type HeadlineInput = {
1383
- call: 'headline';
1384
- /**
1385
- * Counts results, not evals: each case of a family counts one, as the tally line does, and
1386
- * `total` is their sum, so no count is the model's arithmetic; `errors` (outside any eval) aren't
1387
- * results and stay out of it. The eval count is `evals.length`.
1388
- */
1389
- tally: SuiteResult['tally'] & {
1390
- total: number;
1391
- unit: 'results';
1392
- };
1393
- evals: Array<{
1394
- path: string;
1395
- name: string;
1396
- verdict: string;
1397
- answer: string | null;
1398
- }>;
1399
- retry?: string[];
1400
- };
1401
- /** A fact as the model sees it: its short id (`F1`), which is what it cites. */
1402
- type ModelFact = InsightFact & {
1403
- id: string;
1404
- };
1405
- /** An attempt as the model sees it: `A1`, its cell's fact id, and its lines citable as `A1#L<n>`. */
1406
- type ModelAttempt = Omit<InsightAttempt, 'id' | 'cell'> & {
1407
- id: string;
1408
- cell: string;
1409
- };
1410
- /** One eval's call as the model sees it: facts, attempts and row slots by short id. */
1411
- type ModelInsightInput = Omit<InsightInput, 'facts' | 'attempts' | 'slots'> & {
1412
- facts: ModelFact[];
1413
- attempts: ModelAttempt[];
1414
- slots?: Omit<InsightSlots, 'rows'> & {
1415
- rows: string[];
1416
- };
1417
- };
1418
- /** The headline call as the model sees it: each eval by its short id. */
1419
- type ModelHeadlineInput = Omit<HeadlineInput, 'evals'> & {
1420
- evals: Array<HeadlineInput['evals'][number] & {
1421
- id: string;
1422
- }>;
1423
- };
1424
- //#endregion
1425
996
  //#region src/types.d.ts
997
+ /** More than zero and at most 596h: `90s`, `15m`, `1h`, or a number of milliseconds. */
1426
998
  type Duration = number | `${number}${'ms' | 's' | 'm' | 'h'}`;
1427
999
  /** When an event started: wall clock, and ms since its turn started. */
1428
1000
  type EventTime = {
@@ -1433,13 +1005,17 @@ type FileEditEvent = {
1433
1005
  type: 'file-edit';
1434
1006
  toolCallId: string;
1435
1007
  path: string;
1436
- kind: 'create' | 'modify' | 'delete';
1008
+ /**
1009
+ * `write` is a whole-file write whose result doesn't say whether the file existed before
1010
+ * (Antigravity's, fx's, Goose's, Grok Build's and pi's write tools, for example).
1011
+ */
1012
+ kind: 'create' | 'modify' | 'write' | 'delete';
1437
1013
  /** Line counts, or null when the harness didn't send the content (e.g. deletes). */
1438
1014
  added: number | null;
1439
1015
  removed: number | null;
1440
1016
  };
1441
1017
  /**
1442
- * Normalised agent events. Each tool call is one event, merged with its result; `durationMs`
1018
+ * Normalized agent events. Each tool call is one event, merged with its result; `durationMs`
1443
1019
  * is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
1444
1020
  * when the harness never reported the end. Long shell or edit calls set `stalled: true`.
1445
1021
  */
@@ -1496,6 +1072,18 @@ type ExecResult = {
1496
1072
  stdout: string;
1497
1073
  stderr: string;
1498
1074
  };
1075
+ /** How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient. */
1076
+ type ExecRetry = {
1077
+ /** Extra tries after a non-zero exit, a whole number from 0. Default 0. */
1078
+ retries?: number;
1079
+ /** Milliseconds to wait before each extra try. Default 1000. */
1080
+ interval?: number;
1081
+ };
1082
+ /** One run of a `c.exec` command that had `retries`. */
1083
+ type ExecTry = {
1084
+ exitCode: number;
1085
+ durationMs: number;
1086
+ };
1499
1087
  /** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
1500
1088
  type CaptureWindow = {
1501
1089
  /** In CSS pixels. */
@@ -1569,8 +1157,8 @@ type CaptureOptions = {
1569
1157
  /** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
1570
1158
  browser?: boolean;
1571
1159
  /**
1572
- * Vision model for `instructions`. Cascades like `question`. Default is the analysis
1573
- * model when that model accepts images; never Jev.
1160
+ * Vision model for `instructions`. Cascades like `question`. Default is `analysis.model`, else
1161
+ * the attempt's judge model, whichever is first a language model; never Jev.
1574
1162
  */
1575
1163
  model?: string;
1576
1164
  };
@@ -1604,6 +1192,7 @@ type Capture = {
1604
1192
  * default `name` is derived from the path. `wait` and `instructions` are call-only.
1605
1193
  */
1606
1194
  type CaptureCall = {
1195
+ /** The `artifacts/<name>.png` file name: 1 to 100 of `a-z`, `0-9`, `_` and `-`, starting with a letter or digit. Unique within the attempt. */
1607
1196
  name?: string;
1608
1197
  path?: string;
1609
1198
  viewport?: Viewport;
@@ -1665,7 +1254,7 @@ type TurnData = {
1665
1254
  };
1666
1255
  type Turn = TurnData & {
1667
1256
  /** Runs in the eval's sandbox. Same as `c.exec`. */
1668
- exec(command: string): Promise<ExecResult>;
1257
+ exec(command: string, options?: ExecRetry): Promise<ExecResult>;
1669
1258
  readFile(path: string): Promise<string | null>;
1670
1259
  };
1671
1260
  /** Any JSON value. */
@@ -1730,6 +1319,11 @@ type JudgeOptions = {
1730
1319
  * attempt's state. Merged key by key down the cascade.
1731
1320
  */
1732
1321
  state?: JudgeStateOptions;
1322
+ /**
1323
+ * The gateway the judge's calls go through, by name or by its helper. It skips the config's
1324
+ * `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
1325
+ */
1326
+ gateway?: GatewayName | GatewayDescriptor;
1733
1327
  };
1734
1328
  /**
1735
1329
  * What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
@@ -1798,6 +1392,12 @@ type Verdict = {
1798
1392
  model: string;
1799
1393
  /** The model id the provider says served the call, when it reports one. */
1800
1394
  resolvedModel?: string;
1395
+ /** The gateway from the config's `gateway` list the call went through; unset without one. */
1396
+ gateway?: GatewayName;
1397
+ /** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
1398
+ upstream?: string;
1399
+ /** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
1400
+ costUsd?: number;
1801
1401
  /** Captures this call sent the judge as file parts, when any. */
1802
1402
  artifacts?: JudgeArtifactRef[];
1803
1403
  };
@@ -1805,7 +1405,7 @@ type Verdict = {
1805
1405
  * What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished,
1806
1406
  * so the judges and checks after it still run.
1807
1407
  */
1808
- type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
1408
+ type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel' | 'upstream' | 'costUsd'> & {
1809
1409
  verdict: 'error';
1810
1410
  score: null;
1811
1411
  error: string;
@@ -1822,11 +1422,17 @@ type AttemptContext = {
1822
1422
  * `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
1823
1423
  */
1824
1424
  judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1825
- exec(command: string): Promise<ExecResult>;
1425
+ /**
1426
+ * Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
1427
+ * many extra times, `interval` ms apart; it resolves with the last try.
1428
+ */
1429
+ exec(command: string, options?: ExecRetry & {
1430
+ background?: false;
1431
+ }): Promise<ExecResult>;
1826
1432
  exec(command: string, options: {
1827
1433
  background: true;
1828
1434
  }): Promise<BackgroundHandle>;
1829
- exec(command: string, options?: {
1435
+ exec(command: string, options?: ExecRetry & {
1830
1436
  background?: boolean;
1831
1437
  }): Promise<ExecResult | BackgroundHandle>;
1832
1438
  /**
@@ -1863,13 +1469,18 @@ type EvalFn = (c: AttemptContext) => Promise<void>;
1863
1469
  /** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
1864
1470
  type FakeHarnessDescriptor = {
1865
1471
  kind: 'fake';
1472
+ /**
1473
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1474
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1475
+ */
1476
+ gateway?: GatewayName;
1866
1477
  /** Cell id; defaults to `fake`. */
1867
1478
  id?: string;
1868
1479
  /** Files the fake agent writes into the sandbox on every turn. */
1869
1480
  files?: Record<string, string>;
1870
1481
  /** The fake agent's final message on every turn; default `Done.`. */
1871
1482
  text?: string;
1872
- /** Simulated turn length, honours the abort signal. */
1483
+ /** Simulated turn length, honors the abort signal. */
1873
1484
  delayMs?: number;
1874
1485
  /** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
1875
1486
  ignoreAbort?: boolean;
@@ -1949,6 +1560,11 @@ type FakeScriptStep = {
1949
1560
  */
1950
1561
  type CursorHarnessDescriptor = {
1951
1562
  kind: 'cursor';
1563
+ /**
1564
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1565
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1566
+ */
1567
+ gateway?: GatewayName;
1952
1568
  /** A model id `Cursor.models.list()` returns for the key, e.g. `composer-2.5`. */
1953
1569
  model: string;
1954
1570
  /**
@@ -1976,6 +1592,11 @@ type ClaudeCodeEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
1976
1592
  /** Claude Code through `@ai-sdk/harness-claude-code`, made by `claudeCode()`. */
1977
1593
  type ClaudeCodeHarnessDescriptor = {
1978
1594
  kind: 'claude-code';
1595
+ /**
1596
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1597
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1598
+ */
1599
+ gateway?: GatewayName;
1979
1600
  /** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
1980
1601
  model: string;
1981
1602
  /**
@@ -1996,6 +1617,11 @@ type ClaudeCodeHarnessDescriptor = {
1996
1617
  /** The Antigravity CLI through `@reeval/harness-antigravity`, made by `antigravity()`. */
1997
1618
  type AntigravityHarnessDescriptor = {
1998
1619
  kind: 'antigravity';
1620
+ /**
1621
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1622
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1623
+ */
1624
+ gateway?: GatewayName;
1999
1625
  /** A model slug `agy models` lists for a Gemini API key, e.g. `gemini-3.8-flash-medium`. */
2000
1626
  model: string;
2001
1627
  /**
@@ -2020,6 +1646,11 @@ type AntigravityHarnessDescriptor = {
2020
1646
  */
2021
1647
  type OpenCodeHarnessDescriptor = {
2022
1648
  kind: 'opencode';
1649
+ /**
1650
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1651
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1652
+ */
1653
+ gateway?: GatewayName;
2023
1654
  /**
2024
1655
  * OpenCode's model id, `<provider>/<model>`, e.g. `google/gemini-3.8-flash` or
2025
1656
  * `anthropic/claude-haiku-4-5`; the provider picks the key.
@@ -2046,6 +1677,11 @@ type OpenCodeHarnessDescriptor = {
2046
1677
  */
2047
1678
  type GooseHarnessDescriptor = {
2048
1679
  kind: 'goose';
1680
+ /**
1681
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1682
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1683
+ */
1684
+ gateway?: GatewayName;
2049
1685
  /** A Gemini model id, run on Goose's `google` provider, e.g. `gemini-3.8-flash`. */
2050
1686
  model: string;
2051
1687
  /**
@@ -2069,6 +1705,11 @@ type GooseHarnessDescriptor = {
2069
1705
  */
2070
1706
  type PiHarnessDescriptor = {
2071
1707
  kind: 'pi';
1708
+ /**
1709
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1710
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1711
+ */
1712
+ gateway?: GatewayName;
2072
1713
  /** A Gemini model id, run on pi's `google` provider, e.g. `gemini-3.8-flash`. */
2073
1714
  model: string;
2074
1715
  /**
@@ -2092,6 +1733,11 @@ type PiHarnessDescriptor = {
2092
1733
  */
2093
1734
  type CodexHarnessDescriptor = {
2094
1735
  kind: 'codex';
1736
+ /**
1737
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1738
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1739
+ */
1740
+ gateway?: GatewayName;
2095
1741
  /** An OpenAI model id Codex runs on the Responses API, e.g. `gpt-5.6-luna`. */
2096
1742
  model: string;
2097
1743
  /**
@@ -2111,17 +1757,30 @@ type CodexHarnessDescriptor = {
2111
1757
  };
2112
1758
  /**
2113
1759
  * fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
2114
- * OpenAI's API by the model id, and reports each turn's tokens.
1760
+ * OpenAI's API by the model id, or on OpenRouter or Vertex AI, and reports each turn's tokens.
2115
1761
  */
2116
1762
  type FxHarnessDescriptor = {
2117
1763
  kind: 'fx';
1764
+ /**
1765
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1766
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1767
+ */
1768
+ gateway?: GatewayName;
2118
1769
  /**
2119
1770
  * A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
2120
1771
  * `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
2121
- * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-4.1`.
1772
+ * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`. On `openrouter()`, either
1773
+ * runs on OpenRouter's Chat Completions API with `OPENROUTER_API_KEY`, a bare id as `openai/<id>`.
1774
+ * On `vertex()`, a Gemini model id, `google/<id>` or a bare `gemini-…` id, run on Vertex AI's
1775
+ * Chat Completions endpoint, e.g. `google/gemini-3.8-flash`.
2122
1776
  */
2123
1777
  model: string;
2124
- /** Cell id; defaults to `fx:<model>`. */
1778
+ /**
1779
+ * Reasoning effort, sent as `reasoning_effort` on each model call; only on `vertex()`, since fx
1780
+ * sends none itself. Values are what the model takes there, e.g. `low`, `medium` or `high`.
1781
+ */
1782
+ effort?: string;
1783
+ /** Cell id; defaults to `fx:<model>`, or `fx:<model>[effort=<value>]` when `effort` is set. */
2125
1784
  id?: string;
2126
1785
  /** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
2127
1786
  maxConcurrent?: number;
@@ -2137,6 +1796,11 @@ type FxHarnessDescriptor = {
2137
1796
  */
2138
1797
  type GrokBuildHarnessDescriptor = {
2139
1798
  kind: 'grok-build';
1799
+ /**
1800
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1801
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1802
+ */
1803
+ gateway?: GatewayName;
2140
1804
  /** An xAI model id Grok Build runs on the xAI API, e.g. `grok-4.6`. */
2141
1805
  model: string;
2142
1806
  /**
@@ -2154,6 +1818,62 @@ type GrokBuildHarnessDescriptor = {
2154
1818
  */
2155
1819
  startTimeout?: Duration;
2156
1820
  };
1821
+ /**
1822
+ * A gateway the config lists by name: `direct` is each provider's own key, as without `gateway`;
1823
+ * `vercel` is Vercel AI Gateway; `openrouter` is OpenRouter; `vertex` is
1824
+ * Google Cloud's Vertex AI.
1825
+ */
1826
+ type GatewayName = 'direct' | 'vercel' | 'openrouter' | 'vertex';
1827
+ /**
1828
+ * Each provider's own key: made by `direct()` in `reeval/gateway`. Every `gateway` list ends with
1829
+ * it unless it's listed elsewhere, where it's tried in that place.
1830
+ */
1831
+ type DirectGatewayDescriptor = {
1832
+ kind: 'direct';
1833
+ /**
1834
+ * `false` turns off the fallback to each provider's own key, so a cell no other gateway in the
1835
+ * list serves is refused at plan time. Default `true`.
1836
+ */
1837
+ enabled?: boolean;
1838
+ };
1839
+ /**
1840
+ * Vercel AI Gateway, with `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`): made by `vercel()` in
1841
+ * `reeval/gateway`. It serves Claude Code, Codex, Goose, pi, OpenCode, fx and Grok Build.
1842
+ */
1843
+ type VercelGatewayDescriptor = {
1844
+ kind: 'vercel';
1845
+ };
1846
+ /**
1847
+ * OpenRouter, with `OPENROUTER_API_KEY`: made by `openrouter()` in `reeval/gateway`. It serves
1848
+ * Claude Code, Codex, Goose, OpenCode, fx, Grok Build and pi.
1849
+ */
1850
+ type OpenRouterGatewayDescriptor = {
1851
+ kind: 'openrouter';
1852
+ /**
1853
+ * Let OpenRouter fail over to another upstream provider when the one it picked can't serve a
1854
+ * call. Off by default, so reeval's own calls (the judge, analysis, compare questions and
1855
+ * captures) send `allow_fallbacks: false`. The harnesses' calls take OpenRouter's own routing
1856
+ * either way.
1857
+ */
1858
+ fallbacks?: boolean;
1859
+ };
1860
+ /** A model publisher on Vertex AI, which may serve its models in a region of its own. */
1861
+ type VertexPublisher = 'anthropic' | 'google' | 'xai';
1862
+ /**
1863
+ * Google Cloud's Vertex AI, on Application Default Credentials: made by `vertex()` in
1864
+ * `reeval/gateway`.
1865
+ */
1866
+ type VertexGatewayDescriptor = {
1867
+ kind: 'vertex';
1868
+ /** The Google Cloud project the calls are billed to; defaults to `GOOGLE_CLOUD_PROJECT`. */
1869
+ project?: string;
1870
+ /** The region the models are called in, e.g. `us-east5`; defaults to `global`. */
1871
+ location?: string;
1872
+ /** A region per publisher, where it differs from `location`, e.g. `{ anthropic: 'us-east5' }`. */
1873
+ regions?: Partial<Record<VertexPublisher, string>>;
1874
+ };
1875
+ /** One entry of the config's `gateway` list. */
1876
+ type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor | OpenRouterGatewayDescriptor | VertexGatewayDescriptor;
2157
1877
  /** A real coding agent, driven through its AI SDK harness adapter. */
2158
1878
  type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
2159
1879
  /** The agent an attempt runs: a real harness, or the fake one. */
@@ -2359,12 +2079,13 @@ type SetupStep = string | {
2359
2079
  cwd?: string;
2360
2080
  };
2361
2081
  /**
2362
- * Where the sandbox may connect: anywhere (`'allow-all'`, the default), nowhere (`'deny-all'`), or
2363
- * only the listed hosts (`'*.example.com'` matches subdomains). Under a restricted policy the
2364
- * harness's own hosts, brokered secrets' `hosts` and MCP servers' hosts are allowed too; any
2365
- * other request gets a refusal, and the attempt goes on.
2082
+ * Where the sandbox may connect: any host over HTTP(S) through the egress gateway (`'proxy'`, the
2083
+ * default), anywhere including raw TCP and UDP around the gateway (`'allow-all'`), nowhere
2084
+ * (`'deny-all'`), or only the listed hosts (`'*.example.com'` matches subdomains). Under a
2085
+ * restricted policy the harness's own hosts, brokered secrets' `hosts` and MCP servers' hosts are
2086
+ * allowed too; any other request gets a refusal, and the attempt goes on.
2366
2087
  */
2367
- type EgressPolicy = 'allow-all' | 'deny-all' | {
2088
+ type EgressPolicy = 'proxy' | 'allow-all' | 'deny-all' | {
2368
2089
  allow: readonly string[];
2369
2090
  };
2370
2091
  /**
@@ -2651,45 +2372,40 @@ type CompareOptions<E = Environment> = {
2651
2372
  minLift?: number;
2652
2373
  /** Harnesses (ids or descriptors) to compare on; the others run the control only. */
2653
2374
  harness?: Array<string | HarnessDescriptor>;
2654
- /**
2655
- * Asked of pairs of attempts after the run, always a list. They annotate the compare with
2656
- * win rates and never change pass, fail or the exit code.
2657
- */
2658
- questions?: CompareQuestion[];
2659
- /**
2660
- * The factor every question varies when pairing, when a question does not name its own
2661
- * `across`. `'environment'` holds the harness; `'harness'` holds the environment. A list of
2662
- * both, or omitted, asks both pair kinds.
2663
- */
2664
- across?: PairFactor | PairFactor[];
2665
2375
  };
2666
2376
  /**
2667
- * A question for pairwise judgement: its text, or the text and the `'provider/model'` that
2668
- * answers it. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the
2669
- * judge's when it's a language model.
2670
- * `across` is the factor it varies when pairing; without it (and without a one-factor
2671
- * `compare.across`) both pair kinds run. `state` and `artifacts` choose what each side shows.
2377
+ * A question a language model answers after the attempts: its text, or the text and its options.
2378
+ * With `across`, it picks between two attempts that differ only in that factor, in both orders;
2379
+ * without it, it's asked of each attempt on its own. Without a model, the cascaded
2380
+ * `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
2381
+ * `state` and `artifacts` choose what it's shown of each attempt.
2672
2382
  */
2673
- type CompareQuestion = string | {
2383
+ type Question = string | {
2674
2384
  question: string;
2675
2385
  /** `'provider/model'` that answers it, over the cascaded `question.model`. */
2676
2386
  model?: string;
2677
- /** The factor it varies when pairing. */
2678
- across?: PairFactor;
2679
- /** What each side shows, over the cascaded `question.state`, key by key. */
2387
+ /**
2388
+ * The factor it varies when pairing, or both: `'environment'` holds the harness,
2389
+ * `'harness'` holds the environment. Unset, it's asked of each attempt on its own.
2390
+ */
2391
+ across?: PairFactor | PairFactor[];
2392
+ /** The answers a question asked of each attempt picks from, with a short reason. */
2393
+ choices?: string[];
2394
+ /** What it's shown of each attempt, over the cascaded `question.state`, key by key. */
2680
2395
  state?: QuestionStateOptions;
2681
- /** Each side's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
2396
+ /** Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
2682
2397
  artifacts?: true | string[];
2683
2398
  };
2684
2399
  /**
2685
- * What each side of a compare question shows: the keys of `judge.state` but `contents` (the
2686
- * attempts' workspaces are gone when questions are asked). An unset key keeps its default, which
2687
- * is what a question shows with no `state` at all: the task and the diff.
2400
+ * What a question is shown of each attempt: the keys of `judge.state` but `contents` (the
2401
+ * attempts' workspaces are gone when questions are asked). An unset key keeps its default: a
2402
+ * pairwise question shows the task and the diff; one asked of each attempt also shows the final
2403
+ * message and the transcript.
2688
2404
  */
2689
2405
  type QuestionStateOptions = {
2690
2406
  /** The prompts the attempt was sent, once for both sides when they match. Default true. */
2691
2407
  task?: boolean;
2692
- /** The final message: the agent's own account of its work. Default false. */
2408
+ /** The final message: the agent's own account of its work. Default false in a pair, true per attempt. */
2693
2409
  output?: boolean;
2694
2410
  /** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
2695
2411
  files?: boolean;
@@ -2701,7 +2417,8 @@ type QuestionStateOptions = {
2701
2417
  /**
2702
2418
  * The transcript as text lines: `true` is messages and tool calls with tool output clipped,
2703
2419
  * `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
2704
- * harness's tools, so a pair of two harnesses isn't blind. Default false.
2420
+ * harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per
2421
+ * attempt.
2705
2422
  */
2706
2423
  transcript?: boolean | 'tools' | 'full';
2707
2424
  /**
@@ -2722,20 +2439,20 @@ type ResolvedQuestionStateOptions = {
2722
2439
  turns: 'last' | 'all';
2723
2440
  checks: boolean | string[];
2724
2441
  };
2725
- /** Defaults for every compare question: `model` and `artifacts` replaced, `state` merged key by key. */
2442
+ /** Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key. */
2726
2443
  type QuestionOptions = {
2727
2444
  /** `'provider/model'` that answers every question below it without its own `model`. */
2728
2445
  model?: string;
2729
- /** What each side shows. */
2446
+ /** What it's shown of each attempt. */
2730
2447
  state?: QuestionStateOptions;
2731
- /** Each side's own captures, by name; `true` sends all of them. Default none. */
2448
+ /** Each attempt's own captures, by name; `true` sends all of them. Default none. */
2732
2449
  artifacts?: true | string[];
2733
2450
  };
2734
- /** A matrix factor a compare question varies when pairing; the others are held. */
2451
+ /** A matrix factor a pairwise question varies; the others are held. */
2735
2452
  type PairFactor = 'environment' | 'harness';
2736
2453
  /** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
2737
2454
  type CompareCut = 'within' | 'across';
2738
- /** One judgement of one pair of attempts, in one order. */
2455
+ /** One judgment of one pair of attempts, in one order. */
2739
2456
  type PairVote = {
2740
2457
  question: string;
2741
2458
  cut: CompareCut;
@@ -2760,7 +2477,7 @@ type PairVote = {
2760
2477
  b: string[];
2761
2478
  };
2762
2479
  };
2763
- /** A compare question's answer: every vote, and each side's wins over decided votes per cut. */
2480
+ /** A pairwise question's answer: every vote, and each side's wins over decided votes per cut. */
2764
2481
  type CompareAnswer = {
2765
2482
  question: string;
2766
2483
  /** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
@@ -2784,6 +2501,28 @@ type CompareAnswer = {
2784
2501
  /** `questions/<n>.json` under the eval's directory: what the model was sent, per pair and order. */
2785
2502
  input?: FileRef;
2786
2503
  };
2504
+ /** A question asked of one attempt on its own, and the model's answer. Never gates. */
2505
+ type AttemptAnswer = {
2506
+ question: string;
2507
+ /** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
2508
+ model: string;
2509
+ /** The model id the provider says served the call, when it reports one. */
2510
+ resolvedModel?: string;
2511
+ /** What the attempt showed, every key resolved. */
2512
+ stateOptions: ResolvedQuestionStateOptions;
2513
+ /** The captures it sent, as asked: `true` for all, or their names. */
2514
+ artifacts?: true | string[];
2515
+ /** The answers it picked from; unset on a free-text question. */
2516
+ choices?: string[];
2517
+ /** One of `choices`; unset on a free-text question and when the call failed. */
2518
+ choice?: string;
2519
+ /** The model's answer, or its reason for `choice`; capped, and empty when the call failed. */
2520
+ text: string;
2521
+ /** Why there's no answer: the model call failed. Outcomes never change. */
2522
+ error?: string;
2523
+ /** `questions/<n>.json` under the attempt's directory: what the model was sent. */
2524
+ input?: FileRef;
2525
+ };
2787
2526
  /** An option a profile's `limit` can narrow, and `pin` can protect. */
2788
2527
  type PinnableKey = 'harness' | 'runs' | 'compare';
2789
2528
  /**
@@ -2807,7 +2546,11 @@ type EvalOptionFields = {
2807
2546
  * Not an environment: it does not invent skills, files or a name.
2808
2547
  */
2809
2548
  sandbox?: SandboxDescriptor;
2810
- /** Attempts per cell. */
2549
+ /**
2550
+ * Attempts per cell. Unset at every level, a compare with variants runs 4 a side, the fewest
2551
+ * whose lift can reach a verdict, and anything else runs once, a pooled `evaluate.each` case
2552
+ * included.
2553
+ */
2811
2554
  runs?: number;
2812
2555
  /** How each cell's attempts decide its outcome; default `'all'`. */
2813
2556
  gate?: GatePolicy;
@@ -2822,15 +2565,38 @@ type EvalOptionFields = {
2822
2565
  judge?: JudgeOptions;
2823
2566
  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
2824
2567
  capture?: CaptureOptions;
2825
- /** Defaults for every compare question: `model`, and `state` merged key by key, down the cascade. */
2568
+ /** Defaults for every question: `model`, and `state` merged key by key, down the cascade. */
2826
2569
  question?: QuestionOptions;
2570
+ /**
2571
+ * What a language model is asked about the attempts, always a list; they never change pass,
2572
+ * fail or the exit code. A question with `across` picks between pairs of attempts once they all
2573
+ * end; one without is asked of each attempt as it ends. Replaces the inherited list, so `[]`
2574
+ * asks none.
2575
+ */
2576
+ questions?: Question[];
2827
2577
  /** Per attempt. */
2828
2578
  timeout?: Duration;
2829
2579
  /** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
2830
2580
  tags?: string[];
2581
+ /**
2582
+ * The eval's version, part of its fingerprint: bump it when a change the eval file doesn't show
2583
+ * (a helper, a fixture, data it reads) changes what it measures. A group's applies to the evals
2584
+ * inside it that set none.
2585
+ */
2586
+ revision?: Revision;
2831
2587
  /** Keys a profile's `limit` may not narrow for this group or eval. */
2832
2588
  pin?: PinnableKey[];
2833
2589
  };
2590
+ /** An eval's `revision`: a whole number or a non-empty string, compared exactly. */
2591
+ type Revision = number | string;
2592
+ /**
2593
+ * An eval's own source as hashes: its body and case data as declared, and its options and its
2594
+ * enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
2595
+ */
2596
+ type EvalSource = DeclaredSource & {
2597
+ /** sha256 of the canonical options. */
2598
+ options: string;
2599
+ };
2834
2600
  /**
2835
2601
  * `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
2836
2602
  * A compare with a `baseline` names every arm, so the cascaded `environment` is not applied, and
@@ -2879,7 +2645,7 @@ type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
2879
2645
  type HistoryConfig = {
2880
2646
  /** Committed history folder (default: `.history` in the directory the evals live in). */
2881
2647
  dir?: string;
2882
- /** Artifacts recorded next to each run: every eval's, failing evals' only, or none. */
2648
+ /** Artifacts recorded next to each run: every attempt's, failed attempts' only, or none. */
2883
2649
  artifacts?: ArtifactLevel;
2884
2650
  };
2885
2651
  /** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
@@ -2911,7 +2677,7 @@ type ProfileDefaults = {
2911
2677
  timeout?: Duration;
2912
2678
  /** Screenshot defaults for `c.capture()`, shallow-merged over the config's. */
2913
2679
  capture?: CaptureOptions;
2914
- /** Defaults for every compare question, merged over the config's. */
2680
+ /** Defaults for every question, merged over the config's. */
2915
2681
  question?: QuestionOptions;
2916
2682
  };
2917
2683
  /** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
@@ -2939,6 +2705,8 @@ type Profile = {
2939
2705
  record?: RecordMode;
2940
2706
  /** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
2941
2707
  tags?: string[];
2708
+ /** Replaces the config's `gateway` list for this profile's runs. */
2709
+ gateway?: GatewayDescriptor[];
2942
2710
  };
2943
2711
  /** A canned analysis for tests; see `fakeAnalysis()` in `reeval/judge`. */
2944
2712
  type FakeAnalysisModel = {
@@ -2946,95 +2714,47 @@ type FakeAnalysisModel = {
2946
2714
  /** The model's structured answer, or a function of the digest it was given. */
2947
2715
  output?: unknown | ((digest: unknown) => unknown);
2948
2716
  /**
2949
- * How it answers compare questions: a fixed pick, a function of the blinded pair, or
2717
+ * How it answers pairwise questions: a fixed pick, a function of the blinded pair, or
2950
2718
  * `{ error }` to fail every call. A pick may be `{ winner, reason? }`. Default `'tie'`.
2951
2719
  */
2952
2720
  vote?: FakePick | ((pair: BlindPair) => FakePick) | {
2953
2721
  error: string;
2954
2722
  };
2955
2723
  /**
2956
- * What insights get back, in `reeval run` and `reeval analyze --insights`: fixed insights, a
2957
- * function of each call's input (`input.call` is `classify` or `insights`, once each per eval,
2958
- * or `headline` once; it may return a promise), or `{ error }` to fail every call. It counts a
2959
- * token per 4 characters of each call's input and answer, so `analysis.maxTokens` applies.
2724
+ * How it answers questions asked of each attempt: a fixed answer, a function of the attempt as
2725
+ * the model sees it, or `{ error }` to fail every call. An answer may be `{ choice?, text? }`.
2726
+ * Default the first of the question's `choices`, with the text `fake answer`.
2960
2727
  */
2961
- insights?: FakeInsights | ((input: FakeInsightsInput) => FakeInsights | Promise<FakeInsights>) | {
2728
+ answer?: FakeAnswer | ((attempt: BlindAttempt) => FakeAnswer) | {
2962
2729
  error: string;
2963
2730
  };
2964
2731
  };
2965
- /** What a fake insights function is given: one eval's scoped facts, or the run's evals for the headline. */
2966
- type FakeInsightsInput = ModelInsightInput | ModelHeadlineInput;
2967
- /**
2968
- * A fake insight answer. Each call takes its part: `intent` the classification, `answer`,
2969
- * `findings`, `rows`, `namedTools` and `remarks` the eval's, `headline` the run's. Whatever's
2970
- * left out is empty, and the intent defaults to `job: 'other'`.
2971
- */
2972
- type FakeInsights = {
2973
- intent?: {
2974
- job: 'choose' | 'finish' | 'other';
2975
- question?: string;
2976
- subject?: {
2977
- tool?: string;
2978
- change?: string;
2979
- };
2980
- name?: 'states-intent' | 'no-intent' | 'contradicts';
2981
- /** `low` falls back to `job: 'other'`, as a real classification does. */
2982
- confidence?: 'high' | 'medium' | 'low';
2983
- };
2984
- answer?: {
2985
- text: string;
2986
- cites: string[];
2987
- } | null;
2988
- findings?: Array<{
2989
- text: string;
2990
- cites: string[];
2991
- }>;
2992
- rows?: Array<{
2993
- row: string;
2994
- text: string;
2995
- cites: string[];
2996
- }>;
2997
- namedTools?: Array<{
2998
- attempt: string;
2999
- tools: string[];
3000
- }>;
3001
- /**
3002
- * The eval's remarks, as the model's JSON would have them: `source.at` and `fix.at` are
3003
- * `L<n>` lines of the eval's source, `evidence` fact ids (`F1`) or attempt lines (`A1#L3`).
3004
- * They go through the same gates as a model's, so one missing a part is dropped.
3005
- */
3006
- remarks?: Array<{
3007
- kind: string;
3008
- text: string;
3009
- source?: {
3010
- at: string;
3011
- quote: string;
3012
- } | null;
3013
- evidence?: string[];
3014
- fix?: {
3015
- at: string;
3016
- edit: string;
3017
- } | null;
3018
- }>;
3019
- headline?: {
3020
- text: string;
3021
- cites: string[];
3022
- } | null;
3023
- };
3024
2732
  /** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
3025
2733
  type FakeVote = 'A' | 'B' | 'tie';
3026
- /** A fake compare-question pick, optionally with a reason. */
2734
+ /** A fake pairwise pick, optionally with a reason. */
3027
2735
  type FakePick = FakeVote | {
3028
2736
  winner: FakeVote;
3029
2737
  reason?: string;
3030
2738
  };
3031
- /** What a compare question's model is shown: the prompt, and the two sides in slot order. */
2739
+ /** A fake per-attempt answer: its text, or a choice and its reason. */
2740
+ type FakeAnswer = string | {
2741
+ choice?: string;
2742
+ text?: string;
2743
+ };
2744
+ /** What a pairwise question's model is shown: the prompt, and the two sides in slot order. */
3032
2745
  type BlindPair = {
3033
2746
  question: string;
3034
2747
  prompt: string;
3035
2748
  a: BlindSide;
3036
2749
  b: BlindSide;
3037
2750
  };
2751
+ /** What a per-attempt question's model is shown: the prompt, its choices, and the attempt. */
2752
+ type BlindAttempt = {
2753
+ question: string;
2754
+ prompt: string;
2755
+ choices?: string[];
2756
+ attempt: BlindSide;
2757
+ };
3038
2758
  /** A turn of one side, with the keys its question's `state` turned on. */
3039
2759
  type BlindTurn = {
3040
2760
  task?: string;
@@ -3042,9 +2762,9 @@ type BlindTurn = {
3042
2762
  transcript?: string;
3043
2763
  };
3044
2764
  /**
3045
- * One side of a blinded pair: only the keys its question's `state` turned on. With
3046
- * `turns: 'all'`, `task`, `output` and `transcript` are under `turns`; a task both sides share is
3047
- * the pair's prompt's, not either side's.
2765
+ * One side of a blinded pair, or the attempt a per-attempt question is asked of: only the keys
2766
+ * its question's `state` turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
2767
+ * under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
3048
2768
  */
3049
2769
  type BlindSide = BlindTurn & {
3050
2770
  turns?: BlindTurn[];
@@ -3061,7 +2781,7 @@ type BlindSide = BlindTurn & {
3061
2781
  command: string;
3062
2782
  notRun: true;
3063
2783
  }>;
3064
- /** Its captures, in the order sent, labelled by position; a name it lacks is `missing`. */
2784
+ /** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
3065
2785
  artifacts?: Array<{
3066
2786
  label: string;
3067
2787
  mediaType: 'image/png';
@@ -3071,24 +2791,13 @@ type BlindSide = BlindTurn & {
3071
2791
  missing: true;
3072
2792
  }>;
3073
2793
  };
3074
- /** Insights and `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
2794
+ /** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
3075
2795
  type AnalysisConfig = {
3076
2796
  /**
3077
- * `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze`
3078
- * and compare questions fall back to the judge's when it's a language model.
2797
+ * `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Unset, `reeval analyze` and questions
2798
+ * fall back to the judge's when it's a language model.
3079
2799
  */
3080
2800
  model?: string | FakeAnalysisModel;
3081
- /**
3082
- * `reeval run` classifies each eval and explains its result with `model` once the attempts
3083
- * finish (default true when `model` is set; `--no-insights` turns it off for one run).
3084
- */
3085
- insights?: boolean;
3086
- /**
3087
- * Tokens (in and out) that one run's insights may use, over every classification, insight and
3088
- * headline call (default 250000). Past it, the evals left print without insights; the run's
3089
- * outcome and exit code never change.
3090
- */
3091
- maxTokens?: number;
3092
2801
  /** Added to the prompt, e.g. "Fewer tool calls count as a win. The skill under test is reeval-good." */
3093
2802
  instructions?: string;
3094
2803
  /** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
@@ -3108,13 +2817,13 @@ type ReevalConfig = {
3108
2817
  sandbox?: SandboxDescriptor;
3109
2818
  /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
3110
2819
  environment?: Environment;
3111
- /** Attempts per cell; default 1. */
2820
+ /** Attempts per cell; unset, a compare with variants runs 4 a side and anything else once. */
3112
2821
  runs?: number;
3113
2822
  /** How each cell's attempts decide its outcome; default `'all'`. */
3114
2823
  gate?: GatePolicy;
3115
2824
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
3116
2825
  env?: EnvVars;
3117
- /** Where the sandbox may connect; default `'allow-all'`. */
2826
+ /** Where the sandbox may connect; default `'proxy'`. */
3118
2827
  egress?: EgressPolicy;
3119
2828
  /** Defaults for `c.judge()`, merged field by field down the cascade. */
3120
2829
  judge?: ConfigJudgeOptions;
@@ -3125,7 +2834,7 @@ type ReevalConfig = {
3125
2834
  * binary when `c.capture()` runs; it does not change this object.
3126
2835
  */
3127
2836
  capture?: CaptureOptions;
3128
- /** Defaults for every compare question: the model that answers it and what each side shows. Merged down the cascade. */
2837
+ /** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
3129
2838
  question?: QuestionOptions;
3130
2839
  /** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
3131
2840
  concurrency?: number;
@@ -3142,7 +2851,7 @@ type ReevalConfig = {
3142
2851
  ci?: string;
3143
2852
  local?: string;
3144
2853
  };
3145
- /** `unbrokered: 'error'` refuses Docker secrets without `hosts` instead of warning (default `'warn'`). */
2854
+ /** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers written as plain strings, instead of warning (default `'warn'`). */
3146
2855
  secrets?: {
3147
2856
  unbrokered?: 'warn' | 'error';
3148
2857
  };
@@ -3154,8 +2863,14 @@ type ReevalConfig = {
3154
2863
  keep?: number;
3155
2864
  traceBufferMb?: number;
3156
2865
  };
3157
- /** Insights during `reeval run`, `reeval analyze`, and compare questions without a model. */
2866
+ /** `reeval analyze`, and questions without a model of their own. */
3158
2867
  analysis?: AnalysisConfig;
2868
+ /**
2869
+ * The gateways model calls may go through, in the order you prefer them: each harness takes
2870
+ * the first that can serve it. `direct()`, each provider's own key, ends the list unless it's listed
2871
+ * in another place or turned off with `direct({ enabled: false })`. Default `[direct()]`.
2872
+ */
2873
+ gateway?: GatewayDescriptor[];
3159
2874
  };
3160
2875
  /** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
3161
2876
  type ConfigHelpers = EnvironmentHelpers & {
@@ -3307,12 +3022,20 @@ type EnvironmentRecord = {
3307
3022
  sha256: string;
3308
3023
  }>;
3309
3024
  vars: string[];
3025
+ /**
3026
+ * Every secret, by the variable it's set as in the sandbox; an MCP header's (`secret.fromEnv`,
3027
+ * `secret.template`, a plugin's `${VAR}`) by each host variable it reads.
3028
+ */
3310
3029
  secrets: string[];
3311
- /** Secrets the sandbox only had placeholders for, swapped in by the egress gateway. */
3030
+ /**
3031
+ * Secrets the sandbox only had placeholders for, swapped in by the egress gateway, named as in
3032
+ * `secrets`. A variable it also got as its real value somewhere (an `http://` header, an env
3033
+ * secret without `hosts`) isn't listed.
3034
+ */
3312
3035
  brokered: string[];
3313
3036
  /** Optional secrets whose host variable was unset, so they weren't injected. */
3314
3037
  unset: string[];
3315
- /** The policy the attempt ran under, from the options and the environment; in the digest unless `allow-all`. */
3038
+ /** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
3316
3039
  egress: EgressPolicy;
3317
3040
  /**
3318
3041
  * The git repository the workspace started as: the repo as written (a local one relative to the
@@ -3331,15 +3054,22 @@ type EnvironmentRecord = {
3331
3054
  };
3332
3055
  /**
3333
3056
  * An attempt's network traffic. `allowed` is every host a restricted policy let through, its own
3334
- * and the automatic ones; null under `allow-all`. `recorded` is false when the sandbox can't see
3335
- * the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between the
3336
- * sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the harness
3337
- * always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands out.
3057
+ * and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
3058
+ * can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
3059
+ * the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
3060
+ * harness always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
3061
+ * out.
3338
3062
  */
3339
3063
  type EgressRecord = {
3340
3064
  policy: EgressPolicy;
3341
3065
  allowed: string[] | null;
3342
3066
  recorded: boolean;
3067
+ /**
3068
+ * True when traffic could leave around the gateway (`allow-all` in a container that kept its
3069
+ * network): `contacted` lists only what went through the gateway, and anything that ignored the
3070
+ * proxy (raw TCP and UDP, SSH, database drivers) went out unrecorded. Unset otherwise.
3071
+ */
3072
+ partial?: true;
3343
3073
  contacted: Array<{
3344
3074
  host: string;
3345
3075
  port: number;
@@ -3391,6 +3121,16 @@ type StartRetry = {
3391
3121
  /** How long it waited before giving up on that start. */
3392
3122
  waitedMs: number;
3393
3123
  };
3124
+ /**
3125
+ * A process the setup steps left running in a Docker sandbox with a `to: 'checks'` secret without
3126
+ * `hosts` in its environment: it holds the real value while the agent works.
3127
+ */
3128
+ type SetupProcess = {
3129
+ /** Its command line, cut at 200 characters. */
3130
+ command: string;
3131
+ /** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
3132
+ secrets: string[];
3133
+ };
3394
3134
  /** One `expect` or `expect.soft` matcher call an attempt made. */
3395
3135
  type AssertionRecord = {
3396
3136
  /**
@@ -3436,6 +3176,14 @@ type SandboxWarning = {
3436
3176
  /** The line reeval prints, after `WARNING: ` (root) or `warning: ` (capabilities). */
3437
3177
  message: string;
3438
3178
  };
3179
+ /** Where a cell's harness resolved in the `gateway` list, and the entries before it that couldn't serve it. */
3180
+ type ResolvedGateway = {
3181
+ name: GatewayName;
3182
+ skipped: Array<{
3183
+ gateway: GatewayName;
3184
+ why: string;
3185
+ }>;
3186
+ };
3439
3187
  //#endregion
3440
- export { GooseHarnessDescriptor as $, cellSamples as $n, fromModel as $t, DirectoryOptions as A, FamilyRate as An, Secret as At, EvalFn as B, DIRECTIONAL_MIN as Bn, Verdict as Bt, CompareCut as C, messageLine as Cn, PluginSource as Ct, ConfigJudgeOptions as D, CellRecord as Dn, ReevalConfig as Dt, ConfigHelpers as E, AttemptRecord as En, ProfileLimit as Et, EnvVars as F, JudgementRecord as Fn, TokenUsage as Ft, FakeHarnessDescriptor as G, EffortSample as Gn, INSIGHTS_FILE as Gt, EvalOptionsData as H, EFFORT_MIN_RUNS as Hn, Wait as Ht, Environment as I, Lift as In, TranscriptEvent as It, FakeJudgeModel as J, TOKENS_UNREPORTED as Jn, NO_MODEL as Jt, FakeInsights as K, EffortVerdict as Kn, Job as Kt, EnvironmentHelpers as L, RunRecord as Ln, Turn as Lt, Duration as M, FileRef as Mn, ServeHandle as Mt, EgressPolicy as N, HistoryCell as Nn, SetupStep as Nt, CursorHarnessDescriptor as O, EvalEntry as On, SandboxDescriptor as Ot, EgressRecord as P, HistoryEntry as Pn, SkillSource as Pt, FxHarnessDescriptor as Q, cellEffort as Qn, Sentence as Qt, EnvironmentInput as R, StackSummary as Rn, TurnData as Rt, CompareAnswer as S, factSegment as Sn, PluginPart as St, CompareQuestion as T, ArtifactRef as Tn, ProfileDefaults as Tt, ExecResult as U, EffortContrast as Un, WebEventType as Ut, EvalOptions as V, EFFORT_ALPHA as Vn, Viewport as Vt, FakeAnalysisModel as W, EffortMetric as Wn, EvalInsight as Wt, FakeScriptStep as X, bothUnderATenth as Xn, Remark as Xt, FakeSandboxDescriptor as Y, VERDICT_TEXT as Yn, REMARK_KINDS as Yt, FakeVote as Z, cellContrast as Zn, RunInsights as Zt, Cell as _, cancelledOf as _n, Outcome as _t, BackgroundHandle as a, CheckResult as an, effortSample as ar, HarnessOption as at, ClaudeCodeHarnessDescriptor as b, effectOf as bn, PiHarnessDescriptor as bt, Capture as c, EvalResult as cn, formatCount as cr, JudgeArtifactRef as ct, CaptureOptions as d, PooledArm as dn, tokenValues as dr, JudgeModel as dt, Across as en, contrastEffort as er, GrokBuildHarnessDescriptor as et, CaptureUse as f, QuestionResult as fn, JudgeOptions as ft, CaptureWindow as g, SuiteTally as gn, OpenCodeHarnessDescriptor as gt, CaptureWait as h, SuiteResult as hn, McpServer as ht, AttemptContext as i, CheckOutcome as in, effortOf as ir, HarnessEdit as it, DockerSandboxDescriptor as j, FamilyRecord as jn, SecretTemplate as jt, Diff as k, FamilyLift as kn, SandboxWarning as kt, CaptureCall as l, Interval as ln, formatRatio as lr, JudgeCallOptions as lt, CaptureUseScript as m, ResultOutcome as mn, JudgeStateOptions as mt, AnalysisConfig as n, CellError as nn, effortChangeLabel as nr, HarnessData as nt, BlindPair as o, Effect as on, effortSummary as or, HistoryConfig as ot, CaptureUseAction as p, ResultFile as pn, JudgeState as pt, FakeInsightsInput as q, Spread as qn, MODEL_REMARK_KINDS as qt, AntigravityHarnessDescriptor as r, CellResult as rn, effortLabel as rr, HarnessDescriptor as rt, Budget as s, EffortCounters as sn, effortVerdict as sr, HostFile as st, AgentHarnessDescriptor as t, CaseResult as tn, directional as tr, GroupOptions as tt, CaptureInstruction as u, JudgeResult as un, formatSeconds as ur, JudgeFailure as ut, CellRole as v, checkEffect as vn, PairFactor as vt, CompareOptions as w, suiteResult as wn, Profile as wt, CodexHarnessDescriptor as x, evalAcross as xn, PinnableKey as xt, ClaudeCodeEffort as y, countedOf as yn, PairVote as yt, EnvironmentSpec as z, CellEffort as zn, TurnStatus as zt };
3441
- //# sourceMappingURL=types-B-9b1MDV.d.ts.map
3188
+ export { FakeVote as $, VertexPublisher as $t, Diff as A, VERDICT_TEXT as An, Profile as At, EnvironmentInput as B, effortSample as Bn, Secret as Bt, CodexHarnessDescriptor as C, EFFORT_MIN_RUNS as Cn, Outcome as Ct, ConfigHelpers as D, EffortVerdict as Dn, PinnableKey as Dt, CompareOptions as E, EffortSample as En, PiHarnessDescriptor as Et, EgressPolicy as F, contrastEffort as Fn, QuestionStateOptions as Ft, ExecResult as G, formatSeconds as Gn, TokenUsage as Gt, ErrorKind as H, effortVerdict as Hn, ServeHandle as Ht, EgressRecord as I, directional as In, ReevalConfig as It, FakeAnswer as J, DeclaredNode as Jn, TurnData as Jt, ExecRetry as K, tokenValues as Kn, TranscriptEvent as Kt, EnvVars as L, effortChangeLabel as Ln, ResolvedGateway as Lt, DirectoryOptions as M, cellContrast as Mn, ProfileLimit as Mt, DockerSandboxDescriptor as N, cellEffort as Nn, Question as Nt, ConfigJudgeOptions as O, Spread as On, PluginPart as Ot, Duration as P, cellSamples as Pn, QuestionOptions as Pt, FakeScriptStep as Q, VertexGatewayDescriptor as Qt, Environment as R, effortLabel as Rn, SandboxDescriptor as Rt, ClaudeCodeHarnessDescriptor as S, EFFORT_ALPHA as Sn, OpenRouterGatewayDescriptor as St, CompareCut as T, EffortMetric as Tn, PairVote as Tt, EvalFn as U, formatCount as Un, SetupStep as Ut, EnvironmentSpec as V, effortSummary as Vn, SecretTemplate as Vt, EvalOptions as W, formatRatio as Wn, SkillSource as Wt, FakeJudgeModel as X, VercelGatewayDescriptor as Xt, FakeHarnessDescriptor as Y, TurnStatus as Yt, FakeSandboxDescriptor as Z, Verdict as Zt, CaptureWait as _, RunRecord as _n, JudgeOptions as _t, AttemptContext as a, CellRecord as an, GroupOptions as at, CellRole as b, CellEffort as bn, McpServer as bt, BlindPair as c, FamilyLift as cn, HarnessEdit as ct, CaptureCall as d, FamilyRecord as dn, HostFile as dt, Viewport as en, FxHarnessDescriptor as et, CaptureInstruction as f, FileRef as fn, JSONValue as ft, CaptureUseScript as g, Lift as gn, JudgeModel as gt, CaptureUseAction as h, JudgmentRecord as hn, JudgeFailure as ht, AttemptAnswer as i, AttemptRecord as in, GrokBuildHarnessDescriptor as it, DirectGatewayDescriptor as j, bothUnderATenth as jn, ProfileDefaults as jt, CursorHarnessDescriptor as k, TOKENS_UNREPORTED as kn, PluginSource as kt, Budget as l, FamilyLiftSummary as ln, HarnessOption as lt, CaptureUse as m, HistoryEntry as mn, JudgeCallOptions as mt, AnalysisConfig as n, WebEventType as nn, GatewayName as nt, BackgroundHandle as o, CheckTally as on, HarnessData as ot, CaptureOptions as p, HistoryCell as pn, JudgeArtifactRef as pt, FakeAnalysisModel as q, DeclarationTree as qn, Turn as qt, AntigravityHarnessDescriptor as r, ArtifactRef as rn, GooseHarnessDescriptor as rt, BlindAttempt as s, EvalEntry as sn, HarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, GatewayDescriptor as tt, Capture as u, FamilyRate as un, HistoryConfig as ut, CaptureWindow as v, StackSummary as vn, JudgeState as vt, CompareAnswer as w, EffortContrast as wn, PairFactor as wt, ClaudeCodeEffort as x, DIRECTIONAL_MIN as xn, OpenCodeHarnessDescriptor as xt, Cell as y, UpstreamCount as yn, JudgeStateOptions as yt, EnvironmentHelpers as z, effortOf as zn, SandboxWarning as zt };
3189
+ //# sourceMappingURL=types-coMWHaU2.d.ts.map