reeval 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (331) hide show
  1. package/README.md +117 -259
  2. package/THIRD_PARTY_NOTICES.md +121 -121
  3. package/assets/package.json +1 -1
  4. package/assets/pnpm-lock.yaml +41 -30
  5. package/dist/{adapter-DigeVFWC.js → adapter-BloGVZ_a.js} +20 -8
  6. package/dist/adapter-BloGVZ_a.js.map +1 -0
  7. package/dist/{agent-CatUEo5h.js → agent-m-_t-IuW.js} +54 -26
  8. package/dist/{agent-CatUEo5h.js.map → agent-m-_t-IuW.js.map} +1 -1
  9. package/dist/{analysis-Bxub9LMT.js → analysis-C2-aEbr7.js} +4 -4
  10. package/dist/{analysis-Bxub9LMT.js.map → analysis-C2-aEbr7.js.map} +1 -1
  11. package/dist/app-files-C-jwhRDt.js +11 -0
  12. package/dist/app-files-C-jwhRDt.js.map +1 -0
  13. package/dist/{assemble-CF063ms6.js → assemble-BDEpelpi.js} +95 -38
  14. package/dist/assemble-BDEpelpi.js.map +1 -0
  15. package/dist/assemble-C5bfncHU.js +1 -0
  16. package/dist/{attempt-CGPV3BZY.js → attempt-jI0OWxlz.js} +2 -2
  17. package/dist/{attempt-CGPV3BZY.js.map → attempt-jI0OWxlz.js.map} +1 -1
  18. package/dist/{capture-BiD7lVaM.js → capture-CVRwRTnb.js} +574 -38
  19. package/dist/capture-CVRwRTnb.js.map +1 -0
  20. package/dist/capture-CherbjzB.js +1 -0
  21. package/dist/{cli-B5zQCR2r.js → cli-C_4gaJ50.js} +145 -142
  22. package/dist/cli-C_4gaJ50.js.map +1 -0
  23. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
  24. package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
  25. package/dist/cli.js +2 -2
  26. package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
  27. package/dist/client-D5hNjpEx.js.map +1 -0
  28. package/dist/{command-C94CQNh8.js → command-BWAuwQNk.js} +12 -12
  29. package/dist/{command-C94CQNh8.js.map → command-BWAuwQNk.js.map} +1 -1
  30. package/dist/{command-CaZgCTIw.js → command-BXLUfw4h.js} +14 -14
  31. package/dist/command-BXLUfw4h.js.map +1 -0
  32. package/dist/{command-CVtgyqlM.js → command-Be6BSE2L.js} +12 -12
  33. package/dist/{command-CVtgyqlM.js.map → command-Be6BSE2L.js.map} +1 -1
  34. package/dist/{command-BMHXHqMT.js → command-Bxdy62dW.js} +15 -14
  35. package/dist/command-Bxdy62dW.js.map +1 -0
  36. package/dist/{command-DTgBZR0k.js → command-I1euLLio.js} +35 -15
  37. package/dist/command-I1euLLio.js.map +1 -0
  38. package/dist/{commands-CjpGBUWr.js → commands-jzigbNFb.js} +23 -14
  39. package/dist/commands-jzigbNFb.js.map +1 -0
  40. package/dist/{commands-BXzIDB3X.js → commands-p1S4nw_H.js} +1 -1
  41. package/dist/config-BrdHA9bJ.js +3549 -0
  42. package/dist/config-BrdHA9bJ.js.map +1 -0
  43. package/dist/{contrasts-CcoMLC_y.js → contrasts-DRxMQQTC.js} +3 -2
  44. package/dist/contrasts-DRxMQQTC.js.map +1 -0
  45. package/dist/cost-D4elx933.js +359 -0
  46. package/dist/cost-D4elx933.js.map +1 -0
  47. package/dist/cost-DmnAod9x.js +1 -0
  48. package/dist/db-CAfksinX.js +2 -0
  49. package/dist/{db-BeqRiWLl.js → db-DO0p7xgP.js} +25 -8
  50. package/dist/db-DO0p7xgP.js.map +1 -0
  51. package/dist/{dist-CMjp3qqF.js → dist-CYrBwyHA.js} +2 -2
  52. package/dist/{dist-CMjp3qqF.js.map → dist-CYrBwyHA.js.map} +1 -1
  53. package/dist/{docker-BbTlMmYi.js → docker-BnRlTpZA.js} +3 -3
  54. package/dist/{docker-BbTlMmYi.js.map → docker-BnRlTpZA.js.map} +1 -1
  55. package/dist/{docker-CCXdbREk.js → docker-D01slDF3.js} +29 -28
  56. package/dist/docker-D01slDF3.js.map +1 -0
  57. package/dist/{docker-DC69dKxB.js → docker-DivfZl2u.js} +1 -1
  58. package/dist/{doctor-CqttrZcs.js → doctor-DQ2p0euG.js} +136 -38
  59. package/dist/doctor-DQ2p0euG.js.map +1 -0
  60. package/dist/{environment-BBz1Yes9.js → environment-CO0P0soF.js} +31 -11
  61. package/dist/environment-CO0P0soF.js.map +1 -0
  62. package/dist/{evaluate-CXQ1o-pd.js → evaluate-BUmsN7yG.js} +68 -29
  63. package/dist/evaluate-BUmsN7yG.js.map +1 -0
  64. package/dist/{evaluate-yjZKUexW.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
  65. package/dist/{facts-DtENlf9s.js → facts-DzOUTreB.js} +13 -5
  66. package/dist/facts-DzOUTreB.js.map +1 -0
  67. package/dist/{files-B-eUpCZ1.js → files-B-CJeeEh.js} +14 -9
  68. package/dist/{files-B-eUpCZ1.js.map → files-B-CJeeEh.js.map} +1 -1
  69. package/dist/files-Bp5j5BqI.js +1 -0
  70. package/dist/files-C2sjmXhk.js +1 -0
  71. package/dist/{files-BsQbYz2i.js → files-C_gQNVB-.js} +4 -4
  72. package/dist/{files-BsQbYz2i.js.map → files-C_gQNVB-.js.map} +1 -1
  73. package/dist/{files-COHQLEHE.js → files-kUCrswI6.js} +3 -3
  74. package/dist/{files-COHQLEHE.js.map → files-kUCrswI6.js.map} +1 -1
  75. package/dist/gateway.d.ts +17 -0
  76. package/dist/gateway.js +17 -0
  77. package/dist/gateway.js.map +1 -0
  78. package/dist/{generate-Cs2TMH_k.js → generate-DZ7vp69F.js} +18 -18
  79. package/dist/generate-DZ7vp69F.js.map +1 -0
  80. package/dist/{git-DhykwgKF.js → git-Bk8DSwHy.js} +3 -3
  81. package/dist/{git-DhykwgKF.js.map → git-Bk8DSwHy.js.map} +1 -1
  82. package/dist/{harness-BAjw1fcr.js → harness-COVXSwJ6.js} +46 -15
  83. package/dist/harness-COVXSwJ6.js.map +1 -0
  84. package/dist/harness.d.ts +26 -18
  85. package/dist/harness.js +1 -1
  86. package/dist/{header-kSmWatcO.js → header-BLPP6aCU.js} +2 -2
  87. package/dist/{header-kSmWatcO.js.map → header-BLPP6aCU.js.map} +1 -1
  88. package/dist/{images-BmJRuTpm.js → images-DzLeFjLx.js} +8 -8
  89. package/dist/{images-BmJRuTpm.js.map → images-DzLeFjLx.js.map} +1 -1
  90. package/dist/index.d.ts +3 -45
  91. package/dist/index.js +5 -6
  92. package/dist/init-RgCQXZhb.js +1 -0
  93. package/dist/{init-Bwc9byQr.js → init-SRalM5xF.js} +6 -8
  94. package/dist/init-SRalM5xF.js.map +1 -0
  95. package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
  96. package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
  97. package/dist/{invocations-CQTbP092.js → invocations-DzyKSJBa.js} +3 -3
  98. package/dist/{invocations-CQTbP092.js.map → invocations-DzyKSJBa.js.map} +1 -1
  99. package/dist/judge-yLvjC5A3.js +385 -0
  100. package/dist/judge-yLvjC5A3.js.map +1 -0
  101. package/dist/judge.d.ts +7 -6
  102. package/dist/judge.js +7 -5
  103. package/dist/judge.js.map +1 -1
  104. package/dist/match-Bz0nSdzN.js +204 -0
  105. package/dist/match-Bz0nSdzN.js.map +1 -0
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
  108. package/dist/{model-B-mel-Gv.js → model-1ZLr9Dcv.js} +59 -35
  109. package/dist/model-1ZLr9Dcv.js.map +1 -0
  110. package/dist/model-B45lDUV1.js +68 -0
  111. package/dist/model-B45lDUV1.js.map +1 -0
  112. package/dist/model-BTZq8HIZ.js +1 -0
  113. package/dist/{model-CWlAqmbU.js → model-CgnOpss4.js} +58 -316
  114. package/dist/model-CgnOpss4.js.map +1 -0
  115. package/dist/model-Ch8kX87u.js +1 -0
  116. package/dist/{node-D0mtjRAO.js → node-BSq5qHnQ.js} +12 -5
  117. package/dist/node-BSq5qHnQ.js.map +1 -0
  118. package/dist/{opencode-providers-J9colP0-.js → opencode-providers-klkqgzvb.js} +72 -62
  119. package/dist/opencode-providers-klkqgzvb.js.map +1 -0
  120. package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
  121. package/dist/pairs-CHfZNRT6.js.map +1 -0
  122. package/dist/{patch-D4oluWzH.js → patch-DWf-I_Q_.js} +2 -2
  123. package/dist/{patch-D4oluWzH.js.map → patch-DWf-I_Q_.js.map} +1 -1
  124. package/dist/{plan-OkwUcJQ9.js → plan-BIwUzlB8.js} +63 -26
  125. package/dist/plan-BIwUzlB8.js.map +1 -0
  126. package/dist/{plugins-CUWMnSNb.js → plugins-Dkp_H3pB.js} +5 -5
  127. package/dist/{plugins-CUWMnSNb.js.map → plugins-Dkp_H3pB.js.map} +1 -1
  128. package/dist/{preflight-B6pY89bF.js → preflight-D9y-HvZQ.js} +1 -1
  129. package/dist/{preflight-Bjm7_ylE.js → preflight-DJCD_BDn.js} +3 -3
  130. package/dist/{preflight-Bjm7_ylE.js.map → preflight-DJCD_BDn.js.map} +1 -1
  131. package/dist/{prepare-DbdgtZKe.js → prepare-BJG_pD54.js} +2 -2
  132. package/dist/{prepare-CQmK3SBn.js → prepare-CIxCGG0M.js} +8 -8
  133. package/dist/{prepare-CQmK3SBn.js.map → prepare-CIxCGG0M.js.map} +1 -1
  134. package/dist/prices-BcLEXvdY.js +250 -0
  135. package/dist/prices-BcLEXvdY.js.map +1 -0
  136. package/dist/{provenance-B3Y17oCC.js → provenance-CfZpFpAh.js} +3 -3
  137. package/dist/{provenance-B3Y17oCC.js.map → provenance-CfZpFpAh.js.map} +1 -1
  138. package/dist/{providers-CXN2RFz-.js → providers-D1Zxp2he.js} +1 -1
  139. package/dist/{providers-eoniqK6w.js → providers-DFk_6W4K.js} +5 -5
  140. package/dist/{providers-eoniqK6w.js.map → providers-DFk_6W4K.js.map} +1 -1
  141. package/dist/{query-CLRZmWBT.js → query-CemclveB.js} +2 -2
  142. package/dist/{query-CLRZmWBT.js.map → query-CemclveB.js.map} +1 -1
  143. package/dist/query-Cvbwl804.js +1 -0
  144. package/dist/{questions-1XOulevV.js → questions-bAudIc04.js} +309 -47
  145. package/dist/questions-bAudIc04.js.map +1 -0
  146. package/dist/{redact-DJ9JCShu.js → redact-CRGdfV2s.js} +4 -2
  147. package/dist/redact-CRGdfV2s.js.map +1 -0
  148. package/dist/{remarks-DlJCqWRs.js → remarks-CXwSnSjR.js} +9 -9
  149. package/dist/{remarks-DlJCqWRs.js.map → remarks-CXwSnSjR.js.map} +1 -1
  150. package/dist/remarks-CyAu2DiB.js +1 -0
  151. package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
  152. package/dist/remarks-mujJv1Ey.js.map +1 -0
  153. package/dist/{remote-cE2VWfkJ.js → remote-CKcEKja4.js} +129 -174
  154. package/dist/remote-CKcEKja4.js.map +1 -0
  155. package/dist/resolve-B5-uqmJ6.js +1 -0
  156. package/dist/{resolve-C4zeccCH.js → resolve-CBoKzQMP.js} +9 -10
  157. package/dist/resolve-CBoKzQMP.js.map +1 -0
  158. package/dist/{results-Drtt0uFl.js → results-CMgZEANy.js} +8 -8
  159. package/dist/{results-Drtt0uFl.js.map → results-CMgZEANy.js.map} +1 -1
  160. package/dist/{resume-BEFazUfy.js → resume-B9SOSnK9.js} +3 -3
  161. package/dist/{resume-BEFazUfy.js.map → resume-B9SOSnK9.js.map} +1 -1
  162. package/dist/{root-DICDHCtp.js → root-Dlff-zp8.js} +5 -5
  163. package/dist/root-Dlff-zp8.js.map +1 -0
  164. package/dist/{rules-BFzQqPIm.js → rules-VhftPOOT.js} +53 -4
  165. package/dist/rules-VhftPOOT.js.map +1 -0
  166. package/dist/{run-BeMy92Us.js → run-CwIEK16q.js} +12 -12
  167. package/dist/run-CwIEK16q.js.map +1 -0
  168. package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
  169. package/dist/runtime-BgQRyCLl.js.map +1 -0
  170. package/dist/{sandbox-CQaJj7n9.js → sandbox-BOYZ0hlo.js} +2 -2
  171. package/dist/{sandbox-CQaJj7n9.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
  172. package/dist/sandbox.d.ts +2 -1
  173. package/dist/sandbox.js +1 -1
  174. package/dist/server-CHRSgKRp.js +1 -0
  175. package/dist/{server-xK0les4f.js → server-TGW4WyAz.js} +26 -27
  176. package/dist/{server-xK0les4f.js.map → server-TGW4WyAz.js.map} +1 -1
  177. package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
  178. package/dist/shared-BDUzStad.js.map +1 -0
  179. package/dist/shared-CHvO7eLi.d.ts +1 -0
  180. package/dist/{source-Q1hEKbCs.js → source-B1WvBIQt.js} +2 -2
  181. package/dist/{source-Q1hEKbCs.js.map → source-B1WvBIQt.js.map} +1 -1
  182. package/dist/source-DAzztZwU.js +1 -0
  183. package/dist/src-1jHbFaGW.js +1 -0
  184. package/dist/{src-LekQrr_S.js → src-8zoX_qrb.js} +614 -171
  185. package/dist/src-8zoX_qrb.js.map +1 -0
  186. package/dist/{src-C0CeKiLU.js → src-BYLM3hEz.js} +4 -3
  187. package/dist/src-BYLM3hEz.js.map +1 -0
  188. package/dist/{src-CFp86GtV.js → src-CHIwOy0o.js} +87 -55
  189. package/dist/{src-CFp86GtV.js.map → src-CHIwOy0o.js.map} +1 -1
  190. package/dist/src-C_9wgowC.js +2 -0
  191. package/dist/{src-C7_1SqDP.js → src-ChhtI6BP.js} +2 -2
  192. package/dist/{src-C7_1SqDP.js.map → src-ChhtI6BP.js.map} +1 -1
  193. package/dist/src-CyaDNG9r.js +2 -0
  194. package/dist/src-D005Cbqz.js +2 -0
  195. package/dist/src-DIfRbp5a.js +3 -0
  196. package/dist/{src-BhmaorOU.js → src-DKPVX2Us.js} +43 -17
  197. package/dist/{src-BhmaorOU.js.map → src-DKPVX2Us.js.map} +1 -1
  198. package/dist/src-Dg8121Xy.js +2 -0
  199. package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
  200. package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
  201. package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
  202. package/dist/src-DjRI-IeE.js.map +1 -0
  203. package/dist/{src-DiLMkIr9.js → src-DmUgeTlE.js} +47 -17
  204. package/dist/src-DmUgeTlE.js.map +1 -0
  205. package/dist/src-ElBqEN-S.js +2 -0
  206. package/dist/{src-qGoFkuJF.js → src-NSdQ9zI6.js} +88 -23
  207. package/dist/src-NSdQ9zI6.js.map +1 -0
  208. package/dist/src-YHGaUrGE.js +2 -0
  209. package/dist/src-mJVxw1Jm.js +2 -0
  210. package/dist/{state-BmZ8QVZ8.js → state-B1t143wG.js} +22 -20
  211. package/dist/{state-BmZ8QVZ8.js.map → state-B1t143wG.js.map} +1 -1
  212. package/dist/state-vznMh2wl.js +1 -0
  213. package/dist/stats-DSy5HfOv.js +315 -0
  214. package/dist/stats-DSy5HfOv.js.map +1 -0
  215. package/dist/suite-BsWGHzlG.js +2 -0
  216. package/dist/{suite-C8sw2pox.js → suite-WgT7m6Vs.js} +76 -13
  217. package/dist/suite-WgT7m6Vs.js.map +1 -0
  218. package/dist/{summary-0oQNGskv.js → summary-DIGEVr08.js} +58 -17
  219. package/dist/summary-DIGEVr08.js.map +1 -0
  220. package/dist/testing.d.ts +174 -2
  221. package/dist/testing.js +14 -3
  222. package/dist/testing.js.map +1 -0
  223. package/dist/types-C5vI3g1C.js.map +1 -1
  224. package/dist/{types-GraZtfHN.d.ts → types-DXBbP9Mk.d.ts} +302 -72
  225. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  226. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  227. package/dist/usage-DF3_p5pE.js +1 -0
  228. package/dist/{validate-BdoXUuL_.js → validate-BhwqrVsm.js} +48 -16
  229. package/dist/validate-BhwqrVsm.js.map +1 -0
  230. package/dist/view/index.d.ts +143 -91
  231. package/dist/view/index.js +7 -6
  232. package/dist/views-BW_C5eVH.js +2 -0
  233. package/dist/{views-BoUoX4_A.js → views-sXpGE6a2.js} +10 -8
  234. package/dist/views-sXpGE6a2.js.map +1 -0
  235. package/dist/{worker-entry-D0-GErmU.js → worker-entry-Duo9oAeT.js} +8 -8
  236. package/dist/{worker-entry-D0-GErmU.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
  237. package/dist/{workspace-tjZFkjxC.js → workspace-CDI68wtX.js} +45 -11
  238. package/dist/workspace-CDI68wtX.js.map +1 -0
  239. package/dist/workspace-QZCacP8A.js +1 -0
  240. package/package.json +8 -2
  241. package/skills/eval-rubric/SKILL.md +9 -0
  242. package/dist/adapter-DigeVFWC.js.map +0 -1
  243. package/dist/assemble-CF063ms6.js.map +0 -1
  244. package/dist/assemble-CJoD_Rzr.js +0 -1
  245. package/dist/assert-EUIoRdIt.js.map +0 -1
  246. package/dist/capture-BiD7lVaM.js.map +0 -1
  247. package/dist/capture-Dp3q7K7Y.js +0 -1
  248. package/dist/cli-B5zQCR2r.js.map +0 -1
  249. package/dist/client-fcwgwOAD.js +0 -106
  250. package/dist/client-fcwgwOAD.js.map +0 -1
  251. package/dist/command-BMHXHqMT.js.map +0 -1
  252. package/dist/command-CaZgCTIw.js.map +0 -1
  253. package/dist/command-DTgBZR0k.js.map +0 -1
  254. package/dist/commands-CjpGBUWr.js.map +0 -1
  255. package/dist/config-Dv5MG7Rp.js +0 -753
  256. package/dist/config-Dv5MG7Rp.js.map +0 -1
  257. package/dist/contrasts-CcoMLC_y.js.map +0 -1
  258. package/dist/db-BeqRiWLl.js.map +0 -1
  259. package/dist/db-CJDRVQal.js +0 -2
  260. package/dist/docker-CCXdbREk.js.map +0 -1
  261. package/dist/doctor-CqttrZcs.js.map +0 -1
  262. package/dist/egress-BfK_O9ce.js +0 -1241
  263. package/dist/egress-BfK_O9ce.js.map +0 -1
  264. package/dist/environment-BBz1Yes9.js.map +0 -1
  265. package/dist/evaluate-CXQ1o-pd.js.map +0 -1
  266. package/dist/facts-DtENlf9s.js.map +0 -1
  267. package/dist/files-DqCBW7i8.js +0 -1
  268. package/dist/files-Ha-jsQXf.js +0 -1
  269. package/dist/generate-Cs2TMH_k.js.map +0 -1
  270. package/dist/harness-BAjw1fcr.js.map +0 -1
  271. package/dist/init-Bwc9byQr.js.map +0 -1
  272. package/dist/init-D7Y1ny1z.js +0 -1
  273. package/dist/judge-Dt6KW6_O.js +0 -153
  274. package/dist/judge-Dt6KW6_O.js.map +0 -1
  275. package/dist/model-B-mel-Gv.js.map +0 -1
  276. package/dist/model-BGKYVpoj.js +0 -1
  277. package/dist/model-C7tSwT7f.js +0 -44
  278. package/dist/model-C7tSwT7f.js.map +0 -1
  279. package/dist/model-CWlAqmbU.js.map +0 -1
  280. package/dist/model-DHY-dfoO.js +0 -1
  281. package/dist/node-D0mtjRAO.js.map +0 -1
  282. package/dist/opencode-providers-J9colP0-.js.map +0 -1
  283. package/dist/options-CPUW97_7.js +0 -1116
  284. package/dist/options-CPUW97_7.js.map +0 -1
  285. package/dist/pairs-DP6XOJby.js.map +0 -1
  286. package/dist/plan-OkwUcJQ9.js.map +0 -1
  287. package/dist/query-CmBDYt0_.js +0 -1
  288. package/dist/questions-1XOulevV.js.map +0 -1
  289. package/dist/redact-DJ9JCShu.js.map +0 -1
  290. package/dist/remarks-1521zwfW.js.map +0 -1
  291. package/dist/remarks-BhAV1H4V.js +0 -1
  292. package/dist/remote-cE2VWfkJ.js.map +0 -1
  293. package/dist/resolve-C4zeccCH.js.map +0 -1
  294. package/dist/resolve-D30yZYmB.js +0 -1
  295. package/dist/root-DICDHCtp.js.map +0 -1
  296. package/dist/rules-BFzQqPIm.js.map +0 -1
  297. package/dist/run-BeMy92Us.js.map +0 -1
  298. package/dist/runtime-PX_oXiCS.js.map +0 -1
  299. package/dist/server-V4-2w0l8.js +0 -1
  300. package/dist/shared-CA61BuPJ.js.map +0 -1
  301. package/dist/source-BDow_oW8.js +0 -1
  302. package/dist/src-B9L9yGKI.js +0 -1
  303. package/dist/src-Bcah6ZkT.js +0 -3
  304. package/dist/src-C0CeKiLU.js.map +0 -1
  305. package/dist/src-C0xowZ3C.js +0 -2
  306. package/dist/src-CHIfIKUu.js +0 -2
  307. package/dist/src-DIL0jST0.js.map +0 -1
  308. package/dist/src-DLtyz4w5.js +0 -2
  309. package/dist/src-DPIjmReZ.js +0 -2
  310. package/dist/src-DXNh5mTr.js +0 -2
  311. package/dist/src-DiLMkIr9.js.map +0 -1
  312. package/dist/src-LekQrr_S.js.map +0 -1
  313. package/dist/src-bCnsA6Vp.js +0 -2
  314. package/dist/src-qGoFkuJF.js.map +0 -1
  315. package/dist/src-ynBVosfX.js +0 -2
  316. package/dist/state-BQR_BCm4.js +0 -1
  317. package/dist/state-CeR6oz0k.js +0 -66
  318. package/dist/state-CeR6oz0k.js.map +0 -1
  319. package/dist/studio-app/assets/index-CnYJijyI.js +0 -58
  320. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  321. package/dist/studio-app/favicon.svg +0 -1
  322. package/dist/studio-app/index.html +0 -15
  323. package/dist/suite-BaHPmAGy.js +0 -1
  324. package/dist/suite-C8sw2pox.js.map +0 -1
  325. package/dist/summary-0oQNGskv.js.map +0 -1
  326. package/dist/usage-BDo2q_yu.js +0 -1
  327. package/dist/validate-BdoXUuL_.js.map +0 -1
  328. package/dist/views-BoUoX4_A.js.map +0 -1
  329. package/dist/views-noVQNCUP.js +0 -2
  330. package/dist/workspace-B90YDU_a.js +0 -1
  331. package/dist/workspace-tjZFkjxC.js.map +0 -1
@@ -200,6 +200,8 @@ type JudgementRecord = {
200
200
  model: string;
201
201
  /** The model id the provider says served the call, when it reports one. */
202
202
  resolvedModel?: string;
203
+ /** The gateway from the config's `gateway` list the call went through; unset without one. */
204
+ gateway?: string;
203
205
  /**
204
206
  * `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
205
207
  * verdict can be followed across runs (and an edited question starts a new series).
@@ -257,6 +259,12 @@ type AttemptRecord = {
257
259
  * recorded them.
258
260
  */
259
261
  servedModels: string[] | null;
262
+ /**
263
+ * The models the harness's successful API calls asked for, when nothing named the one that
264
+ * served them (fx on Vercel AI Gateway with a Gemini model): the agent's word, not the
265
+ * gateway's, so never part of `servedModels`. Absent when there's none.
266
+ */
267
+ askedModels?: string[];
260
268
  /**
261
269
  * The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
262
270
  * null when the factory left it unset. Fast is not this.
@@ -320,11 +328,19 @@ type AttemptRecord = {
320
328
  judgements: JudgementRecord[];
321
329
  /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
322
330
  startRetries?: StartRetry[];
331
+ /**
332
+ * Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
333
+ * `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
334
+ */
323
335
  checks: Array<{
324
336
  command: string;
325
337
  exitCode: number;
326
338
  durationMs: number;
327
339
  log?: FileRef;
340
+ /** The `retries` it was given, when more than 0: it could run `retries + 1` times. */
341
+ retries?: number;
342
+ /** Every try, in order, when it was given `retries`; its log has each try's output. */
343
+ tries?: ExecTry[];
328
344
  }>;
329
345
  /** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
330
346
  assertions?: AssertionRecord[];
@@ -333,6 +349,13 @@ type AttemptRecord = {
333
349
  count?: number;
334
350
  }>;
335
351
  files: AttemptFiles;
352
+ /**
353
+ * The eval's questions asked of this attempt on its own (those without `across`), in question
354
+ * order; unset without any. They annotate: `outcome` never depends on them. Asked once the
355
+ * attempt ends, when it ran to a pass or a fail. Each answer's `input` is `questions/<n>.json`
356
+ * in the attempt's directory, `<n>` the question's place in the eval's `questions`.
357
+ */
358
+ answers?: AttemptAnswer[];
336
359
  };
337
360
  type EvalEntry = {
338
361
  /** `file › group › … › name`, as shown. */
@@ -367,8 +390,8 @@ type EvalEntry = {
367
390
  cells: CellRecord[];
368
391
  attempts: AttemptRecord[];
369
392
  /**
370
- * The compare's questions, answered after its attempts by pairwise votes, one per question;
371
- * unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
393
+ * The eval's pairwise questions (those with `across`), answered after its attempts by votes,
394
+ * one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
372
395
  * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
373
396
  * (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
374
397
  * `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
@@ -523,6 +546,8 @@ type FingerprintParts = {
523
546
  judge: string;
524
547
  /** Over the attempt's context files. */
525
548
  context: string;
549
+ /** The gateway its model calls went through; unset on `direct`, each provider's own key. */
550
+ gateway?: string;
526
551
  };
527
552
  /** One harness and environment of an eval, and how its attempts went. */
528
553
  type CellRecord = Cell & {
@@ -532,6 +557,11 @@ type CellRecord = Cell & {
532
557
  /** The environment's digest: the join key into `RunRecord.environments`; null for none. */
533
558
  environmentDigest: string | null;
534
559
  harnessKind: string;
560
+ /**
561
+ * The gateway in the config's `gateway` list its harness resolved to (`direct` is each
562
+ * provider's own key); unset when no config, profile or pin named one.
563
+ */
564
+ gateway?: string;
535
565
  /** The configured harness model, when it has one. */
536
566
  model: string | null;
537
567
  /**
@@ -661,6 +691,10 @@ type RunRecord = {
661
691
  attempts?: number;
662
692
  duration?: string | number;
663
693
  } | null;
694
+ /**
695
+ * The run's settings. `sandbox` is the id of each sandbox the planned attempts ran on, after the
696
+ * profile's defaults, joined with `, `.
697
+ */
664
698
  config: {
665
699
  sandbox: string;
666
700
  environment: string | null;
@@ -848,8 +882,6 @@ type Effect = 'better' | 'worse' | 'same' | 'unclear';
848
882
  * `harness` when it has several harnesses, `both`, or `none`.
849
883
  */
850
884
  type Across = 'none' | 'change' | 'harness' | 'both';
851
- /** The fewest counted attempts a side for an effect to be anything but `unclear`. */
852
- declare const EFFECT_MIN_COUNTED = 4;
853
885
  /**
854
886
  * One finished run's facts. Every node has a `path`, a stable id to cite it by, and `paths`
855
887
  * lists them all in the order the nodes appear.
@@ -883,7 +915,8 @@ type SuiteResult = {
883
915
  errors: string[];
884
916
  /**
885
917
  * Outcomes counted per block: each family case is one, each eval with no cases is one, and a
886
- * family whose pooled gate failed is one more fail. `errors` counts the run's `errors`.
918
+ * family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
919
+ * `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
887
920
  */
888
921
  tally: SuiteTally;
889
922
  /** Sorted by path. */
@@ -894,6 +927,7 @@ type SuiteResult = {
894
927
  type SuiteTally = {
895
928
  errors: number;
896
929
  pass: number;
930
+ notGated: number;
897
931
  fail: number;
898
932
  infraError: number;
899
933
  cancelled: number;
@@ -934,13 +968,14 @@ type EvalResult = {
934
968
  /** A family's variants against their baseline, pooled over the cases; null on a single eval. */
935
969
  pooled: PooledArm[] | null;
936
970
  /**
937
- * Its compare questions' answers. On a family, each question asked of any case, in first-asked
938
- * order, with its votes and wins summed over the cases that answered it.
971
+ * Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
972
+ * each question asked of any case, in first-asked order, summed over the cases that answered it.
939
973
  */
940
- questions: QuestionResult[];
974
+ questions: Array<QuestionResult | AttemptQuestionResult>;
941
975
  };
942
- /** A compare question: each side's wins over the pairs the model decided. Never gates. */
976
+ /** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
943
977
  type QuestionResult = {
978
+ kind: 'pairs';
944
979
  question: string;
945
980
  /** Pairs voted, each counted once per order it was asked in. */
946
981
  votes: number;
@@ -953,6 +988,27 @@ type QuestionResult = {
953
988
  /** Why some or all of its votes are missing: the model call failed. */
954
989
  error: string | null;
955
990
  };
991
+ /** A question asked of each attempt on its own: what each one answered. Never gates. */
992
+ type AttemptQuestionResult = {
993
+ kind: 'attempts';
994
+ question: string;
995
+ /** Attempts it was asked of, failed calls included. */
996
+ asked: number;
997
+ /** How many attempts picked each choice, in the question's order; null on a free-text question. */
998
+ choices: Array<{
999
+ choice: string;
1000
+ n: number;
1001
+ }> | null;
1002
+ /** Each attempt's answer, in attempt order; failed calls left out. */
1003
+ answers: Array<{
1004
+ attempt: string;
1005
+ choice: string | null;
1006
+ text: string;
1007
+ }>;
1008
+ /** Calls that failed, and the first one's error. */
1009
+ failed: number;
1010
+ error: string | null;
1011
+ };
956
1012
  type CaseResult = {
957
1013
  path: string;
958
1014
  /** The case's own eval id. */
@@ -1031,6 +1087,12 @@ type CheckResult = {
1031
1087
  effect: Effect | null;
1032
1088
  /** Set when the check is a judge verdict. */
1033
1089
  judge: JudgeResult | null;
1090
+ /**
1091
+ * A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
1092
+ * it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
1093
+ * when none did.
1094
+ */
1095
+ retried?: string;
1034
1096
  };
1035
1097
  /**
1036
1098
  * `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
@@ -1105,6 +1167,13 @@ declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'enviro
1105
1167
  * when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
1106
1168
  */
1107
1169
  declare function messageLine(text: string): string;
1170
+ /** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
1171
+ declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
1172
+ /**
1173
+ * A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
1174
+ * gating it, so it reads `not gated`, never `pass`.
1175
+ */
1176
+ declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
1108
1177
  /** The facts of a run, from its record. Pure: the same record always gives the same result. */
1109
1178
  declare function suiteResult(record: RunRecord): SuiteResult;
1110
1179
  //#endregion
@@ -1258,7 +1327,7 @@ declare const REMARK_KINDS: Record<Remark['kind'], {
1258
1327
  severity: 'concern' | 'suggestion';
1259
1328
  }>;
1260
1329
  /** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
1261
- declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "slow-mcp-start" | "too-few-tries")[];
1330
+ declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "proxy-only-egress" | "slow-mcp-start" | "too-few-tries")[];
1262
1331
  /** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
1263
1332
  type Remark = {
1264
1333
  /**
@@ -1268,7 +1337,7 @@ type Remark = {
1268
1337
  id: string;
1269
1338
  evalId: string;
1270
1339
  /** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
1271
- kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start';
1340
+ kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start' | 'proxy-only-egress';
1272
1341
  /** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
1273
1342
  severity: 'concern' | 'suggestion';
1274
1343
  /** One plain sentence: what's wrong. */
@@ -1494,6 +1563,18 @@ type ExecResult = {
1494
1563
  stdout: string;
1495
1564
  stderr: string;
1496
1565
  };
1566
+ /** How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient. */
1567
+ type ExecRetry = {
1568
+ /** Extra tries after a non-zero exit, a whole number from 0. Default 0. */
1569
+ retries?: number;
1570
+ /** Milliseconds to wait before each extra try. Default 1000. */
1571
+ interval?: number;
1572
+ };
1573
+ /** One run of a `c.exec` command that had `retries`. */
1574
+ type ExecTry = {
1575
+ exitCode: number;
1576
+ durationMs: number;
1577
+ };
1497
1578
  /** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
1498
1579
  type CaptureWindow = {
1499
1580
  /** In CSS pixels. */
@@ -1567,8 +1648,8 @@ type CaptureOptions = {
1567
1648
  /** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
1568
1649
  browser?: boolean;
1569
1650
  /**
1570
- * Vision model for `instructions`. Cascades like `question`. Default is the analysis
1571
- * model when that model accepts images; never Jev.
1651
+ * Vision model for `instructions`. Cascades like `question`. Default is `analysis.model`, else
1652
+ * the attempt's judge model, whichever is first a language model; never Jev.
1572
1653
  */
1573
1654
  model?: string;
1574
1655
  };
@@ -1663,7 +1744,7 @@ type TurnData = {
1663
1744
  };
1664
1745
  type Turn = TurnData & {
1665
1746
  /** Runs in the eval's sandbox. Same as `c.exec`. */
1666
- exec(command: string): Promise<ExecResult>;
1747
+ exec(command: string, options?: ExecRetry): Promise<ExecResult>;
1667
1748
  readFile(path: string): Promise<string | null>;
1668
1749
  };
1669
1750
  /** Any JSON value. */
@@ -1728,6 +1809,11 @@ type JudgeOptions = {
1728
1809
  * attempt's state. Merged key by key down the cascade.
1729
1810
  */
1730
1811
  state?: JudgeStateOptions;
1812
+ /**
1813
+ * The gateway the judge's calls go through, by name or by its helper. It skips the config's
1814
+ * `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
1815
+ */
1816
+ gateway?: GatewayName | GatewayDescriptor;
1731
1817
  };
1732
1818
  /**
1733
1819
  * What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
@@ -1796,6 +1882,8 @@ type Verdict = {
1796
1882
  model: string;
1797
1883
  /** The model id the provider says served the call, when it reports one. */
1798
1884
  resolvedModel?: string;
1885
+ /** The gateway from the config's `gateway` list the call went through; unset without one. */
1886
+ gateway?: GatewayName;
1799
1887
  /** Captures this call sent the judge as file parts, when any. */
1800
1888
  artifacts?: JudgeArtifactRef[];
1801
1889
  };
@@ -1820,11 +1908,17 @@ type AttemptContext = {
1820
1908
  * `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
1821
1909
  */
1822
1910
  judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1823
- exec(command: string): Promise<ExecResult>;
1911
+ /**
1912
+ * Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
1913
+ * many extra times, `interval` ms apart; it resolves with the last try.
1914
+ */
1915
+ exec(command: string, options?: ExecRetry & {
1916
+ background?: false;
1917
+ }): Promise<ExecResult>;
1824
1918
  exec(command: string, options: {
1825
1919
  background: true;
1826
1920
  }): Promise<BackgroundHandle>;
1827
- exec(command: string, options?: {
1921
+ exec(command: string, options?: ExecRetry & {
1828
1922
  background?: boolean;
1829
1923
  }): Promise<ExecResult | BackgroundHandle>;
1830
1924
  /**
@@ -1861,6 +1955,11 @@ type EvalFn = (c: AttemptContext) => Promise<void>;
1861
1955
  /** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
1862
1956
  type FakeHarnessDescriptor = {
1863
1957
  kind: 'fake';
1958
+ /**
1959
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
1960
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
1961
+ */
1962
+ gateway?: GatewayName;
1864
1963
  /** Cell id; defaults to `fake`. */
1865
1964
  id?: string;
1866
1965
  /** Files the fake agent writes into the sandbox on every turn. */
@@ -1947,6 +2046,11 @@ type FakeScriptStep = {
1947
2046
  */
1948
2047
  type CursorHarnessDescriptor = {
1949
2048
  kind: 'cursor';
2049
+ /**
2050
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
2051
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2052
+ */
2053
+ gateway?: GatewayName;
1950
2054
  /** A model id `Cursor.models.list()` returns for the key, e.g. `composer-2.5`. */
1951
2055
  model: string;
1952
2056
  /**
@@ -1974,6 +2078,11 @@ type ClaudeCodeEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
1974
2078
  /** Claude Code through `@ai-sdk/harness-claude-code`, made by `claudeCode()`. */
1975
2079
  type ClaudeCodeHarnessDescriptor = {
1976
2080
  kind: 'claude-code';
2081
+ /**
2082
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
2083
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2084
+ */
2085
+ gateway?: GatewayName;
1977
2086
  /** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
1978
2087
  model: string;
1979
2088
  /**
@@ -1994,6 +2103,11 @@ type ClaudeCodeHarnessDescriptor = {
1994
2103
  /** The Antigravity CLI through `@reeval/harness-antigravity`, made by `antigravity()`. */
1995
2104
  type AntigravityHarnessDescriptor = {
1996
2105
  kind: 'antigravity';
2106
+ /**
2107
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
2108
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2109
+ */
2110
+ gateway?: GatewayName;
1997
2111
  /** A model slug `agy models` lists for a Gemini API key, e.g. `gemini-3.8-flash-medium`. */
1998
2112
  model: string;
1999
2113
  /**
@@ -2018,6 +2132,11 @@ type AntigravityHarnessDescriptor = {
2018
2132
  */
2019
2133
  type OpenCodeHarnessDescriptor = {
2020
2134
  kind: 'opencode';
2135
+ /**
2136
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
2137
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2138
+ */
2139
+ gateway?: GatewayName;
2021
2140
  /**
2022
2141
  * OpenCode's model id, `<provider>/<model>`, e.g. `google/gemini-3.8-flash` or
2023
2142
  * `anthropic/claude-haiku-4-5`; the provider picks the key.
@@ -2044,6 +2163,11 @@ type OpenCodeHarnessDescriptor = {
2044
2163
  */
2045
2164
  type GooseHarnessDescriptor = {
2046
2165
  kind: 'goose';
2166
+ /**
2167
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
2168
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2169
+ */
2170
+ gateway?: GatewayName;
2047
2171
  /** A Gemini model id, run on Goose's `google` provider, e.g. `gemini-3.8-flash`. */
2048
2172
  model: string;
2049
2173
  /**
@@ -2067,6 +2191,11 @@ type GooseHarnessDescriptor = {
2067
2191
  */
2068
2192
  type PiHarnessDescriptor = {
2069
2193
  kind: 'pi';
2194
+ /**
2195
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
2196
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2197
+ */
2198
+ gateway?: GatewayName;
2070
2199
  /** A Gemini model id, run on pi's `google` provider, e.g. `gemini-3.8-flash`. */
2071
2200
  model: string;
2072
2201
  /**
@@ -2090,6 +2219,11 @@ type PiHarnessDescriptor = {
2090
2219
  */
2091
2220
  type CodexHarnessDescriptor = {
2092
2221
  kind: 'codex';
2222
+ /**
2223
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
2224
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2225
+ */
2226
+ gateway?: GatewayName;
2093
2227
  /** An OpenAI model id Codex runs on the Responses API, e.g. `gpt-5.6-luna`. */
2094
2228
  model: string;
2095
2229
  /**
@@ -2113,10 +2247,15 @@ type CodexHarnessDescriptor = {
2113
2247
  */
2114
2248
  type FxHarnessDescriptor = {
2115
2249
  kind: 'fx';
2250
+ /**
2251
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
2252
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2253
+ */
2254
+ gateway?: GatewayName;
2116
2255
  /**
2117
2256
  * A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
2118
2257
  * `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
2119
- * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-4.1`.
2258
+ * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`.
2120
2259
  */
2121
2260
  model: string;
2122
2261
  /** Cell id; defaults to `fx:<model>`. */
@@ -2135,6 +2274,11 @@ type FxHarnessDescriptor = {
2135
2274
  */
2136
2275
  type GrokBuildHarnessDescriptor = {
2137
2276
  kind: 'grok-build';
2277
+ /**
2278
+ * The gateway this harness's model calls go through, by name. It skips the config's `gateway`
2279
+ * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2280
+ */
2281
+ gateway?: GatewayName;
2138
2282
  /** An xAI model id Grok Build runs on the xAI API, e.g. `grok-4.6`. */
2139
2283
  model: string;
2140
2284
  /**
@@ -2152,6 +2296,24 @@ type GrokBuildHarnessDescriptor = {
2152
2296
  */
2153
2297
  startTimeout?: Duration;
2154
2298
  };
2299
+ /**
2300
+ * A gateway the config lists by name: `direct` is each provider's own key, as without `gateway`;
2301
+ * `vercel` is Vercel AI Gateway.
2302
+ */
2303
+ type GatewayName = 'direct' | 'vercel';
2304
+ /** Each provider's own key, as without `gateway`: made by `direct()` in `reeval/gateway`. */
2305
+ type DirectGatewayDescriptor = {
2306
+ kind: 'direct';
2307
+ };
2308
+ /**
2309
+ * Vercel AI Gateway, with `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`): made by `vercel()` in
2310
+ * `reeval/gateway`. It serves Claude Code, Codex, Goose, pi, OpenCode, fx and Grok Build.
2311
+ */
2312
+ type VercelGatewayDescriptor = {
2313
+ kind: 'vercel';
2314
+ };
2315
+ /** One entry of the config's `gateway` list. */
2316
+ type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor;
2155
2317
  /** A real coding agent, driven through its AI SDK harness adapter. */
2156
2318
  type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
2157
2319
  /** The agent an attempt runs: a real harness, or the fake one. */
@@ -2357,12 +2519,13 @@ type SetupStep = string | {
2357
2519
  cwd?: string;
2358
2520
  };
2359
2521
  /**
2360
- * Where the sandbox may connect: anywhere (`'allow-all'`, the default), nowhere (`'deny-all'`), or
2361
- * only the listed hosts (`'*.example.com'` matches subdomains). Under a restricted policy the
2362
- * harness's own hosts, brokered secrets' `hosts` and MCP servers' hosts are allowed too; any
2363
- * other request gets a refusal, and the attempt goes on.
2522
+ * Where the sandbox may connect: any host over HTTP(S) through the egress gateway (`'proxy'`, the
2523
+ * default), anywhere including raw TCP and UDP around the gateway (`'allow-all'`), nowhere
2524
+ * (`'deny-all'`), or only the listed hosts (`'*.example.com'` matches subdomains). Under a
2525
+ * restricted policy the harness's own hosts, brokered secrets' `hosts` and MCP servers' hosts are
2526
+ * allowed too; any other request gets a refusal, and the attempt goes on.
2364
2527
  */
2365
- type EgressPolicy = 'allow-all' | 'deny-all' | {
2528
+ type EgressPolicy = 'proxy' | 'allow-all' | 'deny-all' | {
2366
2529
  allow: readonly string[];
2367
2530
  };
2368
2531
  /**
@@ -2649,45 +2812,40 @@ type CompareOptions<E = Environment> = {
2649
2812
  minLift?: number;
2650
2813
  /** Harnesses (ids or descriptors) to compare on; the others run the control only. */
2651
2814
  harness?: Array<string | HarnessDescriptor>;
2652
- /**
2653
- * Asked of pairs of attempts after the run, always a list. They annotate the compare with
2654
- * win rates and never change pass, fail or the exit code.
2655
- */
2656
- questions?: CompareQuestion[];
2657
- /**
2658
- * The factor every question varies when pairing, when a question does not name its own
2659
- * `across`. `'environment'` holds the harness; `'harness'` holds the environment. A list of
2660
- * both, or omitted, asks both pair kinds.
2661
- */
2662
- across?: PairFactor | PairFactor[];
2663
2815
  };
2664
2816
  /**
2665
- * A question for pairwise judgement: its text, or the text and the `'provider/model'` that
2666
- * answers it. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the
2667
- * judge's when it's a language model.
2668
- * `across` is the factor it varies when pairing; without it (and without a one-factor
2669
- * `compare.across`) both pair kinds run. `state` and `artifacts` choose what each side shows.
2817
+ * A question a language model answers after the attempts: its text, or the text and its options.
2818
+ * With `across`, it picks between two attempts that differ only in that factor, in both orders;
2819
+ * without it, it's asked of each attempt on its own. Without a model, the cascaded
2820
+ * `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
2821
+ * `state` and `artifacts` choose what it's shown of each attempt.
2670
2822
  */
2671
- type CompareQuestion = string | {
2823
+ type Question = string | {
2672
2824
  question: string;
2673
2825
  /** `'provider/model'` that answers it, over the cascaded `question.model`. */
2674
2826
  model?: string;
2675
- /** The factor it varies when pairing. */
2676
- across?: PairFactor;
2677
- /** What each side shows, over the cascaded `question.state`, key by key. */
2827
+ /**
2828
+ * The factor it varies when pairing, or both: `'environment'` holds the harness,
2829
+ * `'harness'` holds the environment. Unset, it's asked of each attempt on its own.
2830
+ */
2831
+ across?: PairFactor | PairFactor[];
2832
+ /** The answers a question asked of each attempt picks from, with a short reason. */
2833
+ choices?: string[];
2834
+ /** What it's shown of each attempt, over the cascaded `question.state`, key by key. */
2678
2835
  state?: QuestionStateOptions;
2679
- /** Each side's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
2836
+ /** Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
2680
2837
  artifacts?: true | string[];
2681
2838
  };
2682
2839
  /**
2683
- * What each side of a compare question shows: the keys of `judge.state` but `contents` (the
2684
- * attempts' workspaces are gone when questions are asked). An unset key keeps its default, which
2685
- * is what a question shows with no `state` at all: the task and the diff.
2840
+ * What a question is shown of each attempt: the keys of `judge.state` but `contents` (the
2841
+ * attempts' workspaces are gone when questions are asked). An unset key keeps its default: a
2842
+ * pairwise question shows the task and the diff; one asked of each attempt also shows the final
2843
+ * message and the transcript.
2686
2844
  */
2687
2845
  type QuestionStateOptions = {
2688
2846
  /** The prompts the attempt was sent, once for both sides when they match. Default true. */
2689
2847
  task?: boolean;
2690
- /** The final message: the agent's own account of its work. Default false. */
2848
+ /** The final message: the agent's own account of its work. Default false in a pair, true per attempt. */
2691
2849
  output?: boolean;
2692
2850
  /** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
2693
2851
  files?: boolean;
@@ -2699,7 +2857,8 @@ type QuestionStateOptions = {
2699
2857
  /**
2700
2858
  * The transcript as text lines: `true` is messages and tool calls with tool output clipped,
2701
2859
  * `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
2702
- * harness's tools, so a pair of two harnesses isn't blind. Default false.
2860
+ * harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per
2861
+ * attempt.
2703
2862
  */
2704
2863
  transcript?: boolean | 'tools' | 'full';
2705
2864
  /**
@@ -2720,16 +2879,16 @@ type ResolvedQuestionStateOptions = {
2720
2879
  turns: 'last' | 'all';
2721
2880
  checks: boolean | string[];
2722
2881
  };
2723
- /** Defaults for every compare question: `model` and `artifacts` replaced, `state` merged key by key. */
2882
+ /** Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key. */
2724
2883
  type QuestionOptions = {
2725
2884
  /** `'provider/model'` that answers every question below it without its own `model`. */
2726
2885
  model?: string;
2727
- /** What each side shows. */
2886
+ /** What it's shown of each attempt. */
2728
2887
  state?: QuestionStateOptions;
2729
- /** Each side's own captures, by name; `true` sends all of them. Default none. */
2888
+ /** Each attempt's own captures, by name; `true` sends all of them. Default none. */
2730
2889
  artifacts?: true | string[];
2731
2890
  };
2732
- /** A matrix factor a compare question varies when pairing; the others are held. */
2891
+ /** A matrix factor a pairwise question varies; the others are held. */
2733
2892
  type PairFactor = 'environment' | 'harness';
2734
2893
  /** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
2735
2894
  type CompareCut = 'within' | 'across';
@@ -2758,7 +2917,7 @@ type PairVote = {
2758
2917
  b: string[];
2759
2918
  };
2760
2919
  };
2761
- /** A compare question's answer: every vote, and each side's wins over decided votes per cut. */
2920
+ /** A pairwise question's answer: every vote, and each side's wins over decided votes per cut. */
2762
2921
  type CompareAnswer = {
2763
2922
  question: string;
2764
2923
  /** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
@@ -2782,6 +2941,28 @@ type CompareAnswer = {
2782
2941
  /** `questions/<n>.json` under the eval's directory: what the model was sent, per pair and order. */
2783
2942
  input?: FileRef;
2784
2943
  };
2944
+ /** A question asked of one attempt on its own, and the model's answer. Never gates. */
2945
+ type AttemptAnswer = {
2946
+ question: string;
2947
+ /** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
2948
+ model: string;
2949
+ /** The model id the provider says served the call, when it reports one. */
2950
+ resolvedModel?: string;
2951
+ /** What the attempt showed, every key resolved. */
2952
+ stateOptions: ResolvedQuestionStateOptions;
2953
+ /** The captures it sent, as asked: `true` for all, or their names. */
2954
+ artifacts?: true | string[];
2955
+ /** The answers it picked from; unset on a free-text question. */
2956
+ choices?: string[];
2957
+ /** One of `choices`; unset on a free-text question and when the call failed. */
2958
+ choice?: string;
2959
+ /** The model's answer, or its reason for `choice`; capped, and empty when the call failed. */
2960
+ text: string;
2961
+ /** Why there's no answer: the model call failed. Outcomes never change. */
2962
+ error?: string;
2963
+ /** `questions/<n>.json` under the attempt's directory: what the model was sent. */
2964
+ input?: FileRef;
2965
+ };
2785
2966
  /** An option a profile's `limit` can narrow, and `pin` can protect. */
2786
2967
  type PinnableKey = 'harness' | 'runs' | 'compare';
2787
2968
  /**
@@ -2820,8 +3001,15 @@ type EvalOptionFields = {
2820
3001
  judge?: JudgeOptions;
2821
3002
  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
2822
3003
  capture?: CaptureOptions;
2823
- /** Defaults for every compare question: `model`, and `state` merged key by key, down the cascade. */
3004
+ /** Defaults for every question: `model`, and `state` merged key by key, down the cascade. */
2824
3005
  question?: QuestionOptions;
3006
+ /**
3007
+ * What a language model is asked about the attempts, always a list; they never change pass,
3008
+ * fail or the exit code. A question with `across` picks between pairs of attempts once they all
3009
+ * end; one without is asked of each attempt as it ends. Replaces the inherited list, so `[]`
3010
+ * asks none.
3011
+ */
3012
+ questions?: Question[];
2825
3013
  /** Per attempt. */
2826
3014
  timeout?: Duration;
2827
3015
  /** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
@@ -2909,7 +3097,7 @@ type ProfileDefaults = {
2909
3097
  timeout?: Duration;
2910
3098
  /** Screenshot defaults for `c.capture()`, shallow-merged over the config's. */
2911
3099
  capture?: CaptureOptions;
2912
- /** Defaults for every compare question, merged over the config's. */
3100
+ /** Defaults for every question, merged over the config's. */
2913
3101
  question?: QuestionOptions;
2914
3102
  };
2915
3103
  /** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
@@ -2937,6 +3125,8 @@ type Profile = {
2937
3125
  record?: RecordMode;
2938
3126
  /** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
2939
3127
  tags?: string[];
3128
+ /** Replaces the config's `gateway` list for this profile's runs. */
3129
+ gateway?: GatewayDescriptor[];
2940
3130
  };
2941
3131
  /** A canned analysis for tests; see `fakeAnalysis()` in `reeval/judge`. */
2942
3132
  type FakeAnalysisModel = {
@@ -2944,12 +3134,20 @@ type FakeAnalysisModel = {
2944
3134
  /** The model's structured answer, or a function of the digest it was given. */
2945
3135
  output?: unknown | ((digest: unknown) => unknown);
2946
3136
  /**
2947
- * How it answers compare questions: a fixed pick, a function of the blinded pair, or
3137
+ * How it answers pairwise questions: a fixed pick, a function of the blinded pair, or
2948
3138
  * `{ error }` to fail every call. A pick may be `{ winner, reason? }`. Default `'tie'`.
2949
3139
  */
2950
3140
  vote?: FakePick | ((pair: BlindPair) => FakePick) | {
2951
3141
  error: string;
2952
3142
  };
3143
+ /**
3144
+ * How it answers questions asked of each attempt: a fixed answer, a function of the attempt as
3145
+ * the model sees it, or `{ error }` to fail every call. An answer may be `{ choice?, text? }`.
3146
+ * Default the first of the question's `choices`, with the text `fake answer`.
3147
+ */
3148
+ answer?: FakeAnswer | ((attempt: BlindAttempt) => FakeAnswer) | {
3149
+ error: string;
3150
+ };
2953
3151
  /**
2954
3152
  * What insights get back, in `reeval run` and `reeval analyze --insights`: fixed insights, a
2955
3153
  * function of each call's input (`input.call` is `classify` or `insights`, once each per eval,
@@ -3021,18 +3219,30 @@ type FakeInsights = {
3021
3219
  };
3022
3220
  /** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
3023
3221
  type FakeVote = 'A' | 'B' | 'tie';
3024
- /** A fake compare-question pick, optionally with a reason. */
3222
+ /** A fake pairwise pick, optionally with a reason. */
3025
3223
  type FakePick = FakeVote | {
3026
3224
  winner: FakeVote;
3027
3225
  reason?: string;
3028
3226
  };
3029
- /** What a compare question's model is shown: the prompt, and the two sides in slot order. */
3227
+ /** A fake per-attempt answer: its text, or a choice and its reason. */
3228
+ type FakeAnswer = string | {
3229
+ choice?: string;
3230
+ text?: string;
3231
+ };
3232
+ /** What a pairwise question's model is shown: the prompt, and the two sides in slot order. */
3030
3233
  type BlindPair = {
3031
3234
  question: string;
3032
3235
  prompt: string;
3033
3236
  a: BlindSide;
3034
3237
  b: BlindSide;
3035
3238
  };
3239
+ /** What a per-attempt question's model is shown: the prompt, its choices, and the attempt. */
3240
+ type BlindAttempt = {
3241
+ question: string;
3242
+ prompt: string;
3243
+ choices?: string[];
3244
+ attempt: BlindSide;
3245
+ };
3036
3246
  /** A turn of one side, with the keys its question's `state` turned on. */
3037
3247
  type BlindTurn = {
3038
3248
  task?: string;
@@ -3040,9 +3250,9 @@ type BlindTurn = {
3040
3250
  transcript?: string;
3041
3251
  };
3042
3252
  /**
3043
- * One side of a blinded pair: only the keys its question's `state` turned on. With
3044
- * `turns: 'all'`, `task`, `output` and `transcript` are under `turns`; a task both sides share is
3045
- * the pair's prompt's, not either side's.
3253
+ * One side of a blinded pair, or the attempt a per-attempt question is asked of: only the keys
3254
+ * its question's `state` turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
3255
+ * under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
3046
3256
  */
3047
3257
  type BlindSide = BlindTurn & {
3048
3258
  turns?: BlindTurn[];
@@ -3073,7 +3283,7 @@ type BlindSide = BlindTurn & {
3073
3283
  type AnalysisConfig = {
3074
3284
  /**
3075
3285
  * `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze`
3076
- * and compare questions fall back to the judge's when it's a language model.
3286
+ * and questions fall back to the judge's when it's a language model.
3077
3287
  */
3078
3288
  model?: string | FakeAnalysisModel;
3079
3289
  /**
@@ -3112,7 +3322,7 @@ type ReevalConfig = {
3112
3322
  gate?: GatePolicy;
3113
3323
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
3114
3324
  env?: EnvVars;
3115
- /** Where the sandbox may connect; default `'allow-all'`. */
3325
+ /** Where the sandbox may connect; default `'proxy'`. */
3116
3326
  egress?: EgressPolicy;
3117
3327
  /** Defaults for `c.judge()`, merged field by field down the cascade. */
3118
3328
  judge?: ConfigJudgeOptions;
@@ -3123,7 +3333,7 @@ type ReevalConfig = {
3123
3333
  * binary when `c.capture()` runs; it does not change this object.
3124
3334
  */
3125
3335
  capture?: CaptureOptions;
3126
- /** Defaults for every compare question: the model that answers it and what each side shows. Merged down the cascade. */
3336
+ /** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
3127
3337
  question?: QuestionOptions;
3128
3338
  /** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
3129
3339
  concurrency?: number;
@@ -3152,8 +3362,13 @@ type ReevalConfig = {
3152
3362
  keep?: number;
3153
3363
  traceBufferMb?: number;
3154
3364
  };
3155
- /** Insights during `reeval run`, `reeval analyze`, and compare questions without a model. */
3365
+ /** Insights during `reeval run`, `reeval analyze`, and questions without a model. */
3156
3366
  analysis?: AnalysisConfig;
3367
+ /**
3368
+ * The gateways model calls may go through, in the order you prefer them: each harness takes
3369
+ * the first that can serve it. Default `[direct()]`, each provider's own key.
3370
+ */
3371
+ gateway?: GatewayDescriptor[];
3157
3372
  };
3158
3373
  /** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
3159
3374
  type ConfigHelpers = EnvironmentHelpers & {
@@ -3310,7 +3525,7 @@ type EnvironmentRecord = {
3310
3525
  brokered: string[];
3311
3526
  /** Optional secrets whose host variable was unset, so they weren't injected. */
3312
3527
  unset: string[];
3313
- /** The policy the attempt ran under, from the options and the environment; in the digest unless `allow-all`. */
3528
+ /** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
3314
3529
  egress: EgressPolicy;
3315
3530
  /**
3316
3531
  * The git repository the workspace started as: the repo as written (a local one relative to the
@@ -3329,15 +3544,22 @@ type EnvironmentRecord = {
3329
3544
  };
3330
3545
  /**
3331
3546
  * An attempt's network traffic. `allowed` is every host a restricted policy let through, its own
3332
- * and the automatic ones; null under `allow-all`. `recorded` is false when the sandbox can't see
3333
- * the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between the
3334
- * sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the harness
3335
- * always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands out.
3547
+ * and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
3548
+ * can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
3549
+ * the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
3550
+ * harness always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
3551
+ * out.
3336
3552
  */
3337
3553
  type EgressRecord = {
3338
3554
  policy: EgressPolicy;
3339
3555
  allowed: string[] | null;
3340
3556
  recorded: boolean;
3557
+ /**
3558
+ * True when traffic could leave around the gateway (`allow-all` in a container that kept its
3559
+ * network): `contacted` lists only what went through the gateway, and anything that ignored the
3560
+ * proxy (raw TCP and UDP, SSH, database drivers) went out unrecorded. Unset otherwise.
3561
+ */
3562
+ partial?: true;
3341
3563
  contacted: Array<{
3342
3564
  host: string;
3343
3565
  port: number;
@@ -3434,6 +3656,14 @@ type SandboxWarning = {
3434
3656
  /** The line reeval prints, after `WARNING: ` (root) or `warning: ` (capabilities). */
3435
3657
  message: string;
3436
3658
  };
3659
+ /** Where a cell's harness resolved in the `gateway` list, and the entries before it that couldn't serve it. */
3660
+ type ResolvedGateway = {
3661
+ name: GatewayName;
3662
+ skipped: Array<{
3663
+ gateway: GatewayName;
3664
+ why: string;
3665
+ }>;
3666
+ };
3437
3667
  //#endregion
3438
- export { GooseHarnessDescriptor as $, cellEffort as $n, fromModel as $t, DirectoryOptions as A, FamilyLift as An, Secret as At, EvalFn as B, CellEffort as Bn, Verdict as Bt, CompareCut as C, factSegment as Cn, PluginSource as Ct, ConfigJudgeOptions as D, AttemptRecord as Dn, ReevalConfig as Dt, ConfigHelpers as E, ArtifactRef as En, ProfileLimit as Et, EnvVars as F, HistoryEntry as Fn, TokenUsage as Ft, FakeHarnessDescriptor as G, EffortMetric as Gn, INSIGHTS_FILE as Gt, EvalOptionsData as H, EFFORT_ALPHA as Hn, Wait as Ht, Environment as I, JudgementRecord as In, TranscriptEvent as It, FakeJudgeModel as J, Spread as Jn, NO_MODEL as Jt, FakeInsights as K, EffortSample as Kn, Job as Kt, EnvironmentHelpers as L, Lift as Ln, Turn as Lt, Duration as M, FamilyRecord as Mn, ServeHandle as Mt, EgressPolicy as N, FileRef as Nn, SetupStep as Nt, CursorHarnessDescriptor as O, CellRecord as On, SandboxDescriptor as Ot, EgressRecord as P, HistoryCell as Pn, SkillSource as Pt, FxHarnessDescriptor as Q, cellContrast as Qn, Sentence as Qt, EnvironmentInput as R, RunRecord as Rn, TurnData as Rt, CompareAnswer as S, evalAcross as Sn, PluginPart as St, CompareQuestion as T, suiteResult as Tn, ProfileDefaults as Tt, ExecResult as U, EFFORT_MIN_RUNS as Un, WebEventType as Ut, EvalOptions as V, DIRECTIONAL_MIN as Vn, Viewport as Vt, FakeAnalysisModel as W, EffortContrast as Wn, EvalInsight as Wt, FakeScriptStep as X, VERDICT_TEXT as Xn, Remark as Xt, FakeSandboxDescriptor as Y, TOKENS_UNREPORTED as Yn, REMARK_KINDS as Yt, FakeVote as Z, bothUnderATenth as Zn, RunInsights as Zt, Cell as _, SuiteTally as _n, Outcome as _t, BackgroundHandle as a, CheckResult as an, effortOf as ar, HarnessOption as at, ClaudeCodeHarnessDescriptor as b, countedOf as bn, PiHarnessDescriptor as bt, Capture as c, EffortCounters as cn, effortVerdict as cr, JudgeArtifactRef as ct, CaptureOptions as d, JudgeResult as dn, formatSeconds as dr, JudgeModel as dt, Across as en, cellSamples as er, GrokBuildHarnessDescriptor as et, CaptureUse as f, PooledArm as fn, tokenValues as fr, JudgeOptions as ft, CaptureWindow as g, SuiteResult as gn, OpenCodeHarnessDescriptor as gt, CaptureWait as h, ResultOutcome as hn, McpServer as ht, AttemptContext as i, CheckOutcome as in, effortLabel as ir, HarnessEdit as it, DockerSandboxDescriptor as j, FamilyRate as jn, SecretTemplate as jt, Diff as k, EvalEntry as kn, SandboxWarning as kt, CaptureCall as l, EvalResult as ln, formatCount as lr, JudgeCallOptions as lt, CaptureUseScript as m, ResultFile as mn, JudgeStateOptions as mt, AnalysisConfig as n, CellError as nn, directional as nr, HarnessData as nt, BlindPair as o, EFFECT_MIN_COUNTED as on, effortSample as or, HistoryConfig as ot, CaptureUseAction as p, QuestionResult as pn, JudgeState as pt, FakeInsightsInput as q, EffortVerdict as qn, MODEL_REMARK_KINDS as qt, AntigravityHarnessDescriptor as r, CellResult as rn, effortChangeLabel as rr, HarnessDescriptor as rt, Budget as s, Effect as sn, effortSummary as sr, HostFile as st, AgentHarnessDescriptor as t, CaseResult as tn, contrastEffort as tr, GroupOptions as tt, CaptureInstruction as u, Interval as un, formatRatio as ur, JudgeFailure as ut, CellRole as v, cancelledOf as vn, PairFactor as vt, CompareOptions as w, messageLine as wn, Profile as wt, CodexHarnessDescriptor as x, effectOf as xn, PinnableKey as xt, ClaudeCodeEffort as y, checkEffect as yn, PairVote as yt, EnvironmentSpec as z, StackSummary as zn, TurnStatus as zt };
3439
- //# sourceMappingURL=types-GraZtfHN.d.ts.map
3668
+ export { FakeJudgeModel as $, RunRecord as $n, Verdict as $t, CursorHarnessDescriptor as A, SuiteTally as An, PluginPart as At, EnvironmentHelpers as B, suiteResult as Bn, SandboxDescriptor as Bt, ClaudeCodeHarnessDescriptor as C, Interval as Cn, effortVerdict as Cr, McpServer as Ct, CompareOptions as D, ResultFile as Dn, tokenValues as Dr, PairVote as Dt, CompareCut as E, QuestionResult as En, formatSeconds as Er, PairFactor as Et, Duration as F, evalAcross as Fn, Question as Ft, EvalOptionsData as G, FamilyLift as Gn, SetupStep as Gt, EnvironmentSpec as H, AttemptRecord as Hn, Secret as Ht, EgressPolicy as I, factSegment as In, QuestionOptions as It, FakeAnalysisModel as J, FileRef as Jn, TranscriptEvent as Jt, ExecResult as K, FamilyRate as Kn, SkillSource as Kt, EgressRecord as L, isNotGatedResult as Ln, QuestionStateOptions as Lt, DirectGatewayDescriptor as M, checkEffect as Mn, Profile as Mt, DirectoryOptions as N, countedOf as Nn, ProfileDefaults as Nt, ConfigHelpers as O, ResultOutcome as On, PiHarnessDescriptor as Ot, DockerSandboxDescriptor as P, effectOf as Pn, ProfileLimit as Pt, FakeInsightsInput as Q, Lift as Qn, VercelGatewayDescriptor as Qt, EnvVars as R, isStudy as Rn, ReevalConfig as Rt, ClaudeCodeEffort as S, EvalResult as Sn, effortSummary as Sr, JudgeStateOptions as St, CompareAnswer as T, PooledArm as Tn, formatRatio as Tr, Outcome as Tt, EvalFn as U, CellRecord as Un, SecretTemplate as Ut, EnvironmentInput as V, ArtifactRef as Vn, SandboxWarning as Vt, EvalOptions as W, EvalEntry as Wn, ServeHandle as Wt, FakeHarnessDescriptor as X, HistoryEntry as Xn, TurnData as Xt, FakeAnswer as Y, HistoryCell as Yn, Turn as Yt, FakeInsights as Z, JudgementRecord as Zn, TurnStatus as Zt, CaptureUseScript as _, CellResult as _n, directional as _r, JudgeCallOptions as _t, AttemptAnswer as a, Job as an, EffortContrast as ar, GatewayName as at, Cell as b, Effect as bn, effortOf as br, JudgeOptions as bt, BlindAttempt as c, REMARK_KINDS as cn, EffortVerdict as cr, GroupOptions as ct, Capture as d, Sentence as dn, VERDICT_TEXT as dr, HarnessEdit as dt, Viewport as en, StackSummary as er, FakeSandboxDescriptor as et, CaptureCall as f, fromModel as fn, bothUnderATenth as fr, HarnessOption as ft, CaptureUseAction as g, CellError as gn, contrastEffort as gr, JudgeArtifactRef as gt, CaptureUse as h, CaseResult as hn, cellSamples as hr, JSONValue as ht, AssertionRecord as i, INSIGHTS_FILE as in, EFFORT_MIN_RUNS as ir, GatewayDescriptor as it, Diff as j, cancelledOf as jn, PluginSource as jt, ConfigJudgeOptions as k, SuiteResult as kn, PinnableKey as kt, BlindPair as l, Remark as ln, Spread as lr, HarnessData as lt, CaptureOptions as m, AttemptQuestionResult as mn, cellEffort as mr, HostFile as mt, AnalysisConfig as n, WebEventType as nn, DIRECTIONAL_MIN as nr, FakeVote as nt, AttemptContext as o, MODEL_REMARK_KINDS as on, EffortMetric as or, GooseHarnessDescriptor as ot, CaptureInstruction as p, Across as pn, cellContrast as pr, HistoryConfig as pt, ExecRetry as q, FamilyRecord as qn, TokenUsage as qt, AntigravityHarnessDescriptor as r, EvalInsight as rn, EFFORT_ALPHA as rr, FxHarnessDescriptor as rt, BackgroundHandle as s, NO_MODEL as sn, EffortSample as sr, GrokBuildHarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, CellEffort as tr, FakeScriptStep as tt, Budget as u, RunInsights as un, TOKENS_UNREPORTED as ur, HarnessDescriptor as ut, CaptureWait as v, CheckOutcome as vn, effortChangeLabel as vr, JudgeFailure as vt, CodexHarnessDescriptor as w, JudgeResult as wn, formatCount as wr, OpenCodeHarnessDescriptor as wt, CellRole as x, EffortCounters as xn, effortSample as xr, JudgeState as xt, CaptureWindow as y, CheckResult as yn, effortLabel as yr, JudgeModel as yt, Environment as z, messageLine as zn, ResolvedGateway as zt };
3669
+ //# sourceMappingURL=types-DXBbP9Mk.d.ts.map