reeval 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (380) hide show
  1. package/README.md +1 -1
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-CptK032x.js → adapter-C60z3wLq.js} +5 -4
  4. package/dist/adapter-C60z3wLq.js.map +1 -0
  5. package/dist/{agent-B63OzUX9.js → agent-BmsMXZXx.js} +30 -10
  6. package/dist/{agent-B63OzUX9.js.map → agent-BmsMXZXx.js.map} +1 -1
  7. package/dist/agents-DXuFxtGC.js +1151 -0
  8. package/dist/agents-DXuFxtGC.js.map +1 -0
  9. package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
  10. package/dist/app-files-DT5mJyV4.js.map +1 -0
  11. package/dist/assemble-CE4pHKIa.js +1 -0
  12. package/dist/{assemble-Me7Whp-y.js → assemble-DM2PNU4R.js} +152 -129
  13. package/dist/assemble-DM2PNU4R.js.map +1 -0
  14. package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
  15. package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
  16. package/dist/{attempt-host-Cik0SoIn.js → attempt-host-7VviAJrl.js} +6 -6
  17. package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-7VviAJrl.js.map} +1 -1
  18. package/dist/boat-BjLqFJIZ.js +108 -0
  19. package/dist/boat-BjLqFJIZ.js.map +1 -0
  20. package/dist/boat-DSNgZt-t.js +322 -0
  21. package/dist/boat-DSNgZt-t.js.map +1 -0
  22. package/dist/bridge/index.mjs +401 -79
  23. package/dist/capture-BzotD2hs.js +1 -0
  24. package/dist/{capture-C3RmjPph.js → capture-Ccf0Hmyx.js} +27 -25
  25. package/dist/capture-Ccf0Hmyx.js.map +1 -0
  26. package/dist/cases-Dy0mifal.js +1 -0
  27. package/dist/cases-z9E6DCd5.js +639 -0
  28. package/dist/cases-z9E6DCd5.js.map +1 -0
  29. package/dist/{cli-BBOlXKPG.js → cli-DTuLSlmW.js} +442 -707
  30. package/dist/cli-DTuLSlmW.js.map +1 -0
  31. package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
  32. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  33. package/dist/cli.js +2 -2
  34. package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
  35. package/dist/client-KM4C5r5_.js.map +1 -0
  36. package/dist/{command-DhiwsZl1.js → command-C2C7P5qo.js} +9 -9
  37. package/dist/{command-DhiwsZl1.js.map → command-C2C7P5qo.js.map} +1 -1
  38. package/dist/{command-CFQ9c1hk.js → command-D8oMB2fa.js} +16 -16
  39. package/dist/command-D8oMB2fa.js.map +1 -0
  40. package/dist/{command-fO0BFsTq.js → command-DSL7I-SI.js} +37 -37
  41. package/dist/command-DSL7I-SI.js.map +1 -0
  42. package/dist/{command-qGS802uX.js → command-Ff0T-1Hi.js} +21 -80
  43. package/dist/command-Ff0T-1Hi.js.map +1 -0
  44. package/dist/{commands-xYTMNTPv.js → commands-Bk9NG1s_.js} +1 -1
  45. package/dist/{commands-CC60TOT5.js → commands-xAOi_VEk.js} +186 -85
  46. package/dist/commands-xAOi_VEk.js.map +1 -0
  47. package/dist/{config-C2UtS9vy.js → config-SvK8pjEK.js} +261 -1708
  48. package/dist/config-SvK8pjEK.js.map +1 -0
  49. package/dist/{context-BajfiH0S.js → context-BdgCao9Q.js} +3 -2
  50. package/dist/context-BdgCao9Q.js.map +1 -0
  51. package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
  52. package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
  53. package/dist/cost-Cr26Alhq.js +1 -0
  54. package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-DEzan4IT.js} +3 -3
  55. package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-DEzan4IT.js.map} +1 -1
  56. package/dist/{db-C5SZ0510.js → db-BzOJ4Js4.js} +130 -84
  57. package/dist/db-BzOJ4Js4.js.map +1 -0
  58. package/dist/db-CQRZ3URX.js +2 -0
  59. package/dist/debug-CF4p4Lp_.js +1 -0
  60. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  61. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  62. package/dist/{default-effort-FEidNAIi.js → default-effort-BBDPbJ4D.js} +1 -1
  63. package/dist/{default-effort-BVmx7r7L.js → default-effort-BDKIxJyC.js} +4 -4
  64. package/dist/{default-effort-BVmx7r7L.js.map → default-effort-BDKIxJyC.js.map} +1 -1
  65. package/dist/{dist-ByYk_wAs.js → dist-Db7ifchA.js} +37 -8
  66. package/dist/dist-Db7ifchA.js.map +1 -0
  67. package/dist/{docker-BlSopHpZ.js → docker--mukta3L.js} +1 -1
  68. package/dist/{docker-B2F8-PGC.js → docker-DFrjoNs-.js} +3 -3
  69. package/dist/{docker-B2F8-PGC.js.map → docker-DFrjoNs-.js.map} +1 -1
  70. package/dist/{docker-Dto8ROCq.js → docker-DG_uFgn2.js} +109 -77
  71. package/dist/docker-DG_uFgn2.js.map +1 -0
  72. package/dist/{doctor-BcvfKpBX.js → doctor-DWK4ttex.js} +84 -27
  73. package/dist/doctor-DWK4ttex.js.map +1 -0
  74. package/dist/{environment-CC1RV21n.js → environment-cPMCOk-h.js} +79 -39
  75. package/dist/environment-cPMCOk-h.js.map +1 -0
  76. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  77. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  78. package/dist/{eval-source-CzxdgFBW.js → eval-source-C9b1h8L6.js} +10 -10
  79. package/dist/{eval-source-CzxdgFBW.js.map → eval-source-C9b1h8L6.js.map} +1 -1
  80. package/dist/{evaluate-NTNJ8T9H.js → evaluate-B12Tosvq.js} +377 -142
  81. package/dist/evaluate-B12Tosvq.js.map +1 -0
  82. package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
  83. package/dist/{files-aAzoGNCB.js → files-BOBdC0uI.js} +5 -5
  84. package/dist/{files-aAzoGNCB.js.map → files-BOBdC0uI.js.map} +1 -1
  85. package/dist/files-C1Rk9FYA.js +1 -0
  86. package/dist/files-DMQ9-Vv5.js +1 -0
  87. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  88. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  89. package/dist/files-Di0xDbCc.js +1 -0
  90. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  91. package/dist/files-DlQshsks.js.map +1 -0
  92. package/dist/{files-DsXULUi5.js → files-F5EG0WPT.js} +5 -5
  93. package/dist/{files-DsXULUi5.js.map → files-F5EG0WPT.js.map} +1 -1
  94. package/dist/files-G5jf8wBp.js +1 -0
  95. package/dist/{files-CZXrkPja.js → files-s28a6czq.js} +54 -13
  96. package/dist/{files-CZXrkPja.js.map → files-s28a6czq.js.map} +1 -1
  97. package/dist/files-tiNWe8Ih.js +64 -0
  98. package/dist/files-tiNWe8Ih.js.map +1 -0
  99. package/dist/gate-CgYdaprG.js +49 -0
  100. package/dist/gate-CgYdaprG.js.map +1 -0
  101. package/dist/gateway.d.ts +1 -1
  102. package/dist/{git-C1MB5bjr.js → git-BpoqEgDV.js} +2 -2
  103. package/dist/{git-C1MB5bjr.js.map → git-BpoqEgDV.js.map} +1 -1
  104. package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
  105. package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
  106. package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
  107. package/dist/harness-DXv2gnaT.js.map +1 -0
  108. package/dist/harness.d.ts +1 -1
  109. package/dist/harness.js +1 -1
  110. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  111. package/dist/ids-C_advdJA.js.map +1 -0
  112. package/dist/{images-CNayXCp2.js → images-D2jFze2v.js} +98 -10
  113. package/dist/images-D2jFze2v.js.map +1 -0
  114. package/dist/index.d.ts +23 -3
  115. package/dist/index.js +5 -5
  116. package/dist/{init-D5HtmyMI.js → init-Bi1Q3lRi.js} +7 -6
  117. package/dist/init-Bi1Q3lRi.js.map +1 -0
  118. package/dist/init-CUw65o36.js +1 -0
  119. package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
  120. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  121. package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
  122. package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
  123. package/dist/judge-COMXT1Hx.js +230 -0
  124. package/dist/judge-COMXT1Hx.js.map +1 -0
  125. package/dist/judge.d.ts +1 -1
  126. package/dist/judge.js +1 -1
  127. package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
  128. package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
  129. package/dist/kinds-Bh2I35id.js +698 -0
  130. package/dist/kinds-Bh2I35id.js.map +1 -0
  131. package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
  132. package/dist/layout-Cm85tQYk.js.map +1 -0
  133. package/dist/list-DuThsE28.js +1348 -0
  134. package/dist/list-DuThsE28.js.map +1 -0
  135. package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
  136. package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
  137. package/dist/{live-uRURFDMZ.js → live-DL0se4Zd.js} +7 -7
  138. package/dist/{live-uRURFDMZ.js.map → live-DL0se4Zd.js.map} +1 -1
  139. package/dist/live-DZLmj0iG.js +1 -0
  140. package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
  141. package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
  142. package/dist/{model-CXVVB4RK.js → model-CkNVAxob.js} +1 -1
  143. package/dist/model-Cv4GuTsZ.js +1 -0
  144. package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
  145. package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
  146. package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
  147. package/dist/model-V3mMXp6x.js.map +1 -0
  148. package/dist/{model-BxBWFOZn.js → model-cgt5uvu8.js} +5 -5
  149. package/dist/{model-BxBWFOZn.js.map → model-cgt5uvu8.js.map} +1 -1
  150. package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
  151. package/dist/node-D7pAU--0.js.map +1 -0
  152. package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
  153. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  154. package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
  155. package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  156. package/dist/options-DPUmk7v_.js +1 -0
  157. package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
  158. package/dist/pairs-BGb134Vu.js.map +1 -0
  159. package/dist/{patch-ug5HpMOj.js → patch-DAi1nCuH.js} +2 -2
  160. package/dist/{patch-ug5HpMOj.js.map → patch-DAi1nCuH.js.map} +1 -1
  161. package/dist/plain-BYUf_fUV.js +267 -0
  162. package/dist/plain-BYUf_fUV.js.map +1 -0
  163. package/dist/{plan-DqnzfDH1.js → plan-CsPilzpk.js} +110 -39
  164. package/dist/plan-CsPilzpk.js.map +1 -0
  165. package/dist/{plugins-B2KQ0y3P.js → plugins-BfnhazOv.js} +10 -267
  166. package/dist/plugins-BfnhazOv.js.map +1 -0
  167. package/dist/{preflight-DyzmvK7i.js → preflight-BixaE4BA.js} +16 -9
  168. package/dist/preflight-BixaE4BA.js.map +1 -0
  169. package/dist/preflight-Bn3ianye.js +1 -0
  170. package/dist/{preflight-DVXEvzFU.js → preflight-D1C-iH0U.js} +1 -1
  171. package/dist/preflight-GkpgrBxP.js +68 -0
  172. package/dist/preflight-GkpgrBxP.js.map +1 -0
  173. package/dist/{prepare-BcSRa7-3.js → prepare-C--rM_wO.js} +2 -2
  174. package/dist/{prepare-21WMQZyt.js → prepare-DQEDXoDp.js} +9 -9
  175. package/dist/{prepare-21WMQZyt.js.map → prepare-DQEDXoDp.js.map} +1 -1
  176. package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
  177. package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
  178. package/dist/{provenance-iRxjp4Pu.js → provenance-BXyYwUC8.js} +4 -3
  179. package/dist/provenance-BXyYwUC8.js.map +1 -0
  180. package/dist/provider-BnUoz4KB.js +1 -0
  181. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  182. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  183. package/dist/{providers-vXMXuwn-.js → providers-1p4HKYIp.js} +152 -9
  184. package/dist/providers-1p4HKYIp.js.map +1 -0
  185. package/dist/providers-B-ijCktf.js +1 -0
  186. package/dist/query-BYhyf0LR.js +1 -0
  187. package/dist/{query-BhaKQ07P.js → query-DhhU9ADI.js} +2 -2
  188. package/dist/{query-BhaKQ07P.js.map → query-DhhU9ADI.js.map} +1 -1
  189. package/dist/{questions-BzdLdV3C.js → questions-CMAq4pr8.js} +9 -9
  190. package/dist/{questions-BzdLdV3C.js.map → questions-CMAq4pr8.js.map} +1 -1
  191. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  192. package/dist/record-6IUzwP9K.js.map +1 -0
  193. package/dist/{remarks-CauU408t.js → remarks-BNABzemt.js} +115 -35
  194. package/dist/remarks-BNABzemt.js.map +1 -0
  195. package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
  196. package/dist/remarks-DPCONFxZ.js.map +1 -0
  197. package/dist/{remote-Dwy3fyo3.js → remote-U7fNt0UP.js} +6 -6
  198. package/dist/{remote-Dwy3fyo3.js.map → remote-U7fNt0UP.js.map} +1 -1
  199. package/dist/resolve-C14Pmtej.js +1 -0
  200. package/dist/{resolve-B1zMnPwK.js → resolve-Cx3a9UQ2.js} +10 -9
  201. package/dist/resolve-Cx3a9UQ2.js.map +1 -0
  202. package/dist/{results-CfC0id_u.js → results-CQKXmHym.js} +6 -6
  203. package/dist/{results-CfC0id_u.js.map → results-CQKXmHym.js.map} +1 -1
  204. package/dist/{resume-Dkhrpjow.js → resume-POpZa1ge.js} +169 -27
  205. package/dist/resume-POpZa1ge.js.map +1 -0
  206. package/dist/retry-CmG809Mp.js +105 -0
  207. package/dist/retry-CmG809Mp.js.map +1 -0
  208. package/dist/{runtime-DcvQcr36.js → runtime-Xm9oeC7o.js} +26 -7
  209. package/dist/runtime-Xm9oeC7o.js.map +1 -0
  210. package/dist/sandbox/boat.d.ts +26 -0
  211. package/dist/sandbox/boat.js +37 -0
  212. package/dist/sandbox/boat.js.map +1 -0
  213. package/dist/{sandbox-8zCrtuCf.js → sandbox-DrlL4yqb.js} +31 -3
  214. package/dist/sandbox-DrlL4yqb.js.map +1 -0
  215. package/dist/sandbox.d.ts +20 -2
  216. package/dist/sandbox.js +1 -1
  217. package/dist/{server-Dplq8g0r.js → server-C00zd6zl.js} +9 -9
  218. package/dist/{server-Dplq8g0r.js.map → server-C00zd6zl.js.map} +1 -1
  219. package/dist/server-C1UJcpUh.js +1 -0
  220. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  221. package/dist/shared-Cfi45Bvy.js.map +1 -0
  222. package/dist/slug-IHh3f770.js +136 -0
  223. package/dist/slug-IHh3f770.js.map +1 -0
  224. package/dist/{source-NWHqn5-o.js → source-DDgr48Gn.js} +2 -2
  225. package/dist/{source-NWHqn5-o.js.map → source-DDgr48Gn.js.map} +1 -1
  226. package/dist/source-DJHmQv_j.js +1 -0
  227. package/dist/src-1dBuXzs4.js +3 -0
  228. package/dist/{src-CUpoVVb8.js → src-BKpioayC.js} +2 -2
  229. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  230. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  231. package/dist/src-BY1cCaoJ.js +2 -0
  232. package/dist/src-BbZYaQ8b.js +2 -0
  233. package/dist/src-CDFH2vVN.js +2 -0
  234. package/dist/src-CMW5R9Q0.js +1 -0
  235. package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
  236. package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
  237. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  238. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  239. package/dist/{src-B9oE_mNS.js → src-CrV2WA2i.js} +2 -2
  240. package/dist/src-CsFaawcZ.js +2 -0
  241. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  242. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  243. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  244. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  245. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  246. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  247. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  248. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  249. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  250. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  251. package/dist/src-wKwznPgt.js +2 -0
  252. package/dist/{src-BQ6BiXoc.js → src-y0nOeRRE.js} +610 -72
  253. package/dist/src-y0nOeRRE.js.map +1 -0
  254. package/dist/{state-DDCGUhva.js → state-328JpnHA.js} +17 -17
  255. package/dist/{state-DDCGUhva.js.map → state-328JpnHA.js.map} +1 -1
  256. package/dist/state-Ca55Gk_O.js +1 -0
  257. package/dist/suite-CKBa6MKt.js +2 -0
  258. package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
  259. package/dist/suite-COglUhPd.js.map +1 -0
  260. package/dist/summary-DDy9DfyJ.js +148 -0
  261. package/dist/summary-DDy9DfyJ.js.map +1 -0
  262. package/dist/tart-cli-C-GsbVsQ.js +1 -0
  263. package/dist/tart-cli-CU0nFS-L.js +55 -0
  264. package/dist/tart-cli-CU0nFS-L.js.map +1 -0
  265. package/dist/tart-dWoJvWHc.js +586 -0
  266. package/dist/tart-dWoJvWHc.js.map +1 -0
  267. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  268. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  269. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  270. package/dist/tart-ref-DYZijjrU.js +55 -0
  271. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  272. package/dist/testing.d.ts +2 -2
  273. package/dist/testing.js +4 -4
  274. package/dist/trace-Dx2NBE1O.js +311 -0
  275. package/dist/trace-Dx2NBE1O.js.map +1 -0
  276. package/dist/traits-DKnYkqeK.js +66 -0
  277. package/dist/traits-DKnYkqeK.js.map +1 -0
  278. package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
  279. package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
  280. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  281. package/dist/types-C2mLpph3.js.map +1 -0
  282. package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
  283. package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
  284. package/dist/vercel-CGlSBv63.js.map +1 -0
  285. package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
  286. package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
  287. package/dist/vertex-O88TCwgY.js +1 -0
  288. package/dist/view/index.d.ts +101 -58
  289. package/dist/view/index.js +9 -8
  290. package/dist/views-B0Yt_Q1S.js +2 -0
  291. package/dist/{views-BISrsIjw.js → views-BROacjBv.js} +7 -6
  292. package/dist/views-BROacjBv.js.map +1 -0
  293. package/dist/{worker-entry-DJD_nHkS.js → worker-entry-Smb4bNDc.js} +11 -8
  294. package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-Smb4bNDc.js.map} +1 -1
  295. package/dist/workspace-C0dDP75l.js +1 -0
  296. package/dist/{workspace-BO1VhUR9.js → workspace-DNDTEk8v.js} +6 -6
  297. package/dist/{workspace-BO1VhUR9.js.map → workspace-DNDTEk8v.js.map} +1 -1
  298. package/package.json +24 -11
  299. package/skills/eval-rubric/SKILL.md +15 -7
  300. package/dist/adapter-CptK032x.js.map +0 -1
  301. package/dist/app-files-sgqwfjtn.js.map +0 -1
  302. package/dist/assemble-CxBHkavM.js +0 -1
  303. package/dist/assemble-Me7Whp-y.js.map +0 -1
  304. package/dist/capture-C3RmjPph.js.map +0 -1
  305. package/dist/capture-YPwxp_iK.js +0 -1
  306. package/dist/cli-BBOlXKPG.js.map +0 -1
  307. package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
  308. package/dist/client-tZ2WBDxf.js.map +0 -1
  309. package/dist/command-CFQ9c1hk.js.map +0 -1
  310. package/dist/command-fO0BFsTq.js.map +0 -1
  311. package/dist/command-qGS802uX.js.map +0 -1
  312. package/dist/commands-CC60TOT5.js.map +0 -1
  313. package/dist/config-C2UtS9vy.js.map +0 -1
  314. package/dist/context-BajfiH0S.js.map +0 -1
  315. package/dist/contrasts-BjOg7wyr.js +0 -101
  316. package/dist/contrasts-BjOg7wyr.js.map +0 -1
  317. package/dist/cost-BJU0MAji.js +0 -1
  318. package/dist/db-C5SZ0510.js.map +0 -1
  319. package/dist/db-puzLzX4k.js +0 -2
  320. package/dist/debug-BnXV7JbX.js +0 -1
  321. package/dist/dist-ByYk_wAs.js.map +0 -1
  322. package/dist/docker-Dto8ROCq.js.map +0 -1
  323. package/dist/doctor-BcvfKpBX.js.map +0 -1
  324. package/dist/environment-CC1RV21n.js.map +0 -1
  325. package/dist/evaluate-NTNJ8T9H.js.map +0 -1
  326. package/dist/files-COW5JBJM.js +0 -1
  327. package/dist/files-DFtuRQJx.js +0 -1
  328. package/dist/files-oOYNR7ZH.js +0 -1
  329. package/dist/files-p-uTdOjA.js.map +0 -1
  330. package/dist/gate-Cp_pQoEe.js.map +0 -1
  331. package/dist/harness-DYSC4zMh.js.map +0 -1
  332. package/dist/ids-DuK6xl4w.js.map +0 -1
  333. package/dist/images-CNayXCp2.js.map +0 -1
  334. package/dist/init-8o6W3_fn.js +0 -1
  335. package/dist/init-D5HtmyMI.js.map +0 -1
  336. package/dist/ink-list-Ca-5dunO.js.map +0 -1
  337. package/dist/judge-DYOF69Oe.js.map +0 -1
  338. package/dist/list-CM_YSJXq.js +0 -374
  339. package/dist/list-CM_YSJXq.js.map +0 -1
  340. package/dist/live-DQc8jr6o.js +0 -1
  341. package/dist/model-Ce9l9_rF.js.map +0 -1
  342. package/dist/model-Cxzo4cjw.js +0 -1
  343. package/dist/node-DhIRSrrK.js.map +0 -1
  344. package/dist/opencode-providers-Co7plCea.js.map +0 -1
  345. package/dist/plan-DqnzfDH1.js.map +0 -1
  346. package/dist/plugins-B2KQ0y3P.js.map +0 -1
  347. package/dist/preflight-DyzmvK7i.js.map +0 -1
  348. package/dist/provenance-iRxjp4Pu.js.map +0 -1
  349. package/dist/provider-DCdTHQpk.js +0 -1
  350. package/dist/providers-D0ob_tra.js +0 -1
  351. package/dist/providers-vXMXuwn-.js.map +0 -1
  352. package/dist/query-CvgUsvep.js +0 -1
  353. package/dist/record-DhzwkSXc.js.map +0 -1
  354. package/dist/remarks-CauU408t.js.map +0 -1
  355. package/dist/remarks-NMgPd30U.js.map +0 -1
  356. package/dist/resolve-B1zMnPwK.js.map +0 -1
  357. package/dist/resolve-BIIeVT12.js +0 -1
  358. package/dist/resume-Dkhrpjow.js.map +0 -1
  359. package/dist/runtime-DcvQcr36.js.map +0 -1
  360. package/dist/sandbox-8zCrtuCf.js.map +0 -1
  361. package/dist/server-0tuj8VSa.js +0 -1
  362. package/dist/shared-ogqPJ0uo.js.map +0 -1
  363. package/dist/source-llwLewOU.js +0 -1
  364. package/dist/src-BQ6BiXoc.js.map +0 -1
  365. package/dist/src-BXnv0ViU.js +0 -2
  366. package/dist/src-BiS8YNN_.js +0 -3
  367. package/dist/src-BkH4GOxu.js +0 -1
  368. package/dist/src-C-tH5ebd.js +0 -2
  369. package/dist/src-Cg1bFs9g.js +0 -2
  370. package/dist/src-DV_UorZv.js +0 -2
  371. package/dist/src-XxUpr2sa.js +0 -2
  372. package/dist/state-DZbAO95a.js +0 -1
  373. package/dist/suite-M8_7xio7.js.map +0 -1
  374. package/dist/suite-u2KJnGpY.js +0 -2
  375. package/dist/summary-Uzc9osfV.js.map +0 -1
  376. package/dist/types-C4OkYhZg.js.map +0 -1
  377. package/dist/vertex-h_peaZTN.js +0 -1
  378. package/dist/views-BISrsIjw.js.map +0 -1
  379. package/dist/views-Bfab8fM9.js +0 -2
  380. package/dist/workspace-B8EOCuxl.js +0 -1
@@ -257,13 +257,23 @@ type UpstreamCount = {
257
257
  provider: string;
258
258
  calls: number;
259
259
  };
260
+ /** An infra error an attempt ended with before it ran again, and when that try started. */
261
+ type AttemptRetry = {
262
+ error: AttemptError;
263
+ at: string | null;
264
+ };
260
265
  type AttemptRecord = {
261
- /** `<eval key> # <cell key>#<n>`: unique in the run. */
266
+ /**
267
+ * `<eval key> # <cell key>#<n>`, or `<eval key> [<case>] # <cell key>#<n>` on an eval with
268
+ * `cases`: unique in the run.
269
+ */
262
270
  key: string;
263
- /** `<cell key>#<n>`: unique in its evaluation. */
271
+ /** `<cell key>#<n>`, or `<cell key>#<case>#<n>` on an eval with `cases`: unique in its evaluation. */
264
272
  id: string;
265
273
  /** The cell's key; see `EvalEntry.cells`. */
266
274
  cell: string;
275
+ /** The case's key, on an eval with `cases`; see `EvalEntry.cases`. */
276
+ case?: string;
267
277
  /** Run number within the cell, from 1. */
268
278
  n: number;
269
279
  /**
@@ -277,6 +287,11 @@ type AttemptRecord = {
277
287
  startedAt: string | null;
278
288
  /** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
279
289
  imageId: string | null;
290
+ /**
291
+ * The image a `tart()` attempt's VM was cloned from, with the digest Tart had for it. Absent in
292
+ * other sandboxes, and in attempts that didn't reach their sandbox.
293
+ */
294
+ vmImage?: VmImage;
280
295
  /** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
281
296
  env: {
282
297
  vars: string[];
@@ -405,6 +420,13 @@ type AttemptRecord = {
405
420
  judgements: JudgmentRecord[];
406
421
  /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
407
422
  startRetries?: StartRetry[];
423
+ /**
424
+ * The infra errors this attempt ended with before `reeval run --retry-errors` or `--resume` ran
425
+ * it again, oldest first: each `{ error, at }`, `at` being when that try started (null when it
426
+ * never did). The attempt's own `outcome` and `error` are its last try's. Absent when it never
427
+ * ran again.
428
+ */
429
+ retries?: AttemptRetry[];
408
430
  /**
409
431
  * Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
410
432
  * without `hosts` in their environment, so they hold its real value while the agent works and
@@ -497,9 +519,15 @@ type EvalEntry = {
497
519
  /**
498
520
  * Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
499
521
  * `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
500
- * cell's failures don't fail the evaluation.
522
+ * cell's failures don't fail the evaluation. With `cases`, a cell's outcome is over every case.
501
523
  */
502
524
  outcome: Outcome;
525
+ /**
526
+ * In records from before an eval with `cases` was one entry, `family` on a case whose
527
+ * `gate: { passRate }` its family held over the mean of its cases. Never written now: see
528
+ * `CaseRecord.verdict`.
529
+ */
530
+ verdict?: 'family';
503
531
  /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
504
532
  todo: boolean;
505
533
  /**
@@ -521,13 +549,63 @@ type EvalEntry = {
521
549
  * with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
522
550
  */
523
551
  answers?: CompareAnswer[];
524
- /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
552
+ /**
553
+ * In records from before an eval with `cases` was one entry, set on each case, which was an
554
+ * entry of its own. Never written now; reading such a record folds the cases into one entry.
555
+ */
525
556
  family?: {
526
557
  id: string;
527
558
  name: string;
528
559
  case: number;
529
560
  cases: number;
530
561
  };
562
+ /**
563
+ * The environments (and harnesses) `where` left out, one entry per case and environment,
564
+ * including those of a case it left out everywhere; unset when none.
565
+ */
566
+ excluded?: ExcludedPair[];
567
+ /**
568
+ * An eval with `cases`: each case that has attempts in this run, in list order, with its own
569
+ * cells. `cells` and `attempts` are over every case. Unset without cases.
570
+ */
571
+ cases?: CaseRecord[];
572
+ /** With `cases`: how many the list has, `cases` holding those with attempts in this run. */
573
+ caseCount?: number;
574
+ /**
575
+ * With `cases` and `compare`: each variant cell against its control, paired by case. Its
576
+ * `minLift` gates the lift's lower bound, and a lift below it fails the variant cell.
577
+ */
578
+ lifts?: FamilyLift[];
579
+ };
580
+ /** One case of an eval with `cases`, and how its attempts went. */
581
+ type CaseRecord = {
582
+ /** The case's key: `nextjs` for the case keyed `nextjs`. */
583
+ key: string;
584
+ /** Its place in the case list, from 0. */
585
+ index: number;
586
+ /**
587
+ * The name it ran under in a record from before an eval with `cases` was one entry, when that
588
+ * isn't `<eval name> [<key>]` (a name template's, `fixes GH-101` for `fixes $id`).
589
+ */
590
+ name?: string;
591
+ /** `EvalEntry.source.case`: the hash of the case's data. */
592
+ hash?: string;
593
+ /**
594
+ * What the case did, over its own cells by each cell's gate. Under a `gate: { passRate }`, the
595
+ * eval's cell gates on the mean over the cases, so `verdict` says this outcome decides nothing.
596
+ */
597
+ outcome: Outcome;
598
+ /** `family` when a cell's `gate: { passRate }` gates the mean over the cases, not this case. */
599
+ verdict?: 'family';
600
+ /** pass@1 over its gated cells' attempts. */
601
+ passRate: number | null;
602
+ fingerprint: string;
603
+ /** Its own cells: their attempts are those with this `case`. */
604
+ cells: CellRecord[];
605
+ /** The environments (and harnesses) `where` left this case out of; unset when none. */
606
+ excluded?: ExcludedPair[];
607
+ /** The eval's pairwise questions, answered over this case's attempts; unset without any. */
608
+ answers?: CompareAnswer[];
531
609
  };
532
610
  /** A 95% interval on a pooled rate or difference. */
533
611
  type FamilyInterval = {
@@ -541,7 +619,7 @@ type FamilyInterval = {
541
619
  */
542
620
  method: 'wilson' | 'newcombe-paired' | 't-cases';
543
621
  };
544
- /** One cell's pass@1 over a family's cases. */
622
+ /** One cell's pass@1 over an eval's cases. */
545
623
  type FamilyRate = {
546
624
  /** The cell's key, the same in every case. */
547
625
  cell: string;
@@ -553,13 +631,19 @@ type FamilyRate = {
553
631
  /** The mean of the cases' pass@1; at one attempt a case, `pass / counted`. */
554
632
  rate: number;
555
633
  interval: FamilyInterval;
634
+ /** The cell's `gate: { passRate }`: with cases it gates `rate`, not each case's pass@1. */
635
+ passRate?: number;
636
+ /** Set with `passRate`: `fail` when `rate` is below it. */
637
+ outcome?: 'pass' | 'fail';
556
638
  };
639
+ /** A cell's pass@1 over an eval's cases. */
640
+ type CellRate = Omit<FamilyRate, 'cell'>;
557
641
  /**
558
642
  * `lift` / `drop`: the interval excludes 0. `noise`: it doesn't. `insufficient`: fewer than 6
559
643
  * discordant pairs (one attempt a case) or 6 paired cases (several), where no test can decide.
560
644
  */
561
645
  type FamilyVerdict = 'lift' | 'drop' | 'noise' | 'insufficient';
562
- /** One effort metric over a family's paired cases. */
646
+ /** One effort metric over an eval's paired cases. */
563
647
  type FamilyEffortMetric = {
564
648
  /** Cases with a value on both sides. */
565
649
  pairs: number;
@@ -580,7 +664,7 @@ type FamilyEffort = {
580
664
  /** Over the cases where every counted attempt reported usage; omitted when none did. */
581
665
  tokens?: FamilyEffortMetric;
582
666
  };
583
- /** A variant cell against its control over a family's cases, paired by case. */
667
+ /** A variant cell against its control over an eval's cases, paired by case. */
584
668
  type FamilyLift = {
585
669
  /** The variant cell's key. */
586
670
  variant: string;
@@ -604,13 +688,17 @@ type FamilyLift = {
604
688
  /** The interval spans 0. */
605
689
  withinNoise: boolean;
606
690
  verdict: FamilyVerdict;
607
- /** The compare's `minLift`: on a family it gates this lower bound, not each case's. */
691
+ /** The compare's `minLift`: with cases it gates this lower bound, not each case's. */
608
692
  minLift?: number;
609
693
  /** `fail` when `interval.lo` is below `minLift`. */
610
694
  outcome: 'pass' | 'fail';
611
695
  effort?: FamilyEffort;
612
696
  };
613
- /** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an evaluation. */
697
+ /**
698
+ * In records from before an eval with `cases` was one entry: the eval over its cases, each case
699
+ * an entry of its own. Reading such a record folds it into the eval's `cases`, `lifts` and each
700
+ * cell's `rate`.
701
+ */
614
702
  type FamilyRecord = {
615
703
  /** `file › group › … › template`. */
616
704
  id: string;
@@ -620,13 +708,24 @@ type FamilyRecord = {
620
708
  name: string;
621
709
  /** Cases in the list. */
622
710
  cases: number;
623
- /** Cases with an evaluation in this run; fewer when a filter or tag left some out. */
711
+ /**
712
+ * Cases with an attempt that ran (not skipped) in this run; fewer when a filter or tag left
713
+ * some out, or an interrupted run or a filtered resume hasn't run them yet.
714
+ */
624
715
  ran: number;
625
716
  /** One per cell. */
626
717
  rates: FamilyRate[];
627
718
  /** One per variant cell, with `compare`. */
628
719
  lifts: FamilyLift[];
629
- /** `fail` when a lift is below its `minLift`. The cases still pass or fail on their own. */
720
+ /**
721
+ * Every case and environment (and harness) pair `where` left out, including those of a case it
722
+ * left out everywhere, which has no evaluation; unset when none.
723
+ */
724
+ excluded?: ExcludedPair[];
725
+ /**
726
+ * `fail` when a lift is below its `minLift` or a rate below its `passRate`. A case still fails
727
+ * on its own under `gate: 'all'`.
728
+ */
630
729
  outcome: 'pass' | 'fail';
631
730
  };
632
731
  /** A variant cell's pass@1 against the control's, on the same harness and sandbox. */
@@ -663,6 +762,7 @@ type FingerprintParts = {
663
762
  /** Over the harness descriptor, less concurrency caps. */
664
763
  harness: string;
665
764
  environment: string | null;
765
+ /** The Docker image's id, or the digest of the image a `tart()` VM was cloned from. */
666
766
  image: string | null;
667
767
  agentCli: string | null;
668
768
  /** Over the judge's model, threshold and version. */
@@ -723,7 +823,13 @@ type CellRecord = Cell & {
723
823
  * reports the worst outcome. A variant below its `minLift` fails too.
724
824
  */
725
825
  outcome: Outcome;
826
+ /** Unset on an eval with several `cases`, whose variants have `EvalEntry.lifts`. */
726
827
  lift?: Lift;
828
+ /**
829
+ * On an eval with several `cases`: the cell's pass@1 over them. A `gate: { passRate }` gates
830
+ * `rate.rate`, the mean of the cases' pass@1, and sets `rate.outcome`.
831
+ */
832
+ rate?: CellRate;
727
833
  /**
728
834
  * Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
729
835
  * attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
@@ -785,6 +891,8 @@ type RunInvocation = {
785
891
  filters: string[];
786
892
  /** `--tag`, without the profile's own tags. */
787
893
  tags: string[];
894
+ /** `--trait`, as given (`key=value`); unset without any. */
895
+ traits?: string[];
788
896
  };
789
897
  type RunRecord = {
790
898
  kind: 'reeval-run';
@@ -795,7 +903,13 @@ type RunRecord = {
795
903
  status: 'complete' | 'interrupted' | 'budget';
796
904
  startedAt: string;
797
905
  endedAt: string;
906
+ /** How long the run ran, counting each `--retry-errors` of it but not the time between them. */
798
907
  durationMs: number;
908
+ /**
909
+ * When each `reeval run --retry-errors` of the run started, oldest first; unset for a run never
910
+ * retried. A `--resume` of the run while it's `interrupted` finishes the last one.
911
+ */
912
+ retriedAt?: string[];
799
913
  author: string;
800
914
  git: GitInfo | null;
801
915
  runner: RunnerInfo;
@@ -830,6 +944,7 @@ type RunRecord = {
830
944
  maxVariants: number;
831
945
  };
832
946
  harness?: string[];
947
+ traits?: Record<string, string | string[]>;
833
948
  };
834
949
  } | null;
835
950
  /** How `reeval run` was started; `--resume` plans with it again. Unset in older records. */
@@ -878,7 +993,10 @@ type RunRecord = {
878
993
  };
879
994
  summary: RunSummary;
880
995
  evals: EvalEntry[];
881
- /** Each pooled `evaluate.each` that ran; unset when none did. */
996
+ /**
997
+ * In records from before an eval with `cases` was one entry, each eval with `cases` that ran.
998
+ * Never written now; reading such a record folds it into its eval.
999
+ */
882
1000
  families?: FamilyRecord[];
883
1001
  /**
884
1002
  * These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
@@ -991,7 +1109,11 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
991
1109
  /** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
992
1110
  canceled?: number;
993
1111
  };
994
- /** A family's pooled lift as the history listing keeps it: enough to compare its gate. */
1112
+ /** One case of an eval with `cases` in a `HistoryEntry`. */
1113
+ type HistoryCase = Pick<CaseRecord, 'key' | 'outcome' | 'verdict' | 'fingerprint' | 'hash'> & {
1114
+ cells: HistoryCell[];
1115
+ };
1116
+ /** A pooled lift as the history listing keeps it: enough to compare its gate. */
995
1117
  type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
996
1118
  lo: number;
997
1119
  hi: number;
@@ -1036,12 +1158,14 @@ type HistoryEntry = {
1036
1158
  outcome: Outcome;
1037
1159
  passRate: number | null;
1038
1160
  cells: HistoryCell[];
1039
- }>;
1040
- /** The run's pooled families; unset when none ran. */
1041
- families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'> & {
1042
- /** The eval ids of its cases in this run. */
1043
- caseIds: string[];
1044
- lifts: FamilyLiftSummary[];
1161
+ /** An eval with `cases`: each case that ran, with its own cells; unset without cases. */
1162
+ cases?: HistoryCase[];
1163
+ /** With `cases`: how many the list has. */
1164
+ caseCount?: number;
1165
+ /** With `cases` and `compare`: each variant's lift paired by case. */
1166
+ lifts?: FamilyLiftSummary[];
1167
+ /** With `cases`: its cells' rates over the cases that a `gate: { passRate }` gates; unset when none. */
1168
+ rates?: Array<Pick<FamilyRate, 'cell' | 'rate'> & Required<Pick<FamilyRate, 'passRate' | 'outcome'>>>;
1045
1169
  }>;
1046
1170
  /** The run's errors outside any attempt; unset when there were none. */
1047
1171
  errors?: string[];
@@ -1621,6 +1745,49 @@ type AttemptContext = {
1621
1745
  };
1622
1746
  };
1623
1747
  type EvalFn = (c: AttemptContext) => Promise<void>;
1748
+ /** The body of an eval with `cases`: the attempt and its case. */
1749
+ type CaseFn<T> = (c: AttemptContext, item: T) => Promise<void>;
1750
+ /**
1751
+ * One eval over several inputs: each case runs the body with every cell and run, and the cases
1752
+ * pool into one pass rate (and, with `compare`, one lift paired by case). Per eval only.
1753
+ */
1754
+ type CaseOptions<T> = {
1755
+ /** The inputs, each JSON data; the body gets one as its second argument. */
1756
+ cases: readonly T[];
1757
+ /**
1758
+ * Each case's key, which names it in the output and the record. Without it, a string or a
1759
+ * finite number is its own key, and an object uses its `id`, else its `name`.
1760
+ */
1761
+ caseKey?: (item: T) => string;
1762
+ /**
1763
+ * Which case and environment pairs run, decided at plan time. It sees each environment's
1764
+ * `traits`, and with `compare` the control, whose answer holds for every variant. Reading
1765
+ * `harness` decides each harness on its own. The pairs it leaves out are listed in `--dry-run`
1766
+ * and the record's `excluded`.
1767
+ */
1768
+ where?: Where<T>;
1769
+ };
1770
+ /** Filter facts about an environment, for `where`: never sent to the agent, never prompt data. */
1771
+ type Traits = Readonly<Record<string, string | readonly string[]>>;
1772
+ /** `where`'s predicate: true runs the case on that environment (and harness). */
1773
+ type Where<T> = (pair: {
1774
+ case: T;
1775
+ environment: {
1776
+ readonly name: string;
1777
+ /** The environment's traits; reading a key that no listed environment defines is refused. */
1778
+ readonly traits: Readonly<Record<string, string | readonly string[] | undefined>>;
1779
+ };
1780
+ /** The harness id; reading it decides each harness on its own. */
1781
+ readonly harness: string;
1782
+ }) => boolean;
1783
+ /** A case and environment pair `where` left out. */
1784
+ type ExcludedPair = {
1785
+ /** The case key. */
1786
+ case: string;
1787
+ environment: string;
1788
+ /** Set only when `where` read the harness. */
1789
+ harness?: string;
1790
+ };
1624
1791
  /** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
1625
1792
  type FakeHarnessDescriptor = {
1626
1793
  kind: 'fake';
@@ -1639,6 +1806,11 @@ type FakeHarnessDescriptor = {
1639
1806
  delayMs?: number;
1640
1807
  /** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
1641
1808
  ignoreAbort?: boolean;
1809
+ /**
1810
+ * Each turn's recorded agent time, exactly, whatever the wall clock says: a test can compare
1811
+ * agent time without scheduling jitter, which four runs a side can tell apart.
1812
+ */
1813
+ turnMs?: number;
1642
1814
  /** Throw before the first turn starts, to simulate an infra failure. */
1643
1815
  failWith?: string;
1644
1816
  /** The first this many session starts of an attempt never become ready, even when aborted (`true`: every one). */
@@ -1672,6 +1844,14 @@ type FakeHarnessDescriptor = {
1672
1844
  };
1673
1845
  /** Chance (0–1) that an attempt writes no files and says so, for flaky pass rates. */
1674
1846
  flakeRate?: number;
1847
+ /** Tokens the fake agent reports for every turn, as `turn.usage`; without it, null. */
1848
+ usage?: {
1849
+ input: number;
1850
+ output: number;
1851
+ cacheRead: number;
1852
+ cacheWrite: number;
1853
+ reasoning: number | null;
1854
+ };
1675
1855
  /**
1676
1856
  * Real sandbox actions, one list per `c.send` (the last list repeats), run in order before
1677
1857
  * `files`. `exec` runs as the agent: with the sandbox's own variables, none of the checks'.
@@ -2099,7 +2279,62 @@ type DockerSandboxDescriptor = {
2099
2279
  /** Processes and threads at once. Default 1024. */
2100
2280
  pids?: number;
2101
2281
  };
2102
- type SandboxDescriptor = FakeSandboxDescriptor | DockerSandboxDescriptor;
2282
+ /** A macOS VM on this Mac, cloned from `image` for each attempt by Tart, made by `tart()`. */
2283
+ type TartSandboxDescriptor = {
2284
+ kind: 'tart';
2285
+ /** Cell id; defaults to `tart:<image>`, the image's last path segment. */
2286
+ id?: string;
2287
+ /**
2288
+ * The VM each attempt clones: an OCI image such as
2289
+ * `ghcr.io/cirruslabs/macos-sequoia-base:latest`, or a local VM's name. Tart must have it
2290
+ * before the run: `reeval image build` (or `tart pull`) pulls it, and `reeval run` refuses one
2291
+ * `tart list` doesn't show.
2292
+ */
2293
+ image: string;
2294
+ /**
2295
+ * The most Tart VMs at once, across the run: 1 or 2, and 2 when unset, since macOS runs at most
2296
+ * two macOS VMs at a time.
2297
+ */
2298
+ maxConcurrent?: number;
2299
+ /** CPUs each clone gets; the image's when unset. */
2300
+ cpus?: number;
2301
+ /** Memory each clone gets: `'8g'`, `'4096m'`, or bytes; the image's when unset. */
2302
+ memory?: string | number;
2303
+ /**
2304
+ * Reach the VM over SSH when its Tart Guest Agent doesn't answer `tart exec`: as `user`
2305
+ * (`admin` when unset), with `key`, a private key the image trusts, absolute or relative to the
2306
+ * file it's written in.
2307
+ */
2308
+ ssh?: {
2309
+ user?: string;
2310
+ key: string;
2311
+ };
2312
+ };
2313
+ /** A boat.dev machine size: `small` is 2 vCPUs and 4 GB. */
2314
+ type BoatMachineType = 'small' | 'default' | 'large' | 'xlarge';
2315
+ /** The host variable holding a Boat API key, made by `secret.fromEnv`. */
2316
+ type BoatApiKey = {
2317
+ kind: 'secret';
2318
+ from: string;
2319
+ };
2320
+ /** A boat.dev VM per attempt, made by `boat()` from `reeval/sandbox/boat`. */
2321
+ type BoatSandboxDescriptor = {
2322
+ kind: 'boat';
2323
+ /** Cell id; defaults to `boat`, or `boat:<type>` with a `type`. */
2324
+ id?: string;
2325
+ /** The host variable with the Boat API key, read on the host only. */
2326
+ apiKey: BoatApiKey;
2327
+ /**
2328
+ * The start of every sandbox name reeval gives Boat, so a run's cleanup finds only its own:
2329
+ * lowercase letters, digits and dashes.
2330
+ */
2331
+ prefix: string;
2332
+ /** The machine size. Default `small`. */
2333
+ type?: BoatMachineType;
2334
+ /** The most Boat sandboxes at once, across the run. */
2335
+ maxConcurrent?: number;
2336
+ };
2337
+ type SandboxDescriptor = FakeSandboxDescriptor | DockerSandboxDescriptor | TartSandboxDescriptor | BoatSandboxDescriptor;
2103
2338
  /**
2104
2339
  * A host environment variable, redacted everywhere reeval writes and recorded by name only. Made by
2105
2340
  * the `secret` helper of `defineEnvironment` and `defineConfig` callbacks.
@@ -2310,6 +2545,12 @@ type EnvironmentSpec = {
2310
2545
  * parent's plugin of the same name.
2311
2546
  */
2312
2547
  plugins?: PluginSource[];
2548
+ /**
2549
+ * Filter facts for `where`, such as `{ stack: 'next' }`: a key to a string or a list of
2550
+ * strings. A child's key replaces its parent's value whole; keys it doesn't set are inherited.
2551
+ * Recorded on the cell, and never written to the sandbox, the prompt or the judge.
2552
+ */
2553
+ traits?: Traits;
2313
2554
  };
2314
2555
  /**
2315
2556
  * A part of a plugin, as Claude Code's layout bundles it: `skills/<name>/SKILL.md`,
@@ -2420,6 +2661,8 @@ type Environment = {
2420
2661
  readonly name: string;
2421
2662
  /** Its `extend` chain, root first. */
2422
2663
  readonly layers: readonly EnvironmentLayer[];
2664
+ /** Its layers' `traits`, merged by key with the child's winning; `{}` when none. */
2665
+ readonly traits: Traits;
2423
2666
  /**
2424
2667
  * A child environment layered on this one; wherever it sets a key this one has, it wins.
2425
2668
  * `sandbox` is replaced entirely when the child sets one; when the child omits it, the
@@ -2521,26 +2764,39 @@ type HarnessEdit = {
2521
2764
  type HarnessData = HarnessDescriptor | HarnessDescriptor[] | HarnessEdit;
2522
2765
  /** `HarnessData`, or a function that edits the inherited list (TypeScript only). */
2523
2766
  type HarnessOption = HarnessData | ((inherited: HarnessDescriptor[]) => HarnessDescriptor | HarnessDescriptor[]);
2524
- type CompareGate = 'variants' | 'all' | 'none';
2525
2767
  /**
2526
- * The baseline and each variant run as their own environments, with the same prompt and checks.
2768
+ * An edit of the inherited environment list: `add` appends environments, `only` keeps the
2769
+ * inherited ones with these names, in their order.
2770
+ */
2771
+ type EnvironmentEdit<E = Environment> = {
2772
+ add: E | E[];
2773
+ } | {
2774
+ only: string | string[];
2775
+ };
2776
+ /** One environment, a list (each is its own cell), or an edit of the inherited list, as data. */
2777
+ type EnvironmentsData<E = EnvironmentData> = E | E[] | EnvironmentEdit<E>;
2778
+ /** `EnvironmentsData`, or a function that edits the inherited list (TypeScript only). */
2779
+ type EnvironmentsOption = EnvironmentsData<Environment> | ((inherited: Environment[]) => Environment | Environment[]);
2780
+ type CompareGate = 'variants' | 'all' | 'observe';
2781
+ /**
2782
+ * The control and each variant run as their own environments, with the same prompt and checks.
2527
2783
  * `compare` does not layer them. Share setup first with `environment.extend` / `extendEnvironment`.
2528
2784
  * `false` is not a value here: a profile turns compare off with `limit.compare`.
2529
2785
  */
2530
2786
  type CompareOptions<E = Environment> = {
2531
2787
  /**
2532
- * The control. Not layered under the variants. Without it, the control is the environment the
2533
- * eval would otherwise run in (the cascaded `environment`), else an empty one named `baseline`.
2788
+ * The environment each variant is measured against. Not layered under the variants, and it
2789
+ * replaces any `environments` the eval inherits.
2534
2790
  */
2535
- baseline?: E;
2791
+ control: E;
2536
2792
  /** Other environments, each planned and run as itself. */
2537
2793
  variants: E[];
2538
- /** Which cells decide pass/fail: `variants` (default), `all`, or `none` (report only). */
2794
+ /** Which cells decide pass/fail: `variants` (default), `all`, or `observe` (none of them). */
2539
2795
  gate?: CompareGate;
2540
2796
  /** Lift in points from -1 to 1; a variant cell below it fails. */
2541
2797
  minLift?: number;
2542
2798
  /** Harnesses (ids or descriptors) to compare on; the others run the control only. */
2543
- harness?: Array<string | HarnessDescriptor>;
2799
+ harnesses?: Array<string | HarnessDescriptor>;
2544
2800
  };
2545
2801
  /**
2546
2802
  * A question a language model answers after the attempts: its text, or the text and its options.
@@ -2691,18 +2947,22 @@ type AttemptAnswer = {
2691
2947
  input?: FileRef;
2692
2948
  };
2693
2949
  /** An option a profile's `limit` can narrow, and `pin` can protect. */
2694
- type PinnableKey = 'harness' | 'runs' | 'compare';
2950
+ type PinnableKey = 'harnesses' | 'runs' | 'compare';
2695
2951
  /**
2696
2952
  * How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
2697
2953
  * passes (pass^k); `{ passRate }` passes when its pass@1 (passes over passes and fails) is at
2698
- * least that share.
2954
+ * least that share; `'observe'` gates nothing, so the eval's attempts run and are recorded, and
2955
+ * only run and infra errors fail it.
2699
2956
  */
2700
- type GatePolicy = 'all' | {
2957
+ type GatePolicy = 'all' | 'observe' | {
2701
2958
  passRate: number;
2702
2959
  };
2703
- /** A cell's gate: its policy, or `'none'` when its failures don't fail the eval (a control's). */
2704
- type CellGate = GatePolicy | 'none';
2705
- /** A compare object: `baseline` and `variants`. Same as `CompareOptions`. */
2960
+ /**
2961
+ * A cell's gate: its policy, or `'none'` when its failures don't fail the eval (a control's, or
2962
+ * any cell under `gate: 'observe'`).
2963
+ */
2964
+ type CellGate = Exclude<GatePolicy, 'observe'> | 'none';
2965
+ /** A compare object: `control` and `variants`. Same as `CompareOptions`. */
2706
2966
  type CompareSpec<E = Environment> = CompareOptions<E>;
2707
2967
  type OddDigit = '1' | '3' | '5' | '7' | '9';
2708
2968
  /**
@@ -2717,12 +2977,12 @@ type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${s
2717
2977
  type Runs<N extends number = number> = number | {
2718
2978
  bestOf: OddAtLeast3<N>;
2719
2979
  };
2720
- /** Options other than `environment` and `compare`, which are exclusive of each other. */
2980
+ /** Options other than `environments` and `compare`, which are exclusive of each other. */
2721
2981
  type EvalOptionFields<N extends number = number> = {
2722
2982
  /** Which agents run the eval; each harness is its own cell. */
2723
- harness?: HarnessOption;
2983
+ harnesses?: HarnessOption;
2724
2984
  /**
2725
- * The default machine when the resolved environment omits `sandbox`. Cascades like `harness`.
2985
+ * The default machine when the resolved environment omits `sandbox`. Cascades like `harnesses`.
2726
2986
  * Not an environment: it does not invent skills, files or a name.
2727
2987
  */
2728
2988
  sandbox?: SandboxDescriptor;
@@ -2730,7 +2990,7 @@ type EvalOptionFields<N extends number = number> = {
2730
2990
  * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
2731
2991
  * a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
2732
2992
  * Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
2733
- * a verdict, and anything else runs once, a pooled `evaluate.each` case included.
2993
+ * a verdict, and anything else runs once, a case of an eval with `cases` included.
2734
2994
  */
2735
2995
  runs?: Runs<N>;
2736
2996
  /** How each cell's attempts decide its outcome; default `'all'`. */
@@ -2784,7 +3044,7 @@ type EvalSource = {
2784
3044
  body: string;
2785
3045
  /**
2786
3046
  * sha256 of the case's data as canonical JSON, with secrets in its strings masked as the record
2787
- * masks them; set on each case of an `evaluate.each`.
3047
+ * masks them; set on each case of an eval with `cases`.
2788
3048
  */
2789
3049
  case?: string;
2790
3050
  /** sha256 of the canonical options. */
@@ -2792,45 +3052,38 @@ type EvalSource = {
2792
3052
  };
2793
3053
  /**
2794
3054
  * `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
2795
- * A compare with a `baseline` names every arm, so the cascaded `environment` is not applied, and
2796
- * the same object can't set both. Without a `baseline`, the cascaded `environment` (this object's
2797
- * own, or an inherited one) is the control.
3055
+ * A compare names every arm, so an inherited `environments` is not applied, and the same object
3056
+ * can't set both.
2798
3057
  */
2799
- type EnvironmentOrCompare<E> = {
2800
- /** What this group or eval runs in, and the control of a compare without a `baseline`. */
2801
- environment?: E;
3058
+ type EnvironmentOrCompare<E, Envs> = {
3059
+ /**
3060
+ * What this group or eval runs in: one environment, or a list, each its own cell beside
3061
+ * each harness. `{ add }` and `{ only }` edit the inherited list.
3062
+ */
3063
+ environments?: Envs;
2802
3064
  compare?: never;
2803
3065
  } | {
2804
- /** Forbidden beside `compare.baseline`, which names the control. */
2805
- environment?: never;
2806
- /** Environments to run side by side; `baseline` is the control. */
2807
- compare: CompareSpec<E> & {
2808
- baseline: E;
2809
- };
2810
- } | {
2811
- /** The control: the compare has no `baseline`. */
2812
- environment?: E;
2813
- /** Environments to run side by side against `environment`, the control. */
2814
- compare: CompareSpec<E> & {
2815
- baseline?: undefined;
2816
- };
3066
+ /** Forbidden beside `compare`, whose `control` names what the eval runs in. */
3067
+ environments?: never;
3068
+ /** Environments to run side by side; `control` is what each variant is measured against. */
3069
+ compare: CompareSpec<E>;
2817
3070
  };
2818
3071
  /** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
2819
- type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
3072
+ type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment, EnvironmentsOption>;
2820
3073
  type GroupOptions<N extends number = number> = EvalOptions<N>;
2821
3074
  /**
2822
3075
  * `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
2823
3076
  * by name or a fake judge, and environments without `extend`. The same `environment` / compare
2824
3077
  * object split as `EvalOptions`.
2825
3078
  */
2826
- type EvalOptionsData = Omit<EvalOptionFields, 'harness' | 'judge'> & {
3079
+ type EvalOptionsData = Omit<EvalOptionFields, 'harnesses' | 'judge'> & {
2827
3080
  /** Which agents run the eval; each harness is its own cell. */
2828
- harness?: HarnessData;
3081
+ harnesses?: HarnessData;
2829
3082
  /** Defaults for `c.judge()`, merged field by field down the cascade. */
2830
3083
  judge?: ConfigJudgeOptions;
2831
- } & EnvironmentOrCompare<EnvironmentData>;
3084
+ } & EnvironmentOrCompare<EnvironmentData, EnvironmentsData>;
2832
3085
  /** An `Environment` as data, as JSON carries it: without `extend` / `extendEnvironment`. */
2833
- type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend'>;
3086
+ type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend' | 'traits'>;
2834
3087
  /** The config file is plain data sent to workers, so its judge model can't be an object. */
2835
3088
  type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
2836
3089
  /** `'provider/model'`, or a fake judge. */
@@ -2852,11 +3105,11 @@ type Budget = {
2852
3105
  /** What evals get when they don't say otherwise: under groups and evals. */
2853
3106
  type ProfileDefaults = {
2854
3107
  /** The agents every eval runs on, unless a group or eval replaces them. */
2855
- harness?: HarnessDescriptor | HarnessDescriptor[];
3108
+ harnesses?: HarnessDescriptor | HarnessDescriptor[];
2856
3109
  /** The default machine when the resolved environment omits `sandbox`. */
2857
3110
  sandbox?: SandboxDescriptor;
2858
- /** What every eval runs in when it names no environment. */
2859
- environment?: Environment;
3111
+ /** What every eval runs in when it names no environment: one, or a list (a cell each). */
3112
+ environments?: Environment | Environment[];
2860
3113
  /** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
2861
3114
  runs?: Runs;
2862
3115
  /** How each cell's attempts decide its outcome. */
@@ -2885,7 +3138,13 @@ type ProfileLimit = {
2885
3138
  maxVariants: number;
2886
3139
  };
2887
3140
  /** Harness ids to allow; evals left with none are skipped. */
2888
- harness?: string[];
3141
+ harnesses?: string[];
3142
+ /**
3143
+ * Cells to run, by their environment's traits (with `compare`, the control's): each key's trait
3144
+ * must equal the value, or one of a list's, or for a list trait contain it. Evals left with no
3145
+ * cell are skipped.
3146
+ */
3147
+ traits?: Record<string, string | string[]>;
2889
3148
  };
2890
3149
  /** How a run is sized: chosen with `--profile`, `REEVAL_PROFILE` or `defaultProfile`. */
2891
3150
  type Profile = {
@@ -3013,14 +3272,17 @@ type ReevalConfig = {
3013
3272
  /** Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`. */
3014
3273
  include?: string[];
3015
3274
  /** The agents every eval runs on, unless a group or eval replaces them; default Cursor. */
3016
- harness?: HarnessDescriptor | HarnessDescriptor[];
3275
+ harnesses?: HarnessDescriptor | HarnessDescriptor[];
3017
3276
  /**
3018
3277
  * The default machine when the resolved environment omits `sandbox`. Not an environment:
3019
3278
  * it does not invent skills, files or a name. Implicit `docker()` when nobody sets one.
3020
3279
  */
3021
3280
  sandbox?: SandboxDescriptor;
3022
- /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
3023
- environment?: Environment;
3281
+ /**
3282
+ * What every eval runs in when it names no environment and has no compare: one, or a list (a
3283
+ * cell each).
3284
+ */
3285
+ environments?: Environment | Environment[];
3024
3286
  /**
3025
3287
  * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
3026
3288
  * variants runs 4 a side and anything else once.
@@ -3158,9 +3420,8 @@ type PhaseTimings = {
3158
3420
  type CellRole = 'control' | 'variant';
3159
3421
  /**
3160
3422
  * A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
3161
- * environment's sandbox id. With `compare`, the control is `baseline` (or the cascaded
3162
- * environment, or an empty `baseline`) and each variant is that environment on its own,
3163
- * including its sandbox.
3423
+ * environment's sandbox id. With `compare`, the control is `compare.control` and each variant is
3424
+ * that environment on its own, including its sandbox.
3164
3425
  */
3165
3426
  type Cell = {
3166
3427
  /** The harness id, the environment name, and the sandbox id, joined with `/`. */
@@ -3180,9 +3441,16 @@ type Cell = {
3180
3441
  * eight-character hash.
3181
3442
  */
3182
3443
  sandbox: string;
3444
+ /**
3445
+ * `variant` on a compare's variant cell, else `control`: the compare's control, or any cell of
3446
+ * an eval without `compare`, each environment of an `environments` grid included. A variant's
3447
+ * lift is against the `control` cell on its harness and sandbox.
3448
+ */
3183
3449
  role: CellRole;
3184
3450
  /** The variant's name, for variant cells. */
3185
3451
  variant?: string;
3452
+ /** The environment's merged traits; unset when it has none. Not part of the key. */
3453
+ traits?: Traits;
3186
3454
  };
3187
3455
  /** A plugin as an attempt placed it. */
3188
3456
  type PluginRecord = {
@@ -3358,6 +3626,13 @@ type AttemptImage = {
3358
3626
  digest: string | null;
3359
3627
  };
3360
3628
  };
3629
+ /** The image a `tart()` attempt's VM was cloned from. */
3630
+ type VmImage = {
3631
+ /** The image as `tart({ image })` names it. */
3632
+ ref: string;
3633
+ /** The OCI manifest digest (`sha256:…`) Tart had for it; null for a local VM, which has none. */
3634
+ digest: string | null;
3635
+ };
3361
3636
  /** A harness session start that timed out and was started again. */
3362
3637
  type StartRetry = {
3363
3638
  /** `<harness kind>: session did not become ready within <startTimeout>`. */
@@ -3488,10 +3763,11 @@ type CaptureStep = {
3488
3763
  };
3489
3764
  /** Which environment ran as the control, so the studio can show it. */
3490
3765
  type EvalCompareMeta = {
3766
+ /** The control's environment name. */
3491
3767
  baseline: string;
3492
3768
  /**
3493
- * Set when `compare.baseline` was omitted: the control is the eval's cascaded `environment`, or
3494
- * an empty environment named `baseline` when there was none.
3769
+ * Not written, since a compare names its control. A record that has it ran the eval's cascaded
3770
+ * environment (`'environment'`) or an empty one named `baseline` (`'empty'`) as the control.
3495
3771
  */
3496
3772
  from?: 'environment' | 'empty';
3497
3773
  };
@@ -3524,5 +3800,5 @@ type ResolvedGateway = {
3524
3800
  why: string;
3525
3801
  }>;
3526
3802
  };
3527
- export { FakeAnalysisModel as $, effortVerdict as $n, StatusRange as $t, CompareCut as A, UpstreamCount as An, OpenRouterGatewayDescriptor as At, EgressPolicy as B, TOKENS_UNREPORTED as Bn, ProfileLimit as Bt, CaptureWindow as C, HistoryCell as Cn, JudgeModel as Ct, ClaudeCodeHarnessDescriptor as D, RunInvocation as Dn, McpServer as Dt, ClaudeCodeEffort as E, Lift as En, JudgeSubject as Et, Diff as F, EffortContrast as Fn, PinnableKey as Ft, EnvironmentInput as G, cellSamples as Gn, ResolvedGateway as Gt, EnvVars as H, bothUnderATenth as Hn, QuestionInputs as Ht, DirectGatewayDescriptor as I, EffortMetric as In, PluginPart as It, EvalFn as J, effortChangeLabel as Jn, Secret as Jt, EnvironmentSpec as K, contrastEffort as Kn, SandboxDescriptor as Kt, DirectoryOptions as L, EffortSample as Ln, PluginSource as Lt, ConfigHelpers as M, DIRECTIONAL_MIN as Mn, PairFactor as Mt, ConfigJudgeOptions as N, EFFORT_ALPHA as Nn, PairVote as Nt, CodexHarnessDescriptor as O, RunRecord as On, Observation as Ot, CursorHarnessDescriptor as P, EFFORT_MIN_RUNS as Pn, PiHarnessDescriptor as Pt, ExecRetry as Q, effortSummary as Qn, SkillSource as Qt, DockerSandboxDescriptor as R, EffortVerdict as Rn, Profile as Rt, CaptureWait as S, FileRef as Sn, JudgeInputs as St, CellRole as T, JudgmentRecord as Tn, JudgeState as Tt, Environment as U, cellContrast as Un, QuestionOptions as Ut, EgressRecord as V, VERDICT_TEXT as Vn, Question as Vt, EnvironmentHelpers as W, cellEffort as Wn, ReevalConfig as Wt, EvalOptionsData as X, effortOf as Xn, ServeHandle as Xt, EvalOptions as Y, effortLabel as Yn, SecretTemplate as Yt, ExecResult as Z, effortSample as Zn, SetupStep as Zt, CapturePage as _, FamilyLift as _n, HostFile as _t, AttemptContext as a, VercelGatewayDescriptor as an, FakeVote as at, CaptureUseAction as b, FamilyRecord as bn, JudgeCallOptions as bt, BlindPair as c, VertexPublisher as cn, GatewayName as ct, CaptureCall as d, WebEventType as dn, GroupOptions as dt, TokenUsage as en, filesChanged as er, FakeAnswer as et, CaptureConsoleMessage as f, ArtifactRef as fn, HarnessData as ft, CaptureOptions as g, EvalEntry as gn, HistoryConfig as gt, CaptureKey as h, CheckTally as hn, HarnessOption as ht, AttemptAnswer as i, TurnStatus as in, tokenValues as ir, FakeScriptStep as it, CompareOptions as j, CellEffort as jn, Outcome as jt, CompareAnswer as k, StackSummary as kn, OpenCodeHarnessDescriptor as kt, Budget as l, Viewport as ln, GooseHarnessDescriptor as lt, CaptureInstruction as m, CellRecord as mn, HarnessEdit as mt, AnalysisConfig as n, Turn as nn, formatRatio as nr, FakeJudgeModel as nt, BackgroundHandle as o, Verdict as on, FxHarnessDescriptor as ot, CaptureFailedRequest as p, AttemptRecord as pn, HarnessDescriptor as pt, ErrorKind as q, directional as qn, SandboxWarning as qt, AntigravityHarnessDescriptor as r, TurnData as rn, formatSeconds as rr, FakeSandboxDescriptor as rt, BlindAttempt as s, VertexGatewayDescriptor as sn, GatewayDescriptor as st, AgentHarnessDescriptor as t, TranscriptEvent as tn, formatCount as tr, FakeHarnessDescriptor as tt, Capture as u, Wait as un, GrokBuildHarnessDescriptor as ut, CapturePageFile as v, FamilyLiftSummary as vn, JSONValue as vt, Cell as w, HistoryEntry as wn, JudgeOptions as wt, CaptureUseScript as x, FamilyVerdict as xn, JudgeFailure as xt, CaptureUse as y, FamilyRate as yn, JudgeArtifactRef as yt, Duration as z, Spread as zn, ProfileDefaults as zt };
3528
- //# sourceMappingURL=types-DGvexkcp.d.ts.map
3803
+ export { EvalFn as $, directional as $n, Secret as $t, CellRole as A, HistoryEntry as An, JudgeState as At, Diff as B, EFFORT_MIN_RUNS as Bn, PinnableKey as Bt, CaptureUseAction as C, FamilyLift as Cn, JSONValue as Ct, CaseFn as D, FamilyVerdict as Dn, JudgeInputs as Dt, CaptureWindow as E, FamilyRecord as En, JudgeFailure as Et, CompareCut as F, StackSummary as Fn, OpenRouterGatewayDescriptor as Ft, EgressPolicy as G, Spread as Gn, ProfileLimit as Gt, DirectoryOptions as H, EffortMetric as Hn, PluginSource as Ht, CompareOptions as I, UpstreamCount as In, Outcome as It, Environment as J, bothUnderATenth as Jn, QuestionOptions as Jt, EgressRecord as K, TOKENS_UNREPORTED as Kn, Question as Kt, ConfigHelpers as L, CellEffort as Ln, PairFactor as Lt, ClaudeCodeHarnessDescriptor as M, Lift as Mn, McpServer as Mt, CodexHarnessDescriptor as N, RunInvocation as Nn, Observation as Nt, CaseOptions as O, FileRef as On, JudgeModel as Ot, CompareAnswer as P, RunRecord as Pn, OpenCodeHarnessDescriptor as Pt, ErrorKind as Q, contrastEffort as Qn, SandboxWarning as Qt, ConfigJudgeOptions as R, DIRECTIONAL_MIN as Rn, PairVote as Rt, CaptureUse as S, EvalEntry as Sn, HostFile as St, CaptureWait as T, FamilyRate as Tn, JudgeCallOptions as Tt, DockerSandboxDescriptor as U, EffortSample as Un, Profile as Ut, DirectGatewayDescriptor as V, EffortContrast as Vn, PluginPart as Vt, Duration as W, EffortVerdict as Wn, ProfileDefaults as Wt, EnvironmentInput as X, cellEffort as Xn, ResolvedGateway as Xt, EnvironmentHelpers as Y, cellContrast as Yn, ReevalConfig as Yt, EnvironmentSpec as Z, cellSamples as Zn, SandboxDescriptor as Zt, CaptureInstruction as _, WebEventType as _n, HarnessData as _t, AttemptContext as a, TartSandboxDescriptor as an, effortVerdict as ar, FakeAnswer as at, CapturePage as b, CellRecord as bn, HarnessOption as bt, BlindPair as c, Turn as cn, formatRatio as cr, FakeSandboxDescriptor as ct, BoatSandboxDescriptor as d, VercelGatewayDescriptor as dn, FxHarnessDescriptor as dt, SecretTemplate as en, effortChangeLabel as er, EvalOptions as et, Budget as f, Verdict as fn, GatewayDescriptor as ft, CaptureFailedRequest as g, Wait as gn, GroupOptions as gt, CaptureConsoleMessage as h, Viewport as hn, GrokBuildHarnessDescriptor as ht, AttemptAnswer as i, StatusRange as in, effortSummary as ir, FakeAnalysisModel as it, ClaudeCodeEffort as j, JudgmentRecord as jn, JudgeSubject as jt, Cell as k, HistoryCell as kn, JudgeOptions as kt, BoatApiKey as l, TurnData as ln, formatSeconds as lr, FakeScriptStep as lt, CaptureCall as m, VertexPublisher as mn, GooseHarnessDescriptor as mt, AnalysisConfig as n, SetupStep as nn, effortOf as nr, ExecResult as nt, BackgroundHandle as o, TokenUsage as on, filesChanged as or, FakeHarnessDescriptor as ot, Capture as p, VertexGatewayDescriptor as pn, GatewayName as pt, EnvVars as q, VERDICT_TEXT as qn, QuestionInputs as qt, AntigravityHarnessDescriptor as r, SkillSource as rn, effortSample as rr, ExecRetry as rt, BlindAttempt as s, TranscriptEvent as sn, formatCount as sr, FakeJudgeModel as st, AgentHarnessDescriptor as t, ServeHandle as tn, effortLabel as tr, EvalOptionsData as tt, BoatMachineType as u, TurnStatus as un, tokenValues as ur, FakeVote as ut, CaptureKey as v, ArtifactRef as vn, HarnessDescriptor as vt, CaptureUseScript as w, FamilyLiftSummary as wn, JudgeArtifactRef as wt, CapturePageFile as x, CheckTally as xn, HistoryConfig as xt, CaptureOptions as y, AttemptRecord as yn, HarnessEdit as yt, CursorHarnessDescriptor as z, EFFORT_ALPHA as zn, PiHarnessDescriptor as zt };
3804
+ //# sourceMappingURL=types-DBwFsGem.d.ts.map