reeval 0.17.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
  4. package/dist/adapter-C8xPuXYq.js.map +1 -0
  5. package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
  6. package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
  7. package/dist/agents-C6zhkxp3.js +1151 -0
  8. package/dist/agents-C6zhkxp3.js.map +1 -0
  9. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  10. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  11. package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
  12. package/dist/app-files-DT5mJyV4.js.map +1 -0
  13. package/dist/assemble-UPCRrtOz.js +1 -0
  14. package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
  15. package/dist/assemble-t6h018An.js.map +1 -0
  16. package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
  17. package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
  18. package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
  19. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
  20. package/dist/boat-BjLqFJIZ.js +108 -0
  21. package/dist/boat-BjLqFJIZ.js.map +1 -0
  22. package/dist/boat-CJCtzfB1.js +322 -0
  23. package/dist/boat-CJCtzfB1.js.map +1 -0
  24. package/dist/bridge/index.mjs +300 -78
  25. package/dist/capture-DoxFUN3J.js +1 -0
  26. package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
  27. package/dist/capture-iBzFiC0j.js.map +1 -0
  28. package/dist/cases-Dy0mifal.js +1 -0
  29. package/dist/cases-z9E6DCd5.js +639 -0
  30. package/dist/cases-z9E6DCd5.js.map +1 -0
  31. package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
  32. package/dist/cli-20Fpvs5H.js.map +1 -0
  33. package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
  34. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
  37. package/dist/client-KM4C5r5_.js.map +1 -0
  38. package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
  39. package/dist/command-B5OyHpOu.js.map +1 -0
  40. package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
  41. package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
  42. package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
  43. package/dist/command-CRh-cb6Z.js.map +1 -0
  44. package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
  45. package/dist/command-aUJvENwy.js.map +1 -0
  46. package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
  47. package/dist/commands-CWy-WndG.js.map +1 -0
  48. package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
  49. package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
  50. package/dist/config-B9mB1kF5.js.map +1 -0
  51. package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
  52. package/dist/context-Bkv3ndjq.js.map +1 -0
  53. package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
  54. package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
  55. package/dist/cost-Cr26Alhq.js +1 -0
  56. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
  57. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
  58. package/dist/db-P2fXw4nx.js +2 -0
  59. package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
  60. package/dist/db-zgC1tvKB.js.map +1 -0
  61. package/dist/debug-CF4p4Lp_.js +1 -0
  62. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  63. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  64. package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
  65. package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
  66. package/dist/default-effort-DGhQmu95.js.map +1 -0
  67. package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
  68. package/dist/dist-CAFD1yfv.js.map +1 -0
  69. package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
  70. package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
  71. package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
  72. package/dist/docker-DAMnm3L9.js.map +1 -0
  73. package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
  74. package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
  75. package/dist/doctor-B2PohneB.js.map +1 -0
  76. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  77. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  78. package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
  79. package/dist/environment-DB8o3hGi.js.map +1 -0
  80. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  81. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  82. package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
  83. package/dist/eval-source-D3sbAzt-.js.map +1 -0
  84. package/dist/evaluate-COdzf7Hl.d.ts +92 -0
  85. package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
  86. package/dist/evaluate-CwKeN_fo.js.map +1 -0
  87. package/dist/files-B4KQUiM7.js +1 -0
  88. package/dist/files-BEwfjD8J.js +64 -0
  89. package/dist/files-BEwfjD8J.js.map +1 -0
  90. package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
  91. package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
  92. package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
  93. package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
  94. package/dist/files-C1Rk9FYA.js +1 -0
  95. package/dist/files-C3mCC8Mn.js +1 -0
  96. package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
  97. package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
  98. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  99. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  100. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  101. package/dist/files-DlQshsks.js.map +1 -0
  102. package/dist/files-DnlA1A7N.js +1 -0
  103. package/dist/gate-CgYdaprG.js +49 -0
  104. package/dist/gate-CgYdaprG.js.map +1 -0
  105. package/dist/gateway.d.ts +1 -1
  106. package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
  107. package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
  108. package/dist/harness-C2berMiM.js +105 -0
  109. package/dist/harness-C2berMiM.js.map +1 -0
  110. package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
  111. package/dist/harness-DXv2gnaT.js.map +1 -0
  112. package/dist/harness.d.ts +1 -1
  113. package/dist/harness.js +1 -1
  114. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  115. package/dist/ids-C_advdJA.js.map +1 -0
  116. package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
  117. package/dist/images-BNgcR77K.js.map +1 -0
  118. package/dist/index.d.ts +23 -3
  119. package/dist/index.js +5 -5
  120. package/dist/init-4RPeD-K-.js +1 -0
  121. package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
  122. package/dist/init-DbuZVRrW.js.map +1 -0
  123. package/dist/ink-list-BpfidQ1h.js +274 -0
  124. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  125. package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
  126. package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
  127. package/dist/judge-COMXT1Hx.js +230 -0
  128. package/dist/judge-COMXT1Hx.js.map +1 -0
  129. package/dist/judge.d.ts +1 -1
  130. package/dist/judge.js +1 -1
  131. package/dist/judgments-CzgpsTqv.js +246 -0
  132. package/dist/judgments-CzgpsTqv.js.map +1 -0
  133. package/dist/kinds-C1oY3qyC.js +697 -0
  134. package/dist/kinds-C1oY3qyC.js.map +1 -0
  135. package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
  136. package/dist/layout-Cm85tQYk.js.map +1 -0
  137. package/dist/list-DuThsE28.js +1348 -0
  138. package/dist/list-DuThsE28.js.map +1 -0
  139. package/dist/live-CJCK11nh.js +1 -0
  140. package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
  141. package/dist/live-CgYbO4Te.js.map +1 -0
  142. package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
  143. package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
  144. package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
  145. package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
  146. package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
  147. package/dist/model-Cv4GuTsZ.js +1 -0
  148. package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
  149. package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
  150. package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
  151. package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
  152. package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
  153. package/dist/model-V3mMXp6x.js.map +1 -0
  154. package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
  155. package/dist/node-D7pAU--0.js.map +1 -0
  156. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
  157. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  158. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
  159. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  160. package/dist/options-CHubslQf.js +1 -0
  161. package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
  162. package/dist/pairs-BGb134Vu.js.map +1 -0
  163. package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
  164. package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
  165. package/dist/plain-BYUf_fUV.js +267 -0
  166. package/dist/plain-BYUf_fUV.js.map +1 -0
  167. package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
  168. package/dist/plan-Cgq9NCEX.js.map +1 -0
  169. package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
  170. package/dist/plugins-BkmPxDiY.js.map +1 -0
  171. package/dist/preflight-4gWSy0Pe.js +1 -0
  172. package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
  173. package/dist/preflight-BElE_q0u.js.map +1 -0
  174. package/dist/preflight-CtlloeOQ.js +68 -0
  175. package/dist/preflight-CtlloeOQ.js.map +1 -0
  176. package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
  177. package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
  178. package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
  179. package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
  180. package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
  181. package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
  182. package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
  183. package/dist/provenance-zsc4--dP.js.map +1 -0
  184. package/dist/provider-BnUoz4KB.js +1 -0
  185. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  186. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  187. package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
  188. package/dist/providers-B8xYMeah.js.map +1 -0
  189. package/dist/providers-CVu8e0xS.js +1 -0
  190. package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
  191. package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
  192. package/dist/query-DIfkUcPh.js +1 -0
  193. package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
  194. package/dist/questions-CWlR9ffu.js.map +1 -0
  195. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  196. package/dist/record-6IUzwP9K.js.map +1 -0
  197. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  198. package/dist/redact-BhjtnjrU.js.map +1 -0
  199. package/dist/remarks-CAvTSq16.js +1 -0
  200. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  201. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  202. package/dist/remarks-DPCONFxZ.js +138 -0
  203. package/dist/remarks-DPCONFxZ.js.map +1 -0
  204. package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
  205. package/dist/remarks-vtkPmQb9.js.map +1 -0
  206. package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
  207. package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
  208. package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
  209. package/dist/resolve-Do0ZaYH6.js.map +1 -0
  210. package/dist/resolve-amw_2Odz.js +1 -0
  211. package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
  212. package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
  213. package/dist/resume-AW5oXzUc.js +691 -0
  214. package/dist/resume-AW5oXzUc.js.map +1 -0
  215. package/dist/retry-ChzISxmy.js +105 -0
  216. package/dist/retry-ChzISxmy.js.map +1 -0
  217. package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
  218. package/dist/runtime-BpW1f1Mx.js.map +1 -0
  219. package/dist/sandbox/boat.d.ts +26 -0
  220. package/dist/sandbox/boat.js +37 -0
  221. package/dist/sandbox/boat.js.map +1 -0
  222. package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
  223. package/dist/sandbox-kSgr7vo5.js.map +1 -0
  224. package/dist/sandbox.d.ts +20 -2
  225. package/dist/sandbox.js +1 -1
  226. package/dist/server-2_gUnVvf.js +1 -0
  227. package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
  228. package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
  229. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  230. package/dist/shared-Cfi45Bvy.js.map +1 -0
  231. package/dist/slug-IHh3f770.js +136 -0
  232. package/dist/slug-IHh3f770.js.map +1 -0
  233. package/dist/source-CY3etKzw.js +1 -0
  234. package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
  235. package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
  236. package/dist/src-6Zzyg4Qd.js +2 -0
  237. package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
  238. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  239. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  240. package/dist/src-BRMsk9nh.js +2 -0
  241. package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
  242. package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
  243. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  244. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  245. package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
  246. package/dist/src-Ck2r_EMT.js.map +1 -0
  247. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  248. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  249. package/dist/src-DB7N8TYJ.js +2 -0
  250. package/dist/src-DPDaAYwh.js +2 -0
  251. package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
  252. package/dist/src-DtodMT59.js +1 -0
  253. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  254. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  255. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  256. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  257. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  258. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  259. package/dist/src-T-OeuJLr.js +2 -0
  260. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  261. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  262. package/dist/src-zntEDlnx.js +3 -0
  263. package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
  264. package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
  265. package/dist/state-kimtIhBP.js +1 -0
  266. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  267. package/dist/stats-C18W4xPX.js.map +1 -0
  268. package/dist/suite-CKBa6MKt.js +2 -0
  269. package/dist/suite-COglUhPd.js +1272 -0
  270. package/dist/suite-COglUhPd.js.map +1 -0
  271. package/dist/summary-DU5zaibx.js +148 -0
  272. package/dist/summary-DU5zaibx.js.map +1 -0
  273. package/dist/tart-BTVXp8rz.js +373 -0
  274. package/dist/tart-BTVXp8rz.js.map +1 -0
  275. package/dist/tart-cli-BIC9HqPg.js +37 -0
  276. package/dist/tart-cli-BIC9HqPg.js.map +1 -0
  277. package/dist/tart-cli-BMrkmBjI.js +1 -0
  278. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  279. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  280. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  281. package/dist/tart-ref-DYZijjrU.js +55 -0
  282. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  283. package/dist/testing.d.ts +47 -37
  284. package/dist/testing.js +4 -4
  285. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  286. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  287. package/dist/trace-Dx2NBE1O.js +311 -0
  288. package/dist/trace-Dx2NBE1O.js.map +1 -0
  289. package/dist/traits-DKnYkqeK.js +66 -0
  290. package/dist/traits-DKnYkqeK.js.map +1 -0
  291. package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
  292. package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
  293. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  294. package/dist/types-C2mLpph3.js.map +1 -0
  295. package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
  296. package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
  297. package/dist/vercel-CGlSBv63.js.map +1 -0
  298. package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
  299. package/dist/vertex-Eb2wqahk.js.map +1 -0
  300. package/dist/vertex-O88TCwgY.js +1 -0
  301. package/dist/view/index.d.ts +238 -90
  302. package/dist/view/index.js +11 -9
  303. package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
  304. package/dist/views-DAIiLiV3.js.map +1 -0
  305. package/dist/views-DtRhj2Tn.js +2 -0
  306. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
  307. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
  308. package/dist/workspace-CcQRT31C.js +1 -0
  309. package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
  310. package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
  311. package/package.json +26 -11
  312. package/skills/eval-rubric/SKILL.md +31 -4
  313. package/dist/adapter-DyvDj4ZN.js.map +0 -1
  314. package/dist/app-files-hpRT5lj_.js.map +0 -1
  315. package/dist/assemble-DTXwCL3m.js +0 -1
  316. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  317. package/dist/capture-BT2fAmpZ.js +0 -1
  318. package/dist/capture-Bdcx9Gap.js.map +0 -1
  319. package/dist/cli-CGd51thd.js.map +0 -1
  320. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  321. package/dist/client-CF-DOjOU.js.map +0 -1
  322. package/dist/command-CsfcwMyc.js.map +0 -1
  323. package/dist/command-CyIb9BSC.js.map +0 -1
  324. package/dist/command-TLmY-rYp.js.map +0 -1
  325. package/dist/commands-C54S45d7.js.map +0 -1
  326. package/dist/config-ImY6AlRE.js.map +0 -1
  327. package/dist/context-BajfiH0S.js.map +0 -1
  328. package/dist/contrasts-DtSX8E_V.js +0 -101
  329. package/dist/contrasts-DtSX8E_V.js.map +0 -1
  330. package/dist/cost-CMzY9VXv.js +0 -1
  331. package/dist/db-ZIk6S2CD.js +0 -2
  332. package/dist/db-sHGpaLo9.js.map +0 -1
  333. package/dist/debug-BnXV7JbX.js +0 -1
  334. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  335. package/dist/dist-ByYk_wAs.js.map +0 -1
  336. package/dist/docker--2ytOHcu.js.map +0 -1
  337. package/dist/doctor-eE65YpEv.js.map +0 -1
  338. package/dist/environment-Cy3FVLR5.js.map +0 -1
  339. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  340. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  341. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  342. package/dist/files-B3d-8Fe0.js +0 -1
  343. package/dist/files-COW5JBJM.js +0 -1
  344. package/dist/files-Do4Rp3yZ.js +0 -1
  345. package/dist/files-p-uTdOjA.js.map +0 -1
  346. package/dist/gate-BoDKeuJu.js.map +0 -1
  347. package/dist/harness-Ch2P81sx.js.map +0 -1
  348. package/dist/ids-DuK6xl4w.js.map +0 -1
  349. package/dist/images-BVZHDf_z.js.map +0 -1
  350. package/dist/init-BWnMbUnc.js.map +0 -1
  351. package/dist/init-s1jN-PCI.js +0 -1
  352. package/dist/judge-ByEUlzOM.js.map +0 -1
  353. package/dist/live-DmJix4c8.js +0 -1
  354. package/dist/live-d_5a5iUv.js.map +0 -1
  355. package/dist/model-8uZBYqV5.js +0 -1
  356. package/dist/model-DOiCiMCD.js.map +0 -1
  357. package/dist/node-CwP1G178.js.map +0 -1
  358. package/dist/opencode-providers-yNKyXD49.js.map +0 -1
  359. package/dist/plan-CIfOFhth.js.map +0 -1
  360. package/dist/plugins-BEs2Am3L.js.map +0 -1
  361. package/dist/preflight-50Qw5e5K.js.map +0 -1
  362. package/dist/provenance-Cc8fFE7Z.js.map +0 -1
  363. package/dist/provider-DCdTHQpk.js +0 -1
  364. package/dist/providers-CP_bBtYm.js +0 -1
  365. package/dist/providers-DZSTlC9R.js.map +0 -1
  366. package/dist/query-kFPuqBv1.js +0 -1
  367. package/dist/questions-C_Y-oXW6.js.map +0 -1
  368. package/dist/record-DhzwkSXc.js.map +0 -1
  369. package/dist/redact-BEEGc37e.js.map +0 -1
  370. package/dist/remarks-0EXMh6AD.js.map +0 -1
  371. package/dist/remarks-BL18Id09.js +0 -1
  372. package/dist/remarks-DeZZT8PK.js +0 -116
  373. package/dist/remarks-DeZZT8PK.js.map +0 -1
  374. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  375. package/dist/resolve-BJSay360.js +0 -1
  376. package/dist/resolve-B_JlQ0La.js.map +0 -1
  377. package/dist/resume-CexsWuGK.js +0 -434
  378. package/dist/resume-CexsWuGK.js.map +0 -1
  379. package/dist/runtime-CSZ1hjlN.js.map +0 -1
  380. package/dist/sandbox-B5W8lLL1.js.map +0 -1
  381. package/dist/server-SBzrf35T.js +0 -1
  382. package/dist/shared-ogqPJ0uo.js.map +0 -1
  383. package/dist/source-Dn1uqPon.js +0 -1
  384. package/dist/src-BKDZO3ra.js +0 -2
  385. package/dist/src-BxrHipV0.js +0 -3
  386. package/dist/src-C9elednc.js +0 -2
  387. package/dist/src-CM7xdJFA.js +0 -2
  388. package/dist/src-CWEGknZo.js +0 -2
  389. package/dist/src-Cpy9MNS3.js.map +0 -1
  390. package/dist/src-DVaAKCsH.js +0 -1
  391. package/dist/src-DurP--_D.js +0 -2
  392. package/dist/state-Cicc9Oa5.js +0 -1
  393. package/dist/stats-D-Gqwqt0.js.map +0 -1
  394. package/dist/suite-BYJ2Q0S-.js +0 -2
  395. package/dist/suite-BtHPBFYL.js +0 -544
  396. package/dist/suite-BtHPBFYL.js.map +0 -1
  397. package/dist/summary-zP3cCXDN.js.map +0 -1
  398. package/dist/types-C4OkYhZg.js.map +0 -1
  399. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  400. package/dist/vertex-BaAz44YN.js +0 -1
  401. package/dist/views-CKoN5NAx.js.map +0 -1
  402. package/dist/views-CmyX255j.js +0 -2
  403. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -1,61 +1,3 @@
1
- /** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
2
- type DeclaredMode = 'run' | 'skip' | 'only';
3
- /** A `group`: options its evals and groups inherit, and its children in declaration order. */
4
- type DeclaredGroup = {
5
- type: 'group';
6
- /** The group's name, as passed to `group`. */
7
- name: string;
8
- mode: DeclaredMode;
9
- /** The group's own options, before the cascade. */
10
- options: EvalOptionsData;
11
- children: DeclaredNode[];
12
- };
13
- /** An `evaluate`: one eval, planned into a test per attempt. */
14
- type DeclaredEval = {
15
- type: 'eval';
16
- /** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
17
- name: string;
18
- mode: DeclaredMode;
19
- /** The eval's own options, before the cascade. */
20
- options: EvalOptionsData;
21
- /** Names the eval's body to whoever runs it. */
22
- body: string;
23
- source: DeclaredSource;
24
- /** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
25
- family?: DeclaredFamily;
26
- };
27
- /**
28
- * An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
29
- * text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
30
- * whitespace, non-ASCII characters as themselves).
31
- */
32
- type DeclaredSource = {
33
- /** sha256 of the body function's source text. */
34
- body: string;
35
- /** sha256 of the case's data; set on each case of an `evaluate.each`. */
36
- case?: string;
37
- };
38
- /** Where a case of a pooled `evaluate.each` sits in its list. */
39
- type DeclaredFamily = {
40
- /** The name template passed to `evaluate.each`. */
41
- name: string;
42
- /** The case's index, from 0. */
43
- case: number;
44
- /** How many cases the list has. */
45
- cases: number;
46
- };
47
- /** `evaluate.todo` or `group.todo`. */
48
- type DeclaredTodo = {
49
- type: 'todo';
50
- /** The name passed to `.todo`. */
51
- name: string;
52
- };
53
- /** One node of the tree: a group, an eval or a todo. */
54
- type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
55
- /** An eval file's top-level groups, evals and todos, in declaration order. */
56
- type DeclarationTree = {
57
- children: DeclaredNode[];
58
- };
59
1
  /** Below this two-sided p, an effort difference is an effect. */
60
2
  declare const EFFORT_ALPHA = 0.05;
61
3
  /** Some counted attempts reported no usage. Not a verdict; never printed as one. */
@@ -315,13 +257,23 @@ type UpstreamCount = {
315
257
  provider: string;
316
258
  calls: number;
317
259
  };
260
+ /** An infra error an attempt ended with before it ran again, and when that try started. */
261
+ type AttemptRetry = {
262
+ error: AttemptError;
263
+ at: string | null;
264
+ };
318
265
  type AttemptRecord = {
319
- /** `<eval key> # <cell key>#<n>`: unique in the run. */
266
+ /**
267
+ * `<eval key> # <cell key>#<n>`, or `<eval key> [<case>] # <cell key>#<n>` on an eval with
268
+ * `cases`: unique in the run.
269
+ */
320
270
  key: string;
321
- /** `<cell key>#<n>`: unique in its evaluation. */
271
+ /** `<cell key>#<n>`, or `<cell key>#<case>#<n>` on an eval with `cases`: unique in its evaluation. */
322
272
  id: string;
323
273
  /** The cell's key; see `EvalEntry.cells`. */
324
274
  cell: string;
275
+ /** The case's key, on an eval with `cases`; see `EvalEntry.cases`. */
276
+ case?: string;
325
277
  /** Run number within the cell, from 1. */
326
278
  n: number;
327
279
  /**
@@ -335,6 +287,11 @@ type AttemptRecord = {
335
287
  startedAt: string | null;
336
288
  /** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
337
289
  imageId: string | null;
290
+ /**
291
+ * The image a `tart()` attempt's VM was cloned from, with the digest Tart had for it. Absent in
292
+ * other sandboxes, and in attempts that didn't reach their sandbox.
293
+ */
294
+ vmImage?: VmImage;
338
295
  /** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
339
296
  env: {
340
297
  vars: string[];
@@ -463,6 +420,13 @@ type AttemptRecord = {
463
420
  judgements: JudgmentRecord[];
464
421
  /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
465
422
  startRetries?: StartRetry[];
423
+ /**
424
+ * The infra errors this attempt ended with before `reeval run --retry-errors` or `--resume` ran
425
+ * it again, oldest first: each `{ error, at }`, `at` being when that try started (null when it
426
+ * never did). The attempt's own `outcome` and `error` are its last try's. Absent when it never
427
+ * ran again.
428
+ */
429
+ retries?: AttemptRetry[];
466
430
  /**
467
431
  * Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
468
432
  * without `hosts` in their environment, so they hold its real value while the agent works and
@@ -516,6 +480,11 @@ type AttemptRecord = {
516
480
  * in the attempt's directory, `<n>` the question's place in the eval's `questions`.
517
481
  */
518
482
  answers?: AttemptAnswer[];
483
+ /**
484
+ * Each `c.observe(name, value)` the attempt made, one per name in first-call order with its last
485
+ * value; unset without any. They annotate: `outcome` never depends on them.
486
+ */
487
+ observations?: Observation[];
519
488
  };
520
489
  type EvalEntry = {
521
490
  /** `file › group › … › name`, as shown. */
@@ -550,11 +519,23 @@ type EvalEntry = {
550
519
  /**
551
520
  * Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
552
521
  * `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
553
- * cell's failures don't fail the evaluation.
522
+ * cell's failures don't fail the evaluation. With `cases`, a cell's outcome is over every case.
554
523
  */
555
524
  outcome: Outcome;
525
+ /**
526
+ * In records from before an eval with `cases` was one entry, `family` on a case whose
527
+ * `gate: { passRate }` its family held over the mean of its cases. Never written now: see
528
+ * `CaseRecord.verdict`.
529
+ */
530
+ verdict?: 'family';
556
531
  /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
557
532
  todo: boolean;
533
+ /**
534
+ * Declared not to run: `evaluate.skip`, inside a `group.skip`, or left out by an `.only` in its
535
+ * file. Its outcome is `skipped`. Unset otherwise, so in a run that isn't `complete` a skipped
536
+ * eval with neither `todo` nor `skip` hasn't run yet. Unset on records from before it.
537
+ */
538
+ skip?: true;
558
539
  /** pass@1, pass / (pass + fail), over the gated cells' attempts; infra errors and skips excluded. */
559
540
  passRate: number | null;
560
541
  cells: CellRecord[];
@@ -568,13 +549,63 @@ type EvalEntry = {
568
549
  * with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
569
550
  */
570
551
  answers?: CompareAnswer[];
571
- /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
552
+ /**
553
+ * In records from before an eval with `cases` was one entry, set on each case, which was an
554
+ * entry of its own. Never written now; reading such a record folds the cases into one entry.
555
+ */
572
556
  family?: {
573
557
  id: string;
574
558
  name: string;
575
559
  case: number;
576
560
  cases: number;
577
561
  };
562
+ /**
563
+ * The environments (and harnesses) `where` left out, one entry per case and environment,
564
+ * including those of a case it left out everywhere; unset when none.
565
+ */
566
+ excluded?: ExcludedPair[];
567
+ /**
568
+ * An eval with `cases`: each case that has attempts in this run, in list order, with its own
569
+ * cells. `cells` and `attempts` are over every case. Unset without cases.
570
+ */
571
+ cases?: CaseRecord[];
572
+ /** With `cases`: how many the list has, `cases` holding those with attempts in this run. */
573
+ caseCount?: number;
574
+ /**
575
+ * With `cases` and `compare`: each variant cell against its control, paired by case. Its
576
+ * `minLift` gates the lift's lower bound, and a lift below it fails the variant cell.
577
+ */
578
+ lifts?: FamilyLift[];
579
+ };
580
+ /** One case of an eval with `cases`, and how its attempts went. */
581
+ type CaseRecord = {
582
+ /** The case's key: `nextjs` for the case keyed `nextjs`. */
583
+ key: string;
584
+ /** Its place in the case list, from 0. */
585
+ index: number;
586
+ /**
587
+ * The name it ran under in a record from before an eval with `cases` was one entry, when that
588
+ * isn't `<eval name> [<key>]` (a name template's, `fixes GH-101` for `fixes $id`).
589
+ */
590
+ name?: string;
591
+ /** `EvalEntry.source.case`: the hash of the case's data. */
592
+ hash?: string;
593
+ /**
594
+ * What the case did, over its own cells by each cell's gate. Under a `gate: { passRate }`, the
595
+ * eval's cell gates on the mean over the cases, so `verdict` says this outcome decides nothing.
596
+ */
597
+ outcome: Outcome;
598
+ /** `family` when a cell's `gate: { passRate }` gates the mean over the cases, not this case. */
599
+ verdict?: 'family';
600
+ /** pass@1 over its gated cells' attempts. */
601
+ passRate: number | null;
602
+ fingerprint: string;
603
+ /** Its own cells: their attempts are those with this `case`. */
604
+ cells: CellRecord[];
605
+ /** The environments (and harnesses) `where` left this case out of; unset when none. */
606
+ excluded?: ExcludedPair[];
607
+ /** The eval's pairwise questions, answered over this case's attempts; unset without any. */
608
+ answers?: CompareAnswer[];
578
609
  };
579
610
  /** A 95% interval on a pooled rate or difference. */
580
611
  type FamilyInterval = {
@@ -588,7 +619,7 @@ type FamilyInterval = {
588
619
  */
589
620
  method: 'wilson' | 'newcombe-paired' | 't-cases';
590
621
  };
591
- /** One cell's pass@1 over a family's cases. */
622
+ /** One cell's pass@1 over an eval's cases. */
592
623
  type FamilyRate = {
593
624
  /** The cell's key, the same in every case. */
594
625
  cell: string;
@@ -600,13 +631,19 @@ type FamilyRate = {
600
631
  /** The mean of the cases' pass@1; at one attempt a case, `pass / counted`. */
601
632
  rate: number;
602
633
  interval: FamilyInterval;
634
+ /** The cell's `gate: { passRate }`: with cases it gates `rate`, not each case's pass@1. */
635
+ passRate?: number;
636
+ /** Set with `passRate`: `fail` when `rate` is below it. */
637
+ outcome?: 'pass' | 'fail';
603
638
  };
639
+ /** A cell's pass@1 over an eval's cases. */
640
+ type CellRate = Omit<FamilyRate, 'cell'>;
604
641
  /**
605
642
  * `lift` / `drop`: the interval excludes 0. `noise`: it doesn't. `insufficient`: fewer than 6
606
643
  * discordant pairs (one attempt a case) or 6 paired cases (several), where no test can decide.
607
644
  */
608
645
  type FamilyVerdict = 'lift' | 'drop' | 'noise' | 'insufficient';
609
- /** One effort metric over a family's paired cases. */
646
+ /** One effort metric over an eval's paired cases. */
610
647
  type FamilyEffortMetric = {
611
648
  /** Cases with a value on both sides. */
612
649
  pairs: number;
@@ -627,7 +664,7 @@ type FamilyEffort = {
627
664
  /** Over the cases where every counted attempt reported usage; omitted when none did. */
628
665
  tokens?: FamilyEffortMetric;
629
666
  };
630
- /** A variant cell against its control over a family's cases, paired by case. */
667
+ /** A variant cell against its control over an eval's cases, paired by case. */
631
668
  type FamilyLift = {
632
669
  /** The variant cell's key. */
633
670
  variant: string;
@@ -651,13 +688,17 @@ type FamilyLift = {
651
688
  /** The interval spans 0. */
652
689
  withinNoise: boolean;
653
690
  verdict: FamilyVerdict;
654
- /** The compare's `minLift`: on a family it gates this lower bound, not each case's. */
691
+ /** The compare's `minLift`: with cases it gates this lower bound, not each case's. */
655
692
  minLift?: number;
656
693
  /** `fail` when `interval.lo` is below `minLift`. */
657
694
  outcome: 'pass' | 'fail';
658
695
  effort?: FamilyEffort;
659
696
  };
660
- /** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an evaluation. */
697
+ /**
698
+ * In records from before an eval with `cases` was one entry: the eval over its cases, each case
699
+ * an entry of its own. Reading such a record folds it into the eval's `cases`, `lifts` and each
700
+ * cell's `rate`.
701
+ */
661
702
  type FamilyRecord = {
662
703
  /** `file › group › … › template`. */
663
704
  id: string;
@@ -667,13 +708,24 @@ type FamilyRecord = {
667
708
  name: string;
668
709
  /** Cases in the list. */
669
710
  cases: number;
670
- /** Cases with an evaluation in this run; fewer when a filter or tag left some out. */
711
+ /**
712
+ * Cases with an attempt that ran (not skipped) in this run; fewer when a filter or tag left
713
+ * some out, or an interrupted run or a filtered resume hasn't run them yet.
714
+ */
671
715
  ran: number;
672
716
  /** One per cell. */
673
717
  rates: FamilyRate[];
674
718
  /** One per variant cell, with `compare`. */
675
719
  lifts: FamilyLift[];
676
- /** `fail` when a lift is below its `minLift`. The cases still pass or fail on their own. */
720
+ /**
721
+ * Every case and environment (and harness) pair `where` left out, including those of a case it
722
+ * left out everywhere, which has no evaluation; unset when none.
723
+ */
724
+ excluded?: ExcludedPair[];
725
+ /**
726
+ * `fail` when a lift is below its `minLift` or a rate below its `passRate`. A case still fails
727
+ * on its own under `gate: 'all'`.
728
+ */
677
729
  outcome: 'pass' | 'fail';
678
730
  };
679
731
  /** A variant cell's pass@1 against the control's, on the same harness and sandbox. */
@@ -710,6 +762,7 @@ type FingerprintParts = {
710
762
  /** Over the harness descriptor, less concurrency caps. */
711
763
  harness: string;
712
764
  environment: string | null;
765
+ /** The Docker image's id, or the digest of the image a `tart()` VM was cloned from. */
713
766
  image: string | null;
714
767
  agentCli: string | null;
715
768
  /** Over the judge's model, threshold and version. */
@@ -770,7 +823,13 @@ type CellRecord = Cell & {
770
823
  * reports the worst outcome. A variant below its `minLift` fails too.
771
824
  */
772
825
  outcome: Outcome;
826
+ /** Unset on an eval with several `cases`, whose variants have `EvalEntry.lifts`. */
773
827
  lift?: Lift;
828
+ /**
829
+ * On an eval with several `cases`: the cell's pass@1 over them. A `gate: { passRate }` gates
830
+ * `rate.rate`, the mean of the cases' pass@1, and sets `rate.outcome`.
831
+ */
832
+ rate?: CellRate;
774
833
  /**
775
834
  * Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
776
835
  * attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
@@ -817,6 +876,24 @@ type Cost = {
817
876
  usage: TokenUsage | null;
818
877
  source: 'reported' | 'unreported';
819
878
  };
879
+ /**
880
+ * The `reeval run` settings that shaped a run's plan, as it was first started. A resume keeps
881
+ * them, so it finishes the attempts the run planned.
882
+ */
883
+ type RunInvocation = {
884
+ /** `--runs`; unset when each cell ran its own `runs`. */
885
+ runs?: number;
886
+ /** The profile the run was sized with; unset when it had none. */
887
+ profile?: string;
888
+ /** `--concurrency` or `REEVAL_CONCURRENCY`; unset when the config or the machine sized it. */
889
+ concurrency?: number;
890
+ /** The filter arguments. */
891
+ filters: string[];
892
+ /** `--tag`, without the profile's own tags. */
893
+ tags: string[];
894
+ /** `--trait`, as given (`key=value`); unset without any. */
895
+ traits?: string[];
896
+ };
820
897
  type RunRecord = {
821
898
  kind: 'reeval-run';
822
899
  schemaVersion: typeof SCHEMA_VERSION;
@@ -826,7 +903,13 @@ type RunRecord = {
826
903
  status: 'complete' | 'interrupted' | 'budget';
827
904
  startedAt: string;
828
905
  endedAt: string;
906
+ /** How long the run ran, counting each `--retry-errors` of it but not the time between them. */
829
907
  durationMs: number;
908
+ /**
909
+ * When each `reeval run --retry-errors` of the run started, oldest first; unset for a run never
910
+ * retried. A `--resume` of the run while it's `interrupted` finishes the last one.
911
+ */
912
+ retriedAt?: string[];
830
913
  author: string;
831
914
  git: GitInfo | null;
832
915
  runner: RunnerInfo;
@@ -861,8 +944,11 @@ type RunRecord = {
861
944
  maxVariants: number;
862
945
  };
863
946
  harness?: string[];
947
+ traits?: Record<string, string | string[]>;
864
948
  };
865
949
  } | null;
950
+ /** How `reeval run` was started; `--resume` plans with it again. Unset in older records. */
951
+ invocation?: RunInvocation;
866
952
  /**
867
953
  * What the run planned before it started, and each level whose sandbox ran as root or with
868
954
  * capabilities added back.
@@ -907,7 +993,10 @@ type RunRecord = {
907
993
  };
908
994
  summary: RunSummary;
909
995
  evals: EvalEntry[];
910
- /** Each pooled `evaluate.each` that ran; unset when none did. */
996
+ /**
997
+ * In records from before an eval with `cases` was one entry, each eval with `cases` that ran.
998
+ * Never written now; reading such a record folds it into its eval.
999
+ */
911
1000
  families?: FamilyRecord[];
912
1001
  /**
913
1002
  * These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
@@ -1020,7 +1109,11 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
1020
1109
  /** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
1021
1110
  canceled?: number;
1022
1111
  };
1023
- /** A family's pooled lift as the history listing keeps it: enough to compare its gate. */
1112
+ /** One case of an eval with `cases` in a `HistoryEntry`. */
1113
+ type HistoryCase = Pick<CaseRecord, 'key' | 'outcome' | 'verdict' | 'fingerprint' | 'hash'> & {
1114
+ cells: HistoryCell[];
1115
+ };
1116
+ /** A pooled lift as the history listing keeps it: enough to compare its gate. */
1024
1117
  type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
1025
1118
  lo: number;
1026
1119
  hi: number;
@@ -1065,12 +1158,14 @@ type HistoryEntry = {
1065
1158
  outcome: Outcome;
1066
1159
  passRate: number | null;
1067
1160
  cells: HistoryCell[];
1068
- }>;
1069
- /** The run's pooled families; unset when none ran. */
1070
- families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'> & {
1071
- /** The eval ids of its cases in this run. */
1072
- caseIds: string[];
1073
- lifts: FamilyLiftSummary[];
1161
+ /** An eval with `cases`: each case that ran, with its own cells; unset without cases. */
1162
+ cases?: HistoryCase[];
1163
+ /** With `cases`: how many the list has. */
1164
+ caseCount?: number;
1165
+ /** With `cases` and `compare`: each variant's lift paired by case. */
1166
+ lifts?: FamilyLiftSummary[];
1167
+ /** With `cases`: its cells' rates over the cases that a `gate: { passRate }` gates; unset when none. */
1168
+ rates?: Array<Pick<FamilyRate, 'cell' | 'rate'> & Required<Pick<FamilyRate, 'passRate' | 'outcome'>>>;
1074
1169
  }>;
1075
1170
  /** The run's errors outside any attempt; unset when there were none. */
1076
1171
  errors?: string[];
@@ -1519,6 +1614,8 @@ type JudgeInputs = {
1519
1614
  transcript?: boolean | 'tools' | 'full';
1520
1615
  /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
1521
1616
  checks?: boolean | string[];
1617
+ /** `c.observe()` values so far, by name; a list shows only those names. */
1618
+ observations?: boolean | string[];
1522
1619
  /**
1523
1620
  * Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
1524
1621
  * capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
@@ -1553,6 +1650,8 @@ type ResolvedJudgeStateOptions = {
1553
1650
  transcript: boolean | 'tools' | 'full';
1554
1651
  turns: 'last' | 'all';
1555
1652
  checks: boolean | string[];
1653
+ /** Set only when the call turned it on, so never `false` as written. */
1654
+ observations?: boolean | string[];
1556
1655
  };
1557
1656
  type Verdict = {
1558
1657
  verdict: 'pass' | 'fail';
@@ -1628,6 +1727,12 @@ type AttemptContext = {
1628
1727
  * made to it afterwards.
1629
1728
  */
1630
1729
  writeFile(path: string, contents: string | Uint8Array): Promise<void>;
1730
+ /**
1731
+ * Records a fact about the attempt under `name`, such as what the agent installed. It never
1732
+ * affects the outcome. Calling it again with a name replaces its value. The value is JSON, at
1733
+ * most 4 KB, and is masked like the rest of the record.
1734
+ */
1735
+ observe(name: string, value: JSONValue): void;
1631
1736
  /** Every turn so far, oldest first. */
1632
1737
  readonly turns: readonly Turn[];
1633
1738
  /** Aborted on timeout or Ctrl-C. */
@@ -1640,6 +1745,49 @@ type AttemptContext = {
1640
1745
  };
1641
1746
  };
1642
1747
  type EvalFn = (c: AttemptContext) => Promise<void>;
1748
+ /** The body of an eval with `cases`: the attempt and its case. */
1749
+ type CaseFn<T> = (c: AttemptContext, item: T) => Promise<void>;
1750
+ /**
1751
+ * One eval over several inputs: each case runs the body with every cell and run, and the cases
1752
+ * pool into one pass rate (and, with `compare`, one lift paired by case). Per eval only.
1753
+ */
1754
+ type CaseOptions<T> = {
1755
+ /** The inputs, each JSON data; the body gets one as its second argument. */
1756
+ cases: readonly T[];
1757
+ /**
1758
+ * Each case's key, which names it in the output and the record. Without it, a string or a
1759
+ * finite number is its own key, and an object uses its `id`, else its `name`.
1760
+ */
1761
+ caseKey?: (item: T) => string;
1762
+ /**
1763
+ * Which case and environment pairs run, decided at plan time. It sees each environment's
1764
+ * `traits`, and with `compare` the control, whose answer holds for every variant. Reading
1765
+ * `harness` decides each harness on its own. The pairs it leaves out are listed in `--dry-run`
1766
+ * and the record's `excluded`.
1767
+ */
1768
+ where?: Where<T>;
1769
+ };
1770
+ /** Filter facts about an environment, for `where`: never sent to the agent, never prompt data. */
1771
+ type Traits = Readonly<Record<string, string | readonly string[]>>;
1772
+ /** `where`'s predicate: true runs the case on that environment (and harness). */
1773
+ type Where<T> = (pair: {
1774
+ case: T;
1775
+ environment: {
1776
+ readonly name: string;
1777
+ /** The environment's traits; reading a key that no listed environment defines is refused. */
1778
+ readonly traits: Readonly<Record<string, string | readonly string[] | undefined>>;
1779
+ };
1780
+ /** The harness id; reading it decides each harness on its own. */
1781
+ readonly harness: string;
1782
+ }) => boolean;
1783
+ /** A case and environment pair `where` left out. */
1784
+ type ExcludedPair = {
1785
+ /** The case key. */
1786
+ case: string;
1787
+ environment: string;
1788
+ /** Set only when `where` read the harness. */
1789
+ harness?: string;
1790
+ };
1643
1791
  /** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
1644
1792
  type FakeHarnessDescriptor = {
1645
1793
  kind: 'fake';
@@ -1658,6 +1806,11 @@ type FakeHarnessDescriptor = {
1658
1806
  delayMs?: number;
1659
1807
  /** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
1660
1808
  ignoreAbort?: boolean;
1809
+ /**
1810
+ * Each turn's recorded agent time, exactly, whatever the wall clock says: a test can compare
1811
+ * agent time without scheduling jitter, which four runs a side can tell apart.
1812
+ */
1813
+ turnMs?: number;
1661
1814
  /** Throw before the first turn starts, to simulate an infra failure. */
1662
1815
  failWith?: string;
1663
1816
  /** The first this many session starts of an attempt never become ready, even when aborted (`true`: every one). */
@@ -1691,6 +1844,14 @@ type FakeHarnessDescriptor = {
1691
1844
  };
1692
1845
  /** Chance (0–1) that an attempt writes no files and says so, for flaky pass rates. */
1693
1846
  flakeRate?: number;
1847
+ /** Tokens the fake agent reports for every turn, as `turn.usage`; without it, null. */
1848
+ usage?: {
1849
+ input: number;
1850
+ output: number;
1851
+ cacheRead: number;
1852
+ cacheWrite: number;
1853
+ reasoning: number | null;
1854
+ };
1694
1855
  /**
1695
1856
  * Real sandbox actions, one list per `c.send` (the last list repeats), run in order before
1696
1857
  * `files`. `exec` runs as the agent: with the sandbox's own variables, none of the checks'.
@@ -2118,7 +2279,62 @@ type DockerSandboxDescriptor = {
2118
2279
  /** Processes and threads at once. Default 1024. */
2119
2280
  pids?: number;
2120
2281
  };
2121
- type SandboxDescriptor = FakeSandboxDescriptor | DockerSandboxDescriptor;
2282
+ /** A macOS VM on this Mac, cloned from `image` for each attempt by Tart, made by `tart()`. */
2283
+ type TartSandboxDescriptor = {
2284
+ kind: 'tart';
2285
+ /** Cell id; defaults to `tart:<image>`, the image's last path segment. */
2286
+ id?: string;
2287
+ /**
2288
+ * The VM each attempt clones: an OCI image such as
2289
+ * `ghcr.io/cirruslabs/macos-sequoia-base:latest`, or a local VM's name. Tart must have it
2290
+ * before the run: `reeval image build` (or `tart pull`) pulls it, and `reeval run` refuses one
2291
+ * `tart list` doesn't show.
2292
+ */
2293
+ image: string;
2294
+ /**
2295
+ * The most Tart VMs at once, across the run: 1 or 2, and 2 when unset, since macOS runs at most
2296
+ * two macOS VMs at a time.
2297
+ */
2298
+ maxConcurrent?: number;
2299
+ /** CPUs each clone gets; the image's when unset. */
2300
+ cpus?: number;
2301
+ /** Memory each clone gets: `'8g'`, `'4096m'`, or bytes; the image's when unset. */
2302
+ memory?: string | number;
2303
+ /**
2304
+ * Reach the VM over SSH when its Tart Guest Agent doesn't answer `tart exec`: as `user`
2305
+ * (`admin` when unset), with `key`, a private key the image trusts, absolute or relative to the
2306
+ * file it's written in.
2307
+ */
2308
+ ssh?: {
2309
+ user?: string;
2310
+ key: string;
2311
+ };
2312
+ };
2313
+ /** A boat.dev machine size: `small` is 2 vCPUs and 4 GB. */
2314
+ type BoatMachineType = 'small' | 'default' | 'large' | 'xlarge';
2315
+ /** The host variable holding a Boat API key, made by `secret.fromEnv`. */
2316
+ type BoatApiKey = {
2317
+ kind: 'secret';
2318
+ from: string;
2319
+ };
2320
+ /** A boat.dev VM per attempt, made by `boat()` from `reeval/sandbox/boat`. */
2321
+ type BoatSandboxDescriptor = {
2322
+ kind: 'boat';
2323
+ /** Cell id; defaults to `boat`, or `boat:<type>` with a `type`. */
2324
+ id?: string;
2325
+ /** The host variable with the Boat API key, read on the host only. */
2326
+ apiKey: BoatApiKey;
2327
+ /**
2328
+ * The start of every sandbox name reeval gives Boat, so a run's cleanup finds only its own:
2329
+ * lowercase letters, digits and dashes.
2330
+ */
2331
+ prefix: string;
2332
+ /** The machine size. Default `small`. */
2333
+ type?: BoatMachineType;
2334
+ /** The most Boat sandboxes at once, across the run. */
2335
+ maxConcurrent?: number;
2336
+ };
2337
+ type SandboxDescriptor = FakeSandboxDescriptor | DockerSandboxDescriptor | TartSandboxDescriptor | BoatSandboxDescriptor;
2122
2338
  /**
2123
2339
  * A host environment variable, redacted everywhere reeval writes and recorded by name only. Made by
2124
2340
  * the `secret` helper of `defineEnvironment` and `defineConfig` callbacks.
@@ -2329,6 +2545,12 @@ type EnvironmentSpec = {
2329
2545
  * parent's plugin of the same name.
2330
2546
  */
2331
2547
  plugins?: PluginSource[];
2548
+ /**
2549
+ * Filter facts for `where`, such as `{ stack: 'next' }`: a key to a string or a list of
2550
+ * strings. A child's key replaces its parent's value whole; keys it doesn't set are inherited.
2551
+ * Recorded on the cell, and never written to the sandbox, the prompt or the judge.
2552
+ */
2553
+ traits?: Traits;
2332
2554
  };
2333
2555
  /**
2334
2556
  * A part of a plugin, as Claude Code's layout bundles it: `skills/<name>/SKILL.md`,
@@ -2439,6 +2661,8 @@ type Environment = {
2439
2661
  readonly name: string;
2440
2662
  /** Its `extend` chain, root first. */
2441
2663
  readonly layers: readonly EnvironmentLayer[];
2664
+ /** Its layers' `traits`, merged by key with the child's winning; `{}` when none. */
2665
+ readonly traits: Traits;
2442
2666
  /**
2443
2667
  * A child environment layered on this one; wherever it sets a key this one has, it wins.
2444
2668
  * `sandbox` is replaced entirely when the child sets one; when the child omits it, the
@@ -2540,26 +2764,39 @@ type HarnessEdit = {
2540
2764
  type HarnessData = HarnessDescriptor | HarnessDescriptor[] | HarnessEdit;
2541
2765
  /** `HarnessData`, or a function that edits the inherited list (TypeScript only). */
2542
2766
  type HarnessOption = HarnessData | ((inherited: HarnessDescriptor[]) => HarnessDescriptor | HarnessDescriptor[]);
2543
- type CompareGate = 'variants' | 'all' | 'none';
2544
2767
  /**
2545
- * The baseline and each variant run as their own environments, with the same prompt and checks.
2768
+ * An edit of the inherited environment list: `add` appends environments, `only` keeps the
2769
+ * inherited ones with these names, in their order.
2770
+ */
2771
+ type EnvironmentEdit<E = Environment> = {
2772
+ add: E | E[];
2773
+ } | {
2774
+ only: string | string[];
2775
+ };
2776
+ /** One environment, a list (each is its own cell), or an edit of the inherited list, as data. */
2777
+ type EnvironmentsData<E = EnvironmentData> = E | E[] | EnvironmentEdit<E>;
2778
+ /** `EnvironmentsData`, or a function that edits the inherited list (TypeScript only). */
2779
+ type EnvironmentsOption = EnvironmentsData<Environment> | ((inherited: Environment[]) => Environment | Environment[]);
2780
+ type CompareGate = 'variants' | 'all' | 'observe';
2781
+ /**
2782
+ * The control and each variant run as their own environments, with the same prompt and checks.
2546
2783
  * `compare` does not layer them. Share setup first with `environment.extend` / `extendEnvironment`.
2547
2784
  * `false` is not a value here: a profile turns compare off with `limit.compare`.
2548
2785
  */
2549
2786
  type CompareOptions<E = Environment> = {
2550
2787
  /**
2551
- * The control. Not layered under the variants. Without it, the control is the environment the
2552
- * eval would otherwise run in (the cascaded `environment`), else an empty one named `baseline`.
2788
+ * The environment each variant is measured against. Not layered under the variants, and it
2789
+ * replaces any `environments` the eval inherits.
2553
2790
  */
2554
- baseline?: E;
2791
+ control: E;
2555
2792
  /** Other environments, each planned and run as itself. */
2556
2793
  variants: E[];
2557
- /** Which cells decide pass/fail: `variants` (default), `all`, or `none` (report only). */
2794
+ /** Which cells decide pass/fail: `variants` (default), `all`, or `observe` (none of them). */
2558
2795
  gate?: CompareGate;
2559
2796
  /** Lift in points from -1 to 1; a variant cell below it fails. */
2560
2797
  minLift?: number;
2561
2798
  /** Harnesses (ids or descriptors) to compare on; the others run the control only. */
2562
- harness?: Array<string | HarnessDescriptor>;
2799
+ harnesses?: Array<string | HarnessDescriptor>;
2563
2800
  };
2564
2801
  /**
2565
2802
  * A question a language model answers after the attempts: its text, or the text and its options.
@@ -2612,6 +2849,8 @@ type QuestionInputs = {
2612
2849
  transcript?: boolean | 'tools' | 'full';
2613
2850
  /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
2614
2851
  checks?: boolean | string[];
2852
+ /** `c.observe()` values, by name; a list shows only those names. */
2853
+ observations?: boolean | string[];
2615
2854
  /** Each attempt's own captures, by name; `true` sends all of them. */
2616
2855
  artifacts?: true | string[];
2617
2856
  };
@@ -2624,6 +2863,8 @@ type ResolvedQuestionStateOptions = {
2624
2863
  transcript: boolean | 'tools' | 'full';
2625
2864
  turns: 'last' | 'all';
2626
2865
  checks: boolean | string[];
2866
+ /** Set only when the question turned it on, so never `false` as written. */
2867
+ observations?: boolean | string[];
2627
2868
  };
2628
2869
  /** Defaults for every question below it. What a question is shown is its own, never a default. */
2629
2870
  type QuestionOptions = {
@@ -2706,18 +2947,22 @@ type AttemptAnswer = {
2706
2947
  input?: FileRef;
2707
2948
  };
2708
2949
  /** An option a profile's `limit` can narrow, and `pin` can protect. */
2709
- type PinnableKey = 'harness' | 'runs' | 'compare';
2950
+ type PinnableKey = 'harnesses' | 'runs' | 'compare';
2710
2951
  /**
2711
2952
  * How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
2712
2953
  * passes (pass^k); `{ passRate }` passes when its pass@1 (passes over passes and fails) is at
2713
- * least that share.
2954
+ * least that share; `'observe'` gates nothing, so the eval's attempts run and are recorded, and
2955
+ * only run and infra errors fail it.
2714
2956
  */
2715
- type GatePolicy = 'all' | {
2957
+ type GatePolicy = 'all' | 'observe' | {
2716
2958
  passRate: number;
2717
2959
  };
2718
- /** A cell's gate: its policy, or `'none'` when its failures don't fail the eval (a control's). */
2719
- type CellGate = GatePolicy | 'none';
2720
- /** A compare object: `baseline` and `variants`. Same as `CompareOptions`. */
2960
+ /**
2961
+ * A cell's gate: its policy, or `'none'` when its failures don't fail the eval (a control's, or
2962
+ * any cell under `gate: 'observe'`).
2963
+ */
2964
+ type CellGate = Exclude<GatePolicy, 'observe'> | 'none';
2965
+ /** A compare object: `control` and `variants`. Same as `CompareOptions`. */
2721
2966
  type CompareSpec<E = Environment> = CompareOptions<E>;
2722
2967
  type OddDigit = '1' | '3' | '5' | '7' | '9';
2723
2968
  /**
@@ -2732,12 +2977,12 @@ type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${s
2732
2977
  type Runs<N extends number = number> = number | {
2733
2978
  bestOf: OddAtLeast3<N>;
2734
2979
  };
2735
- /** Options other than `environment` and `compare`, which are exclusive of each other. */
2980
+ /** Options other than `environments` and `compare`, which are exclusive of each other. */
2736
2981
  type EvalOptionFields<N extends number = number> = {
2737
2982
  /** Which agents run the eval; each harness is its own cell. */
2738
- harness?: HarnessOption;
2983
+ harnesses?: HarnessOption;
2739
2984
  /**
2740
- * The default machine when the resolved environment omits `sandbox`. Cascades like `harness`.
2985
+ * The default machine when the resolved environment omits `sandbox`. Cascades like `harnesses`.
2741
2986
  * Not an environment: it does not invent skills, files or a name.
2742
2987
  */
2743
2988
  sandbox?: SandboxDescriptor;
@@ -2745,7 +2990,7 @@ type EvalOptionFields<N extends number = number> = {
2745
2990
  * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
2746
2991
  * a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
2747
2992
  * Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
2748
- * a verdict, and anything else runs once, a pooled `evaluate.each` case included.
2993
+ * a verdict, and anything else runs once, a case of an eval with `cases` included.
2749
2994
  */
2750
2995
  runs?: Runs<N>;
2751
2996
  /** How each cell's attempts decide its outcome; default `'all'`. */
@@ -2794,51 +3039,51 @@ type Revision = number | string;
2794
3039
  * An eval's own source as hashes: its body and case data as declared, and its options and its
2795
3040
  * enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
2796
3041
  */
2797
- type EvalSource = DeclaredSource & {
3042
+ type EvalSource = {
3043
+ /** sha256 of the body function's source text. */
3044
+ body: string;
3045
+ /**
3046
+ * sha256 of the case's data as canonical JSON, with secrets in its strings masked as the record
3047
+ * masks them; set on each case of an eval with `cases`.
3048
+ */
3049
+ case?: string;
2798
3050
  /** sha256 of the canonical options. */
2799
3051
  options: string;
2800
3052
  };
2801
3053
  /**
2802
3054
  * `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
2803
- * A compare with a `baseline` names every arm, so the cascaded `environment` is not applied, and
2804
- * the same object can't set both. Without a `baseline`, the cascaded `environment` (this object's
2805
- * own, or an inherited one) is the control.
3055
+ * A compare names every arm, so an inherited `environments` is not applied, and the same object
3056
+ * can't set both.
2806
3057
  */
2807
- type EnvironmentOrCompare<E> = {
2808
- /** What this group or eval runs in, and the control of a compare without a `baseline`. */
2809
- environment?: E;
3058
+ type EnvironmentOrCompare<E, Envs> = {
3059
+ /**
3060
+ * What this group or eval runs in: one environment, or a list, each its own cell beside
3061
+ * each harness. `{ add }` and `{ only }` edit the inherited list.
3062
+ */
3063
+ environments?: Envs;
2810
3064
  compare?: never;
2811
3065
  } | {
2812
- /** Forbidden beside `compare.baseline`, which names the control. */
2813
- environment?: never;
2814
- /** Environments to run side by side; `baseline` is the control. */
2815
- compare: CompareSpec<E> & {
2816
- baseline: E;
2817
- };
2818
- } | {
2819
- /** The control: the compare has no `baseline`. */
2820
- environment?: E;
2821
- /** Environments to run side by side against `environment`, the control. */
2822
- compare: CompareSpec<E> & {
2823
- baseline?: undefined;
2824
- };
3066
+ /** Forbidden beside `compare`, whose `control` names what the eval runs in. */
3067
+ environments?: never;
3068
+ /** Environments to run side by side; `control` is what each variant is measured against. */
3069
+ compare: CompareSpec<E>;
2825
3070
  };
2826
3071
  /** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
2827
- type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
3072
+ type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment, EnvironmentsOption>;
2828
3073
  type GroupOptions<N extends number = number> = EvalOptions<N>;
2829
3074
  /**
2830
3075
  * `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
2831
3076
  * by name or a fake judge, and environments without `extend`. The same `environment` / compare
2832
3077
  * object split as `EvalOptions`.
2833
3078
  */
2834
- type EvalOptionsData = Omit<EvalOptionFields, 'harness' | 'judge'> & {
3079
+ type EvalOptionsData = Omit<EvalOptionFields, 'harnesses' | 'judge'> & {
2835
3080
  /** Which agents run the eval; each harness is its own cell. */
2836
- harness?: HarnessData;
3081
+ harnesses?: HarnessData;
2837
3082
  /** Defaults for `c.judge()`, merged field by field down the cascade. */
2838
3083
  judge?: ConfigJudgeOptions;
2839
- } & EnvironmentOrCompare<EnvironmentData>;
3084
+ } & EnvironmentOrCompare<EnvironmentData, EnvironmentsData>;
2840
3085
  /** An `Environment` as data, as JSON carries it: without `extend` / `extendEnvironment`. */
2841
- type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend'>;
3086
+ type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend' | 'traits'>;
2842
3087
  /** The config file is plain data sent to workers, so its judge model can't be an object. */
2843
3088
  type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
2844
3089
  /** `'provider/model'`, or a fake judge. */
@@ -2860,11 +3105,11 @@ type Budget = {
2860
3105
  /** What evals get when they don't say otherwise: under groups and evals. */
2861
3106
  type ProfileDefaults = {
2862
3107
  /** The agents every eval runs on, unless a group or eval replaces them. */
2863
- harness?: HarnessDescriptor | HarnessDescriptor[];
3108
+ harnesses?: HarnessDescriptor | HarnessDescriptor[];
2864
3109
  /** The default machine when the resolved environment omits `sandbox`. */
2865
3110
  sandbox?: SandboxDescriptor;
2866
- /** What every eval runs in when it names no environment. */
2867
- environment?: Environment;
3111
+ /** What every eval runs in when it names no environment: one, or a list (a cell each). */
3112
+ environments?: Environment | Environment[];
2868
3113
  /** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
2869
3114
  runs?: Runs;
2870
3115
  /** How each cell's attempts decide its outcome. */
@@ -2893,7 +3138,13 @@ type ProfileLimit = {
2893
3138
  maxVariants: number;
2894
3139
  };
2895
3140
  /** Harness ids to allow; evals left with none are skipped. */
2896
- harness?: string[];
3141
+ harnesses?: string[];
3142
+ /**
3143
+ * Cells to run, by their environment's traits (with `compare`, the control's): each key's trait
3144
+ * must equal the value, or one of a list's, or for a list trait contain it. Evals left with no
3145
+ * cell are skipped.
3146
+ */
3147
+ traits?: Record<string, string | string[]>;
2897
3148
  };
2898
3149
  /** How a run is sized: chosen with `--profile`, `REEVAL_PROFILE` or `defaultProfile`. */
2899
3150
  type Profile = {
@@ -2985,6 +3236,14 @@ type BlindSide = BlindTurn & {
2985
3236
  command: string;
2986
3237
  notRun: true;
2987
3238
  }>;
3239
+ /** Its `c.observe()` values; a name the question lists that it never observed is `missing`. */
3240
+ observations?: Array<{
3241
+ name: string;
3242
+ value: JSONValue;
3243
+ } | {
3244
+ name: string;
3245
+ missing: true;
3246
+ }>;
2988
3247
  /** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
2989
3248
  artifacts?: Array<{
2990
3249
  label: string;
@@ -3006,21 +3265,24 @@ type AnalysisConfig = {
3006
3265
  instructions?: string;
3007
3266
  /** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
3008
3267
  maxInputTokens?: number;
3009
- /** The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap. */
3268
+ /** The most tokens the model may answer with, reasoning included (default 32000). reeval has no prices, so tokens are the cap. */
3010
3269
  maxOutputTokens?: number;
3011
3270
  };
3012
3271
  type ReevalConfig = {
3013
3272
  /** Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`. */
3014
3273
  include?: string[];
3015
3274
  /** The agents every eval runs on, unless a group or eval replaces them; default Cursor. */
3016
- harness?: HarnessDescriptor | HarnessDescriptor[];
3275
+ harnesses?: HarnessDescriptor | HarnessDescriptor[];
3017
3276
  /**
3018
3277
  * The default machine when the resolved environment omits `sandbox`. Not an environment:
3019
3278
  * it does not invent skills, files or a name. Implicit `docker()` when nobody sets one.
3020
3279
  */
3021
3280
  sandbox?: SandboxDescriptor;
3022
- /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
3023
- environment?: Environment;
3281
+ /**
3282
+ * What every eval runs in when it names no environment and has no compare: one, or a list (a
3283
+ * cell each).
3284
+ */
3285
+ environments?: Environment | Environment[];
3024
3286
  /**
3025
3287
  * Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
3026
3288
  * variants runs 4 a side and anything else once.
@@ -3158,9 +3420,8 @@ type PhaseTimings = {
3158
3420
  type CellRole = 'control' | 'variant';
3159
3421
  /**
3160
3422
  * A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
3161
- * environment's sandbox id. With `compare`, the control is `baseline` (or the cascaded
3162
- * environment, or an empty `baseline`) and each variant is that environment on its own,
3163
- * including its sandbox.
3423
+ * environment's sandbox id. With `compare`, the control is `compare.control` and each variant is
3424
+ * that environment on its own, including its sandbox.
3164
3425
  */
3165
3426
  type Cell = {
3166
3427
  /** The harness id, the environment name, and the sandbox id, joined with `/`. */
@@ -3180,9 +3441,16 @@ type Cell = {
3180
3441
  * eight-character hash.
3181
3442
  */
3182
3443
  sandbox: string;
3444
+ /**
3445
+ * `variant` on a compare's variant cell, else `control`: the compare's control, or any cell of
3446
+ * an eval without `compare`, each environment of an `environments` grid included. A variant's
3447
+ * lift is against the `control` cell on its harness and sandbox.
3448
+ */
3183
3449
  role: CellRole;
3184
3450
  /** The variant's name, for variant cells. */
3185
3451
  variant?: string;
3452
+ /** The environment's merged traits; unset when it has none. Not part of the key. */
3453
+ traits?: Traits;
3186
3454
  };
3187
3455
  /** A plugin as an attempt placed it. */
3188
3456
  type PluginRecord = {
@@ -3358,6 +3626,13 @@ type AttemptImage = {
3358
3626
  digest: string | null;
3359
3627
  };
3360
3628
  };
3629
+ /** The image a `tart()` attempt's VM was cloned from. */
3630
+ type VmImage = {
3631
+ /** The image as `tart({ image })` names it. */
3632
+ ref: string;
3633
+ /** The OCI manifest digest (`sha256:…`) Tart had for it; null for a local VM, which has none. */
3634
+ digest: string | null;
3635
+ };
3361
3636
  /** A harness session start that timed out and was started again. */
3362
3637
  type StartRetry = {
3363
3638
  /** `<harness kind>: session did not become ready within <startTimeout>`. */
@@ -3415,6 +3690,14 @@ type AssertionRecord = {
3415
3690
  /** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
3416
3691
  durationMs?: number;
3417
3692
  };
3693
+ /** A `c.observe(name, value)` call: the last value given for the name. */
3694
+ type Observation = {
3695
+ name: string;
3696
+ /** Masked like the rest of the record. */
3697
+ value: JSONValue;
3698
+ /** How many turns the attempt had sent when the value was given: 0 before the first. */
3699
+ turn: number;
3700
+ };
3418
3701
  /**
3419
3702
  * A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
3420
3703
  * folder), or why it wasn't.
@@ -3480,10 +3763,11 @@ type CaptureStep = {
3480
3763
  };
3481
3764
  /** Which environment ran as the control, so the studio can show it. */
3482
3765
  type EvalCompareMeta = {
3766
+ /** The control's environment name. */
3483
3767
  baseline: string;
3484
3768
  /**
3485
- * Set when `compare.baseline` was omitted: the control is the eval's cascaded `environment`, or
3486
- * an empty environment named `baseline` when there was none.
3769
+ * Not written, since a compare names its control. A record that has it ran the eval's cascaded
3770
+ * environment (`'environment'`) or an empty one named `baseline` (`'empty'`) as the control.
3487
3771
  */
3488
3772
  from?: 'environment' | 'empty';
3489
3773
  };
@@ -3505,7 +3789,7 @@ type SandboxWarning = {
3505
3789
  file: string | null;
3506
3790
  /** The capabilities added back, for `capabilities`. */
3507
3791
  capabilities?: string[];
3508
- /** The line reeval prints, after `WARNING: ` (root) or `warning: ` (capabilities). */
3792
+ /** The line reeval prints after `⚠ Warning:`, in red for root and orange for capabilities. */
3509
3793
  message: string;
3510
3794
  };
3511
3795
  /** Where a cell's harness resolved in the `gateway` list, and the entries before it that couldn't serve it. */
@@ -3516,5 +3800,5 @@ type ResolvedGateway = {
3516
3800
  why: string;
3517
3801
  }>;
3518
3802
  };
3519
- export { FakeAnswer as $, formatSeconds as $n, TranscriptEvent as $t, CompareCut as A, EFFORT_MIN_RUNS as An, PairFactor as At, EgressPolicy as B, cellEffort as Bn, QuestionInputs as Bt, CaptureWindow as C, Lift as Cn, JudgeOptions as Ct, ClaudeCodeHarnessDescriptor as D, CellEffort as Dn, OpenCodeHarnessDescriptor as Dt, ClaudeCodeEffort as E, UpstreamCount as En, McpServer as Et, Diff as F, Spread as Fn, PluginSource as Ft, EnvironmentInput as G, effortLabel as Gn, SandboxWarning as Gt, EnvVars as H, contrastEffort as Hn, ReevalConfig as Ht, DirectGatewayDescriptor as I, TOKENS_UNREPORTED as In, Profile as It, EvalFn as J, effortSummary as Jn, ServeHandle as Jt, EnvironmentSpec as K, effortOf as Kn, Secret as Kt, DirectoryOptions as L, VERDICT_TEXT as Ln, ProfileDefaults as Lt, ConfigHelpers as M, EffortMetric as Mn, PiHarnessDescriptor as Mt, ConfigJudgeOptions as N, EffortSample as Nn, PinnableKey as Nt, CodexHarnessDescriptor as O, DIRECTIONAL_MIN as On, OpenRouterGatewayDescriptor as Ot, CursorHarnessDescriptor as P, EffortVerdict as Pn, PluginPart as Pt, FakeAnalysisModel as Q, formatRatio as Qn, TokenUsage as Qt, DockerSandboxDescriptor as R, bothUnderATenth as Rn, ProfileLimit as Rt, CaptureWait as S, JudgmentRecord as Sn, JudgeModel as St, CellRole as T, StackSummary as Tn, JudgeSubject as Tt, Environment as U, directional as Un, ResolvedGateway as Ut, EgressRecord as V, cellSamples as Vn, QuestionOptions as Vt, EnvironmentHelpers as W, effortChangeLabel as Wn, SandboxDescriptor as Wt, ExecResult as X, filesChanged as Xn, SkillSource as Xt, EvalOptions as Y, effortVerdict as Yn, SetupStep as Yt, ExecRetry as Z, formatCount as Zn, StatusRange as Zt, CapturePage as _, FamilyRate as _n, JSONValue as _t, AttemptContext as a, VertexGatewayDescriptor as an, FxHarnessDescriptor as at, CaptureUseAction as b, HistoryCell as bn, JudgeFailure as bt, BlindPair as c, Wait as cn, GooseHarnessDescriptor as ct, CaptureCall as d, AttemptRecord as dn, HarnessData as dt, Turn as en, tokenValues as er, FakeHarnessDescriptor as et, CaptureConsoleMessage as f, CellRecord as fn, HarnessDescriptor as ft, CaptureOptions as g, FamilyLiftSummary as gn, HostFile as gt, CaptureKey as h, FamilyLift as hn, HistoryConfig as ht, AttemptAnswer as i, Verdict as in, FakeVote as it, CompareOptions as j, EffortContrast as jn, PairVote as jt, CompareAnswer as k, EFFORT_ALPHA as kn, Outcome as kt, Budget as l, WebEventType as ln, GrokBuildHarnessDescriptor as lt, CaptureInstruction as m, EvalEntry as mn, HarnessOption as mt, AnalysisConfig as n, TurnStatus as nn, DeclaredNode as nr, FakeSandboxDescriptor as nt, BackgroundHandle as o, VertexPublisher as on, GatewayDescriptor as ot, CaptureFailedRequest as p, CheckTally as pn, HarnessEdit as pt, ErrorKind as q, effortSample as qn, SecretTemplate as qt, AntigravityHarnessDescriptor as r, VercelGatewayDescriptor as rn, FakeScriptStep as rt, BlindAttempt as s, Viewport as sn, GatewayName as st, AgentHarnessDescriptor as t, TurnData as tn, DeclarationTree as tr, FakeJudgeModel as tt, Capture as u, ArtifactRef as un, GroupOptions as ut, CapturePageFile as v, FamilyRecord as vn, JudgeArtifactRef as vt, Cell as w, RunRecord as wn, JudgeState as wt, CaptureUseScript as x, HistoryEntry as xn, JudgeInputs as xt, CaptureUse as y, FileRef as yn, JudgeCallOptions as yt, Duration as z, cellContrast as zn, Question as zt };
3520
- //# sourceMappingURL=types-BHR1lGPh.d.ts.map
3803
+ export { EvalFn as $, directional as $n, Secret as $t, CellRole as A, HistoryEntry as An, JudgeState as At, Diff as B, EFFORT_MIN_RUNS as Bn, PinnableKey as Bt, CaptureUseAction as C, FamilyLift as Cn, JSONValue as Ct, CaseFn as D, FamilyVerdict as Dn, JudgeInputs as Dt, CaptureWindow as E, FamilyRecord as En, JudgeFailure as Et, CompareCut as F, StackSummary as Fn, OpenRouterGatewayDescriptor as Ft, EgressPolicy as G, Spread as Gn, ProfileLimit as Gt, DirectoryOptions as H, EffortMetric as Hn, PluginSource as Ht, CompareOptions as I, UpstreamCount as In, Outcome as It, Environment as J, bothUnderATenth as Jn, QuestionOptions as Jt, EgressRecord as K, TOKENS_UNREPORTED as Kn, Question as Kt, ConfigHelpers as L, CellEffort as Ln, PairFactor as Lt, ClaudeCodeHarnessDescriptor as M, Lift as Mn, McpServer as Mt, CodexHarnessDescriptor as N, RunInvocation as Nn, Observation as Nt, CaseOptions as O, FileRef as On, JudgeModel as Ot, CompareAnswer as P, RunRecord as Pn, OpenCodeHarnessDescriptor as Pt, ErrorKind as Q, contrastEffort as Qn, SandboxWarning as Qt, ConfigJudgeOptions as R, DIRECTIONAL_MIN as Rn, PairVote as Rt, CaptureUse as S, EvalEntry as Sn, HostFile as St, CaptureWait as T, FamilyRate as Tn, JudgeCallOptions as Tt, DockerSandboxDescriptor as U, EffortSample as Un, Profile as Ut, DirectGatewayDescriptor as V, EffortContrast as Vn, PluginPart as Vt, Duration as W, EffortVerdict as Wn, ProfileDefaults as Wt, EnvironmentInput as X, cellEffort as Xn, ResolvedGateway as Xt, EnvironmentHelpers as Y, cellContrast as Yn, ReevalConfig as Yt, EnvironmentSpec as Z, cellSamples as Zn, SandboxDescriptor as Zt, CaptureInstruction as _, WebEventType as _n, HarnessData as _t, AttemptContext as a, TartSandboxDescriptor as an, effortVerdict as ar, FakeAnswer as at, CapturePage as b, CellRecord as bn, HarnessOption as bt, BlindPair as c, Turn as cn, formatRatio as cr, FakeSandboxDescriptor as ct, BoatSandboxDescriptor as d, VercelGatewayDescriptor as dn, FxHarnessDescriptor as dt, SecretTemplate as en, effortChangeLabel as er, EvalOptions as et, Budget as f, Verdict as fn, GatewayDescriptor as ft, CaptureFailedRequest as g, Wait as gn, GroupOptions as gt, CaptureConsoleMessage as h, Viewport as hn, GrokBuildHarnessDescriptor as ht, AttemptAnswer as i, StatusRange as in, effortSummary as ir, FakeAnalysisModel as it, ClaudeCodeEffort as j, JudgmentRecord as jn, JudgeSubject as jt, Cell as k, HistoryCell as kn, JudgeOptions as kt, BoatApiKey as l, TurnData as ln, formatSeconds as lr, FakeScriptStep as lt, CaptureCall as m, VertexPublisher as mn, GooseHarnessDescriptor as mt, AnalysisConfig as n, SetupStep as nn, effortOf as nr, ExecResult as nt, BackgroundHandle as o, TokenUsage as on, filesChanged as or, FakeHarnessDescriptor as ot, Capture as p, VertexGatewayDescriptor as pn, GatewayName as pt, EnvVars as q, VERDICT_TEXT as qn, QuestionInputs as qt, AntigravityHarnessDescriptor as r, SkillSource as rn, effortSample as rr, ExecRetry as rt, BlindAttempt as s, TranscriptEvent as sn, formatCount as sr, FakeJudgeModel as st, AgentHarnessDescriptor as t, ServeHandle as tn, effortLabel as tr, EvalOptionsData as tt, BoatMachineType as u, TurnStatus as un, tokenValues as ur, FakeVote as ut, CaptureKey as v, ArtifactRef as vn, HarnessDescriptor as vt, CaptureUseScript as w, FamilyLiftSummary as wn, JudgeArtifactRef as wt, CapturePageFile as x, CheckTally as xn, HistoryConfig as xt, CaptureOptions as y, AttemptRecord as yn, HarnessEdit as yt, CursorHarnessDescriptor as z, EFFORT_ALPHA as zn, PiHarnessDescriptor as zt };
3804
+ //# sourceMappingURL=types-DBwFsGem.d.ts.map