reeval 0.17.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
  4. package/dist/adapter-C8xPuXYq.js.map +1 -0
  5. package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
  6. package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
  7. package/dist/agents-C6zhkxp3.js +1151 -0
  8. package/dist/agents-C6zhkxp3.js.map +1 -0
  9. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  10. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  11. package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
  12. package/dist/app-files-DT5mJyV4.js.map +1 -0
  13. package/dist/assemble-UPCRrtOz.js +1 -0
  14. package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
  15. package/dist/assemble-t6h018An.js.map +1 -0
  16. package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
  17. package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
  18. package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
  19. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
  20. package/dist/boat-BjLqFJIZ.js +108 -0
  21. package/dist/boat-BjLqFJIZ.js.map +1 -0
  22. package/dist/boat-CJCtzfB1.js +322 -0
  23. package/dist/boat-CJCtzfB1.js.map +1 -0
  24. package/dist/bridge/index.mjs +300 -78
  25. package/dist/capture-DoxFUN3J.js +1 -0
  26. package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
  27. package/dist/capture-iBzFiC0j.js.map +1 -0
  28. package/dist/cases-Dy0mifal.js +1 -0
  29. package/dist/cases-z9E6DCd5.js +639 -0
  30. package/dist/cases-z9E6DCd5.js.map +1 -0
  31. package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
  32. package/dist/cli-20Fpvs5H.js.map +1 -0
  33. package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
  34. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
  37. package/dist/client-KM4C5r5_.js.map +1 -0
  38. package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
  39. package/dist/command-B5OyHpOu.js.map +1 -0
  40. package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
  41. package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
  42. package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
  43. package/dist/command-CRh-cb6Z.js.map +1 -0
  44. package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
  45. package/dist/command-aUJvENwy.js.map +1 -0
  46. package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
  47. package/dist/commands-CWy-WndG.js.map +1 -0
  48. package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
  49. package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
  50. package/dist/config-B9mB1kF5.js.map +1 -0
  51. package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
  52. package/dist/context-Bkv3ndjq.js.map +1 -0
  53. package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
  54. package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
  55. package/dist/cost-Cr26Alhq.js +1 -0
  56. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
  57. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
  58. package/dist/db-P2fXw4nx.js +2 -0
  59. package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
  60. package/dist/db-zgC1tvKB.js.map +1 -0
  61. package/dist/debug-CF4p4Lp_.js +1 -0
  62. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  63. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  64. package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
  65. package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
  66. package/dist/default-effort-DGhQmu95.js.map +1 -0
  67. package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
  68. package/dist/dist-CAFD1yfv.js.map +1 -0
  69. package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
  70. package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
  71. package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
  72. package/dist/docker-DAMnm3L9.js.map +1 -0
  73. package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
  74. package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
  75. package/dist/doctor-B2PohneB.js.map +1 -0
  76. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  77. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  78. package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
  79. package/dist/environment-DB8o3hGi.js.map +1 -0
  80. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  81. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  82. package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
  83. package/dist/eval-source-D3sbAzt-.js.map +1 -0
  84. package/dist/evaluate-COdzf7Hl.d.ts +92 -0
  85. package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
  86. package/dist/evaluate-CwKeN_fo.js.map +1 -0
  87. package/dist/files-B4KQUiM7.js +1 -0
  88. package/dist/files-BEwfjD8J.js +64 -0
  89. package/dist/files-BEwfjD8J.js.map +1 -0
  90. package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
  91. package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
  92. package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
  93. package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
  94. package/dist/files-C1Rk9FYA.js +1 -0
  95. package/dist/files-C3mCC8Mn.js +1 -0
  96. package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
  97. package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
  98. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  99. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  100. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  101. package/dist/files-DlQshsks.js.map +1 -0
  102. package/dist/files-DnlA1A7N.js +1 -0
  103. package/dist/gate-CgYdaprG.js +49 -0
  104. package/dist/gate-CgYdaprG.js.map +1 -0
  105. package/dist/gateway.d.ts +1 -1
  106. package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
  107. package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
  108. package/dist/harness-C2berMiM.js +105 -0
  109. package/dist/harness-C2berMiM.js.map +1 -0
  110. package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
  111. package/dist/harness-DXv2gnaT.js.map +1 -0
  112. package/dist/harness.d.ts +1 -1
  113. package/dist/harness.js +1 -1
  114. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  115. package/dist/ids-C_advdJA.js.map +1 -0
  116. package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
  117. package/dist/images-BNgcR77K.js.map +1 -0
  118. package/dist/index.d.ts +23 -3
  119. package/dist/index.js +5 -5
  120. package/dist/init-4RPeD-K-.js +1 -0
  121. package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
  122. package/dist/init-DbuZVRrW.js.map +1 -0
  123. package/dist/ink-list-BpfidQ1h.js +274 -0
  124. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  125. package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
  126. package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
  127. package/dist/judge-COMXT1Hx.js +230 -0
  128. package/dist/judge-COMXT1Hx.js.map +1 -0
  129. package/dist/judge.d.ts +1 -1
  130. package/dist/judge.js +1 -1
  131. package/dist/judgments-CzgpsTqv.js +246 -0
  132. package/dist/judgments-CzgpsTqv.js.map +1 -0
  133. package/dist/kinds-C1oY3qyC.js +697 -0
  134. package/dist/kinds-C1oY3qyC.js.map +1 -0
  135. package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
  136. package/dist/layout-Cm85tQYk.js.map +1 -0
  137. package/dist/list-DuThsE28.js +1348 -0
  138. package/dist/list-DuThsE28.js.map +1 -0
  139. package/dist/live-CJCK11nh.js +1 -0
  140. package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
  141. package/dist/live-CgYbO4Te.js.map +1 -0
  142. package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
  143. package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
  144. package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
  145. package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
  146. package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
  147. package/dist/model-Cv4GuTsZ.js +1 -0
  148. package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
  149. package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
  150. package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
  151. package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
  152. package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
  153. package/dist/model-V3mMXp6x.js.map +1 -0
  154. package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
  155. package/dist/node-D7pAU--0.js.map +1 -0
  156. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
  157. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  158. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
  159. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  160. package/dist/options-CHubslQf.js +1 -0
  161. package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
  162. package/dist/pairs-BGb134Vu.js.map +1 -0
  163. package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
  164. package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
  165. package/dist/plain-BYUf_fUV.js +267 -0
  166. package/dist/plain-BYUf_fUV.js.map +1 -0
  167. package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
  168. package/dist/plan-Cgq9NCEX.js.map +1 -0
  169. package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
  170. package/dist/plugins-BkmPxDiY.js.map +1 -0
  171. package/dist/preflight-4gWSy0Pe.js +1 -0
  172. package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
  173. package/dist/preflight-BElE_q0u.js.map +1 -0
  174. package/dist/preflight-CtlloeOQ.js +68 -0
  175. package/dist/preflight-CtlloeOQ.js.map +1 -0
  176. package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
  177. package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
  178. package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
  179. package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
  180. package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
  181. package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
  182. package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
  183. package/dist/provenance-zsc4--dP.js.map +1 -0
  184. package/dist/provider-BnUoz4KB.js +1 -0
  185. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  186. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  187. package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
  188. package/dist/providers-B8xYMeah.js.map +1 -0
  189. package/dist/providers-CVu8e0xS.js +1 -0
  190. package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
  191. package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
  192. package/dist/query-DIfkUcPh.js +1 -0
  193. package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
  194. package/dist/questions-CWlR9ffu.js.map +1 -0
  195. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  196. package/dist/record-6IUzwP9K.js.map +1 -0
  197. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  198. package/dist/redact-BhjtnjrU.js.map +1 -0
  199. package/dist/remarks-CAvTSq16.js +1 -0
  200. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  201. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  202. package/dist/remarks-DPCONFxZ.js +138 -0
  203. package/dist/remarks-DPCONFxZ.js.map +1 -0
  204. package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
  205. package/dist/remarks-vtkPmQb9.js.map +1 -0
  206. package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
  207. package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
  208. package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
  209. package/dist/resolve-Do0ZaYH6.js.map +1 -0
  210. package/dist/resolve-amw_2Odz.js +1 -0
  211. package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
  212. package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
  213. package/dist/resume-AW5oXzUc.js +691 -0
  214. package/dist/resume-AW5oXzUc.js.map +1 -0
  215. package/dist/retry-ChzISxmy.js +105 -0
  216. package/dist/retry-ChzISxmy.js.map +1 -0
  217. package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
  218. package/dist/runtime-BpW1f1Mx.js.map +1 -0
  219. package/dist/sandbox/boat.d.ts +26 -0
  220. package/dist/sandbox/boat.js +37 -0
  221. package/dist/sandbox/boat.js.map +1 -0
  222. package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
  223. package/dist/sandbox-kSgr7vo5.js.map +1 -0
  224. package/dist/sandbox.d.ts +20 -2
  225. package/dist/sandbox.js +1 -1
  226. package/dist/server-2_gUnVvf.js +1 -0
  227. package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
  228. package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
  229. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  230. package/dist/shared-Cfi45Bvy.js.map +1 -0
  231. package/dist/slug-IHh3f770.js +136 -0
  232. package/dist/slug-IHh3f770.js.map +1 -0
  233. package/dist/source-CY3etKzw.js +1 -0
  234. package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
  235. package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
  236. package/dist/src-6Zzyg4Qd.js +2 -0
  237. package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
  238. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  239. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  240. package/dist/src-BRMsk9nh.js +2 -0
  241. package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
  242. package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
  243. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  244. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  245. package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
  246. package/dist/src-Ck2r_EMT.js.map +1 -0
  247. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  248. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  249. package/dist/src-DB7N8TYJ.js +2 -0
  250. package/dist/src-DPDaAYwh.js +2 -0
  251. package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
  252. package/dist/src-DtodMT59.js +1 -0
  253. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  254. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  255. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  256. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  257. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  258. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  259. package/dist/src-T-OeuJLr.js +2 -0
  260. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  261. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  262. package/dist/src-zntEDlnx.js +3 -0
  263. package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
  264. package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
  265. package/dist/state-kimtIhBP.js +1 -0
  266. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  267. package/dist/stats-C18W4xPX.js.map +1 -0
  268. package/dist/suite-CKBa6MKt.js +2 -0
  269. package/dist/suite-COglUhPd.js +1272 -0
  270. package/dist/suite-COglUhPd.js.map +1 -0
  271. package/dist/summary-DU5zaibx.js +148 -0
  272. package/dist/summary-DU5zaibx.js.map +1 -0
  273. package/dist/tart-BTVXp8rz.js +373 -0
  274. package/dist/tart-BTVXp8rz.js.map +1 -0
  275. package/dist/tart-cli-BIC9HqPg.js +37 -0
  276. package/dist/tart-cli-BIC9HqPg.js.map +1 -0
  277. package/dist/tart-cli-BMrkmBjI.js +1 -0
  278. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  279. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  280. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  281. package/dist/tart-ref-DYZijjrU.js +55 -0
  282. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  283. package/dist/testing.d.ts +47 -37
  284. package/dist/testing.js +4 -4
  285. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  286. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  287. package/dist/trace-Dx2NBE1O.js +311 -0
  288. package/dist/trace-Dx2NBE1O.js.map +1 -0
  289. package/dist/traits-DKnYkqeK.js +66 -0
  290. package/dist/traits-DKnYkqeK.js.map +1 -0
  291. package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
  292. package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
  293. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  294. package/dist/types-C2mLpph3.js.map +1 -0
  295. package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
  296. package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
  297. package/dist/vercel-CGlSBv63.js.map +1 -0
  298. package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
  299. package/dist/vertex-Eb2wqahk.js.map +1 -0
  300. package/dist/vertex-O88TCwgY.js +1 -0
  301. package/dist/view/index.d.ts +238 -90
  302. package/dist/view/index.js +11 -9
  303. package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
  304. package/dist/views-DAIiLiV3.js.map +1 -0
  305. package/dist/views-DtRhj2Tn.js +2 -0
  306. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
  307. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
  308. package/dist/workspace-CcQRT31C.js +1 -0
  309. package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
  310. package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
  311. package/package.json +26 -11
  312. package/skills/eval-rubric/SKILL.md +31 -4
  313. package/dist/adapter-DyvDj4ZN.js.map +0 -1
  314. package/dist/app-files-hpRT5lj_.js.map +0 -1
  315. package/dist/assemble-DTXwCL3m.js +0 -1
  316. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  317. package/dist/capture-BT2fAmpZ.js +0 -1
  318. package/dist/capture-Bdcx9Gap.js.map +0 -1
  319. package/dist/cli-CGd51thd.js.map +0 -1
  320. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  321. package/dist/client-CF-DOjOU.js.map +0 -1
  322. package/dist/command-CsfcwMyc.js.map +0 -1
  323. package/dist/command-CyIb9BSC.js.map +0 -1
  324. package/dist/command-TLmY-rYp.js.map +0 -1
  325. package/dist/commands-C54S45d7.js.map +0 -1
  326. package/dist/config-ImY6AlRE.js.map +0 -1
  327. package/dist/context-BajfiH0S.js.map +0 -1
  328. package/dist/contrasts-DtSX8E_V.js +0 -101
  329. package/dist/contrasts-DtSX8E_V.js.map +0 -1
  330. package/dist/cost-CMzY9VXv.js +0 -1
  331. package/dist/db-ZIk6S2CD.js +0 -2
  332. package/dist/db-sHGpaLo9.js.map +0 -1
  333. package/dist/debug-BnXV7JbX.js +0 -1
  334. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  335. package/dist/dist-ByYk_wAs.js.map +0 -1
  336. package/dist/docker--2ytOHcu.js.map +0 -1
  337. package/dist/doctor-eE65YpEv.js.map +0 -1
  338. package/dist/environment-Cy3FVLR5.js.map +0 -1
  339. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  340. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  341. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  342. package/dist/files-B3d-8Fe0.js +0 -1
  343. package/dist/files-COW5JBJM.js +0 -1
  344. package/dist/files-Do4Rp3yZ.js +0 -1
  345. package/dist/files-p-uTdOjA.js.map +0 -1
  346. package/dist/gate-BoDKeuJu.js.map +0 -1
  347. package/dist/harness-Ch2P81sx.js.map +0 -1
  348. package/dist/ids-DuK6xl4w.js.map +0 -1
  349. package/dist/images-BVZHDf_z.js.map +0 -1
  350. package/dist/init-BWnMbUnc.js.map +0 -1
  351. package/dist/init-s1jN-PCI.js +0 -1
  352. package/dist/judge-ByEUlzOM.js.map +0 -1
  353. package/dist/live-DmJix4c8.js +0 -1
  354. package/dist/live-d_5a5iUv.js.map +0 -1
  355. package/dist/model-8uZBYqV5.js +0 -1
  356. package/dist/model-DOiCiMCD.js.map +0 -1
  357. package/dist/node-CwP1G178.js.map +0 -1
  358. package/dist/opencode-providers-yNKyXD49.js.map +0 -1
  359. package/dist/plan-CIfOFhth.js.map +0 -1
  360. package/dist/plugins-BEs2Am3L.js.map +0 -1
  361. package/dist/preflight-50Qw5e5K.js.map +0 -1
  362. package/dist/provenance-Cc8fFE7Z.js.map +0 -1
  363. package/dist/provider-DCdTHQpk.js +0 -1
  364. package/dist/providers-CP_bBtYm.js +0 -1
  365. package/dist/providers-DZSTlC9R.js.map +0 -1
  366. package/dist/query-kFPuqBv1.js +0 -1
  367. package/dist/questions-C_Y-oXW6.js.map +0 -1
  368. package/dist/record-DhzwkSXc.js.map +0 -1
  369. package/dist/redact-BEEGc37e.js.map +0 -1
  370. package/dist/remarks-0EXMh6AD.js.map +0 -1
  371. package/dist/remarks-BL18Id09.js +0 -1
  372. package/dist/remarks-DeZZT8PK.js +0 -116
  373. package/dist/remarks-DeZZT8PK.js.map +0 -1
  374. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  375. package/dist/resolve-BJSay360.js +0 -1
  376. package/dist/resolve-B_JlQ0La.js.map +0 -1
  377. package/dist/resume-CexsWuGK.js +0 -434
  378. package/dist/resume-CexsWuGK.js.map +0 -1
  379. package/dist/runtime-CSZ1hjlN.js.map +0 -1
  380. package/dist/sandbox-B5W8lLL1.js.map +0 -1
  381. package/dist/server-SBzrf35T.js +0 -1
  382. package/dist/shared-ogqPJ0uo.js.map +0 -1
  383. package/dist/source-Dn1uqPon.js +0 -1
  384. package/dist/src-BKDZO3ra.js +0 -2
  385. package/dist/src-BxrHipV0.js +0 -3
  386. package/dist/src-C9elednc.js +0 -2
  387. package/dist/src-CM7xdJFA.js +0 -2
  388. package/dist/src-CWEGknZo.js +0 -2
  389. package/dist/src-Cpy9MNS3.js.map +0 -1
  390. package/dist/src-DVaAKCsH.js +0 -1
  391. package/dist/src-DurP--_D.js +0 -2
  392. package/dist/state-Cicc9Oa5.js +0 -1
  393. package/dist/stats-D-Gqwqt0.js.map +0 -1
  394. package/dist/suite-BYJ2Q0S-.js +0 -2
  395. package/dist/suite-BtHPBFYL.js +0 -544
  396. package/dist/suite-BtHPBFYL.js.map +0 -1
  397. package/dist/summary-zP3cCXDN.js.map +0 -1
  398. package/dist/types-C4OkYhZg.js.map +0 -1
  399. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  400. package/dist/vertex-BaAz44YN.js +0 -1
  401. package/dist/views-CKoN5NAx.js.map +0 -1
  402. package/dist/views-CmyX255j.js +0 -2
  403. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -1,189 +1,23 @@
1
1
  import { r as isTimeout } from "./errors-ttMTnIYX.js";
2
- import { D as profileSandbox } from "./environment-Cy3FVLR5.js";
3
- import { i as evalSlug, l as sha256, n as attemptKey, s as recordFileName, t as attemptId } from "./ids-DuK6xl4w.js";
4
- import { B as profileJudge, H as runsOf, M as harnessId, P as majorityOf, U as sandboxId, Y as mergeJudge, _t as servedEffortOf, ft as UNRECORDED_QUESTION_STATE, gt as resolvedEffortOf, ht as harnessEffort, z as plannedSandboxes } from "./config-ImY6AlRE.js";
5
- import { l as judgeModelName } from "./judge-ByEUlzOM.js";
6
- import { f as redactJsonText, h as redactSplitSecrets, m as redactSecrets } from "./redact-BEEGc37e.js";
7
- import { i as formatDuration } from "./text-D--Q5KsA.js";
8
- import { i as passRateOf, o as worstOutcome, r as gatePart, t as cellOutcome } from "./gate-BoDKeuJu.js";
9
- import { a as mcnemar, d as round3, f as signedRank, l as pairedLiftInterval, n as containsZero, o as meanInterval, p as wilson, r as liftInterval, s as median } from "./stats-D-Gqwqt0.js";
2
+ import { i as formatDuration } from "./text-B8-W3Cvn.js";
10
3
  import { i as sumUsage, r as promptTokens } from "./usage-DJiDnh2D.js";
11
- import { r as patchForJudge, t as collapseDependencies } from "./patch-DvLr8wCZ.js";
12
- import { a as sourcesPart, r as evalFile } from "./eval-source-BTsTRiBW.js";
13
- import { L as checkTries, ct as effortSample } from "./model-DOiCiMCD.js";
14
- import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-CdSjXIfE.js";
15
- import { t as checkVerdict } from "./contrasts-DtSX8E_V.js";
16
- import { n as isCompletedAttempt, o as previousAttempt } from "./resume-CexsWuGK.js";
4
+ import { T as profileSandbox } from "./environment-DB8o3hGi.js";
5
+ import { t as evalSlug } from "./slug-IHh3f770.js";
6
+ import { c as sha256, n as attemptKey, o as recordFileName, t as attemptId } from "./ids-C_advdJA.js";
7
+ import { $ as runsOf, Mt as resolvedEffortOf, Nt as servedEffortOf, St as mergeJudge, U as majorityOf, V as harnessId, X as plannedSandboxes, Z as profileJudge, et as sandboxId, jt as harnessEffort, yt as UNRECORDED_QUESTION_STATE } from "./config-B9mB1kF5.js";
8
+ import { g as redactSplitSecrets, h as redactSecrets, p as redactJsonText } from "./redact-BhjtnjrU.js";
9
+ import { l as judgeModelName } from "./judge-COMXT1Hx.js";
10
+ import { a as liftInterval, m as round3, r as containsZero } from "./stats-C18W4xPX.js";
11
+ import { i as passRateOf, o as worstOutcome, r as gatePart, t as cellOutcome } from "./gate-CgYdaprG.js";
12
+ import { r as patchForJudge, t as collapseDependencies } from "./patch-DH4e_Gkd.js";
13
+ import { i as evalFile, o as sourcesPart } from "./eval-source-D3sbAzt-.js";
14
+ import { c as foldCases, i as checkTallies, o as discrimination } from "./cases-z9E6DCd5.js";
15
+ import { i as checkTries } from "./judgments-CzgpsTqv.js";
16
+ import { v as familiesOf, y as pooledPassRate } from "./suite-COglUhPd.js";
17
+ import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-BOLt8pJg.js";
18
+ import { a as isKeptAttempt, f as previousAttempt, i as isCompletedAttempt, p as readRetries } from "./resume-AW5oXzUc.js";
17
19
  import { join } from "node:path";
18
20
  import { readFile, stat } from "node:fs/promises";
19
- // src/history/families.ts
20
- const isCounted = (a) => a.outcome === "pass" || a.outcome === "fail";
21
- function sideOf(entry, cell) {
22
- const counted = entry.attempts.filter((a) => a.cell === cell && isCounted(a));
23
- if (!counted.length) return null;
24
- return {
25
- pass: counted.filter((a) => a.outcome === "pass").length,
26
- counted: counted.length,
27
- samples: counted.map(effortSample)
28
- };
29
- }
30
- const clamp = (x, lo, hi) => Math.min(hi, Math.max(lo, x));
31
- const interval = (lo, hi, method, bound) => ({
32
- lo: round3(clamp(lo, ...bound)),
33
- hi: round3(clamp(hi, ...bound)),
34
- level: .95,
35
- method
36
- });
37
- function rateOf(cell, sides) {
38
- const pass = sides.reduce((s, x) => s + x.pass, 0);
39
- const counted = sides.reduce((s, x) => s + x.counted, 0);
40
- const single = sides.every((x) => x.counted === 1);
41
- const rates = sides.map((x) => x.pass / x.counted);
42
- const t = meanInterval(rates);
43
- const w = wilson(pass, counted);
44
- return {
45
- cell,
46
- cases: sides.length,
47
- pass,
48
- counted,
49
- rate: round3(t.mean),
50
- interval: single ? interval(w.low, w.high, "wilson", [0, 1]) : interval(t.low, t.high, "t-cases", [0, 1])
51
- };
52
- }
53
- function metric(pairs) {
54
- const differences = pairs.map(([v, c]) => v - c);
55
- const control = median(pairs.map(([, c]) => c)) ?? 0;
56
- const variant = median(pairs.map(([v]) => v)) ?? 0;
57
- const test = signedRank(differences);
58
- const verdict = differences.filter((d) => d !== 0).length < 6 ? "insufficient" : test === null || test.p >= .05 ? "noise" : test.sign < 0 ? "fewer" : "more";
59
- return {
60
- pairs: pairs.length,
61
- ratio: control === 0 ? null : round3(variant / control),
62
- p: test === null ? null : round3(test.p),
63
- verdict
64
- };
65
- }
66
- function effortOf(pairs) {
67
- const per = (pick) => pairs.flatMap(([v, c]) => {
68
- const vs = v.samples.map(pick);
69
- const cs = c.samples.map(pick);
70
- if (!vs.every((x) => x !== void 0) || !cs.every((x) => x !== void 0)) return [];
71
- return [[median(vs) ?? 0, median(cs) ?? 0]];
72
- });
73
- const tokens = per((s) => s.tokens);
74
- return {
75
- tools: metric(per((s) => s.tools)),
76
- agentMs: metric(per((s) => s.agentMs)),
77
- ...tokens.length ? { tokens: metric(tokens) } : {}
78
- };
79
- }
80
- function liftOf$1(variant, control, cases, minLift) {
81
- const pairs = cases.flatMap((e) => {
82
- const v = sideOf(e, variant);
83
- const c = sideOf(e, control);
84
- return v && c ? [[v, c]] : [];
85
- });
86
- if (!pairs.length) return null;
87
- const single = pairs.every(([v, c]) => v.counted === 1 && c.counted === 1);
88
- const vRates = pairs.map(([v]) => v.pass / v.counted);
89
- const cRates = pairs.map(([, c]) => c.pass / c.counted);
90
- const mean = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
91
- let points;
92
- let range;
93
- let p;
94
- let discordant;
95
- let decided;
96
- if (single) {
97
- const counts = {
98
- both: 0,
99
- variantOnly: 0,
100
- controlOnly: 0,
101
- neither: 0
102
- };
103
- for (const [v, c] of pairs) if (v.pass && c.pass) counts.both++;
104
- else if (v.pass) counts.variantOnly++;
105
- else if (c.pass) counts.controlOnly++;
106
- else counts.neither++;
107
- const paired = pairedLiftInterval(counts);
108
- points = paired.points;
109
- range = interval(paired.low, paired.high, "newcombe-paired", [-1, 1]);
110
- p = round3(mcnemar(counts.variantOnly, counts.controlOnly));
111
- discordant = {
112
- variantOnly: counts.variantOnly,
113
- controlOnly: counts.controlOnly
114
- };
115
- decided = counts.variantOnly + counts.controlOnly >= 6;
116
- } else {
117
- const differences = pairs.map(([v, c]) => (v.pass * c.counted - c.pass * v.counted) / (v.counted * c.counted));
118
- const t = meanInterval(differences);
119
- points = t.mean;
120
- range = interval(t.low, t.high, "t-cases", [-1, 1]);
121
- const rank = signedRank(differences);
122
- p = rank === null ? null : round3(rank.p);
123
- decided = pairs.length >= 6;
124
- }
125
- const verdict = !decided ? "insufficient" : range.lo > 0 ? "lift" : range.hi < 0 ? "drop" : "noise";
126
- return {
127
- variant,
128
- vs: control,
129
- pairs: pairs.length,
130
- variantRate: round3(mean(vRates)),
131
- controlRate: round3(mean(cRates)),
132
- ...discordant ? { discordant } : {},
133
- points: round3(points),
134
- interval: range,
135
- p,
136
- withinNoise: containsZero(range.lo, range.hi),
137
- verdict,
138
- ...minLift !== void 0 ? { minLift } : {},
139
- outcome: minLift !== void 0 && range.lo < minLift ? "fail" : "pass",
140
- effort: effortOf(pairs)
141
- };
142
- }
143
- /**
144
- * The run's pooled families, in the order their first case appears; `minLiftOf` gives a variant
145
- * cell's `compare.minLift`.
146
- */
147
- function familiesOf(evals, minLiftOf) {
148
- const byId = /* @__PURE__ */ new Map();
149
- for (const e of evals) {
150
- if (!e.family || e.todo) continue;
151
- byId.set(e.family.id, [...byId.get(e.family.id) ?? [], e]);
152
- }
153
- return [...byId.values()].map((members) => {
154
- const cases = [...members].sort((a, b) => (a.family?.case ?? 0) - (b.family?.case ?? 0));
155
- const first = cases[0];
156
- const family = first.family;
157
- const cellKeys = [];
158
- for (const e of cases) for (const c of e.cells) if (!cellKeys.includes(c.key)) cellKeys.push(c.key);
159
- const rates = cellKeys.flatMap((key) => {
160
- const sides = cases.flatMap((e) => sideOf(e, key) ?? []);
161
- return sides.length ? [rateOf(key, sides)] : [];
162
- });
163
- const lifts = [];
164
- for (const key of cellKeys) {
165
- const holder = cases.find((e) => e.cells.some((c) => c.key === key && c.role === "variant"));
166
- const variant = holder?.cells.find((c) => c.key === key);
167
- if (!holder || !variant) continue;
168
- const control = holder.cells.find((c) => c.role === "control" && c.harness === variant.harness && c.sandbox === variant.sandbox);
169
- if (!control) continue;
170
- const minLift = cases.flatMap((e) => e.cells.filter((c) => c.key === key)).map(minLiftOf).find((m) => m !== void 0);
171
- const lift = liftOf$1(key, control.key, cases, minLift);
172
- if (lift) lifts.push(lift);
173
- }
174
- return {
175
- id: family.id,
176
- file: first.file,
177
- group: first.group,
178
- name: family.name,
179
- cases: family.cases,
180
- ran: cases.length,
181
- rates,
182
- lifts,
183
- outcome: lifts.some((l) => l.outcome === "fail") ? "fail" : "pass"
184
- };
185
- });
186
- }
187
21
  // src/history/assemble.ts
188
22
  /** Final messages are cut at this size in the record; `transcript.jsonl` keeps them whole. */
189
23
  const TURN_TEXT_LIMIT = 8192;
@@ -266,6 +100,15 @@ function mergeOutcome(e, record) {
266
100
  */
267
101
  const withoutCaps = ({ maxConcurrent: _, startTimeout: __, gateway: ___, ...rest }) => rest;
268
102
  const harnessModel = (h) => h.kind === "fake" ? null : h.model ?? null;
103
+ /**
104
+ * The harness's part of a fingerprint: its descriptor, less the operational limits, and less the
105
+ * fake harness's `failWith`, its stand-in for an outage, which a retry of the attempt outlives.
106
+ */
107
+ const harnessPart = (harness) => {
108
+ if (!harness) return sha256("null");
109
+ const { failWith: _, ...measured } = withoutCaps(harness);
110
+ return sha256(JSON.stringify(measured));
111
+ };
269
112
  /** A cell fingerprint: the hash of its parts, so each part can be compared on its own. */
270
113
  const fingerprintOf = (parts) => sha256(JSON.stringify(parts));
271
114
  /** The gateway's part of a fingerprint: none on `direct`, so a cell without a gateway keeps its fingerprint. */
@@ -473,6 +316,8 @@ function judgeInputs(record, diff, diffPath) {
473
316
  }
474
317
  /** What the run's attempts have in common, gathered while their records are read. */
475
318
  var Seen = class {
319
+ /** Each attempt's earlier infra errors, from staging's `retries.json`, by key. */
320
+ retries = /* @__PURE__ */ new Map();
476
321
  sources = /* @__PURE__ */ new Map();
477
322
  byEval = /* @__PURE__ */ new Map();
478
323
  contexts = /* @__PURE__ */ new Map();
@@ -481,6 +326,8 @@ var Seen = class {
481
326
  environments = /* @__PURE__ */ new Map();
482
327
  harnesses = /* @__PURE__ */ new Map();
483
328
  resolvedHarnesses = /* @__PURE__ */ new Map();
329
+ /** Every pair `where` left out of a pooled family, by family id, from its cases' plans. */
330
+ familyExcluded = /* @__PURE__ */ new Map();
484
331
  /** Keys of the attempts kept from the previous record on resume. */
485
332
  kept = /* @__PURE__ */ new Set();
486
333
  /** Registry digests found by the preflight, by image id. */
@@ -550,18 +397,22 @@ var Seen = class {
550
397
  tags: e.attempt?.tags ?? [],
551
398
  ...e.attempt?.plan.compare ? { compare: e.attempt.plan.compare } : {},
552
399
  ...e.attempt?.family ? { family: familyOf(e.attempt.family) } : {},
400
+ ...e.attempt?.plan.excluded ? { excluded: e.attempt.plan.excluded } : {},
553
401
  fingerprint: "",
554
402
  sources,
555
403
  ...from.source ? { source: from.source } : {},
556
404
  ...from.revision !== void 0 ? { revision: from.revision } : {},
557
405
  outcome: "skipped",
558
406
  todo: e.todo === true,
407
+ ...e.skip ? { skip: true } : {},
559
408
  passRate: null,
560
409
  cells: [],
561
410
  attempts: []
562
411
  };
563
412
  this.byEval.set(e.id, entry);
564
413
  }
414
+ const familyExcluded = e.attempt?.plan.familyExcluded;
415
+ if (entry.family && familyExcluded) this.familyExcluded.set(entry.family.id, familyExcluded);
565
416
  return entry;
566
417
  }
567
418
  };
@@ -578,6 +429,14 @@ const UNKNOWN_CELL = {
578
429
  sandbox: "unknown",
579
430
  role: "control"
580
431
  };
432
+ /** A cell known only by its key, `<harness>/<environment>/<sandbox>`. */
433
+ const cellOfKey = (key) => ({
434
+ ...UNKNOWN_CELL,
435
+ key,
436
+ harness: key.split("/")[0] ?? "unknown",
437
+ environment: key.split("/")[1] ?? "unknown",
438
+ sandbox: key.split("/")[2]?.replace(/#\d+$/, "") ?? "unknown"
439
+ });
581
440
  /** minLift per cell, kept aside while attempts are added; see `rollUp`. */
582
441
  const minLifts = /* @__PURE__ */ new WeakMap();
583
442
  /** Cells whose fingerprint came from an attempt that ran, not one that was skipped. */
@@ -635,13 +494,7 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
635
494
  sandbox: beforeCell.sandbox,
636
495
  role: beforeCell.role,
637
496
  ...beforeCell.variant !== void 0 ? { variant: beforeCell.variant } : {}
638
- }) ?? {
639
- ...UNKNOWN_CELL,
640
- key: kept.cell,
641
- harness: kept.cell.split("/")[0] ?? "unknown",
642
- environment: kept.cell.split("/")[1] ?? "unknown",
643
- sandbox: kept.cell.split("/")[2]?.replace(/#\d+$/, "") ?? "unknown"
644
- };
497
+ }) ?? cellOfKey(kept.cell);
645
498
  const sources = before?.sources ?? [];
646
499
  const { source, revision } = e.attempt ?? before ?? {};
647
500
  const stub = {
@@ -655,7 +508,9 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
655
508
  durationMs: kept.durationMs,
656
509
  errors: [],
657
510
  formattedErrors: [],
658
- meta: void 0
511
+ meta: void 0,
512
+ ...before?.todo ? { todo: true } : {},
513
+ ...before?.skip ? { skip: true } : {}
659
514
  };
660
515
  seen.add(null, cell, stub);
661
516
  seen.addKept(previous, kept, cell, beforeCell?.environmentDigest ?? null, e.attempt !== void 0);
@@ -663,6 +518,7 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
663
518
  source,
664
519
  revision
665
520
  });
521
+ if (!e.attempt && !entry.excluded && before?.excluded) entry.excluded = before.excluded;
666
522
  const created = !entry.cells.some((c) => c.key === cell.key);
667
523
  const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? beforeCell?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway, e.attempt ? e.attempt.bestOf : beforeCell?.bestOf);
668
524
  if (created && beforeCell && !e.attempt) {
@@ -732,7 +588,7 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
732
588
  } else {
733
589
  const parts = {
734
590
  sources: sourcesPart(source, revision, sources),
735
- harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
591
+ harness: harnessPart(e.attempt?.harness),
736
592
  environment: cellRecord.environmentDigest,
737
593
  image: kept.imageId,
738
594
  agentCli: null,
@@ -745,16 +601,22 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
745
601
  cellRecord.fingerprint = fingerprintOf(parts);
746
602
  }
747
603
  }
604
+ /** The earlier infra errors of an attempt that ran again, less the try its record still is. */
605
+ function retriesOf(retries, record) {
606
+ const earlier = (retries ?? []).filter((r) => !(record?.outcome === "infra_error" && r.at === (record.startedAt ?? null) && r.error.message === record.error?.message));
607
+ return earlier.length ? { retries: earlier } : {};
608
+ }
748
609
  /** Reads one attempt's worker record, files its artifacts and adds it to its eval and cell. */
749
- async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts) {
750
- const cell = e.attempt?.cell ?? UNKNOWN_CELL;
751
- const n = e.attempt?.n ?? 1;
610
+ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts, unplanned) {
611
+ const cell = e.attempt?.cell ?? unplanned?.cell ?? UNKNOWN_CELL;
612
+ const n = e.attempt?.n ?? unplanned?.n ?? 1;
752
613
  const kept = previousAttempt(previous, e.key);
753
- if (previous && e.state === "skipped" && kept && isCompletedAttempt(kept.outcome, kept.error)) {
614
+ const record = await readRecord(join(ctx.stagingDir, "evals", recordFileName(e.key)));
615
+ const rerun = kept?.outcome === "infra_error" && !!record && isKeptAttempt(record.outcome, record.error);
616
+ if (previous && e.state === "skipped" && kept && !rerun && isCompletedAttempt(kept.outcome, kept.error)) {
754
617
  await addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts);
755
618
  return;
756
619
  }
757
- const record = await readRecord(join(ctx.stagingDir, "evals", recordFileName(e.key)));
758
620
  const { outcome, error } = mergeOutcome(e, record);
759
621
  if (!seen.sources.has(e.file)) seen.sources.set(e.file, await evalFile(ctx.config.root, e.file));
760
622
  const sources = seen.sources.get(e.file) ?? [];
@@ -776,9 +638,9 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
776
638
  if (record) measured.add(cellRecord);
777
639
  const parts = {
778
640
  sources: sourcesPart(source, revision, sources),
779
- harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
641
+ harness: harnessPart(e.attempt?.harness),
780
642
  environment: environment?.digest ?? null,
781
- image: record?.image?.id ?? null,
643
+ image: record?.image?.id ?? record?.vmImage?.digest ?? null,
782
644
  agentCli: record?.image?.agentCli ?? null,
783
645
  judge,
784
646
  context: sha256(JSON.stringify(context)),
@@ -796,6 +658,7 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
796
658
  seq: null,
797
659
  startedAt: record?.startedAt ?? null,
798
660
  imageId: record?.image?.id ?? null,
661
+ ...record?.vmImage ? { vmImage: record.vmImage } : {},
799
662
  env: record?.env ?? {
800
663
  vars: [],
801
664
  secrets: [],
@@ -812,6 +675,7 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
812
675
  outcome,
813
676
  error,
814
677
  durationMs: e.durationMs,
678
+ ...retriesOf(seen.retries.get(e.key), record),
815
679
  ...await attemptEvidence(record, `${entry.slug}/${evalSlug(cell.key)}/${n}`, out, join(ctx.stagingDir, "evals", recordFileName(e.key)))
816
680
  });
817
681
  }
@@ -926,6 +790,7 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
926
790
  ...snapshots.length ? { snapshots } : {},
927
791
  ...record?.serves?.length ? { serves: record.serves } : {},
928
792
  ...record?.assertions?.length ? { assertions: record.assertions } : {},
793
+ ...record?.observations?.length ? { observations: record.observations } : {},
929
794
  checks: (record?.checks ?? []).map((c, i) => ({
930
795
  command: c.command,
931
796
  exitCode: c.exitCode,
@@ -978,45 +843,6 @@ function liftOf(control, variant, pooled) {
978
843
  };
979
844
  }
980
845
  /**
981
- * Each cell's checks: every label any attempt of the eval recorded, tallied over the cell's
982
- * counted attempts (pass or fail). An attempt that counted without reaching a label, because
983
- * an `expect.require` or a thrown error before it ended the body, counts it `notRun`.
984
- */
985
- function checkTallies(entry) {
986
- const labels = [...new Set(entry.attempts.flatMap((a) => (a.assertions ?? []).map((x) => x.label)))];
987
- if (!labels.length) return;
988
- for (const c of entry.cells) {
989
- const counted = entry.attempts.filter((a) => a.cell === c.key && (a.outcome === "pass" || a.outcome === "fail"));
990
- if (!counted.length) continue;
991
- c.checks = labels.map((label) => {
992
- const tally = {
993
- label,
994
- pass: 0,
995
- fail: 0,
996
- error: 0,
997
- notRun: 0
998
- };
999
- for (const a of counted) {
1000
- const status = a.assertions?.find((x) => x.label === label)?.status;
1001
- if (status === void 0) tally.notRun++;
1002
- else tally[status]++;
1003
- }
1004
- return tally;
1005
- });
1006
- }
1007
- }
1008
- /** Puts the control's tally beside each of the variant's checks, with whether it discriminates. */
1009
- function discrimination(control, variant) {
1010
- for (const check of variant.checks ?? []) {
1011
- const k = control.checks?.find((x) => x.label === check.label);
1012
- if (!k) continue;
1013
- const { label: _label, ...tally } = k;
1014
- check.control = tally;
1015
- const verdict = checkVerdict(check, k);
1016
- if (verdict) check.verdict = verdict;
1017
- }
1018
- }
1019
- /**
1020
846
  * Marks a best-of cell's attempts past its majority's first: `added` for one that ran, `skipped`
1021
847
  * for one the majority before it left unrun.
1022
848
  */
@@ -1034,6 +860,8 @@ function markBestOf(cell, attempts) {
1034
860
  if (a.outcome === "fail") fail++;
1035
861
  }
1036
862
  }
863
+ /** A case of an eval with several cases, which pools its gates over them. */
864
+ const pooled = (entry) => (entry.family?.cases ?? 0) >= 2;
1037
865
  /** Per-cell and per-eval counts, pass rates, lift, outcomes and fingerprints, once every attempt is in. */
1038
866
  function rollUp(evals) {
1039
867
  for (const entry of evals) {
@@ -1053,12 +881,14 @@ function rollUp(evals) {
1053
881
  if (c.role !== "variant") continue;
1054
882
  const control = entry.cells.find((x) => x.role === "control" && x.harness === c.harness && x.sandbox === c.sandbox);
1055
883
  if (control) discrimination(control, c);
1056
- const lift = control && liftOf(control, c, entry.family !== void 0);
884
+ const lift = control && liftOf(control, c, pooled(entry));
1057
885
  if (!lift) continue;
1058
886
  c.lift = lift;
1059
887
  if (lift.minLift !== void 0 && lift.interval.lo < lift.minLift && c.outcome === "pass") c.outcome = "fail";
1060
888
  }
1061
889
  entry.outcome = worstOutcome(entry.cells.map((c) => c.gate === "none" && c.outcome === "fail" ? "pass" : c.outcome));
890
+ if (pooled(entry) && entry.cells.some((c) => pooledPassRate(c) !== void 0)) entry.verdict = "family";
891
+ else delete entry.verdict;
1062
892
  const gated = new Set(entry.cells.filter((c) => c.gate !== "none").map((c) => c.key));
1063
893
  entry.passRate = passRateOf(entry.attempts.filter((a) => gated.has(a.cell)).map((a) => a.outcome));
1064
894
  entry.fingerprint = sha256(JSON.stringify(entry.cells.map((c) => [c.key, c.fingerprint]).sort()));
@@ -1091,6 +921,15 @@ function gitState(input, out) {
1091
921
  }
1092
922
  return state;
1093
923
  }
924
+ /** A retried run's git state as it ran, with its `worktree.patch` carried over byte-exact. */
925
+ async function keptGit(previous, from, out) {
926
+ const ref = previous.git?.worktreePatch;
927
+ if (ref && from) {
928
+ const bytes = await readFile(join(from, ...ref.path.split("/"))).catch(() => null);
929
+ if (bytes) out.files.set(ref.path, bytes);
930
+ }
931
+ return previous.git;
932
+ }
1094
933
  const byName = (key) => (a, b) => key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0;
1095
934
  function stackOf(input, seen, judgeModel, threshold) {
1096
935
  const packages = packageVersions(input.enginePackages);
@@ -1121,6 +960,19 @@ function stackOf(input, seen, judgeModel, threshold) {
1121
960
  fingerprintScope: FINGERPRINT_SCOPE
1122
961
  };
1123
962
  }
963
+ /** The judge's part of every cell's fingerprint, and the judge model's name (null for none). */
964
+ function judgePart(config, profile, reeval, enginePackages) {
965
+ const runJudge = profile ? mergeJudge(config.judge, profileJudge(profile)) : config.judge;
966
+ const model = runJudge.model === void 0 ? null : judgeModelName(runJudge.model);
967
+ return {
968
+ part: sha256(JSON.stringify({
969
+ model,
970
+ threshold: runJudge.threshold,
971
+ version: judgeVersion(reeval, packageVersions(enginePackages), model ?? "")
972
+ })),
973
+ model
974
+ };
975
+ }
1124
976
  async function assembleRun(input) {
1125
977
  const { ctx, engine } = input;
1126
978
  const { config, runId, profile } = ctx;
@@ -1131,18 +983,15 @@ async function assembleRun(input) {
1131
983
  include: config.include,
1132
984
  harness: config.harness.map(withoutCaps),
1133
985
  sandbox: withoutCaps(config.sandbox),
1134
- environment: config.environment ?? null,
986
+ environment: config.environments.length > 1 ? config.environments : config.environments[0] ?? null,
1135
987
  runs: config.runs,
1136
988
  env: config.env,
1137
989
  judge: withoutCaps(config.judge),
1138
990
  timeoutMs: config.timeoutMs
1139
991
  }));
1140
992
  const seen = new Seen(input.sandbox);
1141
- const judge = sha256(JSON.stringify({
1142
- model: judgeModel,
1143
- threshold: runJudge.threshold,
1144
- version: judgeVersion(input.reeval, packageVersions(input.enginePackages), judgeModel ?? "")
1145
- }));
993
+ seen.retries = readRetries(ctx.stagingDir);
994
+ const { part: judge } = judgePart(config, profile, input.reeval, input.enginePackages);
1146
995
  for (const e of engine.evals) await addAttempt(e, ctx, seen, out, judge, input.previous, input.previousArtifacts);
1147
996
  if (input.previous) {
1148
997
  const have = new Set([...seen.byEval.values()].flatMap((entry) => entry.attempts.map((a) => a.key)));
@@ -1163,13 +1012,37 @@ async function assembleRun(input) {
1163
1012
  }, a, seen, out, judge, input.previous, input.previousArtifacts);
1164
1013
  }
1165
1014
  }
1015
+ const added = new Set([...seen.byEval.values()].flatMap((e) => e.attempts.map((a) => a.key)));
1016
+ for (const l of input.leftOut ?? []) {
1017
+ if (added.has(l.key)) continue;
1018
+ const [, ...names] = l.evalId.split(" › ");
1019
+ const stub = {
1020
+ id: l.evalId,
1021
+ key: l.key,
1022
+ file: l.file,
1023
+ group: names.slice(0, -1),
1024
+ name: l.name,
1025
+ attempt: void 0,
1026
+ state: "skipped",
1027
+ durationMs: 0,
1028
+ errors: [],
1029
+ formattedErrors: [],
1030
+ meta: void 0
1031
+ };
1032
+ const was = previousAttempt(input.previous, l.key);
1033
+ if (input.previous && was) await addPreviousAttempt(stub, was, seen, out, judge, input.previous, input.previousArtifacts);
1034
+ else await addAttempt(stub, ctx, seen, out, judge, null, null, {
1035
+ cell: cellOfKey(l.cell),
1036
+ n: l.n
1037
+ });
1038
+ }
1166
1039
  const evals = [...seen.byEval.values()];
1167
1040
  for (const entry of evals) {
1168
1041
  const before = input.previous?.evals.find((p) => p.id === entry.id)?.family;
1169
1042
  if (!entry.family && before) entry.family = before;
1170
1043
  }
1171
1044
  rollUp(evals);
1172
- const families = familiesOf(evals, (cell) => minLifts.get(cell));
1045
+ const families = familiesOf(evals, (cell) => minLifts.get(cell), (id) => seen.familyExcluded.get(id) ?? input.previous?.families?.find((f) => f.id === id)?.excluded);
1173
1046
  sequence(evals);
1174
1047
  for (const e of evals) {
1175
1048
  const asked = input.answers?.evals.get(e.id);
@@ -1191,73 +1064,87 @@ async function assembleRun(input) {
1191
1064
  const attempts = evals.flatMap((e) => e.attempts);
1192
1065
  const count = (o) => attempts.filter((a) => a.outcome === o).length;
1193
1066
  const notGated = evals.flatMap((e) => e.attempts.filter((a) => e.cells.find((c) => c.key === a.cell)?.gate === "none")).filter((a) => a.outcome === "fail").length;
1067
+ const retried = input.retry && input.previous ? input.previous : null;
1068
+ const stack = stackOf(input, seen, judgeModel, runJudge.threshold);
1194
1069
  const ids = (list) => list === void 0 ? void 0 : [list].flat().map(harnessId);
1195
- return {
1196
- record: {
1197
- kind: "reeval-run",
1198
- schemaVersion: 1,
1199
- runId,
1200
- status: input.interrupted ? "interrupted" : input.budgetReached ? "budget" : "complete",
1201
- startedAt: input.startedAt.toISOString(),
1202
- endedAt: input.endedAt.toISOString(),
1203
- durationMs: input.endedAt.getTime() - input.startedAt.getTime(),
1204
- author: input.author,
1205
- git: gitState(input, out),
1206
- runner: input.runner,
1207
- dependencies: { lockfiles: input.lockfiles },
1208
- stack: stackOf(input, seen, judgeModel, runJudge.threshold),
1209
- environments: [...seen.environments.values()].sort(byName((e) => e.name)),
1210
- profile: profile ? {
1211
- name: profile.name,
1212
- reason: profile.reason,
1213
- defaults: JSON.parse(JSON.stringify({
1214
- harness: ids(profile.defaults.harness),
1215
- environment: profile.defaults.environment?.name,
1216
- sandbox: (profile.defaults.environment || profile.defaults.sandbox) && sandboxId(profileSandbox(config, profile.defaults)),
1217
- ...profile.defaults.runs !== void 0 ? runsOf(profile.defaults.runs) : {},
1218
- timeout: profile.defaults.timeout,
1219
- env: profile.defaults.env && Object.keys(profile.defaults.env).sort()
1220
- })),
1221
- limit: profile.limit
1222
- } : null,
1223
- plan: {
1224
- evals: input.plan.evals,
1225
- cells: input.plan.cells,
1226
- attempts: input.plan.attempts,
1227
- estimateMs: Math.round(input.plan.estimateMs),
1228
- sandboxWarnings: input.plan.sandboxWarnings
1229
- },
1230
- budget: Object.keys(input.plan.budget).length ? input.plan.budget : null,
1231
- config: {
1232
- sandbox: plannedSandboxes(input.plan.sandboxes, profileSandbox(config, profile?.defaults)),
1233
- environment: config.environment?.name ?? null,
1234
- ...config.runs !== void 0 ? runsOf(config.runs) : { runs: 1 },
1235
- judgeModel,
1236
- threshold: runJudge.threshold,
1237
- timeoutMs: config.timeoutMs,
1238
- concurrency: input.ctx.concurrency,
1239
- configHash
1240
- },
1241
- cost: costOf(attempts),
1242
- artifacts: "all",
1243
- redaction: { version: 1 },
1244
- summary: {
1245
- evals: evals.length,
1246
- attempts: attempts.length,
1247
- pass: count("pass"),
1248
- fail: count("fail"),
1249
- notGated,
1250
- infraError: count("infra_error"),
1251
- skipped: count("skipped")
1070
+ const record = {
1071
+ kind: "reeval-run",
1072
+ schemaVersion: 1,
1073
+ runId,
1074
+ status: input.interrupted ? "interrupted" : input.budgetReached ? "budget" : input.leftOut?.some((l) => !l.done) ? input.previous?.status === "budget" ? "budget" : "interrupted" : "complete",
1075
+ startedAt: input.startedAt.toISOString(),
1076
+ endedAt: input.endedAt.toISOString(),
1077
+ durationMs: retried ? retried.durationMs + input.endedAt.getTime() - (input.retry?.startedAt ?? input.endedAt).getTime() : input.endedAt.getTime() - input.startedAt.getTime(),
1078
+ ...retried && input.retry ? { retriedAt: [...retried.retriedAt ?? [], ...!input.retry.resume ? [input.retry.startedAt.toISOString()] : input.retry.began ? [input.retry.began.toISOString()] : []] } : {},
1079
+ author: input.author,
1080
+ git: retried ? await keptGit(retried, input.previousArtifacts, out) : gitState(input, out),
1081
+ runner: input.runner,
1082
+ dependencies: retried ? retried.dependencies : { lockfiles: input.lockfiles },
1083
+ stack: retried?.stack ? {
1084
+ ...retried.stack,
1085
+ images: stack.images,
1086
+ harnesses: stack.harnesses,
1087
+ judge: {
1088
+ ...retried.stack.judge,
1089
+ resolvedModels: stack.judge.resolvedModels
1252
1090
  },
1253
- evals,
1254
- ...families.length ? { families } : {},
1255
- ...engine.unhandledErrors.length ? { errors: engine.unhandledErrors.map((e) => redactSecrets(e)) } : {}
1091
+ context: stack.context
1092
+ } : stack,
1093
+ environments: [...seen.environments.values()].sort(byName((e) => e.name)),
1094
+ profile: profile ? {
1095
+ name: profile.name,
1096
+ reason: profile.reason,
1097
+ defaults: JSON.parse(JSON.stringify({
1098
+ harness: ids(profile.defaults.harnesses),
1099
+ environment: profile.defaults.environments && [profile.defaults.environments].flat().map((e) => e.name).join(", "),
1100
+ sandbox: (profile.defaults.environments || profile.defaults.sandbox) && sandboxId(profileSandbox(config, profile.defaults)),
1101
+ ...profile.defaults.runs !== void 0 ? runsOf(profile.defaults.runs) : {},
1102
+ timeout: profile.defaults.timeout,
1103
+ env: profile.defaults.env && Object.keys(profile.defaults.env).sort()
1104
+ })),
1105
+ limit: profile.limit
1106
+ } : null,
1107
+ ...input.invocation ? { invocation: input.invocation } : {},
1108
+ plan: retried ? retried.plan : {
1109
+ evals: input.plan.evals,
1110
+ cells: input.plan.cells,
1111
+ attempts: input.plan.attempts,
1112
+ estimateMs: Math.round(input.plan.estimateMs),
1113
+ sandboxWarnings: input.plan.sandboxWarnings
1256
1114
  },
1115
+ budget: Object.keys(input.plan.budget).length ? input.plan.budget : null,
1116
+ config: {
1117
+ sandbox: plannedSandboxes(input.plan.sandboxes, profileSandbox(config, profile?.defaults)),
1118
+ environment: config.environments.map((e) => e.name).join(", ") || null,
1119
+ ...config.runs !== void 0 ? runsOf(config.runs) : { runs: 1 },
1120
+ judgeModel,
1121
+ threshold: runJudge.threshold,
1122
+ timeoutMs: config.timeoutMs,
1123
+ concurrency: input.ctx.concurrency,
1124
+ configHash
1125
+ },
1126
+ cost: costOf(attempts),
1127
+ artifacts: "all",
1128
+ redaction: { version: 1 },
1129
+ summary: {
1130
+ evals: evals.length,
1131
+ attempts: attempts.length,
1132
+ pass: count("pass"),
1133
+ fail: count("fail"),
1134
+ notGated,
1135
+ infraError: count("infra_error"),
1136
+ skipped: count("skipped")
1137
+ },
1138
+ evals,
1139
+ ...families.length ? { families } : {},
1140
+ ...engine.unhandledErrors.length ? { errors: engine.unhandledErrors.map((e) => redactSecrets(e)) } : {}
1141
+ };
1142
+ return {
1143
+ record: foldCases(record),
1257
1144
  artifacts: out.files,
1258
1145
  moves: out.moves
1259
1146
  };
1260
1147
  }
1261
- export { gatewayPart as a, rawLines as c, turnText as d, fingerprintOf as i, transcriptLines as l, assembleRun as n, judgmentKey as o, capText as r, mergeOutcome as s, OUTPUT_CAP as t, turnError as u };
1148
+ export { gatewayPart as a, judgmentKey as c, transcriptLines as d, turnError as f, fingerprintOf as i, mergeOutcome as l, assembleRun as n, harnessPart as o, turnText as p, capText as r, judgePart as s, OUTPUT_CAP as t, rawLines as u };
1262
1149
 
1263
- //# sourceMappingURL=assemble-Dlh6Xv3U.js.map
1150
+ //# sourceMappingURL=assemble-t6h018An.js.map