reeval 0.17.2 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/README.md +4 -2
  2. package/dist/{adapter-DyvDj4ZN.js → adapter-CptK032x.js} +2 -2
  3. package/dist/{adapter-DyvDj4ZN.js.map → adapter-CptK032x.js.map} +1 -1
  4. package/dist/{agent-v036GEgG.js → agent-B63OzUX9.js} +6 -6
  5. package/dist/{agent-v036GEgG.js.map → agent-B63OzUX9.js.map} +1 -1
  6. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  7. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  8. package/dist/{app-files-hpRT5lj_.js → app-files-sgqwfjtn.js} +4 -4
  9. package/dist/app-files-sgqwfjtn.js.map +1 -0
  10. package/dist/assemble-CxBHkavM.js +1 -0
  11. package/dist/{assemble-Dlh6Xv3U.js → assemble-Me7Whp-y.js} +58 -194
  12. package/dist/assemble-Me7Whp-y.js.map +1 -0
  13. package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
  14. package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
  15. package/dist/{attempt-host-DErH-DYV.js → attempt-host-Cik0SoIn.js} +17 -9
  16. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
  17. package/dist/{capture-Bdcx9Gap.js → capture-C3RmjPph.js} +392 -613
  18. package/dist/capture-C3RmjPph.js.map +1 -0
  19. package/dist/capture-YPwxp_iK.js +1 -0
  20. package/dist/{cli-CGd51thd.js → cli-BBOlXKPG.js} +1075 -457
  21. package/dist/cli-BBOlXKPG.js.map +1 -0
  22. package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
  23. package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
  26. package/dist/client-tZ2WBDxf.js.map +1 -0
  27. package/dist/{command-TLmY-rYp.js → command-CFQ9c1hk.js} +13 -13
  28. package/dist/{command-TLmY-rYp.js.map → command-CFQ9c1hk.js.map} +1 -1
  29. package/dist/{command-CtTdCF3B.js → command-DhiwsZl1.js} +9 -9
  30. package/dist/{command-CtTdCF3B.js.map → command-DhiwsZl1.js.map} +1 -1
  31. package/dist/{command-CyIb9BSC.js → command-fO0BFsTq.js} +102 -37
  32. package/dist/command-fO0BFsTq.js.map +1 -0
  33. package/dist/{command-CsfcwMyc.js → command-qGS802uX.js} +16 -14
  34. package/dist/command-qGS802uX.js.map +1 -0
  35. package/dist/{commands-C54S45d7.js → commands-CC60TOT5.js} +64 -18
  36. package/dist/commands-CC60TOT5.js.map +1 -0
  37. package/dist/{commands-jO9Jj35r.js → commands-xYTMNTPv.js} +1 -1
  38. package/dist/{config-ImY6AlRE.js → config-C2UtS9vy.js} +718 -76
  39. package/dist/config-C2UtS9vy.js.map +1 -0
  40. package/dist/{contrasts-DtSX8E_V.js → contrasts-BjOg7wyr.js} +3 -3
  41. package/dist/{contrasts-DtSX8E_V.js.map → contrasts-BjOg7wyr.js.map} +1 -1
  42. package/dist/cost-BJU0MAji.js +1 -0
  43. package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
  44. package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
  45. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BTK_apL-.js} +2 -2
  46. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
  47. package/dist/{db-sHGpaLo9.js → db-C5SZ0510.js} +72 -14
  48. package/dist/db-C5SZ0510.js.map +1 -0
  49. package/dist/db-puzLzX4k.js +2 -0
  50. package/dist/{default-effort-BZRa-SQP.js → default-effort-BVmx7r7L.js} +8 -5
  51. package/dist/default-effort-BVmx7r7L.js.map +1 -0
  52. package/dist/{default-effort-fpdbYVKm.js → default-effort-FEidNAIi.js} +1 -1
  53. package/dist/{docker-CWuSRj7a.js → docker-B2F8-PGC.js} +4 -4
  54. package/dist/{docker-CWuSRj7a.js.map → docker-B2F8-PGC.js.map} +1 -1
  55. package/dist/{docker-CIzHzwlC.js → docker-BlSopHpZ.js} +1 -1
  56. package/dist/{docker--2ytOHcu.js → docker-Dto8ROCq.js} +5 -5
  57. package/dist/{docker--2ytOHcu.js.map → docker-Dto8ROCq.js.map} +1 -1
  58. package/dist/{doctor-eE65YpEv.js → doctor-BcvfKpBX.js} +20 -20
  59. package/dist/{doctor-eE65YpEv.js.map → doctor-BcvfKpBX.js.map} +1 -1
  60. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  61. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  62. package/dist/{environment-Cy3FVLR5.js → environment-CC1RV21n.js} +3 -3
  63. package/dist/{environment-Cy3FVLR5.js.map → environment-CC1RV21n.js.map} +1 -1
  64. package/dist/{eval-source-BTsTRiBW.js → eval-source-CzxdgFBW.js} +18 -10
  65. package/dist/eval-source-CzxdgFBW.js.map +1 -0
  66. package/dist/evaluate-DToSB2bt.d.ts +101 -0
  67. package/dist/{evaluate-CFFdyYZX.js → evaluate-NTNJ8T9H.js} +129 -51
  68. package/dist/evaluate-NTNJ8T9H.js.map +1 -0
  69. package/dist/{files-tnMgu7-X.js → files-CZXrkPja.js} +5 -5
  70. package/dist/{files-tnMgu7-X.js.map → files-CZXrkPja.js.map} +1 -1
  71. package/dist/files-DFtuRQJx.js +1 -0
  72. package/dist/{files-CQU08uh4.js → files-DsXULUi5.js} +3 -3
  73. package/dist/{files-CQU08uh4.js.map → files-DsXULUi5.js.map} +1 -1
  74. package/dist/{files-CIX-pLl4.js → files-aAzoGNCB.js} +4 -4
  75. package/dist/{files-CIX-pLl4.js.map → files-aAzoGNCB.js.map} +1 -1
  76. package/dist/files-oOYNR7ZH.js +1 -0
  77. package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
  78. package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
  79. package/dist/gateway.d.ts +1 -1
  80. package/dist/{git-BWVkG4Yl.js → git-C1MB5bjr.js} +3 -3
  81. package/dist/{git-BWVkG4Yl.js.map → git-C1MB5bjr.js.map} +1 -1
  82. package/dist/harness-DGsRZDSo.js +105 -0
  83. package/dist/harness-DGsRZDSo.js.map +1 -0
  84. package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
  85. package/dist/harness-DYSC4zMh.js.map +1 -0
  86. package/dist/harness.d.ts +1 -1
  87. package/dist/harness.js +1 -1
  88. package/dist/{images-BVZHDf_z.js → images-CNayXCp2.js} +8 -8
  89. package/dist/{images-BVZHDf_z.js.map → images-CNayXCp2.js.map} +1 -1
  90. package/dist/index.d.ts +3 -3
  91. package/dist/index.js +5 -5
  92. package/dist/init-8o6W3_fn.js +1 -0
  93. package/dist/{init-BWnMbUnc.js → init-D5HtmyMI.js} +5 -5
  94. package/dist/{init-BWnMbUnc.js.map → init-D5HtmyMI.js.map} +1 -1
  95. package/dist/ink-list-Ca-5dunO.js +184 -0
  96. package/dist/ink-list-Ca-5dunO.js.map +1 -0
  97. package/dist/{invocations-BGRs1r_C.js → invocations-DrGXF9dT.js} +3 -3
  98. package/dist/{invocations-BGRs1r_C.js.map → invocations-DrGXF9dT.js.map} +1 -1
  99. package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
  100. package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
  101. package/dist/judge.d.ts +1 -1
  102. package/dist/judge.js +1 -1
  103. package/dist/judgments-m-mb0zv8.js +246 -0
  104. package/dist/judgments-m-mb0zv8.js.map +1 -0
  105. package/dist/list-CM_YSJXq.js +374 -0
  106. package/dist/list-CM_YSJXq.js.map +1 -0
  107. package/dist/{live-d_5a5iUv.js → live-B71o2TkX.js} +21 -3
  108. package/dist/live-B71o2TkX.js.map +1 -0
  109. package/dist/live-DQc8jr6o.js +1 -0
  110. package/dist/{live-ColieOai.js → live-uRURFDMZ.js} +7 -7
  111. package/dist/{live-ColieOai.js.map → live-uRURFDMZ.js.map} +1 -1
  112. package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
  113. package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
  114. package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
  115. package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
  116. package/dist/{model-DBpdRXtO.js → model-BxBWFOZn.js} +43 -19
  117. package/dist/{model-DBpdRXtO.js.map → model-BxBWFOZn.js.map} +1 -1
  118. package/dist/{model-DOXnWH65.js → model-CXVVB4RK.js} +1 -1
  119. package/dist/{model-DOiCiMCD.js → model-Ce9l9_rF.js} +19 -255
  120. package/dist/model-Ce9l9_rF.js.map +1 -0
  121. package/dist/model-Cxzo4cjw.js +1 -0
  122. package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
  123. package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
  124. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
  125. package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
  126. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
  127. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
  128. package/dist/{patch-DvLr8wCZ.js → patch-ug5HpMOj.js} +3 -3
  129. package/dist/{patch-DvLr8wCZ.js.map → patch-ug5HpMOj.js.map} +1 -1
  130. package/dist/{plan-CIfOFhth.js → plan-DqnzfDH1.js} +15 -14
  131. package/dist/plan-DqnzfDH1.js.map +1 -0
  132. package/dist/{plugins-BEs2Am3L.js → plugins-B2KQ0y3P.js} +4 -4
  133. package/dist/{plugins-BEs2Am3L.js.map → plugins-B2KQ0y3P.js.map} +1 -1
  134. package/dist/{preflight-BAIRsTpl.js → preflight-DVXEvzFU.js} +1 -1
  135. package/dist/{preflight-50Qw5e5K.js → preflight-DyzmvK7i.js} +3 -3
  136. package/dist/{preflight-50Qw5e5K.js.map → preflight-DyzmvK7i.js.map} +1 -1
  137. package/dist/{prepare-8XhzhLHX.js → prepare-21WMQZyt.js} +7 -7
  138. package/dist/{prepare-8XhzhLHX.js.map → prepare-21WMQZyt.js.map} +1 -1
  139. package/dist/{prepare-8nvfDFQx.js → prepare-BcSRa7-3.js} +2 -2
  140. package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
  141. package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
  142. package/dist/{provenance-Cc8fFE7Z.js → provenance-iRxjp4Pu.js} +4 -4
  143. package/dist/{provenance-Cc8fFE7Z.js.map → provenance-iRxjp4Pu.js.map} +1 -1
  144. package/dist/{providers-CP_bBtYm.js → providers-D0ob_tra.js} +1 -1
  145. package/dist/{providers-DZSTlC9R.js → providers-vXMXuwn-.js} +5 -5
  146. package/dist/{providers-DZSTlC9R.js.map → providers-vXMXuwn-.js.map} +1 -1
  147. package/dist/{query-DXpxaaTH.js → query-BhaKQ07P.js} +3 -3
  148. package/dist/{query-DXpxaaTH.js.map → query-BhaKQ07P.js.map} +1 -1
  149. package/dist/query-CvgUsvep.js +1 -0
  150. package/dist/{questions-C_Y-oXW6.js → questions-BzdLdV3C.js} +39 -9
  151. package/dist/questions-BzdLdV3C.js.map +1 -0
  152. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  153. package/dist/redact-BhjtnjrU.js.map +1 -0
  154. package/dist/remarks-CAvTSq16.js +1 -0
  155. package/dist/{remarks-E4rpi8Ym.js → remarks-CauU408t.js} +231 -25
  156. package/dist/remarks-CauU408t.js.map +1 -0
  157. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  158. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  159. package/dist/{remarks-DeZZT8PK.js → remarks-NMgPd30U.js} +4 -3
  160. package/dist/{remarks-DeZZT8PK.js.map → remarks-NMgPd30U.js.map} +1 -1
  161. package/dist/{remote-DZ00A8V_.js → remote-Dwy3fyo3.js} +5 -5
  162. package/dist/{remote-DZ00A8V_.js.map → remote-Dwy3fyo3.js.map} +1 -1
  163. package/dist/{resolve-B_JlQ0La.js → resolve-B1zMnPwK.js} +7 -7
  164. package/dist/{resolve-B_JlQ0La.js.map → resolve-B1zMnPwK.js.map} +1 -1
  165. package/dist/resolve-BIIeVT12.js +1 -0
  166. package/dist/{results-BMJHaQbi.js → results-CfC0id_u.js} +7 -7
  167. package/dist/{results-BMJHaQbi.js.map → results-CfC0id_u.js.map} +1 -1
  168. package/dist/{resume-CexsWuGK.js → resume-Dkhrpjow.js} +123 -8
  169. package/dist/resume-Dkhrpjow.js.map +1 -0
  170. package/dist/{runtime-CSZ1hjlN.js → runtime-DcvQcr36.js} +2 -2
  171. package/dist/{runtime-CSZ1hjlN.js.map → runtime-DcvQcr36.js.map} +1 -1
  172. package/dist/{sandbox-B5W8lLL1.js → sandbox-8zCrtuCf.js} +2 -2
  173. package/dist/{sandbox-B5W8lLL1.js.map → sandbox-8zCrtuCf.js.map} +1 -1
  174. package/dist/sandbox.d.ts +1 -1
  175. package/dist/sandbox.js +1 -1
  176. package/dist/server-0tuj8VSa.js +1 -0
  177. package/dist/{server-BfN6q2lH.js → server-Dplq8g0r.js} +7 -7
  178. package/dist/{server-BfN6q2lH.js.map → server-Dplq8g0r.js.map} +1 -1
  179. package/dist/{source-BD6Bg0Tr.js → source-NWHqn5-o.js} +2 -2
  180. package/dist/{source-BD6Bg0Tr.js.map → source-NWHqn5-o.js.map} +1 -1
  181. package/dist/source-llwLewOU.js +1 -0
  182. package/dist/{src-BC2WXULN.js → src-B9oE_mNS.js} +1 -1
  183. package/dist/{src-Cpy9MNS3.js → src-BQ6BiXoc.js} +19 -11
  184. package/dist/{src-Cpy9MNS3.js.map → src-BQ6BiXoc.js.map} +1 -1
  185. package/dist/{src-Dp-fWOfG.js → src-BQXdf-Gw.js} +2 -2
  186. package/dist/{src-Dp-fWOfG.js.map → src-BQXdf-Gw.js.map} +1 -1
  187. package/dist/{src-DurP--_D.js → src-BXnv0ViU.js} +1 -1
  188. package/dist/src-BiS8YNN_.js +3 -0
  189. package/dist/{src-DVaAKCsH.js → src-BkH4GOxu.js} +1 -1
  190. package/dist/{src-CM7xdJFA.js → src-C-tH5ebd.js} +1 -1
  191. package/dist/{src-Cx0HJIJ-.js → src-CUpoVVb8.js} +1 -1
  192. package/dist/{src-C9elednc.js → src-Cg1bFs9g.js} +1 -1
  193. package/dist/{src-BKDZO3ra.js → src-DV_UorZv.js} +1 -1
  194. package/dist/{src-CWEGknZo.js → src-XxUpr2sa.js} +1 -1
  195. package/dist/{state-CdSjXIfE.js → state-DDCGUhva.js} +5 -5
  196. package/dist/{state-CdSjXIfE.js.map → state-DDCGUhva.js.map} +1 -1
  197. package/dist/state-DZbAO95a.js +1 -0
  198. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  199. package/dist/stats-C18W4xPX.js.map +1 -0
  200. package/dist/suite-M8_7xio7.js +1294 -0
  201. package/dist/suite-M8_7xio7.js.map +1 -0
  202. package/dist/suite-u2KJnGpY.js +2 -0
  203. package/dist/{summary-zP3cCXDN.js → summary-Uzc9osfV.js} +47 -13
  204. package/dist/summary-Uzc9osfV.js.map +1 -0
  205. package/dist/testing.d.ts +47 -37
  206. package/dist/testing.js +4 -4
  207. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  208. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  209. package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
  210. package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
  211. package/dist/{types-BHR1lGPh.d.ts → types-DGvexkcp.d.ts} +71 -63
  212. package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
  213. package/dist/vertex-BLQMayc1.js.map +1 -0
  214. package/dist/vertex-h_peaZTN.js +1 -0
  215. package/dist/view/index.d.ts +140 -35
  216. package/dist/view/index.js +8 -7
  217. package/dist/{views-CKoN5NAx.js → views-BISrsIjw.js} +6 -6
  218. package/dist/{views-CKoN5NAx.js.map → views-BISrsIjw.js.map} +1 -1
  219. package/dist/views-Bfab8fM9.js +2 -0
  220. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-DJD_nHkS.js} +5 -5
  221. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
  222. package/dist/workspace-B8EOCuxl.js +1 -0
  223. package/dist/{workspace-Cho30qlM.js → workspace-BO1VhUR9.js} +5 -5
  224. package/dist/{workspace-Cho30qlM.js.map → workspace-BO1VhUR9.js.map} +1 -1
  225. package/package.json +3 -1
  226. package/skills/eval-rubric/SKILL.md +22 -3
  227. package/dist/app-files-hpRT5lj_.js.map +0 -1
  228. package/dist/assemble-DTXwCL3m.js +0 -1
  229. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  230. package/dist/capture-BT2fAmpZ.js +0 -1
  231. package/dist/capture-Bdcx9Gap.js.map +0 -1
  232. package/dist/cli-CGd51thd.js.map +0 -1
  233. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  234. package/dist/client-CF-DOjOU.js.map +0 -1
  235. package/dist/command-CsfcwMyc.js.map +0 -1
  236. package/dist/command-CyIb9BSC.js.map +0 -1
  237. package/dist/commands-C54S45d7.js.map +0 -1
  238. package/dist/config-ImY6AlRE.js.map +0 -1
  239. package/dist/cost-CMzY9VXv.js +0 -1
  240. package/dist/db-ZIk6S2CD.js +0 -2
  241. package/dist/db-sHGpaLo9.js.map +0 -1
  242. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  243. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  244. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  245. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  246. package/dist/files-B3d-8Fe0.js +0 -1
  247. package/dist/files-Do4Rp3yZ.js +0 -1
  248. package/dist/harness-Ch2P81sx.js.map +0 -1
  249. package/dist/init-s1jN-PCI.js +0 -1
  250. package/dist/live-DmJix4c8.js +0 -1
  251. package/dist/live-d_5a5iUv.js.map +0 -1
  252. package/dist/model-8uZBYqV5.js +0 -1
  253. package/dist/model-DOiCiMCD.js.map +0 -1
  254. package/dist/plan-CIfOFhth.js.map +0 -1
  255. package/dist/query-kFPuqBv1.js +0 -1
  256. package/dist/questions-C_Y-oXW6.js.map +0 -1
  257. package/dist/redact-BEEGc37e.js.map +0 -1
  258. package/dist/remarks-0EXMh6AD.js.map +0 -1
  259. package/dist/remarks-BL18Id09.js +0 -1
  260. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  261. package/dist/resolve-BJSay360.js +0 -1
  262. package/dist/resume-CexsWuGK.js.map +0 -1
  263. package/dist/server-SBzrf35T.js +0 -1
  264. package/dist/source-Dn1uqPon.js +0 -1
  265. package/dist/src-BxrHipV0.js +0 -3
  266. package/dist/state-Cicc9Oa5.js +0 -1
  267. package/dist/stats-D-Gqwqt0.js.map +0 -1
  268. package/dist/suite-BYJ2Q0S-.js +0 -2
  269. package/dist/suite-BtHPBFYL.js +0 -544
  270. package/dist/suite-BtHPBFYL.js.map +0 -1
  271. package/dist/summary-zP3cCXDN.js.map +0 -1
  272. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  273. package/dist/vertex-BaAz44YN.js +0 -1
  274. package/dist/views-CmyX255j.js +0 -2
  275. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -1,23 +1,24 @@
1
1
  import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
2
2
  import { n as UsageError } from "./errors-ttMTnIYX.js";
3
- import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-ogqPJ0uo.js";
4
- import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-Cy3FVLR5.js";
5
- import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-BEs2Am3L.js";
6
- import { _ as GATEWAY_NAMES, b as gatewayEntryDataSchema, f as FACTORY, g as harnessProblem, h as harnessDescriptorSchema, u as opencodeModelError, v as directOffPin, x as gatewayNameSchema, y as gatewayDescriptorSchema } from "./harness-Ch2P81sx.js";
3
+ import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty$1, m as TIMER_LIMIT, n as dataProblem, o as genericProblem, p as MAX_TIMER_MS, s as got, t as closest, u as oneOf } from "./shared-ogqPJ0uo.js";
4
+ import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-CC1RV21n.js";
5
+ import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-B2KQ0y3P.js";
6
+ import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DYSC4zMh.js";
7
7
  import { a as fromCwd, l as sha256 } from "./ids-DuK6xl4w.js";
8
- import { a as providerEndpoints, c as providerServedModel, n as openCodeEffortBody, o as providerHosts, r as openCodeEffortRefusal, s as providerModelResponses, u as resolveOpenCodeProvider } from "./opencode-providers-yNKyXD49.js";
9
- import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-ByEUlzOM.js";
10
- import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-6uSpZ5L8.js";
11
- import { r as badCharacterAt } from "./redact-BEEGc37e.js";
12
- import { s as withoutBom } from "./text-D--Q5KsA.js";
13
- import { t as inCI } from "./env-T86lAqUD.js";
8
+ import { a as providerEndpoints, c as providerServedModel, n as openCodeEffortBody, o as providerHosts, r as openCodeEffortRefusal, s as providerModelResponses, u as resolveOpenCodeProvider } from "./opencode-providers-Co7plCea.js";
9
+ import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-DYOF69Oe.js";
10
+ import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-BLQMayc1.js";
11
+ import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
12
+ import { o as withoutBom } from "./text-B8-W3Cvn.js";
13
+ import { t as inCI } from "./env-8x1O8I1M.js";
14
14
  import { u as ownFileURL } from "./root-Co0jhOT5.js";
15
- import { F as HARNESS_HOSTS, I as PACKAGE_HOSTS, P as DOCKER_EGRESS, mt as shellQuote, pt as run } from "./src-Cpy9MNS3.js";
15
+ import { F as HARNESS_HOSTS, I as PACKAGE_HOSTS, P as DOCKER_EGRESS, mt as shellQuote, pt as run } from "./src-BQ6BiXoc.js";
16
16
  import { t as InfraError } from "./types-C4OkYhZg.js";
17
- import { n as gateLabel, u as pairFactors } from "./gate-BoDKeuJu.js";
18
- import "./stats-D-Gqwqt0.js";
19
- import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-DyvDj4ZN.js";
20
- import { t as docker } from "./sandbox-B5W8lLL1.js";
17
+ import { n as gateLabel, u as pairFactors } from "./gate-Cp_pQoEe.js";
18
+ import { u as opencodeModelError } from "./harness-DGsRZDSo.js";
19
+ import "./stats-C18W4xPX.js";
20
+ import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CptK032x.js";
21
+ import { t as docker } from "./sandbox-8zCrtuCf.js";
21
22
  import { existsSync, readFileSync } from "node:fs";
22
23
  import { availableParallelism, tmpdir, totalmem } from "node:os";
23
24
  import { dirname, join, resolve, win32 } from "node:path";
@@ -88,20 +89,22 @@ const sharedInputFields = {
88
89
  task: z.boolean().optional(),
89
90
  output: z.boolean().optional(),
90
91
  files: z.boolean().optional(),
91
- diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
92
+ diff: z.union([z.boolean(), z.array(nonEmpty$1()).min(1)]).optional(),
92
93
  transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
93
- checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
94
+ checks: z.union([z.boolean(), z.array(nonEmpty$1())]).optional(),
95
+ observations: z.union([z.boolean(), z.array(nonEmpty$1())]).optional()
94
96
  };
95
97
  const turnsSchema = z.enum(["last", "all"]);
96
- const { task, output, files, diff, transcript, checks } = sharedInputFields;
98
+ const { task, output, files, diff, transcript, checks, observations } = sharedInputFields;
97
99
  const judgeInputsSchema = z.strictObject({
98
100
  task,
99
101
  output,
100
102
  files,
101
103
  diff,
102
- contents: z.array(nonEmpty()).optional(),
104
+ contents: z.array(nonEmpty$1()).optional(),
103
105
  transcript,
104
106
  checks,
107
+ observations,
105
108
  artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
106
109
  });
107
110
  const judgeSubjectSchema = z.strictObject({
@@ -149,7 +152,8 @@ const VALUES$1 = {
149
152
  contents: "a list of paths or patterns",
150
153
  transcript: "true, false, 'tools' or 'full'",
151
154
  turns: "'last' or 'all'",
152
- checks: "true, false or a list of commands"
155
+ checks: "true, false or a list of commands",
156
+ observations: "true, false or a list of names"
153
157
  };
154
158
  function distance(a, b) {
155
159
  const row = Array.from({ length: b.length + 1 }, (_, i) => i);
@@ -239,7 +243,7 @@ function judgeProblem(judge, site) {
239
243
  const pairFactorSchema = z.enum(["environment", "harness"]);
240
244
  /** Evaluation models score yes/no questions only, so they can't answer a question. */
241
245
  const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
242
- const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
246
+ const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty$1()).min(1)]);
243
247
  const questionInputFields = {
244
248
  ...sharedInputFields,
245
249
  artifacts: questionArtifactsSchema.optional()
@@ -257,9 +261,9 @@ const unique = (list, ctx) => {
257
261
  });
258
262
  };
259
263
  const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
260
- const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
264
+ const choicesSchema = z.array(nonEmpty$1()).min(2).superRefine(unique);
261
265
  const questionSchema = z.strictObject({
262
- question: nonEmpty(),
266
+ question: nonEmpty$1(),
263
267
  model: questionModelSchema.optional(),
264
268
  across: acrossSchema.optional(),
265
269
  choices: choicesSchema.optional(),
@@ -296,7 +300,8 @@ const VALUES = {
296
300
  diff: "true, false or a non-empty list of patterns",
297
301
  transcript: "true, false, 'tools' or 'full'",
298
302
  turns: "'last' or 'all'",
299
- checks: "true, false or a list of commands"
303
+ checks: "true, false or a list of commands",
304
+ observations: "true, false or a list of names"
300
305
  };
301
306
  const describe$2 = (value) => {
302
307
  if (Array.isArray(value)) return `a list of ${value.length}`;
@@ -344,8 +349,8 @@ function acrossProblem(question, across, where) {
344
349
  function choicesProblem(question, choices, where) {
345
350
  if (choicesSchema.safeParse(choices).success) return void 0;
346
351
  const at = `questions "${question}" choices in ${where}`;
347
- if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
348
- const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty().safeParse(c).success);
352
+ if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty$1().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
353
+ const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty$1().safeParse(c).success);
349
354
  if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
350
355
  return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
351
356
  }
@@ -359,7 +364,7 @@ function questionProblem(item, where) {
359
364
  return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
360
365
  }
361
366
  const { question, model, across, choices } = item;
362
- if (!nonEmpty().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
367
+ if (!nonEmpty$1().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
363
368
  const text = question;
364
369
  if (across !== void 0) {
365
370
  const problem = acrossProblem(text, across, where);
@@ -1137,7 +1142,7 @@ const AGENTS = {
1137
1142
  },
1138
1143
  unreported: cursorUnreported,
1139
1144
  mcpServers: cursorMcpServers,
1140
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-DurP--_D.js")).createCursorSdk({
1145
+ adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-BXnv0ViU.js")).createCursorSdk({
1141
1146
  auth,
1142
1147
  ...mcpServers ? { mcpServers } : {},
1143
1148
  ...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
@@ -1209,7 +1214,7 @@ const AGENTS = {
1209
1214
  },
1210
1215
  servedModel: geminiServedModel,
1211
1216
  mcpServers: antigravityMcpServers,
1212
- adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-CWEGknZo.js")).createAntigravity({
1217
+ adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-XxUpr2sa.js")).createAntigravity({
1213
1218
  auth,
1214
1219
  ...vertex ? { vertex } : {},
1215
1220
  ...mcpServers ? { mcpServers } : {},
@@ -1239,7 +1244,7 @@ const AGENTS = {
1239
1244
  servedModel: geminiServedModel,
1240
1245
  forModel: opencodeForModel,
1241
1246
  mcpServers: opencodeMcpServers,
1242
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BxrHipV0.js")).createOpenCode({
1247
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BiS8YNN_.js")).createOpenCode({
1243
1248
  auth,
1244
1249
  model,
1245
1250
  ...vertex ? { vertex } : { vars: opencodeVars(model) },
@@ -1264,14 +1269,14 @@ const AGENTS = {
1264
1269
  },
1265
1270
  servedModel: geminiServedModel,
1266
1271
  mcpServers: gooseMcpServers,
1267
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-CM7xdJFA.js")).createGoose({
1272
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-C-tH5ebd.js")).createGoose({
1268
1273
  ...vertex ? { vertex } : {},
1269
1274
  ...auth["AI_GATEWAY_API_KEY"] ? {
1270
1275
  auth,
1271
- provider: (await import("./src-CM7xdJFA.js")).AI_GATEWAY_PROVIDER
1276
+ provider: (await import("./src-C-tH5ebd.js")).AI_GATEWAY_PROVIDER
1272
1277
  } : auth["OPENROUTER_API_KEY"] ? {
1273
1278
  auth,
1274
- provider: (await import("./src-CM7xdJFA.js")).OPENROUTER_PROVIDER
1279
+ provider: (await import("./src-C-tH5ebd.js")).OPENROUTER_PROVIDER
1275
1280
  } : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
1276
1281
  model,
1277
1282
  ...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
@@ -1294,13 +1299,13 @@ const AGENTS = {
1294
1299
  endpoints: GEMINI_ENDPOINTS,
1295
1300
  servedModel: geminiServedModel,
1296
1301
  mcpServers: piMcpServers,
1297
- adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-BC2WXULN.js")).createPi({
1302
+ adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-B9oE_mNS.js")).createPi({
1298
1303
  ...auth["AI_GATEWAY_API_KEY"] ? {
1299
1304
  auth,
1300
- gateway: (await import("./src-BC2WXULN.js")).PI_AI_GATEWAY
1305
+ gateway: (await import("./src-B9oE_mNS.js")).PI_AI_GATEWAY
1301
1306
  } : auth["OPENROUTER_API_KEY"] ? {
1302
1307
  auth,
1303
- gateway: (await import("./src-BC2WXULN.js")).PI_OPENROUTER
1308
+ gateway: (await import("./src-B9oE_mNS.js")).PI_OPENROUTER
1304
1309
  } : {
1305
1310
  auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
1306
1311
  ...vertex ? { vertex } : {}
@@ -1335,9 +1340,9 @@ const AGENTS = {
1335
1340
  },
1336
1341
  servedModel: openaiServedModel,
1337
1342
  mcpServers: codexMcpServers,
1338
- adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-C9elednc.js")).createCodex({
1343
+ adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-Cg1bFs9g.js")).createCodex({
1339
1344
  auth,
1340
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-C9elednc.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-C9elednc.js")).OPENROUTER_PROVIDER } : {},
1345
+ ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).OPENROUTER_PROVIDER } : {},
1341
1346
  ...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
1342
1347
  ...mcpServers ? { mcpServers } : {},
1343
1348
  ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
@@ -1358,7 +1363,7 @@ const AGENTS = {
1358
1363
  ...fxForModel("provider/model"),
1359
1364
  forModel: fxForModel,
1360
1365
  mcpServers: acpMcpServers,
1361
- adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-BKDZO3ra.js")).createFx({
1366
+ adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-DV_UorZv.js")).createFx({
1362
1367
  auth,
1363
1368
  model,
1364
1369
  ...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
@@ -1386,7 +1391,7 @@ const AGENTS = {
1386
1391
  servedModel: xaiServedModel,
1387
1392
  mcpServers: acpMcpServers,
1388
1393
  adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
1389
- const grok = await import("./src-Cx0HJIJ-.js");
1394
+ const grok = await import("./src-CUpoVVb8.js");
1390
1395
  return grok.createGrokBuild({
1391
1396
  ...vertexChat ? {
1392
1397
  auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
@@ -1742,6 +1747,644 @@ function resolvedEffortOf(models) {
1742
1747
  }
1743
1748
  return out;
1744
1749
  }
1750
+ // src/contract/attempt.ts
1751
+ const nullableNumber = z.number().nullable();
1752
+ const tokenUsageSchema = z.strictObject({
1753
+ input: z.number(),
1754
+ output: z.number(),
1755
+ cacheRead: z.number(),
1756
+ cacheWrite: z.number(),
1757
+ reasoning: nullableNumber
1758
+ });
1759
+ const fileChangeSchema = z.strictObject({
1760
+ path: z.string(),
1761
+ status: z.enum([
1762
+ "added",
1763
+ "modified",
1764
+ "deleted"
1765
+ ])
1766
+ });
1767
+ const diffSchema = z.strictObject({
1768
+ patch: z.string(),
1769
+ files: z.array(fileChangeSchema)
1770
+ });
1771
+ const turnStatusSchema = z.enum([
1772
+ "completed",
1773
+ "aborted",
1774
+ "error"
1775
+ ]);
1776
+ const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
1777
+ const at$1 = {
1778
+ ts: z.string(),
1779
+ atMs: z.number()
1780
+ };
1781
+ const transcriptEventSchema = z.discriminatedUnion("type", [
1782
+ z.strictObject({
1783
+ ...at$1,
1784
+ type: z.enum(["text", "reasoning"]),
1785
+ text: z.string(),
1786
+ durationMs: nullableNumber
1787
+ }),
1788
+ z.strictObject({
1789
+ ...at$1,
1790
+ type: z.literal("file-edit"),
1791
+ toolCallId: z.string(),
1792
+ path: z.string(),
1793
+ kind: z.enum([
1794
+ "create",
1795
+ "modify",
1796
+ "write",
1797
+ "delete"
1798
+ ]),
1799
+ added: nullableNumber,
1800
+ removed: nullableNumber,
1801
+ durationMs: nullableNumber,
1802
+ stalled: z.boolean().optional(),
1803
+ subagent: z.string().optional()
1804
+ }),
1805
+ z.strictObject({
1806
+ ...at$1,
1807
+ type: z.literal("web-search"),
1808
+ toolCallId: z.string(),
1809
+ query: z.string(),
1810
+ references: nullableNumber,
1811
+ ok: z.boolean().nullable().optional(),
1812
+ durationMs: nullableNumber,
1813
+ stalled: z.boolean().optional(),
1814
+ subagent: z.string().optional()
1815
+ }),
1816
+ z.strictObject({
1817
+ ...at$1,
1818
+ type: z.literal("web-fetch"),
1819
+ toolCallId: z.string(),
1820
+ url: z.string(),
1821
+ ok: z.boolean().nullable(),
1822
+ durationMs: nullableNumber,
1823
+ stalled: z.boolean().optional(),
1824
+ subagent: z.string().optional()
1825
+ }),
1826
+ z.strictObject({
1827
+ ...at$1,
1828
+ type: z.literal("tool"),
1829
+ toolCallId: z.string(),
1830
+ tool: z.string(),
1831
+ input: z.unknown(),
1832
+ output: z.unknown(),
1833
+ ok: z.boolean().nullable(),
1834
+ durationMs: nullableNumber,
1835
+ stalled: z.boolean().optional(),
1836
+ subagent: z.string().optional()
1837
+ }),
1838
+ z.strictObject({
1839
+ ...at$1,
1840
+ type: z.literal("error"),
1841
+ message: z.string()
1842
+ }),
1843
+ z.strictObject({
1844
+ ...at$1,
1845
+ type: z.literal("retry"),
1846
+ message: z.string()
1847
+ })
1848
+ ]);
1849
+ const turnDataSchema = z.strictObject({
1850
+ prompt: z.string(),
1851
+ text: z.string(),
1852
+ status: turnStatusSchema,
1853
+ startedAt: z.string(),
1854
+ durationMs: z.number(),
1855
+ transcript: z.array(transcriptEventSchema),
1856
+ usage: tokenUsageSchema.nullable(),
1857
+ model: z.string().nullable(),
1858
+ unreported: z.array(webEventTypeSchema),
1859
+ diff: diffSchema
1860
+ });
1861
+ const execResultSchema = z.strictObject({
1862
+ exitCode: z.number().int(),
1863
+ stdout: z.string(),
1864
+ stderr: z.string()
1865
+ });
1866
+ const execRetrySchema = z.strictObject({
1867
+ retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
1868
+ interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
1869
+ });
1870
+ /** A background command has no exit for `retries` to wait on. */
1871
+ const noBackgroundRetry = (options, ctx) => {
1872
+ if (!options.background) return;
1873
+ for (const key of ["retries", "interval"]) {
1874
+ if (options[key] === void 0) continue;
1875
+ ctx.addIssue({
1876
+ code: "custom",
1877
+ path: [key],
1878
+ message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
1879
+ });
1880
+ }
1881
+ };
1882
+ const execOptionsSchema = z.strictObject({
1883
+ background: z.boolean().optional(),
1884
+ ...execRetrySchema.shape
1885
+ }).superRefine(noBackgroundRetry);
1886
+ /** What's wrong with `c.exec()`'s options, or undefined. */
1887
+ const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
1888
+ const assertionRecordSchema = z.strictObject({
1889
+ label: z.string(),
1890
+ labelled: z.boolean(),
1891
+ matcher: z.string(),
1892
+ status: z.enum([
1893
+ "pass",
1894
+ "fail",
1895
+ "error"
1896
+ ]),
1897
+ soft: z.boolean(),
1898
+ at: z.string().optional(),
1899
+ message: z.string().optional(),
1900
+ durationMs: z.number().optional()
1901
+ });
1902
+ const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
1903
+ const captureConsoleMessageSchema = z.strictObject({
1904
+ level: z.enum([
1905
+ "error",
1906
+ "warning",
1907
+ "info",
1908
+ "log",
1909
+ "debug"
1910
+ ]),
1911
+ text: z.string(),
1912
+ source: z.string(),
1913
+ url: z.string().optional()
1914
+ });
1915
+ const captureFailedRequestSchema = z.strictObject({
1916
+ method: z.string(),
1917
+ url: z.string(),
1918
+ status: z.number().optional(),
1919
+ error: z.string().optional()
1920
+ });
1921
+ const capturePageFields = {
1922
+ status: z.number().optional(),
1923
+ finalUrl: z.string(),
1924
+ console: z.array(captureConsoleMessageSchema),
1925
+ errors: z.array(z.string()),
1926
+ failedRequests: z.array(captureFailedRequestSchema),
1927
+ dropped: z.number().optional()
1928
+ };
1929
+ z.strictObject(capturePageFields);
1930
+ const captureSchema = z.strictObject({
1931
+ name: z.string(),
1932
+ port: z.number(),
1933
+ path: z.string(),
1934
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]),
1935
+ file: z.strictObject({
1936
+ path: z.string(),
1937
+ sha256: z.string(),
1938
+ bytes: z.number()
1939
+ }),
1940
+ ...capturePageFields
1941
+ });
1942
+ const judgeArtifactRefSchema = z.strictObject({
1943
+ name: z.string(),
1944
+ path: z.string(),
1945
+ sha256: z.string(),
1946
+ bytes: z.number(),
1947
+ mediaType: z.literal("image/png")
1948
+ });
1949
+ const verdictFields = {
1950
+ threshold: z.number(),
1951
+ question: z.string(),
1952
+ model: z.string(),
1953
+ gateway: gatewayNameSchema.optional(),
1954
+ artifacts: z.array(judgeArtifactRefSchema).optional()
1955
+ };
1956
+ const verdictSchema = z.strictObject({
1957
+ verdict: z.enum(["pass", "fail"]),
1958
+ score: z.number(),
1959
+ ...verdictFields,
1960
+ resolvedModel: z.string().optional(),
1961
+ upstream: z.string().optional(),
1962
+ costUsd: z.number().optional()
1963
+ });
1964
+ const judgeFailureSchema = z.strictObject({
1965
+ verdict: z.literal("error"),
1966
+ score: z.null(),
1967
+ ...verdictFields,
1968
+ error: z.string()
1969
+ });
1970
+ /** A pattern, for the matchers that take a string or a `RegExp`. */
1971
+ const patternSchema = z.strictObject({
1972
+ regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
1973
+ flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
1974
+ }).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
1975
+ /** A pattern in the runner's own language, which reeval never compiles. */
1976
+ function hasAnsweredPattern(value) {
1977
+ if (Array.isArray(value)) return value.some(hasAnsweredPattern);
1978
+ if (typeof value !== "object" || value === null) return false;
1979
+ return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
1980
+ }
1981
+ const answeredPatternSchema = z.strictObject({
1982
+ matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
1983
+ answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
1984
+ }).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
1985
+ const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
1986
+ const matcherArgSchema = z.union([
1987
+ patternSchema,
1988
+ answeredPatternSchema,
1989
+ jsonValueSchema
1990
+ ]).superRefine((value, ctx) => {
1991
+ if (!isPatternLike(value)) return;
1992
+ for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
1993
+ }).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
1994
+ const matcherSubjectSchema = z.union([
1995
+ z.strictObject({ turn: turnIndex }),
1996
+ z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
1997
+ z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
1998
+ z.strictObject({
1999
+ exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
2000
+ result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
2001
+ }),
2002
+ z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
2003
+ ]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
2004
+ const nonEmpty = z.string().regex(/\S/, "must not be empty");
2005
+ const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
2006
+ const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
2007
+ const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
2008
+ const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
2009
+ const captureWaitSchema = z.enum(["load", "idle"]);
2010
+ const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
2011
+ const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
2012
+ const captureCallDataSchema = z.strictObject({
2013
+ name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
2014
+ path: z.string().optional(),
2015
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
2016
+ timeout: durationSchema.optional(),
2017
+ wait: waitSchema.optional(),
2018
+ instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
2019
+ model: z.string().optional()
2020
+ }).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
2021
+ const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
2022
+ const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
2023
+ const judgeCallDataSchema = z.strictObject({
2024
+ model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
2025
+ ...judgeFields,
2026
+ ...sharedInputFields,
2027
+ contents: z.array(nonEmpty).optional(),
2028
+ turns: turnsSchema.optional(),
2029
+ on: judgeOnSchema.optional().describe("The turn `task`, `output`, `transcript` and `diff` read instead of the latest: an earlier turn, or a value graded as the output."),
2030
+ artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
2031
+ }).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, the inputs it shows the judge (at least one turned on) and `on`.");
2032
+ /** An observation's name: one a query, a tally line and a judge's input can carry as it is. */
2033
+ const OBSERVE_NAME = /^[A-Za-z0-9_.:/-]{1,64}$/;
2034
+ const callSiteSchema = z.strictObject({
2035
+ file: nonEmpty.describe("The eval file: absolute, or relative to the project root."),
2036
+ line: z.number().int().min(1).describe("From 1.")
2037
+ }).describe("Where the eval made the call, which the record names beside the check's outcome.");
2038
+ const attemptCallSchema = z.discriminatedUnion("op", [
2039
+ z.strictObject({
2040
+ op: z.literal("send"),
2041
+ prompt: z.string().describe("The turn's prompt.")
2042
+ }).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
2043
+ z.strictObject({
2044
+ op: z.literal("exec"),
2045
+ command: nonEmpty.describe("A shell command, run in the workspace."),
2046
+ background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
2047
+ ...execRetrySchema.shape
2048
+ }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
2049
+ z.strictObject({
2050
+ op: z.literal("serve"),
2051
+ command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
2052
+ port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
2053
+ }).describe("`c.serve(command, port?)`: a background process, once it listens, and its port."),
2054
+ z.strictObject({
2055
+ op: z.literal("kill"),
2056
+ handle: handleId
2057
+ }).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
2058
+ z.strictObject({
2059
+ op: z.literal("capture"),
2060
+ target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
2061
+ options: captureCallDataSchema.optional()
2062
+ }).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
2063
+ z.strictObject({
2064
+ op: z.literal("readFile"),
2065
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
2066
+ }).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
2067
+ z.strictObject({
2068
+ op: z.literal("writeFile"),
2069
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
2070
+ contents: z.string().describe("The file's contents, in `encoding`."),
2071
+ encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
2072
+ }).describe("`c.writeFile(path, contents)`, creating its directories."),
2073
+ z.strictObject({
2074
+ op: z.literal("observe"),
2075
+ name: z.string().regex(OBSERVE_NAME, "must be 1 to 64 letters, digits, _ . : / or -").describe("What the value is of, e.g. `installed`."),
2076
+ value: jsonValueSchema.describe("Any JSON, at most 4 KB once serialized.")
2077
+ }).describe("`c.observe(name, value)`: records a fact about the attempt that never affects its outcome. A name observed again keeps the last value. Fails after the body's `done`."),
2078
+ z.strictObject({
2079
+ op: z.literal("expect"),
2080
+ subject: matcherSubjectSchema,
2081
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
2082
+ args: z.array(matcherArgSchema).describe("The matcher's arguments."),
2083
+ not: z.boolean().describe("Called as `expect(…).not`."),
2084
+ soft: z.boolean().describe("Called as `expect`: a failure fails the attempt at its end and the body carries on. False for `expect.require`, whose failure the runner raises, ending the body."),
2085
+ label: z.string().optional().describe("`expect`'s second argument."),
2086
+ at: callSiteSchema.optional(),
2087
+ poll: z.strictObject({
2088
+ interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
2089
+ timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
2090
+ }).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
2091
+ }).superRefine((call, ctx) => {
2092
+ if (call.poll && "value" in call.subject) ctx.addIssue({
2093
+ code: "custom",
2094
+ path: ["subject"],
2095
+ message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
2096
+ });
2097
+ if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
2098
+ code: "custom",
2099
+ path: ["subject", "result"],
2100
+ message: "can't be polled: drop result so reeval runs the command on each try"
2101
+ });
2102
+ if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
2103
+ code: "custom",
2104
+ path: ["args"],
2105
+ message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
2106
+ });
2107
+ }).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
2108
+ z.strictObject({
2109
+ op: z.literal("record"),
2110
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
2111
+ subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
2112
+ label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
2113
+ status: z.enum(["pass", "fail"]),
2114
+ message: z.string().optional().describe("Why it failed."),
2115
+ soft: z.boolean().optional().describe("The body carried on after a failure, as after an `expect`. Default false: the check raised, as Python's `assert` does."),
2116
+ at: callSiteSchema.optional()
2117
+ }).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
2118
+ z.strictObject({
2119
+ op: z.literal("judge"),
2120
+ question: nonEmpty.describe("A yes/no question about the inputs `options` turns on."),
2121
+ options: judgeCallDataSchema
2122
+ }).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
2123
+ ]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
2124
+ const attemptErrorSchema = z.looseObject({
2125
+ name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
2126
+ message: z.string()
2127
+ }).describe("Why a call failed, in reeval's words.");
2128
+ const id = nonEmpty.describe("Pairs a reply with its request.");
2129
+ const attemptRequestSchema = z.strictObject({
2130
+ id,
2131
+ call: attemptCallSchema
2132
+ }).describe("A call, numbered so its reply can find it.");
2133
+ z.union([
2134
+ z.discriminatedUnion("op", [
2135
+ z.strictObject({
2136
+ id,
2137
+ ok: z.literal(true),
2138
+ op: z.literal("send"),
2139
+ turn: turnDataSchema
2140
+ }),
2141
+ z.strictObject({
2142
+ id,
2143
+ ok: z.literal(true),
2144
+ op: z.literal("exec"),
2145
+ result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
2146
+ }),
2147
+ z.strictObject({
2148
+ id,
2149
+ ok: z.literal(true),
2150
+ op: z.literal("serve"),
2151
+ handle: handleId,
2152
+ port: z.number().describe("The container port it listens on.")
2153
+ }),
2154
+ z.strictObject({
2155
+ id,
2156
+ ok: z.literal(true),
2157
+ op: z.literal("kill")
2158
+ }),
2159
+ z.strictObject({
2160
+ id,
2161
+ ok: z.literal(true),
2162
+ op: z.literal("capture"),
2163
+ capture: captureSchema
2164
+ }),
2165
+ z.strictObject({
2166
+ id,
2167
+ ok: z.literal(true),
2168
+ op: z.literal("readFile"),
2169
+ contents: z.string().nullable().describe("Null when there is no such file.")
2170
+ }),
2171
+ z.strictObject({
2172
+ id,
2173
+ ok: z.literal(true),
2174
+ op: z.literal("writeFile")
2175
+ }),
2176
+ z.strictObject({
2177
+ id,
2178
+ ok: z.literal(true),
2179
+ op: z.literal("observe")
2180
+ }),
2181
+ z.strictObject({
2182
+ id,
2183
+ ok: z.literal(true),
2184
+ op: z.literal("expect"),
2185
+ assertion: assertionRecordSchema
2186
+ }),
2187
+ z.strictObject({
2188
+ id,
2189
+ ok: z.literal(true),
2190
+ op: z.literal("record"),
2191
+ assertion: assertionRecordSchema
2192
+ }),
2193
+ z.strictObject({
2194
+ id,
2195
+ ok: z.literal(true),
2196
+ op: z.literal("judge"),
2197
+ verdict: z.union([verdictSchema, judgeFailureSchema])
2198
+ })
2199
+ ]),
2200
+ z.strictObject({
2201
+ id,
2202
+ ok: z.literal(true),
2203
+ op: z.literal("expect"),
2204
+ untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
2205
+ }),
2206
+ z.strictObject({
2207
+ id,
2208
+ ok: z.literal(false),
2209
+ op: z.enum([
2210
+ "send",
2211
+ "exec",
2212
+ "readFile",
2213
+ "writeFile",
2214
+ "observe",
2215
+ "expect",
2216
+ "record",
2217
+ "judge",
2218
+ "serve",
2219
+ "kill",
2220
+ "capture"
2221
+ ]),
2222
+ error: attemptErrorSchema
2223
+ })
2224
+ ]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
2225
+ const cellRoleSchema = z.enum(["control", "variant"]);
2226
+ const cellSchema = z.strictObject({
2227
+ key: z.string(),
2228
+ harness: z.string(),
2229
+ environment: z.string(),
2230
+ sandbox: z.string(),
2231
+ role: cellRoleSchema,
2232
+ variant: z.string().optional()
2233
+ });
2234
+ z.strictObject({
2235
+ type: z.literal("start"),
2236
+ attempt: z.strictObject({
2237
+ id: nonEmpty,
2238
+ cell: cellSchema,
2239
+ n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
2240
+ }).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
2241
+ file: nonEmpty.describe("The eval file, relative to the project."),
2242
+ body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
2243
+ }).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
2244
+ z.strictObject({
2245
+ type: z.literal("abort"),
2246
+ reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
2247
+ message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
2248
+ }).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
2249
+ z.strictObject({
2250
+ type: z.literal("done"),
2251
+ error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect.require` is one.")
2252
+ }).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
2253
+ const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
2254
+ const OTHER_KEYS = [
2255
+ "id",
2256
+ "call",
2257
+ "turn",
2258
+ "attempt",
2259
+ "value",
2260
+ "regexp",
2261
+ "flags",
2262
+ "matchedBy",
2263
+ "answers",
2264
+ ...Object.keys(judgeCallDataSchema.shape),
2265
+ ...Object.keys(captureSchema.shape),
2266
+ ...Object.keys(captureCallDataSchema.shape),
2267
+ "port",
2268
+ "handle",
2269
+ "exec",
2270
+ "result",
2271
+ "readFile",
2272
+ "interval",
2273
+ "timeout",
2274
+ "ms"
2275
+ ];
2276
+ /** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
2277
+ const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
2278
+ // src/attempt/observe.ts
2279
+ /** The most JSON one value may serialize to, masked. */
2280
+ const MAX_OBSERVED_BYTES = 4096;
2281
+ /** The deepest a value may nest arrays and objects, so a check of it can't run out of stack. */
2282
+ const MAX_DEPTH = 64;
2283
+ const at = (path) => path ? ` at ${path}` : "";
2284
+ /** Why `value` isn't JSON, naming where, or undefined. */
2285
+ function notJson(value, path, seen, depth = 0) {
2286
+ if (value === null || typeof value === "string" || typeof value === "boolean") return void 0;
2287
+ if (typeof value === "number") return Number.isFinite(value) ? void 0 : `${value}${at(path)} isn't a finite number`;
2288
+ if (typeof value !== "object") return `a ${typeof value}${at(path)} isn't JSON`;
2289
+ if (seen.has(value)) return `the value refers to itself${at(path)}`;
2290
+ if (depth >= MAX_DEPTH) return `it nests deeper than ${MAX_DEPTH} levels${at(path)}`;
2291
+ const proto = Object.getPrototypeOf(value);
2292
+ if (!Array.isArray(value) && proto !== Object.prototype && proto !== null) return `a ${value.constructor?.name ?? "class instance"}${at(path)} isn't plain JSON`;
2293
+ seen.add(value);
2294
+ const entries = Array.isArray(value) ? Array.from(value, (v, i) => [`${path}[${i}]`, v]) : Object.entries(value).map(([k, v]) => [path ? `${path}.${k}` : k, v]);
2295
+ for (const [p, v] of entries) {
2296
+ const problem = notJson(v, p, seen, depth + 1);
2297
+ if (problem) return problem;
2298
+ }
2299
+ seen.delete(value);
2300
+ }
2301
+ /**
2302
+ * `value` with each number whose digits are a secret's masked too: `redactValue` leaves numbers
2303
+ * as they are, but an observed value goes to a judge's or question's model as it's recorded.
2304
+ */
2305
+ function maskNumbers(value) {
2306
+ if (typeof value === "number") return redactSecrets(String(value)) === String(value) ? value : REDACTED;
2307
+ if (Array.isArray(value)) return value.map(maskNumbers);
2308
+ if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, maskNumbers(v)]));
2309
+ return value;
2310
+ }
2311
+ /**
2312
+ * An observation's name as it's recorded and shown: masked as its value is. A name that holds a
2313
+ * secret is a value too, and a judge's or question's list of names is matched against this.
2314
+ */
2315
+ const observedName = (name) => redactSecrets(name);
2316
+ /** Each attempt's names as its body gave them, by the name they're recorded as. */
2317
+ const givenNames = /* @__PURE__ */ new WeakMap();
2318
+ /**
2319
+ * Whether this attempt observed `name` as given: its masked name alone may be another secret's.
2320
+ * An attempt that never called `c.observe` has observed nothing.
2321
+ */
2322
+ const observedHere = (a, name) => givenNames.get(a)?.get(observedName(name)) === name;
2323
+ /**
2324
+ * `c.observe`: keeps the last value per name, masked, with the turn it came after. Two names that
2325
+ * mask to the same name are refused rather than renamed: a name is the key that tallies, compare
2326
+ * and `attempt_values` group by across attempts, and a numbered stand-in would depend on the
2327
+ * order of each attempt's calls, joining values that don't belong together.
2328
+ */
2329
+ function observer(a) {
2330
+ const raw = /* @__PURE__ */ new Map();
2331
+ givenNames.set(a, raw);
2332
+ return (given, value) => {
2333
+ if (typeof given !== "string" || !OBSERVE_NAME.test(given)) throw new UsageError(`c.observe() needs a name of 1 to 64 letters, digits, _ . : / or - (got ${JSON.stringify(typeof given === "string" ? observedName(given) : given)})`);
2334
+ const name = observedName(given);
2335
+ const earlier = raw.get(name);
2336
+ if (earlier !== void 0 && earlier !== given) throw new UsageError(`c.observe("${name}") names a secret, and its masked name is another observation's: name the value without the secret`);
2337
+ if (a.ended) throw new UsageError(`c.observe("${name}") came after the eval's body returned`);
2338
+ if (a.signal.aborted) throw new UsageError(`c.observe("${name}") came after the attempt timed out or was canceled`);
2339
+ const problem = notJson(value, "", /* @__PURE__ */ new Set());
2340
+ if (problem) throw new UsageError(`c.observe("${name}") takes JSON: ${problem}`);
2341
+ const masked = maskNumbers(redactValue(value));
2342
+ const bytes = Buffer.byteLength(JSON.stringify(masked));
2343
+ if (bytes > 4096) throw new UsageError(`c.observe("${name}") takes at most ${MAX_OBSERVED_BYTES} bytes of JSON (got ${bytes}): observe a count or a summary instead`);
2344
+ a.record.observations ??= [];
2345
+ const list = a.record.observations;
2346
+ const same = list.find((o) => o.name === name);
2347
+ if (!same && list.length >= 50) throw new UsageError(`c.observe("${name}") is one name too many: an attempt observes at most 50`);
2348
+ raw.set(name, given);
2349
+ const observation = {
2350
+ name,
2351
+ value: masked,
2352
+ turn: a.turns.length
2353
+ };
2354
+ if (same) Object.assign(same, observation);
2355
+ else list.push(observation);
2356
+ };
2357
+ }
2358
+ // src/judge/state-options.ts
2359
+ /**
2360
+ * What one call turned on, every key set (`false` when off), in the record's key order, so equal
2361
+ * selections serialize the same.
2362
+ */
2363
+ function resolveStateOptions(call) {
2364
+ const observations = observedNames(call.observations);
2365
+ return {
2366
+ task: call.task ?? false,
2367
+ output: call.output ?? false,
2368
+ files: call.files ?? false,
2369
+ diff: call.diff ?? false,
2370
+ contents: call.contents ?? [],
2371
+ transcript: call.transcript ?? false,
2372
+ turns: call.turns ?? "last",
2373
+ checks: call.checks ?? false,
2374
+ ...observationsOn(observations) ? { observations } : {}
2375
+ };
2376
+ }
2377
+ /** A list of observation names as they're recorded, so a secret in one is masked. */
2378
+ const observedNames = (value) => Array.isArray(value) ? value.map(observedName) : value;
2379
+ /** `observations` when it turns the input on: `true` or a non-empty list. */
2380
+ const observationsOn = (value) => value === true || Array.isArray(value) && value.length > 0;
2381
+ /** `inner` over `outer`, field by field. */
2382
+ function mergeJudge(outer, inner) {
2383
+ return inner ? {
2384
+ ...outer,
2385
+ ...inner
2386
+ } : outer;
2387
+ }
1745
2388
  // src/compare/state.ts
1746
2389
  /** The error a per-attempt answer carries when the run stopped before a model answered it. */
1747
2390
  const STOPPED_FIRST = "the run stopped before it was answered";
@@ -1764,6 +2407,7 @@ const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
1764
2407
  * question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
1765
2408
  */
1766
2409
  function resolveQuestionState(state) {
2410
+ const observations = observedNames(state?.observations);
1767
2411
  return {
1768
2412
  task: state?.task ?? false,
1769
2413
  output: state?.output ?? false,
@@ -1771,7 +2415,8 @@ function resolveQuestionState(state) {
1771
2415
  diff: state?.diff ?? false,
1772
2416
  transcript: state?.transcript ?? false,
1773
2417
  turns: state?.turns ?? "all",
1774
- checks: state?.checks ?? false
2418
+ checks: state?.checks ?? false,
2419
+ ...observationsOn(observations) ? { observations } : {}
1775
2420
  };
1776
2421
  }
1777
2422
  /** `inner` over `outer`, field by field. */
@@ -1792,7 +2437,7 @@ function withQuestionDefaults(q, defaults) {
1792
2437
  const isOff = (value) => value === false || Array.isArray(value) && !value.length;
1793
2438
  /** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
1794
2439
  function questionShows(state, artifacts) {
1795
- const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
2440
+ const on = [...KEYS.filter((k) => k !== "turns" && !isOff(state[k])), ...state.observations ? ["observations"] : []];
1796
2441
  return artifacts === void 0 ? on : [...on, "artifacts"];
1797
2442
  }
1798
2443
  // src/contract/compare.ts
@@ -1905,7 +2550,7 @@ const addressFor = (listeners, port) => {
1905
2550
  const all = listeners.filter((l) => l.port === port).map((l) => l.address);
1906
2551
  return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
1907
2552
  };
1908
- /** `want` is taken whoever listens on it: reeval allocated it, or the eval named it. */
2553
+ /** `want` is taken whoever listens on it: `servePort` found it free when the command started. */
1909
2554
  function decide(s) {
1910
2555
  const { listeners, alive } = s.probe;
1911
2556
  const ports = newPorts(listeners, s.before);
@@ -1962,6 +2607,24 @@ async function hostListening(port) {
1962
2607
  socket.once("error", () => resolve(false));
1963
2608
  })) return host;
1964
2609
  }
2610
+ /** How many times `c.serve` allocates `PORT` again when the sandbox already listens on it. */
2611
+ const ALLOCATE_TRIES = 5;
2612
+ /**
2613
+ * The port `c.serve` sets as `PORT` and waits on: `port`, or one from `allocate` that nothing
2614
+ * listens on yet. Throws when something already listens on `port`, since that server, not the
2615
+ * command's, would answer the checks: c.serve only stops what it started.
2616
+ */
2617
+ async function servePort(command, port, allocate, taken) {
2618
+ if (port === void 0) {
2619
+ for (let i = 0; i < ALLOCATE_TRIES; i++) {
2620
+ const want = await allocate();
2621
+ if (!await taken(want)) return want;
2622
+ }
2623
+ throw new Error(`c.serve(${JSON.stringify(command)}) found every port it allocated for PORT already in use in the sandbox`);
2624
+ }
2625
+ if (await taken(port)) throw new Error(`c.serve(${JSON.stringify(command)}, ${port}): port ${port} was already in use before c.serve started (a server the agent left running?), so the checks would reach that server, not this command's. Stop it first, or drop the port to serve on PORT`);
2626
+ return port;
2627
+ }
1965
2628
  const sleep = (ms, signal) => new Promise((resolve) => {
1966
2629
  const done = () => {
1967
2630
  clearTimeout(timer);
@@ -2261,16 +2924,16 @@ function resolveOutput(input) {
2261
2924
  "errors-only",
2262
2925
  "none"
2263
2926
  ]);
2264
- const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only";
2927
+ const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "none";
2265
2928
  const color = env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions;
2266
2929
  return {
2267
2930
  mode,
2268
2931
  statusLine,
2932
+ interactive: statusLine && !!input.stdinTTY && outputLogs === "none",
2269
2933
  logOrder,
2270
2934
  logPrefix,
2271
2935
  outputLogs,
2272
2936
  color,
2273
- rich: statusLine && color,
2274
2937
  githubActions,
2275
2938
  heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
2276
2939
  };
@@ -2398,30 +3061,6 @@ function egressLabel({ policy, auto }) {
2398
3061
  const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
2399
3062
  return auto.length ? `${own} + auto: ${auto.join(", ")}` : own;
2400
3063
  }
2401
- // src/judge/state-options.ts
2402
- /**
2403
- * What one call turned on, every key set (`false` when off), in the record's key order, so equal
2404
- * selections serialize the same.
2405
- */
2406
- function resolveStateOptions(call) {
2407
- return {
2408
- task: call.task ?? false,
2409
- output: call.output ?? false,
2410
- files: call.files ?? false,
2411
- diff: call.diff ?? false,
2412
- contents: call.contents ?? [],
2413
- transcript: call.transcript ?? false,
2414
- turns: call.turns ?? "last",
2415
- checks: call.checks ?? false
2416
- };
2417
- }
2418
- /** `inner` over `outer`, field by field. */
2419
- function mergeJudge(outer, inner) {
2420
- return inner ? {
2421
- ...outer,
2422
- ...inner
2423
- } : outer;
2424
- }
2425
3064
  const KNOWN = [...[
2426
3065
  "harness",
2427
3066
  "sandbox",
@@ -2754,9 +3393,8 @@ function normalizeCompare(value) {
2754
3393
  };
2755
3394
  }
2756
3395
  /**
2757
- * Runs per cell when no level sets `runs`: a compare with variants runs `EFFECT_MIN_COUNTED` a
2758
- * side, the fewest whose lift can reach a verdict; anything else, a pooled family's cases
2759
- * included, runs once.
3396
+ * Runs per cell when no level sets `runs`: a compare with variants runs `COMPARE_RUNS` a side;
3397
+ * anything else, a pooled family's cases included, runs once.
2760
3398
  */
2761
3399
  const COMPARE_DEFAULT_RUNS = 4;
2762
3400
  function applyLevel(out, options, where, egress = "own") {
@@ -3257,7 +3895,11 @@ const GATEWAYS = {
3257
3895
  vertex: { cantServe: (h) => {
3258
3896
  if (h.kind === "fake") return void 0;
3259
3897
  if (!VERTEX_ROUTES[h.kind]) return VERTEX_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vertex AI yet`;
3260
- return vertexModelRefusalFor(h) ?? (h.kind === "opencode" ? openCodeEffortOn(h.model, "vertex", harnessEffort(h)) : void 0);
3898
+ const effort = harnessEffort(h);
3899
+ const refused = vertexModelRefusalFor(h);
3900
+ if (refused) return refused;
3901
+ if (h.kind === "claude-code" && effort !== void 0) return claudeCodeTakesEffort(h.model, "vertex") ? void 0 : `Claude Code sends no effort to ${claudeCodeModelName(h.model, "vertex")} on Vertex AI, where it sends one only to the models its catalog lists: drop effort, or run one of them, e.g. claude-sonnet-5-5`;
3902
+ return h.kind === "opencode" ? openCodeEffortOn(h.model, "vertex", effort) : void 0;
3261
3903
  } }
3262
3904
  };
3263
3905
  /** OpenCode's id for `model` on `gateway`, as that gateway's route writes it. */
@@ -3886,7 +4528,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3886
4528
  }
3887
4529
  const ANALYSIS_DEFAULTS = {
3888
4530
  maxInputTokens: 2e4,
3889
- maxOutputTokens: 8e3
4531
+ maxOutputTokens: 32e3
3890
4532
  };
3891
4533
  function checkAnalysis(analysis, file) {
3892
4534
  if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
@@ -4054,6 +4696,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
4054
4696
  return at ? [`${name} (character ${at})`] : [];
4055
4697
  });
4056
4698
  }
4057
- export { allowedHosts as $, evalPlan as A, refusedMcp as At, profileJudge as B, judgeProblem as Bt, openCodeModelOn as C, brokeringFor as Ct, cascade as D, isAgentKind as Dt, brokeredIn as E, expectedBlockedFor as Et, matchesTags as F, NO_JUDGE_INPUT as Ft, sandboxSetBy as G, decideState as Gt, runsOf as H, turnsSchema as Ht, plainHeaderError as I, fakeJudgeDataSchema as It, unbrokeredError as J, secretVarsIn as K, filePartsOf as Kt, plainHeaders as L, inputsOn as Lt, harnessId as M, writePluginFiles as Mt, levelOf as N, captureWindowSchema as Nt, checkLimitHarness as O, keyRejected as Ot, majorityOf as P, INPUT_KEYS as Pt, DEFAULT_EGRESS_ALLOW as Q, plannedSandboxIds as R, judgeFields as Rt, fallbackHint as S, authHint as St, routedHarness as T, credentialOf as Tt, sandboxId as U, artifactRefsOf as Ut, refuseIllegalCompare as V, sharedInputFields as Vt, sandboxProvider as W, badJudgeArtifacts as Wt, resolveStateOptions as X, mergeJudge as Y, DEFAULT_EGRESS as Z, fellBackToDirect as _, servedEffortOf as _t, configPathFor as a, unenforceable as at, resolveModelGateway as b, agentFor as bt, findConfig as c, awaitListening as ct, malformedCredentials as d, STOPPED_FIRST as dt, ciRunner as et, missingCredentials as f, UNRECORDED_QUESTION_STATE as ft, selectProfile as g, resolvedEffortOf as gt, runJudgeModel as h, harnessEffort as ht, configLabel as i, isDirect as it, expand as j, refusedPlugins as jt, checkOptions as k, layoutOf as kt, hostCapacity as l, parseProbe as lt, resolveConcurrency as m, resolveQuestionState as mt, analysisModelOf as n, egressLabel as nt, defaultConcurrency as o, resolveOutput as ot, requireConfig as p, questionShows as pt, unbrokered as q, resolveJudgeArtifacts as qt, configFilesPhrase as r, gatewayPolicy as rt, defineConfig as s, createFakeSandbox as st, CONFIG_FILES as t, directOnCloud as tt, loadConfig as u, probeScript as ut, gatewayScope as v, AGENTS as vt, resolveGateway as w, claudeMcpServers as wt, effortRefusedOn as x, agentMcp as xt, modelGatewayOrNone as y, agentAuth as yt, plannedSandboxes as z, judgeModelNameSchema as zt };
4699
+ export { directOnCloud as $, evalPlan as A, agentMcp as At, profileJudge as B, refusedPlugins as Bt, openCodeModelOn as C, execOptionsProblem as Ct, cascade as D, AGENTS as Dt, brokeredIn as E, servedEffortOf as Et, matchesTags as F, expectedBlockedFor as Ft, sandboxSetBy as G, judgeProblem as Gt, runsOf as H, INPUT_KEYS as Ht, plainHeaderError as I, isAgentKind as It, unbrokeredError as J, decideState as Jt, secretVarsIn as K, artifactRefsOf as Kt, plainHeaders as L, keyRejected as Lt, harnessId as M, brokeringFor as Mt, levelOf as N, claudeMcpServers as Nt, checkLimitHarness as O, agentAuth as Ot, majorityOf as P, credentialOf as Pt, ciRunner as Q, plannedSandboxIds as R, layoutOf as Rt, fallbackHint as S, attemptRequestProblem as St, routedHarness as T, resolvedEffortOf as Tt, sandboxId as U, NO_JUDGE_INPUT as Ut, refuseIllegalCompare as V, writePluginFiles as Vt, sandboxProvider as W, inputsOn as Wt, DEFAULT_EGRESS_ALLOW as X, resolveJudgeArtifacts as Xt, DEFAULT_EGRESS as Y, filePartsOf as Yt, allowedHosts as Z, fellBackToDirect as _, observedHere as _t, configPathFor as a, createFakeSandbox as at, resolveModelGateway as b, CAPTURE_NAME as bt, findConfig as c, parseProbe as ct, malformedCredentials as d, STOPPED_FIRST as dt, egressLabel as et, missingCredentials as f, UNRECORDED_QUESTION_STATE as ft, selectProfile as g, resolveStateOptions as gt, runJudgeModel as h, mergeJudge as ht, configLabel as i, resolveOutput as it, expand as j, authHint as jt, checkOptions as k, agentFor as kt, hostCapacity as l, probeScript as lt, resolveConcurrency as m, resolveQuestionState as mt, analysisModelOf as n, isDirect as nt, defaultConcurrency as o, awaitListening as ot, requireConfig as p, questionShows as pt, unbrokered as q, badJudgeArtifacts as qt, configFilesPhrase as r, unenforceable as rt, defineConfig as s, hostListening as st, CONFIG_FILES as t, gatewayPolicy as tt, loadConfig as u, servePort as ut, gatewayScope as v, observedName as vt, resolveGateway as w, harnessEffort as wt, effortRefusedOn as x, CAPTURE_NAME_RULE as xt, modelGatewayOrNone as y, observer as yt, plannedSandboxes as z, refusedMcp as zt };
4058
4700
 
4059
- //# sourceMappingURL=config-ImY6AlRE.js.map
4701
+ //# sourceMappingURL=config-C2UtS9vy.js.map