reeval 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (481) hide show
  1. package/CHANGELOG.md +841 -0
  2. package/README.md +15 -15
  3. package/assets/gateway-models.mjs +65 -31
  4. package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
  5. package/dist/adapter-Di9H2etg.js.map +1 -0
  6. package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
  7. package/dist/agent-B-6GFDqA.js.map +1 -0
  8. package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
  9. package/dist/agents-KBTYhSSh.js.map +1 -0
  10. package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
  11. package/dist/app-files-Cmh9FGr5.js.map +1 -0
  12. package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
  13. package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
  14. package/dist/assemble-DNVBTBp_.js.map +1 -0
  15. package/dist/attach-EbDGKGTO.js +152 -0
  16. package/dist/attach-EbDGKGTO.js.map +1 -0
  17. package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
  18. package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
  19. package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
  20. package/dist/attempt-host-DQvynDHk.js.map +1 -0
  21. package/dist/boat-Bzz8L-KQ.js +701 -0
  22. package/dist/boat-Bzz8L-KQ.js.map +1 -0
  23. package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
  24. package/dist/boat-Dek8ftIt.js.map +1 -0
  25. package/dist/bridge/index.mjs +6 -1
  26. package/dist/bundle-DZstHDeW.js +718 -0
  27. package/dist/bundle-DZstHDeW.js.map +1 -0
  28. package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
  29. package/dist/capture-BkbPsfej.js.map +1 -0
  30. package/dist/capture-CUAvk9KM.js +1 -0
  31. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  32. package/dist/cases-CM-MQRQp.js.map +1 -0
  33. package/dist/cases-FvhbM8M5.js +1 -0
  34. package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
  35. package/dist/cli-BgBBuxxJ.js.map +1 -0
  36. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
  37. package/dist/cli-spec-B5L9n-DV.js.map +1 -0
  38. package/dist/cli.js +2 -2
  39. package/dist/client-DMEYR3MA.js +807 -0
  40. package/dist/client-DMEYR3MA.js.map +1 -0
  41. package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
  42. package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
  43. package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
  44. package/dist/command-D_w_kDpI.js.map +1 -0
  45. package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
  46. package/dist/command-Dwlh0Vhu.js.map +1 -0
  47. package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
  48. package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
  49. package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
  50. package/dist/commands-Cuf57OYb.js.map +1 -0
  51. package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
  52. package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
  53. package/dist/config-C0RKt00M.js.map +1 -0
  54. package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
  55. package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
  56. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  57. package/dist/cost-C21gGrju.js.map +1 -0
  58. package/dist/cost-IM80LyGD.js +1 -0
  59. package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
  60. package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
  61. package/dist/db-3o8s2EyD.js +2 -0
  62. package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
  63. package/dist/db-ErnJ-gMD.js.map +1 -0
  64. package/dist/debug-CIXzB4wK.js.map +1 -1
  65. package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
  66. package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
  67. package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
  68. package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
  69. package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
  70. package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
  71. package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
  72. package/dist/docker-DrMnpL84.js +1 -0
  73. package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
  74. package/dist/doctor-B7utCSdh.js.map +1 -0
  75. package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
  76. package/dist/environment-C0Gg8kyN.js.map +1 -0
  77. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  78. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  79. package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
  80. package/dist/eval-source-CsW5VIXS.js.map +1 -0
  81. package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
  82. package/dist/evaluate-BHQyAszQ.js.map +1 -0
  83. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
  84. package/dist/fake-bootstrap-VpisF1WE.js +15 -0
  85. package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
  86. package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
  87. package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
  88. package/dist/files-BjOJw4sn.js +1 -0
  89. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  90. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  91. package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
  92. package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
  93. package/dist/files-CAzVG3Rr.js +1 -0
  94. package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
  95. package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
  96. package/dist/files-D-Lir4aE.js +1 -0
  97. package/dist/files-DRlJ0Xr2.js +1 -0
  98. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  99. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  100. package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
  101. package/dist/files-qjeqXnfE.js.map +1 -0
  102. package/dist/gateway.d.ts +1 -1
  103. package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
  104. package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
  105. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  106. package/dist/harness-FcTyyPco.js.map +1 -0
  107. package/dist/harness-install-Dxho1yaU.js +53 -0
  108. package/dist/harness-install-Dxho1yaU.js.map +1 -0
  109. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  110. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  111. package/dist/harness.d.ts +11 -8
  112. package/dist/harness.js +1 -1
  113. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  114. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  115. package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
  116. package/dist/images-Ch1y1t6_.js.map +1 -0
  117. package/dist/index.d.ts +46 -7
  118. package/dist/index.js +6 -6
  119. package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
  120. package/dist/init-C2HqCwnt.js.map +1 -0
  121. package/dist/init-CM9xwyre.js +1 -0
  122. package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
  123. package/dist/ink-list-BbKXElcQ.js.map +1 -0
  124. package/dist/inputs-5UIDpjz-.js +190 -0
  125. package/dist/inputs-5UIDpjz-.js.map +1 -0
  126. package/dist/inputs-load-C82HOKnb.js +1 -0
  127. package/dist/inputs-load-CGPyigMC.js +95 -0
  128. package/dist/inputs-load-CGPyigMC.js.map +1 -0
  129. package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
  130. package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
  131. package/dist/jiti-radm1Soi.js +52 -0
  132. package/dist/jiti-radm1Soi.js.map +1 -0
  133. package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
  134. package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
  135. package/dist/judge.d.ts +3 -1
  136. package/dist/judge.js +1 -1
  137. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  138. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  139. package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
  140. package/dist/layout-Czh5wOpz.js.map +1 -0
  141. package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
  142. package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
  143. package/dist/live-C9Et_BVT.js +1 -0
  144. package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
  145. package/dist/live-DGgIrVw0.js.map +1 -0
  146. package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
  147. package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
  148. package/dist/machine-service-2lIv5UWF.js +1 -0
  149. package/dist/machine-service-Ct_1XMMi.js +520 -0
  150. package/dist/machine-service-Ct_1XMMi.js.map +1 -0
  151. package/dist/machines-BnquHcBZ.js +120 -0
  152. package/dist/machines-BnquHcBZ.js.map +1 -0
  153. package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
  154. package/dist/model-Bcy4Qjmm.js +1 -0
  155. package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
  156. package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
  157. package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
  158. package/dist/model-DRa7yv6K.js.map +1 -0
  159. package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
  160. package/dist/model-E5X8YBJM.js.map +1 -0
  161. package/dist/model-calls-B0BoQfJ4.js +95 -0
  162. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  163. package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
  164. package/dist/node-BjuhqQwa.js.map +1 -0
  165. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  166. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  167. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
  168. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
  169. package/dist/options-C989kuVE.js +1 -0
  170. package/dist/own-reads-CYmBlrHK.js +15 -0
  171. package/dist/own-reads-CYmBlrHK.js.map +1 -0
  172. package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
  173. package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
  174. package/dist/pi-providers-HygmL1y-.js +552 -0
  175. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  176. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  177. package/dist/plain-Ctn8oUdp.js.map +1 -0
  178. package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
  179. package/dist/plan-BcT4G5jj.js.map +1 -0
  180. package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
  181. package/dist/plugins-cC0sqoDA.js.map +1 -0
  182. package/dist/preflight-BMVDBcux.js +1 -0
  183. package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
  184. package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
  185. package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
  186. package/dist/preflight-DySHXeFJ.js.map +1 -0
  187. package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
  188. package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
  189. package/dist/prepare-nbX1CXXg.js +409 -0
  190. package/dist/prepare-nbX1CXXg.js.map +1 -0
  191. package/dist/prices-B1vm_lPk.js +1 -0
  192. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  193. package/dist/prices-BrY_ihLs.js.map +1 -0
  194. package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
  195. package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
  196. package/dist/provider-DHT20Gl8.js +1 -0
  197. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  198. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  199. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  200. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  201. package/dist/providers-BkLGzaSY.js +1 -0
  202. package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
  203. package/dist/providers-bNgfKa6m.js.map +1 -0
  204. package/dist/query-B2SQddoV.js +1 -0
  205. package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
  206. package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
  207. package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
  208. package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
  209. package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
  210. package/dist/record--vHreTLk.js.map +1 -0
  211. package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
  212. package/dist/redact-t-L3VGy8.js.map +1 -0
  213. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  214. package/dist/remarks-BpdM_2RC.js.map +1 -0
  215. package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
  216. package/dist/remarks-CYFOsnz5.js.map +1 -0
  217. package/dist/remarks-CleoDlYw.js +1 -0
  218. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  219. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  220. package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
  221. package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
  222. package/dist/resolve-Bm9ZyFR-.js +1 -0
  223. package/dist/resolve-Bul8lHaa.js +205 -0
  224. package/dist/resolve-Bul8lHaa.js.map +1 -0
  225. package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
  226. package/dist/resolve-DorSHDBs.js.map +1 -0
  227. package/dist/resources-DqdwihhT.js +62 -0
  228. package/dist/resources-DqdwihhT.js.map +1 -0
  229. package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
  230. package/dist/results-DhMgRkR0.js.map +1 -0
  231. package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
  232. package/dist/resume-C8Ihfy-C.js.map +1 -0
  233. package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
  234. package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
  235. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  236. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  237. package/dist/routes-Cp4gAjBZ.js +427 -0
  238. package/dist/routes-Cp4gAjBZ.js.map +1 -0
  239. package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
  240. package/dist/runtime-CPUH0Vk5.js.map +1 -0
  241. package/dist/sandbox/boat.d.ts +1 -1
  242. package/dist/sandbox/boat.js +1 -1
  243. package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
  244. package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
  245. package/dist/sandbox.d.ts +1 -1
  246. package/dist/sandbox.js +1 -1
  247. package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
  248. package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
  249. package/dist/server-ZaMzUNmV.js +1 -0
  250. package/dist/services-BG8xuU82.js +39 -0
  251. package/dist/services-BG8xuU82.js.map +1 -0
  252. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  253. package/dist/shared-DtZwIAfy.js.map +1 -0
  254. package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
  255. package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
  256. package/dist/source-BcBDj-R0.js +1 -0
  257. package/dist/src-B2eHTCnp.js +3 -0
  258. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  259. package/dist/src-B5r1d23Y.js.map +1 -0
  260. package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
  261. package/dist/src-Bb3k9Kn_.js.map +1 -0
  262. package/dist/src-BfOlH_Bu.js +2 -0
  263. package/dist/src-BuGc40IF.js +2 -0
  264. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  265. package/dist/src-C-ChdnG6.js.map +1 -0
  266. package/dist/src-C2K5_afU.js +2 -0
  267. package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
  268. package/dist/src-C3OhA5pw.js.map +1 -0
  269. package/dist/src-CSVJjghq.js +4 -0
  270. package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
  271. package/dist/src-CWfy8ct8.js.map +1 -0
  272. package/dist/src-CcrQQB0_.js +2 -0
  273. package/dist/src-CotUxxQV.js +2 -0
  274. package/dist/src-D-PwBaBB.js +2 -0
  275. package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
  276. package/dist/src-DNWiU__5.js.map +1 -0
  277. package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
  278. package/dist/src-DwcTn4JY.js.map +1 -0
  279. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  280. package/dist/src-boB6D-5T.js.map +1 -0
  281. package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
  282. package/dist/src-c3ZQu7Uu.js.map +1 -0
  283. package/dist/src-jYqNkT-F.js +1 -0
  284. package/dist/start-Db5VRcRZ.js +660 -0
  285. package/dist/start-Db5VRcRZ.js.map +1 -0
  286. package/dist/state-CKIjZZRK.js +1 -0
  287. package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
  288. package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
  289. package/dist/store/local.d.ts +8 -0
  290. package/dist/store/local.js +16 -0
  291. package/dist/store/local.js.map +1 -0
  292. package/dist/store/s3.d.ts +10 -0
  293. package/dist/store/s3.js +18 -0
  294. package/dist/store/s3.js.map +1 -0
  295. package/dist/suite-CH1aaAbs.js +2 -0
  296. package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
  297. package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
  298. package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
  299. package/dist/summary-BK5Sa_N0.js.map +1 -0
  300. package/dist/sync-DUi_P2UJ.js +559 -0
  301. package/dist/sync-DUi_P2UJ.js.map +1 -0
  302. package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
  303. package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
  304. package/dist/tart-BQ_Ap0l4.js +508 -0
  305. package/dist/tart-BQ_Ap0l4.js.map +1 -0
  306. package/dist/tart-CzViW3WZ.js +186 -0
  307. package/dist/tart-CzViW3WZ.js.map +1 -0
  308. package/dist/tart-DmVuVsmX.js +1 -0
  309. package/dist/tart-bake-D1RttYfM.js +148 -0
  310. package/dist/tart-bake-D1RttYfM.js.map +1 -0
  311. package/dist/tart-bake-DjFcsG3N.js +1 -0
  312. package/dist/tart-cli-BOcluha1.js +1 -0
  313. package/dist/tart-cli-gA9xEaQL.js +89 -0
  314. package/dist/tart-cli-gA9xEaQL.js.map +1 -0
  315. package/dist/tart-hosts-CHNrfV6u.js +313 -0
  316. package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
  317. package/dist/tart-hosts-CWse-ASo.js +1 -0
  318. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
  319. package/dist/tart-images-CLBjfF9u.js.map +1 -0
  320. package/dist/testing.d.ts +36 -2
  321. package/dist/testing.js +4 -4
  322. package/dist/tool-class-krsmEo44.js +433 -0
  323. package/dist/tool-class-krsmEo44.js.map +1 -0
  324. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  325. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  326. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  327. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  328. package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
  329. package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
  330. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  331. package/dist/types-CbcnJxCy.js.map +1 -0
  332. package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
  333. package/dist/ui-DKYjTVpn.js +1455 -0
  334. package/dist/ui-DKYjTVpn.js.map +1 -0
  335. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  336. package/dist/usage-CwIRPPZ9.js.map +1 -0
  337. package/dist/vercel-models-Ci33llXz.js +21 -0
  338. package/dist/vercel-models-Ci33llXz.js.map +1 -0
  339. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  340. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  341. package/dist/vertex-DXfcXYKn.js +1 -0
  342. package/dist/view/index.d.ts +32 -3
  343. package/dist/view/index.js +8 -8
  344. package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
  345. package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
  346. package/dist/views-DCjtw1jF.js +2 -0
  347. package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
  348. package/dist/worker-entry-DrAYYt0M.js.map +1 -0
  349. package/dist/workspace-CnRbV8cM.js +1 -0
  350. package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
  351. package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
  352. package/docker/Dockerfile +1 -1
  353. package/package.json +15 -1
  354. package/skills/eval-rubric/SKILL.md +10 -0
  355. package/skills/reeval/SKILL.md +370 -0
  356. package/skills/reeval/references/capture.md +48 -0
  357. package/skills/reeval/references/cli.md +162 -0
  358. package/skills/reeval/references/compare.md +180 -0
  359. package/skills/reeval/references/config.md +147 -0
  360. package/skills/reeval/references/environments.md +218 -0
  361. package/skills/reeval/references/evals.md +241 -0
  362. package/skills/reeval/references/mistakes.md +39 -0
  363. package/skills/reeval/references/vocabulary.md +19 -0
  364. package/skills/setup/SKILL.md +219 -0
  365. package/skills/setup/references/templates.md +258 -0
  366. package/dist/adapter-C60z3wLq.js.map +0 -1
  367. package/dist/agent-BmsMXZXx.js.map +0 -1
  368. package/dist/agents-DXuFxtGC.js.map +0 -1
  369. package/dist/app-files-DT5mJyV4.js.map +0 -1
  370. package/dist/assemble-DM2PNU4R.js.map +0 -1
  371. package/dist/attempt-host-7VviAJrl.js.map +0 -1
  372. package/dist/boat-BjLqFJIZ.js.map +0 -1
  373. package/dist/boat-DSNgZt-t.js +0 -322
  374. package/dist/boat-DSNgZt-t.js.map +0 -1
  375. package/dist/capture-BzotD2hs.js +0 -1
  376. package/dist/capture-Ccf0Hmyx.js.map +0 -1
  377. package/dist/cases-Dy0mifal.js +0 -1
  378. package/dist/cases-z9E6DCd5.js.map +0 -1
  379. package/dist/cli-DTuLSlmW.js.map +0 -1
  380. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  381. package/dist/client-KM4C5r5_.js +0 -387
  382. package/dist/client-KM4C5r5_.js.map +0 -1
  383. package/dist/command-DSL7I-SI.js.map +0 -1
  384. package/dist/command-Ff0T-1Hi.js.map +0 -1
  385. package/dist/commands-xAOi_VEk.js.map +0 -1
  386. package/dist/config-SvK8pjEK.js.map +0 -1
  387. package/dist/cost-BdRgySQl.js.map +0 -1
  388. package/dist/cost-Cr26Alhq.js +0 -1
  389. package/dist/db-BzOJ4Js4.js.map +0 -1
  390. package/dist/db-CQRZ3URX.js +0 -2
  391. package/dist/docker--mukta3L.js +0 -1
  392. package/dist/docker-DG_uFgn2.js +0 -246
  393. package/dist/docker-DG_uFgn2.js.map +0 -1
  394. package/dist/doctor-DWK4ttex.js.map +0 -1
  395. package/dist/environment-cPMCOk-h.js.map +0 -1
  396. package/dist/eval-source-C9b1h8L6.js.map +0 -1
  397. package/dist/evaluate-B12Tosvq.js.map +0 -1
  398. package/dist/files-C1Rk9FYA.js +0 -1
  399. package/dist/files-DMQ9-Vv5.js +0 -1
  400. package/dist/files-Di0xDbCc.js +0 -1
  401. package/dist/files-G5jf8wBp.js +0 -1
  402. package/dist/files-s28a6czq.js.map +0 -1
  403. package/dist/harness-DXv2gnaT.js.map +0 -1
  404. package/dist/images-D2jFze2v.js.map +0 -1
  405. package/dist/init-Bi1Q3lRi.js.map +0 -1
  406. package/dist/init-CUw65o36.js +0 -1
  407. package/dist/ink-list-BpfidQ1h.js.map +0 -1
  408. package/dist/kinds-Bh2I35id.js +0 -698
  409. package/dist/kinds-Bh2I35id.js.map +0 -1
  410. package/dist/layout-Cm85tQYk.js.map +0 -1
  411. package/dist/live-CgYbO4Te.js.map +0 -1
  412. package/dist/live-DZLmj0iG.js +0 -1
  413. package/dist/matchers-CXLn-1jm.js +0 -148
  414. package/dist/matchers-CXLn-1jm.js.map +0 -1
  415. package/dist/model-Cv4GuTsZ.js +0 -1
  416. package/dist/model-CwwkrCb7.js.map +0 -1
  417. package/dist/model-cgt5uvu8.js.map +0 -1
  418. package/dist/node-D7pAU--0.js.map +0 -1
  419. package/dist/options-DPUmk7v_.js +0 -1
  420. package/dist/plain-BYUf_fUV.js.map +0 -1
  421. package/dist/plan-CsPilzpk.js.map +0 -1
  422. package/dist/plugins-BfnhazOv.js.map +0 -1
  423. package/dist/preflight-Bn3ianye.js +0 -1
  424. package/dist/preflight-GkpgrBxP.js.map +0 -1
  425. package/dist/prepare-DQEDXoDp.js +0 -268
  426. package/dist/prepare-DQEDXoDp.js.map +0 -1
  427. package/dist/prices-5m3SzMZ3.js.map +0 -1
  428. package/dist/provider-BnUoz4KB.js +0 -1
  429. package/dist/providers-1p4HKYIp.js.map +0 -1
  430. package/dist/providers-B-ijCktf.js +0 -1
  431. package/dist/query-BYhyf0LR.js +0 -1
  432. package/dist/record-6IUzwP9K.js.map +0 -1
  433. package/dist/redact-BhjtnjrU.js.map +0 -1
  434. package/dist/remarks-BNABzemt.js.map +0 -1
  435. package/dist/remarks-CAvTSq16.js +0 -1
  436. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  437. package/dist/resolve-C14Pmtej.js +0 -1
  438. package/dist/resolve-Cx3a9UQ2.js.map +0 -1
  439. package/dist/results-CQKXmHym.js.map +0 -1
  440. package/dist/resume-POpZa1ge.js.map +0 -1
  441. package/dist/runtime-Xm9oeC7o.js.map +0 -1
  442. package/dist/server-C1UJcpUh.js +0 -1
  443. package/dist/shared-Cfi45Bvy.js.map +0 -1
  444. package/dist/source-DJHmQv_j.js +0 -1
  445. package/dist/src-1dBuXzs4.js +0 -3
  446. package/dist/src-BKpioayC.js +0 -2
  447. package/dist/src-BNQznj4z.js.map +0 -1
  448. package/dist/src-BY1cCaoJ.js +0 -2
  449. package/dist/src-BbZYaQ8b.js +0 -2
  450. package/dist/src-CDFH2vVN.js +0 -2
  451. package/dist/src-CMW5R9Q0.js +0 -1
  452. package/dist/src-CV_7DkPa.js.map +0 -1
  453. package/dist/src-Cb2MWozp.js.map +0 -1
  454. package/dist/src-CrV2WA2i.js +0 -2
  455. package/dist/src-CsFaawcZ.js +0 -2
  456. package/dist/src-DAzaCTcN.js.map +0 -1
  457. package/dist/src-DvGns8cz.js.map +0 -1
  458. package/dist/src-Dw79VLa3.js.map +0 -1
  459. package/dist/src-KxS91P7L.js.map +0 -1
  460. package/dist/src-YvblOrBU.js.map +0 -1
  461. package/dist/src-wKwznPgt.js +0 -2
  462. package/dist/src-y0nOeRRE.js.map +0 -1
  463. package/dist/state-Ca55Gk_O.js +0 -1
  464. package/dist/suite-CKBa6MKt.js +0 -2
  465. package/dist/summary-DDy9DfyJ.js.map +0 -1
  466. package/dist/tart-cli-C-GsbVsQ.js +0 -1
  467. package/dist/tart-cli-CU0nFS-L.js +0 -55
  468. package/dist/tart-cli-CU0nFS-L.js.map +0 -1
  469. package/dist/tart-dWoJvWHc.js +0 -586
  470. package/dist/tart-dWoJvWHc.js.map +0 -1
  471. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  472. package/dist/tool-target-DS5d72eH.js +0 -40
  473. package/dist/tool-target-DS5d72eH.js.map +0 -1
  474. package/dist/types-C2mLpph3.js.map +0 -1
  475. package/dist/usage-DJiDnh2D.js.map +0 -1
  476. package/dist/vercel-CGlSBv63.js +0 -344
  477. package/dist/vercel-CGlSBv63.js.map +0 -1
  478. package/dist/vertex-O88TCwgY.js +0 -1
  479. package/dist/views-B0Yt_Q1S.js +0 -2
  480. package/dist/worker-entry-Smb4bNDc.js.map +0 -1
  481. package/dist/workspace-C0dDP75l.js +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"workspace-DNDTEk8v.js","names":[],"sources":["../src/workspace.ts"],"mappings":";;;;;;;;;;;AAsBA,IAAa,iBAAb,cAAoC,MAAM;CACxC,OAAgB;CAChB;CACA,YAAY,SAAiB,OAAgB;EAC3C,MAAM,OAAO;EACb,KAAK,QAAQ;CACf;AACF;AAEA,MAAa,gBAAgB,MAC3B,KAAK,UAAU;CAAC,EAAE;CAAK,EAAE,OAAO;CAAM,EAAE,QAAQ;AAAI,CAAC;AAEvD,MAAM,YAAY,SAAiB,KAAK,MAAM,WAAW,SAAS,KAAK;AACvE,MAAM,gBAAgB,MAAc,UAClC,KAAK,MAAM,WAAW,SAAS,YAAY,GAAG,MAAM,UAAU;AAEhE,MAAM,WAAW;;AAKjB,SAAgB,iBAAiB,QAAyB;CACxD,IAAI,4EAA4E,KAAK,MAAM,GACzF,OAAO;CACT,IACE,wLAAwL,KACtL,MACF,GAEA,OAAO;CACT,IACE,wOAAwO,KACtO,MACF,GAEA,OAAO;CACT,IACE,sMAAsM,KACpM,MACF,GAEA,OAAO;CACT,OAAO;AACT;AAEA,MAAM,aAAa,WACjB,OACG,MAAM,IAAI,CAAC,CACX,KAAK,MAAM,EAAE,QAAQ,sBAAsB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CACtD,KAAK,OAAO,KAAK;AAGtB,MAAM,UAA6B,QAAQ,IAAI,kBAC3C,CAAC,IACD,EAAE,iBAAiB,uBAAuB;AAE9C,eAAe,MAAM,KAAa,MAAgB,OAAgB,MAAyB,CAAC,GAAG;CAC7F,MAAM,IAAI,MAAM,IAAI,KAAK,MAAM;EAC7B,KAAK;GAAE,GAAG;GAAS,GAAG;EAAI;EAC1B,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;CACzC,CAAC;CACD,OAAO;EAAE,GAAG;EAAG,MAAM,EAAE,OAAO,SAAS,MAAM,CAAC,CAAC,KAAK;CAAE;AACxD;AAEA,MAAM,SAAS,OAAe,IAAI,SAAS,YAAY,WAAW,SAAS,EAAE,CAAC;;;;;AAM9E,eAAe,WACb,MACA,GACA,MACA,QACA,KAC4B;CAC5B,MAAM,MAAM,KAAK,SAAS,IAAI,GAAG,SAAS,OAAO,EAAE,GAAG,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC;CACpE,UAAU,KAAK,EAAE,WAAW,KAAK,CAAC;CAClC,MAAM,QAAQ,OAAO,SAAmB;EACtC,MAAM,IAAI,MAAM,MAAM,KAAK,IAAI;EAC/B,IAAI,EAAE,SAAS,GACb,MAAM,IAAI,eACR,GAAG,KAAK,iCAAiC,IAAI,IAAI,UAAU,EAAE,MAAM,KACnE,KACF;CAEJ;CACA,MAAM,MAAM;EAAC;EAAQ;EAAM;CAAQ,CAAC;CACpC,MAAM,MAAM;EAAC;EAAU;EAAqB,EAAE;CAAG,CAAC;CAClD,MAAM,MAAM;EAAC;EAAU;EAA0B;CAAM,CAAC;CAExD,MAAM,QAAQ,OAAO,MAAgB,OAAgB,SAAmB,CAAC,MAAM;EAC7E,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,MAAM,IAAI,MAAM,MACd,KACA;IAAC,GAAG;IAAQ;IAAS;IAAM;IAAa;IAAyB,GAAG;GAAI,GACxE,OACA,GACF;GACA,IAAI,EAAE,SAAS,GAAG;GAClB,MAAM,UAAU,iBAAiB,EAAE,MAAM;GACzC,IAAI,YAAY,eAAe,IAAI,OAAO,QAAQ;IAChD,MAAM,MAAM,OAAO,MAAM,CAAC;IAC1B;GACF;GACA,MAAM,WAAW,SAAS,EAAE,QAAQ,GAAG,MAAM,YAAY,SAAU,MAAM,UAAU,EAAE,GAAG,CAAE;EAC5F;CACF;CAEA,IAAI;CACJ,IAAI,EAAE,OAAO,SAAS,KAAK,EAAE,GAAG,GAAG;EAEjC,MAAM,QAAQ,MAAM,MAClB,KACA;GAAC;GAAa;GAAY;GAAM,GAAG,EAAE,IAAI;EAAU,GACnD,KAAA,GACA,EACE,mBAAmB,IACrB,CACF;EACA,IAAI,MAAM,SAAS,GAAG,SAAS,MAAM;CACvC;CACA,IAAI,CAAC,QAAQ;EACX,MAAM,OAAO,qBAAqB,QAAQ,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS,KAAK;EAE9E,MAAM,MAAM;GAAC;GAAW;GAAK,GADd,EAAE,OAAO,CAAC,oBAAoB,IAAI,CAAC;GACV;GAAU,IAAI,EAAE,OAAO,OAAO,GAAG;EAAM,CAAC;EAChF,MAAM,WAAW,MAAM,MAAM,KAAK;GAAC;GAAa;GAAY;GAAM,GAAG,KAAK;EAAU,CAAC;EACrF,MAAM,MAAM,KAAK;GAAC;GAAc;GAAM;EAAI,CAAC;EAC3C,IAAI,SAAS,SAAS,GACpB,MAAM,IAAI,eACR,GAAG,KAAK,SAAS,EAAE,IAAI,iDACvB,IACF;EAEF,SAAS,SAAS;CACpB;CAEA,MAAM,MAAM,KAAK;EAAC;EAAc,uBAAuB;EAAU;CAAM,CAAC;CAExE,MAAM,OAAO,MAAM,OAAO,KAAK,QAAQ,GAAG,IAAI;CAC9C,MAAM,MAAM,KAAK,SAAS,IAAI,GAAG,SAAS,GAAG,KAAK,KAAK;CACvD,IAAI,CAAC,WAAW,GAAG,GAAG;EAEpB,MAAM,QAAO,MADS,MAAM,KAAK;GAAC;GAAY;GAAa;GAAmB;EAAI,CAAC,EAAA,CAC9D,KAClB,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,WAAW,GAAG,CAAC,CAAC,CAChC,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC;EAExB,IAAI,KAAK,QACP,MAAM,MAAM;GAAC;GAAsB;GAAW;EAAQ,GAAG,GAAG,KAAK,KAAK,IAAI,EAAE,KAAK,CAC/E,MACA,iCACF,CAAC;EAGH,MAAM,WAAW,MAAM,IACrB,KACA,CACE,GAAG;GAAC;GAAM;GAAuB;GAAM;EAAuB,GAC9D,GAAG;GAAC;GAAM;GAA6B;GAAW;GAAgB;EAAI,CACxE,GACA,EAAE,KAAK,QAAQ,CACjB;EACA,IAAI,SAAS,SAAS,GACpB,MAAM,IAAI,eACR,GAAG,KAAK,oBAAoB,KAAK,MAAM,GAAG,EAAE,EAAE,IAAI,UAAU,SAAS,MAAM,KAC3E,KACF;EAEF,UAAU,KAAK,SAAS,IAAI,GAAG,OAAO,GAAG,EAAE,WAAW,KAAK,CAAC;EAC5D,MAAM,UAAU,GAAG,IAAI,GAAG,QAAQ,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS,KAAK;EACtE,cAAc,SAAS,SAAS,MAAM;EACtC,WAAW,SAAS,GAAG;CACzB;CACA,OAAO;EAAE;EAAQ;EAAM;CAAI;AAC7B;;AAGA,eAAe,OAAO,KAAa,QAAgB,GAAiB,MAAc;CAChF,MAAM,KAAK,GAAG,EAAE,MAAM,MAAM,EAAE,IAAI,KAAK,GAAG,UAAU,OAAO,MAAM,GAAG,CAAC,EAAE;CACvE,IAAI,CAAC,EAAE,MAAM,QAAQ,MAAM,MAAM,KAAK,CAAC,aAAa,GAAG,OAAO,QAAQ,CAAC,EAAA,CAAG;CAC1E,MAAM,QAAQ,MAAM,MAAM,KAAK;EAAC;EAAW;EAAQ;EAAM,EAAE;CAAI,CAAC;CAChE,MAAM,CAAC,OAAO,IAAI,QAAQ,MAAM,KAAK,MAAM,GAAI;CAC/C,MAAM,GAAG,MAAM,OAAO,KAAK,MAAM,GAAG;CACpC,IAAI,MAAM,SAAS,KAAK,SAAS,EAAE,QAAQ,SAAS,UAAU,KAAK,OAAO;CAC1E,IAAI,MAAM,SAAS,KAAK,SAAS,EAAE,MACjC,MAAM,IAAI,eAAe,GAAG,KAAK,GAAG,GAAG,KAAK,EAAE,KAAK,+BAA+B,IAAI;CAExF,IAAI,SAAS,MAAM,QAAQ,EAAE,IAAI;CACjC,IAAI,OAAO,MAAM,MAAM,KAAK,CAAC,WAAW,WAAW,MAAM,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;CACxF,IAAI,KAAK,SAAS,GAAG;EACnB,SAAS;EACT,OAAO,MAAM,MAAM,KAAK,CAAC,WAAW,MAAM,CAAC;CAC7C;CACA,MAAM,QACJ,KAAK,SAAS,IACV,KAAK,KACF,MAAM,IAAI,CAAC,CACX,OAAO,OAAO,CAAC,CACf,KAAK,MAAM;EACV,MAAM,CAAC,IAAI,IAAI,IAAI,MAAM,EAAE,MAAM,GAAI;EACrC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,OAAO,SAAS,GAAG,EAAE,KAAK;CAChD,CAAC,IACH,CAAC;CACP,MAAM,QAAQ,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI,KAAK,MAAM,SAAS,KAAK,QAAQ;CAC3E,MAAM,QAAQ,WAAW,MAAM,sBAAsB,GAAG,OAAO;CAC/D,MAAM,IAAI,eACR,GAAG,KAAK,GAAG,GAAG,qBAAqB,EAAE,KAAK,GAAG,MAAM,SAAS,KAAK,MAAM,IAAI,UAAU,MACrF,IACF;AACF;;AAGA,eAAe,UAAU,KAAa;CACpC,IAAI,CAAC,WAAW,GAAG,GAAG,OAAO;CAC7B,MAAM,IAAI,MAAM,MAAM,KAAK,CAAC,aAAa,yBAAyB,CAAC;CACnE,OAAO,EAAE,SAAS,KAAK,EAAE,SAAS;AACpC;AAEA,SAAS,WACP,SACA,QACA,GACA,MACA,UAAU,OACV;CACA,MAAM,OAAO,UAAU,MAAM;CAC7B,QAAQ,SAAR;EACE,KAAK,OAAO;GACV,MAAM,QACJ,EAAE,OAAO,mBAAmB,KAAK,EAAE,GAAG,IAClC,iDACA;GACN,MAAM,QAAQ,UACV,IAAI,EAAE,KAAK,+EAA+E,EAAE,OAAO,OAAO,qEAC1G;GACJ,OAAO,IAAI,eACT,GAAG,KAAK,+BAA+B,EAAE,OAAO,OAAO,kCAAkC,QAAQ,SACjG,IACF;EACF;EACA,KAAK,QACH,OAAO,IAAI,eACT,GAAG,KAAK,0EAA0E,KAAK,uKACvF,IACF;EACF,KAAK,OACH,OAAO,IAAI,eACT,GAAG,KAAK,uBAAuB,KAAK,oFACpC,IACF;EACF,KAAK,aACH,OAAO,IAAI,eAAe,GAAG,KAAK,yBAAyB,KAAK,IAAI,KAAK;EAC3E,SACE,OAAO,IAAI,eAAe,GAAG,KAAK,wBAAwB,KAAK,IAAI,KAAK;CAC5E;AACF;;;;;AAWA,SAAS,SAAS,KAAa,OAAkC;CAC/D,MAAM,IAAI,OAAO,QAAQ,IAAI,oBAAoB,CAAC,KAAK;CACvD,MAAM,SAAS,OAAO,KAAK,kBAAkB,OAAO,CAAC,CAAC,SAAS,QAAQ;CACvE,eAAe,MAAM;CACrB,OAAO;EACL,kBAAkB,OAAO,IAAI,CAAC;GAC7B,kBAAkB,MAAM,QAAQ,IAAI,IAAI,GAAG,CAAC,CAAC,OAAO;GACpD,oBAAoB,MAAM,wBAAwB;CACrD;AACF;;;;;AAMA,eAAsB,kBACpB,MACA,SAC6B;CAC7B,MAAM,2BAAW,IAAI,IAA0D;CAC/E,KAAK,MAAM,KAAK,MAAM;EACpB,MAAM,MAAM,aAAa,EAAE,SAAS;EACpC,MAAM,QAAQ,SAAS,IAAI,GAAG,KAAK;GAAE,WAAW,EAAE;GAAW,OAAO,CAAC;EAAE;EACvE,MAAM,KAAK,eAAe,EAAE;EAC5B,IAAI,CAAC,MAAM,MAAM,SAAS,EAAE,GAAG,MAAM,MAAM,KAAK,EAAE;EAClD,SAAS,IAAI,KAAK,KAAK;CACzB;CACA,MAAM,sBAAM,IAAI,IAA8B;CAE9C,MAAM,wBAAQ,IAAI,IAAiE;CACnF,KAAK,MAAM,KAAK,SAAS,OAAO,GAC9B,MAAM,IAAI,EAAE,UAAU,KAAK,CAAC,GAAI,MAAM,IAAI,EAAE,UAAU,GAAG,KAAK,CAAC,GAAI,CAAC,CAAC;CAEvE,MAAM,QAAQ,IACZ,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,SAAS;EACtC,KAAK,MAAM,EAAE,WAAW,WAAW,MAAM;GACvC,MAAM,QAAQ,eAAe,SAAS;GACtC,MAAM,OAAO,iBAAiB,KAAK,UAAU,KAAK,EAAE;GACpD,MAAM,MAAM,aAAa,SAAS;GAClC,IAAI;IACF,IAAI,MAAyB,CAAC;IAC9B,IAAI,UAAU,OAAO;KACnB,MAAM,QAAQ,QAAQ,IAAI,UAAU;KACpC,IAAI,CAAC,OACH,MAAM,IAAI,eACR,GAAG,KAAK,uBAAuB,UAAU,MAAM,uDAC/C,IACF;KAEF,eAAe,KAAK;KACpB,MAAM,SAAS,UAAU,KAAK,KAAK;IACrC;IACA,MAAM,SAAS,QAAQ,UAAU;IACjC,MAAM,WAAW,MAAM,WAAW,QAAQ,MAAM,WAAW,MAAM,QAAQ,GAAG;IAC5E,IAAI,IAAI,KAAK;KAAE,IAAI;KAAM;KAAW;IAAS,CAAC;GAChD,SAAS,OAAO;IACd,IAAI,EAAE,iBAAiB,iBAAiB,MAAM;IAC9C,MAAM,UAA0B;KAC9B,OAAO,MAAM,QAAQ,WAAW;KAChC,OAAO;KACP,KAAK;KACL,OAAO;KACP,KAAK;KACL,SAAS,MAAM;KACf,OAAO,MAAM;KACb;IACF;IACA,IAAI,IAAI,KAAK;KAAE,IAAI;KAAO;KAAW;KAAS;KAAO;IAAM,CAAC;GAC9D;EACF;CACF,CAAC,CACH;CACA,MAAM,WAAW,CAAC,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAqB;CAC/E,IAAI,CAAC,SAAS,QAAQ,OAAO;CAC7B,MAAM,OAAO,aAAa,QAAQ,MAAM,QAAQ,KAAK;CACrD,UAAU,KAAK,MAAM,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;CAC/C,MAAM,MAAM,OAAO,YACjB,SAAS,KAAK,MAAM,CAClB,aAAa,EAAE,SAAS,GACxB,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,IAAI;EAAE,SAAS,EAAE;EAAS,OAAO,EAAE;EAAO,OAAO,EAAE;CAAM,CACzF,CAAC,CACH;CACA,cAAc,MAAM,KAAK,UAAU,GAAG,CAAC;CACvC,OAAO;AACT;;;;;AAWA,SAAgB,aAAa,MAAgB,UAAwC;CACnF,MAAM,wBAAQ,IAAI,IAAoB;CACtC,KAAK,MAAM,KAAK,UAAU,IAAI,EAAE,IAAI,MAAM,IAAI,aAAa,EAAE,SAAS,GAAG,EAAE,SAAS,IAAI;CACxF,MAAM,2BAAW,IAAI,IAAuD;CAC5E,KAAK,MAAM,EAAE,IAAI,SAAS,OAAO,MAAM;EACrC,IAAI,EAAE,KAAK,SAAS,aAAa,CAAC,EAAE,WAAW;EAC/C,SAAS,IAAI,GAAG,GAAG,IAAI,EAAE,KAAK,WAAW;GAAE,MAAM,EAAE,KAAK;GAAa,WAAW,EAAE;EAAU,CAAC;CAC/F;CACA,MAAM,sBAAM,IAAI,IAAY;CAC5B,KAAK,MAAM,EAAE,IAAI,SAAS,OAAO,MAAM;EACrC,MAAM,UAAU,SAAS,IAAI,GAAG,GAAG,IAAI,EAAE,KAAK,SAAS;EACvD,IAAI,EAAE,KAAK,SAAS,aAAa,CAAC,EAAE,aAAa,CAAC,SAAS;EAC3D,MAAM,CAAC,MAAM,UAAU,CAAC,aAAa,EAAE,SAAS,GAAG,aAAa,QAAQ,SAAS,CAAC;EAClF,MAAM,OAAO,MAAM,IAAI,IAAI;EAC3B,IAAI,SAAS,UAAU,CAAC,QAAQ,SAAS,MAAM,IAAI,MAAM,GAAG;EAC5D,MAAM,SAAS,MAAoB,OAAO,KAAK,UAAU,eAAe,CAAC,CAAC,EAAE;EAC5E,IAAI,IACF,uBAAuB,GAAG,IAAI,EAAE,KAAK,YAAY,OAAO,QAAQ,KAAK,6BAA6B,KAAK,MAAM,GAAG,EAAE,EAAE,KAAK,MAAM,EAAE,SAAS,EAAE,OAAO,MAAM,QAAQ,SAAS,EAAE,gGAC9K;CACF;CACA,OAAO,CAAC,GAAG,GAAG;AAChB;AAEA,MAAa,4BAA4B,MAAc,UACrD,OAAO,aAAa,MAAM,KAAK,GAAG,EAAE,OAAO,KAAK,CAAC;AAMnD,MAAM,uBAAO,IAAI,IAA0C;;;;;AAM3D,SAAgB,kBACd,MACA,OACA,GAC+B;CAC/B,MAAM,OAAO,aAAa,MAAM,KAAK;CACrC,IAAI,CAAC,KAAK,IAAI,IAAI,GAAG;EACnB,IAAI,MAAoC;EACxC,IAAI;GACF,MAAM,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;EAC7C,QAAQ,CAAC;EACT,KAAK,IAAI,MAAM,GAAG;CACpB;CACA,MAAM,QAAQ,KAAK,IAAI,IAAI,CAAC,GAAG,aAAa,CAAC;CAC7C,IAAI,CAAC,OAAO,OAAO,KAAA;CACnB,IAAI,aAAa,OACf,MAAM,IAAI,aAAa;EACrB,GAAG,MAAM;EACT,SAAS,GAAG,MAAM,QAAQ,QAAQ,uBAAuB,KAAK,UAAU,MAAM,KAAK,EAAE,YAAY,MAAM,MAAM,KAAK,IAAI,EAAE;CAC1H,CAAC;CAEH,OAAO,MAAM;AACf"}
1
+ {"version":3,"file":"workspace-DwTe1zo6.js","names":[],"sources":["../src/workspace.ts"],"mappings":";;;;;;;;;;;AAsBA,IAAa,iBAAb,cAAoC,MAAM;CACxC,OAAgB;CAChB;CACA,YAAY,SAAiB,OAAgB;EAC3C,MAAM,OAAO;EACb,KAAK,QAAQ;CACf;AACF;AAEA,MAAa,gBAAgB,MAC3B,KAAK,UAAU;CAAC,EAAE;CAAK,EAAE,OAAO;CAAM,EAAE,QAAQ;AAAI,CAAC;AAEvD,MAAM,YAAY,SAAiB,KAAK,MAAM,WAAW,SAAS,KAAK;AACvE,MAAM,gBAAgB,MAAc,UAClC,KAAK,MAAM,WAAW,SAAS,YAAY,GAAG,MAAM,UAAU;AAEhE,MAAM,WAAW;;AAKjB,SAAgB,iBAAiB,QAAyB;CACxD,IAAI,4EAA4E,KAAK,MAAM,GACzF,OAAO;CACT,IACE,wLAAwL,KACtL,MACF,GAEA,OAAO;CACT,IACE,wOAAwO,KACtO,MACF,GAEA,OAAO;CACT,IACE,sMAAsM,KACpM,MACF,GAEA,OAAO;CACT,OAAO;AACT;AAEA,MAAM,aAAa,WACjB,OACG,MAAM,IAAI,CAAC,CACX,KAAK,MAAM,EAAE,QAAQ,sBAAsB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CACtD,KAAK,OAAO,KAAK;AAGtB,MAAM,UAA6B,QAAQ,IAAI,kBAC3C,CAAC,IACD,EAAE,iBAAiB,uBAAuB;AAE9C,eAAe,MAAM,KAAa,MAAgB,OAAgB,MAAyB,CAAC,GAAG;CAC7F,MAAM,IAAI,MAAM,IAAI,KAAK,MAAM;EAC7B,KAAK;GAAE,GAAG;GAAS,GAAG;EAAI;EAC1B,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;CACzC,CAAC;CACD,OAAO;EAAE,GAAG;EAAG,MAAM,EAAE,OAAO,SAAS,MAAM,CAAC,CAAC,KAAK;CAAE;AACxD;AAEA,MAAM,SAAS,OAAe,IAAI,SAAS,YAAY,WAAW,SAAS,EAAE,CAAC;;;;;AAM9E,eAAe,WACb,MACA,GACA,MACA,QACA,KAC4B;CAC5B,MAAM,MAAM,KAAK,SAAS,IAAI,GAAG,SAAS,OAAO,EAAE,GAAG,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC;CACpE,UAAU,KAAK,EAAE,WAAW,KAAK,CAAC;CAClC,MAAM,QAAQ,OAAO,SAAmB;EACtC,MAAM,IAAI,MAAM,MAAM,KAAK,IAAI;EAC/B,IAAI,EAAE,SAAS,GACb,MAAM,IAAI,eACR,GAAG,KAAK,iCAAiC,IAAI,IAAI,UAAU,EAAE,MAAM,KACnE,KACF;CAEJ;CACA,MAAM,MAAM;EAAC;EAAQ;EAAM;CAAQ,CAAC;CACpC,MAAM,MAAM;EAAC;EAAU;EAAqB,EAAE;CAAG,CAAC;CAClD,MAAM,MAAM;EAAC;EAAU;EAA0B;CAAM,CAAC;CAExD,MAAM,QAAQ,OAAO,MAAgB,OAAgB,SAAmB,CAAC,MAAM;EAC7E,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,MAAM,IAAI,MAAM,MACd,KACA;IAAC,GAAG;IAAQ;IAAS;IAAM;IAAa;IAAyB,GAAG;GAAI,GACxE,OACA,GACF;GACA,IAAI,EAAE,SAAS,GAAG;GAClB,MAAM,UAAU,iBAAiB,EAAE,MAAM;GACzC,IAAI,YAAY,eAAe,IAAI,OAAO,QAAQ;IAChD,MAAM,MAAM,OAAO,MAAM,CAAC;IAC1B;GACF;GACA,MAAM,WAAW,SAAS,EAAE,QAAQ,GAAG,MAAM,YAAY,SAAU,MAAM,UAAU,EAAE,GAAG,CAAE;EAC5F;CACF;CAEA,IAAI;CACJ,IAAI,EAAE,OAAO,SAAS,KAAK,EAAE,GAAG,GAAG;EAEjC,MAAM,QAAQ,MAAM,MAClB,KACA;GAAC;GAAa;GAAY;GAAM,GAAG,EAAE,IAAI;EAAU,GACnD,KAAA,GACA,EACE,mBAAmB,IACrB,CACF;EACA,IAAI,MAAM,SAAS,GAAG,SAAS,MAAM;CACvC;CACA,IAAI,CAAC,QAAQ;EACX,MAAM,OAAO,qBAAqB,QAAQ,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS,KAAK;EAE9E,MAAM,MAAM;GAAC;GAAW;GAAK,GADd,EAAE,OAAO,CAAC,oBAAoB,IAAI,CAAC;GACV;GAAU,IAAI,EAAE,OAAO,OAAO,GAAG;EAAM,CAAC;EAChF,MAAM,WAAW,MAAM,MAAM,KAAK;GAAC;GAAa;GAAY;GAAM,GAAG,KAAK;EAAU,CAAC;EACrF,MAAM,MAAM,KAAK;GAAC;GAAc;GAAM;EAAI,CAAC;EAC3C,IAAI,SAAS,SAAS,GACpB,MAAM,IAAI,eACR,GAAG,KAAK,SAAS,EAAE,IAAI,iDACvB,IACF;EAEF,SAAS,SAAS;CACpB;CAEA,MAAM,MAAM,KAAK;EAAC;EAAc,uBAAuB;EAAU;CAAM,CAAC;CAExE,MAAM,OAAO,MAAM,OAAO,KAAK,QAAQ,GAAG,IAAI;CAC9C,MAAM,MAAM,KAAK,SAAS,IAAI,GAAG,SAAS,GAAG,KAAK,KAAK;CACvD,IAAI,CAAC,WAAW,GAAG,GAAG;EAEpB,MAAM,QAAO,MADS,MAAM,KAAK;GAAC;GAAY;GAAa;GAAmB;EAAI,CAAC,EAAA,CAC9D,KAClB,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,WAAW,GAAG,CAAC,CAAC,CAChC,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC;EAExB,IAAI,KAAK,QACP,MAAM,MAAM;GAAC;GAAsB;GAAW;EAAQ,GAAG,GAAG,KAAK,KAAK,IAAI,EAAE,KAAK,CAC/E,MACA,iCACF,CAAC;EAGH,MAAM,WAAW,MAAM,IACrB,KACA,CACE,GAAG;GAAC;GAAM;GAAuB;GAAM;EAAuB,GAC9D,GAAG;GAAC;GAAM;GAA6B;GAAW;GAAgB;EAAI,CACxE,GACA,EAAE,KAAK,QAAQ,CACjB;EACA,IAAI,SAAS,SAAS,GACpB,MAAM,IAAI,eACR,GAAG,KAAK,oBAAoB,KAAK,MAAM,GAAG,EAAE,EAAE,IAAI,UAAU,SAAS,MAAM,KAC3E,KACF;EAEF,UAAU,KAAK,SAAS,IAAI,GAAG,OAAO,GAAG,EAAE,WAAW,KAAK,CAAC;EAC5D,MAAM,UAAU,GAAG,IAAI,GAAG,QAAQ,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS,KAAK;EACtE,cAAc,SAAS,SAAS,MAAM;EACtC,WAAW,SAAS,GAAG;CACzB;CACA,OAAO;EAAE;EAAQ;EAAM;CAAI;AAC7B;;AAGA,eAAe,OAAO,KAAa,QAAgB,GAAiB,MAAc;CAChF,MAAM,KAAK,GAAG,EAAE,MAAM,MAAM,EAAE,IAAI,KAAK,GAAG,UAAU,OAAO,MAAM,GAAG,CAAC,EAAE;CACvE,IAAI,CAAC,EAAE,MAAM,QAAQ,MAAM,MAAM,KAAK,CAAC,aAAa,GAAG,OAAO,QAAQ,CAAC,EAAA,CAAG;CAC1E,MAAM,QAAQ,MAAM,MAAM,KAAK;EAAC;EAAW;EAAQ;EAAM,EAAE;CAAI,CAAC;CAChE,MAAM,CAAC,OAAO,IAAI,QAAQ,MAAM,KAAK,MAAM,GAAI;CAC/C,MAAM,GAAG,MAAM,OAAO,KAAK,MAAM,GAAG;CACpC,IAAI,MAAM,SAAS,KAAK,SAAS,EAAE,QAAQ,SAAS,UAAU,KAAK,OAAO;CAC1E,IAAI,MAAM,SAAS,KAAK,SAAS,EAAE,MACjC,MAAM,IAAI,eAAe,GAAG,KAAK,GAAG,GAAG,KAAK,EAAE,KAAK,+BAA+B,IAAI;CAExF,IAAI,SAAS,MAAM,QAAQ,EAAE,IAAI;CACjC,IAAI,OAAO,MAAM,MAAM,KAAK,CAAC,WAAW,WAAW,MAAM,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;CACxF,IAAI,KAAK,SAAS,GAAG;EACnB,SAAS;EACT,OAAO,MAAM,MAAM,KAAK,CAAC,WAAW,MAAM,CAAC;CAC7C;CACA,MAAM,QACJ,KAAK,SAAS,IACV,KAAK,KACF,MAAM,IAAI,CAAC,CACX,OAAO,OAAO,CAAC,CACf,KAAK,MAAM;EACV,MAAM,CAAC,IAAI,IAAI,IAAI,MAAM,EAAE,MAAM,GAAI;EACrC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,OAAO,SAAS,GAAG,EAAE,KAAK;CAChD,CAAC,IACH,CAAC;CACP,MAAM,QAAQ,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI,KAAK,MAAM,SAAS,KAAK,QAAQ;CAC3E,MAAM,QAAQ,WAAW,MAAM,sBAAsB,GAAG,OAAO;CAC/D,MAAM,IAAI,eACR,GAAG,KAAK,GAAG,GAAG,qBAAqB,EAAE,KAAK,GAAG,MAAM,SAAS,KAAK,MAAM,IAAI,UAAU,MACrF,IACF;AACF;;AAGA,eAAe,UAAU,KAAa;CACpC,IAAI,CAAC,WAAW,GAAG,GAAG,OAAO;CAC7B,MAAM,IAAI,MAAM,MAAM,KAAK,CAAC,aAAa,yBAAyB,CAAC;CACnE,OAAO,EAAE,SAAS,KAAK,EAAE,SAAS;AACpC;AAEA,SAAS,WACP,SACA,QACA,GACA,MACA,UAAU,OACV;CACA,MAAM,OAAO,UAAU,MAAM;CAC7B,QAAQ,SAAR;EACE,KAAK,OAAO;GACV,MAAM,QACJ,EAAE,OAAO,mBAAmB,KAAK,EAAE,GAAG,IAClC,iDACA;GACN,MAAM,QAAQ,UACV,IAAI,EAAE,KAAK,+EAA+E,EAAE,OAAO,OAAO,qEAC1G;GACJ,OAAO,IAAI,eACT,GAAG,KAAK,+BAA+B,EAAE,OAAO,OAAO,kCAAkC,QAAQ,SACjG,IACF;EACF;EACA,KAAK,QACH,OAAO,IAAI,eACT,GAAG,KAAK,0EAA0E,KAAK,uKACvF,IACF;EACF,KAAK,OACH,OAAO,IAAI,eACT,GAAG,KAAK,uBAAuB,KAAK,oFACpC,IACF;EACF,KAAK,aACH,OAAO,IAAI,eAAe,GAAG,KAAK,yBAAyB,KAAK,IAAI,KAAK;EAC3E,SACE,OAAO,IAAI,eAAe,GAAG,KAAK,wBAAwB,KAAK,IAAI,KAAK;CAC5E;AACF;;;;;AAWA,SAAS,SAAS,KAAa,OAAkC;CAC/D,MAAM,IAAI,OAAO,QAAQ,IAAI,oBAAoB,CAAC,KAAK;CACvD,MAAM,SAAS,OAAO,KAAK,kBAAkB,OAAO,CAAC,CAAC,SAAS,QAAQ;CACvE,eAAe,MAAM;CACrB,OAAO;EACL,kBAAkB,OAAO,IAAI,CAAC;GAC7B,kBAAkB,MAAM,QAAQ,IAAI,IAAI,GAAG,CAAC,CAAC,OAAO;GACpD,oBAAoB,MAAM,wBAAwB;CACrD;AACF;;;;;AAMA,eAAsB,kBACpB,MACA,SAC6B;CAC7B,MAAM,2BAAW,IAAI,IAA0D;CAC/E,KAAK,MAAM,KAAK,MAAM;EACpB,MAAM,MAAM,aAAa,EAAE,SAAS;EACpC,MAAM,QAAQ,SAAS,IAAI,GAAG,KAAK;GAAE,WAAW,EAAE;GAAW,OAAO,CAAC;EAAE;EACvE,MAAM,KAAK,eAAe,EAAE;EAC5B,IAAI,CAAC,MAAM,MAAM,SAAS,EAAE,GAAG,MAAM,MAAM,KAAK,EAAE;EAClD,SAAS,IAAI,KAAK,KAAK;CACzB;CACA,MAAM,sBAAM,IAAI,IAA8B;CAE9C,MAAM,wBAAQ,IAAI,IAAiE;CACnF,KAAK,MAAM,KAAK,SAAS,OAAO,GAC9B,MAAM,IAAI,EAAE,UAAU,KAAK,CAAC,GAAI,MAAM,IAAI,EAAE,UAAU,GAAG,KAAK,CAAC,GAAI,CAAC,CAAC;CAEvE,MAAM,QAAQ,IACZ,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,SAAS;EACtC,KAAK,MAAM,EAAE,WAAW,WAAW,MAAM;GACvC,MAAM,QAAQ,eAAe,SAAS;GACtC,MAAM,OAAO,iBAAiB,KAAK,UAAU,KAAK,EAAE;GACpD,MAAM,MAAM,aAAa,SAAS;GAClC,IAAI;IACF,IAAI,MAAyB,CAAC;IAC9B,IAAI,UAAU,OAAO;KACnB,MAAM,QAAQ,QAAQ,IAAI,UAAU;KACpC,IAAI,CAAC,OACH,MAAM,IAAI,eACR,GAAG,KAAK,uBAAuB,UAAU,MAAM,uDAC/C,IACF;KAEF,eAAe,KAAK;KACpB,MAAM,SAAS,UAAU,KAAK,KAAK;IACrC;IACA,MAAM,SAAS,QAAQ,UAAU;IACjC,MAAM,WAAW,MAAM,WAAW,QAAQ,MAAM,WAAW,MAAM,QAAQ,GAAG;IAC5E,IAAI,IAAI,KAAK;KAAE,IAAI;KAAM;KAAW;IAAS,CAAC;GAChD,SAAS,OAAO;IACd,IAAI,EAAE,iBAAiB,iBAAiB,MAAM;IAC9C,MAAM,UAA0B;KAC9B,OAAO,MAAM,QAAQ,WAAW;KAChC,OAAO;KACP,KAAK;KACL,OAAO;KACP,KAAK;KACL,SAAS,MAAM;KACf,OAAO,MAAM;KACb;IACF;IACA,IAAI,IAAI,KAAK;KAAE,IAAI;KAAO;KAAW;KAAS;KAAO;IAAM,CAAC;GAC9D;EACF;CACF,CAAC,CACH;CACA,MAAM,WAAW,CAAC,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAqB;CAC/E,IAAI,CAAC,SAAS,QAAQ,OAAO;CAC7B,MAAM,OAAO,aAAa,QAAQ,MAAM,QAAQ,KAAK;CACrD,UAAU,KAAK,MAAM,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;CAC/C,MAAM,MAAM,OAAO,YACjB,SAAS,KAAK,MAAM,CAClB,aAAa,EAAE,SAAS,GACxB,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,IAAI;EAAE,SAAS,EAAE;EAAS,OAAO,EAAE;EAAO,OAAO,EAAE;CAAM,CACzF,CAAC,CACH;CACA,cAAc,MAAM,KAAK,UAAU,GAAG,CAAC;CACvC,OAAO;AACT;;;;;AAWA,SAAgB,aAAa,MAAgB,UAAwC;CACnF,MAAM,wBAAQ,IAAI,IAAoB;CACtC,KAAK,MAAM,KAAK,UAAU,IAAI,EAAE,IAAI,MAAM,IAAI,aAAa,EAAE,SAAS,GAAG,EAAE,SAAS,IAAI;CACxF,MAAM,2BAAW,IAAI,IAAuD;CAC5E,KAAK,MAAM,EAAE,IAAI,SAAS,OAAO,MAAM;EACrC,IAAI,EAAE,KAAK,SAAS,aAAa,CAAC,EAAE,WAAW;EAC/C,SAAS,IAAI,GAAG,GAAG,IAAI,EAAE,KAAK,WAAW;GAAE,MAAM,EAAE,KAAK;GAAa,WAAW,EAAE;EAAU,CAAC;CAC/F;CACA,MAAM,sBAAM,IAAI,IAAY;CAC5B,KAAK,MAAM,EAAE,IAAI,SAAS,OAAO,MAAM;EACrC,MAAM,UAAU,SAAS,IAAI,GAAG,GAAG,IAAI,EAAE,KAAK,SAAS;EACvD,IAAI,EAAE,KAAK,SAAS,aAAa,CAAC,EAAE,aAAa,CAAC,SAAS;EAC3D,MAAM,CAAC,MAAM,UAAU,CAAC,aAAa,EAAE,SAAS,GAAG,aAAa,QAAQ,SAAS,CAAC;EAClF,MAAM,OAAO,MAAM,IAAI,IAAI;EAC3B,IAAI,SAAS,UAAU,CAAC,QAAQ,SAAS,MAAM,IAAI,MAAM,GAAG;EAC5D,MAAM,SAAS,MAAoB,OAAO,KAAK,UAAU,eAAe,CAAC,CAAC,EAAE;EAC5E,IAAI,IACF,uBAAuB,GAAG,IAAI,EAAE,KAAK,YAAY,OAAO,QAAQ,KAAK,6BAA6B,KAAK,MAAM,GAAG,EAAE,EAAE,KAAK,MAAM,EAAE,SAAS,EAAE,OAAO,MAAM,QAAQ,SAAS,EAAE,gGAC9K;CACF;CACA,OAAO,CAAC,GAAG,GAAG;AAChB;AAEA,MAAa,4BAA4B,MAAc,UACrD,OAAO,aAAa,MAAM,KAAK,GAAG,EAAE,OAAO,KAAK,CAAC;AAMnD,MAAM,uBAAO,IAAI,IAA0C;;;;;AAM3D,SAAgB,kBACd,MACA,OACA,GAC+B;CAC/B,MAAM,OAAO,aAAa,MAAM,KAAK;CACrC,IAAI,CAAC,KAAK,IAAI,IAAI,GAAG;EACnB,IAAI,MAAoC;EACxC,IAAI;GACF,MAAM,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;EAC7C,QAAQ,CAAC;EACT,KAAK,IAAI,MAAM,GAAG;CACpB;CACA,MAAM,QAAQ,KAAK,IAAI,IAAI,CAAC,GAAG,aAAa,CAAC;CAC7C,IAAI,CAAC,OAAO,OAAO,KAAA;CACnB,IAAI,aAAa,OACf,MAAM,IAAI,aAAa;EACrB,GAAG,MAAM;EACT,SAAS,GAAG,MAAM,QAAQ,QAAQ,uBAAuB,KAAK,UAAU,MAAM,KAAK,EAAE,YAAY,MAAM,MAAM,KAAK,IAAI,EAAE;CAC1H,CAAC;CAEH,OAAO,MAAM;AACf"}
package/docker/Dockerfile CHANGED
@@ -1,6 +1,6 @@
1
1
  # Base image for reeval sandboxes. `reeval image build` builds this, bakes the Cursor harness
2
2
  # bootstrap into it and commits the result as the image evals run in.
3
- FROM node:26-bookworm-slim
3
+ FROM node:26-bookworm-slim@sha256:add5c17c14752c54d2f3d519ec46a3815684d41c33f198bcb09fb998f7dcbf68
4
4
 
5
5
  RUN apt-get update \
6
6
  && apt-get install -y --no-install-recommends bash ca-certificates curl git procps \
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "reeval",
3
- "version": "0.20.0",
3
+ "version": "0.22.0",
4
4
  "description": "reeval: an eval runner for coding agents",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Invertase <oss@invertase.io>",
@@ -38,6 +38,14 @@
38
38
  "types": "./dist/sandbox/boat.d.ts",
39
39
  "default": "./dist/sandbox/boat.js"
40
40
  },
41
+ "./store/local": {
42
+ "types": "./dist/store/local.d.ts",
43
+ "default": "./dist/store/local.js"
44
+ },
45
+ "./store/s3": {
46
+ "types": "./dist/store/s3.d.ts",
47
+ "default": "./dist/store/s3.js"
48
+ },
41
49
  "./judge": {
42
50
  "types": "./dist/judge.d.ts",
43
51
  "default": "./dist/judge.js"
@@ -63,6 +71,7 @@
63
71
  "assets",
64
72
  "skills",
65
73
  "README.md",
74
+ "CHANGELOG.md",
66
75
  "THIRD_PARTY_NOTICES.md"
67
76
  ],
68
77
  "preferUnplugged": true,
@@ -81,6 +90,7 @@
81
90
  "@ai-sdk/provider-utils": "5.0.73",
82
91
  "@ai-sdk/typesafe-ai": "3.0.32",
83
92
  "@balena/dockerignore": "1.0.2",
93
+ "@bradenmacdonald/s3-lite-client": "1.0.1",
84
94
  "@hono/node-server": "2.1.5",
85
95
  "@modelcontextprotocol/server": "2.4.0",
86
96
  "@openrouter/ai-sdk-provider": "3.1.0",
@@ -93,10 +103,14 @@
93
103
  "hono": "4.13.13",
94
104
  "ink": "8.0.0",
95
105
  "jiti": "2.7.0",
106
+ "p-queue": "10.0.0",
107
+ "p-retry": "9.0.0",
96
108
  "picomatch": "4.0.7",
97
109
  "react": "19.3.1",
110
+ "shell-quote": "1.12.0",
98
111
  "std-env": "4.3.0",
99
112
  "string-width": "8.3.0",
113
+ "tarn": "3.1.2",
100
114
  "tinyglobby": "0.2.17",
101
115
  "ws": "8.22.0",
102
116
  "yaml": "2.9.1",
@@ -88,6 +88,16 @@ about what happened in the run, so it prints in every run it happens.
88
88
  **Fix:** broker the secret with the hosts the process sends it to, so it holds a placeholder, or
89
89
  start the process from `c.exec` after the agent's turn.
90
90
 
91
+ ## `setup-tunnel-closed`
92
+
93
+ Concern. Under `egress: { setup, agent }`, a connection the setup phase opened through the egress
94
+ gateway (a background process's CONNECT tunnel or WebSocket) was still open when the agent phase
95
+ started, and the agent's policy refuses its host, so reeval closed it. The attempt's
96
+ `egress.blocked` has it with `closed: true`. Whatever setup left running lost that connection, so
97
+ a check that relies on it may fail for a reason the agent didn't cause. It's about what happened
98
+ in the run, so it prints in every run it happens. **Fix:** stop the process before setup ends, or
99
+ allow its host in `egress.agent` if the agent's work needs it.
100
+
91
101
  ## `snapshot-too-large`
92
102
 
93
103
  Concern. An attempt's workspace was over its `snapshot` form's cap (as a directory, 50 MB of
@@ -0,0 +1,370 @@
1
+ ---
2
+ name: reeval
3
+ description: >
4
+ How to write, run and debug evals with reeval, the eval runner for coding agents (npm package
5
+ reeval; CLI `reeval`, alias `re`). Use whenever a task mentions reeval, reeval.config.ts,
6
+ *.eval.ts files, `reeval run`, evals for a coding agent, comparing skills, MCP servers or
7
+ docs, or lift. Not for changing reeval's own source: contributors follow the repository's
8
+ AGENTS.md.
9
+ ---
10
+
11
+ # reeval
12
+
13
+ This skill is for writing evals with reeval, not for changing reeval itself: to contribute to
14
+ the invertase/reeval repository, follow its root `AGENTS.md` instead.
15
+
16
+ reeval runs a real coding agent (Cursor, Claude Code, Antigravity, OpenCode, Goose, pi,
17
+ Codex, fx or Grok Build) on a task in a fresh Docker sandbox, then checks the result with
18
+ deterministic assertions and an LLM judge. You write evals in TypeScript; `reeval run` runs them.
19
+
20
+ The package and CLI are `reeval`. Full docs: the `docs/` folder of github.com/invertase/reeval (a docs.page
21
+ site; its Reference tab has every option, flag and error).
22
+
23
+ This file covers the common work: set up a project, write an eval with checks and a judge,
24
+ compare a skill, fix a config, read a failure. Read a reference file when the task needs it:
25
+
26
+ | Read | When the task involves |
27
+ |---|---|
28
+ | [references/config.md](references/config.md) | Harness factories and their keys, `effort`, judge models, `reeval.yaml`, path rules, where the sandbox comes from, imports, the options cascade |
29
+ | [references/evals.md](references/evals.md) | `cases`, `caseKey` and `where`, `gate: 'observe'`, `toStayWithin`, `c.serve`, `c.writeFile` hidden tests, the judge's inputs, the judge's input size, judge errors, every matcher |
30
+ | [references/capture.md](references/capture.md) | Web UIs: `c.capture()` screenshots, `instructions` journeys, `artifacts` on judges and questions |
31
+ | [references/compare.md](references/compare.md) | `compare` beyond the example below: the control, gates, lift and effort, labeled checks, `questions`, `skill_files` |
32
+ | [references/environments.md](references/environments.md) | Environments: skills, files, rules, MCP servers, `git()` workspaces, `directory()`, Docker sandboxes, secrets, egress |
33
+ | [references/cli.md](references/cli.md) | The CLI beyond `run`: `query` views, `analyze`, `show`, `compare`, `record`, `studio`, `mcp`, filters, exit codes |
34
+ | [references/mistakes.md](references/mistakes.md) | An error message from reeval you need to map to its fix (the full table) |
35
+ | [../eval-rubric/SKILL.md](../eval-rubric/SKILL.md) | A `Worth knowing:` list after a run, or reviewing an eval's name, prompts and judge questions: one section per remark kind, with its fix |
36
+
37
+ Use reeval's terms in code and messages: eval (`evaluate`), attempt (one run of a cell;
38
+ `runs: n`), harness (`cursor()`, …), cell (harness × environment × sandbox), control and variant,
39
+ lift (variant pass@1 − control pass@1, 95% interval), gate, checks and judge, transcript, effort
40
+ (measured tool calls, agent time, tokens). Not "trial", "baseline/treatment" or "grader";
41
+ [references/vocabulary.md](references/vocabulary.md) maps other frameworks' words to these.
42
+
43
+ ## Workflow
44
+
45
+ Setting reeval up for someone (new to reeval, or "set up reeval here")? Follow the setup
46
+ skill, [../setup/SKILL.md](../setup/SKILL.md), first: it asks what to evaluate and
47
+ writes the first evals. This section is the reference it builds on.
48
+
49
+ ```bash
50
+ npm i -D reeval
51
+ npx reeval init --harness cursor # or --bare: the config, no example eval
52
+ npx reeval run --dry-run # the plan, no keys or Docker
53
+ npx reeval image build # once, for real agents
54
+ npx reeval doctor # Docker, keys and images, with fixes
55
+ npx reeval run
56
+ ```
57
+
58
+ 1. Install: `npm init -y` if there's no `package.json`, then `npm i -D reeval`
59
+ (or a tarball: `npm i -D /path/to/reeval.tgz`). Pre-release: pin the version.
60
+ Node 22.13 or later; npm, pnpm or Yarn.
61
+ 2. `npx reeval init --harness <cursor|claude-code|antigravity|opencode|goose|pi|codex|fx|grok-build>`
62
+ (it only asks in a terminal) writes `reeval.config.ts` (the agent, `analysis.model:
63
+ 'google/gemini-3.8-flash'` for `reeval analyze`, and `pr` and `nightly` profiles), `evals/environments.ts` (one `defineEnvironment`), `evals/greet.eval.ts` (an eval
64
+ in that environment), the `.gitignore` entries (`.reeval/`, `.env*`), an `eval` script and a
65
+ `tsconfig.json` that allows the evals' `.ts` imports, then says whether Docker is running.
66
+ `--bare` writes only the config, `.gitignore` and `package.json`. Where a config exists it
67
+ exits 2 before asking; `--force` replaces only `reeval.config.ts`. Otherwise it only adds:
68
+ it merges into `package.json` and `.gitignore` and skips any other file that exists. Or write
69
+ `reeval.config.ts` and `evals/**/*.eval.ts` (the default `include`) yourself.
70
+ 3. `npx reeval run --dry-run` prints the plan (evals, cells, attempts, time and a cost range)
71
+ without running anything and needs no credentials or Docker. `--dry-run --json` for a program
72
+ to read.
73
+ 4. For real agents, and before any `tart()` run (a run never pulls its macOS image; `tart()`
74
+ runs every real harness, and the build bakes each one's CLI into a copy of the image):
75
+ `npx reeval image build` once (and after upgrading reeval), keys in
76
+ `.env.local` next to `reeval.config.ts` (not the directory you run from), `npx reeval doctor`
77
+ (exit 1 names each problem and its fix; a missing `analysis.model` or its key,
78
+ `GOOGLE_GENERATIVE_AI_API_KEY`, is only a warning; `--json`, `--online` tries the keys), then
79
+ `npx reeval run`. Exit 0 means every gated cell passed. `npx reeval run --json` prints the
80
+ results as one JSON document on stdout (each check `ran` or `not run`, each variant's
81
+ `effect`), with the progress on stderr.
82
+ 5. Read failures in the summary, then `.reeval/runs/<run>/summary.md` and each attempt's directory
83
+ (`transcript.jsonl`, `diff.patch`, `checks/<n>.log`, `judge-input.json`, `setup.log`).
84
+ `npx reeval run -v` prints a timestamped trace per attempt; `.reeval/debug/` keeps it.
85
+ Each attempt's whole final workspace is in its directory there, under `snapshots/final/`, to
86
+ read or run what the agent built.
87
+ `npx reeval query "SELECT … FROM failing_attempts"` names each failed attempt's first failed
88
+ check (more views in [references/cli.md](references/cli.md)).
89
+
90
+ Always rerun `--dry-run` after editing: config mistakes are exit-2 usage errors with a precise
91
+ message, one at a time.
92
+
93
+ ## reeval.config.ts
94
+
95
+ ```ts
96
+ import { defineConfig } from 'reeval'
97
+ import { cursor } from 'reeval/harness'
98
+
99
+ export default defineConfig({
100
+ include: ['evals/**/*.eval.ts'], // default
101
+ harnesses: cursor({ model: 'composer-2.5' }), // model required; CURSOR_API_KEY
102
+ // no default judge model; pin versions
103
+ judge: { model: 'typesafe-ai/jev-1.14.0', threshold: 0.8 },
104
+ // runs: attempts per cell; unset, 1, or 4 for a compare with variants
105
+ timeout: '15m', // per attempt
106
+ })
107
+ ```
108
+
109
+ - Harnesses (`reeval/harness`), each with a required `model`; a list runs each as its own cell:
110
+ `cursor` (`CURSOR_API_KEY`), `claudeCode` (`ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`), `antigravity`
111
+ (`GEMINI_API_KEY`), `opencode` (the provider's key; model `<provider>/<model>`), `goose` and
112
+ `pi` (the provider's key; model `<provider>/<model>`, e.g. `google/gemini-3.8-flash` on `GOOGLE_GENERATIVE_AI_API_KEY`), `codex` (`OPENAI_API_KEY`), `fx` (`OPENAI_API_KEY` for a bare id,
113
+ `AI_GATEWAY_API_KEY` with `vercel()` or `OPENROUTER_API_KEY` with `openrouter()` in `gateway`), `grokBuild` (`XAI_API_KEY`), `fakeHarness`. Details
114
+ and `effort` in [references/config.md](references/config.md).
115
+ - Judge models: `'typesafe-ai/jev-1.14.0'` (`TYPESAFE_AI_API_KEY`, recommended),
116
+ `'google/gemini-3.8-flash'` (`GOOGLE_GENERATIVE_AI_API_KEY`), `anthropic/*` (`ANTHROPIC_API_KEY`)
117
+ and `'openai/gpt-6-luna'` (`OPENAI_API_KEY`; OpenAI's Decisions API takes only its decision
118
+ model, so not `openai/gpt-5.5`), each on its provider's own key. reeval uses no gateway by
119
+ default: `AI_GATEWAY_API_KEY` is read only with `gateway: [vercel()]` in the config (or a judge
120
+ `gateway: 'vercel'` pin, listed there), `OPENROUTER_API_KEY` only with `openrouter()`. As a
121
+ judge, `vercel()` serves decision models only (Jev, `openai/gpt-6-luna-decisions`), and
122
+ `openrouter()` only those on its Decisions API (`typesafe/jev-1.14`), never a language model:
123
+ pin a language-model judge to `gateway: 'direct'`. Always a provider prefix. Every `gateway`
124
+ list ends with `direct()`, so a cell no listed gateway serves falls back to its own key (the
125
+ plan's hint says so); `direct({ enabled: false })` refuses it instead.
126
+ - Imports: `reeval` (`defineConfig`, `defineEnvironment`, `group`, `evaluate`,
127
+ `expect`, `DEFAULT_EGRESS_ALLOW`, `DEFAULT_SKILL_EXCLUDE`, types), `/harness`, `/sandbox`
128
+ (`docker`, `fakeSandbox`), `/judge` (`fakeJudge`, `RECOMMENDED_JUDGE_MODEL`,
129
+ `DEFAULT_THRESHOLD`). There is no `test`, `describe`, `baseline`, `secret` or `skill` export.
130
+ - `defineConfig`, `group` and `evaluate` take the same options (`harnesses`, `sandbox`,
131
+ `environments`, `env`, `judge`, `runs`, `gate`, `timeout`, `tags`, …) and inner levels win;
132
+ `compare` is for groups and evals only. Options are plain objects, never functions. Unknown
133
+ keys are errors with a hint (`repeat` → "Did you mean runs?"), and type errors in a
134
+ `defineConfig` object.
135
+ - `runs: { bestOf: 5 }` (odd, 3 or more) stops each cell once pass or fail has a majority (3 to 5
136
+ attempts) and gates on it, replacing `gate`. Use it to settle a flaky pass/fail for less than a
137
+ fixed count; not on an eval with `compare` (refused: lift needs equal attempts), and a cell
138
+ only reaches `flaky_cells`' 6-attempt `flaky` in one run from `bestOf: 7`.
139
+ - An eval's fingerprint reads its own body, options, case and `revision`, not
140
+ helpers, constants outside the body or fixtures. After changing one of those in a way that
141
+ changes what the eval measures, bump `revision` (`{ revision: 2 }`, on the eval or its group)
142
+ so `reeval compare` says `can't compare … revision 1 → 2` instead of `regressed`.
143
+
144
+ ### Without Docker or API keys: the fake trio
145
+
146
+ ```ts
147
+ import { defineConfig } from 'reeval'
148
+ import { fakeHarness } from 'reeval/harness'
149
+ import { fakeJudge } from 'reeval/judge'
150
+ import { fakeSandbox } from 'reeval/sandbox'
151
+
152
+ export default defineConfig({
153
+ // A scripted "agent": writes these files every turn and replies with text.
154
+ harnesses: fakeHarness({
155
+ files: { 'hello.txt': 'hello\n' },
156
+ text: 'Created hello.txt.',
157
+ }),
158
+ sandbox: fakeSandbox(), // default machine; not an environment
159
+ judge: { model: fakeJudge({ score: 0.9 }) }, // fixed score, no model call
160
+ })
161
+ ```
162
+
163
+ `fakeHarness` also takes `withSkill: { files, text }` (used when the environment gives the agent
164
+ a skill), `script` (one list of `{ exec }`, `{ write }`, `{ reply }`, `{ mcp: 'server/tool' }`
165
+ steps per turn), `layout` (an agent's id, such as `'claude-code'`), `failWith`, `delayMs`,
166
+ `turnMs` (each turn's agent time, exactly: effort comparisons without scheduling jitter),
167
+ `flakeRate`, `reasoning`, `tools` and `id`.
168
+
169
+ `fakeSandbox()` is for testing configs and reeval itself without Docker. It runs commands on
170
+ your host with your tools, brokers no secrets and enforces no egress, so an eval passing there says nothing about its checks in the eval's real sandbox.
171
+ Don't use it to validate an eval, and don't put it on an environment that real harnesses use:
172
+ real Cursor runs need Docker.
173
+
174
+ ## Evals
175
+
176
+ ```ts
177
+ // evals/hello.eval.ts
178
+ import { evaluate, expect, group } from 'reeval'
179
+
180
+ group('hello', () => {
181
+ evaluate('creates hello.txt', async (c) => {
182
+ const turn = await c.send('Create hello.txt containing "hello".')
183
+ await expect(turn).toHaveFile('hello.txt', /hello/) // async: await it
184
+ await expect(c.exec('grep -q hello hello.txt')).toSucceed()
185
+ const q = 'Does hello.txt contain the right content?'
186
+ expect(await c.judge(q, { contents: ['hello.txt'] })).toPass()
187
+ })
188
+ })
189
+ ```
190
+
191
+ - `evaluate(name, [options,] fn)` and `group(name, [options,] fn)`; also `.skip`, `.only`
192
+ (an error in CI), `.todo`. `{ cases: [...] }` runs one eval over a list, with the body
193
+ `(c, item) => …` (cases pool into one pass rate, see [references/evals.md](references/evals.md)).
194
+ `gate: 'observe'` gates nothing: the eval records what agents do. The eval id is `file › group › name`; renaming an
195
+ eval restarts its history. Name the eval after the question it answers
196
+ (`recommends Genkit for a production AI feature`, not `genkit test 2`), so the results and
197
+ `reeval analyze` read in those terms.
198
+ - `await c.send(prompt)` runs one agent turn and returns a `Turn` (`text`, `status`, `diff`,
199
+ `transcript`). Call it again for multi-turn;
200
+ `c.judge(q, { on: firstTurn, task: true, output: true })` grades an earlier one.
201
+ - `await c.exec(cmd)` returns `{ exitCode, stdout, stderr }`. A check that can fail on a
202
+ transient (a hidden test that calls a model or a network service) gets
203
+ `c.exec(cmd, { retries: 2 })`: any non-zero exit runs it again, and the record and output say
204
+ `passed on try 2 of 3`. Don't roll a retry loop into the test itself.
205
+ `c.exec(cmd, { env: { EXPECTED: '42' } })` sets variables for that command only (a hidden
206
+ verify script's inputs), over the environment's; secret-named values are masked, and a brokered
207
+ secret's host value goes in as its placeholder.
208
+ `await c.readFile(path)` the text or `null` (`~/` for home). `await c.writeFile(path, text)` places a file after a turn (a hidden
209
+ test the agent never saw). Web apps: `c.serve` and `c.capture` in
210
+ [references/capture.md](references/capture.md).
211
+ - `await c.judge(question, options)`: the judge sees only the inputs the call turns on, and it
212
+ needs at least one (there are no defaults, here or in any config): `task`, `output`, `files`,
213
+ `diff` (`true` or patterns), `contents` (files whole), `transcript` (`true`, `'tools'`,
214
+ `'full'`), `checks`, `artifacts`. Pick what the question needs: code → `{ diff: ['src/**'] }`
215
+ or `{ contents: ['README.md'] }`, the agent's answer → `{ output: true }`, how it worked →
216
+ `{ transcript: 'tools' }`. `on` and `turns` only select; details in
217
+ [references/evals.md](references/evals.md). Prefer deterministic checks for what can be checked; use the judge for quality, with plain
218
+ yes/no questions that don't name one specific way of doing it.
219
+ - A failed judge call doesn't throw: `c.judge()` returns `{ verdict: 'error', score: null, error }`,
220
+ the later judges and checks still run, and the attempt fails as a judge error at the end, or
221
+ as an infra error (which `--retry-errors` reruns) when the provider never answered and nothing
222
+ else failed.
223
+ `expect(await c.judge(q, { diff: true }), name).toPass()` records each judge's result and carries on.
224
+ - Make the eval trustworthy. Before a real run, swap in the fake harness, and only the harness,
225
+ so the eval runs in its real sandbox (its `docker()` image, `install`, egress and brokered
226
+ secrets): an oracle `fakeHarness` that does the task right must pass, and a canary that cheats
227
+ or gets it wrong must fail. A canary that passes means a check isn't checking. Put them in the
228
+ one config as profiles that set `defaults.harnesses` (and `defaults.judge`), never
229
+ `defaults.sandbox`, then `reeval image build --profile oracle` and
230
+ `reeval run --profile oracle` / `--profile canary` (needs Docker, no keys); see
231
+ [references/evals.md](references/evals.md#trustworthy-evals). Not `fakeSandbox()`.
232
+ Keep grading out of the agent's reach: place hidden tests after the turn with `c.writeFile`, and guard seeded files
233
+ with `expect(await c.readFile(path)).toBe(original)` or `expect(c).not.toHaveChanged(['test/**'])`.
234
+ When the behavior is the outcome, judge the program's output, not the diff:
235
+ `c.judge(q, { on: (await c.exec('node app.js')).stdout, output: true })`.
236
+ - Give the environment a `.gitignore` in `files` (the project's isn't copied in): an agent that
237
+ runs a build otherwise fills the judge's diff with its output.
238
+ - Matchers: `await expect(turn).toHaveFile(path, pattern?)`,
239
+ `expect(await c.exec(cmd)).toSucceed()` (or `await expect(c.exec(cmd)).toSucceed()`),
240
+ `expect(await c.judge(q, { diff: true })).toPass()`, plus Vitest's. Each takes only its own subject:
241
+ `toPass()` on an exec result is a type error, so typecheck evals (`tsc --noEmit`, with
242
+ `typescript` installed).
243
+ Transcript matchers (sync, no `await`): `expect(turn | c).toHaveCalledSkill(name)`,
244
+ `toHaveCalledMcp(server, tool, input?)`, `toHaveCalledTool(name | RegExp, input?)`,
245
+ `toHaveRun(command?)`, `toHaveEdited(path?)`, `toHaveSearched(query?)`,
246
+ `toHaveFetched(url?)`. `toHaveEdited` counts edit tools' events and the files that changed
247
+ while the agent ran the turn, so a shell write counts and a read doesn't. `toHaveRun('genkit')`
248
+ matches anywhere in a command line (`cat genkit.config.ts` too); to ask "did it run the CLI",
249
+ use `toHaveRun({ program: 'genkit', args: /^flow:run\b/ })`, which matches the program each
250
+ segment ran after env assignments, `timeout`, `sudo`, `npx`, `pnpm dlx` and similar wrappers.
251
+ `shellCommands(c)` (from `reeval`) lists those `{ program, args, line, ok, turn }` to count. Guard files with
252
+ `expect(c).not.toHaveChanged(globs)`: its globs are exact, while `toHaveEdited`'s string
253
+ matches any path containing it. `toHaveEdited()` counts files that appear in the workspace
254
+ diff. Gitignored files don't appear there, so use `toHaveFile()` to check them. Label
255
+ requirements with
256
+ `expect`'s second argument: `await expect(c, 'exports greet').toHaveFile('greet.js', /export/)`.
257
+ - Checks don't stop the eval: a failed `expect` is recorded, the body carries on, and the attempt
258
+ fails at its end, with every failed check listed by label and line. Use `expect.require` for
259
+ gates, such as a build the later checks depend on: its failure ends the body there
260
+ (`await expect.require(c.exec('npm run build'), 'builds').toSucceed()`). A caught
261
+ `expect.require` still fails the attempt. Thrown errors (the body's own, a dead sandbox, a
262
+ timeout) still end the attempt. To wait for something, use
263
+ `expect.poll(() => c.exec(cmd), { timeout: 10_000 }).toSucceed()` (Python
264
+ `expect.poll(lambda: c.exec(cmd), timeout=10).to_succeed()`), never `expect` in a retry loop. A
265
+ caught plain Python `assert` records nothing, so `assert` or `if` in a loop also works.
266
+
267
+ ## Environments and compare
268
+
269
+ An environment is what the agent runs *in*: skills, files, rules, MCP servers, env vars, setup
270
+ steps and the workspace's starting code. `compare` runs the same eval in a control and each
271
+ variant and reports lift.
272
+
273
+ ```ts
274
+ import { defineEnvironment, evaluate, expect, group } from 'reeval'
275
+
276
+ const greetsAda = `node -e "import('./greet.js').then((m) =>
277
+ process.exit(m.greet('Ada') === 'Ahoy, Ada!' ? 0 : 1))"`
278
+ const plain = defineEnvironment('plain', {})
279
+ const houseStyle = plain.extendEnvironment('house-style', ({ skill }) => ({
280
+ skills: [skill.local('./skills/house-greeting')], // a dir with SKILL.md, relative to this file
281
+ }))
282
+
283
+ group(
284
+ 'greetings',
285
+ { compare: { control: plain, variants: [houseStyle] } },
286
+ () => {
287
+ evaluate('greets in the house style', async (c) => {
288
+ await c.send('Create greet.js exporting greet(name) in our house style.')
289
+ await expect(c.exec(greetsAda)).toSucceed()
290
+ })
291
+ },
292
+ )
293
+ ```
294
+
295
+ - A spec, or a synchronous callback that destructures the helpers it needs (`skill`, `plugin`,
296
+ `file`, `directory`, `git`, `mcp`, `secret`): `files: { 'docs/api.md': file('../docs/api.md') }`,
297
+ `rules: file('../rules/house.md')` (AGENTS.md; CLAUDE.md for Claude Code),
298
+ `mcp: { docs: mcp.stdio('npx', ['-y', '@acme/docs-mcp@1.0.0']) }`,
299
+ `setup: [{ run: 'npm ci', timeout: '5m' }]`. More in
300
+ [references/environments.md](references/environments.md).
301
+ - Attempts = evals × harnesses × (1 control + variants) × runs. The example is 1 eval, 2 cells
302
+ (`<harness>/plain/<sandbox>` control, `<harness>/house-style/<sandbox>` variant), 8 attempts:
303
+ without `runs`, a compare runs 4 a cell, the fewest whose lift often holds (with `cases`, once a
304
+ case: the lift pairs cases). A `runs` below 4 shows
305
+ only the direction, with a `too-few-tries` remark; `runs: 1` at any level or a profile's `limit.runs: 1` keeps a smoke run cheap.
306
+ - Each variant runs as that environment on its own; `compare` does not layer, inherit or
307
+ extend. Share setup with `parent.extendEnvironment(name, spec)` before `compare`. `control` is
308
+ required, and `environments` beside `compare` is an error: the control goes in `compare`.
309
+ - Only variant cells gate by default; a control's misses don't fail the run. Below four tries a
310
+ side each variant's effect is only a direction; from four, the block adds an effort table
311
+ (median tools, time, tokens per arm), which often shows a skill's effect first. In a compare,
312
+ keep independent requirements as labeled `expect`s, not `expect.require`, so each gets a verdict.
313
+ - `questions: [{ question: 'How did the agent pick its greeting?', transcript: true }]` beside
314
+ `compare` asks a model about each attempt as it ends (`choices: ['yes', 'no']` for a pick); an
315
+ entry with `across: 'environment'` (and, say, `diff: true`) asks which side did better, pair by
316
+ pair, after the run. Each entry turns on at least one input, as on `c.judge()`, except `contents`. Questions
317
+ never gate and need a language model. Details, gates,
318
+ `minLift` and `reeval query skill_files` (which skill files each attempt opened) are in
319
+ [references/compare.md](references/compare.md).
320
+
321
+ ## CLI
322
+
323
+ ```text
324
+ reeval run [filter...] --dry-run [--json] --runs <n> --tag <t> --trait <k=v> --profile <p>
325
+ --config <path> --record --ui plain -v
326
+ reeval show [run] reeval history reeval compare [base] [head]
327
+ reeval query lift reeval query effort
328
+ reeval query "SELECT … FROM failing_attempts"
329
+ reeval analyze reeval studio reeval image build reeval doctor
330
+ ```
331
+
332
+ | Exit code | Means |
333
+ |---|---|
334
+ | 0 | Pass, and `--dry-run` |
335
+ | 1 | A gated failure |
336
+ | 2 | A usage or config error, including a filter that matches nothing |
337
+ | 3 | An infra error, such as a sandbox that failed to start, a harness key the API rejected or a turn the provider stopped |
338
+
339
+ `reeval run` ends with a status list: each eval file, a line per eval under it with its icon and
340
+ `N of M runs passed`, and for a failure, a compare or an eval with remarks, a sentence for each
341
+ way it went (the harnesses, a variant's effect as `helped`, `made it worse` or `same`, the failed
342
+ checks with their messages and lines; for an agent, also the rest of each message, the snapshot
343
+ and each remark's whole sentence); `-v` expands every eval. Then the tally
344
+ (`Done in 12s · 1 failed · 2 passed`; an `error:` outside any eval, such as a stray rejection in an eval file, is
345
+ `1 run error` and fails the run; an eval with `gate: 'observe'` is `1 ran`, never a pass,
346
+ so read its arm rows for what passed). `Worth knowing:` under an eval names what about it needs work
347
+ (a flaky check, too few tries, a slow MCP server), with the line to fix, from fixed rules with no
348
+ model; they never change a verdict or the exit code, and go in `remarks.json` beside the run. The
349
+ eval rubric ([../eval-rubric/SKILL.md](../eval-rubric/SKILL.md)) explains each kind. `summary.md`
350
+ and `reeval show` print the full result blocks, a table per arm and check.
351
+
352
+ Every command uses the nearest `reeval.config.*` (or `reeval.yaml`) in the current directory or
353
+ above, or `--config <path>`. Runs land in `.reeval/runs/`. All views, `analyze` and the MCP
354
+ server: [references/cli.md](references/cli.md).
355
+
356
+ ## Common mistakes (and the fix)
357
+
358
+ | Mistake | Fix |
359
+ |---|---|
360
+ | `compare` in `reeval.config.ts` | Put it on the group or eval |
361
+ | `environments: [a, b]` to measure b, or `compare: [b]` | `compare: { control: a, variants: [b] }` |
362
+ | `compare: { baseline: a, … }` | `compare: { control: a, … }` |
363
+ | `repeat: 3` / `times` | `runs: 3` |
364
+ | Judge model `'gemini-3.8-flash'` | `'google/gemini-3.8-flash'` |
365
+ | Forgetting `await` on `toHaveFile`, or on `toSucceed` given `c.exec()`'s promise | `await expect(...)` |
366
+ | `toPass()` on `c.exec()`'s result | `toSucceed()`; `toPass()` is for `await c.judge()` |
367
+ | `cursor()` with `fakeSandbox()` | Use `fakeHarness()` or `docker()` |
368
+ | Test file named `*.test.ts` | `*.eval.ts` under `evals/` |
369
+
370
+ Every error message and its fix: [references/mistakes.md](references/mistakes.md).
@@ -0,0 +1,48 @@
1
+ # Web UIs: serve, capture and artifacts
2
+
3
+ ```ts
4
+ evaluate('builds the sign-in page', async (c) => {
5
+ await c.send('Build a sign-in page served by `npm start` on $PORT.')
6
+ const app = await c.serve('npm start')
7
+ await c.capture(app, { name: 'sign-in', path: '/login', wait: 'idle' })
8
+ const q = 'Does the screenshot show an email field and a password field?'
9
+ expect(await c.judge(q, { artifacts: ['sign-in'] })).toPass()
10
+ })
11
+ ```
12
+
13
+ - `await c.serve(cmd, port?)` starts `cmd` in the background with `PORT` set (`$PORT` on a bash
14
+ string) and returns a `ServeHandle` once `cmd` listens. Without `port`, a server
15
+ that ignores `PORT` and listens on one new port of its own (`npm run dev` on Vite's 5173, IPv4 or
16
+ IPv6 loopback) gets that port as `handle.port`; no `--port` flags needed. Several new ports, none
17
+ of them `PORT`, throw with the list and the URLs it printed: pass the one to use, `c.serve('npm run dev', 5173)`. An
18
+ exit, or nothing listening in 120 s, throws with its last output. With `port`, only that
19
+ port counts, and a `port` something already listens on (the agent's own dev server, left
20
+ running) throws before `cmd` starts: pass none and serve on `PORT`. `c.exec(cmd, { background:
21
+ true })` starts a command without a port or the wait and returns `{ kill() }`.
22
+ - `await c.capture(port | server, { name?, path?,
23
+ wait?, instructions?, viewport?, model? })` screenshots the tunneled port into
24
+ `artifacts/<name>.png`. The first argument is the container port or a `ServeHandle`;
25
+ default `path` is `/`, default `name` comes from the path. A `name` is 1 to 100 of
26
+ `a-z`, `0-9`, `_` and `-`, starting with a letter or digit. `wait` is once, before the
27
+ first model shot or the saved PNG (`'load'`, `'idle'`, or `{ ms }`). `instructions` is
28
+ a journey of goal strings and `{ wait }` beats (`{ wait: 'idle' }` waits for the
29
+ network to go quiet from that beat, so it covers a click that only fetches) — no
30
+ selectors; the model clicks pixels. `viewport: 'full-page'` is the saved shot (default window + tall PNG).
31
+ - `capture.model` cascades like `question`; the call may override. Default is
32
+ `analysis.model`, else the attempt's judge model (a profile's, group's or eval's),
33
+ whichever is first a language model; never Jev. Gemini's 0–1000 points are scaled to the screenshot. Fixture evals import
34
+ `fakeInstructions` from `reeval/testing` and pass it as `instructions`.
35
+ The same `timeout` is the budget for the whole capture, its instructions included: if it
36
+ runs out the capture fails and there is no `<name>.png`, but the page as the instruction
37
+ stopped is in `artifacts/<name>.step-<n>.png`, which the error names. `capture.browser: false` or
38
+ `REEVAL_CAPTURE_BROWSER=0` makes `c.capture()` throw.
39
+ - The `capture` option shallow-merges camera defaults (`viewport`, `timeout`, `browser`,
40
+ `model`, the vision model) down the cascade; `viewport` replaces the value, and the port is
41
+ not an option.
42
+ - On a judge, `artifacts` is an input like `diff`, set on the call: `true` for every capture
43
+ so far, or `c.capture()` handles or names, sent as file parts. Alone, the judge sees only the
44
+ pictures. Ask about facts in the picture, not whether it "looks
45
+ great". Jev (`typesafe-ai/jev-1.14.0`) takes PNG captures; a provider that cannot take files
46
+ fails that call.
47
+ - On a question, `artifacts: ['sign-in']` (or `true`) sends each attempt's own capture of that
48
+ name as an image (see [compare.md](compare.md)).