reeval 0.14.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (364) hide show
  1. package/README.md +15 -15
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
  5. package/dist/adapter-CRKYPkRt.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
  7. package/dist/agent-BgH4-kFc.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
  9. package/dist/app-files-BMVBr079.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
  11. package/dist/assemble-BVbI6NW-.js.map +1 -0
  12. package/dist/assemble-SiHa4__F.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
  16. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
  18. package/dist/capture-iQ6kYJV0.js.map +1 -0
  19. package/dist/capture-px7Ey95_.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
  21. package/dist/cli-1YQU-66F.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
  28. package/dist/command-8_Cg5v6b.js.map +1 -0
  29. package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
  30. package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
  31. package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
  32. package/dist/command-C-HeGlap.js.map +1 -0
  33. package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
  34. package/dist/command-Cr1zjRvl.js.map +1 -0
  35. package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
  36. package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
  37. package/dist/commands-BvTNZ2-b.js.map +1 -0
  38. package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
  39. package/dist/config-CneBZB84.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-BnXyamuj.js +2 -0
  46. package/dist/db-Cnh3jIv_.js +1530 -0
  47. package/dist/db-Cnh3jIv_.js.map +1 -0
  48. package/dist/debug-CAx3qcaP.js +1 -0
  49. package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
  50. package/dist/debug-CgG3T071.js.map +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
  52. package/dist/docker-BihzJwbV.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
  54. package/dist/docker-Cqq03H72.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
  57. package/dist/doctor-Dl4vMeqt.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
  61. package/dist/environment-Cf5HMjib.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
  63. package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
  64. package/dist/eval-source-CAE4y2Sr.js +146 -0
  65. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
  67. package/dist/evaluate-CUtgyr43.js.map +1 -0
  68. package/dist/evaluate-D35T51AC.d.ts +45 -0
  69. package/dist/files-BEnUJW1e.js +1 -0
  70. package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
  71. package/dist/files-BRViE7m-.js.map +1 -0
  72. package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
  73. package/dist/files-BSqA2sZE.js.map +1 -0
  74. package/dist/files-C3tQtD26.js +1 -0
  75. package/dist/files-CNiJ_fZf.js +37 -0
  76. package/dist/files-CNiJ_fZf.js.map +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/files-Dts7EY3H.js +1 -0
  80. package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
  81. package/dist/gate-BC6SdQZs.js.map +1 -0
  82. package/dist/gateway.d.ts +22 -9
  83. package/dist/gateway.js +30 -8
  84. package/dist/gateway.js.map +1 -1
  85. package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
  86. package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
  87. package/dist/harness-3NVyTSGK.js +392 -0
  88. package/dist/harness-3NVyTSGK.js.map +1 -0
  89. package/dist/harness.d.ts +12 -8
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
  92. package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
  93. package/dist/index.d.ts +99 -11
  94. package/dist/index.js +5 -5
  95. package/dist/init-okoALIah.js +1 -0
  96. package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
  97. package/dist/init-sFBgODhu.js.map +1 -0
  98. package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
  99. package/dist/invocations-C9ejlOYl.js.map +1 -0
  100. package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
  101. package/dist/judge-Bg4v4e_K.js.map +1 -0
  102. package/dist/judge.d.ts +5 -7
  103. package/dist/judge.js +4 -8
  104. package/dist/judge.js.map +1 -1
  105. package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
  106. package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
  107. package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
  108. package/dist/matchers-BKPuaXlj.js.map +1 -0
  109. package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
  110. package/dist/model-2rBZrdZA.js.map +1 -0
  111. package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
  112. package/dist/model-3MJtpho8.js.map +1 -0
  113. package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
  114. package/dist/model-Bz7-j71_.js.map +1 -0
  115. package/dist/model-DkPwypN4.js +1 -0
  116. package/dist/model-bREXVXwT.js +1 -0
  117. package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
  118. package/dist/node-fP-H2uxn.js.map +1 -0
  119. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  120. package/dist/object-BkXGPKWY.js.map +1 -0
  121. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  122. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  123. package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
  124. package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
  125. package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
  126. package/dist/patch-6T9XAipm.js.map +1 -0
  127. package/dist/patch-CquGAGX5.js +76 -0
  128. package/dist/patch-CquGAGX5.js.map +1 -0
  129. package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
  130. package/dist/plan-Dw2BjafJ.js.map +1 -0
  131. package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
  132. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  133. package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
  134. package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
  135. package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
  136. package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
  137. package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
  138. package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
  139. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  140. package/dist/prices-QaS-kjSQ.js.map +1 -0
  141. package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
  142. package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
  143. package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
  144. package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
  145. package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
  146. package/dist/query-Bt6vLJ3m.js +1 -0
  147. package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
  148. package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
  149. package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
  150. package/dist/questions-CkAwePPR.js.map +1 -0
  151. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  152. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  153. package/dist/redact-DVIaahLv.js +658 -0
  154. package/dist/redact-DVIaahLv.js.map +1 -0
  155. package/dist/remarks-Bp6AwPgb.js +1 -0
  156. package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
  157. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  158. package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
  159. package/dist/remarks-IFFe_oty.js.map +1 -0
  160. package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
  161. package/dist/remarks-eBcrIZRu.js.map +1 -0
  162. package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
  163. package/dist/remote-DfJYiP-R.js.map +1 -0
  164. package/dist/resolve-CT4XZIfV.js +1 -0
  165. package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
  166. package/dist/resolve-D_cLa4tW.js.map +1 -0
  167. package/dist/results-DrKJuIQF.js +63 -0
  168. package/dist/results-DrKJuIQF.js.map +1 -0
  169. package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
  170. package/dist/resume-BmIM52MJ.js.map +1 -0
  171. package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
  172. package/dist/runtime-CZVpkr04.js.map +1 -0
  173. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
  174. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  175. package/dist/sandbox.d.ts +1 -1
  176. package/dist/sandbox.js +1 -1
  177. package/dist/server-CKJOnIB0.js +1 -0
  178. package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
  179. package/dist/server-DS8BGhGU.js.map +1 -0
  180. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  181. package/dist/shared-B_lvGqWz.js.map +1 -0
  182. package/dist/source-CgUqpumI.js +1 -0
  183. package/dist/source-DcXv4de-.js +59 -0
  184. package/dist/source-DcXv4de-.js.map +1 -0
  185. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  186. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  187. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  188. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  189. package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
  190. package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
  191. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  192. package/dist/src-BO8AYqJ5.js.map +1 -0
  193. package/dist/src-BTqJbvj6.js +2 -0
  194. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  195. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BkiComGh.js +2 -0
  197. package/dist/src-BvgeZ4tS.js +2 -0
  198. package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
  199. package/dist/src-C0ySiej5.js.map +1 -0
  200. package/dist/src-CgJrz5p0.js +3 -0
  201. package/dist/src-Cp-2Hdib.js +2 -0
  202. package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
  203. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  204. package/dist/src-DSZyg0Uq.js.map +1 -0
  205. package/dist/src-Dk2Na0b-.js +2 -0
  206. package/dist/src-OESRtvOi.js +2 -0
  207. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  208. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  209. package/dist/src-h5uDlk3m.js +1 -0
  210. package/dist/state-C97udoh3.js +1 -0
  211. package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
  212. package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
  213. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  214. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  215. package/dist/suite-ByT1lsdD.js +2 -0
  216. package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
  217. package/dist/suite-DieLXvCT.js.map +1 -0
  218. package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
  219. package/dist/summary-CSzyBVsF.js.map +1 -0
  220. package/dist/tarball-CHcahyNK.js +258 -0
  221. package/dist/tarball-CHcahyNK.js.map +1 -0
  222. package/dist/testing.d.ts +426 -143
  223. package/dist/testing.js +12 -10
  224. package/dist/testing.js.map +1 -1
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  228. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  229. package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
  230. package/dist/transcript-BJrKfgRW.js.map +1 -0
  231. package/dist/types-C5vI3g1C.js.map +1 -1
  232. package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +548 -121
  238. package/dist/view/index.js +9 -9
  239. package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
  240. package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
  241. package/dist/views-Ca-rTixp.js +2 -0
  242. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
  243. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  244. package/dist/workspace-8uHtBSve.js +1 -0
  245. package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
  246. package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +5 -1
  249. package/skills/eval-rubric/SKILL.md +46 -104
  250. package/dist/adapter-BloGVZ_a.js.map +0 -1
  251. package/dist/agent-m-_t-IuW.js.map +0 -1
  252. package/dist/app-files-C-jwhRDt.js.map +0 -1
  253. package/dist/assemble-BDEpelpi.js.map +0 -1
  254. package/dist/assemble-C5bfncHU.js +0 -1
  255. package/dist/capture-CVRwRTnb.js.map +0 -1
  256. package/dist/capture-CherbjzB.js +0 -1
  257. package/dist/cli-C_4gaJ50.js.map +0 -1
  258. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  259. package/dist/client-D5hNjpEx.js.map +0 -1
  260. package/dist/command-BWAuwQNk.js +0 -91
  261. package/dist/command-BWAuwQNk.js.map +0 -1
  262. package/dist/command-BXLUfw4h.js.map +0 -1
  263. package/dist/command-Be6BSE2L.js +0 -335
  264. package/dist/command-Be6BSE2L.js.map +0 -1
  265. package/dist/command-Bxdy62dW.js.map +0 -1
  266. package/dist/commands-jzigbNFb.js.map +0 -1
  267. package/dist/config-BrdHA9bJ.js.map +0 -1
  268. package/dist/cost-D4elx933.js.map +0 -1
  269. package/dist/cost-DmnAod9x.js +0 -1
  270. package/dist/db-CAfksinX.js +0 -2
  271. package/dist/db-DO0p7xgP.js +0 -619
  272. package/dist/db-DO0p7xgP.js.map +0 -1
  273. package/dist/debug-D24NOQH1.js.map +0 -1
  274. package/dist/debug-D7bEv_UV.js +0 -1
  275. package/dist/docker-BnRlTpZA.js.map +0 -1
  276. package/dist/docker-D01slDF3.js.map +0 -1
  277. package/dist/doctor-DQ2p0euG.js.map +0 -1
  278. package/dist/env-BQqj93gu.js.map +0 -1
  279. package/dist/environment-CO0P0soF.js.map +0 -1
  280. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  281. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  282. package/dist/facts-DzOUTreB.js +0 -906
  283. package/dist/facts-DzOUTreB.js.map +0 -1
  284. package/dist/files-B-CJeeEh.js.map +0 -1
  285. package/dist/files-BAt0i00X.js +0 -1
  286. package/dist/files-Bp5j5BqI.js +0 -1
  287. package/dist/files-C2sjmXhk.js +0 -1
  288. package/dist/files-C_gQNVB-.js +0 -65
  289. package/dist/files-C_gQNVB-.js.map +0 -1
  290. package/dist/files-Ca0d6Uq3.js.map +0 -1
  291. package/dist/files-kUCrswI6.js.map +0 -1
  292. package/dist/generate-DZ7vp69F.js +0 -1647
  293. package/dist/generate-DZ7vp69F.js.map +0 -1
  294. package/dist/harness-COVXSwJ6.js +0 -354
  295. package/dist/harness-COVXSwJ6.js.map +0 -1
  296. package/dist/header-BLPP6aCU.js.map +0 -1
  297. package/dist/init-RgCQXZhb.js +0 -1
  298. package/dist/init-SRalM5xF.js.map +0 -1
  299. package/dist/insights-TyAvz9aV.js +0 -68
  300. package/dist/insights-TyAvz9aV.js.map +0 -1
  301. package/dist/invocations-DzyKSJBa.js.map +0 -1
  302. package/dist/judge-yLvjC5A3.js.map +0 -1
  303. package/dist/match-Bz0nSdzN.js.map +0 -1
  304. package/dist/matchers-D4eUqx4j.js.map +0 -1
  305. package/dist/model-1ZLr9Dcv.js.map +0 -1
  306. package/dist/model-B45lDUV1.js.map +0 -1
  307. package/dist/model-BTZq8HIZ.js +0 -1
  308. package/dist/model-CgnOpss4.js.map +0 -1
  309. package/dist/model-Ch8kX87u.js +0 -1
  310. package/dist/node-BSq5qHnQ.js.map +0 -1
  311. package/dist/object-CLf2-tNq.js.map +0 -1
  312. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  313. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  314. package/dist/pairs-CHfZNRT6.js.map +0 -1
  315. package/dist/patch-DWf-I_Q_.js.map +0 -1
  316. package/dist/plan-BIwUzlB8.js.map +0 -1
  317. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  318. package/dist/prices-BcLEXvdY.js.map +0 -1
  319. package/dist/query-Cvbwl804.js +0 -1
  320. package/dist/questions-bAudIc04.js.map +0 -1
  321. package/dist/redact-CRGdfV2s.js +0 -240
  322. package/dist/redact-CRGdfV2s.js.map +0 -1
  323. package/dist/remarks-CXwSnSjR.js +0 -278
  324. package/dist/remarks-CXwSnSjR.js.map +0 -1
  325. package/dist/remarks-CyAu2DiB.js +0 -1
  326. package/dist/remarks-mujJv1Ey.js.map +0 -1
  327. package/dist/remote-CKcEKja4.js.map +0 -1
  328. package/dist/resolve-B5-uqmJ6.js +0 -1
  329. package/dist/resolve-CBoKzQMP.js.map +0 -1
  330. package/dist/results-CMgZEANy.js +0 -78
  331. package/dist/results-CMgZEANy.js.map +0 -1
  332. package/dist/resume-B9SOSnK9.js.map +0 -1
  333. package/dist/rules-VhftPOOT.js.map +0 -1
  334. package/dist/run-CwIEK16q.js +0 -88
  335. package/dist/run-CwIEK16q.js.map +0 -1
  336. package/dist/runtime-BgQRyCLl.js.map +0 -1
  337. package/dist/server-CHRSgKRp.js +0 -1
  338. package/dist/server-TGW4WyAz.js.map +0 -1
  339. package/dist/shared-BDUzStad.js.map +0 -1
  340. package/dist/source-B1WvBIQt.js +0 -40
  341. package/dist/source-B1WvBIQt.js.map +0 -1
  342. package/dist/source-DAzztZwU.js +0 -1
  343. package/dist/src-1jHbFaGW.js +0 -1
  344. package/dist/src-8zoX_qrb.js.map +0 -1
  345. package/dist/src-C_9wgowC.js +0 -2
  346. package/dist/src-ChhtI6BP.js.map +0 -1
  347. package/dist/src-D005Cbqz.js +0 -2
  348. package/dist/src-DIfRbp5a.js +0 -3
  349. package/dist/src-Dg8121Xy.js +0 -2
  350. package/dist/src-ElBqEN-S.js +0 -2
  351. package/dist/src-NSdQ9zI6.js.map +0 -1
  352. package/dist/src-YHGaUrGE.js +0 -2
  353. package/dist/src-mJVxw1Jm.js +0 -2
  354. package/dist/state-vznMh2wl.js +0 -1
  355. package/dist/suite-BsWGHzlG.js +0 -2
  356. package/dist/suite-WgT7m6Vs.js.map +0 -1
  357. package/dist/summary-DIGEVr08.js.map +0 -1
  358. package/dist/text-CrVy5ioL.js.map +0 -1
  359. package/dist/transcript-BTxJpW4R.js.map +0 -1
  360. package/dist/usage-DF3_p5pE.js +0 -1
  361. package/dist/validate-BhwqrVsm.js.map +0 -1
  362. package/dist/views-BW_C5eVH.js +0 -2
  363. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  364. package/dist/workspace-QZCacP8A.js +0 -1
@@ -0,0 +1 @@
1
+ {"version":3,"file":"app-files-BMVBr079.js","names":[],"sources":["../src/studio/app-files.ts"],"mappings":";AAAA,MAAa,YAAY;CAAC,6BAA4B,EAAC,QAAO,6n4DAA4n4D;CAAE,4BAA2B,EAAC,QAAO,0z9pBAA0z9pB;CAAE,eAAc,EAAC,QAAO,yTAAwT;CAAE,cAAa,EAAC,QAAO,ojBAAkjB;AAAC"}
@@ -1,18 +1,20 @@
1
1
  import { r as isTimeout } from "./errors-B6BSMs03.js";
2
- import { T as profileSandbox } from "./environment-CO0P0soF.js";
2
+ import { D as profileSandbox } from "./environment-Cf5HMjib.js";
3
3
  import { i as evalSlug, l as sha256, n as attemptKey, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
4
- import { B as mergeJudge, M as profileJudge, O as harnessId, P as sandboxId, V as resolveStateOptions, at as servedEffortOf, it as resolvedEffortOf, j as plannedSandboxes, nt as resolveQuestionState, rt as harnessEffort } from "./config-BrdHA9bJ.js";
5
- import { c as judgeModelName } from "./judge-yLvjC5A3.js";
6
- import { s as redactSecrets } from "./redact-CRGdfV2s.js";
7
- import { i as formatDuration } from "./text-CrVy5ioL.js";
4
+ import { A as harnessId, B as sandboxId, I as plannedSandboxes, K as mergeJudge, L as profileJudge, M as majorityOf, ft as harnessEffort, lt as UNRECORDED_QUESTION_STATE, mt as servedEffortOf, pt as resolvedEffortOf, z as runsOf } from "./config-CneBZB84.js";
5
+ import { l as judgeModelName } from "./judge-Bg4v4e_K.js";
6
+ import { f as redactJsonText, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
7
+ import { i as formatDuration } from "./text-DTigcA64.js";
8
+ import { i as passRateOf, o as worstOutcome, r as gatePart, t as cellOutcome } from "./gate-BC6SdQZs.js";
9
+ import { a as mcnemar, d as round3, f as signedRank, l as pairedLiftInterval, n as containsZero, o as meanInterval, p as wilson, r as liftInterval, s as median } from "./stats-hfpuYsAM.js";
8
10
  import { i as sumUsage, r as promptTokens } from "./usage-BNY1ABrU.js";
9
- import { r as patchForJudge, t as collapseDependencies } from "./patch-DWf-I_Q_.js";
10
- import { L as checkTries, ct as effortSample } from "./model-CgnOpss4.js";
11
- import { a as mcnemar, d as round3, f as signedRank, l as pairedLiftInterval, n as containsZero, o as meanInterval, p as wilson, r as liftInterval, s as median } from "./stats-DSy5HfOv.js";
12
- import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-B1t143wG.js";
13
- import { t as checkVerdict } from "./contrasts-DRxMQQTC.js";
14
- import { a as previousAttempt, n as isCompletedAttempt } from "./resume-B9SOSnK9.js";
15
- import { dirname, join, relative, resolve, sep } from "node:path";
11
+ import { r as patchForJudge, t as collapseDependencies } from "./patch-6T9XAipm.js";
12
+ import { a as sourcesPart, r as evalFile } from "./eval-source-CAE4y2Sr.js";
13
+ import { I as checkTries, st as effortSample } from "./model-Bz7-j71_.js";
14
+ import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-C_kdhp5v.js";
15
+ import { t as checkVerdict } from "./contrasts-lhFsekco.js";
16
+ import { n as isCompletedAttempt, o as previousAttempt } from "./resume-BmIM52MJ.js";
17
+ import { join } from "node:path";
16
18
  import { readFile, stat } from "node:fs/promises";
17
19
  //#region src/history/families.ts
18
20
  const isCounted = (a) => a.outcome === "pass" || a.outcome === "fail";
@@ -183,65 +185,27 @@ function familiesOf(evals, minLiftOf) {
183
185
  });
184
186
  }
185
187
  //#endregion
186
- //#region src/history/sources.ts
187
- /** `from '…'`, `import '…'`, `import('…')` and `export … from '…'`, with a literal specifier. */
188
- const SPECIFIER = /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+)(['"])((?:\.{1,2}\/|\/)[^'"]+)\1/g;
189
- const CANDIDATES = (path) => {
190
- return [
191
- path,
192
- .../\.(m?js|cjs)$/.exec(path) ? [path.replace(/\.(m?js|cjs)$/, ".ts"), path.replace(/\.(m?js|cjs)$/, ".tsx")] : [],
193
- `${path}.ts`,
194
- `${path}.tsx`,
195
- `${path}.js`,
196
- `${path}.mjs`,
197
- join(path, "index.ts"),
198
- join(path, "index.js")
199
- ];
200
- };
201
- async function isFile(path) {
202
- return (await stat(path).catch(() => null))?.isFile() ?? false;
203
- }
204
- async function resolveImport(from, specifier) {
205
- const base = specifier.startsWith("/") ? specifier : resolve(dirname(from), specifier);
206
- for (const candidate of CANDIDATES(base)) if (await isFile(candidate)) return candidate;
207
- return null;
208
- }
209
- /**
210
- * The eval file and every local module it imports, transitively, by path from `root` and
211
- * content hash, sorted by path. Only relative and absolute specifiers are followed: packages
212
- * (`reeval`, `node:*`, anything in `node_modules`) aren't traced.
213
- */
214
- async function traceSources(root, file) {
215
- const seen = /* @__PURE__ */ new Map();
216
- const queue = [resolve(root, file)];
217
- while (queue.length) {
218
- const path = queue.shift();
219
- if (seen.has(path) || path.split(sep).includes("node_modules")) continue;
220
- const bytes = await readFile(path).catch(() => null);
221
- if (!bytes) continue;
222
- seen.set(path, {
223
- path: relative(root, path).split(sep).join("/"),
224
- sha256: sha256(bytes)
225
- });
226
- for (const match of bytes.toString("utf8").matchAll(SPECIFIER)) {
227
- const found = await resolveImport(path, match[2]);
228
- if (found && !seen.has(found)) queue.push(found);
229
- }
230
- }
231
- return [...seen.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
232
- }
233
- //#endregion
234
188
  //#region src/history/assemble.ts
235
189
  /** Final messages are cut at this size in the record; `transcript.jsonl` keeps them whole. */
236
190
  const TURN_TEXT_LIMIT = 8192;
237
191
  const TURN_ERROR_LIMIT = 1024;
192
+ /**
193
+ * `text` masked when it's longer than `limit`, so a secret the cut would go through is masked
194
+ * whole. Text that fits is masked when the record is written.
195
+ */
196
+ const maskedOver = (text, limit) => text.length > limit ? redactSecrets(text) : text;
238
197
  /** A turn's first error event, such as the provider's "Credit balance is too low". */
239
198
  function turnError(transcript) {
240
199
  const found = transcript?.find((e) => e.type === "error");
241
200
  if (found?.type !== "error") return null;
242
- const message = found.message.trim();
201
+ const message = maskedOver(found.message.trim(), TURN_ERROR_LIMIT);
243
202
  return message.length > TURN_ERROR_LIMIT ? `${message.slice(0, TURN_ERROR_LIMIT)}…` : message;
244
203
  }
204
+ /** A turn's final message, cut at `TURN_TEXT_LIMIT`. */
205
+ function turnText(text) {
206
+ const s = maskedOver(text, TURN_TEXT_LIMIT);
207
+ return s.length > TURN_TEXT_LIMIT ? `${s.slice(0, TURN_TEXT_LIMIT)}… [reeval: truncated ${s.length} characters; the whole message is the turn's last text event in transcript.jsonl]` : s;
208
+ }
245
209
  /** Tokens across every turn that reported them. */
246
210
  function costOf(attempts) {
247
211
  const usage = sumUsage(attempts.map((a) => a.usage));
@@ -263,7 +227,8 @@ const OUTPUT_CAP = 131072;
263
227
  const HEAD = 98304;
264
228
  const TAIL = 16384;
265
229
  /** Keeps evidence readable without letting one `cat` of a lockfile dominate the record. */
266
- function capText(text, cap = OUTPUT_CAP) {
230
+ function capText(raw, cap = OUTPUT_CAP) {
231
+ const text = maskedOver(raw, cap);
267
232
  if (text.length <= cap) return text;
268
233
  const omitted = text.length - HEAD - TAIL;
269
234
  return `${text.slice(0, HEAD)}\n[reeval: truncated ${omitted} of ${text.length} characters, sha256 ${sha256(text)}]\n${text.slice(-16384)}`;
@@ -296,18 +261,6 @@ function mergeOutcome(e, record) {
296
261
  error
297
262
  };
298
263
  }
299
- /** Any failure fails; then infra errors; then all-skipped; else pass. */
300
- function worstOutcome(outcomes) {
301
- if (outcomes.includes("fail")) return "fail";
302
- if (outcomes.includes("infra_error")) return "infra_error";
303
- if (outcomes.length > 0 && outcomes.every((o) => o === "skipped")) return "skipped";
304
- return outcomes.length ? "pass" : "skipped";
305
- }
306
- const passRateOf = (outcomes) => {
307
- const pass = outcomes.filter((o) => o === "pass").length;
308
- const counted = pass + outcomes.filter((o) => o === "fail").length;
309
- return counted ? pass / counted : null;
310
- };
311
264
  /**
312
265
  * The harness as the measurement sees it: without the operational limits (`maxConcurrent`,
313
266
  * `startTimeout`), which don't change results, and without the route, which `gateway` holds.
@@ -319,12 +272,24 @@ const fingerprintOf = (parts) => sha256(JSON.stringify(parts));
319
272
  /** The gateway's part of a fingerprint: none on `direct`, so a cell without a gateway keeps its fingerprint. */
320
273
  const gatewayPart = (gateway) => gateway && gateway.name !== "direct" ? { gateway: gateway.name } : {};
321
274
  /** `<index>:<question hash>`, which follows one verdict across runs. */
322
- const judgementKey = (index, question) => `${index}:${sha256(question).slice(0, 12)}`;
275
+ const judgmentKey = (index, question) => `${index}:${sha256(question).slice(0, 12)}`;
276
+ const JSON_FILE = /\.(?:json|jsonl|ndjson)$/;
323
277
  /** Collects artifact files, redacting before anything is hashed. */
324
278
  var Artifacts = class {
325
279
  files = /* @__PURE__ */ new Map();
280
+ moves = /* @__PURE__ */ new Map();
281
+ /**
282
+ * A snapshot at `source` placed at `path`, or null when it isn't there: its name and size are
283
+ * already in the record, and its files are never masked again or read into memory.
284
+ */
285
+ async move(path, source) {
286
+ if (!await stat(source).catch(() => null)) return null;
287
+ this.moves.set(path, source);
288
+ return path;
289
+ }
290
+ /** JSON and JSON Lines files are masked inside their strings only, so they stay valid. */
326
291
  add(path, content) {
327
- const bytes = Buffer.from(redactSecrets(content));
292
+ const bytes = Buffer.from(JSON_FILE.test(path) ? redactJsonText(content) : redactSecrets(content));
328
293
  this.files.set(path, bytes);
329
294
  return {
330
295
  path,
@@ -367,7 +332,7 @@ function withInput(e, { n, answer, calls }, out) {
367
332
  const doc = {
368
333
  question: answer.question,
369
334
  model: answer.model,
370
- stateOptions: answer.stateOptions ?? resolveQuestionState(void 0, true),
335
+ stateOptions: answer.stateOptions ?? UNRECORDED_QUESTION_STATE,
371
336
  ...answer.artifacts !== void 0 ? { artifacts: answer.artifacts } : {},
372
337
  calls: calls.map(call)
373
338
  };
@@ -436,13 +401,14 @@ function transcriptLines(record) {
436
401
  });
437
402
  return `${lines.join("\n")}\n`;
438
403
  }
439
- /** Every stream part the harness sent, for debugging the normaliser (`REEVAL_RAW_TRANSCRIPT=1`). */
404
+ /** Every stream part the harness sent, for debugging the normalizer (`REEVAL_RAW_TRANSCRIPT=1`). */
440
405
  function rawLines(record) {
441
406
  if (!record.turns.some((t) => t.raw)) return null;
442
- return `${record.turns.flatMap((turn, index) => (turn.raw ?? []).map((part) => JSON.stringify({
407
+ const lines = record.turns.flatMap((turn, index) => (turn.raw ?? []).map((part) => JSON.stringify({
443
408
  turn: index,
444
409
  ...part
445
- }))).join("\n")}\n`;
410
+ })));
411
+ return `${redactSplitSecrets(lines).join("\n")}\n`;
446
412
  }
447
413
  function activity(record) {
448
414
  const events = record.turns.flatMap((t) => t.transcript);
@@ -515,6 +481,8 @@ var Seen = class {
515
481
  environments = /* @__PURE__ */ new Map();
516
482
  harnesses = /* @__PURE__ */ new Map();
517
483
  resolvedHarnesses = /* @__PURE__ */ new Map();
484
+ /** Keys of the attempts kept from the previous record on resume. */
485
+ kept = /* @__PURE__ */ new Set();
518
486
  /** Registry digests found by the preflight, by image id. */
519
487
  digests;
520
488
  constructor(sandbox) {
@@ -547,7 +515,28 @@ var Seen = class {
547
515
  for (const t of record?.turns ?? []) if (t.model) resolved.add(t.model);
548
516
  this.resolvedHarnesses.set(cell.harness, resolved);
549
517
  }
550
- entry(e, sources) {
518
+ /**
519
+ * A kept attempt has no worker record left, so what `add` reads from one comes from the
520
+ * previous record instead: its image, environment, context files and resolved models.
521
+ */
522
+ addKept(previous, kept, cell, environmentDigest, planned) {
523
+ this.kept.add(kept.key);
524
+ for (const c of kept.context) this.contexts.set(`${c.path}\0${c.sha256}`, c);
525
+ for (const j of kept.judgements) if (j.resolvedModel) this.resolvedJudges.add(j.resolvedModel);
526
+ const image = previous.stack.images.find((i) => i.id === kept.imageId);
527
+ if (image && !this.images.has(image.id)) this.images.set(image.id, image);
528
+ const environment = previous.environments.find((x) => x.digest === environmentDigest);
529
+ if (environment) this.environments.set(`${environment.name}\0${environment.digest}`, environment);
530
+ const harness = previous.stack.harnesses.find((h) => h.id === cell.harness);
531
+ if (!planned && harness && !this.harnesses.has(cell.harness)) {
532
+ const { resolvedModels: _, resolvedEffort: __, ...rest } = harness;
533
+ this.harnesses.set(cell.harness, rest);
534
+ }
535
+ const resolved = this.resolvedHarnesses.get(cell.harness) ?? /* @__PURE__ */ new Set();
536
+ for (const t of kept.turns) if (t.model) resolved.add(t.model);
537
+ this.resolvedHarnesses.set(cell.harness, resolved);
538
+ }
539
+ entry(e, sources, from) {
551
540
  let entry = this.byEval.get(e.id);
552
541
  if (!entry) {
553
542
  const key = e.id;
@@ -563,6 +552,8 @@ var Seen = class {
563
552
  ...e.attempt?.family ? { family: familyOf(e.attempt.family) } : {},
564
553
  fingerprint: "",
565
554
  sources,
555
+ ...from.source ? { source: from.source } : {},
556
+ ...from.revision !== void 0 ? { revision: from.revision } : {},
566
557
  outcome: "skipped",
567
558
  todo: e.todo === true,
568
559
  passRate: null,
@@ -589,6 +580,8 @@ const UNKNOWN_CELL = {
589
580
  };
590
581
  /** minLift per cell, kept aside while attempts are added; see `rollUp`. */
591
582
  const minLifts = /* @__PURE__ */ new WeakMap();
583
+ /** Cells whose fingerprint came from an attempt that ran, not one that was skipped. */
584
+ const measured = /* @__PURE__ */ new WeakSet();
592
585
  const EMPTY_PARTS = {
593
586
  sources: "",
594
587
  harness: "",
@@ -598,12 +591,13 @@ const EMPTY_PARTS = {
598
591
  judge: "",
599
592
  context: ""
600
593
  };
601
- function cellOf(entry, cell, gate, harness, gateway) {
594
+ function cellOf(entry, cell, gate, harness, gateway, bestOf) {
602
595
  let found = entry.cells.find((c) => c.key === cell.key);
603
596
  if (!found) {
604
597
  found = {
605
598
  ...cell,
606
599
  ...gateway ? { gateway: gateway.name } : {},
600
+ ...bestOf !== void 0 ? { bestOf } : {},
607
601
  fingerprint: "",
608
602
  fingerprintParts: EMPTY_PARTS,
609
603
  environmentDigest: null,
@@ -631,17 +625,25 @@ async function copyRef(ref, from, out) {
631
625
  return out.add(ref.path, Buffer.from(bytes).toString("utf8"));
632
626
  }
633
627
  /** A completed attempt from a previous assemble of this runId, kept on resume. */
634
- async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts, sources) {
635
- const cell = e.attempt?.cell ?? {
628
+ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts) {
629
+ const before = previous.evals.find((p) => p.id === e.id);
630
+ const beforeCell = before?.cells.find((c) => c.key === kept.cell);
631
+ const cell = e.attempt?.cell ?? (beforeCell && {
632
+ key: beforeCell.key,
633
+ harness: beforeCell.harness,
634
+ environment: beforeCell.environment,
635
+ sandbox: beforeCell.sandbox,
636
+ role: beforeCell.role,
637
+ ...beforeCell.variant !== void 0 ? { variant: beforeCell.variant } : {}
638
+ }) ?? {
636
639
  ...UNKNOWN_CELL,
637
640
  key: kept.cell,
638
641
  harness: kept.cell.split("/")[0] ?? "unknown",
639
642
  environment: kept.cell.split("/")[1] ?? "unknown",
640
643
  sandbox: kept.cell.split("/")[2]?.replace(/#\d+$/, "") ?? "unknown"
641
644
  };
642
- if (!seen.sources.has(e.file) && sources) seen.sources.set(e.file, sources);
643
- if (!seen.sources.has(e.file)) seen.sources.set(e.file, []);
644
- const resolved = sources ?? seen.sources.get(e.file) ?? [];
645
+ const sources = before?.sources ?? [];
646
+ const { source, revision } = e.attempt ?? before ?? {};
645
647
  const stub = {
646
648
  id: e.id,
647
649
  key: e.key,
@@ -656,8 +658,21 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts,
656
658
  meta: void 0
657
659
  };
658
660
  seen.add(null, cell, stub);
659
- const entry = seen.entry(stub, resolved);
660
- const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway);
661
+ seen.addKept(previous, kept, cell, beforeCell?.environmentDigest ?? null, e.attempt !== void 0);
662
+ const entry = seen.entry(stub, sources, {
663
+ source,
664
+ revision
665
+ });
666
+ const created = !entry.cells.some((c) => c.key === cell.key);
667
+ const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? beforeCell?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway, e.attempt ? e.attempt.bestOf : beforeCell?.bestOf);
668
+ if (created && beforeCell && !e.attempt) {
669
+ cellRecord.harnessKind = beforeCell.harnessKind;
670
+ cellRecord.model = beforeCell.model;
671
+ cellRecord.effort = beforeCell.effort;
672
+ if (beforeCell.gateway !== void 0) cellRecord.gateway = beforeCell.gateway;
673
+ }
674
+ const minLift = e.attempt ? e.attempt.minLift : beforeCell?.lift?.minLift ?? previous.families?.find((f) => f.id === before?.family?.id)?.lifts.find((l) => l.variant === kept.cell)?.minLift;
675
+ if (minLift !== void 0) minLifts.set(cellRecord, minLift);
661
676
  const files = {};
662
677
  for (const [name, ref] of Object.entries(kept.files)) {
663
678
  if (!ref) continue;
@@ -686,23 +701,37 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts,
686
701
  ...q,
687
702
  input: await copyRef(q.input, previousArtifacts, out)
688
703
  } : q);
704
+ const { snapshots: keptSnapshots, ...rest } = kept;
705
+ const snapshots = keptSnapshots ? await placeSnapshots(keptSnapshots, (s) => ({
706
+ to: s.path ?? "",
707
+ from: previousArtifacts && s.path && join(previousArtifacts, ...s.path.split("/"))
708
+ }), out) : [];
689
709
  entry.attempts.push({
690
- ...kept,
710
+ ...rest,
691
711
  seq: null,
692
712
  files,
693
713
  checks,
694
- ...answers.length ? { answers } : {}
714
+ ...answers.length ? { answers } : {},
715
+ ...snapshots.length ? { snapshots } : {}
695
716
  });
696
- if (!cellRecord.fingerprint && kept.outcome !== "skipped") {
717
+ if (measured.has(cellRecord)) return;
718
+ measured.add(cellRecord);
719
+ if (beforeCell?.fingerprint) {
720
+ cellRecord.fingerprintParts = beforeCell.fingerprintParts;
721
+ cellRecord.fingerprint = beforeCell.fingerprint;
722
+ cellRecord.environmentDigest = beforeCell.environmentDigest;
723
+ if (cellRecord.overrides.length === 0) cellRecord.overrides = [...beforeCell.overrides];
724
+ } else {
697
725
  const parts = {
698
- sources: sha256(JSON.stringify(resolved)),
726
+ sources: sourcesPart(source, revision, sources),
699
727
  harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
700
728
  environment: cellRecord.environmentDigest,
701
729
  image: kept.imageId,
702
730
  agentCli: null,
703
731
  judge,
704
732
  context: sha256(JSON.stringify(kept.context)),
705
- ...gatewayPart(e.attempt?.gateway)
733
+ ...gatewayPart(e.attempt?.gateway),
734
+ ...gatePart(cellRecord.gate)
706
735
  };
707
736
  cellRecord.fingerprintParts = parts;
708
737
  cellRecord.fingerprint = fingerprintOf(parts);
@@ -713,32 +742,40 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
713
742
  const cell = e.attempt?.cell ?? UNKNOWN_CELL;
714
743
  const n = e.attempt?.n ?? 1;
715
744
  const kept = previousAttempt(previous, e.key);
716
- if (e.state === "skipped" && kept && isCompletedAttempt(kept.outcome, kept.error)) {
717
- await addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts);
745
+ if (previous && e.state === "skipped" && kept && isCompletedAttempt(kept.outcome, kept.error)) {
746
+ await addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts);
718
747
  return;
719
748
  }
720
749
  const record = await readRecord(join(ctx.stagingDir, "evals", recordFileName(e.key)));
721
750
  const { outcome, error } = mergeOutcome(e, record);
722
- if (!seen.sources.has(e.file)) seen.sources.set(e.file, await traceSources(ctx.config.root, e.file));
751
+ if (!seen.sources.has(e.file)) seen.sources.set(e.file, await evalFile(ctx.config.root, e.file));
723
752
  const sources = seen.sources.get(e.file) ?? [];
753
+ const source = e.attempt?.source;
754
+ const revision = e.attempt?.revision;
724
755
  const context = record?.context ?? [];
725
756
  seen.add(record, cell, e);
726
- const entry = seen.entry(e, sources);
727
- const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway);
757
+ const entry = seen.entry(e, sources, {
758
+ source,
759
+ revision
760
+ });
761
+ if (record) entry.sources = sources;
762
+ const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway, e.attempt?.bestOf);
728
763
  if (e.attempt?.minLift !== void 0) minLifts.set(cellRecord, e.attempt.minLift);
729
764
  if (record?.environment?.overrides.length && cellRecord.overrides.length === 0) cellRecord.overrides = [...record.environment.overrides];
730
765
  const environment = record?.environment;
731
766
  if (environment && environment.name !== "-") cellRecord.environmentDigest = environment.digest;
732
767
  if (!cellRecord.fingerprint || record) {
768
+ if (record) measured.add(cellRecord);
733
769
  const parts = {
734
- sources: sha256(JSON.stringify(sources)),
770
+ sources: sourcesPart(source, revision, sources),
735
771
  harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
736
772
  environment: environment?.digest ?? null,
737
773
  image: record?.image?.id ?? null,
738
774
  agentCli: record?.image?.agentCli ?? null,
739
775
  judge,
740
776
  context: sha256(JSON.stringify(context)),
741
- ...gatewayPart(e.attempt?.gateway)
777
+ ...gatewayPart(e.attempt?.gateway),
778
+ ...gatePart(cellRecord.gate)
742
779
  };
743
780
  cellRecord.fingerprintParts = parts;
744
781
  cellRecord.fingerprint = fingerprintOf(parts);
@@ -760,6 +797,8 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
760
797
  egress: record?.egress ?? null,
761
798
  servedModels: record?.servedModels ?? null,
762
799
  ...record?.askedModels?.length ? { askedModels: record.askedModels } : {},
800
+ ...record?.upstreams?.length ? { upstreams: record.upstreams } : {},
801
+ ...record?.reportedCostUsd !== void 0 ? { reportedCostUsd: record.reportedCostUsd } : {},
763
802
  effort: e.attempt ? harnessEffort(e.attempt.harness) ?? null : null,
764
803
  servedEffort: servedEffortOf([...record?.servedModels ?? [], ...(record?.turns ?? []).map((t) => t.model)]) ?? null,
765
804
  outcome,
@@ -768,6 +807,32 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
768
807
  ...await attemptEvidence(record, `${entry.slug}/${evalSlug(cell.key)}/${n}`, out, join(ctx.stagingDir, "evals", recordFileName(e.key)))
769
808
  });
770
809
  }
810
+ /** A snapshot's file or directory name: `final`, or `final.tar.gz` compressed. */
811
+ const fileOf = (s) => s.form === "tar.gz" ? `${s.name}.tar.gz` : s.name;
812
+ /**
813
+ * The attempt's snapshots with their paths in the run's directory, each moved there from where
814
+ * `place` says it waits; one whose files are gone (a resumed run's, from a run that kept none)
815
+ * is dropped.
816
+ */
817
+ async function placeSnapshots(snapshots, place, out) {
818
+ const placed = [];
819
+ for (const s of snapshots) {
820
+ if (s.skipped) {
821
+ placed.push({
822
+ ...s,
823
+ path: null
824
+ });
825
+ continue;
826
+ }
827
+ const { to, from } = place(s);
828
+ const path = from ? await out.move(to, from) : null;
829
+ if (path) placed.push({
830
+ ...s,
831
+ path
832
+ });
833
+ }
834
+ return placed;
835
+ }
771
836
  /** Turns, timings, activity, verdicts, checks and changed files, with their artifact files. */
772
837
  async function attemptEvidence(record, dir, out, stagingRecord) {
773
838
  const files = {};
@@ -798,9 +863,13 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
798
863
  }
799
864
  if (artifacts.length) files.artifacts = artifacts;
800
865
  }
801
- const judgements = (record?.judgements ?? []).map((j, i) => ({
866
+ const snapshots = record?.snapshots ? await placeSnapshots(record.snapshots, (s) => ({
867
+ to: `${dir}/snapshots/${fileOf(s)}`,
868
+ from: stagingRecord && join(stagingRecord.replace(/\.json$/, ".snapshots"), fileOf(s))
869
+ }), out) : [];
870
+ const judgments = (record?.judgements ?? []).map(({ call, ...j }, i) => ({
802
871
  ...j,
803
- key: judgementKey(i, j.question)
872
+ key: judgmentKey(call ?? i, j.question)
804
873
  }));
805
874
  return {
806
875
  turns: (record?.turns ?? []).map((t) => ({
@@ -813,7 +882,7 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
813
882
  unreported: t.unreported ?? [],
814
883
  error: turnError(t.transcript),
815
884
  ...t.mcpStartup ? { mcpStartup: t.mcpStartup } : {},
816
- text: t.text.length > TURN_TEXT_LIMIT ? `${t.text.slice(0, TURN_TEXT_LIMIT)}… [reeval: truncated ${t.text.length} characters; the whole message is the turn's last text event in transcript.jsonl]` : t.text
885
+ text: turnText(t.text)
817
886
  })),
818
887
  usage: sumUsage((record?.turns ?? []).map((t) => t.usage)),
819
888
  ...record?.captureUse ? { captureUse: record.captureUse } : {},
@@ -833,8 +902,11 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
833
902
  stalledTools: 0,
834
903
  maxToolMs: null
835
904
  },
836
- judgements,
905
+ judgements: judgments,
837
906
  ...record?.startRetries?.length ? { startRetries: record.startRetries } : {},
907
+ ...record?.setupProcesses?.length ? { setupProcesses: record.setupProcesses } : {},
908
+ ...snapshots.length ? { snapshots } : {},
909
+ ...record?.serves?.length ? { serves: record.serves } : {},
838
910
  ...record?.assertions?.length ? { assertions: record.assertions } : {},
839
911
  checks: (record?.checks ?? []).map((c, i) => ({
840
912
  command: c.command,
@@ -890,7 +962,7 @@ function liftOf(control, variant, pooled) {
890
962
  /**
891
963
  * Each cell's checks: every label any attempt of the eval recorded, tallied over the cell's
892
964
  * counted attempts (pass or fail). An attempt that counted without reaching a label, because
893
- * a hard `expect` before it failed, counts it `notRun`.
965
+ * an `expect.require` or a thrown error before it ended the body, counts it `notRun`.
894
966
  */
895
967
  function checkTallies(entry) {
896
968
  const labels = [...new Set(entry.attempts.flatMap((a) => (a.assertions ?? []).map((x) => x.label)))];
@@ -927,20 +999,28 @@ function discrimination(control, variant) {
927
999
  }
928
1000
  }
929
1001
  /**
930
- * A cell's outcome by its gate. `'all'` matches `{ passRate: 1 }`: every counted attempt must
931
- * pass, and it is an infra error only when nothing counted. `'none'` takes the worst attempt.
1002
+ * Marks a best-of cell's attempts past its majority's first: `added` for one that ran, `skipped`
1003
+ * for one the majority before it left unrun.
932
1004
  */
933
- function cellOutcome(outcomes, gate) {
934
- if (gate === "none") return worstOutcome(outcomes);
935
- const required = gate === "all" ? 1 : gate.passRate;
936
- const rate = passRateOf(outcomes);
937
- if (rate === null) return worstOutcome(outcomes);
938
- return rate >= required ? "pass" : "fail";
1005
+ function markBestOf(cell, attempts) {
1006
+ if (cell.bestOf === void 0) return;
1007
+ const majority = majorityOf(cell.bestOf);
1008
+ let pass = 0;
1009
+ let fail = 0;
1010
+ for (const a of attempts.filter((x) => x.cell === cell.key).sort((x, y) => x.n - y.n)) {
1011
+ if (a.n > majority) {
1012
+ if (a.outcome !== "skipped") a.bestOf = "added";
1013
+ else if (pass >= majority || fail >= majority) a.bestOf = "skipped";
1014
+ }
1015
+ if (a.outcome === "pass") pass++;
1016
+ if (a.outcome === "fail") fail++;
1017
+ }
939
1018
  }
940
1019
  /** Per-cell and per-eval counts, pass rates, lift, outcomes and fingerprints, once every attempt is in. */
941
1020
  function rollUp(evals) {
942
1021
  for (const entry of evals) {
943
1022
  for (const c of entry.cells) {
1023
+ markBestOf(c, entry.attempts);
944
1024
  const outcomes = entry.attempts.filter((a) => a.cell === c.key).map((a) => a.outcome);
945
1025
  c.runs = outcomes.length;
946
1026
  c.pass = outcomes.filter((o) => o === "pass").length;
@@ -994,7 +1074,7 @@ function gitState(input, out) {
994
1074
  return state;
995
1075
  }
996
1076
  const byName = (key) => (a, b) => key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0;
997
- function stackOf(input, seen, judgeModel, threshold, stateOptions) {
1077
+ function stackOf(input, seen, judgeModel, threshold) {
998
1078
  const packages = packageVersions(input.enginePackages);
999
1079
  return {
1000
1080
  reeval: input.reeval,
@@ -1017,8 +1097,7 @@ function stackOf(input, seen, judgeModel, threshold, stateOptions) {
1017
1097
  model: judgeModel,
1018
1098
  resolvedModels: [...seen.resolvedJudges].sort(),
1019
1099
  threshold,
1020
- version: judgeVersion(input.reeval, packages, judgeModel ?? ""),
1021
- ...stateOptions ? { state: stateOptions } : {}
1100
+ version: judgeVersion(input.reeval, packages, judgeModel ?? "")
1022
1101
  },
1023
1102
  context: [...seen.contexts.values()].sort(byName((c) => c.path)),
1024
1103
  fingerprintScope: FINGERPRINT_SCOPE
@@ -1029,8 +1108,6 @@ async function assembleRun(input) {
1029
1108
  const { config, runId, profile } = ctx;
1030
1109
  const out = new Artifacts();
1031
1110
  const runJudge = profile ? mergeJudge(config.judge, profileJudge(profile)) : config.judge;
1032
- const runStateOptions = runJudge.state && resolveStateOptions(runJudge.state);
1033
- const ownStateOptions = runStateOptions && JSON.stringify(runStateOptions) !== JSON.stringify(resolveStateOptions({})) ? runStateOptions : void 0;
1034
1111
  const judgeModel = runJudge.model === void 0 ? null : judgeModelName(runJudge.model);
1035
1112
  const configHash = sha256(JSON.stringify({
1036
1113
  include: config.include,
@@ -1046,8 +1123,7 @@ async function assembleRun(input) {
1046
1123
  const judge = sha256(JSON.stringify({
1047
1124
  model: judgeModel,
1048
1125
  threshold: runJudge.threshold,
1049
- version: judgeVersion(input.reeval, packageVersions(input.enginePackages), judgeModel ?? ""),
1050
- ...ownStateOptions ? { state: ownStateOptions } : {}
1126
+ version: judgeVersion(input.reeval, packageVersions(input.enginePackages), judgeModel ?? "")
1051
1127
  }));
1052
1128
  for (const e of engine.evals) await addAttempt(e, ctx, seen, out, judge, input.previous, input.previousArtifacts);
1053
1129
  if (input.previous) {
@@ -1066,7 +1142,7 @@ async function assembleRun(input) {
1066
1142
  errors: [],
1067
1143
  formattedErrors: [],
1068
1144
  meta: void 0
1069
- }, a, seen, out, judge, input.previousArtifacts, entry.sources);
1145
+ }, a, seen, out, judge, input.previous, input.previousArtifacts);
1070
1146
  }
1071
1147
  }
1072
1148
  const evals = [...seen.byEval.values()];
@@ -1080,6 +1156,15 @@ async function assembleRun(input) {
1080
1156
  for (const e of evals) {
1081
1157
  const asked = input.answers?.evals.get(e.id);
1082
1158
  if (asked?.length) e.answers = asked.map((q) => withInput(e, q, out));
1159
+ else if (e.attempts.length && e.attempts.every((a) => seen.kept.has(a.key))) {
1160
+ const before = input.previous?.evals.find((p) => p.id === e.id)?.answers ?? [];
1161
+ const answers = [];
1162
+ for (const q of before) answers.push(q.input ? {
1163
+ ...q,
1164
+ input: await copyRef(q.input, input.previousArtifacts, out)
1165
+ } : q);
1166
+ if (answers.length) e.answers = answers;
1167
+ }
1083
1168
  for (const a of e.attempts) {
1084
1169
  const own = input.answers?.attempts.get(a.key);
1085
1170
  if (own?.length) a.answers = own.map((q) => withAttemptInput(e, a, q, out));
@@ -1102,7 +1187,7 @@ async function assembleRun(input) {
1102
1187
  git: gitState(input, out),
1103
1188
  runner: input.runner,
1104
1189
  dependencies: { lockfiles: input.lockfiles },
1105
- stack: stackOf(input, seen, judgeModel, runJudge.threshold, ownStateOptions),
1190
+ stack: stackOf(input, seen, judgeModel, runJudge.threshold),
1106
1191
  environments: [...seen.environments.values()].sort(byName((e) => e.name)),
1107
1192
  profile: profile ? {
1108
1193
  name: profile.name,
@@ -1111,7 +1196,7 @@ async function assembleRun(input) {
1111
1196
  harness: ids(profile.defaults.harness),
1112
1197
  environment: profile.defaults.environment?.name,
1113
1198
  sandbox: (profile.defaults.environment || profile.defaults.sandbox) && sandboxId(profileSandbox(config, profile.defaults)),
1114
- runs: profile.defaults.runs,
1199
+ ...profile.defaults.runs !== void 0 ? runsOf(profile.defaults.runs) : {},
1115
1200
  timeout: profile.defaults.timeout,
1116
1201
  env: profile.defaults.env && Object.keys(profile.defaults.env).sort()
1117
1202
  })),
@@ -1128,7 +1213,7 @@ async function assembleRun(input) {
1128
1213
  config: {
1129
1214
  sandbox: plannedSandboxes(input.plan.sandboxes, profileSandbox(config, profile?.defaults)),
1130
1215
  environment: config.environment?.name ?? null,
1131
- runs: config.runs,
1216
+ ...config.runs !== void 0 ? runsOf(config.runs) : { runs: 1 },
1132
1217
  judgeModel,
1133
1218
  threshold: runJudge.threshold,
1134
1219
  timeoutMs: config.timeoutMs,
@@ -1151,10 +1236,11 @@ async function assembleRun(input) {
1151
1236
  ...families.length ? { families } : {},
1152
1237
  ...engine.unhandledErrors.length ? { errors: engine.unhandledErrors.map((e) => redactSecrets(e)) } : {}
1153
1238
  },
1154
- artifacts: out.files
1239
+ artifacts: out.files,
1240
+ moves: out.moves
1155
1241
  };
1156
1242
  }
1157
1243
  //#endregion
1158
- export { fingerprintOf as a, mergeOutcome as c, cellOutcome as i, transcriptLines as l, assembleRun as n, gatewayPart as o, capText as r, judgementKey as s, OUTPUT_CAP as t, worstOutcome as u };
1244
+ export { gatewayPart as a, rawLines as c, turnText as d, fingerprintOf as i, transcriptLines as l, assembleRun as n, judgmentKey as o, capText as r, mergeOutcome as s, OUTPUT_CAP as t, turnError as u };
1159
1245
 
1160
- //# sourceMappingURL=assemble-BDEpelpi.js.map
1246
+ //# sourceMappingURL=assemble-BVbI6NW-.js.map