reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -0,0 +1 @@
1
+ {"version":3,"file":"app-files-ChbvRx3P.js","names":[],"sources":["../src/studio/app-files.ts"],"mappings":";AAAA,MAAa,YAAY;CAAC,6BAA4B,EAAC,QAAO,6n4DAA4n4D;CAAE,4BAA2B,EAAC,QAAO,m26pBAAm26pB;CAAE,eAAc,EAAC,QAAO,yTAAwT;CAAE,cAAa,EAAC,QAAO,ojBAAkjB;AAAC"}
@@ -1,18 +1,20 @@
1
1
  import { r as isTimeout } from "./errors-B6BSMs03.js";
2
- import { C as servedEffortOf, S as resolvedEffortOf, b as resolveStateOptions, d as profileJudge, p as sandboxId, s as harnessId, u as plannedSandboxes, x as harnessEffort, y as mergeJudge } from "./options-CyAyLjgJ.js";
3
- import { T as profileSandbox } from "./environment-B4IuURK1.js";
2
+ import { D as profileSandbox } from "./environment-BOFgzFDg.js";
4
3
  import { i as evalSlug, l as sha256, n as attemptKey, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
5
- import { s as redactSecrets } from "./redact-CRGdfV2s.js";
6
- import { c as judgeModelName } from "./judge-CEDIfRiU.js";
7
- import { i as resolveQuestionState } from "./state-CeR6oz0k.js";
8
- import { i as sumUsage, r as promptTokens } from "./usage-CleNKkly.js";
9
- import { r as patchForJudge, t as collapseDependencies } from "./patch-uMq9rCl_.js";
10
- import { _t as round3, dt as containsZero, ft as liftInterval, gt as pairedLiftInterval, ht as median, mt as meanInterval, pt as mcnemar, rt as effortSample, vt as signedRank, yt as wilson } from "./model-UgHDMhI1.js";
11
- import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-C0i_ujQZ.js";
12
- import { t as checkVerdict } from "./contrasts-B4nOhn0Z.js";
13
- import { a as previousAttempt, n as isCompletedAttempt } from "./resume-8-nnW8iO.js";
14
- import { dirname, join, relative, resolve, sep } from "node:path";
15
- import { readFile, stat } from "node:fs/promises";
4
+ import { A as harnessId, F as plannedSandboxes, G as resolveStateOptions, I as profileJudge, R as sandboxId, W as mergeJudge, ct as resolvedEffortOf, lt as servedEffortOf, ot as resolveQuestionState, st as harnessEffort } from "./config-k6m7XbAz.js";
5
+ import { l as judgeModelName } from "./judge-CKdgEXN7.js";
6
+ import { f as redactJsonText, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
7
+ import { i as formatDuration } from "./text-DTigcA64.js";
8
+ import { a as mcnemar, d as round3, f as signedRank, l as pairedLiftInterval, n as containsZero, o as meanInterval, p as wilson, r as liftInterval, s as median } from "./stats-hfpuYsAM.js";
9
+ import { i as sumUsage, r as promptTokens } from "./usage-BNY1ABrU.js";
10
+ import { r as patchForJudge, t as collapseDependencies } from "./patch-DKJiC5Sq.js";
11
+ import { a as sourcesPart, r as evalFile } from "./eval-source-Dje3rERU.js";
12
+ import { Et as worstOutcome, I as checkTries, St as cellOutcome, Tt as passRateOf, st as effortSample, wt as gatePart } from "./model-CTnyvh7H.js";
13
+ import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-DrJ687ut.js";
14
+ import { t as checkVerdict } from "./contrasts-DLZF4dGF.js";
15
+ import { n as isCompletedAttempt, o as previousAttempt } from "./resume-j-rcIryi.js";
16
+ import { join } from "node:path";
17
+ import { readFile } from "node:fs/promises";
16
18
  //#region src/history/families.ts
17
19
  const isCounted = (a) => a.outcome === "pass" || a.outcome === "fail";
18
20
  function sideOf(entry, cell) {
@@ -182,65 +184,27 @@ function familiesOf(evals, minLiftOf) {
182
184
  });
183
185
  }
184
186
  //#endregion
185
- //#region src/history/sources.ts
186
- /** `from '…'`, `import '…'`, `import('…')` and `export … from '…'`, with a literal specifier. */
187
- const SPECIFIER = /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+)(['"])((?:\.{1,2}\/|\/)[^'"]+)\1/g;
188
- const CANDIDATES = (path) => {
189
- return [
190
- path,
191
- .../\.(m?js|cjs)$/.exec(path) ? [path.replace(/\.(m?js|cjs)$/, ".ts"), path.replace(/\.(m?js|cjs)$/, ".tsx")] : [],
192
- `${path}.ts`,
193
- `${path}.tsx`,
194
- `${path}.js`,
195
- `${path}.mjs`,
196
- join(path, "index.ts"),
197
- join(path, "index.js")
198
- ];
199
- };
200
- async function isFile(path) {
201
- return (await stat(path).catch(() => null))?.isFile() ?? false;
202
- }
203
- async function resolveImport(from, specifier) {
204
- const base = specifier.startsWith("/") ? specifier : resolve(dirname(from), specifier);
205
- for (const candidate of CANDIDATES(base)) if (await isFile(candidate)) return candidate;
206
- return null;
207
- }
208
- /**
209
- * The eval file and every local module it imports, transitively, by path from `root` and
210
- * content hash, sorted by path. Only relative and absolute specifiers are followed: packages
211
- * (`reeval`, `node:*`, anything in `node_modules`) aren't traced.
212
- */
213
- async function traceSources(root, file) {
214
- const seen = /* @__PURE__ */ new Map();
215
- const queue = [resolve(root, file)];
216
- while (queue.length) {
217
- const path = queue.shift();
218
- if (seen.has(path) || path.split(sep).includes("node_modules")) continue;
219
- const bytes = await readFile(path).catch(() => null);
220
- if (!bytes) continue;
221
- seen.set(path, {
222
- path: relative(root, path).split(sep).join("/"),
223
- sha256: sha256(bytes)
224
- });
225
- for (const match of bytes.toString("utf8").matchAll(SPECIFIER)) {
226
- const found = await resolveImport(path, match[2]);
227
- if (found && !seen.has(found)) queue.push(found);
228
- }
229
- }
230
- return [...seen.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
231
- }
232
- //#endregion
233
187
  //#region src/history/assemble.ts
234
188
  /** Final messages are cut at this size in the record; `transcript.jsonl` keeps them whole. */
235
189
  const TURN_TEXT_LIMIT = 8192;
236
190
  const TURN_ERROR_LIMIT = 1024;
191
+ /**
192
+ * `text` masked when it's longer than `limit`, so a secret the cut would go through is masked
193
+ * whole. Text that fits is masked when the record is written.
194
+ */
195
+ const maskedOver = (text, limit) => text.length > limit ? redactSecrets(text) : text;
237
196
  /** A turn's first error event, such as the provider's "Credit balance is too low". */
238
197
  function turnError(transcript) {
239
198
  const found = transcript?.find((e) => e.type === "error");
240
199
  if (found?.type !== "error") return null;
241
- const message = found.message.trim();
200
+ const message = maskedOver(found.message.trim(), TURN_ERROR_LIMIT);
242
201
  return message.length > TURN_ERROR_LIMIT ? `${message.slice(0, TURN_ERROR_LIMIT)}…` : message;
243
202
  }
203
+ /** A turn's final message, cut at `TURN_TEXT_LIMIT`. */
204
+ function turnText(text) {
205
+ const s = maskedOver(text, TURN_TEXT_LIMIT);
206
+ return s.length > TURN_TEXT_LIMIT ? `${s.slice(0, TURN_TEXT_LIMIT)}… [reeval: truncated ${s.length} characters; the whole message is the turn's last text event in transcript.jsonl]` : s;
207
+ }
244
208
  /** Tokens across every turn that reported them. */
245
209
  function costOf(attempts) {
246
210
  const usage = sumUsage(attempts.map((a) => a.usage));
@@ -262,7 +226,8 @@ const OUTPUT_CAP = 131072;
262
226
  const HEAD = 98304;
263
227
  const TAIL = 16384;
264
228
  /** Keeps evidence readable without letting one `cat` of a lockfile dominate the record. */
265
- function capText(text, cap = OUTPUT_CAP) {
229
+ function capText(raw, cap = OUTPUT_CAP) {
230
+ const text = maskedOver(raw, cap);
266
231
  if (text.length <= cap) return text;
267
232
  const omitted = text.length - HEAD - TAIL;
268
233
  return `${text.slice(0, HEAD)}\n[reeval: truncated ${omitted} of ${text.length} characters, sha256 ${sha256(text)}]\n${text.slice(-16384)}`;
@@ -295,30 +260,25 @@ function mergeOutcome(e, record) {
295
260
  error
296
261
  };
297
262
  }
298
- /** Any failure fails; then infra errors; then all-skipped; else pass. */
299
- function worstOutcome(outcomes) {
300
- if (outcomes.includes("fail")) return "fail";
301
- if (outcomes.includes("infra_error")) return "infra_error";
302
- if (outcomes.length > 0 && outcomes.every((o) => o === "skipped")) return "skipped";
303
- return outcomes.length ? "pass" : "skipped";
304
- }
305
- const passRateOf = (outcomes) => {
306
- const pass = outcomes.filter((o) => o === "pass").length;
307
- const counted = pass + outcomes.filter((o) => o === "fail").length;
308
- return counted ? pass / counted : null;
309
- };
310
- /** Without the operational limits (`maxConcurrent`, `startTimeout`), which don't change results. */
311
- const withoutCaps = ({ maxConcurrent: _, startTimeout: __, ...rest }) => rest;
263
+ /**
264
+ * The harness as the measurement sees it: without the operational limits (`maxConcurrent`,
265
+ * `startTimeout`), which don't change results, and without the route, which `gateway` holds.
266
+ */
267
+ const withoutCaps = ({ maxConcurrent: _, startTimeout: __, gateway: ___, ...rest }) => rest;
312
268
  const harnessModel = (h) => h.kind === "fake" ? null : h.model ?? null;
313
269
  /** A cell fingerprint: the hash of its parts, so each part can be compared on its own. */
314
270
  const fingerprintOf = (parts) => sha256(JSON.stringify(parts));
271
+ /** The gateway's part of a fingerprint: none on `direct`, so a cell without a gateway keeps its fingerprint. */
272
+ const gatewayPart = (gateway) => gateway && gateway.name !== "direct" ? { gateway: gateway.name } : {};
315
273
  /** `<index>:<question hash>`, which follows one verdict across runs. */
316
- const judgementKey = (index, question) => `${index}:${sha256(question).slice(0, 12)}`;
274
+ const judgmentKey = (index, question) => `${index}:${sha256(question).slice(0, 12)}`;
275
+ const JSON_FILE = /\.(?:json|jsonl|ndjson)$/;
317
276
  /** Collects artifact files, redacting before anything is hashed. */
318
277
  var Artifacts = class {
319
278
  files = /* @__PURE__ */ new Map();
279
+ /** JSON and JSON Lines files are masked inside their strings only, so they stay valid. */
320
280
  add(path, content) {
321
- const bytes = Buffer.from(redactSecrets(content));
281
+ const bytes = Buffer.from(JSON_FILE.test(path) ? redactJsonText(content) : redactSecrets(content));
322
282
  this.files.set(path, bytes);
323
283
  return {
324
284
  path,
@@ -340,26 +300,28 @@ var Artifacts = class {
340
300
  return this.add(path, `${JSON.stringify(value, null, 2)}\n`);
341
301
  }
342
302
  };
303
+ /** A screenshot with its attempt's id resolved to the capture's path in the record. */
304
+ function screenshotRef(s, byId) {
305
+ if ("missing" in s) return s;
306
+ const { attempt, ...shot } = s;
307
+ const ref = byId.get(attempt)?.files.artifacts?.find((a) => a.name === s.name);
308
+ return ref ? {
309
+ ...shot,
310
+ path: ref.path
311
+ } : shot;
312
+ }
343
313
  /** The answer, with what its model was sent saved as `questions/<n>.json` beside the eval's attempts. */
344
- function withInput(e, { answer, calls }, n, out) {
314
+ function withInput(e, { n, answer, calls }, out) {
345
315
  if (!calls.length) return answer;
346
316
  const byId = new Map(e.attempts.map((a) => [a.id, a]));
347
317
  const call = ({ screenshots, ...rest }) => ({
348
318
  ...rest,
349
- ...screenshots ? { screenshots: screenshots.map((s) => {
350
- if ("missing" in s) return s;
351
- const { attempt, ...shot } = s;
352
- const ref = byId.get(attempt)?.files.artifacts?.find((a) => a.name === s.name);
353
- return ref ? {
354
- ...shot,
355
- path: ref.path
356
- } : shot;
357
- }) } : {}
319
+ ...screenshots ? { screenshots: screenshots.map((s) => screenshotRef(s, byId)) } : {}
358
320
  });
359
321
  const doc = {
360
322
  question: answer.question,
361
323
  model: answer.model,
362
- stateOptions: answer.stateOptions ?? resolveQuestionState(void 0),
324
+ stateOptions: answer.stateOptions ?? resolveQuestionState(void 0, true),
363
325
  ...answer.artifacts !== void 0 ? { artifacts: answer.artifacts } : {},
364
326
  calls: calls.map(call)
365
327
  };
@@ -368,6 +330,26 @@ function withInput(e, { answer, calls }, n, out) {
368
330
  input: out.json(`${e.slug}/questions/${n}.json`, doc)
369
331
  };
370
332
  }
333
+ /** An attempt's own answer, with what its model was sent saved as `questions/<n>.json` in its directory. */
334
+ function withAttemptInput(e, a, { n, answer, call }, out) {
335
+ if (!call) return answer;
336
+ const { screenshots, ...rest } = call;
337
+ const byId = /* @__PURE__ */ new Map([[a.id, a]]);
338
+ const doc = {
339
+ question: answer.question,
340
+ model: answer.model,
341
+ stateOptions: answer.stateOptions,
342
+ ...answer.artifacts !== void 0 ? { artifacts: answer.artifacts } : {},
343
+ ...answer.choices ? { choices: answer.choices } : {},
344
+ ...rest,
345
+ ...screenshots ? { screenshots: screenshots.map((s) => screenshotRef(s, byId)) } : {}
346
+ };
347
+ const dir = `${e.slug}/${evalSlug(a.cell)}/${a.n}`;
348
+ return {
349
+ ...answer,
350
+ input: out.json(`${dir}/questions/${n}.json`, doc)
351
+ };
352
+ }
371
353
  const endOf = (startedAt, durationMs) => new Date(Date.parse(startedAt) + durationMs).toISOString();
372
354
  /** One JSON object per line: each turn's prompt, its events, then a `turn-end`. */
373
355
  function transcriptLines(record) {
@@ -408,13 +390,14 @@ function transcriptLines(record) {
408
390
  });
409
391
  return `${lines.join("\n")}\n`;
410
392
  }
411
- /** Every stream part the harness sent, for debugging the normaliser (`REEVAL_RAW_TRANSCRIPT=1`). */
393
+ /** Every stream part the harness sent, for debugging the normalizer (`REEVAL_RAW_TRANSCRIPT=1`). */
412
394
  function rawLines(record) {
413
395
  if (!record.turns.some((t) => t.raw)) return null;
414
- return `${record.turns.flatMap((turn, index) => (turn.raw ?? []).map((part) => JSON.stringify({
396
+ const lines = record.turns.flatMap((turn, index) => (turn.raw ?? []).map((part) => JSON.stringify({
415
397
  turn: index,
416
398
  ...part
417
- }))).join("\n")}\n`;
399
+ })));
400
+ return `${redactSplitSecrets(lines).join("\n")}\n`;
418
401
  }
419
402
  function activity(record) {
420
403
  const events = record.turns.flatMap((t) => t.transcript);
@@ -440,7 +423,15 @@ const block = (label, text) => {
440
423
  const body = capText(text);
441
424
  return `--- ${label}\n${body}${body.endsWith("\n") || !body ? "" : "\n"}`;
442
425
  };
443
- const checkLog = (c) => `$ ${c.command}\nexit ${c.exitCode}\n${block("stdout", c.stdout)}${block("stderr", c.stderr)}`;
426
+ const output = (r) => `${block("stdout", r.stdout)}${block("stderr", r.stderr)}`;
427
+ /** The command, its exit and output; with `retries`, each try's exit and output in turn. */
428
+ function checkLog(c) {
429
+ const head = `$ ${c.command}\nexit ${c.exitCode}`;
430
+ if (!c.retries || !c.tries?.length) return `${head}\n${output(c)}`;
431
+ const of = c.retries + 1;
432
+ const tries = c.tries.map((t, i) => `\ntry ${i + 1} of ${of}: exit ${t.exitCode} after ${formatDuration(t.durationMs)}\n${output(t)}`);
433
+ return `${head}, ${checkTries(c) ?? `passed on try 1 of ${of}`}\n${tries.join("")}`;
434
+ }
444
435
  /** Dependency and binary noise dropped, and each remaining file's diff capped. */
445
436
  function historyPatch(patch) {
446
437
  return patchForJudge(patch).split(/(?=^diff --git )/m).map((part) => capText(part)).join("");
@@ -479,6 +470,8 @@ var Seen = class {
479
470
  environments = /* @__PURE__ */ new Map();
480
471
  harnesses = /* @__PURE__ */ new Map();
481
472
  resolvedHarnesses = /* @__PURE__ */ new Map();
473
+ /** Keys of the attempts kept from the previous record on resume. */
474
+ kept = /* @__PURE__ */ new Set();
482
475
  /** Registry digests found by the preflight, by image id. */
483
476
  digests;
484
477
  constructor(sandbox) {
@@ -511,7 +504,28 @@ var Seen = class {
511
504
  for (const t of record?.turns ?? []) if (t.model) resolved.add(t.model);
512
505
  this.resolvedHarnesses.set(cell.harness, resolved);
513
506
  }
514
- entry(e, sources) {
507
+ /**
508
+ * A kept attempt has no worker record left, so what `add` reads from one comes from the
509
+ * previous record instead: its image, environment, context files and resolved models.
510
+ */
511
+ addKept(previous, kept, cell, environmentDigest, planned) {
512
+ this.kept.add(kept.key);
513
+ for (const c of kept.context) this.contexts.set(`${c.path}\0${c.sha256}`, c);
514
+ for (const j of kept.judgements) if (j.resolvedModel) this.resolvedJudges.add(j.resolvedModel);
515
+ const image = previous.stack.images.find((i) => i.id === kept.imageId);
516
+ if (image && !this.images.has(image.id)) this.images.set(image.id, image);
517
+ const environment = previous.environments.find((x) => x.digest === environmentDigest);
518
+ if (environment) this.environments.set(`${environment.name}\0${environment.digest}`, environment);
519
+ const harness = previous.stack.harnesses.find((h) => h.id === cell.harness);
520
+ if (!planned && harness && !this.harnesses.has(cell.harness)) {
521
+ const { resolvedModels: _, resolvedEffort: __, ...rest } = harness;
522
+ this.harnesses.set(cell.harness, rest);
523
+ }
524
+ const resolved = this.resolvedHarnesses.get(cell.harness) ?? /* @__PURE__ */ new Set();
525
+ for (const t of kept.turns) if (t.model) resolved.add(t.model);
526
+ this.resolvedHarnesses.set(cell.harness, resolved);
527
+ }
528
+ entry(e, sources, from) {
515
529
  let entry = this.byEval.get(e.id);
516
530
  if (!entry) {
517
531
  const key = e.id;
@@ -527,6 +541,8 @@ var Seen = class {
527
541
  ...e.attempt?.family ? { family: familyOf(e.attempt.family) } : {},
528
542
  fingerprint: "",
529
543
  sources,
544
+ ...from.source ? { source: from.source } : {},
545
+ ...from.revision !== void 0 ? { revision: from.revision } : {},
530
546
  outcome: "skipped",
531
547
  todo: e.todo === true,
532
548
  passRate: null,
@@ -553,6 +569,8 @@ const UNKNOWN_CELL = {
553
569
  };
554
570
  /** minLift per cell, kept aside while attempts are added; see `rollUp`. */
555
571
  const minLifts = /* @__PURE__ */ new WeakMap();
572
+ /** Cells whose fingerprint came from an attempt that ran, not one that was skipped. */
573
+ const measured = /* @__PURE__ */ new WeakSet();
556
574
  const EMPTY_PARTS = {
557
575
  sources: "",
558
576
  harness: "",
@@ -562,11 +580,12 @@ const EMPTY_PARTS = {
562
580
  judge: "",
563
581
  context: ""
564
582
  };
565
- function cellOf(entry, cell, gate, harness) {
583
+ function cellOf(entry, cell, gate, harness, gateway) {
566
584
  let found = entry.cells.find((c) => c.key === cell.key);
567
585
  if (!found) {
568
586
  found = {
569
587
  ...cell,
588
+ ...gateway ? { gateway: gateway.name } : {},
570
589
  fingerprint: "",
571
590
  fingerprintParts: EMPTY_PARTS,
572
591
  environmentDigest: null,
@@ -594,17 +613,25 @@ async function copyRef(ref, from, out) {
594
613
  return out.add(ref.path, Buffer.from(bytes).toString("utf8"));
595
614
  }
596
615
  /** A completed attempt from a previous assemble of this runId, kept on resume. */
597
- async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts, sources) {
598
- const cell = e.attempt?.cell ?? {
616
+ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts) {
617
+ const before = previous.evals.find((p) => p.id === e.id);
618
+ const beforeCell = before?.cells.find((c) => c.key === kept.cell);
619
+ const cell = e.attempt?.cell ?? (beforeCell && {
620
+ key: beforeCell.key,
621
+ harness: beforeCell.harness,
622
+ environment: beforeCell.environment,
623
+ sandbox: beforeCell.sandbox,
624
+ role: beforeCell.role,
625
+ ...beforeCell.variant !== void 0 ? { variant: beforeCell.variant } : {}
626
+ }) ?? {
599
627
  ...UNKNOWN_CELL,
600
628
  key: kept.cell,
601
629
  harness: kept.cell.split("/")[0] ?? "unknown",
602
630
  environment: kept.cell.split("/")[1] ?? "unknown",
603
631
  sandbox: kept.cell.split("/")[2]?.replace(/#\d+$/, "") ?? "unknown"
604
632
  };
605
- if (!seen.sources.has(e.file) && sources) seen.sources.set(e.file, sources);
606
- if (!seen.sources.has(e.file)) seen.sources.set(e.file, []);
607
- const resolved = sources ?? seen.sources.get(e.file) ?? [];
633
+ const sources = before?.sources ?? [];
634
+ const { source, revision } = e.attempt ?? before ?? {};
608
635
  const stub = {
609
636
  id: e.id,
610
637
  key: e.key,
@@ -619,8 +646,21 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts,
619
646
  meta: void 0
620
647
  };
621
648
  seen.add(null, cell, stub);
622
- const entry = seen.entry(stub, resolved);
623
- const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? "all", e.attempt?.harness);
649
+ seen.addKept(previous, kept, cell, beforeCell?.environmentDigest ?? null, e.attempt !== void 0);
650
+ const entry = seen.entry(stub, sources, {
651
+ source,
652
+ revision
653
+ });
654
+ const created = !entry.cells.some((c) => c.key === cell.key);
655
+ const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? beforeCell?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway);
656
+ if (created && beforeCell && !e.attempt) {
657
+ cellRecord.harnessKind = beforeCell.harnessKind;
658
+ cellRecord.model = beforeCell.model;
659
+ cellRecord.effort = beforeCell.effort;
660
+ if (beforeCell.gateway !== void 0) cellRecord.gateway = beforeCell.gateway;
661
+ }
662
+ const minLift = e.attempt ? e.attempt.minLift : beforeCell?.lift?.minLift ?? previous.families?.find((f) => f.id === before?.family?.id)?.lifts.find((l) => l.variant === kept.cell)?.minLift;
663
+ if (minLift !== void 0) minLifts.set(cellRecord, minLift);
624
664
  const files = {};
625
665
  for (const [name, ref] of Object.entries(kept.files)) {
626
666
  if (!ref) continue;
@@ -644,21 +684,36 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts,
644
684
  ...c,
645
685
  log: c.log ? await copyRef(c.log, previousArtifacts, out) : c.log
646
686
  });
687
+ const answers = [];
688
+ for (const q of kept.answers ?? []) answers.push(q.input ? {
689
+ ...q,
690
+ input: await copyRef(q.input, previousArtifacts, out)
691
+ } : q);
647
692
  entry.attempts.push({
648
693
  ...kept,
649
694
  seq: null,
650
695
  files,
651
- checks
696
+ checks,
697
+ ...answers.length ? { answers } : {}
652
698
  });
653
- if (!cellRecord.fingerprint && kept.outcome !== "skipped") {
699
+ if (measured.has(cellRecord)) return;
700
+ measured.add(cellRecord);
701
+ if (beforeCell?.fingerprint) {
702
+ cellRecord.fingerprintParts = beforeCell.fingerprintParts;
703
+ cellRecord.fingerprint = beforeCell.fingerprint;
704
+ cellRecord.environmentDigest = beforeCell.environmentDigest;
705
+ if (cellRecord.overrides.length === 0) cellRecord.overrides = [...beforeCell.overrides];
706
+ } else {
654
707
  const parts = {
655
- sources: sha256(JSON.stringify(resolved)),
708
+ sources: sourcesPart(source, revision, sources),
656
709
  harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
657
710
  environment: cellRecord.environmentDigest,
658
711
  image: kept.imageId,
659
712
  agentCli: null,
660
713
  judge,
661
- context: sha256(JSON.stringify(kept.context))
714
+ context: sha256(JSON.stringify(kept.context)),
715
+ ...gatewayPart(e.attempt?.gateway),
716
+ ...gatePart(cellRecord.gate)
662
717
  };
663
718
  cellRecord.fingerprintParts = parts;
664
719
  cellRecord.fingerprint = fingerprintOf(parts);
@@ -669,31 +724,40 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
669
724
  const cell = e.attempt?.cell ?? UNKNOWN_CELL;
670
725
  const n = e.attempt?.n ?? 1;
671
726
  const kept = previousAttempt(previous, e.key);
672
- if (e.state === "skipped" && kept && isCompletedAttempt(kept.outcome, kept.error)) {
673
- await addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts);
727
+ if (previous && e.state === "skipped" && kept && isCompletedAttempt(kept.outcome, kept.error)) {
728
+ await addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts);
674
729
  return;
675
730
  }
676
731
  const record = await readRecord(join(ctx.stagingDir, "evals", recordFileName(e.key)));
677
732
  const { outcome, error } = mergeOutcome(e, record);
678
- if (!seen.sources.has(e.file)) seen.sources.set(e.file, await traceSources(ctx.config.root, e.file));
733
+ if (!seen.sources.has(e.file)) seen.sources.set(e.file, await evalFile(ctx.config.root, e.file));
679
734
  const sources = seen.sources.get(e.file) ?? [];
735
+ const source = e.attempt?.source;
736
+ const revision = e.attempt?.revision;
680
737
  const context = record?.context ?? [];
681
738
  seen.add(record, cell, e);
682
- const entry = seen.entry(e, sources);
683
- const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? "all", e.attempt?.harness);
739
+ const entry = seen.entry(e, sources, {
740
+ source,
741
+ revision
742
+ });
743
+ if (record) entry.sources = sources;
744
+ const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway);
684
745
  if (e.attempt?.minLift !== void 0) minLifts.set(cellRecord, e.attempt.minLift);
685
746
  if (record?.environment?.overrides.length && cellRecord.overrides.length === 0) cellRecord.overrides = [...record.environment.overrides];
686
747
  const environment = record?.environment;
687
748
  if (environment && environment.name !== "-") cellRecord.environmentDigest = environment.digest;
688
749
  if (!cellRecord.fingerprint || record) {
750
+ if (record) measured.add(cellRecord);
689
751
  const parts = {
690
- sources: sha256(JSON.stringify(sources)),
752
+ sources: sourcesPart(source, revision, sources),
691
753
  harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
692
754
  environment: environment?.digest ?? null,
693
755
  image: record?.image?.id ?? null,
694
756
  agentCli: record?.image?.agentCli ?? null,
695
757
  judge,
696
- context: sha256(JSON.stringify(context))
758
+ context: sha256(JSON.stringify(context)),
759
+ ...gatewayPart(e.attempt?.gateway),
760
+ ...gatePart(cellRecord.gate)
697
761
  };
698
762
  cellRecord.fingerprintParts = parts;
699
763
  cellRecord.fingerprint = fingerprintOf(parts);
@@ -714,6 +778,9 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
714
778
  context,
715
779
  egress: record?.egress ?? null,
716
780
  servedModels: record?.servedModels ?? null,
781
+ ...record?.askedModels?.length ? { askedModels: record.askedModels } : {},
782
+ ...record?.upstreams?.length ? { upstreams: record.upstreams } : {},
783
+ ...record?.reportedCostUsd !== void 0 ? { reportedCostUsd: record.reportedCostUsd } : {},
717
784
  effort: e.attempt ? harnessEffort(e.attempt.harness) ?? null : null,
718
785
  servedEffort: servedEffortOf([...record?.servedModels ?? [], ...(record?.turns ?? []).map((t) => t.model)]) ?? null,
719
786
  outcome,
@@ -752,9 +819,9 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
752
819
  }
753
820
  if (artifacts.length) files.artifacts = artifacts;
754
821
  }
755
- const judgements = (record?.judgements ?? []).map((j, i) => ({
822
+ const judgments = (record?.judgements ?? []).map(({ call, ...j }, i) => ({
756
823
  ...j,
757
- key: judgementKey(i, j.question)
824
+ key: judgmentKey(call ?? i, j.question)
758
825
  }));
759
826
  return {
760
827
  turns: (record?.turns ?? []).map((t) => ({
@@ -767,7 +834,7 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
767
834
  unreported: t.unreported ?? [],
768
835
  error: turnError(t.transcript),
769
836
  ...t.mcpStartup ? { mcpStartup: t.mcpStartup } : {},
770
- text: t.text.length > TURN_TEXT_LIMIT ? `${t.text.slice(0, TURN_TEXT_LIMIT)}… [reeval: truncated ${t.text.length} characters; the whole message is the turn's last text event in transcript.jsonl]` : t.text
837
+ text: turnText(t.text)
771
838
  })),
772
839
  usage: sumUsage((record?.turns ?? []).map((t) => t.usage)),
773
840
  ...record?.captureUse ? { captureUse: record.captureUse } : {},
@@ -787,14 +854,22 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
787
854
  stalledTools: 0,
788
855
  maxToolMs: null
789
856
  },
790
- judgements,
857
+ judgements: judgments,
791
858
  ...record?.startRetries?.length ? { startRetries: record.startRetries } : {},
859
+ ...record?.setupProcesses?.length ? { setupProcesses: record.setupProcesses } : {},
792
860
  ...record?.assertions?.length ? { assertions: record.assertions } : {},
793
861
  checks: (record?.checks ?? []).map((c, i) => ({
794
862
  command: c.command,
795
863
  exitCode: c.exitCode,
796
864
  durationMs: c.durationMs,
797
- log: out.add(`${dir}/checks/${i + 1}.log`, checkLog(c))
865
+ log: out.add(`${dir}/checks/${i + 1}.log`, checkLog(c)),
866
+ ...c.retries && c.tries ? {
867
+ retries: c.retries,
868
+ tries: c.tries.map((t) => ({
869
+ exitCode: t.exitCode,
870
+ durationMs: t.durationMs
871
+ }))
872
+ } : {}
798
873
  })),
799
874
  changedFiles: changedFiles(record?.diff?.files ?? []),
800
875
  files
@@ -873,17 +948,6 @@ function discrimination(control, variant) {
873
948
  if (verdict) check.verdict = verdict;
874
949
  }
875
950
  }
876
- /**
877
- * A cell's outcome by its gate. `'all'` matches `{ passRate: 1 }`: every counted attempt must
878
- * pass, and it is an infra error only when nothing counted. `'none'` takes the worst attempt.
879
- */
880
- function cellOutcome(outcomes, gate) {
881
- if (gate === "none") return worstOutcome(outcomes);
882
- const required = gate === "all" ? 1 : gate.passRate;
883
- const rate = passRateOf(outcomes);
884
- if (rate === null) return worstOutcome(outcomes);
885
- return rate >= required ? "pass" : "fail";
886
- }
887
951
  /** Per-cell and per-eval counts, pass rates, lift, outcomes and fingerprints, once every attempt is in. */
888
952
  function rollUp(evals) {
889
953
  for (const entry of evals) {
@@ -1013,7 +1077,7 @@ async function assembleRun(input) {
1013
1077
  errors: [],
1014
1078
  formattedErrors: [],
1015
1079
  meta: void 0
1016
- }, a, seen, out, judge, input.previousArtifacts, entry.sources);
1080
+ }, a, seen, out, judge, input.previous, input.previousArtifacts);
1017
1081
  }
1018
1082
  }
1019
1083
  const evals = [...seen.byEval.values()];
@@ -1025,8 +1089,21 @@ async function assembleRun(input) {
1025
1089
  const families = familiesOf(evals, (cell) => minLifts.get(cell));
1026
1090
  sequence(evals);
1027
1091
  for (const e of evals) {
1028
- const asked = input.answers?.get(e.id);
1029
- if (asked?.length) e.answers = asked.map((q, i) => withInput(e, q, i + 1, out));
1092
+ const asked = input.answers?.evals.get(e.id);
1093
+ if (asked?.length) e.answers = asked.map((q) => withInput(e, q, out));
1094
+ else if (e.attempts.length && e.attempts.every((a) => seen.kept.has(a.key))) {
1095
+ const before = input.previous?.evals.find((p) => p.id === e.id)?.answers ?? [];
1096
+ const answers = [];
1097
+ for (const q of before) answers.push(q.input ? {
1098
+ ...q,
1099
+ input: await copyRef(q.input, input.previousArtifacts, out)
1100
+ } : q);
1101
+ if (answers.length) e.answers = answers;
1102
+ }
1103
+ for (const a of e.attempts) {
1104
+ const own = input.answers?.attempts.get(a.key);
1105
+ if (own?.length) a.answers = own.map((q) => withAttemptInput(e, a, q, out));
1106
+ }
1030
1107
  }
1031
1108
  const attempts = evals.flatMap((e) => e.attempts);
1032
1109
  const count = (o) => attempts.filter((a) => a.outcome === o).length;
@@ -1071,7 +1148,7 @@ async function assembleRun(input) {
1071
1148
  config: {
1072
1149
  sandbox: plannedSandboxes(input.plan.sandboxes, profileSandbox(config, profile?.defaults)),
1073
1150
  environment: config.environment?.name ?? null,
1074
- runs: config.runs,
1151
+ runs: config.runs ?? 1,
1075
1152
  judgeModel,
1076
1153
  threshold: runJudge.threshold,
1077
1154
  timeoutMs: config.timeoutMs,
@@ -1098,6 +1175,6 @@ async function assembleRun(input) {
1098
1175
  };
1099
1176
  }
1100
1177
  //#endregion
1101
- export { fingerprintOf as a, transcriptLines as c, cellOutcome as i, worstOutcome as l, assembleRun as n, judgementKey as o, capText as r, mergeOutcome as s, OUTPUT_CAP as t };
1178
+ export { gatewayPart as a, rawLines as c, turnText as d, fingerprintOf as i, transcriptLines as l, assembleRun as n, judgmentKey as o, capText as r, mergeOutcome as s, OUTPUT_CAP as t, turnError as u };
1102
1179
 
1103
- //# sourceMappingURL=assemble-BO93VAk1.js.map
1180
+ //# sourceMappingURL=assemble-SKQGW_QF.js.map