reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -0,0 +1,861 @@
1
+ import { s as recordFileName } from "./ids-Bp6ut7ZD.js";
2
+ import { it as STOPPED_FIRST, ot as resolveQuestionState, rt as DEFAULT_QUESTION_STATE } from "./config-k6m7XbAz.js";
3
+ import { m as redactSecrets } from "./redact-DVIaahLv.js";
4
+ import { n as clip } from "./text-DTigcA64.js";
5
+ import { c as ratesOf, d as slots, i as pairFactors, m as voteReason, n as checksHeld, o as pairsOf, p as voteOf, t as acrossOf, u as sideName } from "./pairs-puYi-zj2.js";
6
+ import { a as isFake, i as hostModel, r as cannedModel } from "./model-Bq3G0OG1.js";
7
+ import { n as judgeDiff } from "./patch-DKJiC5Sq.js";
8
+ import { t as transcriptForJudge } from "./transcript-DxPaWHTZ.js";
9
+ import { join } from "node:path";
10
+ import { z } from "zod";
11
+ import { readFile } from "node:fs/promises";
12
+ import { Output, generateText } from "ai";
13
+ //#region src/compare/questions.ts
14
+ const DEFAULT_SHOWS = { state: DEFAULT_QUESTION_STATE };
15
+ const isOn = (value) => value !== false && !(Array.isArray(value) && !value.length);
16
+ /** `a`, `a and b`, `a, b and c`. */
17
+ const listed = (parts) => parts.length < 2 ? parts[0] ?? "" : `${parts.slice(0, -1).join(", ")} and ${parts.at(-1)}`;
18
+ /** The parts an attempt is shown as, for the system prompt. */
19
+ function shownParts(shows, taskEach) {
20
+ const { state } = shows;
21
+ return [
22
+ ...taskEach && state.task ? ["the prompts it was sent"] : [],
23
+ ...state.output ? ["its final message"] : [],
24
+ ...isOn(state.transcript) ? ["its transcript"] : [],
25
+ ...state.files ? ["the files it changed"] : [],
26
+ ...isOn(state.diff) ? ["the changes it made to the project, as a unified diff"] : [],
27
+ ...isOn(state.checks) ? ["the commands run on its work, with their exit codes and output"] : [],
28
+ ...shows.artifacts !== void 0 ? ["screenshots of its work"] : []
29
+ ];
30
+ }
31
+ /** The system prompt, naming the parts each attempt is shown as. */
32
+ function questionSystem(shows = DEFAULT_SHOWS, taskEach = false) {
33
+ const parts = shownParts(shows, taskEach);
34
+ return `You compare two attempts by coding agents at the same task and answer one question about them.
35
+ Each attempt is shown as ${listed(parts)}, labeled A or B. The labels are assigned at random: which attempt comes first says nothing about it.
36
+ Judge only by the question, from what is shown. Answer A or B for the attempt the question favors, or tie when what is shown doesn't separate them on it. Give a short reason for the pick.`;
37
+ }
38
+ const voteSchema = z.object({
39
+ winner: z.enum([
40
+ "A",
41
+ "B",
42
+ "tie"
43
+ ]),
44
+ reason: z.string().optional()
45
+ });
46
+ /** Roughly four characters a token: each side gets half of `maxInputTokens`. */
47
+ const CHARS_PER_TOKEN = 4;
48
+ /** The turns a side shows: the last, or every one. */
49
+ const shownTurns = (side, state) => {
50
+ const turns = side.turns ?? [];
51
+ return state.turns === "all" ? turns : turns.slice(-1);
52
+ };
53
+ /** The task both sides were given, when they were given the same prompts. */
54
+ function sharedTask(first, second, state) {
55
+ if (!state.task) return null;
56
+ const a = shownTurns(first, state).map((t) => t.prompt);
57
+ const b = shownTurns(second, state).map((t) => t.prompt);
58
+ if (!a.length || a.length !== b.length || a.some((p, i) => p !== b[i])) return null;
59
+ return a.join("\n\n");
60
+ }
61
+ /** What a screenshot is labelled: its place on its side of a pair, or on the one attempt shown. */
62
+ const screenshotLabel = (side, i, of) => side ? `Attempt ${side}, screenshot ${i} of ${of}` : `Screenshot ${i} of ${of}`;
63
+ function buildSide(attempt, label, shows, taskShared) {
64
+ const { state } = shows;
65
+ const turns = shownTurns(attempt, state).map((t) => ({
66
+ ...state.task && !taskShared ? { task: redactSecrets(t.prompt) } : {},
67
+ ...state.output ? { output: redactSecrets(t.text ?? "") } : {},
68
+ ...state.transcript ? { transcript: redactSecrets(transcriptForJudge(t.transcript ?? [], state.transcript).join("\n")) } : {}
69
+ }));
70
+ const shown = judgeDiff({
71
+ patch: attempt.diff,
72
+ files: attempt.files ?? []
73
+ }, Array.isArray(state.diff) ? { patterns: state.diff } : {});
74
+ const patch = redactSecrets(shown.patch);
75
+ const side = {
76
+ ...state.turns === "all" ? turns.some((t) => Object.keys(t).length) ? { turns } : {} : turns[0] ?? {},
77
+ ...state.files ? { files: shown.files } : {},
78
+ ...state.diff !== false ? { diff: patch.trim() ? patch : "(no changes)" } : {}
79
+ };
80
+ const checks = checksOf(attempt, state.checks);
81
+ if (checks) side.checks = checks;
82
+ const images = [];
83
+ const sent = [];
84
+ if (shows.artifacts !== void 0) {
85
+ const captures = attempt.captures ?? [];
86
+ const names = shows.artifacts === true ? captures.map((c) => c.name) : shows.artifacts;
87
+ side.artifacts = names.map((name, i) => {
88
+ const at = screenshotLabel(label, i + 1, names.length);
89
+ const capture = captures.find((c) => c.name === name);
90
+ const side = label ?? "A";
91
+ if (!capture) {
92
+ images.push({
93
+ label: `${at}: not captured`,
94
+ data: null,
95
+ side,
96
+ name
97
+ });
98
+ return {
99
+ label: at,
100
+ missing: true
101
+ };
102
+ }
103
+ sent.push(name);
104
+ images.push({
105
+ label: at,
106
+ data: capture.data,
107
+ side,
108
+ name
109
+ });
110
+ return {
111
+ label: at,
112
+ mediaType: "image/png",
113
+ bytes: capture.data.byteLength
114
+ };
115
+ });
116
+ }
117
+ return {
118
+ side,
119
+ images,
120
+ sent
121
+ };
122
+ }
123
+ function checksOf(attempt, which) {
124
+ if (which === false) return void 0;
125
+ const ran = attempt.checks ?? [];
126
+ const shown = ({ command, exitCode, stdout, stderr }) => ({
127
+ command,
128
+ exitCode,
129
+ stdout: redactSecrets(stdout),
130
+ stderr: redactSecrets(stderr)
131
+ });
132
+ if (which === true) return ran.map(shown);
133
+ return which.flatMap((command) => {
134
+ const runs = ran.filter((r) => r.command === command);
135
+ return runs.length ? runs.map(shown) : [{
136
+ command,
137
+ notRun: true
138
+ }];
139
+ });
140
+ }
141
+ const tag = (name, body) => [
142
+ `<${name}>`,
143
+ body,
144
+ `</${name}>`
145
+ ];
146
+ /** A side's text in the prompt: each part it shows, in its own tag. */
147
+ function sideText(side) {
148
+ const turnLines = (t) => [
149
+ ...t.task !== void 0 ? tag("task", t.task) : [],
150
+ ...t.output !== void 0 ? tag("output", t.output) : [],
151
+ ...t.transcript !== void 0 ? tag("transcript", t.transcript) : []
152
+ ];
153
+ return [
154
+ ...side.turns ? side.turns.flatMap((t, i) => [
155
+ `<turn n="${i + 1}">`,
156
+ ...turnLines(t),
157
+ "</turn>"
158
+ ]) : turnLines(side),
159
+ ...side.files ? tag("files", side.files.join("\n") || "(none)") : [],
160
+ ...side.diff !== void 0 ? tag("diff", side.diff) : [],
161
+ ...side.checks ? tag("checks", side.checks.map((c) => "notRun" in c ? `$ ${c.command} → not run` : [
162
+ `$ ${c.command} → exit ${c.exitCode}`,
163
+ ...c.stdout ? [c.stdout.trimEnd()] : [],
164
+ ...c.stderr ? [`stderr: ${c.stderr.trimEnd()}`] : []
165
+ ].join("\n")).join("\n")) : [],
166
+ ...side.artifacts ? tag("screenshots", side.artifacts.map((a) => "missing" in a ? `${a.label}: not captured` : a.label).join("\n")) : []
167
+ ].join("\n");
168
+ }
169
+ /** A side's text, cut to `chars` when it's longer. */
170
+ function cut(text, chars) {
171
+ if (text.length <= chars) return text;
172
+ return `${text.slice(0, chars)}\n[reeval: attempt cut at ${chars} of ${text.length} characters]`;
173
+ }
174
+ /** What the model is shown for one order of a pair: no harness, arm, run, attempt id or capture name. */
175
+ function blindInput(question, pair, swapped, maxInputTokens, shows = DEFAULT_SHOWS) {
176
+ const [first, second] = slots(pair, swapped);
177
+ const chars = Math.floor(maxInputTokens * CHARS_PER_TOKEN / 2);
178
+ const task = sharedTask(first, second, shows.state);
179
+ const a = buildSide(first, "A", shows, task !== null);
180
+ const b = buildSide(second, "B", shows, task !== null);
181
+ const prompt = [
182
+ `Question: ${question}`,
183
+ ...task ? [
184
+ "",
185
+ "The task both attempts were given:",
186
+ "<task>",
187
+ task,
188
+ "</task>"
189
+ ] : [],
190
+ "",
191
+ "<attempt label=\"A\">",
192
+ cut(sideText(a.side), chars),
193
+ "</attempt>",
194
+ "",
195
+ "<attempt label=\"B\">",
196
+ cut(sideText(b.side), chars),
197
+ "</attempt>"
198
+ ].join("\n");
199
+ const [sentFirst, sentSecond] = swapped ? [b.sent, a.sent] : [a.sent, b.sent];
200
+ return {
201
+ blind: {
202
+ question,
203
+ prompt: redactSecrets(prompt),
204
+ a: a.side,
205
+ b: b.side
206
+ },
207
+ system: questionSystem(shows, task === null),
208
+ images: [...a.images, ...b.images],
209
+ sent: {
210
+ a: sentFirst,
211
+ b: sentSecond
212
+ }
213
+ };
214
+ }
215
+ function fakeAnswer(fake, pair) {
216
+ const vote = fake.vote ?? "tie";
217
+ if (typeof vote === "object" && "error" in vote) return cannedModel({ error: vote.error });
218
+ const pick = typeof vote === "function" ? vote(pair) : vote;
219
+ const winner = typeof pick === "string" ? pick : pick.winner;
220
+ const reason = typeof pick === "string" ? void 0 : pick.reason;
221
+ return cannedModel({ text: JSON.stringify({
222
+ winner,
223
+ ...reason !== void 0 ? { reason } : {}
224
+ }) });
225
+ }
226
+ /**
227
+ * The model call's system prompt and input: the prompt as text, or, with screenshots, one user
228
+ * message of the prompt then each screenshot's label and image part.
229
+ */
230
+ function questionRequest(input) {
231
+ const { blind, images } = input;
232
+ if (!images.length) return {
233
+ system: input.system,
234
+ prompt: blind.prompt
235
+ };
236
+ return {
237
+ system: input.system,
238
+ messages: [{
239
+ role: "user",
240
+ content: [{
241
+ type: "text",
242
+ text: blind.prompt
243
+ }, ...images.flatMap((image) => [{
244
+ type: "text",
245
+ text: image.label
246
+ }, ...image.data ? [{
247
+ type: "file",
248
+ mediaType: "image/png",
249
+ data: image.data
250
+ }] : []])]
251
+ }]
252
+ };
253
+ }
254
+ async function ask(model, gateway, settings, input, maxOutputTokens, signal) {
255
+ const result = await generateText({
256
+ model: isFake(model) ? fakeAnswer(model, input.blind) : await hostModel(model, gateway, "question model", settings),
257
+ ...questionRequest(input),
258
+ output: Output.object({
259
+ schema: voteSchema,
260
+ name: "vote"
261
+ }),
262
+ maxOutputTokens,
263
+ maxRetries: 3,
264
+ ...signal ? { abortSignal: signal } : {}
265
+ });
266
+ const { winner, reason } = result.output;
267
+ return {
268
+ pick: winner,
269
+ reason: voteReason(reason),
270
+ resolvedModel: isFake(model) ? null : result.response?.modelId ?? null
271
+ };
272
+ }
273
+ /** Runs `work` over `items`, at most `limit` at once, keeping their order. */
274
+ async function pooled(items, limit, work) {
275
+ const out = new Array(items.length);
276
+ let next = 0;
277
+ const lanes = Array.from({ length: Math.min(limit, items.length) }, async () => {
278
+ while (next < items.length) {
279
+ const i = next++;
280
+ out[i] = await work(items[i]);
281
+ }
282
+ });
283
+ await Promise.all(lanes);
284
+ return out;
285
+ }
286
+ function askedCall(pair, swapped, input, error) {
287
+ const [first, second] = slots(pair, swapped);
288
+ const shown = {
289
+ A: first,
290
+ B: second
291
+ };
292
+ return {
293
+ cut: pair.cut,
294
+ ...pair.cut === "within" ? { harness: pair.harness } : { arm: pair.arm },
295
+ run: pair.run,
296
+ a: sideName(pair.cut, pair.a),
297
+ b: sideName(pair.cut, pair.b),
298
+ swapped,
299
+ labels: {
300
+ A: sideName(pair.cut, first),
301
+ B: sideName(pair.cut, second)
302
+ },
303
+ system: input.system,
304
+ prompt: input.blind.prompt,
305
+ ...input.images.length ? { screenshots: input.images.map((image) => {
306
+ const label = image.label.replace(/: not captured$/, "");
307
+ return image.data ? {
308
+ label,
309
+ side: sideName(pair.cut, shown[image.side]),
310
+ name: image.name,
311
+ attempt: shown[image.side].id ?? ""
312
+ } : {
313
+ label,
314
+ missing: true
315
+ };
316
+ }) } : {},
317
+ ...error !== void 0 ? { error } : {}
318
+ };
319
+ }
320
+ const errorText = (error) => redactSecrets((error instanceof Error ? error.message : String(error)).split("\n")[0] ?? "");
321
+ /**
322
+ * Why the question can't be asked: a command or capture it names that none of `sides` has, so
323
+ * each would show it missing. Undefined when each one is on some side.
324
+ */
325
+ function missingEverywhere(question, shows, sides, who = "paired attempt") {
326
+ const { checks } = shows.state;
327
+ if (Array.isArray(checks)) {
328
+ const command = checks.find((c) => !sides.some((s) => s.checks?.some((r) => r.command === c)));
329
+ if (command !== void 0) return `question "${question}" state.checks names a command no ${who} ran: ${JSON.stringify(command)}`;
330
+ }
331
+ if (Array.isArray(shows.artifacts)) {
332
+ const name = shows.artifacts.find((n) => !sides.some((s) => s.captures?.some((c) => c.name === n)));
333
+ if (name !== void 0) return `question "${question}" artifacts names a capture no ${who} has: ${JSON.stringify(name)}`;
334
+ }
335
+ }
336
+ /**
337
+ * Asks one question of every pair, each twice with the order swapped. Votes that came back are
338
+ * kept when a call fails, and the first failure is the answer's `error`.
339
+ */
340
+ async function answerQuestion(options) {
341
+ const { question, model, pairs } = options;
342
+ const shows = {
343
+ state: resolveQuestionState(options.state, true),
344
+ ...options.artifacts !== void 0 ? { artifacts: options.artifacts } : {}
345
+ };
346
+ const head = {
347
+ question,
348
+ model: isFake(model) ? "fake" : model
349
+ };
350
+ const asked = {
351
+ stateOptions: shows.state,
352
+ ...shows.artifacts !== void 0 ? { artifacts: shows.artifacts } : {}
353
+ };
354
+ const missing = missingEverywhere(question, shows, [...new Set(pairs.flatMap((p) => [p.a, p.b]))]);
355
+ if (missing) return {
356
+ ...head,
357
+ ...asked,
358
+ votes: [],
359
+ rates: [],
360
+ error: missing
361
+ };
362
+ const calls = pairs.flatMap((pair) => [false, true].map((swapped) => ({
363
+ pair,
364
+ swapped
365
+ }))).map((call, at) => ({
366
+ ...call,
367
+ at
368
+ }));
369
+ const served = /* @__PURE__ */ new Set();
370
+ let error;
371
+ const made = new Array(calls.length).fill(null);
372
+ const votes = await pooled(calls, options.concurrency ?? 4, async ({ pair, swapped, at }) => {
373
+ if (error !== void 0) return null;
374
+ const input = blindInput(question, pair, swapped, options.maxInputTokens, shows);
375
+ try {
376
+ const { pick, reason, resolvedModel } = await ask(model, options.gateway, {
377
+ fallbacks: !!options.fallbacks,
378
+ ...options.vertex ? { vertex: options.vertex } : {}
379
+ }, input, options.maxOutputTokens, options.signal);
380
+ made[at] = askedCall(pair, swapped, input);
381
+ if (resolvedModel) served.add(resolvedModel);
382
+ const vote = voteOf(question, pair, swapped, pick, reason);
383
+ return shows.artifacts !== void 0 ? {
384
+ ...vote,
385
+ artifacts: input.sent
386
+ } : vote;
387
+ } catch (e) {
388
+ const why = errorText(e);
389
+ made[at] = askedCall(pair, swapped, input, why);
390
+ error ??= why;
391
+ return null;
392
+ }
393
+ });
394
+ options.calls?.push(...made.filter((c) => c !== null));
395
+ const kept = votes.filter((v) => v !== null);
396
+ return {
397
+ ...head,
398
+ ...served.size ? { resolvedModel: [...served].sort().join(", ") } : {},
399
+ ...asked,
400
+ votes: kept,
401
+ rates: ratesOf(kept),
402
+ ...error !== void 0 ? { error } : {}
403
+ };
404
+ }
405
+ /** Stored answers to a question asked of one attempt are at most this long. */
406
+ const ANSWER_TEXT_CAP = 1e3;
407
+ /** The system prompt for a question asked of one attempt, naming the parts it's shown as. */
408
+ function attemptQuestionSystem(shows, choices) {
409
+ const how = choices ? "Pick the one of the choices that answers it, and give a short reason for the pick." : "Answer it in a few sentences.";
410
+ return `You answer one question about an attempt by a coding agent at a task.
411
+ The attempt is shown as ${listed(shownParts(shows, true))}.
412
+ Answer only from what is shown. ${how}`;
413
+ }
414
+ function attemptInput(question, attempt, maxInputTokens, shows, choices) {
415
+ const built = buildSide(attempt, null, shows, false);
416
+ const prompt = [
417
+ `Question: ${question}`,
418
+ ...choices ? [
419
+ "",
420
+ "Choices:",
421
+ ...choices.map((c) => `- ${c}`)
422
+ ] : [],
423
+ "",
424
+ "<attempt>",
425
+ cut(sideText(built.side), maxInputTokens * CHARS_PER_TOKEN),
426
+ "</attempt>"
427
+ ].join("\n");
428
+ return {
429
+ blind: {
430
+ question,
431
+ prompt: redactSecrets(prompt),
432
+ ...choices ? { choices } : {},
433
+ attempt: built.side
434
+ },
435
+ system: attemptQuestionSystem(shows, choices),
436
+ images: built.images,
437
+ sent: built.sent
438
+ };
439
+ }
440
+ function fakeAttemptAnswer(fake, attempt) {
441
+ const answer = fake.answer;
442
+ if (answer && typeof answer === "object" && "error" in answer) return cannedModel({ error: answer.error });
443
+ const given = answer === void 0 ? {} : typeof answer === "function" ? answer(attempt) : answer;
444
+ const { choice, text } = typeof given === "string" ? {
445
+ text: given,
446
+ choice: void 0
447
+ } : given;
448
+ return cannedModel({ text: JSON.stringify({
449
+ ...attempt.choices ? { choice: choice ?? attempt.choices[0] } : {},
450
+ answer: text ?? "fake answer"
451
+ }) });
452
+ }
453
+ const answerSchema = (choices) => choices ? z.object({
454
+ choice: z.enum(choices),
455
+ answer: z.string()
456
+ }) : z.object({ answer: z.string() });
457
+ /** Asks one question of one attempt. A failed call is the answer's `error`, with empty `text`. */
458
+ async function answerAttempt(options) {
459
+ const { question, model, attempt, choices } = options;
460
+ const shows = {
461
+ state: resolveQuestionState(options.state, false),
462
+ ...options.artifacts !== void 0 ? { artifacts: options.artifacts } : {}
463
+ };
464
+ const head = {
465
+ question,
466
+ model: isFake(model) ? "fake" : model,
467
+ stateOptions: shows.state,
468
+ ...shows.artifacts !== void 0 ? { artifacts: shows.artifacts } : {},
469
+ ...choices ? { choices } : {}
470
+ };
471
+ let input;
472
+ try {
473
+ input = attemptInput(question, attempt, options.maxInputTokens, shows, choices);
474
+ const result = await generateText({
475
+ model: isFake(model) ? fakeAttemptAnswer(model, input.blind) : await hostModel(model, options.gateway, "question model", {
476
+ fallbacks: !!options.fallbacks,
477
+ ...options.vertex ? { vertex: options.vertex } : {}
478
+ }),
479
+ ...questionRequest(input),
480
+ output: Output.object({
481
+ schema: answerSchema(choices),
482
+ name: "answer"
483
+ }),
484
+ maxOutputTokens: options.maxOutputTokens,
485
+ maxRetries: 3,
486
+ ...options.signal ? { abortSignal: options.signal } : {}
487
+ });
488
+ const out = result.output;
489
+ const served = isFake(model) ? null : result.response?.modelId ?? null;
490
+ return {
491
+ answer: {
492
+ ...head,
493
+ ...served ? { resolvedModel: served } : {},
494
+ ...out.choice !== void 0 ? { choice: out.choice } : {},
495
+ text: clip(redactSecrets(out.answer.trim()), ANSWER_TEXT_CAP)
496
+ },
497
+ call: attemptCall(input, attempt)
498
+ };
499
+ } catch (e) {
500
+ const why = errorText(e);
501
+ return {
502
+ answer: {
503
+ ...head,
504
+ text: "",
505
+ error: why
506
+ },
507
+ ...input ? { call: attemptCall(input, attempt, why) } : {}
508
+ };
509
+ }
510
+ }
511
+ /** The call a question asked of one attempt made, as its `questions/<n>.json` keeps it. */
512
+ const attemptCall = (input, attempt, error) => ({
513
+ system: input.system,
514
+ prompt: input.blind.prompt,
515
+ ...input.images.length ? { screenshots: input.images.map((image) => {
516
+ const label = image.label.replace(/: not captured$/, "");
517
+ return image.data ? {
518
+ label,
519
+ side: "attempt",
520
+ name: image.name,
521
+ attempt: attempt.id ?? ""
522
+ } : {
523
+ label,
524
+ missing: true
525
+ };
526
+ }) } : {},
527
+ ...error !== void 0 ? { error } : {}
528
+ });
529
+ /** Ran to a pass or a fail: what a question asked of one attempt is asked of. */
530
+ const answerable = (e, record) => record !== null && (e.state === "passed" || e.state === "failed" && record.error?.kind !== "infra" && record.error?.kind !== "aborted");
531
+ /** Runs `work` at most `limit` at a time, in the order it was asked. */
532
+ function limiter(limit) {
533
+ let running = 0;
534
+ const waiting = [];
535
+ return async (work) => {
536
+ if (running >= limit) await new Promise((go) => waiting.push(go));
537
+ else running++;
538
+ try {
539
+ return await work();
540
+ } finally {
541
+ const next = waiting.shift();
542
+ if (next) next();
543
+ else running--;
544
+ }
545
+ };
546
+ }
547
+ /**
548
+ * A staged attempt as a pair side: its record's turns, diff and checks, and whether its checks
549
+ * held. One a resume skipped as already completed is judged by its staged outcome.
550
+ */
551
+ function stagedAttempt(e, record) {
552
+ const cell = e.attempt?.cell;
553
+ if (!cell) return null;
554
+ const outcome = e.state === "skipped" ? record?.outcome : e.state === "passed" ? "pass" : "fail";
555
+ const checksPassed = record !== null && outcome !== void 0 && checksHeld(outcome, record.error);
556
+ return {
557
+ harness: cell.harness,
558
+ arm: cell.environment,
559
+ run: e.attempt?.n ?? 1,
560
+ diff: record?.diff?.patch ?? "",
561
+ files: record?.diff?.files ?? [],
562
+ checksPassed,
563
+ control: cell.role === "control",
564
+ turns: record?.turns.map((t) => ({
565
+ prompt: t.prompt,
566
+ text: t.text,
567
+ transcript: t.transcript
568
+ })) ?? [],
569
+ checks: record?.checks.map(({ command, exitCode, stdout, stderr }) => ({
570
+ command,
571
+ exitCode,
572
+ stdout,
573
+ stderr
574
+ })) ?? [],
575
+ id: `${cell.key}#${e.attempt?.n ?? 1}`
576
+ };
577
+ }
578
+ /** The last try's output in a check log (`--- stdout` then `--- stderr`, as assembly writes it). */
579
+ function logOutput(log) {
580
+ const out = log?.lastIndexOf("--- stdout\n") ?? -1;
581
+ const err = log?.lastIndexOf("--- stderr\n") ?? -1;
582
+ if (!log || out < 0 || err < out) return {
583
+ stdout: "",
584
+ stderr: ""
585
+ };
586
+ return {
587
+ stdout: log.slice(out + 11, err),
588
+ stderr: log.slice(err + 11)
589
+ };
590
+ }
591
+ /**
592
+ * An attempt a resume kept from the previous record, as a pair side: its turns, `diff.patch`,
593
+ * transcript, check logs and captures, read back from the previous record's artifact files.
594
+ */
595
+ async function keptAttempt(e, kept, artifacts, captures) {
596
+ const cell = e.attempt?.cell;
597
+ if (!cell) return null;
598
+ const path = (ref) => join(artifacts ?? "", ...ref.path.split("/"));
599
+ const text = (ref) => ref && artifacts ? readFile(path(ref), "utf8").catch(() => null) : Promise.resolve(null);
600
+ const events = /* @__PURE__ */ new Map();
601
+ for (const line of (await text(kept.files.transcript))?.split("\n") ?? []) {
602
+ if (!line) continue;
603
+ const { turn, ...event } = JSON.parse(line);
604
+ if (event.type === "prompt" || event.type === "turn-end") continue;
605
+ events.set(turn, [...events.get(turn) ?? [], event]);
606
+ }
607
+ const checks = [];
608
+ for (const c of kept.checks) checks.push({
609
+ command: c.command,
610
+ exitCode: c.exitCode,
611
+ ...logOutput(await text(c.log))
612
+ });
613
+ const read = captures && artifacts ? kept.files.artifacts ?? [] : [];
614
+ const shots = await Promise.all(read.map(async (a) => {
615
+ const data = await readFile(path(a)).catch(() => null);
616
+ return data ? [{
617
+ name: a.name,
618
+ data: new Uint8Array(data)
619
+ }] : [];
620
+ }));
621
+ return {
622
+ harness: cell.harness,
623
+ arm: cell.environment,
624
+ run: e.attempt?.n ?? kept.n,
625
+ diff: await text(kept.files.diff) ?? "",
626
+ checksPassed: checksHeld(kept.outcome, kept.error),
627
+ control: cell.role === "control",
628
+ turns: kept.turns.map((t, i) => {
629
+ const said = events.get(i) ?? [];
630
+ const last = t.text.includes("… [reeval: truncated ") ? said.findLast((x) => x.type === "text") : void 0;
631
+ return {
632
+ prompt: t.prompt,
633
+ text: last?.type === "text" ? last.text : t.text,
634
+ transcript: said
635
+ };
636
+ }),
637
+ checks,
638
+ ...captures ? { captures: shots.flat() } : {},
639
+ id: `${cell.key}#${e.attempt?.n ?? kept.n}`
640
+ };
641
+ }
642
+ const readStaged = async (stagingDir, key) => JSON.parse(await readFile(join(stagingDir, "evals", recordFileName(key)), "utf8").catch(() => "null"));
643
+ /** A staged attempt's captures, read from the PNGs beside its record; a missing file is left out. */
644
+ async function stagedCaptures(stagingDir, key, record) {
645
+ const dir = join(stagingDir, "evals", recordFileName(key).replace(/\.json$/, ".artifacts"));
646
+ return (await Promise.all((record?.artifacts ?? []).map(async ({ name }) => {
647
+ const data = await readFile(join(dir, `${name}.png`)).catch(() => null);
648
+ return data ? [{
649
+ name,
650
+ data: new Uint8Array(data)
651
+ }] : [];
652
+ }))).flat();
653
+ }
654
+ /** A per-attempt question's answer when no model answered it. */
655
+ const unanswered = (q, error) => ({
656
+ n: q.n,
657
+ answer: {
658
+ question: q.question,
659
+ model: isFake(q.model) ? "fake" : q.model,
660
+ stateOptions: resolveQuestionState(q.state, false),
661
+ ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {},
662
+ ...q.choices ? { choices: q.choices } : {},
663
+ text: "",
664
+ error
665
+ }
666
+ });
667
+ /**
668
+ * Asks each attempt its eval's per-attempt questions as soon as it ends, and an eval's pairwise
669
+ * questions as soon as its last planned attempt ends, while the rest of the run goes on; `settle`
670
+ * starts any pairwise ones left (an eval a filter or budget cut short) and waits for all.
671
+ */
672
+ function questionRunner(options) {
673
+ const byId = new Map(options.evals.map((e) => [e.row.id, e]));
674
+ const ended = /* @__PURE__ */ new Map();
675
+ const started = /* @__PURE__ */ new Map();
676
+ const perAttempt = /* @__PURE__ */ new Map();
677
+ const done = /* @__PURE__ */ new Map();
678
+ const shown = /* @__PURE__ */ new Map();
679
+ const asking = /* @__PURE__ */ new Map();
680
+ const controller = new AbortController();
681
+ const live = () => !controller.signal.aborted;
682
+ const limit = limiter(options.concurrency ?? 4);
683
+ const pairwiseOf = (e) => e.questions.filter((q) => q.across !== void 0);
684
+ const eachOf = (e) => e.questions.filter((q) => q.across === void 0);
685
+ const keptAttempts = new Map((options.resumed?.previous?.evals ?? []).flatMap((e) => e.attempts.map((a) => [a.key, a])));
686
+ const isCompleted = (a) => a.state === "skipped" && options.resumed?.completed.has(a.key) === true;
687
+ const side = async (a, record, captures) => {
688
+ const kept = isCompleted(a) ? keptAttempts.get(a.key) : void 0;
689
+ if (kept) return keptAttempt(a, kept, options.resumed?.artifacts ?? null, captures);
690
+ const found = stagedAttempt(a, record);
691
+ if (!found || !captures) return found;
692
+ return {
693
+ ...found,
694
+ captures: await stagedCaptures(options.stagingDir, a.key, record)
695
+ };
696
+ };
697
+ const askEach = (id, e, result) => {
698
+ const questions = eachOf(e);
699
+ if (!questions.length || perAttempt.has(result.key) || !live()) return;
700
+ const asked = (async () => {
701
+ const record = await readStaged(options.stagingDir, result.key);
702
+ if (!answerable(result, record)) return [];
703
+ const attempt = await side(result, record, questions.some((q) => q.artifacts !== void 0));
704
+ if (!attempt) return [];
705
+ shown.set(result.key, attempt);
706
+ asking.set(result.key, questions);
707
+ return Promise.all(questions.map(async (q) => {
708
+ const got = await limit(() => answerAttempt({
709
+ question: q.question,
710
+ model: q.model,
711
+ ...q.gateway ? { gateway: q.gateway } : {},
712
+ ...q.fallbacks ? { fallbacks: q.fallbacks } : {},
713
+ ...q.vertex ? { vertex: q.vertex } : {},
714
+ attempt,
715
+ maxInputTokens: options.maxInputTokens,
716
+ maxOutputTokens: options.maxOutputTokens,
717
+ ...q.state ? { state: q.state } : {},
718
+ ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {},
719
+ ...q.choices ? { choices: q.choices } : {},
720
+ signal: controller.signal
721
+ }));
722
+ if (live()) options.onAttemptAnswer?.(id, attempt.id ?? result.key, got.answer);
723
+ return {
724
+ n: q.n,
725
+ ...got
726
+ };
727
+ }));
728
+ })().catch((error) => questions.map((q) => {
729
+ const asked = unanswered(q, errorText(error));
730
+ if (live()) options.onAttemptAnswer?.(id, result.key, asked.answer);
731
+ return asked;
732
+ }));
733
+ perAttempt.set(result.key, asked);
734
+ asked.then((got) => {
735
+ if (live() && got.length) done.set(result.key, got);
736
+ });
737
+ };
738
+ const unshown = (e, results) => {
739
+ const sides = results.flatMap((r) => shown.get(r.key) ?? []);
740
+ for (const q of eachOf(e)) {
741
+ const shows = {
742
+ state: resolveQuestionState(q.state, false),
743
+ ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
744
+ };
745
+ const missing = sides.length && missingEverywhere(q.question, shows, sides, "attempt");
746
+ if (!missing) continue;
747
+ options.onUnshown?.(e.row.id, missing);
748
+ for (const r of results) {
749
+ const got = done.get(r.key);
750
+ if (!got) continue;
751
+ done.set(r.key, got.map((x) => {
752
+ if (x.n !== q.n) return x;
753
+ const { choice: _, resolvedModel: __, ...head } = x.answer;
754
+ return {
755
+ ...x,
756
+ answer: {
757
+ ...head,
758
+ text: "",
759
+ error: missing
760
+ }
761
+ };
762
+ }));
763
+ }
764
+ }
765
+ };
766
+ const start = (id) => {
767
+ const e = byId.get(id);
768
+ if (!e || started.has(id)) return;
769
+ const questions = pairwiseOf(e);
770
+ if (!questions.length) return;
771
+ const attempts = ended.get(id) ?? [];
772
+ const wantsCaptures = questions.some((q) => q.artifacts !== void 0);
773
+ const answers = (async () => {
774
+ const staged = (await Promise.all(attempts.map(async (a) => side(a, await readStaged(options.stagingDir, a.key), wantsCaptures)))).filter((s) => s !== null).sort((x, y) => (x.id ?? "") < (y.id ?? "") ? -1 : 1);
775
+ return Promise.all(questions.map(async (q) => {
776
+ const calls = [];
777
+ const answer = await answerQuestion({
778
+ question: q.question,
779
+ model: q.model,
780
+ ...q.gateway ? { gateway: q.gateway } : {},
781
+ ...q.fallbacks ? { fallbacks: q.fallbacks } : {},
782
+ ...q.vertex ? { vertex: q.vertex } : {},
783
+ pairs: pairsOf(staged, acrossOf(pairFactors(q.across))),
784
+ maxInputTokens: options.maxInputTokens,
785
+ maxOutputTokens: options.maxOutputTokens,
786
+ ...q.state ? { state: q.state } : {},
787
+ ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {},
788
+ signal: controller.signal,
789
+ calls
790
+ });
791
+ if (live()) options.onAnswer?.(id, answer);
792
+ return {
793
+ n: q.n,
794
+ answer,
795
+ calls
796
+ };
797
+ }));
798
+ })().catch((error) => questions.map((q) => {
799
+ const answer = {
800
+ question: q.question,
801
+ model: isFake(q.model) ? "fake" : q.model,
802
+ stateOptions: resolveQuestionState(q.state, true),
803
+ ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {},
804
+ votes: [],
805
+ rates: [],
806
+ error: errorText(error)
807
+ };
808
+ if (live()) options.onAnswer?.(id, answer);
809
+ return {
810
+ n: q.n,
811
+ answer,
812
+ calls: []
813
+ };
814
+ }));
815
+ started.set(id, answers);
816
+ };
817
+ return {
818
+ onEvalEnd(result) {
819
+ const e = byId.get(result.id);
820
+ if (!e) return;
821
+ askEach(result.id, e, result);
822
+ const list = ended.get(result.id) ?? [];
823
+ list.push(result);
824
+ ended.set(result.id, list);
825
+ if (list.filter((r) => !isCompleted(r)).length >= e.row.plan.attempts) start(result.id);
826
+ },
827
+ /** Stops calls in flight and asks nothing more: the run was canceled. */
828
+ cancel() {
829
+ controller.abort();
830
+ },
831
+ /**
832
+ * What a canceled run keeps: each attempt whose answers all came back before the cancel,
833
+ * and, for an attempt the cancel cut short, each question as an error, so the record shows
834
+ * it went unanswered. Waits for the aborted calls to unwind.
835
+ */
836
+ async answered() {
837
+ await Promise.all(perAttempt.values());
838
+ const attempts = new Map(done);
839
+ for (const [key, questions] of asking) if (!attempts.has(key)) attempts.set(key, questions.map((q) => unanswered(q, STOPPED_FIRST)));
840
+ return {
841
+ evals: /* @__PURE__ */ new Map(),
842
+ attempts
843
+ };
844
+ },
845
+ async settle() {
846
+ for (const id of byId.keys()) if (ended.has(id)) start(id);
847
+ const evals = /* @__PURE__ */ new Map();
848
+ for (const [id, answers] of started) evals.set(id, await answers);
849
+ for (const answers of perAttempt.values()) await answers;
850
+ for (const [id, e] of byId) unshown(e, ended.get(id) ?? []);
851
+ return {
852
+ evals,
853
+ attempts: new Map(done)
854
+ };
855
+ }
856
+ };
857
+ }
858
+ //#endregion
859
+ export { questionRunner };
860
+
861
+ //# sourceMappingURL=questions-BRhU8MzB.js.map