reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -1,19 +1,9 @@
1
- import { $n as cellSamples, $t as fromModel, An as FamilyRate, Bn as DIRECTIONAL_MIN, Cn as messageLine, Dn as CellRecord, En as AttemptRecord, Fn as JudgementRecord, Ft as TokenUsage, Gn as EffortSample, Gt as INSIGHTS_FILE, Hn as EFFORT_MIN_RUNS, In as Lift, It as TranscriptEvent, Jn as TOKENS_UNREPORTED, Jt as NO_MODEL, Kn as EffortVerdict, Kt as Job, Ln as RunRecord, Mn as FileRef, Nn as HistoryCell, On as EvalEntry, Pn as HistoryEntry, Qn as cellEffort, Qt as Sentence, Rn as StackSummary, Sn as factSegment, Tn as ArtifactRef, Un as EffortContrast, Vn as EFFORT_ALPHA, Wn as EffortMetric, Wt as EvalInsight, Xn as bothUnderATenth, Xt as Remark, Yn as VERDICT_TEXT, Yt as REMARK_KINDS, Zn as cellContrast, Zt as RunInsights, _n as cancelledOf, _t as Outcome, an as CheckResult, ar as effortSample, bn as effectOf, cn as EvalResult, cr as formatCount, dn as PooledArm, dr as tokenValues, en as Across, er as contrastEffort, fn as QuestionResult, gn as SuiteTally, hn as SuiteResult, in as CheckOutcome, ir as effortOf, jn as FamilyRecord, kn as FamilyLift, kt as SandboxWarning, ln as Interval, lr as formatRatio, mn as ResultOutcome, nn as CellError, nr as effortChangeLabel, on as Effect, or as effortSummary, pn as ResultFile, qn as Spread, qt as MODEL_REMARK_KINDS, rn as CellResult, rr as effortLabel, sn as EffortCounters, sr as effortVerdict, tn as CaseResult, tr as directional, un as JudgeResult, ur as formatSeconds, vn as checkEffect, wn as suiteResult, xn as evalAcross, yn as countedOf, zn as CellEffort, zt as TurnStatus } from "../types-B-9b1MDV.js";
2
- //#region src/stats.d.ts
3
- /**
4
- * Counted attempts a side before a pass-rate lift can be called: below it, every view says
5
- * `insufficient` (`can't tell yet`, `unclear`), whatever the interval. A product floor that keeps
6
- * `reeval run`, `reeval query`, the judge-key lift and `reeval analyze` agreeing, not one that
7
- * makes the stated 95% exact: at 4 a side, a 4/4 against 0/4 split happens by chance about 7% of
8
- * the time when nothing changed.
9
- */
10
- export declare const EFFECT_MIN_COUNTED = 4;
11
- //#endregion
1
+ import { An as VERDICT_TEXT, Bn as effortSample, Cn as EFFORT_MIN_RUNS, Ct as Outcome, Dn as EffortVerdict, En as EffortSample, Fn as contrastEffort, Gn as formatSeconds, Gt as TokenUsage, H as ErrorKind, Hn as effortVerdict, In as directional, Kn as tokenValues, Kt as TranscriptEvent, Ln as effortChangeLabel, Mn as cellContrast, Nn as cellEffort, On as Spread, Pn as cellSamples, Rn as effortLabel, Sn as EFFORT_ALPHA, Tn as EffortMetric, Un as formatCount, Vn as effortSummary, Wn as formatRatio, Yt as TurnStatus, _n as RunRecord, an as CellRecord, bn as CellEffort, cn as FamilyLift, dn as FamilyRecord, fn as FileRef, gn as Lift, hn as JudgmentRecord, in as AttemptRecord, jn as bothUnderATenth, kn as TOKENS_UNREPORTED, ln as FamilyLiftSummary, mn as HistoryEntry, on as CheckTally, pn as HistoryCell, rn as ArtifactRef, sn as EvalEntry, un as FamilyRate, vn as StackSummary, wn as EffortContrast, xn as DIRECTIONAL_MIN, yn as UpstreamCount, zn as effortOf, zt as SandboxWarning } from "../types-coMWHaU2.js";
12
2
  //#region src/view/analysis.d.ts
13
3
  /** Bumped when the system prompt or output schema changes: an older analysis is superseded. */
14
- export declare const ANALYSIS_PROMPT_VERSION = "analysis-v10";
4
+ export declare const ANALYSIS_PROMPT_VERSION = "analysis-v11";
15
5
  /** Bumped when the digest's shape or selection rules change. */
16
- export declare const DIGEST_VERSION = 11;
6
+ export declare const DIGEST_VERSION = 12;
17
7
  export type Confidence = 'high' | 'medium' | 'low';
18
8
  export type AnalysisClaim = {
19
9
  text: string;
@@ -74,7 +64,7 @@ export type AnalysisValidation = {
74
64
  export type AnalysisScope = {
75
65
  kind: 'run';
76
66
  head: string;
77
- /** Resolved once, so a nearer ancestor recorded later doesn't change what this analysed. */
67
+ /** Resolved once, so a nearer ancestor recorded later doesn't change what this analyzed. */
78
68
  base: string | null;
79
69
  /** An eval filter (substring of the eval id), or null for every eval. */
80
70
  evals: string | null;
@@ -96,7 +86,7 @@ export type AnalysisDoc = {
96
86
  promptVersion: string;
97
87
  model: string;
98
88
  resolvedModel: string | null;
99
- route: 'direct' | 'gateway' | 'fake';
89
+ route: 'direct' | 'gateway' | 'openrouter' | 'vertex' | 'fake';
100
90
  usage: {
101
91
  inputTokens: number | null;
102
92
  outputTokens: number | null;
@@ -112,7 +102,7 @@ export type AnalysisDoc = {
112
102
  evalSlug: string;
113
103
  attempt: string;
114
104
  }>;
115
- /** Each cited fact or evidence id, summarised in one line. */
105
+ /** Each cited fact or evidence id, summarized in one line. */
116
106
  cited: Record<string, string>;
117
107
  /**
118
108
  * Why the analysis failed; its `analysis` is then empty. `reeval analyze` treats a cached one as
@@ -166,14 +156,27 @@ export declare function sortAnalyses(entries: AnalysisEntry[]): AnalysisEntry[];
166
156
  export declare const isSuperseded: (doc: AnalysisDoc) => boolean;
167
157
  //#endregion
168
158
  //#region src/view/compare.d.ts
169
- export type CompareStatus = 'regressed' | 'fixed' | 'same' | 'drift' | 'added' | 'removed' | 'infra';
159
+ export type CompareStatus = 'regressed' | 'fixed' | 'same' | 'gate' | 'drift' | 'added' | 'removed' | 'infra';
170
160
  export type CompareRow = {
171
161
  id: string;
172
162
  status: CompareStatus;
173
163
  base: Outcome | null;
174
164
  head: Outcome | null;
175
- /** For `drift`: what changed, e.g. `evals/helpers.ts changed` or `harness config changed`. */
165
+ /**
166
+ * For `drift`: what changed, e.g. `body changed`, `revision 2 → 3` or `harness config changed`. On
167
+ * any other row, `gate changed` when the cells' gate did.
168
+ */
176
169
  changes?: string[];
170
+ /**
171
+ * When the cells' gate changed but nothing else did: each side's gate, and base's outcome with
172
+ * its recorded attempts scored under head's gate. `status` compares that with head, so a flip
173
+ * the gate alone explains is `gate`, not `regressed` or `fixed`.
174
+ */
175
+ gate?: {
176
+ base: string;
177
+ head: string;
178
+ rescored: Outcome;
179
+ };
177
180
  /**
178
181
  * When the runs ran different cells of the eval: the cells only head or only base has. Then
179
182
  * `status`, `base` and `head` are over the cells both have.
@@ -185,10 +188,24 @@ export type CompareRow = {
185
188
  /** Cells in both runs, with each side's lift (variants) and effort. */
186
189
  perCell?: CompareCell[];
187
190
  /**
188
- * The sides whose `fail` is only attempts the run's interrupt cancelled, which `reeval show`
189
- * reads as `cancelled`. Such an eval is never `regressed` or `fixed`: it's `drift`.
191
+ * The sides whose `fail` is only attempts the run's interrupt canceled, which `reeval show`
192
+ * reads as `canceled`. Such an eval is never `regressed` or `fixed`: it's `drift`.
190
193
  */
191
- cancelled?: Array<'base' | 'head'>;
194
+ canceled?: Array<'base' | 'head'>;
195
+ /**
196
+ * `family`: a pooled `evaluate.each`'s gate, its `id` the family's. `base` and `head` are its
197
+ * pooled outcome (`fail` when a lift is below its `minLift`); its cases have rows of their own.
198
+ * For such a row, `cells` are the variants whose pooled lift only one run has.
199
+ */
200
+ kind?: 'family';
201
+ /** On a family row: each variant's pooled lift on each side, null where that run has none. */
202
+ lifts?: CompareFamilyLift[];
203
+ };
204
+ export type CompareFamilyLift = {
205
+ key: string;
206
+ vs: string;
207
+ base: Omit<FamilyLiftSummary, 'variant' | 'vs'> | null;
208
+ head: Omit<FamilyLiftSummary, 'variant' | 'vs'> | null;
192
209
  };
193
210
  export type CompareCell = {
194
211
  key: string;
@@ -222,17 +239,26 @@ export declare function stackChanges(base: StackSource, head: StackSource): Stac
222
239
  * A cell `seen` doesn't have yet is new coverage, not a change.
223
240
  */
224
241
  export declare function cellsChanged(seen: Map<string, string>, evalId: string, cells: ReadonlyArray<Pick<HistoryCell, 'key' | 'fingerprint'>>): boolean;
225
- type ComparableEval = Pick<HistoryEntry['evals'][number], 'id' | 'fingerprint' | 'outcome' | 'sources' | 'cells'>;
226
- type Comparable = Pick<HistoryEntry, 'stack'> & {
242
+ type ComparableEval = Pick<HistoryEntry['evals'][number], 'id' | 'fingerprint' | 'outcome' | 'sources' | 'source' | 'revision' | 'cells'>;
243
+ type Comparable = Pick<HistoryEntry, 'stack' | 'families'> & {
227
244
  evals: ComparableEval[];
228
245
  };
229
246
  /** Which source files were added, removed or changed. */
230
247
  export declare function sourceChanges(before: ComparableEval['sources'], after: ComparableEval['sources']): string[];
231
- /** Why two runs of an eval aren't comparable: the source files and fingerprint parts that differ. */
248
+ /**
249
+ * Why two runs of an eval aren't comparable: what in its source differs (`body`, `options`,
250
+ * `case data`, `revision 2 → 3`; a record from before `source`, its files), then the fingerprint
251
+ * parts that differ. Each reason but the revision ends in ` changed`.
252
+ */
232
253
  export declare function whyChanged(base: ComparableEval, head: ComparableEval): string[];
233
254
  /** An eval only head has. */
234
255
  export declare const addedRow: (h: ComparableEval) => CompareRow;
256
+ /** The rows of a run with nothing to compare it with: every eval and gated family is new. */
257
+ export declare const newRows: (head: Comparable) => CompareRow[];
258
+ /** Each eval's row, then each family's that either run gated. */
235
259
  export declare function compare(base: Comparable, head: Comparable): CompareRow[];
260
+ /** The key that tells a family's row from an eval's, whose ids can coincide. */
261
+ export declare const rowKey: (row: Pick<CompareRow, 'id' | 'kind'>) => string;
236
262
  //#endregion
237
263
  //#region src/view/debug.d.ts
238
264
  /**
@@ -282,6 +308,13 @@ export type DebugRequest = {
282
308
  /** Brokered secrets and the harness credential swapped in, by name. */
283
309
  substituted: string[];
284
310
  ms: number;
311
+ /**
312
+ * On a request the gateway sent the harness's key on, or a model call whose response it reads,
313
+ * the reasoning settings its JSON body carried, as
314
+ * `<path>=<value>` joined by spaces: `reasoning_effort=high`,
315
+ * `generationConfig.thinkingConfig.thinkingLevel=LOW`.
316
+ */
317
+ requestEffort?: string;
285
318
  };
286
319
  /** What the studio's Debug tab shows for one attempt. */
287
320
  export type DebugView = {
@@ -342,7 +375,7 @@ export declare const familyCases: (family: Pick<FamilyRecord, 'id'>, evals: read
342
375
  /** The derived header of one run: what listings, indexes and comparisons need without the record. */
343
376
  export declare function runHeader(record: RunRecord, recorded: boolean): HistoryEntry;
344
377
  //#endregion
345
- //#region src/view/judgements.d.ts
378
+ //#region src/view/judgments.d.ts
346
379
  /** Short name for a judge question, e.g. `login` from `login?`. */
347
380
  export declare function judgeLabel(question: string): string;
348
381
  /**
@@ -369,7 +402,7 @@ export type JudgeKeyRate = {
369
402
  /** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides qualify. */
370
403
  export declare function cellJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey: string): JudgeKeyRate[];
371
404
  /**
372
- * The judge rates a cell's summary line adds to its checks: a labelled judge whose label is one
405
+ * The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
373
406
  * of the cell's checks against its control is already there.
374
407
  */
375
408
  export declare function summaryJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cell: Pick<CellRecord, 'key' | 'checks'>): JudgeKeyRate[];
@@ -383,20 +416,20 @@ export type CollapsedJudge = {
383
416
  key: string;
384
417
  };
385
418
  /** Each key, or the lowest score plus how many judges there were. */
386
- export declare function collapsedJudges(judgements: readonly JudgementRecord[]): {
419
+ export declare function collapsedJudges(judgments: readonly JudgmentRecord[]): {
387
420
  shown: CollapsedJudge[];
388
421
  extra: number;
389
422
  };
390
423
  /** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
391
- export declare function collapsedJudgeLabel(judgements: readonly JudgementRecord[]): string | null;
424
+ export declare function collapsedJudgeLabel(judgments: readonly JudgmentRecord[]): string | null;
392
425
  /**
393
- * Each judgement's `expect` label, by position: the attempt's `toPass` assertions (less those on a
394
- * failed judge call, which recorded no judgement) in call order, when they are exactly as many as
395
- * its judgements. `check` is that assertion's label, the key of its check; a judgement no label
426
+ * Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
427
+ * failed judge call, which recorded no judgment) in call order, when they are exactly as many as
428
+ * its judgments. `check` is that assertion's label, the key of its check; a judgment no label
396
429
  * reaches is named by its question.
397
430
  */
398
- export declare function labelledJudgements(a: Pick<AttemptRecord, 'judgements' | 'assertions'>): Array<{
399
- judgement: JudgementRecord;
431
+ export declare function labeledJudgments(a: Pick<AttemptRecord, 'judgements' | 'assertions'>): Array<{
432
+ judgment: JudgmentRecord;
400
433
  label: string;
401
434
  labelled: boolean;
402
435
  check?: string;
@@ -408,6 +441,24 @@ export declare function cellUsage(e: Pick<EvalEntry, 'attempts'>, cellKey: strin
408
441
  * order, when they are exactly as many as its checks; undefined where no label reaches.
409
442
  */
410
443
  export declare function checkLabels(a: Pick<AttemptRecord, 'checks' | 'assertions'>): Array<string | undefined>;
444
+ /**
445
+ * Each `toSucceed` label's `c.exec` check, by position, when the attempt's `toSucceed` assertions
446
+ * are exactly as many as its checks; empty otherwise.
447
+ */
448
+ export declare function checksByLabel<C>(a: {
449
+ checks: readonly C[];
450
+ } & Pick<AttemptRecord, 'assertions'>): Map<string, C>;
451
+ /**
452
+ * How a `c.exec` given `retries` ended: `passed on try 2 of 3`, `failed all 3 tries`, or null
453
+ * when it had none or passed on its first try.
454
+ */
455
+ export declare function triesNote(exitCode: number, tried: number, of: number): string | null;
456
+ /** {@link triesNote} for a recorded check. */
457
+ export declare const checkTries: (c: {
458
+ exitCode: number;
459
+ retries?: number;
460
+ tries?: readonly unknown[];
461
+ }) => string | null;
411
462
  /** A check's command on one line: the first non-empty line, clipped; its log has the rest. */
412
463
  export declare function commandLine(command: string, max?: number): string;
413
464
  //#endregion
@@ -439,7 +490,7 @@ export type LiveCounts = {
439
490
  skipped: number;
440
491
  running: number;
441
492
  };
442
- /** What the run is doing: running attempts, cancelling them, or assembling its record. */
493
+ /** What the run is doing: running attempts, canceling them, or assembling its record. */
443
494
  export type LivePhase = 'running' | 'cancelling' | 'finishing';
444
495
  export type LiveProgress = {
445
496
  schemaVersion: 1;
@@ -673,8 +724,32 @@ export type AnswerRow = {
673
724
  /** `questions/<n>.json` under the run's directory: what the model was sent; null when not kept. */
674
725
  input: string | null;
675
726
  };
676
- /** One row per compare question, with each cut's win rates in the record's order. */
727
+ /** One row per pairwise question, with each cut's win rates in the record's order. */
677
728
  export declare function answerRows(e: Pick<EvalEntry, 'answers'>): AnswerRow[];
729
+ /** One attempt's answer to a question asked of it on its own. */
730
+ export type AttemptAnswerItem = {
731
+ /** The attempt's id, `<cell key>#<n>`. */
732
+ attempt: string;
733
+ outcome: Outcome;
734
+ choice: string | null;
735
+ text: string;
736
+ error: string | null;
737
+ /** `questions/<n>.json` in the attempt's directory: what the model was sent; null when not kept. */
738
+ input: string | null;
739
+ };
740
+ export type AttemptAnswerRow = {
741
+ question: string;
742
+ /** The model that answered: as served, when the provider said, else as configured. */
743
+ model: string;
744
+ /** How many attempts picked each choice, in the question's order; null on a free-text question. */
745
+ choices: Array<{
746
+ choice: string;
747
+ n: number;
748
+ }> | null;
749
+ answers: AttemptAnswerItem[];
750
+ };
751
+ /** One row per question asked of each attempt on its own, in first-asked order. */
752
+ export declare function attemptAnswerRows(e: Pick<EvalEntry, 'attempts'>): AttemptAnswerRow[];
678
753
  /** Whether an eval needs the cells table: several cells, or several runs of one. */
679
754
  export declare const isMatrix: (e: Pick<EvalEntry, 'cells'>) => boolean;
680
755
  export type EvalHistoryPoint = {
@@ -699,11 +774,15 @@ export type CompareViewRow = {
699
774
  /** See `CompareRow`: then pass@1 is over the cells both runs have too. */
700
775
  cells?: CompareRow['cells'];
701
776
  changes?: string[];
702
- /** Mean judge score per run; null without judgements or without the record. */
777
+ gate?: CompareRow['gate'];
778
+ /** Mean judge score per run; null without judgments or without the record. */
703
779
  baseScore: number | null;
704
780
  headScore: number | null;
705
781
  /** Cells in both runs, with lift and effort on each side. */
706
782
  perCell: CompareCell[];
783
+ /** See `CompareRow`: a pooled family's gate, with no slug, pass@1 or score of its own. */
784
+ kind?: 'family';
785
+ lifts?: CompareRow['lifts'];
707
786
  };
708
787
  export declare const COMPARE_ORDER: CompareStatus[];
709
788
  /**
@@ -712,6 +791,11 @@ export declare const COMPARE_ORDER: CompareStatus[];
712
791
  * the eval line labels it `gated`: `2/4 · pass@1 100% gated`.
713
792
  */
714
793
  export declare const hasUngatedCells: (cells: readonly Pick<HistoryCell, 'gate'>[]) => boolean;
794
+ /** A study's pass: every cell `gate: 'none'`, so nothing it saw passed or failed it. */
795
+ export declare const isNotGatedPass: (e: {
796
+ outcome: Outcome;
797
+ cells: readonly Pick<HistoryCell, 'gate'>[];
798
+ }) => boolean;
715
799
  /**
716
800
  * An expected miss: a failed attempt in an ungated cell (a control, by default). It's reported
717
801
  * apart from failures, since it fails neither its eval nor the run.
@@ -767,6 +851,12 @@ export type TranscriptItem = {
767
851
  /** Parses `transcript.jsonl`, skipping lines that aren't JSON (a truncated tail, say). */
768
852
  export declare function parseTranscript(text: string): TranscriptLine[];
769
853
  export declare function transcriptItems(lines: TranscriptLine[]): TranscriptItem[];
854
+ export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string) => EvalEntry | undefined;
855
+ export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
856
+ /** The attempt to open by default: the first failure, else the first attempt. */
857
+ export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
858
+ //#endregion
859
+ //#region src/view/patch.d.ts
770
860
  export type PatchFile = {
771
861
  path: string;
772
862
  status: 'added' | 'deleted' | 'modified' | 'renamed' | 'binary';
@@ -774,12 +864,13 @@ export type PatchFile = {
774
864
  removed: number;
775
865
  text: string;
776
866
  };
867
+ /**
868
+ * The path one file's part of a git patch changes: the new path, or the old one for a deletion.
869
+ * Read from its header lines, before the first `@@`, where git C-quotes an awkward name.
870
+ */
871
+ export declare function patchPath(part: string): string | null;
777
872
  /** Splits a git patch into one entry per file, with line counts. */
778
873
  export declare function splitPatch(patch: string): PatchFile[];
779
- export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string) => EvalEntry | undefined;
780
- export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
781
- /** The attempt to open by default: the first failure, else the first attempt. */
782
- export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
783
874
  //#endregion
784
875
  //#region src/view/paths.d.ts
785
876
  /**
@@ -828,6 +919,57 @@ export declare function parseDataPath(path: string): DataTarget | null;
828
919
  /** Artifacts are untrusted agent output: JSON stays JSON, PNGs are images, else plain text. */
829
920
  export declare const artifactType: (name: string) => string;
830
921
  //#endregion
922
+ //#region src/view/remarks.d.ts
923
+ /** `<history>/<run>/remarks.json`, beside the record. */
924
+ export declare const REMARKS_FILE = "remarks.json";
925
+ /** A run's remarks, `<history>/<run>/remarks.json`. Written only when the run raised one. */
926
+ export type RunRemarks = {
927
+ kind: 'reeval-remarks';
928
+ schemaVersion: 1;
929
+ runId: string;
930
+ createdAt: string;
931
+ /**
932
+ * Every remark this run raised, ranked, including those an earlier run printed first
933
+ * (`firstSeenRun`), which don't print again.
934
+ */
935
+ remarks: Remark[];
936
+ };
937
+ /** Each remark kind's severity. */
938
+ export declare const REMARK_KINDS: Record<Remark['kind'], {
939
+ severity: 'concern' | 'suggestion';
940
+ }>;
941
+ /** A remark about the eval's quality, not about the agent. */
942
+ export type Remark = {
943
+ /**
944
+ * sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
945
+ * per kind (and server) per eval version.
946
+ */
947
+ id: string;
948
+ evalId: string;
949
+ kind: 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'slow-mcp-start' | 'proxy-only-egress' | 'setup-process-secret';
950
+ /** Fixed by kind (`REMARK_KINDS`). */
951
+ severity: 'concern' | 'suggestion';
952
+ /** One plain sentence: what's wrong. */
953
+ text: string;
954
+ /** Facts from this run (or history facts) showing it mattered now. */
955
+ evidence: string[];
956
+ /** An edit at a source location (`source:<file>#L<n>`) in the eval. */
957
+ fix: {
958
+ at: string;
959
+ edit: string;
960
+ };
961
+ /**
962
+ * The first run that printed this `id`, or this run when none has: it prints only in that run,
963
+ * except the kinds about the environment, which print in every run they happen.
964
+ */
965
+ firstSeenRun: string;
966
+ /**
967
+ * Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
968
+ * when the cap or a run without `-v` left it out, so a later run prints it.
969
+ */
970
+ printed?: boolean;
971
+ };
972
+ //#endregion
831
973
  //#region src/view/record.d.ts
832
974
  /** A file from a newer reeval, whose schemaVersion this reeval doesn't know. */
833
975
  export declare class NewerSchemaError extends Error {
@@ -846,8 +988,328 @@ export declare function readableRecord(record: RunRecord, what?: string): RunRec
846
988
  export declare const readableTranscriptLine: (line: TranscriptLine) => TranscriptLine;
847
989
  /** An analysis document as every reader sees it; one from a newer reeval throws. */
848
990
  export declare const readableAnalysis: (doc: AnalysisDoc, what?: string) => AnalysisDoc;
849
- /** A run's insights as every reader sees them; a file from a newer reeval throws. */
850
- export declare const readableInsights: (doc: RunInsights, what?: string) => RunInsights;
991
+ /** A run's remarks as every reader sees them; a file from a newer reeval throws. */
992
+ export declare const readableRemarks: (doc: RunRemarks, what?: string) => RunRemarks;
993
+ //#endregion
994
+ //#region src/stats.d.ts
995
+ /**
996
+ * Counted attempts a side before a pass-rate lift can be called: below it, every view says
997
+ * `insufficient` (`can't tell yet`, `unclear`), whatever the interval. A product floor that keeps
998
+ * `reeval run`, `reeval query`, the judge-key lift and `reeval analyze` agreeing, not one that
999
+ * makes the stated 95% exact: at 4 a side, a 4/4 against 0/4 split happens by chance about 7% of
1000
+ * the time when nothing changed.
1001
+ */
1002
+ export declare const EFFECT_MIN_COUNTED = 4;
1003
+ //#endregion
1004
+ //#region src/view/suite.d.ts
1005
+ /**
1006
+ * A variant against its baseline: `better` or `worse` when the stored 95% interval on the
1007
+ * difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
1008
+ * including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
1009
+ */
1010
+ export type Effect = 'better' | 'worse' | 'same' | 'unclear';
1011
+ /**
1012
+ * What an eval's cells compare: `change` when it has variants (or several environments),
1013
+ * `harness` when it has several harnesses, `both`, or `none`.
1014
+ */
1015
+ export type Across = 'none' | 'change' | 'harness' | 'both';
1016
+ /**
1017
+ * One finished run's facts. Every node has a `path`, a stable id that remarks cite it by.
1018
+ *
1019
+ * A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
1020
+ *
1021
+ * - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
1022
+ * (`evals/a.eval.ts › fixes $id`);
1023
+ * - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
1024
+ * - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
1025
+ * - `<cell>/checks/<label>`: a check, by its assertion label;
1026
+ * - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
1027
+ *
1028
+ * A case segment never has a raw `+` and a cell segment always has two, so the two can't be
1029
+ * confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
1030
+ * fact in every run that has it.
1031
+ */
1032
+ export type SuiteResult = {
1033
+ runId: string;
1034
+ status: RunRecord['status'];
1035
+ /**
1036
+ * The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
1037
+ * an attempt, and `skipped` on an interrupted run.
1038
+ */
1039
+ outcome: Outcome;
1040
+ /**
1041
+ * Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
1042
+ * awaited, a worker that died. Any one fails the run.
1043
+ */
1044
+ errors: string[];
1045
+ /**
1046
+ * Outcomes counted per block: each family case is one, each eval with no cases is one, and a
1047
+ * family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
1048
+ * `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
1049
+ */
1050
+ tally: SuiteTally;
1051
+ /** Sorted by path. */
1052
+ files: ResultFile[];
1053
+ };
1054
+ export type SuiteTally = {
1055
+ errors: number;
1056
+ pass: number;
1057
+ notGated: number;
1058
+ fail: number;
1059
+ infraError: number;
1060
+ canceled: number;
1061
+ skipped: number;
1062
+ };
1063
+ /**
1064
+ * An eval's or a family case's outcome: the record's, except `canceled` where its only failed
1065
+ * gated attempts are ones the run's interrupt canceled, since the agent didn't fail those.
1066
+ */
1067
+ export type ResultOutcome = Outcome | 'canceled';
1068
+ export type ResultFile = {
1069
+ path: string;
1070
+ evals: EvalResult[];
1071
+ };
1072
+ export type EvalResult = {
1073
+ path: string;
1074
+ /** The eval's id, or the family's. */
1075
+ id: string;
1076
+ group: string[];
1077
+ /** The eval's name; on a family, its template. */
1078
+ name: string;
1079
+ /** A family's name template (`fixes $id`); null on a single eval. */
1080
+ template: string | null;
1081
+ /** Source order in the file. */
1082
+ order: number;
1083
+ across: Across;
1084
+ /**
1085
+ * A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
1086
+ * unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
1087
+ */
1088
+ family: boolean;
1089
+ /** A placeholder (`evaluate.todo`) that never ran. */
1090
+ todo: boolean;
1091
+ /** On a family: `fail` when a case failed or its pooled gate did. */
1092
+ outcome: ResultOutcome;
1093
+ cells: CellResult[] | null;
1094
+ cases: CaseResult[] | null;
1095
+ /** A family's variants against their baseline, pooled over the cases; null on a single eval. */
1096
+ pooled: PooledArm[] | null;
1097
+ /**
1098
+ * Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
1099
+ * each question asked of any case, in first-asked order, summed over the cases that answered it.
1100
+ */
1101
+ questions: Array<QuestionResult | AttemptQuestionResult>;
1102
+ };
1103
+ /** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
1104
+ export type QuestionResult = {
1105
+ kind: 'pairs';
1106
+ question: string;
1107
+ /** Pairs voted, each counted once per order it was asked in. */
1108
+ votes: number;
1109
+ rates: Array<{
1110
+ cut: 'within' | 'across';
1111
+ name: string;
1112
+ wins: number;
1113
+ n: number;
1114
+ }>;
1115
+ /** Why some or all of its votes are missing: the model call failed. */
1116
+ error: string | null;
1117
+ };
1118
+ /** A question asked of each attempt on its own: what each one answered. Never gates. */
1119
+ export type AttemptQuestionResult = {
1120
+ kind: 'attempts';
1121
+ question: string;
1122
+ /** Attempts it was asked of, failed calls included. */
1123
+ asked: number;
1124
+ /** How many attempts picked each choice, in the question's order; null on a free-text question. */
1125
+ choices: Array<{
1126
+ choice: string;
1127
+ n: number;
1128
+ }> | null;
1129
+ /** Each attempt's answer, in attempt order; failed calls left out. */
1130
+ answers: Array<{
1131
+ attempt: string;
1132
+ choice: string | null;
1133
+ text: string;
1134
+ }>;
1135
+ /** Calls that failed, and the first one's error. */
1136
+ failed: number;
1137
+ error: string | null;
1138
+ };
1139
+ export type CaseResult = {
1140
+ path: string;
1141
+ /** The case's own eval id. */
1142
+ id: string;
1143
+ /** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
1144
+ label: string;
1145
+ name: string;
1146
+ /** Its place in the case list. */
1147
+ order: number;
1148
+ outcome: ResultOutcome;
1149
+ cells: CellResult[];
1150
+ };
1151
+ export type CellResult = {
1152
+ path: string;
1153
+ key: string;
1154
+ harness: string;
1155
+ /** `-` when no environment is set. */
1156
+ environment: string;
1157
+ sandbox: string;
1158
+ /** The variant's name; null on a control. */
1159
+ variant: string | null;
1160
+ /** The control of an eval that has variants: the arm the variants are measured against. */
1161
+ baseline: boolean;
1162
+ /** On a variant, its baseline cell's key; else null. */
1163
+ vs: string | null;
1164
+ /** The cell's place in the eval's cells. */
1165
+ order: number;
1166
+ /** Its failures fail the eval (its gate isn't `'none'`). */
1167
+ gated: boolean;
1168
+ outcome: Outcome;
1169
+ /** Every attempt: `passed + failed + infraError + skipped`. */
1170
+ attempts: number;
1171
+ passed: number;
1172
+ failed: number;
1173
+ /** Infrastructure errors, which are neither passes nor fails. */
1174
+ infraError: number;
1175
+ skipped: number;
1176
+ fingerprint: string;
1177
+ /** The skill sources and files the cell's environment placed, as written. */
1178
+ context: string[];
1179
+ /**
1180
+ * Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
1181
+ * attempt (every one an infra error or skipped) lists each as `not run`.
1182
+ */
1183
+ checks: CheckResult[];
1184
+ /** On a variant, against its baseline; null on a control. */
1185
+ effect: Effect | null;
1186
+ /** The stored 95% interval on the variant's difference from its baseline. Not for display. */
1187
+ interval: Interval | null;
1188
+ /** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
1189
+ minLift: number | null;
1190
+ /** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
1191
+ effort: EffortCounters | null;
1192
+ /** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
1193
+ effortDiffers: boolean;
1194
+ /** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
1195
+ errors: CellError[];
1196
+ /** Web searches and fetches per counted attempt; null where the harness can't report them. */
1197
+ searches: Array<number | null>;
1198
+ fetches: Array<number | null>;
1199
+ /**
1200
+ * The upstream providers a gateway (OpenRouter) said served the cell's agent calls, with how
1201
+ * many, summed over its attempts in first-seen order; absent when it named none.
1202
+ */
1203
+ upstreams?: UpstreamCount[];
1204
+ /** The same for the cell's judge calls; absent when the gateway named none. */
1205
+ judgeUpstreams?: UpstreamCount[];
1206
+ };
1207
+ /** A cell's attempts the run's interrupt canceled: recorded as fails, but not the agent's. */
1208
+ export declare const canceledOf: (cell: CellResult) => number;
1209
+ /** A cell's counted tries: its passes and fails, less the ones the interrupt canceled. */
1210
+ export declare const countedOf: (cell: CellResult) => number;
1211
+ export type CheckResult = {
1212
+ path: string;
1213
+ label: string;
1214
+ /** First-seen order across the eval's attempts. */
1215
+ order: number;
1216
+ outcome: CheckOutcome;
1217
+ /**
1218
+ * On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
1219
+ * null on a control or without one.
1220
+ */
1221
+ effect: Effect | null;
1222
+ /** Set when the check is a judge verdict. */
1223
+ judge: JudgeResult | null;
1224
+ /**
1225
+ * A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
1226
+ * it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
1227
+ * when none did.
1228
+ */
1229
+ retried?: string;
1230
+ };
1231
+ /**
1232
+ * `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
1233
+ * none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
1234
+ * reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
1235
+ * first failure's message.
1236
+ */
1237
+ export type CheckOutcome = {
1238
+ status: 'not run';
1239
+ } | {
1240
+ status: 'ran';
1241
+ passed: number;
1242
+ failed: number;
1243
+ errored: number;
1244
+ notRun: number;
1245
+ assertion: string | null;
1246
+ };
1247
+ /**
1248
+ * One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
1249
+ * an error), its message's first line, and how often.
1250
+ */
1251
+ export type CellError = {
1252
+ kind: ErrorKind | 'turn';
1253
+ message: string;
1254
+ attempts: number;
1255
+ };
1256
+ export type JudgeResult = {
1257
+ question: string;
1258
+ threshold: number;
1259
+ /** One per counted attempt that judged it. */
1260
+ scores: number[];
1261
+ };
1262
+ export type Interval = {
1263
+ low: number;
1264
+ high: number;
1265
+ };
1266
+ /** A null counter is one the harness didn't record; a recorded zero stays zero. */
1267
+ export type EffortCounters = {
1268
+ tools: number | null;
1269
+ agentMs: number | null;
1270
+ tokens: number | null;
1271
+ };
1272
+ export type PooledArm = {
1273
+ path: string;
1274
+ /** The variant cell's key, the same in every case. */
1275
+ key: string;
1276
+ vs: string;
1277
+ effect: Effect;
1278
+ /** Cases with counted attempts on both sides. */
1279
+ pairs: number;
1280
+ interval: Interval;
1281
+ /** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
1282
+ minLift: number | null;
1283
+ outcome: 'pass' | 'fail';
1284
+ };
1285
+ type Side = {
1286
+ pass: number;
1287
+ counted: number;
1288
+ };
1289
+ /**
1290
+ * The effect of `variant` against `control`, from the interval on their difference. A side
1291
+ * with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
1292
+ */
1293
+ export declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
1294
+ /** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
1295
+ export declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
1296
+ /** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
1297
+ export declare const factSegment: (s: string) => string;
1298
+ export declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
1299
+ /**
1300
+ * An error's one line: its first line with any letter or digit in it, joined with the next one
1301
+ * when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
1302
+ */
1303
+ export declare function messageLine(text: string): string;
1304
+ /** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
1305
+ export declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
1306
+ /**
1307
+ * A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
1308
+ * gating it, so it reads `not gated`, never `pass`.
1309
+ */
1310
+ export declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
1311
+ /** The facts of a run, from its record. Pure: the same record always gives the same result. */
1312
+ export declare function suiteResult(record: RunRecord): SuiteResult;
851
1313
  //#endregion
852
- export { Across, type ArtifactRef, type AttemptRecord, CaseResult, CellEffort, CellError, type CellRecord, CellResult, CheckOutcome, CheckResult, DIRECTIONAL_MIN, EFFORT_ALPHA, EFFORT_MIN_RUNS, Effect, EffortContrast, EffortCounters, EffortMetric, EffortSample, EffortVerdict, type EvalEntry, EvalInsight, EvalResult, type FamilyLift, type FamilyRate, type FamilyRecord, type FileRef, type HistoryCell, type HistoryEntry, INSIGHTS_FILE, Interval, Job, JudgeResult, type JudgementRecord, MODEL_REMARK_KINDS, NO_MODEL, type Outcome, PooledArm, QuestionResult, REMARK_KINDS, Remark, ResultFile, ResultOutcome, RunInsights, type RunRecord, type SandboxWarning, Sentence, Spread, type StackSummary, SuiteResult, SuiteTally, TOKENS_UNREPORTED, type TranscriptEvent, VERDICT_TEXT, bothUnderATenth, cancelledOf, cellContrast, cellEffort, cellSamples, checkEffect, contrastEffort, countedOf, directional, effectOf, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, evalAcross, factSegment, formatCount, formatRatio, formatSeconds, fromModel, messageLine, suiteResult, tokenValues };
1314
+ export { type ArtifactRef, type AttemptRecord, CellEffort, type CellRecord, DIRECTIONAL_MIN, EFFORT_ALPHA, EFFORT_MIN_RUNS, EffortContrast, EffortMetric, EffortSample, EffortVerdict, type EvalEntry, type FamilyLift, type FamilyRate, type FamilyRecord, type FileRef, type HistoryCell, type HistoryEntry, type JudgmentRecord, type Outcome, type RunRecord, type SandboxWarning, Spread, type StackSummary, TOKENS_UNREPORTED, type TranscriptEvent, VERDICT_TEXT, bothUnderATenth, cellContrast, cellEffort, cellSamples, contrastEffort, directional, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, formatCount, formatRatio, formatSeconds, tokenValues };
853
1315
  //# sourceMappingURL=index.d.ts.map