reeval 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. package/README.md +3 -1
  2. package/bin/mcp-timer.cjs +224 -0
  3. package/dist/{adapter-Bwy3QxAK.js → adapter-DigeVFWC.js} +3 -3
  4. package/dist/{adapter-Bwy3QxAK.js.map → adapter-DigeVFWC.js.map} +1 -1
  5. package/dist/{agent-DpE80WPz.js → agent-CatUEo5h.js} +21 -6
  6. package/dist/{agent-DpE80WPz.js.map → agent-CatUEo5h.js.map} +1 -1
  7. package/dist/{assemble-D9GWr-HS.js → assemble-CF063ms6.js} +19 -17
  8. package/dist/assemble-CF063ms6.js.map +1 -0
  9. package/dist/assemble-CJoD_Rzr.js +1 -0
  10. package/dist/bridge/index.mjs +2 -1
  11. package/dist/{capture-BQlGHiMC.js → capture-BiD7lVaM.js} +25 -17
  12. package/dist/{capture-BQlGHiMC.js.map → capture-BiD7lVaM.js.map} +1 -1
  13. package/dist/capture-Dp3q7K7Y.js +1 -0
  14. package/dist/{cli-ClpPT8kn.js → cli-B5zQCR2r.js} +161 -116
  15. package/dist/cli-B5zQCR2r.js.map +1 -0
  16. package/dist/{cli-spec-qmEnQ25_.js → cli-spec-BXIBUlaK.js} +34 -15
  17. package/dist/cli-spec-BXIBUlaK.js.map +1 -0
  18. package/dist/cli.js +2 -2
  19. package/dist/{command-TgbzPizb.js → command-BMHXHqMT.js} +15 -14
  20. package/dist/command-BMHXHqMT.js.map +1 -0
  21. package/dist/command-C94CQNh8.js +91 -0
  22. package/dist/command-C94CQNh8.js.map +1 -0
  23. package/dist/command-CVtgyqlM.js +335 -0
  24. package/dist/command-CVtgyqlM.js.map +1 -0
  25. package/dist/{command-DbAe1Rzb.js → command-CaZgCTIw.js} +14 -17
  26. package/dist/command-CaZgCTIw.js.map +1 -0
  27. package/dist/{command-BlM0y8jN.js → command-DTgBZR0k.js} +6 -6
  28. package/dist/{command-BlM0y8jN.js.map → command-DTgBZR0k.js.map} +1 -1
  29. package/dist/{commands-CthqwkAn.js → commands-BXzIDB3X.js} +1 -1
  30. package/dist/commands-CjpGBUWr.js +666 -0
  31. package/dist/commands-CjpGBUWr.js.map +1 -0
  32. package/dist/{config-DrVSEVN0.js → config-Dv5MG7Rp.js} +27 -48
  33. package/dist/config-Dv5MG7Rp.js.map +1 -0
  34. package/dist/{context-C1mxXh80.js → context-Dgn48BqT.js} +13 -2
  35. package/dist/context-Dgn48BqT.js.map +1 -0
  36. package/dist/{contrasts-5zsj32gk.js → contrasts-CcoMLC_y.js} +3 -3
  37. package/dist/{contrasts-5zsj32gk.js.map → contrasts-CcoMLC_y.js.map} +1 -1
  38. package/dist/db-BeqRiWLl.js +602 -0
  39. package/dist/db-BeqRiWLl.js.map +1 -0
  40. package/dist/db-CJDRVQal.js +2 -0
  41. package/dist/{dist-CuY4ehEk.js → dist-CMjp3qqF.js} +2 -2
  42. package/dist/{dist-CuY4ehEk.js.map → dist-CMjp3qqF.js.map} +1 -1
  43. package/dist/{docker-DOvjji9I.js → docker-BbTlMmYi.js} +2 -2
  44. package/dist/{docker-DOvjji9I.js.map → docker-BbTlMmYi.js.map} +1 -1
  45. package/dist/{docker-D1HZDS6q.js → docker-CCXdbREk.js} +20 -4
  46. package/dist/{docker-D1HZDS6q.js.map → docker-CCXdbREk.js.map} +1 -1
  47. package/dist/{docker-DNmMPOBb.js → docker-DC69dKxB.js} +1 -1
  48. package/dist/{doctor-D2MbNO8L.js → doctor-CqttrZcs.js} +57 -22
  49. package/dist/doctor-CqttrZcs.js.map +1 -0
  50. package/dist/{egress-2-qzPvGi.js → egress-BfK_O9ce.js} +435 -30
  51. package/dist/egress-BfK_O9ce.js.map +1 -0
  52. package/dist/env-BQqj93gu.js +21 -0
  53. package/dist/env-BQqj93gu.js.map +1 -0
  54. package/dist/{environment-Cca6a4wB.js → environment-BBz1Yes9.js} +224 -10
  55. package/dist/environment-BBz1Yes9.js.map +1 -0
  56. package/dist/{evaluate-DKIl54Hb.js → evaluate-CXQ1o-pd.js} +539 -559
  57. package/dist/evaluate-CXQ1o-pd.js.map +1 -0
  58. package/dist/{evaluate-DMyaGQEj.d.ts → evaluate-yjZKUexW.d.ts} +2 -2
  59. package/dist/facts-DtENlf9s.js +898 -0
  60. package/dist/facts-DtENlf9s.js.map +1 -0
  61. package/dist/{files-BpUYBwGu.js → files-B-eUpCZ1.js} +30 -7
  62. package/dist/files-B-eUpCZ1.js.map +1 -0
  63. package/dist/files-BsQbYz2i.js +65 -0
  64. package/dist/files-BsQbYz2i.js.map +1 -0
  65. package/dist/{files-BXHZFWmx.js → files-COHQLEHE.js} +6 -17
  66. package/dist/files-COHQLEHE.js.map +1 -0
  67. package/dist/files-DqCBW7i8.js +1 -0
  68. package/dist/files-Ha-jsQXf.js +1 -0
  69. package/dist/generate-Cs2TMH_k.js +1647 -0
  70. package/dist/generate-Cs2TMH_k.js.map +1 -0
  71. package/dist/{git-Crl3b87o.js → git-DhykwgKF.js} +2 -2
  72. package/dist/{git-Crl3b87o.js.map → git-DhykwgKF.js.map} +1 -1
  73. package/dist/{harness-CEuntrla.js → harness-BAjw1fcr.js} +2 -2
  74. package/dist/{harness-CEuntrla.js.map → harness-BAjw1fcr.js.map} +1 -1
  75. package/dist/harness.d.ts +1 -1
  76. package/dist/harness.js +1 -1
  77. package/dist/{header-DHQv_ZrF.js → header-kSmWatcO.js} +10 -3
  78. package/dist/{header-DHQv_ZrF.js.map → header-kSmWatcO.js.map} +1 -1
  79. package/dist/{images-Bi3aGxb9.js → images-BmJRuTpm.js} +61 -13
  80. package/dist/images-BmJRuTpm.js.map +1 -0
  81. package/dist/index.d.ts +3 -3
  82. package/dist/index.js +4 -4
  83. package/dist/init-Bwc9byQr.js +543 -0
  84. package/dist/init-Bwc9byQr.js.map +1 -0
  85. package/dist/init-D7Y1ny1z.js +1 -0
  86. package/dist/insights-B2Xmhz9o.js +64 -0
  87. package/dist/insights-B2Xmhz9o.js.map +1 -0
  88. package/dist/{invocations-BYDPe3fV.js → invocations-CQTbP092.js} +3 -3
  89. package/dist/{invocations-BYDPe3fV.js.map → invocations-CQTbP092.js.map} +1 -1
  90. package/dist/{judge-DW_GDOmB.js → judge-Dt6KW6_O.js} +5 -3
  91. package/dist/judge-Dt6KW6_O.js.map +1 -0
  92. package/dist/judge.d.ts +7 -6
  93. package/dist/judge.js +8 -7
  94. package/dist/judge.js.map +1 -1
  95. package/dist/{live-jlfszAbb.js → live-DYd2QEyR.js} +2 -2
  96. package/dist/{live-jlfszAbb.js.map → live-DYd2QEyR.js.map} +1 -1
  97. package/dist/{model-D0IcOQao.js → model-B-mel-Gv.js} +43 -18
  98. package/dist/model-B-mel-Gv.js.map +1 -0
  99. package/dist/model-BGKYVpoj.js +1 -0
  100. package/dist/{model-DaVCY3rb.js → model-C7tSwT7f.js} +2 -2
  101. package/dist/{model-DaVCY3rb.js.map → model-C7tSwT7f.js.map} +1 -1
  102. package/dist/{model-DaX4bNLZ.js → model-CWlAqmbU.js} +135 -40
  103. package/dist/model-CWlAqmbU.js.map +1 -0
  104. package/dist/model-DHY-dfoO.js +1 -0
  105. package/dist/{node-B3ydHo6t.js → node-D0mtjRAO.js} +3 -3
  106. package/dist/{node-B3ydHo6t.js.map → node-D0mtjRAO.js.map} +1 -1
  107. package/dist/{options-C0d0AzeR.js → options-CPUW97_7.js} +7 -9
  108. package/dist/options-CPUW97_7.js.map +1 -0
  109. package/dist/{pairs-CnInRu-G.js → pairs-DP6XOJby.js} +2 -2
  110. package/dist/{pairs-CnInRu-G.js.map → pairs-DP6XOJby.js.map} +1 -1
  111. package/dist/{patch-B1bnfhN-.js → patch-D4oluWzH.js} +2 -2
  112. package/dist/{patch-B1bnfhN-.js.map → patch-D4oluWzH.js.map} +1 -1
  113. package/dist/{plan-DgQxp9yy.js → plan-OkwUcJQ9.js} +21 -8
  114. package/dist/plan-OkwUcJQ9.js.map +1 -0
  115. package/dist/plugins-CUWMnSNb.js +654 -0
  116. package/dist/plugins-CUWMnSNb.js.map +1 -0
  117. package/dist/{preflight-DlQvbySb.js → preflight-B6pY89bF.js} +1 -1
  118. package/dist/{preflight-BRzD30L5.js → preflight-Bjm7_ylE.js} +14 -10
  119. package/dist/preflight-Bjm7_ylE.js.map +1 -0
  120. package/dist/{prepare-B_y-AFMv.js → prepare-CQmK3SBn.js} +70 -23
  121. package/dist/prepare-CQmK3SBn.js.map +1 -0
  122. package/dist/prepare-DbdgtZKe.js +2 -0
  123. package/dist/{provenance-DQ0AXn6c.js → provenance-B3Y17oCC.js} +3 -3
  124. package/dist/{provenance-DQ0AXn6c.js.map → provenance-B3Y17oCC.js.map} +1 -1
  125. package/dist/providers-CXN2RFz-.js +1 -0
  126. package/dist/{providers-DoTq1jHW.js → providers-eoniqK6w.js} +24 -10
  127. package/dist/providers-eoniqK6w.js.map +1 -0
  128. package/dist/{query-BO-ux7RB.js → query-CLRZmWBT.js} +3 -3
  129. package/dist/{query-BO-ux7RB.js.map → query-CLRZmWBT.js.map} +1 -1
  130. package/dist/query-CmBDYt0_.js +1 -0
  131. package/dist/{questions-C2dym2ce.js → questions-1XOulevV.js} +6 -6
  132. package/dist/{questions-C2dym2ce.js.map → questions-1XOulevV.js.map} +1 -1
  133. package/dist/{record-BKmMvZ6O.js → record-kk9M1hj-.js} +8 -4
  134. package/dist/record-kk9M1hj-.js.map +1 -0
  135. package/dist/remarks-1521zwfW.js +38 -0
  136. package/dist/remarks-1521zwfW.js.map +1 -0
  137. package/dist/remarks-BhAV1H4V.js +1 -0
  138. package/dist/remarks-DlJCqWRs.js +278 -0
  139. package/dist/remarks-DlJCqWRs.js.map +1 -0
  140. package/dist/{remote-CPPW86Q9.js → remote-cE2VWfkJ.js} +3 -3
  141. package/dist/{remote-CPPW86Q9.js.map → remote-cE2VWfkJ.js.map} +1 -1
  142. package/dist/resolve-C4zeccCH.js +724 -0
  143. package/dist/resolve-C4zeccCH.js.map +1 -0
  144. package/dist/resolve-D30yZYmB.js +1 -0
  145. package/dist/{results-BE2f-1XS.js → results-Drtt0uFl.js} +33 -8
  146. package/dist/results-Drtt0uFl.js.map +1 -0
  147. package/dist/{resume-Ca1c8vix.js → resume-BEFazUfy.js} +4 -4
  148. package/dist/{resume-Ca1c8vix.js.map → resume-BEFazUfy.js.map} +1 -1
  149. package/dist/rules-BFzQqPIm.js +317 -0
  150. package/dist/rules-BFzQqPIm.js.map +1 -0
  151. package/dist/run-BeMy92Us.js +88 -0
  152. package/dist/run-BeMy92Us.js.map +1 -0
  153. package/dist/{sandbox-DLrsvu7U.js → sandbox-CQaJj7n9.js} +2 -2
  154. package/dist/{sandbox-DLrsvu7U.js.map → sandbox-CQaJj7n9.js.map} +1 -1
  155. package/dist/sandbox.d.ts +1 -1
  156. package/dist/sandbox.js +1 -1
  157. package/dist/server-V4-2w0l8.js +1 -0
  158. package/dist/{server-TUY5xA_A.js → server-xK0les4f.js} +8 -8
  159. package/dist/{server-TUY5xA_A.js.map → server-xK0les4f.js.map} +1 -1
  160. package/dist/source-BDow_oW8.js +1 -0
  161. package/dist/{source-DJGS0a0u.js → source-Q1hEKbCs.js} +2 -2
  162. package/dist/{source-DJGS0a0u.js.map → source-Q1hEKbCs.js.map} +1 -1
  163. package/dist/src-B9L9yGKI.js +1 -0
  164. package/dist/{src-CLoOMVWI.js → src-Bcah6ZkT.js} +1 -1
  165. package/dist/{src-DvCg0NBs.js → src-BhmaorOU.js} +28 -6
  166. package/dist/{src-DvCg0NBs.js.map → src-BhmaorOU.js.map} +1 -1
  167. package/dist/{src-CF17Ug08.js → src-C0CeKiLU.js} +19 -4
  168. package/dist/{src-CF17Ug08.js.map → src-C0CeKiLU.js.map} +1 -1
  169. package/dist/src-C0xowZ3C.js +2 -0
  170. package/dist/src-CHIfIKUu.js +2 -0
  171. package/dist/src-DLtyz4w5.js +2 -0
  172. package/dist/src-DPIjmReZ.js +2 -0
  173. package/dist/{src-BxfCKA3_.js → src-DXNh5mTr.js} +1 -1
  174. package/dist/{src-FGXhkEkH.js → src-DiLMkIr9.js} +142 -13
  175. package/dist/src-DiLMkIr9.js.map +1 -0
  176. package/dist/{src-ftu5wPSt.js → src-LekQrr_S.js} +209 -29
  177. package/dist/src-LekQrr_S.js.map +1 -0
  178. package/dist/src-bCnsA6Vp.js +2 -0
  179. package/dist/{src-u8_ixn_F.js → src-cbCb-MPy.js} +20 -4
  180. package/dist/src-cbCb-MPy.js.map +1 -0
  181. package/dist/{src-DvK0EIZx.js → src-qGoFkuJF.js} +55 -6
  182. package/dist/{src-DvK0EIZx.js.map → src-qGoFkuJF.js.map} +1 -1
  183. package/dist/{src-DSYYPEOj.js → src-ynBVosfX.js} +1 -1
  184. package/dist/state-BQR_BCm4.js +1 -0
  185. package/dist/{state-DfpHox6b.js → state-BmZ8QVZ8.js} +12 -12
  186. package/dist/{state-DfpHox6b.js.map → state-BmZ8QVZ8.js.map} +1 -1
  187. package/dist/studio-app/assets/{index-CBkhNAv8.js → index-CnYJijyI.js} +9 -9
  188. package/dist/studio-app/index.html +1 -1
  189. package/dist/suite-BaHPmAGy.js +1 -0
  190. package/dist/{suite-Cw_7G2RM.js → suite-C8sw2pox.js} +80 -28
  191. package/dist/suite-C8sw2pox.js.map +1 -0
  192. package/dist/{summary-BUKfU-9l.js → summary-0oQNGskv.js} +273 -47
  193. package/dist/summary-0oQNGskv.js.map +1 -0
  194. package/dist/testing.d.ts +2 -2
  195. package/dist/testing.js +2 -2
  196. package/dist/{text-gEcKjAyp.js → text-CrVy5ioL.js} +18 -2
  197. package/dist/text-CrVy5ioL.js.map +1 -0
  198. package/dist/{transcript-BNNANU8A.js → transcript-BTxJpW4R.js} +2 -2
  199. package/dist/{transcript-BNNANU8A.js.map → transcript-BTxJpW4R.js.map} +1 -1
  200. package/dist/types-C5vI3g1C.js.map +1 -1
  201. package/dist/{types-BYdlwMwo.d.ts → types-GraZtfHN.d.ts} +848 -34
  202. package/dist/usage-BDo2q_yu.js +1 -0
  203. package/dist/{command-Bj39y-Wm.js → validate-BdoXUuL_.js} +15 -372
  204. package/dist/validate-BdoXUuL_.js.map +1 -0
  205. package/dist/view/index.d.ts +20 -257
  206. package/dist/view/index.js +7 -6
  207. package/dist/{views-B8w2J-gD.js → views-BoUoX4_A.js} +8 -98
  208. package/dist/views-BoUoX4_A.js.map +1 -0
  209. package/dist/views-noVQNCUP.js +2 -0
  210. package/dist/{worker-entry-DNnveLqx.js → worker-entry-D0-GErmU.js} +2 -2
  211. package/dist/{worker-entry-DNnveLqx.js.map → worker-entry-D0-GErmU.js.map} +1 -1
  212. package/dist/{workspace-C_X0tdv6.js → workspace-B90YDU_a.js} +1 -1
  213. package/dist/{workspace-C4RA3srV.js → workspace-tjZFkjxC.js} +4 -4
  214. package/dist/{workspace-C4RA3srV.js.map → workspace-tjZFkjxC.js.map} +1 -1
  215. package/package.json +3 -1
  216. package/skills/eval-rubric/SKILL.md +130 -0
  217. package/dist/agent-D483ru7D.js +0 -334
  218. package/dist/agent-D483ru7D.js.map +0 -1
  219. package/dist/assemble-D5KGrwnJ.js +0 -1
  220. package/dist/assemble-D9GWr-HS.js.map +0 -1
  221. package/dist/capture-CYGLpWxR.js +0 -1
  222. package/dist/cli-ClpPT8kn.js.map +0 -1
  223. package/dist/cli-spec-qmEnQ25_.js.map +0 -1
  224. package/dist/command-Bj39y-Wm.js.map +0 -1
  225. package/dist/command-C-cVjlny.js +0 -1
  226. package/dist/command-DbAe1Rzb.js.map +0 -1
  227. package/dist/command-TgbzPizb.js.map +0 -1
  228. package/dist/commands-YUwlyTbr.js +0 -228
  229. package/dist/commands-YUwlyTbr.js.map +0 -1
  230. package/dist/config-DrVSEVN0.js.map +0 -1
  231. package/dist/context-C1mxXh80.js.map +0 -1
  232. package/dist/db-CKg5NtiW.js +0 -2
  233. package/dist/db-DfhipHEF.js +0 -1416
  234. package/dist/db-DfhipHEF.js.map +0 -1
  235. package/dist/doctor-D2MbNO8L.js.map +0 -1
  236. package/dist/egress-2-qzPvGi.js.map +0 -1
  237. package/dist/environment-Cca6a4wB.js.map +0 -1
  238. package/dist/evaluate-DKIl54Hb.js.map +0 -1
  239. package/dist/files-BXHZFWmx.js.map +0 -1
  240. package/dist/files-BpUYBwGu.js.map +0 -1
  241. package/dist/files-Bw8lWEjg.js +0 -1
  242. package/dist/files-D6J0iSdx.js +0 -1
  243. package/dist/images-Bi3aGxb9.js.map +0 -1
  244. package/dist/init-Cl3f3NBT.js +0 -360
  245. package/dist/init-Cl3f3NBT.js.map +0 -1
  246. package/dist/init-fhOXED8F.js +0 -1
  247. package/dist/judge-DW_GDOmB.js.map +0 -1
  248. package/dist/model-CXhrziBe.js +0 -1
  249. package/dist/model-D0IcOQao.js.map +0 -1
  250. package/dist/model-DaX4bNLZ.js.map +0 -1
  251. package/dist/model-_bxNrk2h.js +0 -1
  252. package/dist/options-C0d0AzeR.js.map +0 -1
  253. package/dist/plan-DgQxp9yy.js.map +0 -1
  254. package/dist/preflight-BRzD30L5.js.map +0 -1
  255. package/dist/prepare-B_y-AFMv.js.map +0 -1
  256. package/dist/prepare-DCKTLHN3.js +0 -2
  257. package/dist/providers-B9yTdIqF.js +0 -1
  258. package/dist/providers-DoTq1jHW.js.map +0 -1
  259. package/dist/query-CRQfsw0e.js +0 -1
  260. package/dist/record-BKmMvZ6O.js.map +0 -1
  261. package/dist/results-BE2f-1XS.js.map +0 -1
  262. package/dist/server-Dp8eOeK5.js +0 -1
  263. package/dist/source-DdbLGR1M.js +0 -1
  264. package/dist/src-CcrQbpG8.js +0 -2
  265. package/dist/src-DCFtJRp0.js +0 -2
  266. package/dist/src-DYO3DdnQ.js +0 -1
  267. package/dist/src-FGXhkEkH.js.map +0 -1
  268. package/dist/src-WD-zgQYZ.js +0 -2
  269. package/dist/src-_oI9AePb.js +0 -2
  270. package/dist/src-ftu5wPSt.js.map +0 -1
  271. package/dist/src-qsKCqv7a.js +0 -2
  272. package/dist/src-u8_ixn_F.js.map +0 -1
  273. package/dist/state-DpAxYyED.js +0 -1
  274. package/dist/suite-Cw_7G2RM.js.map +0 -1
  275. package/dist/suite-DamAEnNF.js +0 -1
  276. package/dist/summary-BUKfU-9l.js.map +0 -1
  277. package/dist/summary-Baa4rTHS.js +0 -192
  278. package/dist/summary-Baa4rTHS.js.map +0 -1
  279. package/dist/text-gEcKjAyp.js.map +0 -1
  280. package/dist/tools-DLHgetpx.js +0 -249
  281. package/dist/tools-DLHgetpx.js.map +0 -1
  282. package/dist/views-B8w2J-gD.js.map +0 -1
  283. package/dist/views-DblnI3kA.js +0 -2
@@ -1,1416 +0,0 @@
1
- import { n as toolTarget, t as SKILL_LOAD_TOOLS } from "./tool-target-D2mcYlSQ.js";
2
- import { n as UsageError } from "./errors-B6BSMs03.js";
3
- import { t as RUN_ID } from "./paths-BmR-7tx3.js";
4
- import { c as relativeFile, l as sha256 } from "./ids-Bp6ut7ZD.js";
5
- import { d as ownRequire, t as BUILD_ID } from "./root-DICDHCtp.js";
6
- import { n as isRuntimeState } from "./context-C1mxXh80.js";
7
- import { $ as contrastEffort, dt as liftInterval, gt as round3, mt as median, rt as effortOf, vt as wilson } from "./model-DaX4bNLZ.js";
8
- import { r as pairTallies } from "./pairs-CnInRu-G.js";
9
- import { m as warnUnreadable, p as scratchDir, s as readRecord, u as recordNames } from "./files-BpUYBwGu.js";
10
- import { n as contrastPairs, t as checkVerdict } from "./contrasts-5zsj32gk.js";
11
- import { dirname, join } from "node:path";
12
- import { mkdir, readFile, stat } from "node:fs/promises";
13
- //#region src/history/rows.ts
14
- /** A statement's rows as `Row`, the columns its SQL selects: SQLite checks none of them. */
15
- const rowsOf = (statement, ...params) => statement.all(...params);
16
- //#endregion
17
- //#region src/history/tool-class.ts
18
- const TOOL_CLASSES = [
19
- "context",
20
- "dependency",
21
- "web",
22
- "verify",
23
- "workspace"
24
- ];
25
- const SHELL_TOOL = /shell|bash|terminal|command|exec/i;
26
- const WEB_TOOL = /web|fetch|browse|url/i;
27
- const DEPENDENCY = /(^|\/)(node_modules|site-packages|dist-packages|vendor|\.venv|\.cargo\/registry|go\/pkg\/mod)(\/|$)/;
28
- const VERIFY = /\b(?:(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:test|build|lint|typecheck|check)|npx\s+(?:tsc|vitest|jest|eslint|biome)|tsc|vitest|jest|pytest|eslint|cargo\s+(?:test|build|check)|go\s+(?:test|build|vet))\b/;
29
- /** Words of a shell command, quotes removed; `;`, `|` and `&&` split words. */
30
- function shellWords(command) {
31
- const words = [];
32
- let word = "";
33
- let quote = null;
34
- let started = false;
35
- const flush = () => {
36
- if (started) words.push(word);
37
- word = "";
38
- started = false;
39
- };
40
- for (const ch of command) {
41
- if (quote) {
42
- if (ch === quote) quote = null;
43
- else word += ch;
44
- continue;
45
- }
46
- if (ch === "\"" || ch === "'") {
47
- quote = ch;
48
- started = true;
49
- } else if (/\s/.test(ch)) flush();
50
- else if (ch === ";" || ch === "|" || ch === "&") {
51
- flush();
52
- words.push(ch);
53
- } else {
54
- word += ch;
55
- started = true;
56
- }
57
- }
58
- flush();
59
- return words;
60
- }
61
- /** A command's words, with a `bash -c '…'` (or `-lc`) script split into its own words. */
62
- function commandWords(command) {
63
- const words = shellWords(command);
64
- return words.flatMap((w, i) => i > 0 && /^-[a-z]*c$/.test(words[i - 1]) && /\s/.test(w) ? commandWords(w) : [w]);
65
- }
66
- const EXCLUDING_FLAG = /^--(?:exclude|exclude-dir|ignore|ignore-dir)(?:=|$)/;
67
- /** The words of a command that name something it reads, without the ones it excludes. */
68
- function readWords(words) {
69
- const out = [];
70
- let test = false;
71
- for (let i = 0; i < words.length; i++) {
72
- const w = words[i];
73
- const prev = words[i - 1];
74
- if (w === "[" || w === "[[" || w === "test") test = true;
75
- else if (w === "]" || w === "]]" || w === ";" || w === "|" || w === "&") test = false;
76
- if (test && /^-[defsrwx]$/.test(w)) {
77
- i++;
78
- continue;
79
- }
80
- if (EXCLUDING_FLAG.test(w)) {
81
- if (!w.includes("=")) i++;
82
- continue;
83
- }
84
- if ((w === "-path" || w === "-name" || w === "-ipath" || w === "-wholename") && i + 1 < words.length) {
85
- const pruned = words[i + 2] === "-prune";
86
- if (prev === "-not" || prev === "!" || pruned) {
87
- i++;
88
- continue;
89
- }
90
- }
91
- if ((w === "-g" || w === "--glob" || w === "--iglob") && words[i + 1]?.startsWith("!")) {
92
- i++;
93
- continue;
94
- }
95
- if (/^--i?glob=!/.test(w)) continue;
96
- if (w === "-v" && i > 0) {
97
- i++;
98
- continue;
99
- }
100
- out.push(w);
101
- }
102
- return out;
103
- }
104
- /** `/home/<user>/x` and `/root/x` as `~/x`; `./x` as `x`. */
105
- const normalise = (p) => p.replace(/^\/home\/[^/]+\//, "~/").replace(/^\/root\//, "~/").replace(/^\$HOME\//, "~/").replace(/^\.\//, "");
106
- const GLOB = /[*?[{]/;
107
- /** `*`, `**` and `?` as wildcards; every other character, `[` included, literally. */
108
- function wildcards(glob) {
109
- let re = "";
110
- for (let i = 0; i < glob.length; i++) {
111
- const ch = glob[i];
112
- if (ch === "*") {
113
- if (glob[i + 1] === "*") {
114
- re += ".*";
115
- i++;
116
- } else re += "[^/]*";
117
- } else if (ch === "?") re += "[^/]";
118
- else re += ch.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
119
- }
120
- return re;
121
- }
122
- /** A glob as a regex, with `{a,b}` alternatives. Words come from transcripts, so it never throws. */
123
- function globRegex(glob) {
124
- let re = "";
125
- let from = 0;
126
- for (let i = 0; i < glob.length; i++) {
127
- if (glob[i] !== "{") continue;
128
- const end = glob.indexOf("}", i);
129
- if (end < 0) break;
130
- re += wildcards(glob.slice(from, i));
131
- re += `(?:${glob.slice(i + 1, end).split(",").map(wildcards).join("|")})`;
132
- from = end + 1;
133
- i = end;
134
- }
135
- return new RegExp(`^${re}${wildcards(glob.slice(from))}$`);
136
- }
137
- /** The declared files one word names: the file itself, one of a glob's matches, or a directory's. */
138
- function filesOf(word, files) {
139
- const w = normalise(word).replace(/\/+$/, "");
140
- if (!w) return {
141
- files: [],
142
- dir: false
143
- };
144
- const forms = (f) => f.startsWith("~/") ? [f, f.slice(2)] : [f];
145
- const matches = (f, test) => forms(f).some((form) => test(form) || !w.startsWith("~/") && test(`~/${form}`));
146
- if (GLOB.test(w)) {
147
- const re = globRegex(w);
148
- const fromWorkspace = /^[*?[{]/.test(w);
149
- return {
150
- files: files.filter((f) => fromWorkspace ? !f.startsWith("~/") && re.test(f) : matches(f, (form) => re.test(form))),
151
- dir: false
152
- };
153
- }
154
- const exact = files.filter((f) => matches(f, (form) => form === w || form.endsWith(`/${w}`)));
155
- if (exact.length) return {
156
- files: exact,
157
- dir: false
158
- };
159
- return {
160
- files: [],
161
- dir: files.filter((f) => matches(f, (form) => form.startsWith(`${w}/`))).length > 0
162
- };
163
- }
164
- /** A tool call's class, and the declared files it read (for `skill_files`). */
165
- function classifyCall(tool, target, ctx) {
166
- if (SKILL_LOAD_TOOLS.has(tool)) {
167
- if (!ctx.skills.includes(target)) return {
168
- class: "workspace",
169
- files: []
170
- };
171
- return {
172
- class: "context",
173
- files: ctx.files.filter((f) => f.endsWith(`skills/${target}/SKILL.md`))
174
- };
175
- }
176
- if (ctx.mcp.some((s) => tool.startsWith(`mcp__${s}__`) || tool.startsWith(`${s}__`))) return {
177
- class: "context",
178
- files: []
179
- };
180
- const shell = SHELL_TOOL.test(tool);
181
- const words = shell ? readWords(commandWords(target)) : [target];
182
- const files = /* @__PURE__ */ new Set();
183
- let context = false;
184
- for (const w of words) {
185
- const hit = filesOf(w, ctx.files);
186
- for (const f of hit.files) files.add(f);
187
- if (hit.files.length || hit.dir) context = true;
188
- if (ctx.skills.some((s) => normalise(w).includes(`skills/${s}/`) || normalise(w).endsWith(`skills/${s}`))) context = true;
189
- }
190
- if (context) return {
191
- class: "context",
192
- files: [...files]
193
- };
194
- if (words.some((w) => DEPENDENCY.test(normalise(w))) || shell && words.includes("--help")) return {
195
- class: "dependency",
196
- files: []
197
- };
198
- if (!shell && WEB_TOOL.test(tool)) return {
199
- class: "web",
200
- files: []
201
- };
202
- if (shell && words.some((w) => w === "curl" || w === "wget") && /https?:\/\//.test(target)) return {
203
- class: "web",
204
- files: []
205
- };
206
- if (shell) {
207
- const command = target.trim();
208
- if (ctx.verify.some((v) => v.trim().length >= 4 && command.includes(v.trim()))) return {
209
- class: "verify",
210
- files: []
211
- };
212
- if (VERIFY.test(command)) return {
213
- class: "verify",
214
- files: []
215
- };
216
- }
217
- return {
218
- class: "workspace",
219
- files: []
220
- };
221
- }
222
- /** A failing verdict this close to its threshold points at the judge, not the agent. */
223
- const JUDGE_MARGIN = .1;
224
- const DERIVED_TABLES = `
225
- CREATE TABLE effort_m (
226
- run_id TEXT, eval_key TEXT, cell_key TEXT, role TEXT, variant TEXT, harness TEXT,
227
- environment TEXT, n INTEGER, tools_median REAL, tools_q1 REAL, tools_q3 REAL,
228
- agent_s_median REAL, agent_s_q1 REAL, agent_s_q3 REAL, file_edits_median REAL,
229
- tokens_median REAL, tokens_q1 REAL, tokens_q3 REAL,
230
- control_key TEXT, control_n INTEGER, control_tools_median REAL, control_agent_s_median REAL,
231
- control_tokens_median REAL,
232
- tools_ratio REAL, agent_s_ratio REAL, agent_s_delta REAL, tokens_ratio REAL,
233
- tools_p REAL, agent_s_p REAL, tokens_p REAL,
234
- separated INTEGER, time_separated INTEGER, tokens_separated INTEGER,
235
- verdict TEXT, time_verdict TEXT, tokens_verdict TEXT, tokens_note TEXT,
236
- context_calls_median REAL, dependency_calls_median REAL, web_calls_median REAL,
237
- verify_calls_median REAL, workspace_calls_median REAL
238
- );
239
- CREATE TABLE contrasts_m (
240
- run_id TEXT, eval_key TEXT, harness TEXT, sandbox TEXT, a_key TEXT, b_key TEXT,
241
- a_environment TEXT, b_environment TEXT, a_pass INTEGER, a_counted INTEGER, b_pass INTEGER,
242
- b_counted INTEGER, points REAL, lo REAL, hi REAL, pass_verdict TEXT,
243
- a_tools_median REAL, b_tools_median REAL, tools_ratio REAL, tools_p REAL, separated INTEGER,
244
- verdict TEXT, agent_s_ratio REAL, time_separated INTEGER, time_verdict TEXT,
245
- tokens_ratio REAL, tokens_separated INTEGER, tokens_verdict TEXT,
246
- checks INTEGER, discriminating TEXT
247
- );
248
- CREATE TABLE factor_effects_m (
249
- factor TEXT, level TEXT, baseline TEXT, context TEXT, tier TEXT, run_id TEXT, eval_key TEXT,
250
- strata INTEGER, runs INTEGER, variant_pass INTEGER, variant_counted INTEGER,
251
- control_pass INTEGER, control_counted INTEGER, points REAL, lo REAL, hi REAL,
252
- within_noise INTEGER, pass_verdict TEXT, tools_ratio REAL, tools_ratios TEXT,
253
- agent_s_ratio REAL, tools_p REAL, consistent INTEGER, separated INTEGER, effort_verdict TEXT,
254
- tokens_ratio REAL, tokens_p REAL, tokens_verdict TEXT, tokens_note TEXT,
255
- note TEXT
256
- );
257
- CREATE TABLE flaky_m (
258
- eval_key TEXT, cell_key TEXT, fingerprint TEXT, runs INTEGER, flaky_runs INTEGER,
259
- first_seen TEXT, last_seen TEXT, pass INTEGER, counted INTEGER, infra_error INTEGER,
260
- pass_at_1 REAL, lo REAL, hi REAL, verdict TEXT, cause TEXT, min_attempts INTEGER
261
- );
262
- CREATE TABLE drift_m (
263
- run_id TEXT, prev_run_id TEXT, eval_key TEXT, cell_key TEXT, changed TEXT,
264
- changed_paths TEXT, noise INTEGER, real_changed TEXT
265
- );
266
- `;
267
- const FAMILY_COLUMNS = [
268
- "run_id",
269
- "family_id",
270
- "harness",
271
- "variant",
272
- "pairs",
273
- "points",
274
- "lo",
275
- "hi",
276
- "within_noise",
277
- "verdict",
278
- "tools_ratio",
279
- "tools_p",
280
- "effort_verdict",
281
- "agent_s_ratio",
282
- "tokens_ratio",
283
- "tokens_p",
284
- "tokens_verdict"
285
- ];
286
- const DERIVED = [
287
- "effort_m",
288
- "contrasts_m",
289
- "factor_effects_m",
290
- "flaky_m",
291
- "drift_m"
292
- ];
293
- const PARTS = [
294
- ["sources", "fp_sources"],
295
- ["harness", "fp_harness"],
296
- ["environment", "fp_environment"],
297
- ["image", "fp_image"],
298
- ["agent_cli", "fp_agent_cli"],
299
- ["judge", "fp_judge"],
300
- ["context", "fp_context"]
301
- ];
302
- const r4 = (x) => x === null ? null : Math.round(x * 1e4) / 1e4;
303
- const s1 = (ms) => Math.round(ms / 100) / 10;
304
- const cellId = (run, evalKey, cell) => `${run}\0${evalKey}\0${cell}`;
305
- function passCols(v, c) {
306
- const lift = liftInterval({
307
- pass: v.pass,
308
- n: v.counted
309
- }, {
310
- pass: c.pass,
311
- n: c.counted
312
- });
313
- const noise = lift.low <= 0 && lift.high >= 0;
314
- return {
315
- variant_pass: v.pass,
316
- variant_counted: v.counted,
317
- control_pass: c.pass,
318
- control_counted: c.counted,
319
- points: round3(lift.points),
320
- lo: round3(lift.low),
321
- hi: round3(lift.high),
322
- within_noise: noise ? 1 : 0,
323
- pass_verdict: Math.min(v.counted, c.counted) < 2 ? "insufficient" : noise ? "noise" : "effect"
324
- };
325
- }
326
- function tokenCols(e) {
327
- return {
328
- tokens_ratio: r4(e?.tokensRatio ?? null),
329
- tokens_p: r4(e?.tokensP ?? null),
330
- tokens_verdict: e?.tokensVerdict ?? null,
331
- tokens_note: e?.tokensNote ?? null
332
- };
333
- }
334
- function effortCols(e) {
335
- return {
336
- tools_ratio: r4(e?.toolsRatio ?? null),
337
- tools_ratios: e?.toolsRatio === null || !e ? null : `×${e.toolsRatio.toFixed(2)}`,
338
- agent_s_ratio: r4(e?.agentMsRatio ?? null),
339
- tools_p: r4(e?.toolsP ?? null),
340
- consistent: e && (e.verdict === "fewer" || e.verdict === "more") ? 1 : 0,
341
- separated: e?.separated ? 1 : 0,
342
- effort_verdict: e?.verdict ?? null,
343
- ...tokenCols(e)
344
- };
345
- }
346
- /** Across strata: an effect when ≥ 2/3 of them show it in the same direction at p < 0.05. */
347
- function pooledEffort(strata) {
348
- const known = strata.filter((s) => s !== null);
349
- const k = strata.length;
350
- const count = (v) => known.filter((s) => s.verdict === v).length;
351
- const needed = Math.ceil(2 * k / 3);
352
- let verdict = "noise";
353
- if (count("fewer") >= needed && count("more") === 0) verdict = "fewer";
354
- else if (count("more") >= needed && count("fewer") === 0) verdict = "more";
355
- else if (known.length && count("insufficient") === known.length) verdict = "insufficient";
356
- const sorted = [...known.map((s) => s.toolsRatio).filter((r) => r !== null)].sort((a, b) => a - b);
357
- const mid = sorted.length ? sorted.length % 2 ? sorted[(sorted.length - 1) / 2] : (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]) / 2 : null;
358
- return {
359
- tools_ratio: r4(mid),
360
- tools_ratios: known.map((s) => s.toolsRatio === null ? "×–" : `×${s.toolsRatio.toFixed(2)}`).join(" / "),
361
- agent_s_ratio: null,
362
- tools_p: r4(known.length ? Math.max(...known.map((s) => s.toolsP ?? 1)) : null),
363
- consistent: verdict === "fewer" || verdict === "more" ? count(verdict) : 0,
364
- separated: known.filter((s) => s.separated).length,
365
- effort_verdict: verdict,
366
- ...pooledTokens(known)
367
- };
368
- }
369
- /**
370
- * Token verdict over the strata that reported usage. A stratum without token samples is
371
- * dropped from this pool only — not `insufficient`, and it does not blank tokens on the rest.
372
- */
373
- function pooledTokens(known) {
374
- const withTokens = known.filter((s) => s.tokensVerdict !== null);
375
- if (withTokens.length === 0) return {
376
- tokens_ratio: null,
377
- tokens_p: null,
378
- tokens_verdict: null,
379
- tokens_note: null
380
- };
381
- const k = withTokens.length;
382
- const count = (v) => withTokens.filter((s) => s.tokensVerdict === v).length;
383
- const needed = Math.ceil(2 * k / 3);
384
- let verdict = "noise";
385
- if (count("fewer") >= needed && count("more") === 0) verdict = "fewer";
386
- else if (count("more") >= needed && count("fewer") === 0) verdict = "more";
387
- else if (count("insufficient") === k) verdict = "insufficient";
388
- const sorted = [...withTokens.map((s) => s.tokensRatio).filter((r) => r !== null)].sort((a, b) => a - b);
389
- const mid = sorted.length ? sorted.length % 2 ? sorted[(sorted.length - 1) / 2] : (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]) / 2 : null;
390
- return {
391
- tokens_ratio: r4(mid),
392
- tokens_p: r4(Math.max(...withTokens.map((s) => s.tokensP ?? 1))),
393
- tokens_verdict: verdict,
394
- tokens_note: known.some((s) => s.tokensNote) || withTokens.length < known.length ? "some cells did not report usage" : null
395
- };
396
- }
397
- const merge = (sides) => ({
398
- pass: sides.reduce((n, s) => n + s.pass, 0),
399
- counted: sides.reduce((n, s) => n + s.counted, 0),
400
- samples: sides.flatMap((s) => s.samples)
401
- });
402
- function insertAll(db, table, rows) {
403
- if (rows.length === 0) return;
404
- const cols = rowsOf(db.prepare(`PRAGMA table_info("${table}")`)).map((c) => c.name);
405
- const statement = db.prepare(`INSERT INTO ${table} (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`);
406
- for (const row of rows) statement.run(...cols.map((c) => row[c] ?? null));
407
- }
408
- /** Recomputes every derived table from the projected rows; run inside the sync's transaction. */
409
- function deriveFacts(db) {
410
- for (const table of DERIVED) db.exec(`DELETE FROM ${table}`);
411
- const cells = rowsOf(db.prepare("SELECT c.*, r.started_at FROM cells c JOIN runs r USING (run_id) ORDER BY c.run_id, c.rowid"));
412
- const attempts = rowsOf(db.prepare(`SELECT run_id, eval_key, cell_key, key, outcome, error_kind, tools, agent_ms, file_edits,
413
- input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens
414
- FROM attempts ORDER BY run_id, rowid`));
415
- const nearThreshold = new Set(rowsOf(db.prepare(`SELECT DISTINCT run_id, attempt_key FROM judgements WHERE verdict = 'fail' AND ABS(score - threshold) <= ${JUDGE_MARGIN}`)).map((r) => `${r.run_id}\0${r.attempt_key}`));
416
- const diffs = new Map(rowsOf(db.prepare(`SELECT run_id, attempt_key, sha256 FROM artifacts WHERE kind = 'diff' AND bytes > 0`)).flatMap((r) => r.sha256 ? [[`${r.run_id}\0${r.attempt_key}`, r.sha256]] : []));
417
- const samples = /* @__PURE__ */ new Map();
418
- const byCell = /* @__PURE__ */ new Map();
419
- for (const a of attempts) {
420
- const id = cellId(a.run_id, a.eval_key, a.cell_key);
421
- byCell.set(id, [...byCell.get(id) ?? [], a]);
422
- if (a.outcome !== "pass" && a.outcome !== "fail") continue;
423
- const tokens = a.input_tokens === null ? void 0 : a.input_tokens + (a.output_tokens ?? 0) + (a.cache_read_tokens ?? 0) + (a.cache_write_tokens ?? 0) + (a.reasoning_tokens ?? 0);
424
- samples.set(id, [...samples.get(id) ?? [], {
425
- tools: a.tools,
426
- agentMs: a.agent_ms,
427
- fileEdits: a.file_edits,
428
- ...tokens !== void 0 ? { tokens } : {}
429
- }]);
430
- }
431
- const side = (c) => ({
432
- pass: c.pass,
433
- counted: c.pass + c.fail,
434
- samples: samples.get(cellId(c.run_id, c.eval_key, c.key)) ?? []
435
- });
436
- const find = (run, evalKey, key) => cells.find((c) => c.run_id === run && c.eval_key === evalKey && c.key === key);
437
- const classCounts = /* @__PURE__ */ new Map();
438
- for (const r of rowsOf(db.prepare("SELECT run_id, attempt_key, class, COUNT(*) AS n FROM tool_calls GROUP BY run_id, attempt_key, class"))) {
439
- const id = `${r.run_id}\0${r.attempt_key}`;
440
- const m = classCounts.get(id) ?? /* @__PURE__ */ new Map();
441
- m.set(r.class, r.n);
442
- classCounts.set(id, m);
443
- }
444
- const classMedians = (c) => {
445
- const known = (byCell.get(cellId(c.run_id, c.eval_key, c.key)) ?? []).filter((a) => a.outcome === "pass" || a.outcome === "fail").flatMap((a) => {
446
- const m = classCounts.get(`${a.run_id}\0${a.key}`);
447
- return m ? [m] : [];
448
- });
449
- return Object.fromEntries(TOOL_CLASSES.map((k) => [`${k}_calls_median`, known.length ? median(known.map((m) => m.get(k) ?? 0)) ?? null : null]));
450
- };
451
- insertAll(db, "effort_m", cells.map((c) => {
452
- const own = effortOf(side(c).samples);
453
- const control = c.lift_vs ? find(c.run_id, c.eval_key, c.lift_vs) : void 0;
454
- const vs = control ? contrastEffort(side(c).samples, side(control).samples) : null;
455
- return {
456
- run_id: c.run_id,
457
- eval_key: c.eval_key,
458
- cell_key: c.key,
459
- role: c.role,
460
- variant: c.variant,
461
- harness: c.harness,
462
- environment: c.environment,
463
- n: own?.n ?? 0,
464
- tools_median: own?.tools.median ?? null,
465
- tools_q1: own?.tools.q1 ?? null,
466
- tools_q3: own?.tools.q3 ?? null,
467
- agent_s_median: own ? s1(own.agentMs.median) : null,
468
- agent_s_q1: own ? s1(own.agentMs.q1) : null,
469
- agent_s_q3: own ? s1(own.agentMs.q3) : null,
470
- file_edits_median: own?.fileEdits ?? null,
471
- tokens_median: own?.tokens?.median ?? null,
472
- tokens_q1: own?.tokens?.q1 ?? null,
473
- tokens_q3: own?.tokens?.q3 ?? null,
474
- control_key: control?.key ?? null,
475
- control_n: vs?.control.n ?? null,
476
- control_tools_median: vs?.control.tools.median ?? null,
477
- control_agent_s_median: vs ? s1(vs.control.agentMs.median) : null,
478
- control_tokens_median: vs?.control.tokens?.median ?? null,
479
- tools_ratio: r4(vs?.toolsRatio ?? null),
480
- agent_s_ratio: r4(vs?.agentMsRatio ?? null),
481
- agent_s_delta: vs ? s1(vs.agentMsDelta) : null,
482
- tokens_ratio: r4(vs?.tokensRatio ?? null),
483
- tools_p: r4(vs?.toolsP ?? null),
484
- agent_s_p: r4(vs?.agentMsP ?? null),
485
- tokens_p: r4(vs?.tokensP ?? null),
486
- separated: vs ? vs.separated ? 1 : 0 : null,
487
- time_separated: vs ? vs.timeSeparated ? 1 : 0 : null,
488
- tokens_separated: vs?.tokensSeparated == null ? null : vs.tokensSeparated ? 1 : 0,
489
- verdict: vs?.verdict ?? null,
490
- time_verdict: vs?.timeVerdict ?? null,
491
- tokens_verdict: vs?.tokensVerdict ?? null,
492
- tokens_note: vs?.tokensNote ?? null,
493
- ...classMedians(c)
494
- };
495
- }));
496
- const checkRows = rowsOf(db.prepare("SELECT run_id, eval_key, cell_key, label, pass, fail, error, not_run FROM cell_checks"));
497
- const checksOf = /* @__PURE__ */ new Map();
498
- for (const k of checkRows) {
499
- const id = cellId(k.run_id, k.eval_key, k.cell_key);
500
- const m = checksOf.get(id) ?? /* @__PURE__ */ new Map();
501
- m.set(k.label, {
502
- pass: k.pass,
503
- fail: k.fail,
504
- error: k.error,
505
- notRun: k.not_run
506
- });
507
- checksOf.set(id, m);
508
- }
509
- const evalCells = /* @__PURE__ */ new Map();
510
- for (const c of cells) {
511
- const id = `${c.run_id}\0${c.eval_key}`;
512
- evalCells.set(id, [...evalCells.get(id) ?? [], c]);
513
- }
514
- const contrastRows = [];
515
- for (const group of evalCells.values()) for (const [a, b] of contrastPairs(group)) {
516
- const x = side(a);
517
- const y = side(b);
518
- const e = contrastEffort(x.samples, y.samples);
519
- const ka = checksOf.get(cellId(a.run_id, a.eval_key, a.key)) ?? /* @__PURE__ */ new Map();
520
- const kb = checksOf.get(cellId(b.run_id, b.eval_key, b.key)) ?? /* @__PURE__ */ new Map();
521
- const verdicts = [...ka].flatMap(([label, t]) => {
522
- const other = kb.get(label);
523
- const verdict = other && checkVerdict(t, other);
524
- return verdict ? [{
525
- label,
526
- verdict
527
- }] : [];
528
- });
529
- const pass = passCols(x, y);
530
- contrastRows.push({
531
- run_id: a.run_id,
532
- eval_key: a.eval_key,
533
- harness: a.harness,
534
- sandbox: a.sandbox,
535
- a_key: a.key,
536
- b_key: b.key,
537
- a_environment: a.environment,
538
- b_environment: b.environment,
539
- a_pass: x.pass,
540
- a_counted: x.counted,
541
- b_pass: y.pass,
542
- b_counted: y.counted,
543
- points: pass.points ?? null,
544
- lo: pass.lo ?? null,
545
- hi: pass.hi ?? null,
546
- pass_verdict: pass.pass_verdict ?? null,
547
- a_tools_median: e?.variant.tools.median ?? null,
548
- b_tools_median: e?.control.tools.median ?? null,
549
- tools_ratio: r4(e?.toolsRatio ?? null),
550
- tools_p: r4(e?.toolsP ?? null),
551
- separated: e ? e.separated ? 1 : 0 : null,
552
- verdict: e?.verdict ?? null,
553
- agent_s_ratio: r4(e?.agentMsRatio ?? null),
554
- time_separated: e ? e.timeSeparated ? 1 : 0 : null,
555
- time_verdict: e?.timeVerdict ?? null,
556
- tokens_ratio: r4(e?.tokensRatio ?? null),
557
- tokens_separated: e?.tokensSeparated == null ? null : e.tokensSeparated ? 1 : 0,
558
- tokens_verdict: e?.tokensVerdict ?? null,
559
- checks: verdicts.length,
560
- discriminating: verdicts.filter((v) => v.verdict === "discriminates").map((v) => v.label).join(" · ") || null
561
- });
562
- }
563
- insertAll(db, "contrasts_m", contrastRows);
564
- const pairs = [];
565
- const groups = /* @__PURE__ */ new Map();
566
- for (const c of cells) {
567
- const id = `${c.run_id}\0${c.eval_key}`;
568
- groups.set(id, [...groups.get(id) ?? [], c]);
569
- }
570
- for (const group of groups.values()) {
571
- for (const v of group) {
572
- const control = v.lift_vs ? group.find((c) => c.key === v.lift_vs) : void 0;
573
- if (!control) continue;
574
- pairs.push({
575
- factor: "environment",
576
- level: v.variant ?? v.environment,
577
- baseline: "control",
578
- context: v.harness,
579
- run: v.run_id,
580
- evalKey: v.eval_key,
581
- pool: `${v.comparable}\0${control.comparable}`,
582
- v: side(v),
583
- c: side(control)
584
- });
585
- }
586
- const bySetup = /* @__PURE__ */ new Map();
587
- for (const c of group) {
588
- const id = `${c.variant ?? c.role}\0${c.sandbox}`;
589
- bySetup.set(id, [...bySetup.get(id) ?? [], c]);
590
- }
591
- for (const same of bySetup.values()) {
592
- const base = same[0];
593
- for (const other of same.slice(1)) {
594
- if (other.harness === base.harness) continue;
595
- pairs.push({
596
- factor: other.harness_kind === base.harness_kind ? "model" : "harness",
597
- level: other.harness,
598
- baseline: base.harness,
599
- context: other.variant ?? "control",
600
- run: other.run_id,
601
- evalKey: other.eval_key,
602
- pool: `${other.comparable}\0${base.comparable}`,
603
- v: side(other),
604
- c: side(base)
605
- });
606
- }
607
- }
608
- }
609
- const factorRows = [];
610
- const head = (p) => ({
611
- factor: p.factor,
612
- level: p.level,
613
- baseline: p.baseline,
614
- context: p.context
615
- });
616
- for (const p of pairs) factorRows.push({
617
- ...head(p),
618
- tier: "controlled",
619
- run_id: p.run,
620
- eval_key: p.evalKey,
621
- strata: 1,
622
- runs: 1,
623
- ...passCols(p.v, p.c),
624
- ...effortCols(contrastEffort(p.v.samples, p.c.samples)),
625
- note: null
626
- });
627
- const pooled = /* @__PURE__ */ new Map();
628
- for (const p of pairs) {
629
- const id = [
630
- p.factor,
631
- p.level,
632
- p.baseline,
633
- p.context,
634
- p.evalKey,
635
- p.pool
636
- ].join("\0");
637
- pooled.set(id, [...pooled.get(id) ?? [], p]);
638
- }
639
- const strata = /* @__PURE__ */ new Map();
640
- for (const group of pooled.values()) {
641
- const first = group[0];
642
- const v = merge(group.map((p) => p.v));
643
- const c = merge(group.map((p) => p.c));
644
- if (group.length > 1) factorRows.push({
645
- ...head(first),
646
- tier: "pooled",
647
- run_id: null,
648
- eval_key: first.evalKey,
649
- strata: 1,
650
- runs: group.length,
651
- ...passCols(v, c),
652
- ...effortCols(contrastEffort(v.samples, c.samples)),
653
- note: null
654
- });
655
- const id = [
656
- first.factor,
657
- first.level,
658
- first.baseline,
659
- first.context
660
- ].join("\0");
661
- strata.set(id, [...strata.get(id) ?? [], {
662
- evalKey: first.evalKey,
663
- pool: first.pool,
664
- v,
665
- c
666
- }]);
667
- }
668
- for (const [id, list] of strata) {
669
- const poolsByEval = /* @__PURE__ */ new Map();
670
- for (const s of list) poolsByEval.set(s.evalKey, /* @__PURE__ */ new Set([...poolsByEval.get(s.evalKey) ?? [], s.pool]));
671
- const comparable = list.filter((s) => (poolsByEval.get(s.evalKey)?.size ?? 0) === 1);
672
- if (new Set(comparable.map((s) => s.evalKey)).size < 2) continue;
673
- const [factor, level, baseline, context] = id.split("\0");
674
- const v = merge(comparable.map((s) => s.v));
675
- const c = merge(comparable.map((s) => s.c));
676
- factorRows.push({
677
- ...head({
678
- factor,
679
- level,
680
- baseline,
681
- context
682
- }),
683
- tier: "pooled",
684
- run_id: null,
685
- eval_key: null,
686
- strata: comparable.length,
687
- runs: null,
688
- ...passCols(v, c),
689
- ...pooledEffort(comparable.map((s) => contrastEffort(s.v.samples, s.c.samples))),
690
- note: "pass@1: summed counts; effort: an effect in at least 2/3 of strata"
691
- });
692
- }
693
- const familyLifts = rowsOf(db.prepare(`SELECT ${FAMILY_COLUMNS.join(", ")} FROM family_lift_m`));
694
- for (const f of familyLifts) factorRows.push({
695
- factor: "environment",
696
- level: f.variant,
697
- baseline: "control",
698
- context: f.harness,
699
- tier: "paired",
700
- run_id: f.run_id,
701
- eval_key: f.family_id,
702
- strata: f.pairs,
703
- runs: 1,
704
- points: f.points,
705
- lo: f.lo,
706
- hi: f.hi,
707
- within_noise: f.within_noise,
708
- pass_verdict: f.verdict === "insufficient" ? "insufficient" : f.within_noise ? "noise" : "effect",
709
- tools_ratio: f.tools_ratio,
710
- agent_s_ratio: f.agent_s_ratio,
711
- tools_p: f.tools_p,
712
- effort_verdict: f.effort_verdict,
713
- tokens_ratio: f.tokens_ratio,
714
- tokens_p: f.tokens_p,
715
- tokens_verdict: f.tokens_verdict,
716
- note: "paired by case over a pooled evaluate.each: strata is the paired cases"
717
- });
718
- const driftRows = [];
719
- const series = /* @__PURE__ */ new Map();
720
- for (const c of cells) {
721
- const id = `${c.eval_key}\0${c.key}`;
722
- series.set(id, [...series.get(id) ?? [], c]);
723
- }
724
- const contextOf = db.prepare("SELECT path, sha256 FROM attempt_context WHERE run_id = ? AND attempt_key = ?");
725
- const cellContext = (c) => {
726
- const files = /* @__PURE__ */ new Map();
727
- for (const a of byCell.get(cellId(c.run_id, c.eval_key, c.key)) ?? []) for (const f of rowsOf(contextOf, c.run_id, a.key)) files.set(f.path, f.sha256);
728
- return files;
729
- };
730
- for (const list of series.values()) {
731
- list.sort((a, b) => a.run_id < b.run_id ? -1 : 1);
732
- for (let i = 1; i < list.length; i++) {
733
- const prev = list[i - 1];
734
- const cur = list[i];
735
- if (prev.fingerprint === cur.fingerprint) continue;
736
- const changed = PARTS.filter(([, col]) => prev[col] !== cur[col]).map(([name]) => name);
737
- let paths = [];
738
- if (changed.includes("context")) {
739
- const a = cellContext(prev);
740
- const b = cellContext(cur);
741
- paths = [.../* @__PURE__ */ new Set([...a.keys(), ...b.keys()])].filter((p) => a.get(p) !== b.get(p)).sort();
742
- }
743
- const contextNoise = paths.length > 0 && paths.every(isRuntimeState);
744
- const real = changed.filter((part) => part !== "context" || !contextNoise);
745
- driftRows.push({
746
- run_id: cur.run_id,
747
- prev_run_id: prev.run_id,
748
- eval_key: cur.eval_key,
749
- cell_key: cur.key,
750
- changed: changed.join(" "),
751
- changed_paths: paths.length ? paths.join(", ") : null,
752
- noise: real.length === 0 ? 1 : 0,
753
- real_changed: real.join(" ") || null
754
- });
755
- if (real.length === 0) continue;
756
- const judgeOnly = real.length === 1 && real[0] === "judge";
757
- const sourcesChanged = real.includes("sources");
758
- const cols = passCols(side(cur), side(prev));
759
- if (sourcesChanged) cols.pass_verdict = "not_comparable";
760
- factorRows.push({
761
- factor: real.length === 1 ? real[0] : "confounded",
762
- level: cur.run_id,
763
- baseline: prev.run_id,
764
- context: `${cur.eval_key} # ${cur.key}`,
765
- tier: "observed",
766
- run_id: cur.run_id,
767
- eval_key: cur.eval_key,
768
- strata: 1,
769
- runs: 2,
770
- ...cols,
771
- ...effortCols(contrastEffort(side(cur).samples, side(prev).samples)),
772
- note: sourcesChanged ? "the eval changed: not comparable, not an effect of the thing under test" : judgeOnly ? "the judge changed: a measurement change, not an effect of the thing under test" : real.length > 1 ? `confounded: ${real.join(", ")} changed together` : null
773
- });
774
- }
775
- }
776
- insertAll(db, "drift_m", driftRows);
777
- const levels = (pick) => [...new Set(cells.map(pick))];
778
- const single = [
779
- ["harness", levels((c) => c.harness_kind)],
780
- ["model", levels((c) => `${c.harness_kind}:${c.model ?? "-"}`)],
781
- ["sandbox", levels((c) => c.sandbox)]
782
- ];
783
- if (cells.length && !cells.some((c) => c.role === "variant")) single.push(["environment", levels((c) => c.environment)]);
784
- for (const [factor, values] of single) {
785
- if (values.length !== 1) continue;
786
- factorRows.push({
787
- factor,
788
- level: values[0],
789
- baseline: null,
790
- context: null,
791
- tier: "not_measured",
792
- note: "one level in every run, so its effect is not measured"
793
- });
794
- }
795
- insertAll(db, "factor_effects_m", factorRows);
796
- const flaky = /* @__PURE__ */ new Map();
797
- for (const c of cells) {
798
- const id = `${c.eval_key}\0${c.key}\0${c.comparable}`;
799
- flaky.set(id, [...flaky.get(id) ?? [], c]);
800
- }
801
- const flakyRows = [];
802
- for (const list of flaky.values()) {
803
- const pass = list.reduce((n, c) => n + c.pass, 0);
804
- const fail = list.reduce((n, c) => n + c.fail, 0);
805
- if (!(pass > 0 && fail > 0)) continue;
806
- const first = list[0];
807
- const infra = list.reduce((n, c) => n + c.infra_error, 0);
808
- const failing = list.flatMap((c) => (byCell.get(cellId(c.run_id, c.eval_key, c.key)) ?? []).filter((a) => a.outcome === "fail"));
809
- const w = wilson(pass, pass + fail);
810
- const passedDiffs = new Set(list.flatMap((c) => (byCell.get(cellId(c.run_id, c.eval_key, c.key)) ?? []).flatMap((a) => {
811
- const sha = a.outcome === "pass" && diffs.get(`${a.run_id}\0${a.key}`);
812
- return sha ? [sha] : [];
813
- })));
814
- const sameWork = failing.length > 0 && failing.every((a) => a.error_kind === "assertion" && passedDiffs.has(diffs.get(`${a.run_id}\0${a.key}`) ?? ""));
815
- const cause = infra > 0 || failing.some((a) => a.error_kind === "infra") ? "infra" : failing.some((a) => nearThreshold.has(`${a.run_id}\0${a.key}`)) ? "judge" : sameWork ? "check" : "agent";
816
- const starts = list.map((c) => c.started_at).sort();
817
- flakyRows.push({
818
- eval_key: first.eval_key,
819
- cell_key: first.key,
820
- fingerprint: first.comparable,
821
- runs: list.length,
822
- flaky_runs: list.filter((c) => c.pass > 0 && c.fail > 0).length,
823
- first_seen: starts[0] ?? null,
824
- last_seen: starts.at(-1) ?? null,
825
- pass,
826
- counted: pass + fail,
827
- infra_error: infra,
828
- pass_at_1: round3(pass / (pass + fail)),
829
- lo: round3(w.low),
830
- hi: round3(w.high),
831
- verdict: pass + fail >= 6 ? "flaky" : "insufficient",
832
- cause,
833
- min_attempts: 6
834
- });
835
- }
836
- flakyRows.sort((a, b) => String(a.first_seen).localeCompare(String(b.first_seen)));
837
- insertAll(db, "flaky_m", flakyRows);
838
- }
839
- //#endregion
840
- //#region src/history/db.ts
841
- /** Bump when the tables or views change: a database from another version is rebuilt. */
842
- const DB_VERSION = 11;
843
- /** This build of reeval, like the header cache: another reeval re-derives every row. */
844
- const BUILD = BUILD_ID;
845
- const historyDbPath = (root) => join(root, ".reeval", "history.db");
846
- /** `node:sqlite` is in Node from 22.13 (package `engines`); before 24 it warns on load. */
847
- const SQLITE_NODE = "22.13.0";
848
- function nodeAtLeast(version, min = SQLITE_NODE) {
849
- const [a = 0, b = 0, c = 0] = version.replace(/^v/, "").split(".").map(Number);
850
- const [x = 0, y = 0, z = 0] = min.split(".").map(Number);
851
- return a !== x ? a > x : b !== y ? b > y : c >= z;
852
- }
853
- function loadSqlite() {
854
- if (!nodeAtLeast(process.versions.node)) throw new UsageError(`reeval query needs Node ${SQLITE_NODE} or later for node:sqlite (this is Node ${process.versions.node}).`);
855
- const emit = process.emitWarning;
856
- process.emitWarning = ((warning, ...rest) => {
857
- const text = typeof warning === "string" ? warning : warning.message;
858
- if (/SQLite/.test(text)) return;
859
- return emit.call(process, warning, ...rest);
860
- });
861
- try {
862
- return ownRequire("node:sqlite");
863
- } finally {
864
- process.emitWarning = emit;
865
- }
866
- }
867
- const TABLES = `
868
- CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
869
- -- The record file each run was read from, and its size and mtime when it was.
870
- CREATE TABLE files (run_id TEXT PRIMARY KEY, path TEXT NOT NULL, size INTEGER NOT NULL, mtime_ms REAL NOT NULL);
871
- CREATE TABLE runs (
872
- run_id TEXT PRIMARY KEY, recorded INTEGER NOT NULL, status TEXT, started_at TEXT, ended_at TEXT,
873
- duration_ms INTEGER, author TEXT, git_sha TEXT, branch TEXT, dirty INTEGER, runner TEXT,
874
- ci_provider TEXT, ci_url TEXT, pr_number INTEGER, pr_base TEXT, reeval TEXT, judge_model TEXT,
875
- judge_version TEXT, profile TEXT, evals INTEGER, attempts INTEGER, pass INTEGER, fail INTEGER,
876
- not_gated INTEGER, infra_error INTEGER, skipped INTEGER
877
- );
878
- CREATE TABLE evals (
879
- run_id TEXT NOT NULL, key TEXT NOT NULL, id TEXT, slug TEXT, file TEXT, groups TEXT, name TEXT,
880
- tags TEXT, fingerprint TEXT, outcome TEXT, pass_at_1 REAL, sources TEXT, family_id TEXT,
881
- case_index INTEGER,
882
- PRIMARY KEY (run_id, key)
883
- );
884
- -- Each pooled evaluate.each per run: its cases, and each cell's pass@1 over them (the mean of the
885
- -- cases' pass@1).
886
- CREATE TABLE families (
887
- run_id TEXT NOT NULL, family_id TEXT NOT NULL, file TEXT, groups TEXT, name TEXT,
888
- cases INTEGER, ran INTEGER, cell_key TEXT NOT NULL, cell_cases INTEGER, pass INTEGER,
889
- counted INTEGER, pass_at_1 REAL, lo REAL, hi REAL, method TEXT, outcome TEXT,
890
- PRIMARY KEY (run_id, family_id, cell_key)
891
- );
892
- -- Every compare-question vote: a pair of sides (arms within a harness, harnesses across an arm)
893
- -- on one run, shown in one order (swapped = 1: b first), and the side picked or 'tie'.
894
- CREATE TABLE votes (
895
- run_id TEXT NOT NULL, eval_key TEXT NOT NULL, question TEXT NOT NULL, cut TEXT, held TEXT,
896
- pair_run INTEGER, a TEXT, b TEXT, winner TEXT, swapped INTEGER, reason TEXT
897
- );
898
- -- Each pair of sides' head-to-head per question: a pair decides when both orders agree (or one
899
- -- ties); a split is a pair whose orders picked opposite sides, split_first those that picked
900
- -- the side shown first both times.
901
- CREATE TABLE question_pairs_m (
902
- run_id TEXT NOT NULL, eval_key TEXT NOT NULL, question TEXT NOT NULL, cut TEXT, held TEXT,
903
- a TEXT, b TEXT, pairs INTEGER, a_wins INTEGER, b_wins INTEGER, ties INTEGER, splits INTEGER,
904
- splits_first INTEGER
905
- );
906
- CREATE TABLE family_lift_m (
907
- run_id TEXT NOT NULL, family_id TEXT NOT NULL, name TEXT, harness TEXT, variant TEXT,
908
- cell_key TEXT, control TEXT,
909
- pairs INTEGER, variant_pass_at_1 REAL, control_pass_at_1 REAL, discordant_vc INTEGER,
910
- discordant_cv INTEGER, points REAL, lo REAL, hi REAL, method TEXT, p REAL,
911
- within_noise INTEGER, verdict TEXT, min_lift REAL, outcome TEXT, tools_ratio REAL,
912
- tools_p REAL, effort_verdict TEXT, agent_s_ratio REAL, agent_s_p REAL, time_verdict TEXT,
913
- tokens_ratio REAL, tokens_p REAL, tokens_verdict TEXT
914
- );
915
- CREATE TABLE cells (
916
- run_id TEXT NOT NULL, eval_key TEXT NOT NULL, key TEXT NOT NULL, role TEXT, variant TEXT,
917
- harness TEXT, harness_kind TEXT, model TEXT, reasoning_effort TEXT, environment TEXT,
918
- environment_digest TEXT, sandbox TEXT, gate TEXT, gate_pass_rate REAL, fingerprint TEXT,
919
- fp_sources TEXT, fp_harness TEXT, fp_environment TEXT, fp_image TEXT, fp_agent_cli TEXT,
920
- fp_judge TEXT, fp_context TEXT, runs INTEGER, pass INTEGER, fail INTEGER, infra_error INTEGER,
921
- skipped INTEGER, pass_at_1 REAL, outcome TEXT, lift_vs TEXT, lift_points REAL, lift_lo REAL,
922
- lift_hi REAL, within_noise INTEGER, min_lift REAL, comparable TEXT,
923
- PRIMARY KEY (run_id, eval_key, key)
924
- );
925
- CREATE TABLE attempts (
926
- run_id TEXT NOT NULL, eval_key TEXT NOT NULL, cell_key TEXT NOT NULL, key TEXT NOT NULL,
927
- id TEXT, n INTEGER, seq INTEGER, started_at TEXT, outcome TEXT, error_kind TEXT,
928
- error_message TEXT, duration_ms INTEGER, image_id TEXT, turns INTEGER, tools INTEGER,
929
- file_edits INTEGER, setup_ms INTEGER, agent_ms INTEGER, checks_ms INTEGER, judge_ms INTEGER,
930
- last_text TEXT, input_tokens INTEGER, output_tokens INTEGER, cache_read_tokens INTEGER,
931
- cache_write_tokens INTEGER, reasoning_tokens INTEGER, served_model TEXT, oom_kills INTEGER,
932
- reasoning_effort TEXT, served_reasoning_effort TEXT,
933
- PRIMARY KEY (run_id, key)
934
- );
935
- CREATE TABLE judgements (
936
- run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, key TEXT NOT NULL, question TEXT,
937
- verdict TEXT, score REAL, threshold REAL, model TEXT, resolved_model TEXT, input_path TEXT,
938
- PRIMARY KEY (run_id, attempt_key, key)
939
- );
940
- CREATE TABLE environments (
941
- run_id TEXT NOT NULL, name TEXT NOT NULL, digest TEXT NOT NULL, parent TEXT, overrides TEXT,
942
- PRIMARY KEY (run_id, digest)
943
- );
944
- CREATE TABLE env_skills (
945
- run_id TEXT NOT NULL, digest TEXT NOT NULL, name TEXT NOT NULL, source TEXT, commit_sha TEXT,
946
- sha256 TEXT, placement TEXT
947
- );
948
- CREATE TABLE artifacts (
949
- run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, kind TEXT NOT NULL, path TEXT NOT NULL,
950
- sha256 TEXT, bytes INTEGER
951
- );
952
- -- What the agent could see, by content hash; runtime = 1 for an agent CLI's own session state,
953
- -- declared = 1 for a file the environment declared (its skills' files, rules and config).
954
- CREATE TABLE attempt_context (
955
- run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, path TEXT NOT NULL, sha256 TEXT, runtime INTEGER,
956
- declared INTEGER
957
- );
958
- -- Every tool call in each attempt's transcript: its line in transcript.jsonl (from 1), target and
959
- -- class (context, dependency, web, verify or workspace: a heuristic, see tool-class.ts).
960
- CREATE TABLE tool_calls (
961
- run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, line INTEGER NOT NULL, turn INTEGER,
962
- tool TEXT, target TEXT, ok INTEGER, duration_ms INTEGER, class TEXT
963
- );
964
- -- Each placed file a tool call named (directly or through a glob), by the call's line.
965
- CREATE TABLE context_reads (
966
- run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, line INTEGER NOT NULL, path TEXT NOT NULL
967
- );
968
- -- Every expect / expect.soft matcher call each attempt made, in call order (seq from 1).
969
- CREATE TABLE assertions (
970
- run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, eval_key TEXT NOT NULL, cell_key TEXT NOT NULL,
971
- seq INTEGER NOT NULL, label TEXT NOT NULL, labelled INTEGER, matcher TEXT, status TEXT,
972
- soft INTEGER, message TEXT, duration_ms INTEGER
973
- );
974
- -- Each check (assertion label) per cell over its counted attempts; on a variant, its control's too.
975
- CREATE TABLE cell_checks (
976
- run_id TEXT NOT NULL, eval_key TEXT NOT NULL, cell_key TEXT NOT NULL, label TEXT NOT NULL,
977
- pass INTEGER, fail INTEGER, error INTEGER, not_run INTEGER, control_pass INTEGER,
978
- control_fail INTEGER, control_error INTEGER, control_not_run INTEGER, verdict TEXT
979
- );
980
- ${DERIVED_TABLES}
981
- CREATE INDEX assertions_by_attempt ON assertions (run_id, attempt_key);
982
- CREATE INDEX cells_by_eval ON cells (eval_key, key);
983
- CREATE INDEX tool_calls_by_attempt ON tool_calls (run_id, attempt_key);
984
- CREATE INDEX context_by_attempt ON attempt_context (run_id, attempt_key);
985
- CREATE INDEX context_reads_by_attempt ON context_reads (run_id, attempt_key);
986
- CREATE INDEX attempts_by_cell ON attempts (run_id, eval_key, cell_key);
987
- CREATE INDEX env_skills_by_digest ON env_skills (run_id, digest);
988
- `;
989
- /** Canned questions about the history, as views to filter. */
990
- const VIEWS = `
991
- -- Variant against control, per harness, per run.
992
- CREATE VIEW lift AS
993
- SELECT c.run_id, r.started_at, r.branch, c.eval_key, c.harness, c.model, c.variant,
994
- c.lift_vs AS control, c.pass AS variant_pass, c.pass + c.fail AS variant_counted,
995
- k.pass AS control_pass, k.pass + k.fail AS control_counted, c.pass_at_1 AS variant_pass_at_1,
996
- k.pass_at_1 AS control_pass_at_1, c.lift_points AS points,
997
- c.lift_lo AS lo, c.lift_hi AS hi, c.within_noise, c.min_lift, c.outcome, c.fingerprint
998
- FROM cells c JOIN runs r USING (run_id)
999
- JOIN cells k ON k.run_id = c.run_id AND k.eval_key = c.eval_key AND k.key = c.lift_vs
1000
- WHERE c.role = 'variant';
1001
-
1002
- -- Each variant against its control over a pooled evaluate.each's cases, paired by case, per run.
1003
- CREATE VIEW family_lift AS
1004
- SELECT f.*, r.started_at, r.branch
1005
- FROM family_lift_m f JOIN runs r USING (run_id);
1006
-
1007
- -- Cells with mixed outcomes, pooled over runs whose fingerprints match once agent runtime state is
1008
- -- left out; 'flaky' needs FLAKY_MIN_ATTEMPTS counted attempts, else 'insufficient'.
1009
- CREATE VIEW flaky_cells AS SELECT * FROM flaky_m;
1010
-
1011
- -- Each run where a cell's fingerprint differs from its previous run: which parts, which context
1012
- -- paths, and noise = 1 when only agent runtime state changed (not a real change).
1013
- CREATE VIEW drift AS SELECT * FROM drift_m;
1014
-
1015
- -- Tool calls, agent time and tokens per cell, and a variant's against its control in the same run.
1016
- CREATE VIEW effort AS SELECT * FROM effort_m;
1017
-
1018
- -- Each pair of arms (cells a and b) on one harness and sandbox, when an eval has two variants or
1019
- -- more: pass@1 lift of a over b, effort with its verdicts and separation, and the checks that
1020
- -- tell them apart. A variant against its control is here too; gating stays variant vs control.
1021
- CREATE VIEW contrasts AS SELECT * FROM contrasts_m;
1022
-
1023
- -- Each compare question's pairs of sides, head to head: wins when both orders agree, and splits
1024
- -- when the order decided. The pooled win rate counts a split as a win for each side.
1025
- CREATE VIEW question_pairs AS SELECT * FROM question_pairs_m;
1026
-
1027
- -- What a factor (environment, harness, model) did, by evidence tier: controlled (one run),
1028
- -- pooled (comparable runs, and across evals), observed (over time), not_measured (one level).
1029
- CREATE VIEW factor_effects AS SELECT * FROM factor_effects_m;
1030
-
1031
- -- Whether each attempt read each skill its environment placed, and at which transcript line.
1032
- -- A read is a tool whose target is under skills/<name>/, or a dedicated load
1033
- -- (OpenCode skill, Goose load_skill) whose target is the skill name.
1034
- CREATE VIEW skill_reads AS
1035
- SELECT a.run_id, a.eval_key, a.cell_key, a.key AS attempt_key, a.id AS attempt_id, c.variant,
1036
- s.name AS skill, s.placement,
1037
- (SELECT MIN(t.line) FROM tool_calls t WHERE t.run_id = a.run_id AND t.attempt_key = a.key
1038
- AND (t.target LIKE '%skills/' || s.name || '/%'
1039
- OR (t.tool IN ('skill', 'Skill', 'load_skill') AND t.target = s.name))) AS read_line,
1040
- (SELECT COUNT(*) FROM tool_calls t WHERE t.run_id = a.run_id AND t.attempt_key = a.key
1041
- AND (t.target LIKE '%skills/' || s.name || '/%'
1042
- OR (t.tool IN ('skill', 'Skill', 'load_skill') AND t.target = s.name))) AS reads
1043
- FROM attempts a
1044
- JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
1045
- JOIN env_skills s ON s.run_id = c.run_id AND s.digest = c.environment_digest;
1046
-
1047
- -- Tool calls per attempt by class: context (a placed skill, rules file or other placed file, a
1048
- -- skill load, an MCP tool of the environment), dependency (node_modules, site-packages, vendor,
1049
- -- --help), web, verify (a command the eval's checks run, or test/build/lint/tsc) and workspace.
1050
- -- A heuristic: shell commands are split into words and exclusions are not counted.
1051
- CREATE VIEW context_use AS
1052
- SELECT a.run_id, a.eval_key, a.cell_key, c.environment, a.key AS attempt_key, a.id AS attempt_id,
1053
- t.class, COUNT(*) AS calls, MIN(t.line) AS first_line,
1054
- (SELECT x.target FROM tool_calls x WHERE x.run_id = a.run_id AND x.attempt_key = a.key
1055
- AND x.class = t.class ORDER BY x.line LIMIT 1) AS example
1056
- FROM tool_calls t
1057
- JOIN attempts a ON a.run_id = t.run_id AND a.key = t.attempt_key
1058
- JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
1059
- GROUP BY a.run_id, a.key, t.class;
1060
-
1061
- -- Each file the environment declared (its skills' files, rules), per attempt: how many tool calls
1062
- -- named it, directly or through a glob, and the first one's transcript line (null: never read).
1063
- CREATE VIEW skill_files AS
1064
- SELECT a.run_id, a.eval_key, a.cell_key, c.environment, a.key AS attempt_key,
1065
- a.id AS attempt_id, x.path,
1066
- (SELECT COUNT(*) FROM context_reads r WHERE r.run_id = a.run_id AND r.attempt_key = a.key
1067
- AND r.path = x.path) AS reads,
1068
- (SELECT MIN(r.line) FROM context_reads r WHERE r.run_id = a.run_id AND r.attempt_key = a.key
1069
- AND r.path = x.path) AS first_line
1070
- FROM attempt_context x
1071
- JOIN attempts a ON a.run_id = x.run_id AND a.key = x.attempt_key
1072
- JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
1073
- WHERE x.declared = 1;
1074
-
1075
- -- Failed attempts and infra errors (an OOM kill among them) with the cell's harness, model, each
1076
- -- of its environment's skills and the label of the first check that failed.
1077
- CREATE VIEW failing_attempts AS
1078
- SELECT a.run_id, r.started_at, a.eval_key, a.key AS attempt_key, a.cell_key, c.harness,
1079
- c.model, c.environment, s.name AS skill, a.outcome, a.error_kind, a.error_message,
1080
- a.duration_ms, a.oom_kills,
1081
- (SELECT x.label FROM assertions x WHERE x.run_id = a.run_id AND x.attempt_key = a.key
1082
- AND x.status = 'fail' ORDER BY x.seq LIMIT 1) AS first_failed_check
1083
- FROM attempts a JOIN runs r USING (run_id)
1084
- JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
1085
- LEFT JOIN env_skills s ON s.run_id = c.run_id AND s.digest = c.environment_digest
1086
- WHERE a.outcome IN ('fail', 'infra_error');
1087
-
1088
- -- Each check of a variant cell beside its control's: whether it tells them apart.
1089
- CREATE VIEW check_discrimination AS
1090
- SELECT k.run_id, k.eval_key, c.harness, c.variant, c.key AS cell_key, k.label,
1091
- k.pass AS variant_pass, k.fail AS variant_fail, k.not_run AS variant_not_run,
1092
- k.control_pass, k.control_fail, k.control_not_run, k.verdict
1093
- FROM cell_checks k
1094
- JOIN cells c ON c.run_id = k.run_id AND c.eval_key = k.eval_key AND c.key = k.cell_key
1095
- WHERE c.role = 'variant' AND k.verdict IS NOT NULL;
1096
-
1097
- -- Each check (expect label) per cell per run, on every cell (control and variant alike), over the
1098
- -- cell's counted attempts. rate = pass / (pass + fail): an error (the matcher couldn't answer) and
1099
- -- a not-run (a hard expect before it failed) are left out; null when neither passed nor failed.
1100
- CREATE VIEW check_rates AS
1101
- SELECT k.run_id, k.eval_key, k.cell_key, c.harness, c.environment, c.role, k.label, k.pass,
1102
- k.fail, k.error, k.not_run, CAST(k.pass AS REAL) / NULLIF(k.pass + k.fail, 0) AS rate
1103
- FROM cell_checks k
1104
- JOIN cells c ON c.run_id = k.run_id AND c.eval_key = k.eval_key AND c.key = k.cell_key;
1105
-
1106
- -- One eval per run: outcome and pass@1, with the branch and pull request.
1107
- CREATE VIEW eval_trend AS
1108
- SELECT e.eval_key, r.run_id, r.started_at, r.branch, r.pr_number, r.recorded, e.outcome,
1109
- e.pass_at_1, e.fingerprint
1110
- FROM (SELECT key AS eval_key, * FROM evals) e JOIN runs r USING (run_id);
1111
-
1112
- -- Each judge question's pass rate per cell per run, where it differs from that cell's last run.
1113
- CREATE VIEW verdict_flips AS
1114
- WITH rates AS (
1115
- SELECT j.run_id, a.eval_key, a.cell_key, j.key AS judgement_key, j.question,
1116
- AVG(j.verdict = 'pass') AS pass_rate, COUNT(*) AS n
1117
- FROM judgements j JOIN attempts a ON a.run_id = j.run_id AND a.key = j.attempt_key
1118
- GROUP BY j.run_id, a.eval_key, a.cell_key, j.key
1119
- ), ordered AS (
1120
- SELECT *, LAG(pass_rate) OVER w AS prev_pass_rate, LAG(run_id) OVER w AS prev_run_id
1121
- FROM rates WINDOW w AS (PARTITION BY eval_key, cell_key, judgement_key ORDER BY run_id)
1122
- )
1123
- SELECT run_id, prev_run_id, eval_key, cell_key, judgement_key, question, prev_pass_rate,
1124
- pass_rate, n
1125
- FROM ordered WHERE prev_run_id IS NOT NULL AND pass_rate IS NOT prev_pass_rate;
1126
- `;
1127
- const VIEW_NAMES = [
1128
- "lift",
1129
- "family_lift",
1130
- "effort",
1131
- "contrasts",
1132
- "question_pairs",
1133
- "factor_effects",
1134
- "skill_reads",
1135
- "context_use",
1136
- "skill_files",
1137
- "flaky_cells",
1138
- "drift",
1139
- "failing_attempts",
1140
- "eval_trend",
1141
- "verdict_flips",
1142
- "check_discrimination",
1143
- "check_rates"
1144
- ];
1145
- const json = (value) => JSON.stringify(value);
1146
- /** The agent CLI's home config, which its harness writes (MCP servers, settings), not the eval. */
1147
- const HARNESS_CONFIG = /^~\/.*(^|\/)(config\.(toml|ya?ml|json)|settings(\.local)?\.json|mcp\.json)$/;
1148
- /**
1149
- * A placed file the environment declared: under one of its skills, or a rules file outside any
1150
- * skills directory. Not an agent CLI's own state or home config, its bundled skills or the
1151
- * adapter's skill index.
1152
- */
1153
- const declaredContext = (path, skills) => !isRuntimeState(path) && !path.endsWith("/.ai-sdk-harness-skills.json") && !HARNESS_CONFIG.test(path) && (skills.some((s) => path.includes(`skills/${s}/`)) || !/(^|\/)skills(-[^/]+)?\//.test(path));
1154
- /** `attempts.last_text` is the agent's final message, capped: most "why" questions start there. */
1155
- const LAST_TEXT_LIMIT = 2048;
1156
- /** The tool calls in a transcript.jsonl, numbered by line from 1; lines that aren't JSON are skipped. */
1157
- function toolCalls(transcript) {
1158
- const calls = [];
1159
- transcript.split("\n").forEach((line, i) => {
1160
- if (!line.trim()) return;
1161
- let event;
1162
- try {
1163
- event = JSON.parse(line);
1164
- } catch {
1165
- return;
1166
- }
1167
- if (event?.type !== "tool") return;
1168
- calls.push({
1169
- line: i + 1,
1170
- turn: typeof event.turn === "number" ? event.turn : null,
1171
- tool: String(event.tool ?? ""),
1172
- target: toolTarget(event.input, String(event.tool ?? "")),
1173
- ok: event.ok !== false,
1174
- durationMs: typeof event.durationMs === "number" ? event.durationMs : null
1175
- });
1176
- });
1177
- return calls;
1178
- }
1179
- /** `classifyCall`, but a call its heuristic can't read counts as workspace: one transcript line can't stop the sync. */
1180
- function classifyOrWorkspace(tool, target, ctx) {
1181
- try {
1182
- return classifyCall(tool, target, ctx);
1183
- } catch {
1184
- return {
1185
- class: "workspace",
1186
- files: []
1187
- };
1188
- }
1189
- }
1190
- /**
1191
- * What a cell's results can be pooled on across runs: its fingerprint parts, with the context
1192
- * part replaced by the context files that aren't an agent CLI's own runtime state.
1193
- */
1194
- function comparableFingerprint(cell, attempts) {
1195
- const { context: _context, ...parts } = cell.fingerprintParts;
1196
- const files = /* @__PURE__ */ new Map();
1197
- for (const a of attempts) {
1198
- if (a.cell !== cell.key) continue;
1199
- for (const c of a.context) if (!isRuntimeState(c.path)) files.set(c.path, c.sha256);
1200
- }
1201
- return sha256(json({
1202
- parts,
1203
- context: [...files].sort(([a], [b]) => a < b ? -1 : 1)
1204
- }));
1205
- }
1206
- function insertRun(db, record, recorded, transcripts) {
1207
- const run = (sql, ...values) => db.prepare(sql).run(...values);
1208
- const { runner, summary, git } = record;
1209
- run(`INSERT INTO runs VALUES (${Array(26).fill("?").join(", ")})`, record.runId, recorded ? 1 : 0, record.status, record.startedAt, record.endedAt, record.durationMs, record.author, git?.sha ?? null, git?.branch ?? null, git ? git.dirty ? 1 : 0 : null, runner.kind, runner.kind === "ci" ? runner.provider : null, runner.kind === "ci" ? runner.runUrl : null, runner.kind === "ci" ? runner.pr?.number ?? null : null, runner.kind === "ci" ? runner.pr?.baseRef ?? null : null, record.stack.reeval, record.stack.judge.model, record.stack.judge.version, record.profile?.name ?? null, summary.evals, summary.attempts, summary.pass, summary.fail, summary.notGated, summary.infraError, summary.skipped);
1210
- for (const env of record.environments) {
1211
- run("INSERT OR IGNORE INTO environments VALUES (?, ?, ?, ?, ?)", record.runId, env.name, env.digest, env.parent, json(env.overrides));
1212
- for (const s of env.skills) run("INSERT INTO env_skills VALUES (?, ?, ?, ?, ?, ?, ?)", record.runId, env.digest, s.name, s.source, s.commit, s.sha256, s.placement);
1213
- }
1214
- const envOf = new Map(record.environments.map((env) => [env.digest, env]));
1215
- for (const e of record.evals) {
1216
- const cellEnv = new Map(e.cells.map((c) => [c.key, c.environmentDigest ?? ""]));
1217
- run(`INSERT INTO evals VALUES (${Array(14).fill("?").join(", ")})`, record.runId, e.key, e.id, e.slug, e.file, json(e.group), e.name, json(e.tags), e.fingerprint, e.outcome, e.passRate, json(e.sources), e.family?.id ?? null, e.family?.case ?? null);
1218
- for (const c of e.cells) {
1219
- const p = c.fingerprintParts;
1220
- run(`INSERT INTO cells VALUES (${Array(36).fill("?").join(", ")})`, record.runId, e.key, c.key, c.role, c.variant ?? null, c.harness, c.harnessKind, c.model, c.effort ?? null, c.environment, c.environmentDigest, c.sandbox, typeof c.gate === "string" ? c.gate : "passRate", typeof c.gate === "string" ? null : c.gate.passRate, c.fingerprint, p.sources, p.harness, p.environment, p.image, p.agentCli, p.judge, p.context, c.runs, c.pass, c.fail, c.infraError, c.skipped, c.passRate, c.outcome, c.lift?.vs ?? null, c.lift?.points ?? null, c.lift?.interval.lo ?? null, c.lift?.interval.hi ?? null, c.lift ? c.lift.withinNoise ? 1 : 0 : null, c.lift?.minLift ?? null, comparableFingerprint(c, e.attempts));
1221
- for (const k of c.checks ?? []) run(`INSERT INTO cell_checks VALUES (${Array(13).fill("?").join(", ")})`, record.runId, e.key, c.key, k.label, k.pass, k.fail, k.error, k.notRun, k.control?.pass ?? null, k.control?.fail ?? null, k.control?.error ?? null, k.control?.notRun ?? null, k.verdict ?? null);
1222
- }
1223
- for (const answer of e.answers ?? []) {
1224
- for (const v of answer.votes) run(`INSERT INTO votes VALUES (${Array(11).fill("?").join(", ")})`, record.runId, e.key, answer.question, v.cut, v.harness ?? v.arm ?? null, v.run, v.a, v.b, v.winner, v.swapped ? 1 : 0, v.reason ?? null);
1225
- for (const t of pairTallies(answer.votes)) run(`INSERT INTO question_pairs_m VALUES (${Array(13).fill("?").join(", ")})`, record.runId, e.key, answer.question, t.cut, t.held, t.a, t.b, t.pairs, t.aWins, t.bWins, t.ties, t.splits, t.splitsFirst);
1226
- }
1227
- for (const a of e.attempts) {
1228
- run(`INSERT INTO attempts VALUES (${Array(30).fill("?").join(", ")})`, record.runId, e.key, a.cell, a.key, a.id, a.n, a.seq, a.startedAt, a.outcome, a.error?.kind ?? null, a.error?.message ?? null, a.durationMs, a.imageId, a.turns.length, a.activity.tools, a.activity.fileEdits, a.timings.setupMs, a.timings.agentMs, a.timings.checksMs, a.timings.judgeMs, a.turns.at(-1)?.text.slice(0, 2048) ?? null, a.usage?.input ?? null, a.usage?.output ?? null, a.usage?.cacheRead ?? null, a.usage?.cacheWrite ?? null, a.usage?.reasoning ?? null, a.turns.findLast((t) => t.model)?.model ?? null, a.error?.resource?.cause === "oom" ? a.error.resource.kills : 0, a.effort ?? null, a.servedEffort ?? null);
1229
- const env = envOf.get(cellEnv.get(a.cell) ?? "");
1230
- const skills = env?.skills.map((x) => x.name) ?? [];
1231
- for (const c of a.context) run("INSERT INTO attempt_context VALUES (?, ?, ?, ?, ?, ?)", record.runId, a.key, c.path, c.sha256, isRuntimeState(c.path) ? 1 : 0, declaredContext(c.path, skills) ? 1 : 0);
1232
- for (const [i, x] of (a.assertions ?? []).entries()) run(`INSERT INTO assertions VALUES (${Array(12).fill("?").join(", ")})`, record.runId, a.key, e.key, a.cell, i + 1, x.label, x.labelled ? 1 : 0, x.matcher, x.status, x.soft ? 1 : 0, x.message ?? null, x.durationMs ?? null);
1233
- const transcript = transcripts.get(a.key);
1234
- const ctx = {
1235
- skills,
1236
- files: a.context.map((c) => c.path).filter((p) => declaredContext(p, skills)),
1237
- mcp: env?.mcp ?? [],
1238
- verify: a.checks.map((c) => c.command)
1239
- };
1240
- for (const t of transcript ? toolCalls(transcript) : []) {
1241
- const call = classifyOrWorkspace(t.tool, t.target, ctx);
1242
- run("INSERT INTO tool_calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", record.runId, a.key, t.line, t.turn, t.tool, t.target, t.ok ? 1 : 0, t.durationMs, call.class);
1243
- for (const path of call.files) run("INSERT INTO context_reads VALUES (?, ?, ?, ?)", record.runId, a.key, t.line, path);
1244
- }
1245
- for (const j of a.judgements) run("INSERT OR IGNORE INTO judgements VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", record.runId, a.key, j.key, j.question, j.verdict, j.score, j.threshold, j.model, j.resolvedModel ?? null, a.files.judgeInput?.path ?? null);
1246
- for (const [kind, ref] of Object.entries(a.files)) {
1247
- if (ref == null || Array.isArray(ref) || typeof ref.path !== "string") continue;
1248
- run("INSERT INTO artifacts VALUES (?, ?, ?, ?, ?, ?)", record.runId, a.key, kind, ref.path, ref.sha256, ref.bytes);
1249
- }
1250
- a.checks.forEach((c, i) => {
1251
- if (!c.log) return;
1252
- run("INSERT INTO artifacts VALUES (?, ?, ?, ?, ?, ?)", record.runId, a.key, `check:${i + 1}`, c.log.path, c.log.sha256, c.log.bytes);
1253
- });
1254
- }
1255
- }
1256
- for (const f of record.families ?? []) {
1257
- for (const r of f.rates) run(`INSERT INTO families VALUES (${Array(16).fill("?").join(", ")})`, record.runId, f.id, f.file, json(f.group), f.name, f.cases, f.ran, r.cell, r.cases, r.pass, r.counted, r.rate, r.interval.lo, r.interval.hi, r.interval.method, f.outcome);
1258
- for (const l of f.lifts) {
1259
- const e = l.effort;
1260
- const cell = record.evals.find((x) => x.family?.id === f.id && x.cells.some((c) => c.key === l.variant))?.cells.find((c) => c.key === l.variant);
1261
- run(`INSERT INTO family_lift_m VALUES (${Array(30).fill("?").join(", ")})`, record.runId, f.id, f.name, cell?.harness ?? null, cell?.variant ?? cell?.environment ?? null, l.variant, l.vs, l.pairs, l.variantRate, l.controlRate, l.discordant?.variantOnly ?? null, l.discordant?.controlOnly ?? null, l.points, l.interval.lo, l.interval.hi, l.interval.method, l.p, l.withinNoise ? 1 : 0, l.verdict, l.minLift ?? null, l.outcome, e?.tools.ratio ?? null, e?.tools.p ?? null, e?.tools.verdict ?? null, e?.agentMs.ratio ?? null, e?.agentMs.p ?? null, e?.agentMs.verdict ?? null, e?.tokens?.ratio ?? null, e?.tokens?.p ?? null, e?.tokens?.verdict ?? null);
1262
- }
1263
- }
1264
- }
1265
- const DATA_TABLES = [
1266
- "runs",
1267
- "evals",
1268
- "families",
1269
- "family_lift_m",
1270
- "votes",
1271
- "question_pairs_m",
1272
- "cells",
1273
- "attempts",
1274
- "judgements",
1275
- "environments",
1276
- "env_skills",
1277
- "artifacts",
1278
- "attempt_context",
1279
- "tool_calls",
1280
- "context_reads",
1281
- "assertions",
1282
- "cell_checks"
1283
- ];
1284
- function removeRun(db, runId) {
1285
- for (const table of DATA_TABLES) db.prepare(`DELETE FROM ${table} WHERE run_id = ?`).run(runId);
1286
- db.prepare("DELETE FROM files WHERE run_id = ?").run(runId);
1287
- }
1288
- function freshSchema(db) {
1289
- const objects = rowsOf(db.prepare("SELECT type, name FROM sqlite_schema WHERE type IN ('table', 'view')"));
1290
- for (const o of objects) {
1291
- if (o.name.startsWith("sqlite_")) continue;
1292
- db.exec(`DROP ${o.type === "view" ? "VIEW" : "TABLE"} IF EXISTS "${o.name}"`);
1293
- }
1294
- db.exec(TABLES);
1295
- db.exec(VIEWS);
1296
- db.prepare("INSERT INTO meta VALUES (?, ?), (?, ?)").run("version", String(DB_VERSION), "build", String(BUILD));
1297
- }
1298
- function current(db) {
1299
- try {
1300
- const rows = rowsOf(db.prepare("SELECT key, value FROM meta"));
1301
- const meta = Object.fromEntries(rows.map((r) => [r.key, r.value]));
1302
- return meta.version === String(DB_VERSION) && meta.build === String(BUILD);
1303
- } catch {
1304
- return false;
1305
- }
1306
- }
1307
- /** Each attempt's transcript.jsonl, by attempt key, where the run kept it. */
1308
- async function readTranscripts(dir, record) {
1309
- const out = /* @__PURE__ */ new Map();
1310
- for (const e of record.evals) for (const a of e.attempts) {
1311
- const ref = a.files.transcript;
1312
- if (!ref) continue;
1313
- const text = await readFile(join(dir, record.runId, ...ref.path.split("/")), "utf8").catch(() => null);
1314
- if (text !== null) out.set(a.key, text);
1315
- }
1316
- return out;
1317
- }
1318
- /**
1319
- * Brings the projection up to date with the record files: recorded runs in `historyDir` win over
1320
- * the same run's scratch copy in `.reeval/runs`. Only new, changed or removed files are touched.
1321
- */
1322
- async function sync(db, root, historyDir) {
1323
- const rebuilt = !current(db);
1324
- if (rebuilt) freshSchema(db);
1325
- const wanted = /* @__PURE__ */ new Map();
1326
- for (const [dir, recorded] of [[historyDir, true], [scratchDir(root), false]]) for (const name of await recordNames(dir) ?? []) {
1327
- const runId = name.slice(0, -5);
1328
- if (!RUN_ID.test(runId) || wanted.has(runId)) continue;
1329
- wanted.set(runId, {
1330
- path: join(dir, name),
1331
- recorded
1332
- });
1333
- }
1334
- const known = new Map(rowsOf(db.prepare("SELECT run_id, path, size, mtime_ms FROM files")).map((r) => [r.run_id, r]));
1335
- let added = 0;
1336
- let removed = 0;
1337
- db.exec("BEGIN");
1338
- try {
1339
- for (const runId of known.keys()) if (!wanted.has(runId)) {
1340
- removeRun(db, runId);
1341
- removed++;
1342
- }
1343
- for (const [runId, { path, recorded }] of wanted) {
1344
- const info = await stat(path);
1345
- const rel = relativeFile(root, path);
1346
- const had = known.get(runId);
1347
- if (had && had.path === rel && had.size === info.size && had.mtime_ms === info.mtimeMs) continue;
1348
- let record;
1349
- try {
1350
- record = await readRecord(path);
1351
- } catch (error) {
1352
- warnUnreadable(path, error);
1353
- continue;
1354
- }
1355
- if (had) removeRun(db, runId);
1356
- insertRun(db, record, recorded, await readTranscripts(dirname(path), record));
1357
- db.prepare("INSERT INTO files VALUES (?, ?, ?, ?)").run(runId, rel, info.size, info.mtimeMs);
1358
- added++;
1359
- }
1360
- if (added || removed || rebuilt) deriveFacts(db);
1361
- db.exec("COMMIT");
1362
- } catch (error) {
1363
- db.exec("ROLLBACK");
1364
- throw error;
1365
- }
1366
- return {
1367
- runs: wanted.size,
1368
- added,
1369
- removed,
1370
- rebuilt
1371
- };
1372
- }
1373
- /** Opens `.reeval/history.db`, synced with the records; statements after this are read-only. */
1374
- async function openHistoryDb(root, historyDir, options = {}) {
1375
- const { DatabaseSync } = loadSqlite();
1376
- const path = historyDbPath(root);
1377
- await mkdir(dirname(path), { recursive: true });
1378
- const writer = new DatabaseSync(path);
1379
- let synced;
1380
- try {
1381
- writer.exec("PRAGMA busy_timeout = 5000");
1382
- if (options.rebuild) writer.exec("DROP TABLE IF EXISTS meta");
1383
- synced = await sync(writer, root, historyDir);
1384
- } finally {
1385
- writer.close();
1386
- }
1387
- const db = new DatabaseSync(path, { readOnly: true });
1388
- db.exec("PRAGMA busy_timeout = 5000");
1389
- return {
1390
- sync: synced,
1391
- query(sql, params = []) {
1392
- try {
1393
- const statement = db.prepare(sql);
1394
- const rows = Array.isArray(params) ? statement.all(...params) : statement.all(params);
1395
- return {
1396
- columns: typeof statement.columns === "function" ? statement.columns().map((c) => c.name) : Object.keys(rows[0] ?? {}),
1397
- rows
1398
- };
1399
- } catch (error) {
1400
- const message = error.message;
1401
- throw new UsageError(/readonly|read-only/i.test(message) ? `reeval query is read-only: .reeval/history.db is a projection of the run records (${message})` : `reeval query: ${message}`);
1402
- }
1403
- },
1404
- schema() {
1405
- return rowsOf(db.prepare(`SELECT name, type FROM sqlite_schema WHERE type IN ('table', 'view') AND name NOT IN ('meta', 'files', 'family_lift_m', ${DERIVED.map((d) => `'${d}'`).join(", ")}) AND name NOT LIKE 'sqlite_%' ORDER BY type, name`)).map((o) => ({
1406
- ...o,
1407
- columns: rowsOf(db.prepare(`PRAGMA table_info("${o.name}")`)).map((c) => c.name)
1408
- }));
1409
- },
1410
- close: () => db.close()
1411
- };
1412
- }
1413
- //#endregion
1414
- export { historyDbPath as a, toolCalls as c, comparableFingerprint as i, TOOL_CLASSES as l, SQLITE_NODE as n, nodeAtLeast as o, VIEW_NAMES as r, openHistoryDb as s, LAST_TEXT_LIMIT as t };
1415
-
1416
- //# sourceMappingURL=db-DfhipHEF.js.map