pi-mega-compact 0.11.4 → 0.11.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (381) hide show
  1. package/dist/extensions/dashboard-server/api-contracts/cache-stripes.js +8 -0
  2. package/dist/extensions/dashboard-server/api-contracts/endpoints.js +32 -0
  3. package/dist/extensions/dashboard-server/api-contracts/memory-map.js +1 -0
  4. package/dist/extensions/dashboard-server/api-contracts/memory.js +8 -0
  5. package/dist/extensions/dashboard-server/api-contracts/provider-cache.js +1 -0
  6. package/dist/extensions/dashboard-server/api-contracts/setup.js +10 -0
  7. package/dist/extensions/dashboard-server/index-reader.js +6 -2
  8. package/dist/extensions/dashboard-server/routes-cache.js +135 -47
  9. package/dist/extensions/dashboard-server/routes-game.js +3 -0
  10. package/dist/extensions/dashboard-server/routes-memory-map.js +50 -0
  11. package/dist/extensions/dashboard-server/routes-memory.js +51 -0
  12. package/dist/extensions/dashboard-server/routes-repo.js +2 -2
  13. package/dist/extensions/dashboard-server/routes-setup.js +267 -0
  14. package/dist/extensions/dashboard-server/routes.js +4 -0
  15. package/dist/extensions/dashboard-server/server.js +13 -1
  16. package/dist/extensions/mega-commands.js +189 -1
  17. package/dist/extensions/mega-compact.js +5 -0
  18. package/dist/extensions/mega-config.js +7 -0
  19. package/dist/extensions/mega-events/agent-handlers.js +19 -4
  20. package/dist/extensions/mega-events/compact-handlers.js +1 -0
  21. package/dist/extensions/mega-events/context-handler.js +45 -92
  22. package/dist/extensions/mega-events/context-hwm.js +51 -0
  23. package/dist/extensions/mega-events/error-classifier.js +16 -5
  24. package/dist/extensions/mega-events/mirror-append.js +107 -0
  25. package/dist/extensions/mega-events/outage-advisor.js +7 -1
  26. package/dist/extensions/mega-events/perf-handler.js +134 -2
  27. package/dist/extensions/mega-events/recall-tail.js +59 -0
  28. package/dist/extensions/mega-events/separated-prompt.js +248 -0
  29. package/dist/extensions/mega-events/session-handlers.js +15 -8
  30. package/dist/extensions/mega-runtime/env-loader.js +56 -0
  31. package/dist/extensions/mega-runtime/reset-runtime.js +6 -0
  32. package/dist/extensions/mega-runtime/runtime-snapshot.js +1 -0
  33. package/dist/extensions/mega-runtime/runtime.js +7 -0
  34. package/dist/extensions/mega-runtime/snapshot.js +1 -0
  35. package/dist/extensions/mega-runtime/widget.js +5 -0
  36. package/dist/src/cache-stripe-impl.js +288 -0
  37. package/dist/src/cache-stripe.js +11 -0
  38. package/dist/src/config/stopwords.js +24 -0
  39. package/dist/src/config.js +17 -0
  40. package/dist/src/embedder.js +41 -2
  41. package/dist/src/memoryGraph.js +299 -0
  42. package/dist/src/memoryStats.js +75 -0
  43. package/dist/src/monitoring.js +13 -0
  44. package/dist/src/prefix-break.js +43 -0
  45. package/dist/src/pricing.js +1 -1
  46. package/dist/src/queryExpansion.js +163 -0
  47. package/dist/src/queryReformulation/cache.js +123 -0
  48. package/dist/src/queryReformulation/rrf.js +29 -0
  49. package/dist/src/queryReformulation/tfidf.js +61 -0
  50. package/dist/src/queryReformulation/vagueness.js +33 -0
  51. package/dist/src/queryReformulation.js +225 -0
  52. package/dist/src/recallMetrics.js +258 -0
  53. package/dist/src/store/sqlite/cache-stripes.js +133 -0
  54. package/dist/src/store/sqlite/fts5-search.js +68 -0
  55. package/dist/src/store/sqlite/perf-samples.js +145 -16
  56. package/dist/src/store/sqlite/schema.js +60 -1
  57. package/dist/src/store/sqlite.js +1 -0
  58. package/dist/src/store/turns/memory-store.js +8 -0
  59. package/dist/src/store/turns/sqlite-store.js +14 -3
  60. package/dist/src/store/turns/types.js +18 -1
  61. package/dist/src/store/vectorIndex.js +55 -10
  62. package/dist/src/tiered-router/types.js +18 -0
  63. package/dist/src/tieredRouter.js +421 -0
  64. package/dist/src/vectorStore.js +8 -6
  65. package/extensions/dashboard-client/dist/assets/{AchievementTiles-vEo2MtE2.js → AchievementTiles-DR9cLGDL.js} +2 -2
  66. package/extensions/dashboard-client/dist/assets/{AchievementTiles-vEo2MtE2.js.map → AchievementTiles-DR9cLGDL.js.map} +1 -1
  67. package/extensions/dashboard-client/dist/assets/AchievementsTab-svNUgJ1e.js +2 -0
  68. package/extensions/dashboard-client/dist/assets/{AchievementsTab-DjxfYrDc.js.map → AchievementsTab-svNUgJ1e.js.map} +1 -1
  69. package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js +2 -0
  70. package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js.map +1 -0
  71. package/extensions/dashboard-client/dist/assets/{ConfigTab-Cm6ymbmx.js → ConfigTab-CmmWheuW.js} +2 -2
  72. package/extensions/dashboard-client/dist/assets/{ConfigTab-Cm6ymbmx.js.map → ConfigTab-CmmWheuW.js.map} +1 -1
  73. package/extensions/dashboard-client/dist/assets/{EventsTab-DXWl_YzI.js → EventsTab-CFE7jLsa.js} +2 -2
  74. package/extensions/dashboard-client/dist/assets/{EventsTab-DXWl_YzI.js.map → EventsTab-CFE7jLsa.js.map} +1 -1
  75. package/extensions/dashboard-client/dist/assets/GameTab-CJAX_th_.js +2 -0
  76. package/extensions/dashboard-client/dist/assets/{GameTab-BLDudcO5.js.map → GameTab-CJAX_th_.js.map} +1 -1
  77. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DLkKCztn.js → MaintenanceTab-P4SUsvMf.js} +2 -2
  78. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DLkKCztn.js.map → MaintenanceTab-P4SUsvMf.js.map} +1 -1
  79. package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js +2 -0
  80. package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js.map +1 -0
  81. package/extensions/dashboard-client/dist/assets/{MetricsTab-Cq4u-rZF.js → MetricsTab-CHQscixy.js} +2 -2
  82. package/extensions/dashboard-client/dist/assets/{MetricsTab-Cq4u-rZF.js.map → MetricsTab-CHQscixy.js.map} +1 -1
  83. package/extensions/dashboard-client/dist/assets/OverviewTab-DBs4DJdd.js +2 -0
  84. package/extensions/dashboard-client/dist/assets/{OverviewTab-D4kGvsiQ.js.map → OverviewTab-DBs4DJdd.js.map} +1 -1
  85. package/extensions/dashboard-client/dist/assets/ReposTab-DeFz76L5.js +2 -0
  86. package/extensions/dashboard-client/dist/assets/ReposTab-DeFz76L5.js.map +1 -0
  87. package/extensions/dashboard-client/dist/assets/{SessionsTab-DZC2-GCn.js → SessionsTab-BWNsj1OB.js} +2 -2
  88. package/extensions/dashboard-client/dist/assets/{SessionsTab-DZC2-GCn.js.map → SessionsTab-BWNsj1OB.js.map} +1 -1
  89. package/extensions/dashboard-client/dist/assets/SetupTab-uS6A0xmw.js +2 -0
  90. package/extensions/dashboard-client/dist/assets/SetupTab-uS6A0xmw.js.map +1 -0
  91. package/extensions/dashboard-client/dist/assets/TimeSavedCard-CiqCOQuL.js +2 -0
  92. package/extensions/dashboard-client/dist/assets/TimeSavedCard-CiqCOQuL.js.map +1 -0
  93. package/extensions/dashboard-client/dist/assets/{TopicsTab-01G71hv1.js → TopicsTab-BWWQxltv.js} +2 -2
  94. package/extensions/dashboard-client/dist/assets/{TopicsTab-01G71hv1.js.map → TopicsTab-BWWQxltv.js.map} +1 -1
  95. package/extensions/dashboard-client/dist/assets/{TurnsTab-DLRg0LZT.js → TurnsTab-CVz4f4-o.js} +2 -2
  96. package/extensions/dashboard-client/dist/assets/{TurnsTab-DLRg0LZT.js.map → TurnsTab-CVz4f4-o.js.map} +1 -1
  97. package/extensions/dashboard-client/dist/assets/format-COjDIVGj.js +2 -0
  98. package/extensions/dashboard-client/dist/assets/format-COjDIVGj.js.map +1 -0
  99. package/extensions/dashboard-client/dist/assets/index-CUNn22iV.css +1 -0
  100. package/extensions/dashboard-client/dist/assets/index-DrktNOoT.js +42 -0
  101. package/extensions/dashboard-client/dist/assets/index-DrktNOoT.js.map +1 -0
  102. package/extensions/dashboard-client/dist/assets/pricing-CvE4NllQ.js +2 -0
  103. package/extensions/dashboard-client/dist/assets/pricing-CvE4NllQ.js.map +1 -0
  104. package/extensions/dashboard-client/dist/assets/{useSSE-BBYt5UPB.js → useSSE-Bf5LOLSi.js} +2 -2
  105. package/extensions/dashboard-client/dist/assets/{useSSE-BBYt5UPB.js.map → useSSE-Bf5LOLSi.js.map} +1 -1
  106. package/extensions/dashboard-client/dist/index.html +2 -2
  107. package/extensions/dashboard-client/src/App.tsx +28 -8
  108. package/extensions/dashboard-client/src/api/client.ts +29 -0
  109. package/extensions/dashboard-client/src/components/ActiveReposTable.tsx +21 -12
  110. package/extensions/dashboard-client/src/components/CacheHitRateTrendCard.tsx +100 -0
  111. package/extensions/dashboard-client/src/components/ProviderCacheCard.tsx +56 -0
  112. package/extensions/dashboard-client/src/components/SavingsByModelTable.tsx +38 -57
  113. package/extensions/dashboard-client/src/components/StripeDistributionCard.tsx +110 -0
  114. package/extensions/dashboard-client/src/components/TabBar.tsx +49 -4
  115. package/extensions/dashboard-client/src/main.tsx +1 -0
  116. package/extensions/dashboard-client/src/styles/base.css +30 -0
  117. package/extensions/dashboard-client/src/styles/cache.css +167 -0
  118. package/extensions/dashboard-client/src/tabs/CacheTab.tsx +75 -12
  119. package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +440 -0
  120. package/extensions/dashboard-client/src/tabs/SetupTab.tsx +418 -0
  121. package/extensions/dashboard-client/tsconfig.json +3 -2
  122. package/extensions/dashboard-client/vite.config.ts +2 -0
  123. package/extensions/dashboard-server/api-contracts/cache-stripes.ts +49 -0
  124. package/extensions/dashboard-server/api-contracts/endpoints.ts +66 -3
  125. package/extensions/dashboard-server/api-contracts/index.ts +22 -0
  126. package/extensions/dashboard-server/api-contracts/memory-map.ts +50 -0
  127. package/extensions/dashboard-server/api-contracts/memory.ts +27 -0
  128. package/extensions/dashboard-server/api-contracts/multi-repo.ts +231 -231
  129. package/extensions/dashboard-server/api-contracts/provider-cache.ts +37 -0
  130. package/extensions/dashboard-server/api-contracts/setup.ts +80 -0
  131. package/extensions/dashboard-server/index-reader.ts +138 -121
  132. package/extensions/dashboard-server/routes-cache.ts +186 -75
  133. package/extensions/dashboard-server/routes-game.ts +3 -0
  134. package/extensions/dashboard-server/routes-memory-map.ts +105 -0
  135. package/extensions/dashboard-server/routes-memory.ts +63 -0
  136. package/extensions/dashboard-server/routes-repo.ts +36 -36
  137. package/extensions/dashboard-server/routes-setup.ts +293 -0
  138. package/extensions/dashboard-server/routes.ts +4 -0
  139. package/extensions/dashboard-server/server.ts +12 -0
  140. package/extensions/mega-commands.ts +211 -1
  141. package/extensions/mega-compact.ts +5 -0
  142. package/extensions/mega-config.ts +27 -0
  143. package/extensions/mega-events/agent-handlers.ts +20 -4
  144. package/extensions/mega-events/compact-handlers.ts +1 -0
  145. package/extensions/mega-events/context-handler.ts +53 -110
  146. package/extensions/mega-events/context-hwm.ts +85 -0
  147. package/extensions/mega-events/error-classifier.ts +15 -4
  148. package/extensions/mega-events/mirror-append.ts +155 -0
  149. package/extensions/mega-events/outage-advisor.ts +11 -5
  150. package/extensions/mega-events/perf-handler.ts +190 -6
  151. package/extensions/mega-events/recall-tail.ts +88 -0
  152. package/extensions/mega-events/separated-prompt.ts +329 -0
  153. package/extensions/mega-events/session-handlers.ts +14 -7
  154. package/extensions/mega-runtime/env-loader.ts +55 -0
  155. package/extensions/mega-runtime/helpers.ts +14 -0
  156. package/extensions/mega-runtime/reset-runtime.ts +6 -0
  157. package/extensions/mega-runtime/runtime-snapshot.ts +1 -0
  158. package/extensions/mega-runtime/runtime.ts +7 -0
  159. package/extensions/mega-runtime/snapshot.ts +6 -0
  160. package/extensions/mega-runtime/widget-types.ts +3 -0
  161. package/extensions/mega-runtime/widget.ts +5 -0
  162. package/package.json +5 -2
  163. package/src/cache-stripe-impl.ts +385 -0
  164. package/src/cache-stripe.ts +18 -0
  165. package/src/config/stopwords.ts +25 -0
  166. package/src/config.ts +26 -0
  167. package/src/embedder.ts +42 -2
  168. package/src/memoryGraph.ts +438 -0
  169. package/src/memoryStats.ts +107 -0
  170. package/src/monitoring.ts +35 -0
  171. package/src/prefix-break.ts +83 -0
  172. package/src/pricing.ts +2 -2
  173. package/src/queryExpansion.ts +206 -0
  174. package/src/queryReformulation/cache.ts +169 -0
  175. package/src/queryReformulation/rrf.ts +51 -0
  176. package/src/queryReformulation/tfidf.ts +79 -0
  177. package/src/queryReformulation/vagueness.ts +38 -0
  178. package/src/queryReformulation.ts +355 -0
  179. package/src/recallMetrics.ts +388 -0
  180. package/src/store/sqlite/cache-stripes.ts +196 -0
  181. package/src/store/sqlite/fts5-search.ts +90 -0
  182. package/src/store/sqlite/perf-samples.ts +218 -23
  183. package/src/store/sqlite/schema.ts +60 -1
  184. package/src/store/sqlite.ts +1 -0
  185. package/src/store/turns/memory-store.ts +10 -1
  186. package/src/store/turns/sqlite-store.ts +27 -15
  187. package/src/store/turns/types.ts +21 -0
  188. package/src/store/vectorIndex.ts +71 -11
  189. package/src/tiered-router/types.ts +64 -0
  190. package/src/tieredRouter.ts +489 -0
  191. package/src/vectorStore.ts +9 -5
  192. package/dist/extensions/conflict-scan.test.js +0 -115
  193. package/dist/extensions/dashboard-server/api-contracts.test.js +0 -884
  194. package/dist/extensions/dashboard-server/html.test.js +0 -52
  195. package/dist/extensions/dashboard-server/perf-server.test.js +0 -80
  196. package/dist/extensions/dashboard-server/routes-cache.test.js +0 -248
  197. package/dist/extensions/dashboard-server/server.test.js +0 -120
  198. package/dist/extensions/dashboard-server-s32.test.js +0 -181
  199. package/dist/extensions/dashboard-server.test.js +0 -339
  200. package/dist/extensions/mega-cache-replay.test.js +0 -400
  201. package/dist/extensions/mega-compact-s38.test.js +0 -1383
  202. package/dist/extensions/mega-compact.test.js +0 -1112
  203. package/dist/extensions/mega-events.test.js +0 -47
  204. package/dist/extensions/mega-game-cmds.test.js +0 -163
  205. package/dist/extensions/mega-runtime/snapshot.test.js +0 -127
  206. package/dist/extensions/mega-runtime/state.test.js +0 -266
  207. package/dist/extensions/mega-runtime/widget.test.js +0 -343
  208. package/dist/extensions/mega-shutdown-widget.test.js +0 -121
  209. package/dist/extensions/mega-teamrun.test.js +0 -156
  210. package/dist/extensions/mega-trim.test.js +0 -58
  211. package/dist/extensions/mega-turn-store.test.js +0 -68
  212. package/dist/src/boundary.test.js +0 -170
  213. package/dist/src/compact.test.js +0 -78
  214. package/dist/src/config/themes.test.js +0 -94
  215. package/dist/src/dedup/dedup.test.js +0 -41
  216. package/dist/src/dedup/l1.test.js +0 -50
  217. package/dist/src/dedup/raptor/buildHistory.test.js +0 -292
  218. package/dist/src/dedup/raptor/multilevel-serve.test.js +0 -229
  219. package/dist/src/dedup/raptor/multilevel.test.js +0 -238
  220. package/dist/src/dedup/raptor/promote.test.js +0 -69
  221. package/dist/src/dedup/raptor/raptor.test.js +0 -248
  222. package/dist/src/dedup/raptor/retrieval.test.js +0 -95
  223. package/dist/src/dedup/raptor/serve-gate.test.js +0 -298
  224. package/dist/src/dedup/sprint12.test.js +0 -219
  225. package/dist/src/dedup-engine.test.js +0 -472
  226. package/dist/src/driftDetection.test.js +0 -87
  227. package/dist/src/e2e.test.js +0 -698
  228. package/dist/src/engine.test.js +0 -111
  229. package/dist/src/extractive.test.js +0 -147
  230. package/dist/src/fork.test.js +0 -81
  231. package/dist/src/game/scoring.test.js +0 -98
  232. package/dist/src/httpEmbedder.test.js +0 -277
  233. package/dist/src/importance.test.js +0 -372
  234. package/dist/src/intent.test.js +0 -78
  235. package/dist/src/log.test.js +0 -42
  236. package/dist/src/mechanical-fix.test.js +0 -65
  237. package/dist/src/memory.test.js +0 -70
  238. package/dist/src/memoryConsolidate.test.js +0 -38
  239. package/dist/src/memoryOps.test.js +0 -129
  240. package/dist/src/memoryRecall.test.js +0 -171
  241. package/dist/src/memoryRoundtrip.test.js +0 -137
  242. package/dist/src/metrics/turns.test.js +0 -152
  243. package/dist/src/mirror/mirror.test.js +0 -185
  244. package/dist/src/raptor-inject-summaries.test.js +0 -162
  245. package/dist/src/ratio.bench.test.js +0 -897
  246. package/dist/src/recall.integration.test.js +0 -77
  247. package/dist/src/recall.test.js +0 -379
  248. package/dist/src/sprint14.test.js +0 -219
  249. package/dist/src/sprint4x-rag-verification.test.js +0 -93
  250. package/dist/src/store/compression.test.js +0 -139
  251. package/dist/src/store/memoryIndex.test.js +0 -51
  252. package/dist/src/store/migrate.test.js +0 -139
  253. package/dist/src/store/pgOpenGuard.test.js +0 -74
  254. package/dist/src/store/phase01.test.js +0 -81
  255. package/dist/src/store/phase04.test.js +0 -54
  256. package/dist/src/store/sprint10.test.js +0 -186
  257. package/dist/src/store/sqlite/game-achievements.test.js +0 -67
  258. package/dist/src/store/sqlite/game-scores.test.js +0 -106
  259. package/dist/src/store/sqlite/game-state.test.js +0 -76
  260. package/dist/src/store/sqlite/global-index.test.js +0 -136
  261. package/dist/src/store/sqlite/mechanical-fix.test.js +0 -146
  262. package/dist/src/store/sqlite/perf-samples.test.js +0 -146
  263. package/dist/src/store/sqlite/raptor.test.js +0 -106
  264. package/dist/src/store/sqlite/turns.test.js +0 -183
  265. package/dist/src/store/sqlite.cachehit.test.js +0 -55
  266. package/dist/src/store/sqlite.dbmaint.test.js +0 -158
  267. package/dist/src/store/sqlite.dbmirror.test.js +0 -184
  268. package/dist/src/store/turns/contract-compliance.test.js +0 -428
  269. package/dist/src/store/turns/memory-store.test.js +0 -9
  270. package/dist/src/store/turns/migrations.test.js +0 -138
  271. package/dist/src/store/turns/sqlite-store.test.js +0 -144
  272. package/dist/src/store/vectorIndex.test.js +0 -123
  273. package/dist/src/store.test.js +0 -161
  274. package/dist/src/supersede.test.js +0 -36
  275. package/dist/src/topics/cluster.test.js +0 -150
  276. package/dist/src/topics/store.test.js +0 -117
  277. package/dist/src/types.test.js +0 -9
  278. package/dist/src/vector-search-cache.test.js +0 -157
  279. package/dist/src/vectorStore.test.js +0 -479
  280. package/dist/src/wiki.test.js +0 -171
  281. package/extensions/conflict-scan.test.ts +0 -129
  282. package/extensions/dashboard-client/dist/assets/AchievementsTab-DjxfYrDc.js +0 -2
  283. package/extensions/dashboard-client/dist/assets/CacheTab-Dnpv94wq.js +0 -2
  284. package/extensions/dashboard-client/dist/assets/CacheTab-Dnpv94wq.js.map +0 -1
  285. package/extensions/dashboard-client/dist/assets/GameTab-BLDudcO5.js +0 -2
  286. package/extensions/dashboard-client/dist/assets/OverviewTab-D4kGvsiQ.js +0 -2
  287. package/extensions/dashboard-client/dist/assets/ReposTab-C0Wnrf2u.js +0 -2
  288. package/extensions/dashboard-client/dist/assets/ReposTab-C0Wnrf2u.js.map +0 -1
  289. package/extensions/dashboard-client/dist/assets/TimeSavedCard-UqljSyP-.js +0 -2
  290. package/extensions/dashboard-client/dist/assets/TimeSavedCard-UqljSyP-.js.map +0 -1
  291. package/extensions/dashboard-client/dist/assets/index-BEdKlpAB.css +0 -1
  292. package/extensions/dashboard-client/dist/assets/index-BX0bcgLu.js +0 -42
  293. package/extensions/dashboard-client/dist/assets/index-BX0bcgLu.js.map +0 -1
  294. package/extensions/dashboard-server/api-contracts.test.ts +0 -1065
  295. package/extensions/dashboard-server/html.test.ts +0 -62
  296. package/extensions/dashboard-server/perf-server.test.ts +0 -101
  297. package/extensions/dashboard-server/routes-cache.test.ts +0 -302
  298. package/extensions/dashboard-server/server.test.ts +0 -131
  299. package/extensions/dashboard-server-s32.test.ts +0 -195
  300. package/extensions/dashboard-server.test.ts +0 -382
  301. package/extensions/mega-cache-replay.test.ts +0 -526
  302. package/extensions/mega-compact-s38.test.ts +0 -1433
  303. package/extensions/mega-compact.test.ts +0 -1509
  304. package/extensions/mega-events.test.ts +0 -55
  305. package/extensions/mega-game-cmds.test.ts +0 -197
  306. package/extensions/mega-runtime/snapshot.test.ts +0 -139
  307. package/extensions/mega-runtime/state.test.ts +0 -262
  308. package/extensions/mega-runtime/widget.test.ts +0 -468
  309. package/extensions/mega-shutdown-widget.test.ts +0 -141
  310. package/extensions/mega-teamrun.test.ts +0 -176
  311. package/extensions/mega-trim.test.ts +0 -64
  312. package/extensions/mega-turn-store.test.ts +0 -108
  313. package/src/boundary.test.ts +0 -187
  314. package/src/compact.test.ts +0 -99
  315. package/src/config/themes.test.ts +0 -116
  316. package/src/dedup/dedup.test.ts +0 -46
  317. package/src/dedup/l1.test.ts +0 -57
  318. package/src/dedup/raptor/buildHistory.test.ts +0 -353
  319. package/src/dedup/raptor/multilevel-serve.test.ts +0 -273
  320. package/src/dedup/raptor/multilevel.test.ts +0 -325
  321. package/src/dedup/raptor/promote.test.ts +0 -82
  322. package/src/dedup/raptor/raptor.test.ts +0 -297
  323. package/src/dedup/raptor/retrieval.test.ts +0 -118
  324. package/src/dedup/raptor/serve-gate.test.ts +0 -348
  325. package/src/dedup/sprint12.test.ts +0 -242
  326. package/src/dedup-engine.test.ts +0 -670
  327. package/src/driftDetection.test.ts +0 -100
  328. package/src/e2e.test.ts +0 -843
  329. package/src/engine.test.ts +0 -123
  330. package/src/extractive.test.ts +0 -174
  331. package/src/fork.test.ts +0 -94
  332. package/src/game/scoring.test.ts +0 -103
  333. package/src/httpEmbedder.test.ts +0 -286
  334. package/src/importance.test.ts +0 -538
  335. package/src/intent.test.ts +0 -87
  336. package/src/log.test.ts +0 -47
  337. package/src/mechanical-fix.test.ts +0 -70
  338. package/src/memory.test.ts +0 -92
  339. package/src/memoryConsolidate.test.ts +0 -47
  340. package/src/memoryOps.test.ts +0 -148
  341. package/src/memoryRecall.test.ts +0 -194
  342. package/src/memoryRoundtrip.test.ts +0 -155
  343. package/src/metrics/turns.test.ts +0 -165
  344. package/src/mirror/mirror.test.ts +0 -240
  345. package/src/raptor-inject-summaries.test.ts +0 -228
  346. package/src/ratio.bench.test.ts +0 -1316
  347. package/src/recall.integration.test.ts +0 -96
  348. package/src/recall.test.ts +0 -467
  349. package/src/sprint14.test.ts +0 -245
  350. package/src/sprint4x-rag-verification.test.ts +0 -119
  351. package/src/store/compression.test.ts +0 -168
  352. package/src/store/memoryIndex.test.ts +0 -61
  353. package/src/store/migrate.test.ts +0 -158
  354. package/src/store/pgOpenGuard.test.ts +0 -89
  355. package/src/store/phase01.test.ts +0 -88
  356. package/src/store/phase04.test.ts +0 -58
  357. package/src/store/sprint10.test.ts +0 -214
  358. package/src/store/sqlite/game-achievements.test.ts +0 -80
  359. package/src/store/sqlite/game-scores.test.ts +0 -132
  360. package/src/store/sqlite/game-state.test.ts +0 -89
  361. package/src/store/sqlite/global-index.test.ts +0 -222
  362. package/src/store/sqlite/mechanical-fix.test.ts +0 -162
  363. package/src/store/sqlite/perf-samples.test.ts +0 -169
  364. package/src/store/sqlite/raptor.test.ts +0 -139
  365. package/src/store/sqlite/turns.test.ts +0 -218
  366. package/src/store/sqlite.cachehit.test.ts +0 -68
  367. package/src/store/sqlite.dbmaint.test.ts +0 -193
  368. package/src/store/sqlite.dbmirror.test.ts +0 -229
  369. package/src/store/turns/contract-compliance.test.ts +0 -562
  370. package/src/store/turns/memory-store.test.ts +0 -15
  371. package/src/store/turns/migrations.test.ts +0 -161
  372. package/src/store/turns/sqlite-store.test.ts +0 -161
  373. package/src/store/vectorIndex.test.ts +0 -140
  374. package/src/store.test.ts +0 -191
  375. package/src/supersede.test.ts +0 -42
  376. package/src/topics/cluster.test.ts +0 -176
  377. package/src/topics/store.test.ts +0 -132
  378. package/src/types.test.ts +0 -10
  379. package/src/vector-search-cache.test.ts +0 -190
  380. package/src/vectorStore.test.ts +0 -533
  381. package/src/wiki.test.ts +0 -217
@@ -0,0 +1,299 @@
1
+ /**
2
+ * memoryGraph.ts — S46 visual memory map: build a graph of graphs.
3
+ *
4
+ * Nodes = checkpoints (with optional RAPTOR level annotations).
5
+ * Edges = temporal adjacency, semantic similarity (cosine above threshold), and
6
+ * RAPTOR parent–child links.
7
+ *
8
+ * pi-agnostic (no pi runtime types). Zero network calls (PREVENT-PI-004).
9
+ * Synchronous read-only interface — non-fatal on store errors.
10
+ *
11
+ * Re-exports the public types as both MemoryGraph* and shorter aliases
12
+ * (GraphNode, GraphEdge, GraphMetadata) so dashboard api-contracts can import
13
+ * from the canonical source.
14
+ */
15
+ import { getStateDir, normalizeSessionId } from "./store.js";
16
+ import { openStore, decodeEmbedding } from "./store/sqlite/utils.js";
17
+ import { listRaptorNodes } from "./store/sqlite/raptor.js";
18
+ import { Logger } from "./log.js";
19
+ const log = new Logger();
20
+ // ---------------------------------------------------------------------------
21
+ // Defaults
22
+ // ---------------------------------------------------------------------------
23
+ const DEFAULT_SIMILARITY_THRESHOLD = 0.7;
24
+ const DEFAULT_MAX_EDGES_PER_NODE = 4;
25
+ // ---------------------------------------------------------------------------
26
+ // Public API
27
+ // ---------------------------------------------------------------------------
28
+ /**
29
+ * Build a memory graph from the store.
30
+ *
31
+ * Opens the SQLite store, reads all (or session-filtered) checkpoints,
32
+ * computes cosine similarity edges, temporal edges, and RAPTOR parent edges.
33
+ * Returns a MemoryGraph or an empty graph on failure (non-fatal).
34
+ */
35
+ export function buildMemoryGraph(opts = {}) {
36
+ const { similarityThreshold = DEFAULT_SIMILARITY_THRESHOLD, maxEdgesPerNode = DEFAULT_MAX_EDGES_PER_NODE, sessionId, stateDir = getStateDir(), includeRaptorEdges = true, includeTemporalEdges = true, } = opts;
37
+ const builtAt = Date.now();
38
+ try {
39
+ const nodes = loadNodes(sessionId, stateDir);
40
+ const nodeIndex = buildNodeIndex(nodes);
41
+ const edges = [];
42
+ // Temporal edges: consecutive checkpoints within the same session
43
+ if (includeTemporalEdges) {
44
+ edges.push(...buildTemporalEdges(nodes));
45
+ }
46
+ // Semantic edges: cosine similarity above threshold
47
+ edges.push(...buildSemanticEdges(nodes, nodeIndex, similarityThreshold, maxEdgesPerNode));
48
+ // RAPTOR parent–child edges
49
+ if (includeRaptorEdges) {
50
+ edges.push(...buildRaptorEdges(nodes, sessionId, stateDir, nodeIndex));
51
+ }
52
+ // Deduplicate edges (keep highest weight for each source-target pair)
53
+ const dedupedEdges = deduplicateEdges(edges);
54
+ // Count sessions
55
+ const sessionSet = new Set();
56
+ for (const n of nodes) {
57
+ sessionSet.add(n.sessionId);
58
+ }
59
+ const meta = {
60
+ sessionCount: sessionSet.size,
61
+ totalCheckpoints: nodes.length,
62
+ totalEdges: dedupedEdges.length,
63
+ semanticEdgeCount: dedupedEdges.filter((e) => e.type === "semantic").length,
64
+ temporalEdgeCount: dedupedEdges.filter((e) => e.type === "temporal").length,
65
+ raptorEdgeCount: dedupedEdges.filter((e) => e.type === "raptor_parent").length,
66
+ similarityThresholdUsed: similarityThreshold,
67
+ builtAt,
68
+ };
69
+ return { nodes, edges: dedupedEdges, metadata: meta };
70
+ }
71
+ catch (err) {
72
+ log.warn("buildMemoryGraph failed", { error: String(err) });
73
+ return {
74
+ nodes: [],
75
+ edges: [],
76
+ metadata: {
77
+ sessionCount: 0,
78
+ totalCheckpoints: 0,
79
+ totalEdges: 0,
80
+ semanticEdgeCount: 0,
81
+ temporalEdgeCount: 0,
82
+ raptorEdgeCount: 0,
83
+ similarityThresholdUsed: similarityThreshold,
84
+ builtAt,
85
+ },
86
+ };
87
+ }
88
+ }
89
+ // ---------------------------------------------------------------------------
90
+ // Node loading
91
+ // ---------------------------------------------------------------------------
92
+ function loadNodes(sessionId, stateDir) {
93
+ const db = openStore(stateDir);
94
+ const sql = sessionId
95
+ ? "SELECT * FROM context_chunks WHERE session_id = ? ORDER BY id ASC"
96
+ : "SELECT * FROM context_chunks ORDER BY session_id, id ASC";
97
+ const rows = sessionId
98
+ ? db.prepare(sql).all(normalizeSessionId(sessionId))
99
+ : db.prepare(sql).all();
100
+ return rows.map(rowToGraphNode);
101
+ }
102
+ function rowToGraphNode(row) {
103
+ const summary = row.summary ?? "";
104
+ return {
105
+ id: row.id,
106
+ sessionId: row.session_id,
107
+ label: row.id,
108
+ summaryTruncated: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
109
+ tokenEstimate: row.token_estimate ?? 0,
110
+ timestamp: row.created_at ?? Date.now(),
111
+ dedupStatus: row.dedup_status ?? undefined,
112
+ raptorLevel: 0, // enriched below if RAPTOR data exists
113
+ topicSummary: row.topic_summary ?? undefined,
114
+ decisionCount: row.key_decisions
115
+ ? safeJsonCount(row.key_decisions)
116
+ : 0,
117
+ textSnippet: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
118
+ };
119
+ }
120
+ /** Parse a JSON array stored in a text column and return its length. */
121
+ function safeJsonCount(raw) {
122
+ if (!raw)
123
+ return 0;
124
+ try {
125
+ const arr = JSON.parse(raw);
126
+ return Array.isArray(arr) ? arr.length : 0;
127
+ }
128
+ catch {
129
+ return 0;
130
+ }
131
+ }
132
+ // ---------------------------------------------------------------------------
133
+ // Node index building
134
+ // ---------------------------------------------------------------------------
135
+ function buildNodeIndex(nodes) {
136
+ const idx = new Map();
137
+ nodes.forEach((n, i) => idx.set(n.id, i));
138
+ return idx;
139
+ }
140
+ // ---------------------------------------------------------------------------
141
+ // Edges: temporal
142
+ // ---------------------------------------------------------------------------
143
+ function buildTemporalEdges(nodes) {
144
+ const edges = [];
145
+ if (nodes.length < 2)
146
+ return edges;
147
+ let prev = nodes[0];
148
+ for (let i = 1; i < nodes.length; i++) {
149
+ const cur = nodes[i];
150
+ if (cur.sessionId === prev.sessionId) {
151
+ edges.push({
152
+ source: prev.id,
153
+ target: cur.id,
154
+ weight: 1.0,
155
+ type: "temporal",
156
+ });
157
+ }
158
+ prev = cur;
159
+ }
160
+ return edges;
161
+ }
162
+ // ---------------------------------------------------------------------------
163
+ // Edges: semantic (cosine similarity)
164
+ // ---------------------------------------------------------------------------
165
+ function buildSemanticEdges(nodes, _nodeIndex, threshold, maxPerNode) {
166
+ if (nodes.length < 2)
167
+ return [];
168
+ const db = openStore();
169
+ // Retrieve embeddings from DB for each node
170
+ const embeddings = new Map();
171
+ const placeholders = nodes.map(() => "?").join(",");
172
+ const ids = nodes.map((n) => n.id);
173
+ const rows = db
174
+ .prepare(`SELECT id, embedding_blob FROM context_chunks WHERE id IN (${placeholders})`)
175
+ .all(...ids);
176
+ for (const row of rows) {
177
+ const emb = decodeEmbedding(row.embedding_blob);
178
+ if (emb.length > 0) {
179
+ embeddings.set(row.id, emb);
180
+ }
181
+ }
182
+ const edges = [];
183
+ for (const node of nodes) {
184
+ const aId = node.id;
185
+ const aEmb = embeddings.get(aId);
186
+ if (!aEmb)
187
+ continue;
188
+ const candidates = [];
189
+ for (const other of nodes) {
190
+ if (other.id === aId)
191
+ continue;
192
+ // Skip if already connected via temporal or raptor edge
193
+ const bEmb = embeddings.get(other.id);
194
+ if (!bEmb)
195
+ continue;
196
+ const sim = cosineSimilarity(aEmb, bEmb);
197
+ if (sim >= threshold) {
198
+ candidates.push({ target: other.id, weight: sim });
199
+ }
200
+ }
201
+ // Keep top-K highest-weight semantic neighbors
202
+ candidates.sort((a, b) => b.weight - a.weight);
203
+ const top = candidates.slice(0, maxPerNode);
204
+ for (const c of top) {
205
+ edges.push({
206
+ source: aId,
207
+ target: c.target,
208
+ weight: c.weight,
209
+ type: "semantic",
210
+ });
211
+ }
212
+ }
213
+ return edges;
214
+ }
215
+ // ---------------------------------------------------------------------------
216
+ // Edges: RAPTOR parent–child
217
+ // ---------------------------------------------------------------------------
218
+ function buildRaptorEdges(nodes, sessionId, stateDir, nodeIndex) {
219
+ const edges = [];
220
+ try {
221
+ if (sessionId) {
222
+ enrichRaptorForSession(sessionId, stateDir, nodeIndex, nodes, edges);
223
+ }
224
+ else {
225
+ // Load all sessions from DB
226
+ const db = openStore(stateDir);
227
+ const sessionRows = db
228
+ .prepare("SELECT DISTINCT session_id FROM raptor_nodes ORDER BY session_id")
229
+ .all();
230
+ for (const row of sessionRows) {
231
+ enrichRaptorForSession(row.session_id, stateDir, nodeIndex, nodes, edges);
232
+ }
233
+ }
234
+ }
235
+ catch (err) {
236
+ log.warn("RAPTOR edge build failed (non-fatal)", { error: String(err) });
237
+ }
238
+ return edges;
239
+ }
240
+ function enrichRaptorForSession(sessId, stateDir, nodeIndex, nodes, edges) {
241
+ const raptorNodes = listRaptorNodes(sessId, stateDir);
242
+ for (const rn of raptorNodes) {
243
+ const idx = nodeIndex.get(rn.id);
244
+ if (idx !== undefined) {
245
+ // Annotate the graph node with the RAPTOR level
246
+ nodes[idx].raptorLevel = rn.level;
247
+ }
248
+ if (!rn.parentId)
249
+ continue;
250
+ // Only create edge if both nodes are in the graph
251
+ if (nodeIndex.has(rn.id) && nodeIndex.has(rn.parentId)) {
252
+ edges.push({
253
+ source: rn.id,
254
+ target: rn.parentId,
255
+ weight: Math.max(0.1, 1.0 - rn.level * 0.1), // higher levels = weaker link
256
+ type: "raptor_parent",
257
+ });
258
+ }
259
+ }
260
+ }
261
+ // ---------------------------------------------------------------------------
262
+ // Edge deduplication
263
+ // ---------------------------------------------------------------------------
264
+ function deduplicateEdges(edges) {
265
+ const seen = new Map();
266
+ for (const e of edges) {
267
+ const keyA = `${e.source}|${e.target}`;
268
+ const keyB = `${e.target}|${e.source}`;
269
+ const existing = seen.get(keyA) ?? seen.get(keyB);
270
+ if (!existing) {
271
+ seen.set(keyA, e);
272
+ }
273
+ else if (e.weight > existing.weight) {
274
+ // Replace with higher-weight edge
275
+ seen.delete(keyA);
276
+ seen.delete(keyB);
277
+ seen.set(keyA, e);
278
+ }
279
+ }
280
+ return Array.from(seen.values());
281
+ }
282
+ // ---------------------------------------------------------------------------
283
+ // Cosine similarity
284
+ // ---------------------------------------------------------------------------
285
+ function cosineSimilarity(a, b) {
286
+ const len = Math.min(a.length, b.length);
287
+ if (len === 0)
288
+ return 0;
289
+ let dot = 0;
290
+ let normA = 0;
291
+ let normB = 0;
292
+ for (let i = 0; i < len; i++) {
293
+ dot += a[i] * b[i];
294
+ normA += a[i] * a[i];
295
+ normB += b[i] * b[i];
296
+ }
297
+ const denom = Math.sqrt(normA) * Math.sqrt(normB);
298
+ return denom === 0 ? 0 : dot / denom;
299
+ }
@@ -0,0 +1,75 @@
1
+ /**
2
+ * memoryStats.ts — aggregate memory-store statistics (S53B).
3
+ *
4
+ * Reads from the durable SQLite memories table to produce:
5
+ * - totalMemories: all memories in the store
6
+ * - memoriesInLast30Days: memories created within the last 30 days
7
+ * - topStableMemories: top-N memories by recall count (desc)
8
+ * - avgRecallScore: mean recallCount across all memories
9
+ *
10
+ * recallCount = 1 if last_recalled_at IS NOT NULL, else 0.
11
+ * This is the durable signal from recallMemory() — a memory that was
12
+ * recalled at least once is "stable" (still relevant). The exact recall
13
+ * frequency is tracked via turn_recall rows (source='memory') for
14
+ * cross-session analysis, but the memories table provides a reliable
15
+ * on-store signal that is always present without a schema migration.
16
+ *
17
+ * Real-store only (PREVENT-PI-004). No mocks. No network.
18
+ */
19
+ import { openStore } from "./store/sqlite/utils.js";
20
+ /**
21
+ * Aggregate memory statistics from the SQLite store.
22
+ * Always returns a valid MemoryStatsResult; never throws.
23
+ */
24
+ export async function memoryStats(stateDir, opts = {}) {
25
+ const topN = opts.topN ?? 5;
26
+ try {
27
+ const db = openStore(stateDir);
28
+ // ── totalMemories ──────────────────────────────────────────────────────
29
+ const totalRow = db
30
+ .prepare("SELECT COUNT(*) AS n FROM memories")
31
+ .get();
32
+ const totalMemories = totalRow.n;
33
+ // ── memoriesInLast30Days ───────────────────────────────────────────────
34
+ const thirtyDaysAgo = Math.floor(Date.now() / 1000) - 30 * 86_400;
35
+ const recentRow = db
36
+ .prepare("SELECT COUNT(*) AS n FROM memories WHERE created_at >= ?")
37
+ .get(thirtyDaysAgo);
38
+ const memoriesInLast30Days = recentRow.n;
39
+ // ── topStableMemories + avgRecallScore ─────────────────────────────────
40
+ // recallCount: 1 if last_recalled_at IS NOT NULL (recalled at least once),
41
+ // 0 otherwise. Stable memories score high.
42
+ const topRows = db
43
+ .prepare(`SELECT
44
+ m.id,
45
+ m.content AS text,
46
+ m.last_recalled_at AS lastRecalledAt,
47
+ CASE WHEN m.last_recalled_at IS NOT NULL THEN 1 ELSE 0 END AS recallCount
48
+ FROM memories m
49
+ ORDER BY recallCount DESC, m.id ASC
50
+ LIMIT ?`)
51
+ .all(topN);
52
+ // avgRecallScore = (count of recalled memories) / totalMemories.
53
+ // This represents the fraction of memories that are "stable" (recalled at least once).
54
+ const recalledRow = db
55
+ .prepare("SELECT COUNT(*) AS n FROM memories WHERE last_recalled_at IS NOT NULL")
56
+ .get();
57
+ const avgRecallScore = totalMemories > 0 ? recalledRow.n / totalMemories : 0;
58
+ const topStableMemories = topRows.map((r) => ({
59
+ id: r.id,
60
+ text: r.text ?? "",
61
+ recallCount: r.recallCount ?? 0,
62
+ lastRecalledAt: r.lastRecalledAt ?? null,
63
+ }));
64
+ return { totalMemories, memoriesInLast30Days, topStableMemories, avgRecallScore };
65
+ }
66
+ catch {
67
+ // Non-fatal: return zeros rather than propagating an error.
68
+ return {
69
+ totalMemories: 0,
70
+ memoriesInLast30Days: 0,
71
+ topStableMemories: [],
72
+ avgRecallScore: 0,
73
+ };
74
+ }
75
+ }
@@ -129,3 +129,16 @@ export function defaultMetricsPath(stateDir = STATE_DIR_DEFAULT) {
129
129
  export function defaultEventsPath(stateDir = STATE_DIR_DEFAULT) {
130
130
  return join(stateDir, "events.log");
131
131
  }
132
+ /**
133
+ * Append a RecallQualityEvent to events.log (best-effort, non-fatal).
134
+ * Follows the same append-one-JSON-line pattern as logDecision.
135
+ */
136
+ export function logRecallQuality(path, ev) {
137
+ try {
138
+ mkdirSync(dirname(path), { recursive: true });
139
+ appendFileSync(path, JSON.stringify(ev) + "\n", "utf-8");
140
+ }
141
+ catch {
142
+ /* best-effort — never break the extension */
143
+ }
144
+ }
@@ -0,0 +1,43 @@
1
+ /**
2
+ * prefix-break.ts — S53A prefix-break classification.
3
+ *
4
+ * Classifies WHY the provider cache prefix broke on a given turn by comparing
5
+ * the break timestamp against the three candidate event timestamps with a
6
+ * configurable tolerance window.
7
+ *
8
+ * The classification is best-effort (non-fatal): null timestamps are absent,
9
+ * and a break outside all tolerance windows is classified as 'other'.
10
+ *
11
+ * Pi-agnostic: no pi runtime types, no network.
12
+ */
13
+ /**
14
+ * Classify why the provider cache prefix broke at `breakTimestamp`.
15
+ *
16
+ * @param breakTimestamp Wall-clock ms of the cache hit % drop sample.
17
+ * @param events Recent event timestamps from the runtime.
18
+ * @param toleranceMs Max ms gap to consider a match (default 2000).
19
+ * @returns Classification result with cause + confidence.
20
+ *
21
+ * Priority: recall > compaction > inject. Confidence = 1 − (gap / (2·tolerance)),
22
+ * capped at [0.5, 1.0]. Returns 'other' when no event is within tolerance.
23
+ */
24
+ export function classifyPrefixBreak(breakTimestamp, events, toleranceMs = 2000) {
25
+ // Ordered by priority (recall wins if multiple match).
26
+ const candidates = [
27
+ { cause: "recall", ts: events.lastRecallAt },
28
+ { cause: "compaction", ts: events.lastCompactAt },
29
+ { cause: "inject", ts: events.lastInjectAt },
30
+ ];
31
+ for (const { cause, ts } of candidates) {
32
+ if (ts == null)
33
+ continue;
34
+ const gap = Math.abs(breakTimestamp - ts);
35
+ if (gap <= toleranceMs) {
36
+ // confidence = 1 − (gap / (2·tolerance)) ∈ [0.5, 1.0]
37
+ // exact match (gap=0) → 1.0; at boundary (gap=tolerance) → 0.5
38
+ const confidence = Math.max(0.5, 1 - gap / (2 * toleranceMs));
39
+ return { cause, confidence };
40
+ }
41
+ }
42
+ return { cause: "other", confidence: 0 };
43
+ }
@@ -34,7 +34,7 @@ export const MODEL_INPUT_RATES = {
34
34
  "gemini-2.0-flash": 0.1e-6,
35
35
  "deepseek-chat": 0.14e-6,
36
36
  "deepseek-reasoner": 0.55e-6,
37
- "codestral": 0.3e-6,
37
+ codestral: 0.3e-6,
38
38
  "mistral-large": 2e-6,
39
39
  "llama-3.1-405b": 1.33e-6,
40
40
  "llama-3.1-70b": 0.8e-6,
@@ -0,0 +1,163 @@
1
+ /**
2
+ * queryExpansion.ts — S45 CRAG query expansion via embedding neighbors.
3
+ *
4
+ * Expands a query with synonym/related-term variants by finding embedding-
5
+ * similar checkpoints and extracting their discriminative terms. Fully local:
6
+ * uses the TrigramEmbedder (shipped default, zero network) and in-process
7
+ * cosine scan — no LLM call, no external API. PREVENT-PI-004 compliant.
8
+ *
9
+ * Pi-agnostic: no pi runtime types or imports from `src/`.
10
+ */
11
+ import { cosineSimilarity, defaultEmbedder } from "./embedder.js";
12
+ import { listCheckpoints } from "./store/sqlite.js";
13
+ const DEFAULT_MAX_TERMS = 5;
14
+ const DEFAULT_MIN_SIM = 0.3;
15
+ /**
16
+ * Tokenize text into lowercased alphanumeric terms, filtering stop words.
17
+ * Internal helper — no export needed outside this module.
18
+ */
19
+ function tokenizeTerms(text) {
20
+ const STOP_WORDS = new Set([
21
+ "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
22
+ "of", "with", "by", "from", "as", "is", "was", "are", "were", "be",
23
+ "been", "has", "have", "had", "do", "does", "did", "will", "would",
24
+ "could", "should", "may", "might", "shall", "can", "not", "no", "nor",
25
+ "so", "if", "then", "than", "that", "this", "these", "those", "it",
26
+ "its", "i", "me", "my", "we", "our", "you", "your", "he", "she",
27
+ "they", "them", "their", "what", "which", "who", "whom", "when",
28
+ "where", "why", "how", "all", "each", "every", "both", "few", "more",
29
+ "most", "other", "some", "such", "only", "own", "same", "so", "than",
30
+ "too", "very", "just", "about", "above", "after", "again", "against",
31
+ "because", "before", "between", "down", "during", "out", "over",
32
+ "through", "under", "up", "also", "into", "off", "onto", "upon",
33
+ ]);
34
+ // Split on non-alphanumeric, lowercase, filter empties and stop words.
35
+ return text
36
+ .toLowerCase()
37
+ .split(/[^a-z0-9]+/)
38
+ .filter((t) => t.length >= 3 && !STOP_WORDS.has(t));
39
+ }
40
+ /**
41
+ * Compute TF-IDF-like term importance scores for a set of documents.
42
+ * Returns a Map from term to score (log-frequency within doc * IDF).
43
+ */
44
+ function termScores(docs) {
45
+ const docCount = docs.length;
46
+ if (docCount === 0)
47
+ return new Map();
48
+ // Document frequency: how many docs each term appears in.
49
+ const df = new Map();
50
+ const docTerms = docs.map((d) => {
51
+ const terms = tokenizeTerms(d);
52
+ const unique = new Set(terms);
53
+ for (const t of unique) {
54
+ df.set(t, (df.get(t) ?? 0) + 1);
55
+ }
56
+ return terms;
57
+ });
58
+ // Compute TF-IDF score for each term across all docs:
59
+ // score = sum over docs of (tf * idf)
60
+ // tf = term frequency within doc (log-smooth)
61
+ // idf = log(1 + docCount / (1 + docFreq))
62
+ const scores = new Map();
63
+ for (let i = 0; i < docTerms.length; i++) {
64
+ const terms = docTerms[i];
65
+ const tf = new Map();
66
+ for (const t of terms)
67
+ tf.set(t, (tf.get(t) ?? 0) + 1);
68
+ for (const [t, count] of tf) {
69
+ const tfVal = 1 + Math.log10(count);
70
+ const docFreq = df.get(t) ?? 1;
71
+ const idf = Math.log10(1 + docCount / (1 + docFreq));
72
+ scores.set(t, (scores.get(t) ?? 0) + tfVal * idf);
73
+ }
74
+ }
75
+ return scores;
76
+ }
77
+ /**
78
+ * Expand a query with related terms discovered from embedding-neighbor
79
+ * checkpoints. Uses TF-IDF term extraction from the most similar checkpoints
80
+ * to find novel, discriminative terms not already in the query.
81
+ *
82
+ * @param query The original search query.
83
+ * @param store VectorStore instance (for checkpoint access + embedder).
84
+ * @returns Array of expansion terms (lowercased, max `maxTerms`).
85
+ */
86
+ export function expandQuery(query, store, opts = {}) {
87
+ const maxTerms = opts.maxTerms ?? DEFAULT_MAX_TERMS;
88
+ const minSim = opts.minSimilarity ?? DEFAULT_MIN_SIM;
89
+ const embedder = opts.embedder ?? store.embedder ?? defaultEmbedder();
90
+ // Embed the query and scan all checkpoints for nearest neighbors.
91
+ const qv = embedder.embed(query);
92
+ const candidates = [];
93
+ const allSessions = collectSessionIds(store.stateDir);
94
+ const seen = new Set();
95
+ for (const sid of allSessions) {
96
+ const cps = listCheckpoints(sid, store.stateDir);
97
+ for (const cp of cps) {
98
+ if (!cp.summary || seen.has(cp.checkpointId))
99
+ continue;
100
+ seen.add(cp.checkpointId);
101
+ const sim = cosineSimilarity(qv, cp.embedding);
102
+ if (sim >= minSim) {
103
+ candidates.push({ summary: cp.summary, score: sim });
104
+ }
105
+ }
106
+ }
107
+ if (candidates.length === 0)
108
+ return [];
109
+ // Sort by similarity descending, take the top 20 for term extraction.
110
+ const topCandidates = candidates
111
+ .sort((a, b) => b.score - a.score)
112
+ .slice(0, 20);
113
+ // Compute TF-IDF scores across the candidate summaries.
114
+ const docTexts = topCandidates.map((c) => c.summary);
115
+ const scores = termScores(docTexts);
116
+ // Filter out query terms: we want NOVEL terms, not the ones already in the query.
117
+ const queryTerms = new Set(tokenizeTerms(query));
118
+ // Sort terms by score descending, filter query terms and short/noise terms.
119
+ const ranked = [...scores.entries()]
120
+ .filter(([term]) => !queryTerms.has(term))
121
+ .sort((a, b) => b[1] - a[1])
122
+ .slice(0, maxTerms)
123
+ .map(([term]) => term);
124
+ return ranked;
125
+ }
126
+ /**
127
+ * Collect unique session IDs that have at least one checkpoint in the store.
128
+ * Scans the SQLite checkpoint table. Returns an empty array on any error
129
+ * (non-fatal — query expansion degrades gracefully).
130
+ */
131
+ function collectSessionIds(stateDir) {
132
+ try {
133
+ // We import listCheckpoints per session dynamically. A simpler approach:
134
+ // scan via a direct SQL query rather than reading every session.
135
+ // Reuse store/sqlite.ts functions — we can read the sessions table.
136
+ // Fallback: try listing checkpoints for all known session IDs via
137
+ // the sessions table.
138
+ const { db } = loadDb(stateDir);
139
+ const rows = db
140
+ .prepare("SELECT DISTINCT session_id FROM checkpoint_epochs ORDER BY session_id")
141
+ .all();
142
+ return rows.map((r) => r.session_id);
143
+ }
144
+ catch {
145
+ return [];
146
+ }
147
+ }
148
+ /**
149
+ * Lazy-load the SQLite database handle for a given stateDir.
150
+ * Cached per stateDir to avoid re-opening the file on every call.
151
+ */
152
+ const _dbCache = new Map();
153
+ function loadDb(stateDir) {
154
+ const cached = _dbCache.get(stateDir);
155
+ if (cached)
156
+ return cached;
157
+ const { DatabaseSync } = require("node:sqlite");
158
+ const { join } = require("node:path");
159
+ const db = new DatabaseSync(join(stateDir, "mega-compact.db"));
160
+ db.exec("PRAGMA journal_mode=WAL");
161
+ _dbCache.set(stateDir, { db });
162
+ return { db };
163
+ }