pi-mega-compact 0.13.4 → 0.13.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. package/README.md +11 -8
  2. package/dist/extensions/dashboard-server/api-contracts/endpoints/registry.js +50 -0
  3. package/dist/extensions/dashboard-server/api-contracts/rag-metrics.js +1 -0
  4. package/dist/extensions/dashboard-server/api-contracts/rag-settings.js +3 -3
  5. package/dist/extensions/dashboard-server/api-contracts/wiki.js +1 -0
  6. package/dist/extensions/dashboard-server/routes-rag-metrics.js +60 -0
  7. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +167 -0
  8. package/dist/extensions/dashboard-server/routes-rag-settings.js +136 -77
  9. package/dist/extensions/dashboard-server/routes-wiki-helpers.js +97 -0
  10. package/dist/extensions/dashboard-server/routes-wiki.js +330 -0
  11. package/dist/extensions/dashboard-server/routes.js +2 -0
  12. package/dist/extensions/dashboard-server/server.js +17 -3
  13. package/dist/extensions/mega-events/context-handler/afterCompact.js +35 -2
  14. package/dist/extensions/mega-pipeline/recall.js +31 -0
  15. package/dist/extensions/mega-turn-store.js +2 -1
  16. package/dist/src/config/turns.js +10 -0
  17. package/dist/src/config.js +2 -0
  18. package/dist/src/recall/async.js +10 -1
  19. package/dist/src/recall/format.js +10 -1
  20. package/dist/src/recall/hydeTelemetry.js +22 -0
  21. package/dist/src/recall/sync.js +28 -2
  22. package/dist/src/store/turns/hydeStore.js +32 -0
  23. package/dist/src/store/turns/index.js +1 -0
  24. package/dist/src/store/turns/schema.js +102 -1
  25. package/dist/src/store/turns/sqlite-store/rows.js +23 -0
  26. package/dist/src/store/turns/sqlite-store/write.js +7 -3
  27. package/dist/src/topics/durability.js +187 -0
  28. package/dist/src/topics/index.js +2 -1
  29. package/dist/src/topics/store.js +28 -9
  30. package/dist/src/wiki/curation-helpers.js +71 -0
  31. package/dist/src/wiki/curation.js +147 -0
  32. package/dist/src/wiki/incremental.js +126 -0
  33. package/dist/src/wiki/index.js +5 -0
  34. package/extensions/dashboard-client/dist/assets/AreaChart-KAmp_6bL.js +70 -0
  35. package/extensions/dashboard-client/dist/assets/{SessionsTab-C30AwyoF.js.map → AreaChart-KAmp_6bL.js.map} +1 -1
  36. package/extensions/dashboard-client/dist/assets/CacheTab-Bk-gm-R0.js +2 -0
  37. package/extensions/dashboard-client/dist/assets/CacheTab-Bk-gm-R0.js.map +1 -0
  38. package/extensions/dashboard-client/dist/assets/EventsTab-Cyylo0of.js +2 -0
  39. package/extensions/dashboard-client/dist/assets/EventsTab-Cyylo0of.js.map +1 -0
  40. package/extensions/dashboard-client/dist/assets/HealthTab-D-IzmV29.js +2 -0
  41. package/extensions/dashboard-client/dist/assets/HealthTab-D-IzmV29.js.map +1 -0
  42. package/extensions/dashboard-client/dist/assets/MaintenanceTab-CrlkCFVj.js +2 -0
  43. package/extensions/dashboard-client/dist/assets/MaintenanceTab-CrlkCFVj.js.map +1 -0
  44. package/extensions/dashboard-client/dist/assets/MemoryMapTab-DHUdZ7RR.js +2 -0
  45. package/extensions/dashboard-client/dist/assets/MemoryMapTab-DHUdZ7RR.js.map +1 -0
  46. package/extensions/dashboard-client/dist/assets/MetricsTab-B7I4HWVg.js +2 -0
  47. package/extensions/dashboard-client/dist/assets/MetricsTab-B7I4HWVg.js.map +1 -0
  48. package/extensions/dashboard-client/dist/assets/OverviewTab-K6j4ZI7u.js +2 -0
  49. package/extensions/dashboard-client/dist/assets/OverviewTab-K6j4ZI7u.js.map +1 -0
  50. package/extensions/dashboard-client/dist/assets/ReposTab-BSRWhn56.js +2 -0
  51. package/extensions/dashboard-client/dist/assets/ReposTab-BSRWhn56.js.map +1 -0
  52. package/extensions/dashboard-client/dist/assets/SessionsTab-BXdizHN1.js +2 -0
  53. package/extensions/dashboard-client/dist/assets/SessionsTab-BXdizHN1.js.map +1 -0
  54. package/extensions/dashboard-client/dist/assets/SetupTab-CAgyfqY3.js +2 -0
  55. package/extensions/dashboard-client/dist/assets/SetupTab-CAgyfqY3.js.map +1 -0
  56. package/extensions/dashboard-client/dist/assets/TimeSavedCard-DwgCzPD9.js +2 -0
  57. package/extensions/dashboard-client/dist/assets/TimeSavedCard-DwgCzPD9.js.map +1 -0
  58. package/extensions/dashboard-client/dist/assets/TurnsTab-CVKcStBp.js +5 -0
  59. package/extensions/dashboard-client/dist/assets/TurnsTab-CVKcStBp.js.map +1 -0
  60. package/extensions/dashboard-client/dist/assets/WikiTab-BNohxTbs.js +2 -0
  61. package/extensions/dashboard-client/dist/assets/WikiTab-BNohxTbs.js.map +1 -0
  62. package/extensions/dashboard-client/dist/assets/button-B4A-_dCA.js +2 -0
  63. package/extensions/dashboard-client/dist/assets/button-B4A-_dCA.js.map +1 -0
  64. package/extensions/dashboard-client/dist/assets/card-Cjn7Ifoe.js +2 -0
  65. package/extensions/dashboard-client/dist/assets/card-Cjn7Ifoe.js.map +1 -0
  66. package/extensions/dashboard-client/dist/assets/index-CQjCLvv7.js +137 -0
  67. package/extensions/dashboard-client/dist/assets/index-CQjCLvv7.js.map +1 -0
  68. package/extensions/dashboard-client/dist/assets/index-CUA6EDRF.css +1 -0
  69. package/extensions/dashboard-client/dist/assets/switch-CcdXX0Zm.js +2 -0
  70. package/extensions/dashboard-client/dist/assets/switch-CcdXX0Zm.js.map +1 -0
  71. package/extensions/dashboard-client/dist/assets/toggle-KdBnFLu3.js +2 -0
  72. package/extensions/dashboard-client/dist/assets/toggle-KdBnFLu3.js.map +1 -0
  73. package/extensions/dashboard-client/dist/assets/{useSSE-p1pOI_-P.js → useSSE-B6t44vzc.js} +2 -2
  74. package/extensions/dashboard-client/dist/assets/{useSSE-p1pOI_-P.js.map → useSSE-B6t44vzc.js.map} +1 -1
  75. package/extensions/dashboard-client/dist/index.html +2 -2
  76. package/extensions/dashboard-client/package-lock.json +1995 -191
  77. package/extensions/dashboard-client/package.json +12 -1
  78. package/extensions/dashboard-client/postcss.config.js +6 -0
  79. package/extensions/dashboard-client/src/App.tsx +127 -88
  80. package/extensions/dashboard-client/src/api/client.ts +109 -10
  81. package/extensions/dashboard-client/src/components/ActiveReposTable.tsx +15 -7
  82. package/extensions/dashboard-client/src/components/ActiveSessionsTable.tsx +67 -55
  83. package/extensions/dashboard-client/src/components/CacheHitRateTrendCard.tsx +17 -13
  84. package/extensions/dashboard-client/src/components/CacheHitsCard.tsx +35 -30
  85. package/extensions/dashboard-client/src/components/CacheStatusPerModel.tsx +33 -28
  86. package/extensions/dashboard-client/src/components/EventCategoryFilter.tsx +5 -6
  87. package/extensions/dashboard-client/src/components/EventStream.tsx +17 -13
  88. package/extensions/dashboard-client/src/components/HydeDetailPanel.tsx +173 -0
  89. package/extensions/dashboard-client/src/components/LegendCard.tsx +16 -9
  90. package/extensions/dashboard-client/src/components/PerfCards.tsx +28 -20
  91. package/extensions/dashboard-client/src/components/PerfChart.tsx +9 -6
  92. package/extensions/dashboard-client/src/components/ProviderCacheCard.tsx +15 -7
  93. package/extensions/dashboard-client/src/components/RagDashboard.tsx +253 -0
  94. package/extensions/dashboard-client/src/components/RepoAllSessionsCard.tsx +15 -10
  95. package/extensions/dashboard-client/src/components/RepoContextStack.tsx +24 -16
  96. package/extensions/dashboard-client/src/components/RepoDetailModal.tsx +52 -49
  97. package/extensions/dashboard-client/src/components/RepoTable.tsx +26 -17
  98. package/extensions/dashboard-client/src/components/SessionContextGauges.tsx +11 -14
  99. package/extensions/dashboard-client/src/components/SessionInfo.tsx +20 -15
  100. package/extensions/dashboard-client/src/components/SessionsMemoryChart.tsx +8 -5
  101. package/extensions/dashboard-client/src/components/SummaryTiles.tsx +26 -17
  102. package/extensions/dashboard-client/src/components/TabBar.tsx +1 -3
  103. package/extensions/dashboard-client/src/components/layout/AppShell.tsx +42 -0
  104. package/extensions/dashboard-client/src/components/layout/BottomBar.tsx +88 -0
  105. package/extensions/dashboard-client/src/components/layout/Sidebar.tsx +83 -0
  106. package/extensions/dashboard-client/src/components/ui/badge.tsx +30 -0
  107. package/extensions/dashboard-client/src/components/ui/button.tsx +44 -0
  108. package/extensions/dashboard-client/src/components/ui/card.tsx +32 -0
  109. package/extensions/dashboard-client/src/components/ui/select.tsx +64 -0
  110. package/extensions/dashboard-client/src/components/ui/switch.tsx +26 -0
  111. package/extensions/dashboard-client/src/components/ui/tabs.tsx +36 -0
  112. package/extensions/dashboard-client/src/components/ui/toggle.tsx +28 -0
  113. package/extensions/dashboard-client/src/components/ui/tooltip.tsx +27 -0
  114. package/extensions/dashboard-client/src/config.ts +10 -0
  115. package/extensions/dashboard-client/src/main.tsx +2 -2
  116. package/extensions/dashboard-client/src/memory-map-layout.ts +1 -2
  117. package/extensions/dashboard-client/src/styles/index.css +79 -0
  118. package/extensions/dashboard-client/src/tabs/CacheTab.tsx +33 -26
  119. package/extensions/dashboard-client/src/tabs/EventsTab.tsx +22 -7
  120. package/extensions/dashboard-client/src/tabs/HealthTab.tsx +77 -46
  121. package/extensions/dashboard-client/src/tabs/MaintenanceTab/ActionsCard.tsx +60 -50
  122. package/extensions/dashboard-client/src/tabs/MaintenanceTab/DbStatsCard.tsx +61 -35
  123. package/extensions/dashboard-client/src/tabs/MaintenanceTab/DebugBundleCard.tsx +34 -23
  124. package/extensions/dashboard-client/src/tabs/MaintenanceTab/HealthMitigationCard.tsx +62 -43
  125. package/extensions/dashboard-client/src/tabs/MaintenanceTab/SchemaHealthCard.tsx +53 -32
  126. package/extensions/dashboard-client/src/tabs/MaintenanceTab.tsx +2 -2
  127. package/extensions/dashboard-client/src/tabs/MemoryMapTab/MemoryMapView.tsx +29 -46
  128. package/extensions/dashboard-client/src/tabs/MemoryMapTab/RaptorTreeView.tsx +10 -34
  129. package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +14 -34
  130. package/extensions/dashboard-client/src/tabs/MetricsTab.tsx +5 -2
  131. package/extensions/dashboard-client/src/tabs/OverviewTab.tsx +11 -5
  132. package/extensions/dashboard-client/src/tabs/ReposTab.tsx +5 -5
  133. package/extensions/dashboard-client/src/tabs/SessionsTab.tsx +47 -33
  134. package/extensions/dashboard-client/src/tabs/SetupTab/EmbedderSetup.tsx +6 -2
  135. package/extensions/dashboard-client/src/tabs/SetupTab/RagHealthCard.tsx +78 -0
  136. package/extensions/dashboard-client/src/tabs/SetupTab/SettingsPanel.tsx +198 -0
  137. package/extensions/dashboard-client/src/tabs/SetupTab/SettingsSection.tsx +50 -0
  138. package/extensions/dashboard-client/src/tabs/SetupTab.tsx +21 -59
  139. package/extensions/dashboard-client/src/tabs/TopicsTab.tsx +75 -40
  140. package/extensions/dashboard-client/src/tabs/TurnsTab.tsx +150 -112
  141. package/extensions/dashboard-client/src/tabs/WikiTab/TopicEvolutionGraph.tsx +274 -0
  142. package/extensions/dashboard-client/src/tabs/WikiTab/TopicEvolutionView.tsx +71 -0
  143. package/extensions/dashboard-client/src/tabs/WikiTab/TopicTimeline.tsx +103 -0
  144. package/extensions/dashboard-client/src/tabs/WikiTab/WikiPage.tsx +213 -0
  145. package/extensions/dashboard-client/src/tabs/WikiTab/WikiPageControls.tsx +188 -0
  146. package/extensions/dashboard-client/src/tabs/WikiTab.tsx +222 -0
  147. package/extensions/dashboard-client/src/tabs/registry.ts +57 -0
  148. package/extensions/dashboard-client/src/utils/cn.ts +7 -0
  149. package/extensions/dashboard-client/tailwind.config.js +46 -0
  150. package/extensions/dashboard-server/api-contracts/core.ts +59 -0
  151. package/extensions/dashboard-server/api-contracts/endpoints/registry.ts +79 -0
  152. package/extensions/dashboard-server/api-contracts/index.ts +48 -2
  153. package/extensions/dashboard-server/api-contracts/rag-metrics.ts +22 -0
  154. package/extensions/dashboard-server/api-contracts/rag-settings.ts +30 -20
  155. package/extensions/dashboard-server/api-contracts/wiki.ts +138 -0
  156. package/extensions/dashboard-server/routes-rag-metrics.ts +82 -0
  157. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +314 -0
  158. package/extensions/dashboard-server/routes-rag-settings.ts +147 -97
  159. package/extensions/dashboard-server/routes-wiki-helpers.ts +126 -0
  160. package/extensions/dashboard-server/routes-wiki.ts +376 -0
  161. package/extensions/dashboard-server/routes.ts +2 -0
  162. package/extensions/dashboard-server/server.ts +18 -2
  163. package/extensions/mega-events/context-handler/afterCompact.ts +44 -1
  164. package/extensions/mega-pipeline/recall.ts +31 -0
  165. package/extensions/mega-turn-store.ts +13 -0
  166. package/package.json +1 -1
  167. package/src/config/turns.ts +17 -0
  168. package/src/config.ts +3 -0
  169. package/src/recall/async.ts +10 -1
  170. package/src/recall/format.ts +15 -2
  171. package/src/recall/hydeTelemetry.ts +44 -0
  172. package/src/recall/sync.ts +34 -2
  173. package/src/recall/types.ts +36 -0
  174. package/src/store/turns/hydeStore.ts +82 -0
  175. package/src/store/turns/index.ts +4 -0
  176. package/src/store/turns/schema.ts +126 -1
  177. package/src/store/turns/sqlite-store/rows.ts +25 -0
  178. package/src/store/turns/sqlite-store/write.ts +20 -2
  179. package/src/store/turns/types.ts +33 -0
  180. package/src/topics/durability.ts +231 -0
  181. package/src/topics/index.ts +7 -1
  182. package/src/topics/store.ts +35 -9
  183. package/src/wiki/curation-helpers.ts +140 -0
  184. package/src/wiki/curation.ts +217 -0
  185. package/src/wiki/incremental.ts +164 -0
  186. package/src/wiki/index.ts +13 -0
  187. package/extensions/dashboard-client/dist/assets/CacheTab-Cqnc7lMg.js +0 -2
  188. package/extensions/dashboard-client/dist/assets/CacheTab-Cqnc7lMg.js.map +0 -1
  189. package/extensions/dashboard-client/dist/assets/EventsTab-tYk3VFAp.js +0 -2
  190. package/extensions/dashboard-client/dist/assets/EventsTab-tYk3VFAp.js.map +0 -1
  191. package/extensions/dashboard-client/dist/assets/HealthTab-D2_kjgxw.js +0 -2
  192. package/extensions/dashboard-client/dist/assets/HealthTab-D2_kjgxw.js.map +0 -1
  193. package/extensions/dashboard-client/dist/assets/MaintenanceTab-Bc-uPYrz.js +0 -2
  194. package/extensions/dashboard-client/dist/assets/MaintenanceTab-Bc-uPYrz.js.map +0 -1
  195. package/extensions/dashboard-client/dist/assets/MemoryMapTab-D1V9g_NC.js +0 -2
  196. package/extensions/dashboard-client/dist/assets/MemoryMapTab-D1V9g_NC.js.map +0 -1
  197. package/extensions/dashboard-client/dist/assets/MetricsTab-DT_XfL96.js +0 -2
  198. package/extensions/dashboard-client/dist/assets/MetricsTab-DT_XfL96.js.map +0 -1
  199. package/extensions/dashboard-client/dist/assets/OverviewTab-BY6KZCme.js +0 -2
  200. package/extensions/dashboard-client/dist/assets/OverviewTab-BY6KZCme.js.map +0 -1
  201. package/extensions/dashboard-client/dist/assets/ReposTab-CeS-tpDM.js +0 -2
  202. package/extensions/dashboard-client/dist/assets/ReposTab-CeS-tpDM.js.map +0 -1
  203. package/extensions/dashboard-client/dist/assets/SessionsTab-C30AwyoF.js +0 -70
  204. package/extensions/dashboard-client/dist/assets/SetupTab-BJUHOVK2.js +0 -2
  205. package/extensions/dashboard-client/dist/assets/SetupTab-BJUHOVK2.js.map +0 -1
  206. package/extensions/dashboard-client/dist/assets/TimeSavedCard-p55K1tFP.js +0 -2
  207. package/extensions/dashboard-client/dist/assets/TimeSavedCard-p55K1tFP.js.map +0 -1
  208. package/extensions/dashboard-client/dist/assets/TopicsTab-D8FqyAbK.js +0 -2
  209. package/extensions/dashboard-client/dist/assets/TopicsTab-D8FqyAbK.js.map +0 -1
  210. package/extensions/dashboard-client/dist/assets/TurnsTab-ju-nXDys.js +0 -2
  211. package/extensions/dashboard-client/dist/assets/TurnsTab-ju-nXDys.js.map +0 -1
  212. package/extensions/dashboard-client/dist/assets/index-BEj1Zahy.js +0 -42
  213. package/extensions/dashboard-client/dist/assets/index-BEj1Zahy.js.map +0 -1
  214. package/extensions/dashboard-client/dist/assets/index-CUNn22iV.css +0 -1
  215. package/extensions/dashboard-client/src/styles/metrics-extra.css +0 -69
  216. package/extensions/dashboard-client/src/styles/turns.css +0 -142
  217. package/extensions/dashboard-client/src/tabs/SetupTab/RagSettingsCard.tsx +0 -160
@@ -12,6 +12,7 @@ import { normalizeSessionId } from "../store.js";
12
12
  import { cosineSimilarity } from "../embedder.js";
13
13
  import { computeRecallMetrics } from "../recallMetrics.js";
14
14
  import { Logger } from "../log.js";
15
+ import type { RecallMetricsSnapshot } from "./types.js";
15
16
 
16
17
  /** Wrap a recall block so the model reads it as restored compacted context. */
17
18
  export function formatRecallBlock(hits: SearchHit[]): string {
@@ -150,7 +151,10 @@ export function raptorOverviewBlock(
150
151
  * B3: Compute recall-quality metrics on the injected hits and log them.
151
152
  * Only called when RAG_RECALL_METRICS() is ON. Non-fatal.
152
153
  */
153
- export function scoreAndLogRecallMetrics(query: string, toInject: SearchHit[]): void {
154
+ export function scoreAndLogRecallMetrics(
155
+ query: string,
156
+ toInject: SearchHit[],
157
+ ): RecallMetricsSnapshot | null {
154
158
  try {
155
159
  const logger = new Logger();
156
160
  const metrics = computeRecallMetrics(query, toInject);
@@ -167,7 +171,16 @@ export function scoreAndLogRecallMetrics(query: string, toInject: SearchHit[]):
167
171
  diversity: metrics.breakdown.diversity,
168
172
  });
169
173
  }
174
+ return {
175
+ hitCount: toInject.length,
176
+ score: metrics.score,
177
+ pass: metrics.pass,
178
+ relevance: metrics.breakdown.relevance,
179
+ coverage: metrics.breakdown.coverage,
180
+ diversity: metrics.breakdown.diversity,
181
+ specificity: metrics.breakdown.specificity,
182
+ };
170
183
  } catch {
171
- /* non-fatal: metrics never break recall */
184
+ return null; // non-fatal: metrics never break recall
172
185
  }
173
186
  }
@@ -0,0 +1,44 @@
1
+ /**
2
+ * recall/hydeTelemetry.ts — pure builder for HydeInvocationInfo.
3
+ *
4
+ * No I/O, no logging: turns a HyDE outcome + hit counts into the
5
+ * serialization-safe telemetry object surfaced on RecallInjectResult.
6
+ */
7
+ import type { HydeInvocationInfo } from "./types.js";
8
+
9
+ export type HydeOutcome = "ran" | "disabled" | "no-llm" | "generation-failed";
10
+
11
+ export function buildHydeInfo(
12
+ outcome: HydeOutcome,
13
+ hypotheticalDoc: string,
14
+ generationMs: number,
15
+ rawHitCount: number,
16
+ hydeHitCount: number,
17
+ fusedHitCount: number,
18
+ ): HydeInvocationInfo {
19
+ const ran = outcome === "ran";
20
+ const skipped = !ran;
21
+ // Lift = fusedHitCount / max(1, rawHitCount) — how much HyDE changed recall
22
+ // breadth. Skipped passes yield 1 (raw hits cascade unchanged) unless there
23
+ // were no raw hits to compare against.
24
+ const lift = fusedHitCount / Math.max(1, rawHitCount);
25
+ return {
26
+ ran,
27
+ skipped,
28
+ reason: outcome,
29
+ hypotheticalDoc,
30
+ generationMs: ran ? generationMs : 0,
31
+ rawHitCount,
32
+ hydeHitCount: ran ? hydeHitCount : 0,
33
+ fusedHitCount,
34
+ lift: Math.round(lift * 100) / 100,
35
+ };
36
+ }
37
+
38
+ export function hydeSkipped(
39
+ reason: "disabled" | "no-llm" | "generation-failed",
40
+ rawHitCount: number,
41
+ fusedHitCount: number,
42
+ ): HydeInvocationInfo {
43
+ return buildHydeInfo(reason, "", 0, rawHitCount, 0, fusedHitCount);
44
+ }
@@ -19,10 +19,16 @@ import { estimateBlockTokens } from "../tokens.js";
19
19
  import { defaultEmbedder, cosineSimilarity } from "../embedder.js";
20
20
  import { RAG_QUERY_REFORMULATION, RAG_TIERED_ROUTER, RAG_RECALL_METRICS, RAG_HYDE_ENABLED } from "../config.js";
21
21
  import { generateHypotheticalDoc, fuseRecallHits } from "../hyde.js";
22
- import type { RecallInjectOptions, RecallInjectResult } from "./types.js";
22
+ import type {
23
+ RecallInjectOptions,
24
+ RecallInjectResult,
25
+ HydeInvocationInfo,
26
+ RecallMetricsSnapshot,
27
+ } from "./types.js";
23
28
  import type { SearchHit } from "../vectorStore.js";
24
29
  import { reformulateRecallQuery, runTieredRecall } from "./reformulate.js";
25
30
  import { formatRecallBlock, scoreAndLogRecallMetrics, raptorOverviewBlock } from "./format.js";
31
+ import { buildHydeInfo, hydeSkipped } from "./hydeTelemetry.js";
26
32
 
27
33
  /**
28
34
  * Recall and inline context from the checkpoint store.
@@ -94,10 +100,17 @@ export function recallAndInline(
94
100
  // RRF-fuses with the raw-query results. Non-fatal: any error → raw-only.
95
101
  // TrigramEmbedder (kind !== "http") skips this — S57 B1 reformulation above
96
102
  // already handles the no-LLM path.
103
+ // H1 telemetry tracking for the HyDE block below.
97
104
  let hydeHits: SearchHit[] | null = null;
105
+ let hydeDoc = "";
106
+ let hydeGenMs = 0;
107
+ let hydeRawCount = 0;
98
108
  if (RAG_HYDE_ENABLED() && embedder.kind === "http") {
109
+ const t0 = Date.now();
99
110
  const hyde = generateHypotheticalDoc(opts.query, embedder);
111
+ hydeGenMs = Date.now() - t0;
100
112
  if (hyde) {
113
+ hydeDoc = hyde;
101
114
  try {
102
115
  const r2 = searchRecall(
103
116
  { sessionId: opts.sessionId, query: hyde, limit, skipInjected: skip },
@@ -120,6 +133,8 @@ export function recallAndInline(
120
133
  );
121
134
  newHits = result.newHits;
122
135
  }
136
+ // H1: the raw-query hit count is taken BEFORE fusion.
137
+ hydeRawCount = newHits.length;
123
138
 
124
139
  // S43 re-plan: fuse the hypothetical-doc hits with the raw-query hits via RRF
125
140
  // when HyDE produced results. The downstream inline dedupe + metrics + format
@@ -162,7 +177,10 @@ export function recallAndInline(
162
177
  // byte-identical). Scores the injected hits against the ORIGINAL query so the
163
178
  // metric measures whether the (possibly expanded) search results stay
164
179
  // relevant to what the user actually asked.
165
- if (RAG_RECALL_METRICS()) scoreAndLogRecallMetrics(opts.query, toInject);
180
+ let recallMetrics: RecallMetricsSnapshot | null = null;
181
+ if (RAG_RECALL_METRICS()) {
182
+ recallMetrics = scoreAndLogRecallMetrics(opts.query, toInject);
183
+ }
166
184
 
167
185
  // F3: format once — one preamble, correct [1..n] numbering.
168
186
  const recallBlock = toInject.length > 0 ? formatRecallBlock(toInject) : "";
@@ -188,10 +206,24 @@ export function recallAndInline(
188
206
  ? overview + "\n" + recallBlock
189
207
  : overview || recallBlock;
190
208
 
209
+ // H1: build the HyDE invocation telemetry for this pass.
210
+ const fusedCount = toInject.length;
211
+ let hydeInfo: HydeInvocationInfo;
212
+ if (RAG_HYDE_ENABLED() && embedder.kind === "http") {
213
+ hydeInfo =
214
+ hydeHits && hydeHits.length > 0
215
+ ? buildHydeInfo("ran", hydeDoc, hydeGenMs, hydeRawCount, hydeHits.length, fusedCount)
216
+ : hydeSkipped(hydeDoc ? "generation-failed" : "no-llm", hydeRawCount, fusedCount);
217
+ } else {
218
+ hydeInfo = hydeSkipped("disabled", hydeRawCount, fusedCount);
219
+ }
220
+
191
221
  return {
192
222
  toInject,
193
223
  report,
194
224
  block,
195
225
  empty: block.length === 0,
226
+ hydeInfo,
227
+ recallMetrics,
196
228
  };
197
229
  }
@@ -46,6 +46,42 @@ export interface RecallInjectResult {
46
46
  block: string;
47
47
  /** True when nothing new was inlined. */
48
48
  empty: boolean;
49
+ /** H1: HyDE invocation telemetry for this recall pass. Non-null always. */
50
+ hydeInfo: HydeInvocationInfo | null;
51
+ /** H1: recall-quality metrics (only populated when RAG_RECALL_METRICS() is ON). */
52
+ recallMetrics: RecallMetricsSnapshot | null;
53
+ }
54
+
55
+ export interface HydeInvocationInfo {
56
+ /** True when HyDE ran (LLM generated a hypothetical doc + embedded + searched). */
57
+ ran: boolean;
58
+ /** True when HyDE was considered but explicitly skipped (e.g. no LLM surface). */
59
+ skipped: boolean;
60
+ /** Human-readable reason: "ran", "disabled", "no-llm", "generation-failed". */
61
+ reason: string;
62
+ /** The hypothetical answer document the LLM produced ("" when skipped/failed). */
63
+ hypotheticalDoc: string;
64
+ /** Wall-clock ms spent generating + embedding the hypothetical doc (0 when skipped). */
65
+ generationMs: number;
66
+ /** Hit count from the raw-query search before fusion. */
67
+ rawHitCount: number;
68
+ /** Hit count from the hypothetical-doc search before fusion. */
69
+ hydeHitCount: number;
70
+ /** Hit count of the RRF-fused result actually injected (after dedupe+cap). */
71
+ fusedHitCount: number;
72
+ /** Lift = fusedHitCount / max(1, rawHitCount) — how much HyDE changed recall breadth. */
73
+ lift: number;
74
+ }
75
+
76
+ /** Slim, serialization-safe slice of RecallQualityResult for persistence + SSE. */
77
+ export interface RecallMetricsSnapshot {
78
+ hitCount: number;
79
+ score: number;
80
+ pass: boolean;
81
+ relevance: number;
82
+ coverage: number;
83
+ diversity: number;
84
+ specificity: number;
49
85
  }
50
86
 
51
87
  export interface MemoryRecallInjectOptions {
@@ -0,0 +1,82 @@
1
+ /**
2
+ * turns/hydeStore.ts — read helpers over the H1 telemetry columns.
3
+ *
4
+ * Pure read aggregations on the turns table. All queries parameterized
5
+ * (PREVENT-002); no mutation here (turns stays append-only).
6
+ */
7
+ import type { DatabaseSync } from "node:sqlite";
8
+
9
+ export interface TurnTelemetryRow {
10
+ turnId: string;
11
+ conversationId: string;
12
+ turnIndex: number;
13
+ role: string;
14
+ endedAt: number;
15
+ hydeRan: number;
16
+ hydeReason: string;
17
+ hydeDoc: string;
18
+ hydeRawCount: number;
19
+ hydeHydeCount: number;
20
+ hydeFusedCount: number;
21
+ hydeLift: number;
22
+ hydeGenerationMs: number;
23
+ recallScore: number;
24
+ recallPass: number;
25
+ recallRelevance: number;
26
+ recallCoverage: number;
27
+ recallDiversity: number;
28
+ recallSpecificity: number;
29
+ }
30
+
31
+ export function listTelemetryTurns(
32
+ db: DatabaseSync,
33
+ opts: { limit?: number } = {},
34
+ ): TurnTelemetryRow[] {
35
+ const limit = Math.max(1, opts.limit ?? 200);
36
+ return db
37
+ .prepare(
38
+ `SELECT id AS turnId, conversation_id AS conversationId,
39
+ turn_index AS turnIndex, role, ended_at AS endedAt,
40
+ hyde_ran AS hydeRan, hyde_reason AS hydeReason, hyde_doc AS hydeDoc,
41
+ hyde_raw_count AS hydeRawCount, hyde_hyde_count AS hydeHydeCount,
42
+ hyde_fused_count AS hydeFusedCount, hyde_lift AS hydeLift,
43
+ hyde_generation_ms AS hydeGenerationMs,
44
+ recall_score AS recallScore, recall_pass AS recallPass,
45
+ recall_relevance AS recallRelevance, recall_coverage AS recallCoverage,
46
+ recall_diversity AS recallDiversity, recall_specificity AS recallSpecificity
47
+ FROM turns
48
+ WHERE hyde_ran = 1 OR recall_pass IS NOT NULL
49
+ ORDER BY ended_at DESC
50
+ LIMIT ?`,
51
+ )
52
+ .all(limit) as unknown as TurnTelemetryRow[];
53
+ }
54
+
55
+ export interface DailyTelemetry {
56
+ day: string;
57
+ recallCount: number;
58
+ hydeRanCount: number;
59
+ avgScore: number | null;
60
+ avgLift: number | null;
61
+ avgGenMs: number | null;
62
+ }
63
+
64
+ export function aggregateDailyTelemetry(
65
+ db: DatabaseSync,
66
+ days: number,
67
+ ): DailyTelemetry[] {
68
+ const since = Date.now() - days * 86_400_000;
69
+ return db
70
+ .prepare(
71
+ `SELECT date(ended_at / 1000, 'unixepoch') AS day,
72
+ COUNT(*) AS recallCount,
73
+ SUM(hyde_ran) AS hydeRanCount,
74
+ AVG(recall_score) AS avgScore,
75
+ AVG(hyde_lift) AS avgLift,
76
+ AVG(hyde_generation_ms) AS avgGenMs
77
+ FROM turns
78
+ WHERE ended_at >= ? AND (hyde_ran = 1 OR recall_pass IS NOT NULL)
79
+ GROUP BY day ORDER BY day ASC`,
80
+ )
81
+ .all(since) as unknown as DailyTelemetry[];
82
+ }
@@ -15,6 +15,8 @@ export type {
15
15
  SessionId,
16
16
  TurnEntry,
17
17
  TurnRecallEntry,
18
+ TurnHydeTelemetry,
19
+ TurnRecallTelemetry,
18
20
  ConversationFork,
19
21
  TurnFilter,
20
22
  PruneReport,
@@ -46,6 +48,8 @@ export {
46
48
  } from "./connection.js";
47
49
  export { initTurnSchema } from "./schema.js";
48
50
  export { migrateTurnTablesIfNeeded } from "./migrations.js";
51
+ export { listTelemetryTurns, aggregateDailyTelemetry } from "./hydeStore.js";
52
+ export type { TurnTelemetryRow, DailyTelemetry } from "./hydeStore.js";
49
53
 
50
54
  import { SqliteTurnStore } from "./sqlite-store.js";
51
55
  import { InMemoryTurnStore } from "./memory-store.js";
@@ -16,7 +16,7 @@
16
16
  import type { DatabaseSync } from "node:sqlite";
17
17
 
18
18
  /** Schema version stamp (written once into turns_meta). */
19
- const SCHEMA_VERSION = 2;
19
+ const SCHEMA_VERSION = 3;
20
20
 
21
21
  /** Create the unified turn-store schema if absent. Idempotent. */
22
22
  export function initTurnSchema(db: DatabaseSync): void {
@@ -162,6 +162,131 @@ export function initTurnSchema(db: DatabaseSync): void {
162
162
  `CREATE INDEX IF NOT EXISTS idx_memory_topics_topic ON memory_topics(topic_id)`,
163
163
  );
164
164
 
165
+ // H1: additive HyDE + recall-quality telemetry columns. ALTER TABLE ADD
166
+ // COLUMN (idempotent via PRAGMA guard) — turns stays append-only; these land
167
+ // in the single INSERT at turn-record time, never via UPDATE.
168
+ const ensureCol = (name: string, ddl: string): void => {
169
+ const has = (
170
+ db.prepare("PRAGMA table_info(turns)").all() as Array<{ name: string }>
171
+ ).some((c) => c.name === name);
172
+ if (!has) db.exec(`ALTER TABLE turns ADD COLUMN ${ddl}`);
173
+ };
174
+ ensureCol("hyde_ran", "hyde_ran INTEGER DEFAULT 0");
175
+ ensureCol("hyde_doc", "hyde_doc TEXT DEFAULT ''");
176
+ ensureCol("hyde_raw_count", "hyde_raw_count INTEGER DEFAULT 0");
177
+ ensureCol("hyde_hyde_count", "hyde_hyde_count INTEGER DEFAULT 0");
178
+ ensureCol("hyde_fused_count", "hyde_fused_count INTEGER DEFAULT 0");
179
+ ensureCol("hyde_lift", "hyde_lift REAL DEFAULT 0");
180
+ ensureCol("hyde_generation_ms", "hyde_generation_ms INTEGER DEFAULT 0");
181
+ ensureCol("hyde_reason", "hyde_reason TEXT DEFAULT ''");
182
+ ensureCol("recall_score", "recall_score REAL DEFAULT 0");
183
+ ensureCol("recall_pass", "recall_pass INTEGER DEFAULT 0");
184
+ ensureCol("recall_relevance", "recall_relevance REAL DEFAULT 0");
185
+ ensureCol("recall_coverage", "recall_coverage REAL DEFAULT 0");
186
+ ensureCol("recall_diversity", "recall_diversity REAL DEFAULT 0");
187
+ ensureCol("recall_specificity", "recall_specificity REAL DEFAULT 0");
188
+
189
+ // ── Wiki Revival (Spec 3): user curation + topic evolution ────────
190
+ const sessionCol = (
191
+ db.prepare("PRAGMA table_info(memory_topics)").all() as Array<{ name: string }>
192
+ ).some((c) => c.name === "session_id");
193
+ if (!sessionCol) {
194
+ db.exec("ALTER TABLE memory_topics ADD COLUMN session_id TEXT");
195
+ }
196
+ db.exec(
197
+ `CREATE INDEX IF NOT EXISTS idx_memory_topics_session ON memory_topics(session_id)`,
198
+ );
199
+
200
+ db.exec(`
201
+ CREATE TABLE IF NOT EXISTS topic_overrides (
202
+ topic_id TEXT NOT NULL,
203
+ kind TEXT NOT NULL CHECK(kind IN ('label','merge','split')),
204
+ custom_label TEXT,
205
+ merged_into TEXT,
206
+ split_from TEXT,
207
+ split_memory_ids TEXT,
208
+ merged_memory_ids TEXT,
209
+ overridden_at INTEGER NOT NULL,
210
+ PRIMARY KEY (topic_id, kind)
211
+ )
212
+ `);
213
+ db.exec(
214
+ `CREATE INDEX IF NOT EXISTS idx_topic_overrides_topic ON topic_overrides(topic_id)`,
215
+ );
216
+
217
+ // W5: add merged_memory_ids column to existing topic_overrides tables (same
218
+ // additive ensureColumn pattern as the HyDE columns). Stores the member ids
219
+ // from the source topic at merge time so merge overrides survive a full
220
+ // replaceTopicModel rebuild (topic_evolution rows are CASCADE-deleted with
221
+ // their parent topic, but topic_overrides rows persist across rebuilds).
222
+ const overridesCols = db
223
+ .prepare("PRAGMA table_info(topic_overrides)")
224
+ .all() as Array<{ name: string }>;
225
+ if (
226
+ overridesCols.length > 0 &&
227
+ !overridesCols.some((c) => c.name === "merged_memory_ids")
228
+ ) {
229
+ db.exec("ALTER TABLE topic_overrides ADD COLUMN merged_memory_ids TEXT");
230
+ }
231
+
232
+ db.exec(`
233
+ CREATE TABLE IF NOT EXISTS topic_evolution (
234
+ topic_id TEXT NOT NULL REFERENCES topics(id) ON DELETE CASCADE,
235
+ memory_id TEXT NOT NULL,
236
+ session_id TEXT,
237
+ assigned_at INTEGER NOT NULL,
238
+ method TEXT NOT NULL CHECK(method IN ('kmeans+tfidf','merge','split','manual')),
239
+ PRIMARY KEY (topic_id, memory_id)
240
+ )
241
+ `);
242
+ db.exec(
243
+ `CREATE INDEX IF NOT EXISTS idx_topic_evolution_topic ON topic_evolution(topic_id, assigned_at)`,
244
+ );
245
+ db.exec(
246
+ `CREATE INDEX IF NOT EXISTS idx_topic_evolution_ts ON topic_evolution(assigned_at)`,
247
+ );
248
+
249
+ // Migration: topic_evolution.topic_id FK lacked ON DELETE CASCADE when the
250
+ // table was first created (W1). Without it, deleting a topic (merge source or
251
+ // replaceTopicModel rebuild) throws SQLITE_CONSTRAINT when evolution rows
252
+ // reference it, rolling back the delete. Recreate the table with CASCADE when
253
+ // the live FK definition is missing it. Non-fatal: a failure just leaves the
254
+ // old definition (rebuilds remain best-effort).
255
+ try {
256
+ // PRAGMA foreign_key_list exposes the referenced table + on-delete rule.
257
+ const fkList = db
258
+ .prepare("PRAGMA foreign_key_list(topic_evolution)")
259
+ .all() as Array<{ table: string; from: string; on_delete: string }>;
260
+ const topicFk = fkList.find((f) => f.from === "topic_id");
261
+ if (topicFk && topicFk.on_delete !== "CASCADE") {
262
+ db.exec("DROP TABLE IF EXISTS topic_evolution_mig");
263
+ db.exec(`
264
+ CREATE TABLE topic_evolution_mig (
265
+ topic_id TEXT NOT NULL REFERENCES topics(id) ON DELETE CASCADE,
266
+ memory_id TEXT NOT NULL,
267
+ session_id TEXT,
268
+ assigned_at INTEGER NOT NULL,
269
+ method TEXT NOT NULL CHECK(method IN ('kmeans+tfidf','merge','split','manual')),
270
+ PRIMARY KEY (topic_id, memory_id)
271
+ )
272
+ `);
273
+ db.exec(
274
+ `INSERT INTO topic_evolution_mig (topic_id, memory_id, session_id, assigned_at, method)
275
+ SELECT topic_id, memory_id, session_id, assigned_at, method FROM topic_evolution`,
276
+ );
277
+ db.exec("DROP TABLE topic_evolution");
278
+ db.exec("ALTER TABLE topic_evolution_mig RENAME TO topic_evolution");
279
+ db.exec(
280
+ `CREATE INDEX IF NOT EXISTS idx_topic_evolution_topic ON topic_evolution(topic_id, assigned_at)`,
281
+ );
282
+ db.exec(
283
+ `CREATE INDEX IF NOT EXISTS idx_topic_evolution_ts ON topic_evolution(assigned_at)`,
284
+ );
285
+ }
286
+ } catch {
287
+ /* non-fatal: migration never breaks turn-DB open */
288
+ }
289
+
165
290
  // Stamp schema version once.
166
291
  const existing = db
167
292
  .prepare("SELECT value FROM turns_meta WHERE key = 'schema_version'")
@@ -32,6 +32,31 @@ export function rowToEntry(r: Record<string, unknown>): TurnEntry {
32
32
  (r.pressure_band as "green" | "yellow" | "red" | null) ?? undefined,
33
33
  model: (r.model as string | null) ?? undefined,
34
34
  epochId: (r.epoch_id as string | null) ?? undefined,
35
+ // H1: telemetry columns (only populated when the row has them non-default).
36
+ hyde:
37
+ r.hyde_ran == null
38
+ ? undefined
39
+ : {
40
+ ran: r.hyde_ran === 1,
41
+ reason: (r.hyde_reason as string) ?? "",
42
+ hypotheticalDoc: (r.hyde_doc as string) ?? "",
43
+ rawHitCount: (r.hyde_raw_count as number) ?? 0,
44
+ hydeHitCount: (r.hyde_hyde_count as number) ?? 0,
45
+ fusedHitCount: (r.hyde_fused_count as number) ?? 0,
46
+ lift: (r.hyde_lift as number) ?? 0,
47
+ generationMs: (r.hyde_generation_ms as number) ?? 0,
48
+ },
49
+ recallMetrics:
50
+ r.recall_score == null
51
+ ? undefined
52
+ : {
53
+ score: (r.recall_score as number) ?? 0,
54
+ pass: (r.recall_pass as number) === 1,
55
+ relevance: (r.recall_relevance as number) ?? 0,
56
+ coverage: (r.recall_coverage as number) ?? 0,
57
+ diversity: (r.recall_diversity as number) ?? 0,
58
+ specificity: (r.recall_specificity as number) ?? 0,
59
+ },
35
60
  };
36
61
  }
37
62
 
@@ -22,8 +22,12 @@ export function appendTurn(ctx: SqliteTurnStoreCtx, entry: TurnEntry): TurnId {
22
22
  db
23
23
  .prepare(
24
24
  `INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
25
- ctx_tokens, ctx_percent, pressure_band, model, epoch_id)
26
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
25
+ ctx_tokens, ctx_percent, pressure_band, model, epoch_id,
26
+ hyde_ran, hyde_doc, hyde_raw_count, hyde_hyde_count,
27
+ hyde_fused_count, hyde_lift, hyde_generation_ms, hyde_reason,
28
+ recall_score, recall_pass, recall_relevance, recall_coverage,
29
+ recall_diversity, recall_specificity)
30
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
27
31
  )
28
32
  .run(
29
33
  entry.conversationId,
@@ -36,6 +40,20 @@ export function appendTurn(ctx: SqliteTurnStoreCtx, entry: TurnEntry): TurnId {
36
40
  entry.pressureBand ?? null,
37
41
  entry.model ?? null,
38
42
  entry.epochId ?? null,
43
+ entry.hyde?.ran == null ? null : entry.hyde.ran ? 1 : 0,
44
+ entry.hyde?.hypotheticalDoc ?? null,
45
+ entry.hyde?.rawHitCount ?? null,
46
+ entry.hyde?.hydeHitCount ?? null,
47
+ entry.hyde?.fusedHitCount ?? null,
48
+ entry.hyde?.lift ?? null,
49
+ entry.hyde?.generationMs ?? null,
50
+ entry.hyde?.reason ?? null,
51
+ entry.recallMetrics?.score ?? null,
52
+ entry.recallMetrics?.pass == null ? null : entry.recallMetrics.pass ? 1 : 0,
53
+ entry.recallMetrics?.relevance ?? null,
54
+ entry.recallMetrics?.coverage ?? null,
55
+ entry.recallMetrics?.diversity ?? null,
56
+ entry.recallMetrics?.specificity ?? null,
39
57
  );
40
58
  } catch (e) {
41
59
  if (
@@ -59,6 +59,39 @@ export interface TurnEntry {
59
59
  model?: string; // model used for this turn
60
60
  /** S50B: the compact epoch that superseded this turn (stamped post-hoc). */
61
61
  epochId?: string;
62
+ /** H1: HyDE invocation telemetry for this turn (persisted to hyde_* columns). */
63
+ hyde?: TurnHydeTelemetry;
64
+ /** H1: recall-quality snapshot for this turn (persisted to recall_* columns). */
65
+ recallMetrics?: TurnRecallTelemetry;
66
+ }
67
+
68
+ /**
69
+ * H1: HyDE telemetry for a turn — a structural subset of the recall-core
70
+ * `HydeInvocationInfo` that maps onto the turns `hyde_*` columns.
71
+ */
72
+ export interface TurnHydeTelemetry {
73
+ ran: boolean;
74
+ /** Skip reason when HyDE didn't run: "disabled" | "no-llm" | "generation-failed". */
75
+ reason: string;
76
+ hypotheticalDoc: string;
77
+ rawHitCount: number;
78
+ hydeHitCount: number;
79
+ fusedHitCount: number;
80
+ lift: number;
81
+ generationMs: number;
82
+ }
83
+
84
+ /**
85
+ * H1: recall-quality snapshot for a turn — a structural subset of the recall-core
86
+ * `RecallMetricsSnapshot` that maps onto the turns `recall_*` columns.
87
+ */
88
+ export interface TurnRecallTelemetry {
89
+ score: number;
90
+ pass: boolean;
91
+ relevance: number;
92
+ coverage: number;
93
+ diversity: number;
94
+ specificity: number;
62
95
  }
63
96
 
64
97
  /** A recall hit recorded during a turn — an immutable, append-only fact. */