pi-mega-compact 0.11.5 → 0.11.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/dist/extensions/dashboard-server/api-contracts/cache-stripes.js +8 -0
  2. package/dist/extensions/dashboard-server/api-contracts/endpoints.js +13 -0
  3. package/dist/extensions/dashboard-server/api-contracts/memory-map.js +1 -0
  4. package/dist/extensions/dashboard-server/routes-cache.js +28 -0
  5. package/dist/extensions/dashboard-server/routes-game.js +3 -0
  6. package/dist/extensions/dashboard-server/routes-memory-map.js +50 -0
  7. package/dist/extensions/dashboard-server/routes-setup.js +105 -0
  8. package/dist/extensions/dashboard-server/routes.js +3 -1
  9. package/dist/extensions/dashboard-server/server.js +7 -1
  10. package/dist/extensions/mega-compact.js +5 -0
  11. package/dist/extensions/mega-config.js +5 -0
  12. package/dist/extensions/mega-events/context-handler.js +10 -2
  13. package/dist/extensions/mega-events/perf-handler.js +83 -1
  14. package/dist/extensions/mega-events/separated-prompt.js +248 -0
  15. package/dist/extensions/mega-runtime/env-loader.js +56 -0
  16. package/dist/extensions/mega-runtime/reset-runtime.js +1 -0
  17. package/dist/extensions/mega-runtime/runtime-snapshot.js +1 -0
  18. package/dist/extensions/mega-runtime/runtime.js +1 -0
  19. package/dist/extensions/mega-runtime/snapshot.js +1 -0
  20. package/dist/extensions/mega-runtime/widget.js +5 -0
  21. package/dist/src/cache-stripe-impl.js +288 -0
  22. package/dist/src/cache-stripe.js +11 -0
  23. package/dist/src/config/stopwords.js +24 -0
  24. package/dist/src/config.js +17 -0
  25. package/dist/src/memoryGraph.js +299 -0
  26. package/dist/src/monitoring.js +13 -0
  27. package/dist/src/queryExpansion.js +163 -0
  28. package/dist/src/queryReformulation/cache.js +123 -0
  29. package/dist/src/queryReformulation/rrf.js +29 -0
  30. package/dist/src/queryReformulation/tfidf.js +61 -0
  31. package/dist/src/queryReformulation/vagueness.js +33 -0
  32. package/dist/src/queryReformulation.js +225 -0
  33. package/dist/src/recall.js +136 -1
  34. package/dist/src/recallMetrics.js +258 -0
  35. package/dist/src/store/sqlite/cache-stripes.js +133 -0
  36. package/dist/src/store/sqlite/fts5-search.js +68 -0
  37. package/dist/src/store/sqlite/perf-samples.js +1 -0
  38. package/dist/src/store/sqlite/schema.js +60 -1
  39. package/dist/src/store/sqlite.js +1 -0
  40. package/dist/src/store/vectorIndex.js +29 -5
  41. package/dist/src/tiered-router/types.js +18 -0
  42. package/dist/src/tieredRouter.js +421 -0
  43. package/dist/src/vectorStore.js +8 -6
  44. package/extensions/dashboard-client/dist/assets/{AchievementTiles-BoL7qBOJ.js → AchievementTiles-DR9cLGDL.js} +2 -2
  45. package/extensions/dashboard-client/dist/assets/{AchievementTiles-BoL7qBOJ.js.map → AchievementTiles-DR9cLGDL.js.map} +1 -1
  46. package/extensions/dashboard-client/dist/assets/{AchievementsTab-BFDE3HYc.js → AchievementsTab-svNUgJ1e.js} +2 -2
  47. package/extensions/dashboard-client/dist/assets/{AchievementsTab-BFDE3HYc.js.map → AchievementsTab-svNUgJ1e.js.map} +1 -1
  48. package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js +2 -0
  49. package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js.map +1 -0
  50. package/extensions/dashboard-client/dist/assets/{ConfigTab-DDFE-ckA.js → ConfigTab-CmmWheuW.js} +2 -2
  51. package/extensions/dashboard-client/dist/assets/{ConfigTab-DDFE-ckA.js.map → ConfigTab-CmmWheuW.js.map} +1 -1
  52. package/extensions/dashboard-client/dist/assets/{EventsTab-B6UxZ7PM.js → EventsTab-CFE7jLsa.js} +2 -2
  53. package/extensions/dashboard-client/dist/assets/{EventsTab-B6UxZ7PM.js.map → EventsTab-CFE7jLsa.js.map} +1 -1
  54. package/extensions/dashboard-client/dist/assets/GameTab-CJAX_th_.js +2 -0
  55. package/extensions/dashboard-client/dist/assets/{GameTab-DehUX-KE.js.map → GameTab-CJAX_th_.js.map} +1 -1
  56. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-0GWRt4wn.js → MaintenanceTab-P4SUsvMf.js} +2 -2
  57. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-0GWRt4wn.js.map → MaintenanceTab-P4SUsvMf.js.map} +1 -1
  58. package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js +2 -0
  59. package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js.map +1 -0
  60. package/extensions/dashboard-client/dist/assets/{MetricsTab-C3rwMVsN.js → MetricsTab-CHQscixy.js} +2 -2
  61. package/extensions/dashboard-client/dist/assets/{MetricsTab-C3rwMVsN.js.map → MetricsTab-CHQscixy.js.map} +1 -1
  62. package/extensions/dashboard-client/dist/assets/{OverviewTab-39O9_x5j.js → OverviewTab-DBs4DJdd.js} +2 -2
  63. package/extensions/dashboard-client/dist/assets/{OverviewTab-39O9_x5j.js.map → OverviewTab-DBs4DJdd.js.map} +1 -1
  64. package/extensions/dashboard-client/dist/assets/{ReposTab-Cfj6Q5sH.js → ReposTab-DeFz76L5.js} +2 -2
  65. package/extensions/dashboard-client/dist/assets/{ReposTab-Cfj6Q5sH.js.map → ReposTab-DeFz76L5.js.map} +1 -1
  66. package/extensions/dashboard-client/dist/assets/{SessionsTab-DXDTCi3r.js → SessionsTab-BWNsj1OB.js} +2 -2
  67. package/extensions/dashboard-client/dist/assets/{SessionsTab-DXDTCi3r.js.map → SessionsTab-BWNsj1OB.js.map} +1 -1
  68. package/extensions/dashboard-client/dist/assets/SetupTab-uS6A0xmw.js +2 -0
  69. package/extensions/dashboard-client/dist/assets/SetupTab-uS6A0xmw.js.map +1 -0
  70. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Cw1dJ4e9.js → TimeSavedCard-CiqCOQuL.js} +2 -2
  71. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Cw1dJ4e9.js.map → TimeSavedCard-CiqCOQuL.js.map} +1 -1
  72. package/extensions/dashboard-client/dist/assets/{TopicsTab-BAw972BV.js → TopicsTab-BWWQxltv.js} +2 -2
  73. package/extensions/dashboard-client/dist/assets/{TopicsTab-BAw972BV.js.map → TopicsTab-BWWQxltv.js.map} +1 -1
  74. package/extensions/dashboard-client/dist/assets/{TurnsTab-Cf5mGIdl.js → TurnsTab-CVz4f4-o.js} +2 -2
  75. package/extensions/dashboard-client/dist/assets/{TurnsTab-Cf5mGIdl.js.map → TurnsTab-CVz4f4-o.js.map} +1 -1
  76. package/extensions/dashboard-client/dist/assets/index-CUNn22iV.css +1 -0
  77. package/extensions/dashboard-client/dist/assets/{index-4gQ7auad.js → index-DrktNOoT.js} +11 -11
  78. package/extensions/dashboard-client/dist/assets/index-DrktNOoT.js.map +1 -0
  79. package/extensions/dashboard-client/dist/assets/{useSSE-DNgMuyP6.js → useSSE-Bf5LOLSi.js} +2 -2
  80. package/extensions/dashboard-client/dist/assets/{useSSE-DNgMuyP6.js.map → useSSE-Bf5LOLSi.js.map} +1 -1
  81. package/extensions/dashboard-client/dist/index.html +2 -2
  82. package/extensions/dashboard-client/src/App.tsx +24 -8
  83. package/extensions/dashboard-client/src/api/client.ts +15 -0
  84. package/extensions/dashboard-client/src/components/CacheHitRateTrendCard.tsx +100 -0
  85. package/extensions/dashboard-client/src/components/StripeDistributionCard.tsx +110 -0
  86. package/extensions/dashboard-client/src/components/TabBar.tsx +49 -4
  87. package/extensions/dashboard-client/src/main.tsx +1 -0
  88. package/extensions/dashboard-client/src/styles/base.css +30 -0
  89. package/extensions/dashboard-client/src/styles/cache.css +167 -0
  90. package/extensions/dashboard-client/src/tabs/CacheTab.tsx +75 -13
  91. package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +440 -0
  92. package/extensions/dashboard-client/src/tabs/SetupTab.tsx +65 -2
  93. package/extensions/dashboard-server/api-contracts/cache-stripes.ts +49 -0
  94. package/extensions/dashboard-server/api-contracts/endpoints.ts +31 -1
  95. package/extensions/dashboard-server/api-contracts/index.ts +16 -1
  96. package/extensions/dashboard-server/api-contracts/memory-map.ts +50 -0
  97. package/extensions/dashboard-server/api-contracts/setup.ts +22 -0
  98. package/extensions/dashboard-server/routes-cache.ts +36 -1
  99. package/extensions/dashboard-server/routes-game.ts +3 -0
  100. package/extensions/dashboard-server/routes-memory-map.ts +105 -0
  101. package/extensions/dashboard-server/routes-setup.ts +112 -0
  102. package/extensions/dashboard-server/routes.ts +3 -1
  103. package/extensions/dashboard-server/server.ts +6 -0
  104. package/extensions/mega-compact.ts +5 -0
  105. package/extensions/mega-config.ts +17 -0
  106. package/extensions/mega-events/context-handler.ts +10 -2
  107. package/extensions/mega-events/perf-handler.ts +111 -1
  108. package/extensions/mega-events/separated-prompt.ts +329 -0
  109. package/extensions/mega-runtime/env-loader.ts +55 -0
  110. package/extensions/mega-runtime/helpers.ts +2 -0
  111. package/extensions/mega-runtime/reset-runtime.ts +1 -0
  112. package/extensions/mega-runtime/runtime-snapshot.ts +1 -0
  113. package/extensions/mega-runtime/runtime.ts +1 -0
  114. package/extensions/mega-runtime/snapshot.ts +6 -0
  115. package/extensions/mega-runtime/widget-types.ts +3 -0
  116. package/extensions/mega-runtime/widget.ts +5 -0
  117. package/package.json +1 -1
  118. package/src/cache-stripe-impl.ts +385 -0
  119. package/src/cache-stripe.ts +18 -0
  120. package/src/config/stopwords.ts +25 -0
  121. package/src/config.ts +26 -0
  122. package/src/memoryGraph.ts +438 -0
  123. package/src/monitoring.ts +35 -0
  124. package/src/queryExpansion.ts +206 -0
  125. package/src/queryReformulation/cache.ts +169 -0
  126. package/src/queryReformulation/rrf.ts +51 -0
  127. package/src/queryReformulation/tfidf.ts +79 -0
  128. package/src/queryReformulation/vagueness.ts +38 -0
  129. package/src/queryReformulation.ts +355 -0
  130. package/src/recall.ts +168 -4
  131. package/src/recallMetrics.ts +388 -0
  132. package/src/store/sqlite/cache-stripes.ts +196 -0
  133. package/src/store/sqlite/fts5-search.ts +90 -0
  134. package/src/store/sqlite/perf-samples.ts +3 -1
  135. package/src/store/sqlite/schema.ts +60 -1
  136. package/src/store/sqlite.ts +1 -0
  137. package/src/store/vectorIndex.ts +38 -7
  138. package/src/tiered-router/types.ts +64 -0
  139. package/src/tieredRouter.ts +489 -0
  140. package/src/vectorStore.ts +9 -5
  141. package/extensions/dashboard-client/dist/assets/CacheTab-eKVc-NV1.js +0 -2
  142. package/extensions/dashboard-client/dist/assets/CacheTab-eKVc-NV1.js.map +0 -1
  143. package/extensions/dashboard-client/dist/assets/GameTab-DehUX-KE.js +0 -2
  144. package/extensions/dashboard-client/dist/assets/SetupTab-mMlfunJa.js +0 -2
  145. package/extensions/dashboard-client/dist/assets/SetupTab-mMlfunJa.js.map +0 -1
  146. package/extensions/dashboard-client/dist/assets/index-4gQ7auad.js.map +0 -1
  147. package/extensions/dashboard-client/dist/assets/index-BEdKlpAB.css +0 -1
@@ -0,0 +1,258 @@
1
+ /**
2
+ * recallMetrics.ts — S45 CRAG quality metrics for recall evaluation.
3
+ *
4
+ * Four metrics evaluate the quality of a set of recall hits:
5
+ * - relevance: average cosine similarity (SearchHit.score) across hits
6
+ * - coverage: fraction of query sub-terms found in at least one hit summary
7
+ * - diversity: average pairwise cosine distance among top-K embeddings (MMR-style)
8
+ * - specificity: inverse document-frequency proxy via average chunk length
9
+ *
10
+ * Pi-agnostic: no pi runtime types, no network calls (PREVENT-PI-004).
11
+ * All config is passed explicitly; defaults are documented but NOT hardcoded
12
+ * inside the metric functions (the four constants from S45A.5 are also received
13
+ * via config, not baked in).
14
+ *
15
+ * Best-effort: all functions handle degenerate inputs (empty hits, missing
16
+ * embeddings, zero query terms) and return sensible defaults instead of NaN.
17
+ */
18
+ import { cosineSimilarity } from "./embedder.js";
19
+ import { STOP_WORDS } from "./config/stopwords.js";
20
+ /** Default config — all values marked "uncalibrated" unless CRAG_CALIBRATED=true. */
21
+ export const DEFAULT_RECALL_QUALITY_CONFIG = {
22
+ minDiversity: 0.3,
23
+ minCoverage: 0.4,
24
+ minRelevance: 0.5,
25
+ minOverallScore: 0.4,
26
+ weights: { relevance: 0.35, coverage: 0.25, diversity: 0.25, specificity: 0.15 },
27
+ specificityDivisor: 300,
28
+ specificityOptimalMin: 100,
29
+ specificityOptimalMax: 500,
30
+ idfBroadenRatio: 0.5,
31
+ stopwords: STOP_WORDS,
32
+ calibrated: false,
33
+ };
34
+ // ---------------------------------------------------------------------------
35
+ // Metric helpers (exported individually for unit testing)
36
+ // ---------------------------------------------------------------------------
37
+ /**
38
+ * S45A.2: Relevance — average of SearchHit.score across all hits.
39
+ * Scores are already cosine similarities in [0, 1].
40
+ * Empty hits → 0.
41
+ */
42
+ export function computeRelevance(hits) {
43
+ if (hits.length === 0)
44
+ return 0;
45
+ let sum = 0;
46
+ for (const h of hits)
47
+ sum += h.score;
48
+ return sum / hits.length;
49
+ }
50
+ /**
51
+ * S45A.3: Coverage — fraction of query subterms present in at least one
52
+ * hit's `checkpoint.summary`. Tokenization removes stopwords and short terms.
53
+ * Zero query terms after filtering → coverage = 1.0 (trivially satisfied).
54
+ */
55
+ export function computeCoverage(query, hits, stopwords = STOP_WORDS) {
56
+ const queryTerms = tokenizeTerms(query, stopwords);
57
+ if (queryTerms.length === 0)
58
+ return 1.0;
59
+ if (hits.length === 0)
60
+ return 0;
61
+ let foundCount = 0;
62
+ for (const qt of queryTerms) {
63
+ const found = hits.some((h) => {
64
+ const summary = h.checkpoint.summary ?? "";
65
+ return summary.toLowerCase().includes(qt);
66
+ });
67
+ if (found)
68
+ foundCount++;
69
+ }
70
+ return foundCount / queryTerms.length;
71
+ }
72
+ /**
73
+ * S45A.4: Diversity — average pairwise cosine distance (1 - similarity)
74
+ * among hit embeddings. Filtered to hits with non-empty embedding arrays.
75
+ *
76
+ * Edge cases:
77
+ * - < 2 valid hits → 1.0 (nothing to compare)
78
+ * - empty-embedding checkpoints are skipped (no NaN path)
79
+ */
80
+ export function computeDiversity(hits) {
81
+ const valid = hits.filter((h) => Array.isArray(h.checkpoint.embedding) && h.checkpoint.embedding.length > 0);
82
+ if (valid.length < 2)
83
+ return 1.0;
84
+ let totalDist = 0;
85
+ let pairCount = 0;
86
+ for (let i = 0; i < valid.length; i++) {
87
+ for (let j = i + 1; j < valid.length; j++) {
88
+ const sim = cosineSimilarity(valid[i].checkpoint.embedding, valid[j].checkpoint.embedding);
89
+ // cosineSimilarity returns 0 for empty vectors, but we've filtered those out.
90
+ // NaN/Infinity could theoretically arise from degenerate floats; guard defensively.
91
+ const dist = Number.isFinite(sim) ? 1 - sim : 1.0;
92
+ totalDist += dist;
93
+ pairCount++;
94
+ }
95
+ }
96
+ return pairCount > 0 ? totalDist / pairCount : 1.0;
97
+ }
98
+ /**
99
+ * S45A.5: Specificity — average chunk length (tokenEstimate) as a proxy
100
+ * for information density.
101
+ *
102
+ * Formula (all constants from config, NOT hardcoded):
103
+ * avg = mean(tokenEstimate across hits; hits missing tokenEstimate → 0)
104
+ * If avg < optimalMin / 2 → 0 (too short to be specific)
105
+ * base = avg / divisor
106
+ * If avg > optimalMax:
107
+ * ramp = 1 - (avg - optimalMax) / optimalMax // linear to 0 at 2*optimalMax
108
+ * capped = max(0, ramp)
109
+ * specificity = min(base, capped)
110
+ * else:
111
+ * specificity = min(base, 1)
112
+ *
113
+ * Edge cases: empty hits → 1.0 (nothing to penalize).
114
+ */
115
+ export function computeSpecificity(hits, config) {
116
+ if (hits.length === 0)
117
+ return 1.0;
118
+ const { specificityDivisor: divisor, specificityOptimalMin: optMin, specificityOptimalMax: optMax } = config;
119
+ let sum = 0;
120
+ for (const h of hits) {
121
+ sum += h.checkpoint.tokenEstimate ?? 0;
122
+ }
123
+ const avg = sum / hits.length;
124
+ // Below optMin/2 → too short to be specific
125
+ if (avg < optMin / 2)
126
+ return 0;
127
+ const base = divisor > 0 ? avg / divisor : 0;
128
+ const clamped = base > 1 ? 1 : base;
129
+ if (avg > optMax) {
130
+ // Linear ramp-down from optMax to 0 at 2*optMax
131
+ const ramp = optMax > 0 ? 1 - (avg - optMax) / optMax : 0;
132
+ const capped = ramp > 0 ? ramp : 0;
133
+ return clamped < capped ? clamped : capped;
134
+ }
135
+ return clamped;
136
+ }
137
+ // ---------------------------------------------------------------------------
138
+ // Composite evaluation
139
+ // ---------------------------------------------------------------------------
140
+ /**
141
+ * Normalize a four-weight vector so its components sum to 1.0.
142
+ * If all weights are 0, return equal weights (0.25 each).
143
+ * Used defensively since users may set non-sum-1 weights via env.
144
+ */
145
+ export function normalizeWeights(w) {
146
+ const sum = w.relevance + w.coverage + w.diversity + w.specificity;
147
+ if (sum === 0)
148
+ return { relevance: 0.25, coverage: 0.25, diversity: 0.25, specificity: 0.25 };
149
+ return {
150
+ relevance: w.relevance / sum,
151
+ coverage: w.coverage / sum,
152
+ diversity: w.diversity / sum,
153
+ specificity: w.specificity / sum,
154
+ };
155
+ }
156
+ /**
157
+ * S45A.6: Evaluate recall quality — compute all four metrics and return a
158
+ * composite result with pass/fail + recommendation.
159
+ */
160
+ export function evaluateRecall(query, hits, config) {
161
+ const weights = normalizeWeights(config.weights);
162
+ const breakdown = {
163
+ relevance: computeRelevance(hits),
164
+ coverage: computeCoverage(query, hits, config.stopwords),
165
+ diversity: computeDiversity(hits),
166
+ specificity: computeSpecificity(hits, config),
167
+ };
168
+ const score = weights.relevance * breakdown.relevance +
169
+ weights.coverage * breakdown.coverage +
170
+ weights.diversity * breakdown.diversity +
171
+ weights.specificity * breakdown.specificity;
172
+ const pass = score >= config.minOverallScore &&
173
+ breakdown.relevance >= config.minRelevance &&
174
+ breakdown.coverage >= config.minCoverage &&
175
+ breakdown.diversity >= config.minDiversity;
176
+ const recommendation = buildRecommendation(breakdown, score, config);
177
+ return {
178
+ pass,
179
+ score,
180
+ breakdown,
181
+ recommendation,
182
+ uncalibrated: !config.calibrated,
183
+ weights,
184
+ };
185
+ }
186
+ // ---------------------------------------------------------------------------
187
+ // Recommendation builder
188
+ // ---------------------------------------------------------------------------
189
+ /**
190
+ * Build a human-readable recommendation based on which metric is lowest.
191
+ * Uses env-overridable thresholds from config (not hardcoded).
192
+ */
193
+ function buildRecommendation(breakdown, score, config) {
194
+ // Check each metric against its threshold (lowest first for specificity).
195
+ const issues = [];
196
+ if (breakdown.diversity < config.minDiversity) {
197
+ issues.push("chunks are too similar — consider expanding K or using MMR");
198
+ }
199
+ if (breakdown.coverage < config.minCoverage) {
200
+ issues.push("query terms not well covered — consider query expansion");
201
+ }
202
+ if (breakdown.relevance < config.minRelevance) {
203
+ issues.push("chunks may be irrelevant — consider stricter threshold");
204
+ }
205
+ if (score < config.minOverallScore) {
206
+ issues.push("quality is low — recommend re-retrieval with expanded query");
207
+ }
208
+ return issues.length > 0 ? issues.join("; ") : null;
209
+ }
210
+ /**
211
+ * Compute recall quality metrics for a set of search hits.
212
+ *
213
+ * Pi-agnostic: takes query text and SearchHit results directly, no pi types.
214
+ * The `config` parameter is optional — when omitted, DEFAULT_RECALL_QUALITY_CONFIG
215
+ * is used (labelled uncalibrated).
216
+ *
217
+ * This is the primary export for external callers (e.g., recall.ts integration).
218
+ */
219
+ export function computeRecallMetrics(query, results, config) {
220
+ const effectiveConfig = {
221
+ ...DEFAULT_RECALL_QUALITY_CONFIG,
222
+ ...config,
223
+ // Merge weights shallowly if both sides exist
224
+ weights: {
225
+ ...DEFAULT_RECALL_QUALITY_CONFIG.weights,
226
+ ...(config?.weights ?? {}),
227
+ },
228
+ };
229
+ const result = evaluateRecall(query, results, effectiveConfig);
230
+ return {
231
+ breakdown: result.breakdown,
232
+ score: result.score,
233
+ pass: result.pass,
234
+ recommendation: result.recommendation,
235
+ uncalibrated: result.uncalibrated,
236
+ };
237
+ }
238
+ // ---------------------------------------------------------------------------
239
+ // Internal helpers
240
+ // ---------------------------------------------------------------------------
241
+ /**
242
+ * Tokenize text into lowercased alphanumeric terms, filtering stop words
243
+ * and terms shorter than 3 characters.
244
+ */
245
+ function tokenizeTerms(text, stopwords) {
246
+ return text
247
+ .toLowerCase()
248
+ .split(/[^a-z0-9]+/)
249
+ .filter((t) => t.length >= 3 && !stopwords.has(t));
250
+ }
251
+ /**
252
+ * Estimate block tokens for a piece of text (rough approximation).
253
+ * Used by the specificity metric fallback when tokenEstimate is missing.
254
+ */
255
+ export function estimateBlockTokens(text) {
256
+ // ~4 chars per token for English text
257
+ return Math.ceil(text.length / 4);
258
+ }
@@ -0,0 +1,133 @@
1
+ /**
2
+ * src/store/sqlite/cache-stripes.ts — Cache stripe distribution reader (A3, PLAN_V2 Phase 4).
3
+ *
4
+ * Reads from the cache_stripes table (schema.ts) to produce stripe distribution
5
+ * stats, stability aggregates, and a composite health score. Pi-agnostic.
6
+ *
7
+ * PREVENT-002: all queries use parameterized SQL.
8
+ */
9
+ import { getStateDir } from "../../store.js";
10
+ import { openStore } from "./utils.js";
11
+ /** Human-readable labels for each stripe tier (0=permanent .. 4=volatile). */
12
+ const STRIPE_LABELS = {
13
+ 0: "Permanent",
14
+ 1: "Epoch",
15
+ 2: "Topic",
16
+ 3: "Thread",
17
+ 4: "Volatile",
18
+ };
19
+ /** Weight per stripe for health scoring (higher = more stable = better). */
20
+ const STRIPE_HEALTH_WEIGHTS = {
21
+ 0: 1.0,
22
+ 1: 0.8,
23
+ 2: 0.5,
24
+ 3: 0.25,
25
+ 4: 0.0,
26
+ };
27
+ /**
28
+ * Read cache stripe distribution from the database.
29
+ *
30
+ * Returns aggregated per-stripe buckets with stability stats and a composite
31
+ * health score. Returns empty buckets (all zeros) when the table is empty,
32
+ * so the dashboard always has a valid response.
33
+ */
34
+ export function readCacheStripes(stateDir = getStateDir()) {
35
+ const db = openStore(stateDir);
36
+ // ── Fetch per-stripe aggregates ──
37
+ const rows = db
38
+ .prepare(`SELECT
39
+ stripe,
40
+ COUNT(*) AS count,
41
+ ROUND(AVG(stability), 4) AS avgStability,
42
+ ROUND(MIN(stability), 4) AS minStability,
43
+ ROUND(MAX(stability), 4) AS maxStability
44
+ FROM cache_stripes
45
+ GROUP BY stripe
46
+ ORDER BY stripe ASC`)
47
+ .all();
48
+ // ── Latest epoch ──
49
+ const epochRow = db
50
+ .prepare(`SELECT epoch_id, MAX(assigned_at) AS last_refresh
51
+ FROM cache_stripes
52
+ WHERE epoch_id IS NOT NULL`)
53
+ .get();
54
+ // ── Total chunk count ──
55
+ const totalRow = db
56
+ .prepare("SELECT COUNT(*) AS cnt FROM cache_stripes")
57
+ .get();
58
+ const totalChunks = totalRow?.cnt ?? 0;
59
+ // ── Build bucket array (always 5 entries, stripe 0–4) ──
60
+ const rowMap = new Map();
61
+ for (const r of rows)
62
+ rowMap.set(r.stripe, r);
63
+ const buckets = [];
64
+ for (let s = 0; s <= 4; s++) {
65
+ const r = rowMap.get(s);
66
+ buckets.push({
67
+ stripe: s,
68
+ label: STRIPE_LABELS[s] ?? `Stripe ${s}`,
69
+ count: r?.count ?? 0,
70
+ avgStability: r?.avgStability ?? 0,
71
+ minStability: r?.minStability ?? 0,
72
+ maxStability: r?.maxStability ?? 0,
73
+ });
74
+ }
75
+ // ── Compute health score ──
76
+ const health = computeCacheHealth(buckets, totalChunks, stateDir);
77
+ return {
78
+ buckets,
79
+ health,
80
+ epochId: epochRow?.epoch_id ?? null,
81
+ lastRefreshAt: epochRow?.last_refresh ?? 0,
82
+ totalChunks,
83
+ };
84
+ }
85
+ /**
86
+ * Compute cache health from bucket distribution.
87
+ *
88
+ * Score = weighted sum of stripe fractions (normalised 0..1).
89
+ * Dominant tier = fraction of chunks in the top 2 stripes (0+1).
90
+ * Label: >= 0.8 = good, >= 0.6 = fair, >= 0.4 = degraded, else poor.
91
+ */
92
+ function computeCacheHealth(buckets, total, stateDir) {
93
+ if (total === 0) {
94
+ return { score: 1, label: "good", dominantTier: 1, churnRate: 0 };
95
+ }
96
+ let weightedSum = 0;
97
+ let dominantCount = 0;
98
+ for (const b of buckets) {
99
+ const w = STRIPE_HEALTH_WEIGHTS[b.stripe] ?? 0;
100
+ weightedSum += (b.count / total) * w;
101
+ if (b.stripe <= 1)
102
+ dominantCount += b.count;
103
+ }
104
+ const dominantTier = dominantCount / total;
105
+ // ── Churn rate: open a new DB connection for the churn query ──
106
+ let churnRate = 0;
107
+ try {
108
+ const churnDb = openStore(stateDir);
109
+ const churnRow = churnDb
110
+ .prepare(`WITH
111
+ ranked AS (
112
+ SELECT chunk_id, epoch_id,
113
+ ROW_NUMBER() OVER (PARTITION BY chunk_id ORDER BY assigned_at DESC) AS rn
114
+ FROM cache_stripes
115
+ ),
116
+ cur AS (SELECT chunk_id, epoch_id FROM ranked WHERE rn = 1),
117
+ prev AS (SELECT chunk_id, epoch_id FROM ranked WHERE rn = 2)
118
+ SELECT
119
+ COUNT(*) AS total_changed
120
+ FROM cur c
121
+ JOIN prev p ON c.chunk_id = p.chunk_id AND c.epoch_id != p.epoch_id`)
122
+ .get();
123
+ if (churnRow && total > 0) {
124
+ churnRate = Math.min(1, (churnRow.total_changed ?? 0) / total);
125
+ }
126
+ }
127
+ catch {
128
+ // churn unavailable — keep 0
129
+ }
130
+ const score = Math.round(weightedSum * 100) / 100;
131
+ const label = score >= 0.8 ? "good" : score >= 0.6 ? "fair" : score >= 0.4 ? "degraded" : "poor";
132
+ return { score, label, dominantTier, churnRate };
133
+ }
@@ -0,0 +1,68 @@
1
+ /**
2
+ * fts5-search.ts — FTS5 trigram search helper (L1 tier in the S44 tiered router).
3
+ *
4
+ * Runs BM25-ranked trigram search against the existing `context_chunks_trgm`
5
+ * virtual table (pg_trgm-equivalent, created in schema.ts). All queries are
6
+ * parameterized (PREVENT-002). Pi-agnostic — pure SQL over a node:sqlite
7
+ * DatabaseSync handle.
8
+ *
9
+ * The virtual table stores (id UNINDEXED, normalized_text) with tokenize='trigram'.
10
+ * FTS5 trigram queries tokenize the user query into overlapping 3-grams at query
11
+ * time automatically — no manual n-gram splitting needed. The BM25 rank from
12
+ * fts5 ranks results by trigram-overlap density.
13
+ */
14
+ /**
15
+ * Search the context_chunks_trgm FTS5 table for checkpoints whose
16
+ * `normalized_text` matches `query` via trigram similarity.
17
+ *
18
+ * @param query Raw search text (FTS5 trigram tokenizer handles n-gram splitting
19
+ * automatically; input is NOT SQL-concatenated — bound as ? param).
20
+ * @param reader A sync-reader handle against the SQLite store.
21
+ * @param limit Max results to return (default 10).
22
+ * @returns BM25-ranked hits, highest score first.
23
+ */
24
+ export function fts5Search(query, reader, limit = 10) {
25
+ const sql = `
26
+ SELECT id, bm25(context_chunks_trgm, 0.0, 1.0) AS score
27
+ FROM context_chunks_trgm
28
+ WHERE context_chunks_trgm MATCH ?
29
+ ORDER BY score
30
+ LIMIT ?
31
+ `;
32
+ const rows = reader
33
+ .prepare(sql)
34
+ .all(query, limit);
35
+ return rows.map((r) => ({ id: r.id, score: r.score }));
36
+ }
37
+ /**
38
+ * Search with an optional session_id filter for scoped FTS5 recall.
39
+ *
40
+ * JOINs the FTS5 virtual table against the real `context_chunks` table on `id`
41
+ * so we can filter by `session_id` while still using FTS5 MATCH. The vtab
42
+ * stores `id UNINDEXED` (it's a row-key passthrough, not searchable text), so
43
+ * the join is safe and efficient via the `idx_chunks_pk` index.
44
+ *
45
+ * @param query Raw search text.
46
+ * @param reader Sync-reader handle against the SQLite store.
47
+ * @param sessionId Scope to a specific session (omit for cross-session search).
48
+ * @param limit Max results (default 10).
49
+ */
50
+ export function fts5SearchScoped(query, reader, sessionId, limit = 10) {
51
+ if (sessionId) {
52
+ const sql = `
53
+ SELECT t.id, bm25(context_chunks_trgm, 0.0, 1.0) AS score
54
+ FROM context_chunks_trgm AS t
55
+ INNER JOIN context_chunks AS c ON c.id = t.id
56
+ WHERE t.context_chunks_trgm MATCH ?
57
+ AND c.session_id = ?
58
+ ORDER BY score
59
+ LIMIT ?
60
+ `;
61
+ const rows = reader
62
+ .prepare(sql)
63
+ .all(query, sessionId, limit);
64
+ return rows.map((r) => ({ id: r.id, score: r.score }));
65
+ }
66
+ // No session filter — plain FTS5 search across all sessions.
67
+ return fts5Search(query, reader, limit);
68
+ }
@@ -28,6 +28,7 @@ export const PERF_KINDS = [
28
28
  "cpu_sys_ms",
29
29
  "db_recompute_ms",
30
30
  "disk_write_ms",
31
+ "cache_health",
31
32
  ];
32
33
  function isPerfKind(k) {
33
34
  return PERF_KINDS.includes(k);
@@ -1,5 +1,5 @@
1
1
  import { ACHIEVEMENT_DEFS } from "../../game/scoring.js";
2
- const SCHEMA_VERSION = 2;
2
+ const SCHEMA_VERSION = 4;
3
3
  /**
4
4
  * Add `column` (with `decl`, e.g. "INTEGER") to `table` if it does not already
5
5
  * exist. Idempotent: checks PRAGMA table_info first, so it is safe to run on
@@ -353,6 +353,65 @@ export function initSchema(db) {
353
353
  );
354
354
  CREATE INDEX IF NOT EXISTS idx_perf_samples_ts ON perf_samples(ts);
355
355
  CREATE INDEX IF NOT EXISTS idx_perf_samples_kind_ts ON perf_samples(kind, ts);
356
+
357
+ -- A1 PLAN_V2 Phase 2: Message Separation tables (SCHEMA_VERSION 3).
358
+ -- Stable conversation thread: user + assistant turns ONLY. Tool results are
359
+ -- stored in tool_results to keep the conversation thread cache-predictable.
360
+ -- Parameterized queries only (PREVENT-002). CREATE TABLE IF NOT EXISTS is
361
+ -- a no-op on re-open — no migration needed.
362
+ CREATE TABLE IF NOT EXISTS conversation_thread (
363
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
364
+ conversation_id TEXT NOT NULL,
365
+ role TEXT NOT NULL CHECK(role IN ('user','assistant')),
366
+ content TEXT NOT NULL,
367
+ turn_index INTEGER NOT NULL,
368
+ timestamp INTEGER NOT NULL
369
+ );
370
+ CREATE INDEX IF NOT EXISTS idx_conv_thread_conv_turn
371
+ ON conversation_thread(conversation_id, turn_index);
372
+
373
+ -- Volatile tool results: appended at END so they never disrupt the
374
+ -- conversation thread prefix (the prompt-cache-friendly prefix).
375
+ CREATE TABLE IF NOT EXISTS tool_results (
376
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
377
+ conversation_id TEXT NOT NULL,
378
+ tool_call_id TEXT NOT NULL,
379
+ tool_result TEXT NOT NULL,
380
+ turn_index INTEGER NOT NULL,
381
+ timestamp INTEGER NOT NULL
382
+ );
383
+ CREATE INDEX IF NOT EXISTS idx_tool_results_conv_turn
384
+ ON tool_results(conversation_id, turn_index);
385
+
386
+ -- A2 PLAN_V2 Phase 3: Vector-Aware Cache Striping (SCHEMA_VERSION 4).
387
+ -- Cache stripe assignments: maps each context chunk to a cache stability
388
+ -- tier (stripe). Layer 0=permanent, 1=epoch, 2=topic, 3=thread, 4=volatile.
389
+ -- stability is the composite score (0.0-1.0) from computeStabilityScore.
390
+ -- Primary key is (chunk_id, epoch_id) so the same chunk can be reassigned
391
+ -- across epochs without conflict. See cache-stripe.ts for scoring details.
392
+ -- Parameterized queries only (PREVENT-002). IF NOT EXISTS is a no-op on re-open.
393
+ CREATE TABLE IF NOT EXISTS cache_stripes (
394
+ chunk_id TEXT NOT NULL,
395
+ stripe INTEGER NOT NULL,
396
+ stability REAL NOT NULL,
397
+ assigned_at INTEGER NOT NULL,
398
+ epoch_id TEXT,
399
+ PRIMARY KEY (chunk_id, epoch_id)
400
+ );
401
+ CREATE INDEX IF NOT EXISTS idx_cs_stripe_stability
402
+ ON cache_stripes(stripe, stability DESC);
403
+
404
+ -- Embedding computation cache: avoids re-embedding content every time
405
+ -- stability scores are recomputed across epochs. Keyed by content_hash
406
+ -- (SHA-256 64-hex-char string). Embeddings are serialized via
407
+ -- Float64Array-to-Buffer to match the embedding_blob convention.
408
+ CREATE TABLE IF NOT EXISTS embedding_cache (
409
+ content_hash TEXT PRIMARY KEY,
410
+ embedding BLOB NOT NULL,
411
+ computed_at INTEGER NOT NULL
412
+ );
413
+ CREATE INDEX IF NOT EXISTS idx_emb_cache_computed
414
+ ON embedding_cache(computed_at DESC);
356
415
  `);
357
416
  // Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
358
417
  // pre-existing table, so new columns added to context_chunks after a store was
@@ -24,4 +24,5 @@ export * from "./sqlite/game-state.js";
24
24
  export * from "./sqlite/game-scores.js";
25
25
  export * from "./sqlite/game-achievements.js";
26
26
  export * from "./sqlite/perf-samples.js";
27
+ export * from "./sqlite/cache-stripes.js";
27
28
  export * from "./sqlite/turns.js";
@@ -195,10 +195,15 @@ export async function upsertEmbedding(repoId, sessionId, checkpointId, embedding
195
195
  if (!pg)
196
196
  return;
197
197
  const lit = toVectorLiteral(embedding);
198
- await pg.query(`INSERT INTO vector_index (repo_id, session_id, checkpoint_id, embedding)
199
- VALUES ($1, $2, $3, $4::vector)
200
- ON CONFLICT (repo_id, session_id, checkpoint_id)
201
- DO UPDATE SET embedding = EXCLUDED.embedding;`, [repoId, sessionId, checkpointId, lit]);
198
+ const upserted = await withOpenTimeout(pg.query(`INSERT INTO vector_index (repo_id, session_id, checkpoint_id, embedding)
199
+ VALUES ($1, $2, $3, $4::vector)
200
+ ON CONFLICT (repo_id, session_id, checkpoint_id)
201
+ DO UPDATE SET embedding = EXCLUDED.embedding;`, [repoId, sessionId, checkpointId, lit]), (reason) => {
202
+ disabled = true;
203
+ logWarn(`upsert query timed out after ${queryTimeoutMs()}ms: ${reason}`);
204
+ }, queryTimeoutMs());
205
+ if (upserted === undefined)
206
+ return; // timed out — non-fatal
202
207
  }
203
208
  catch (err) {
204
209
  disabled = true;
@@ -227,7 +232,12 @@ export async function searchAsync(query, opts = {}) {
227
232
  params.push(repoId);
228
233
  }
229
234
  sql += " ORDER BY embedding <=> $1::vector LIMIT $2";
230
- const res = await pg.query(sql, params);
235
+ const res = await withOpenTimeout(pg.query(sql, params), (reason) => {
236
+ disabled = true;
237
+ logWarn(`search query timed out after ${queryTimeoutMs()}ms: ${reason}`);
238
+ }, queryTimeoutMs());
239
+ if (!res)
240
+ return []; // query timed out — degrade to empty (non-fatal)
231
241
  return res.rows.map((r) => ({
232
242
  repoId: r.repo_id,
233
243
  sessionId: r.session_id,
@@ -252,6 +262,20 @@ function closeTimeoutMs() {
252
262
  }
253
263
  return DEFAULT_CLOSE_TIMEOUT_MS;
254
264
  }
265
+ /** Default cap for a PGlite search query. A WASM query that never resolves would
266
+ * otherwise hang the calling turn (and node --test) forever — the catch below
267
+ * only handles thrown errors, not a stalled promise. 0 disables the guard
268
+ * (preserve original unbounded behavior). Mirrors the open/close timeouts. */
269
+ export const DEFAULT_QUERY_TIMEOUT_MS = 10_000;
270
+ function queryTimeoutMs() {
271
+ const raw = process.env.MEGACOMPACT_PGLITE_QUERY_TIMEOUT_MS;
272
+ if (raw !== undefined) {
273
+ const n = Number(raw);
274
+ if (Number.isFinite(n) && n >= 0)
275
+ return n;
276
+ }
277
+ return DEFAULT_QUERY_TIMEOUT_MS;
278
+ }
255
279
  /**
256
280
  * Race a promise against a timeout. If the inner promise never settles, the
257
281
  * timer fires, resolves with `undefined`, and the caller proceeds. This mirrors
@@ -0,0 +1,18 @@
1
+ /**
2
+ * tiered-router/types.ts — S44 tiered router types, interfaces, and defaults.
3
+ *
4
+ * Shell file: pure data/type definitions. Zero runtime dependencies on store
5
+ * internals. Imported by tieredRouter.ts and any consumer that needs the
6
+ * RecallResult / TieredRouterMetrics shapes.
7
+ */
8
+ // ---------------------------------------------------------------------------
9
+ // Defaults
10
+ // ---------------------------------------------------------------------------
11
+ /** Default LRU cache size. */
12
+ export const DEFAULT_CACHE_SIZE = 256;
13
+ /** Default latency budgets per tier (ms). */
14
+ export const DEFAULT_BUDGET_L0_MS = 1;
15
+ export const DEFAULT_BUDGET_L1_MS = 50;
16
+ export const DEFAULT_BUDGET_L2_MS = 500;
17
+ /** Default cadence for tiered-metrics logging (every N queries). */
18
+ export const DEFAULT_LOG_CADENCE = 100;