pi-mega-compact 0.8.21 → 0.8.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
  2. package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
  3. package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
  4. package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
  5. package/dist/extensions/dashboard-server/html.js +2 -621
  6. package/dist/extensions/dashboard-server/routes-core.js +62 -0
  7. package/dist/extensions/dashboard-server/routes-game.js +323 -0
  8. package/dist/extensions/dashboard-server/routes-repo.js +170 -0
  9. package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
  10. package/dist/extensions/dashboard-server/routes.js +10 -0
  11. package/dist/extensions/dashboard-server/server.js +26 -623
  12. package/dist/extensions/mega-commands.js +4 -3
  13. package/dist/extensions/mega-events/agent-handlers.js +2 -1
  14. package/dist/extensions/mega-events/compact-handlers.js +26 -0
  15. package/dist/extensions/mega-events/session-handlers.js +2 -1
  16. package/dist/extensions/mega-pipeline/compact.js +3 -2
  17. package/dist/extensions/mega-runtime/state.js +7 -7
  18. package/dist/src/dedup/raptor/promote.test.js +5 -5
  19. package/dist/src/dedup/sprint12.test.js +7 -7
  20. package/dist/src/dedup-engine.test.js +29 -29
  21. package/dist/src/e2e.test.js +38 -38
  22. package/dist/src/engine.js +3 -3
  23. package/dist/src/engine.test.js +6 -6
  24. package/dist/src/importance.js +197 -0
  25. package/dist/src/importance.test.js +372 -0
  26. package/dist/src/ratio.bench.test.js +18 -18
  27. package/dist/src/recall.js +6 -5
  28. package/dist/src/recall.test.js +85 -27
  29. package/dist/src/sprint14.test.js +2 -2
  30. package/dist/src/store/migrate.test.js +5 -5
  31. package/dist/src/store/sprint10.test.js +5 -5
  32. package/dist/src/store/sqlite/global-index.js +5 -174
  33. package/dist/src/store/sqlite/global-sessions.js +190 -0
  34. package/dist/src/vector-read.js +168 -0
  35. package/dist/src/vector-search.js +191 -0
  36. package/dist/src/vectorStore.js +10 -297
  37. package/dist/src/vectorStore.test.js +32 -32
  38. package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
  39. package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
  40. package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
  41. package/extensions/dashboard-server/dashboard-client.ts +21 -0
  42. package/extensions/dashboard-server/html.ts +2 -621
  43. package/extensions/dashboard-server/routes-core.ts +113 -0
  44. package/extensions/dashboard-server/routes-game.ts +386 -0
  45. package/extensions/dashboard-server/routes-repo.ts +212 -0
  46. package/extensions/dashboard-server/routes-sessions.ts +195 -0
  47. package/extensions/dashboard-server/routes.ts +13 -0
  48. package/extensions/dashboard-server/server.ts +37 -700
  49. package/extensions/mega-commands.ts +4 -3
  50. package/extensions/mega-events/agent-handlers.ts +2 -1
  51. package/extensions/mega-events/compact-handlers.ts +28 -0
  52. package/extensions/mega-events/session-handlers.ts +2 -1
  53. package/extensions/mega-pipeline/compact.ts +3 -2
  54. package/extensions/mega-runtime/state.ts +7 -7
  55. package/extensions/openclaw-mega-compact.ts +2 -2
  56. package/package.json +1 -1
  57. package/src/dedup/raptor/promote.test.ts +5 -5
  58. package/src/dedup/sprint12.test.ts +7 -7
  59. package/src/dedup-engine.test.ts +30 -30
  60. package/src/e2e.test.ts +38 -38
  61. package/src/engine.test.ts +6 -6
  62. package/src/engine.ts +3 -3
  63. package/src/importance.test.ts +538 -0
  64. package/src/importance.ts +312 -0
  65. package/src/ratio.bench.test.ts +18 -18
  66. package/src/recall.test.ts +101 -29
  67. package/src/recall.ts +9 -9
  68. package/src/sprint14.test.ts +2 -2
  69. package/src/store/migrate.test.ts +5 -5
  70. package/src/store/sprint10.test.ts +5 -5
  71. package/src/store/sqlite/global-index.ts +18 -290
  72. package/src/store/sqlite/global-sessions.ts +291 -0
  73. package/src/vector-read.ts +237 -0
  74. package/src/vector-search.ts +231 -0
  75. package/src/vectorStore.test.ts +32 -32
  76. package/src/vectorStore.ts +29 -356
@@ -0,0 +1,312 @@
1
+ /**
2
+ * importance.ts — S40 importance scoring for compaction.
3
+ *
4
+ * A standalone, deterministic, pi-agnostic scoring module with no side
5
+ * effects. Scores each context item by type, age decay, recency/retention
6
+ * boosts, and selects the top-N for verbatim preservation (bypassing
7
+ * summarization) so high-importance old messages (decisions, errors) are
8
+ * not lost to compaction.
9
+ *
10
+ * PREVENT-PI-001 (anchor floor): importance scoring AUGMENTS, never
11
+ * replaces, the boundary guard. `itemsToPreserve` only marks items the
12
+ * caller already considered compactable; the caller's `computeDropRange`
13
+ * still protects the most recent N user messages.
14
+ * PREVENT-PI-002 (tool pairs): callers preserve pairs; `score()` itself
15
+ * is pair-agnostic.
16
+ * PREVENT-PI-PI-003 (no system role): scored items are surfaced via the
17
+ * existing systemPrompt-prepend path, never as role:"system".
18
+ *
19
+ * @module
20
+ */
21
+ import type { EngineMessage } from "./types.js";
22
+
23
+ /** The 8 content/item types that drive the type multiplier. */
24
+ export enum ContextItemType {
25
+ UserMessage = "user_message",
26
+ AssistantMessage = "assistant_message",
27
+ SystemMessage = "system_message",
28
+ CodeBlock = "code_block",
29
+ Error = "error",
30
+ Decision = "decision",
31
+ FileModification = "file_modification",
32
+ ToolExecution = "tool_execution",
33
+ }
34
+
35
+ /** A scored context item. */
36
+ export interface ScoredItem {
37
+ /** Message index or checkpoint ID (caller-supplied). */
38
+ id: string;
39
+ type: ContextItemType;
40
+ content: string;
41
+ /** Raw role from EngineMessage ("user" | "assistant" | "tool" | "custom"). */
42
+ role: string;
43
+ /** Epoch ms. */
44
+ timestamp: number;
45
+ /** Type-based multiplier, before decay/boost. */
46
+ rawMultiplier: number;
47
+ /** 0–0.7 fraction SUBTRACTED from the score. */
48
+ ageDecay: number;
49
+ /** 1.0 or 1.2. */
50
+ recencyBoost: number;
51
+ /** 1.0 or 3.0. */
52
+ retentionBoost: number;
53
+ /** Composite score. */
54
+ finalScore: number;
55
+ }
56
+
57
+ /** Result of a preservation pass. */
58
+ export interface PreservationResult {
59
+ preservedIds: Set<string>;
60
+ /** Score cutoff used. */
61
+ threshold: number;
62
+ totalScored: number;
63
+ totalPreserved: number;
64
+ }
65
+
66
+ /**
67
+ * Default type multipliers (ported from the Rust reference
68
+ * `router/src/context/importance.rs`). Decisions (2.5x) and errors (2.0x)
69
+ * dominate; system/filler (0.5x) sinks.
70
+ */
71
+ export const DEFAULT_MULTIPLIERS: Record<ContextItemType, number> = {
72
+ [ContextItemType.UserMessage]: 1.5,
73
+ [ContextItemType.AssistantMessage]: 1.0,
74
+ [ContextItemType.SystemMessage]: 0.5,
75
+ [ContextItemType.CodeBlock]: 1.2,
76
+ [ContextItemType.Error]: 2.0,
77
+ [ContextItemType.Decision]: 2.5,
78
+ [ContextItemType.FileModification]: 1.8,
79
+ [ContextItemType.ToolExecution]: 1.3,
80
+ };
81
+
82
+ /**
83
+ * Age decay: fraction to SUBTRACT from the score.
84
+ * Formula: `min(maxDecay, (ageMs / 3_600_000) * decayRatePerHour)`.
85
+ * 0 = fresh, 0.7 = very old. At 14h with 0.05/hr: 14 * 0.05 = 0.70 (capped).
86
+ */
87
+ export function ageDecay(
88
+ itemAgeMs: number,
89
+ decayRatePerHour: number = 0.05,
90
+ maxDecay: number = 0.7,
91
+ ): number {
92
+ if (itemAgeMs <= 0) return 0;
93
+ const hours = itemAgeMs / 3_600_000;
94
+ return Math.min(maxDecay, hours * decayRatePerHour);
95
+ }
96
+
97
+ /** Recency boost: 1.2 if `ageMs < thresholdMs` (default 5min), else 1.0. */
98
+ export function recencyBoost(
99
+ ageMs: number,
100
+ thresholdMs: number = 300_000,
101
+ ): number {
102
+ return ageMs < thresholdMs ? 1.2 : 1.0;
103
+ }
104
+
105
+ /**
106
+ * Retention boost: 3.0 if the user flagged the item, else 1.0.
107
+ * User-flagged items are also inferred by `detectItemType` returning
108
+ * Decision or Error.
109
+ */
110
+ export function retentionBoost(userFlagged: boolean): number {
111
+ return userFlagged ? 3.0 : 1.0;
112
+ }
113
+
114
+ /**
115
+ * Classify a content blob into a `ContextItemType`.
116
+ *
117
+ * Rules are ordered by priority — first match wins:
118
+ * 1. role === "tool" → ToolExecution
119
+ * 2. role === "custom" → SystemMessage
120
+ * 3. error-ish content → Error
121
+ * 4. decision-ish content → Decision
122
+ * 5. fenced code block (≥20 chars) → CodeBlock
123
+ * 6. file-modification verbs → FileModification
124
+ * 7. role === "user" → UserMessage
125
+ * 8. role === "assistant" → AssistantMessage
126
+ * 9. Fallback → AssistantMessage
127
+ */
128
+ export function detectItemType(
129
+ content: string,
130
+ role: "user" | "assistant" | "tool" | "custom",
131
+ ): ContextItemType {
132
+ const c = content ?? "";
133
+ if (role === "tool") return ContextItemType.ToolExecution;
134
+ if (role === "custom") return ContextItemType.SystemMessage;
135
+ // (3) errors — traceback / panic / E#### / exception / failure / crash.
136
+ if (/error|exception|failure|crash|panic|traceback|E\d{4}/i.test(c)) {
137
+ return ContextItemType.Error;
138
+ }
139
+ // (4) decisions — "we decided", "going with", "switching to", etc.
140
+ if (
141
+ /decided|we chose|going with|switching to|using .* instead|final decision|let's go with/i.test(
142
+ c,
143
+ )
144
+ ) {
145
+ return ContextItemType.Decision;
146
+ }
147
+ // (5) fenced code block ≥20 chars.
148
+ if (/```[\s\S]{20,}/.test(c)) {
149
+ return ContextItemType.CodeBlock;
150
+ }
151
+ // (6) file modification verbs.
152
+ if (/(wrote|edited|created|modified|updated|patched)\s+\S+\.\w+/i.test(c)) {
153
+ return ContextItemType.FileModification;
154
+ }
155
+ if (role === "user") return ContextItemType.UserMessage;
156
+ if (role === "assistant") return ContextItemType.AssistantMessage;
157
+ // (9) Fallback.
158
+ return ContextItemType.AssistantMessage;
159
+ }
160
+
161
+ /**
162
+ * Composite score for a single item.
163
+ *
164
+ * Formula:
165
+ * type = detectItemType(content, role)
166
+ * rawMult = multipliers[type] ?? DEFAULT_MULTIPLIERS[type]
167
+ * decay = ageDecay(now - timestamp, decayRatePerHour, maxDecay)
168
+ * recency = recencyBoost(now - timestamp, recencyThresholdMs)
169
+ * retention = retentionBoost(userFlagged)
170
+ * finalScore = rawMult * (1 - decay) * recency * retention
171
+ *
172
+ * Clamps `finalScore` to a minimum of 0.01 (never zero, so an item is
173
+ * always a candidate unless explicitly filtered).
174
+ */
175
+ export function score(
176
+ item: {
177
+ id: string;
178
+ content: string;
179
+ role: string;
180
+ timestamp: number;
181
+ userFlagged?: boolean;
182
+ },
183
+ now: number,
184
+ multipliers?: Partial<Record<ContextItemType, number>>,
185
+ opts?: {
186
+ decayRatePerHour?: number;
187
+ maxDecay?: number;
188
+ recencyThresholdMs?: number;
189
+ },
190
+ ): ScoredItem {
191
+ const content = item.content ?? "";
192
+ // detectItemType expects the 4-role union; narrow defensively.
193
+ const role = (["user", "assistant", "tool", "custom"].includes(item.role)
194
+ ? item.role
195
+ : "assistant") as "user" | "assistant" | "tool" | "custom";
196
+ const type = detectItemType(content, role);
197
+ const rawMultiplier =
198
+ multipliers?.[type] ?? DEFAULT_MULTIPLIERS[type];
199
+ const ageMs = Math.max(0, now - item.timestamp);
200
+ const decay = ageDecay(
201
+ ageMs,
202
+ opts?.decayRatePerHour,
203
+ opts?.maxDecay,
204
+ );
205
+ const recency = recencyBoost(ageMs, opts?.recencyThresholdMs);
206
+ const retention = retentionBoost(item.userFlagged === true);
207
+ const finalScore = Math.max(
208
+ 0.01,
209
+ rawMultiplier * (1 - decay) * recency * retention,
210
+ );
211
+ return {
212
+ id: item.id,
213
+ type,
214
+ content,
215
+ role,
216
+ timestamp: item.timestamp,
217
+ rawMultiplier,
218
+ ageDecay: decay,
219
+ recencyBoost: recency,
220
+ retentionBoost: retention,
221
+ finalScore,
222
+ };
223
+ }
224
+
225
+ /**
226
+ * Score cutoff at the `preserveRatio` percentile boundary.
227
+ *
228
+ * Sorts items by `finalScore` descending; returns the score at the
229
+ * `preserveRatio` quantile. Items with `finalScore >= threshold` are
230
+ * preserved.
231
+ *
232
+ * - `ratio = 1.0` → return 0 (preserve all)
233
+ * - `ratio = 0` → return Infinity (preserve none)
234
+ * - empty input → return Infinity
235
+ */
236
+ export function preservationCutoff(
237
+ items: ScoredItem[],
238
+ preserveRatio: number,
239
+ ): number {
240
+ if (items.length === 0) return Infinity;
241
+ if (preserveRatio >= 1) return 0;
242
+ if (preserveRatio <= 0) return Infinity;
243
+ // Sort descending; the top `preserveRatio` fraction is preserved.
244
+ const sorted = [...items].sort((a, b) => b.finalScore - a.finalScore);
245
+ // Index of the lowest-scoring item in the preserved set.
246
+ // ceil(N * ratio) - 1, clamped to [0, N-1].
247
+ const cutoffIdx = Math.min(
248
+ sorted.length - 1,
249
+ Math.max(0, Math.ceil(sorted.length * preserveRatio) - 1),
250
+ );
251
+ return sorted[cutoffIdx].finalScore;
252
+ }
253
+
254
+ /**
255
+ * Select which items to preserve verbatim based on `preserveRatio`.
256
+ *
257
+ * Returns the IDs of items with `finalScore >= threshold`. The caller
258
+ * is responsible for excluding items inside the anchor window (the
259
+ * boundary guard already protects those).
260
+ */
261
+ export function itemsToPreserve(
262
+ items: ScoredItem[],
263
+ preserveRatio: number,
264
+ ): PreservationResult {
265
+ const threshold = preservationCutoff(items, preserveRatio);
266
+ const preservedIds = new Set(
267
+ items.filter((i) => i.finalScore >= threshold).map((i) => i.id),
268
+ );
269
+ return {
270
+ preservedIds,
271
+ threshold,
272
+ totalScored: items.length,
273
+ totalPreserved: preservedIds.size,
274
+ };
275
+ }
276
+
277
+ /**
278
+ * Convenience: score a list of `EngineMessage`s in position order using
279
+ * approximate ages from their index (1 minute per position, oldest first).
280
+ * Real timestamps should be threaded through the extension adapter when
281
+ * available; this is the position-based fallback documented in S40B-2.
282
+ *
283
+ * The returned `ScoredItem[]` is in the SAME ORDER as the input — callers
284
+ * index back into `messages` by `id` (which is the stringified index).
285
+ */
286
+ export function scoreEngineMessages(
287
+ messages: EngineMessage[],
288
+ now: number,
289
+ multipliers?: Partial<Record<ContextItemType, number>>,
290
+ opts?: {
291
+ decayRatePerHour?: number;
292
+ maxDecay?: number;
293
+ recencyThresholdMs?: number;
294
+ },
295
+ ): ScoredItem[] {
296
+ return messages.map((m, i) => {
297
+ // Oldest message (i=0) is `messages.length` minutes ago; newest is 1 min.
298
+ const timestamp = now - (messages.length - i) * 60_000;
299
+ return score(
300
+ {
301
+ id: String(i),
302
+ content: m.text ?? "",
303
+ role: m.role,
304
+ timestamp,
305
+ userFlagged: false,
306
+ },
307
+ now,
308
+ multipliers,
309
+ opts,
310
+ );
311
+ });
312
+ }
@@ -19,7 +19,7 @@ import * as fs from "node:fs";
19
19
  import * as path from "node:path";
20
20
  import * as os from "node:os";
21
21
 
22
- import { VectorStore, computeRegionHash } from "./vectorStore.js";
22
+ import { VectorStore, computeRegionHash, vectorStats, vectorWasInjected, vectorMarkInjected, vectorTopSimilar } from "./vectorStore.js";
23
23
  import type { SearchHit } from "./vectorStore.js";
24
24
  import { findSuperseded } from "./supersede.js";
25
25
  import { autoCompactCheck, isChatty } from "./compact.js";
@@ -635,7 +635,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
635
635
 
636
636
  // Check stats for each session
637
637
  for (const sid of sessionIds) {
638
- const stats = store.stats(sid);
638
+ const stats = vectorStats(store,sid);
639
639
  console.log(
640
640
  ` ${sid}: ${stats.checkpointCount} checkpoint(s), dedup rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
641
641
  );
@@ -663,7 +663,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
663
663
  // Check stats across sessions
664
664
  let totalCheckpoints = 0;
665
665
  for (let i = 0; i < variations.length; i++) {
666
- const stats = store.stats(`sess_l1_${i}`);
666
+ const stats = vectorStats(store,`sess_l1_${i}`);
667
667
  totalCheckpoints += stats.checkpointCount;
668
668
  }
669
669
 
@@ -693,7 +693,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
693
693
 
694
694
  let totalCheckpoints = 0;
695
695
  for (let i = 0; i < variations.length; i++) {
696
- const stats = store.stats(`sess_l1neg_${i}`);
696
+ const stats = vectorStats(store,`sess_l1neg_${i}`);
697
697
  totalCheckpoints += stats.checkpointCount;
698
698
  }
699
699
 
@@ -733,7 +733,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
733
733
 
734
734
  let totalCheckpoints = 0;
735
735
  for (let i = 0; i < paraphrases.length; i++) {
736
- const stats = store.stats(`sess_l2_${i}`);
736
+ const stats = vectorStats(store,`sess_l2_${i}`);
737
737
  totalCheckpoints += stats.checkpointCount;
738
738
  }
739
739
 
@@ -798,7 +798,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
798
798
  });
799
799
  }
800
800
 
801
- const stats = store.stats(dedupSession);
801
+ const stats = vectorStats(store,dedupSession);
802
802
  const totalAdded = 3 + 4 + 3;
803
803
 
804
804
  console.log(` Total added: ${totalAdded}`);
@@ -1113,7 +1113,7 @@ describe("Store Stats & Dedup Metrics", () => {
1113
1113
  });
1114
1114
  }
1115
1115
 
1116
- const stats = store.stats(session);
1116
+ const stats = vectorStats(store,session);
1117
1117
 
1118
1118
  console.log(` Total added: 5`);
1119
1119
  console.log(` Checkpoints stored: ${stats.checkpointCount}`);
@@ -1136,34 +1136,34 @@ describe("Store Stats & Dedup Metrics", () => {
1136
1136
  const sid = "sess_inject";
1137
1137
 
1138
1138
  assert.equal(
1139
- store.wasInjected(sid, id1),
1139
+ vectorWasInjected(store,sid, id1),
1140
1140
  false,
1141
1141
  "Should not be injected initially",
1142
1142
  );
1143
1143
 
1144
- store.markInjected(sid, id1);
1144
+ vectorMarkInjected(store,sid, id1);
1145
1145
  assert.equal(
1146
- store.wasInjected(sid, id1),
1146
+ vectorWasInjected(store,sid, id1),
1147
1147
  true,
1148
1148
  "Should be injected after mark",
1149
1149
  );
1150
1150
  assert.equal(
1151
- store.wasInjected(sid, id2),
1151
+ vectorWasInjected(store,sid, id2),
1152
1152
  false,
1153
1153
  "Different ID should not be affected",
1154
1154
  );
1155
1155
 
1156
- store.markInjected(sid, id2);
1156
+ vectorMarkInjected(store,sid, id2);
1157
1157
  assert.equal(
1158
- store.wasInjected(sid, id2),
1158
+ vectorWasInjected(store,sid, id2),
1159
1159
  true,
1160
1160
  "Second ID should also be injected",
1161
1161
  );
1162
1162
 
1163
1163
  // Idempotent
1164
- store.markInjected(sid, id1);
1164
+ vectorMarkInjected(store,sid, id1);
1165
1165
  assert.equal(
1166
- store.wasInjected(sid, id1),
1166
+ vectorWasInjected(store,sid, id1),
1167
1167
  true,
1168
1168
  "Re-mark should be idempotent",
1169
1169
  );
@@ -1200,7 +1200,7 @@ describe("Store Stats & Dedup Metrics", () => {
1200
1200
  );
1201
1201
 
1202
1202
  // Mark first hit as injected
1203
- store.markInjected(
1203
+ vectorMarkInjected(store,
1204
1204
  sid,
1205
1205
  results1.hits[0].checkpoint.checkpointId,
1206
1206
  );
@@ -1272,7 +1272,7 @@ describe("topSimilar Edge Cases & Coverage", () => {
1272
1272
  }
1273
1273
 
1274
1274
  for (const n of [1, 3, 5, 10]) {
1275
- const results = store.topSimilar(sid, n);
1275
+ const results = vectorTopSimilar(store,sid, n);
1276
1276
  assert.ok(
1277
1277
  results.length <= n,
1278
1278
  `topSimilar(${n}) returned ${results.length} results (should be <= ${n})`,
@@ -1282,7 +1282,7 @@ describe("topSimilar Edge Cases & Coverage", () => {
1282
1282
  });
1283
1283
 
1284
1284
  it("topSimilar returns empty for empty session", () => {
1285
- const results = store.topSimilar("sess_unknown_empty", 5);
1285
+ const results = vectorTopSimilar(store,"sess_unknown_empty", 5);
1286
1286
  assert.equal(results.length, 0, "Empty session should return no results");
1287
1287
  });
1288
1288
  });
@@ -6,7 +6,13 @@ import { join } from "node:path";
6
6
  import { VectorStore } from "./vectorStore.js";
7
7
  import { compactSession } from "./engine.js";
8
8
  import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "./recall.js";
9
+ import { vectorList } from "./vectorStore.js";
9
10
  import { markInjectedGlobal, wasInjectedGlobal, closeIndexStore } from "./store/sqlite.js";
11
+ import {
12
+ closeVectorIndex,
13
+ initVectorIndex,
14
+ rebuildFromSqlite,
15
+ } from "./store/vectorIndex.js";
10
16
  import type { EngineMessage } from "./types.js";
11
17
 
12
18
  const baseTmp = mkdtempSync(join(tmpdir(), "mc-recall-"));
@@ -119,58 +125,124 @@ test("Fix C: inline dedupe drops a hit already resident in the live window", ()
119
125
  );
120
126
  });
121
127
 
128
+ // ---- S18 cross-repo global injected-set (real-data, no mocks) ----------------
129
+ //
130
+ // Earlier versions of these tests used `as any` mock stores with canned
131
+ // `searchAsync` returns. That was mock data: it asserted recall's orchestration
132
+ // against a fake search result, not the real embed → HNSW → hydrate → inject
133
+ // path. Per the no-mock-data principle, both tests now seed a REAL foreign
134
+ // VectorStore (real checkpoint + real TrigramEmbedder embedding persisted to
135
+ // SQLite), rebuild the real PGlite index from it via `rebuildFromSqlite`, and
136
+ // run the full `recallAndInlineAsync` cross-repo path against a separate self
137
+ // store. The foreign checkpoint hydrates from the foreign store via its real
138
+ // repoId (== stateDir, per VectorStore's repoId convention).
139
+
140
+ async function seedForeignRepo(foreignStateDir: string): Promise<{ sessionId: string; checkpointId: string; summary: string }> {
141
+ // A real VectorStore at the foreign repo's stateDir. repoId == stateDir.
142
+ const foreign = new VectorStore({ stateDir: foreignStateDir, dedupSim: 0.9 });
143
+ const sess = "sess_foreign";
144
+ // Seed a real checkpoint via the real compactSession pipeline. The summary
145
+ // text determines the embedding; the query below must land near it.
146
+ const summary = "foreign repo authentication jwt token validation";
147
+ const result = compactSession(
148
+ {
149
+ sessionId: sess,
150
+ messages: [msg("user", summary), msg("assistant", "ok", "Edit")],
151
+ keepFrom: 2,
152
+ timestamp: 1,
153
+ },
154
+ foreign,
155
+ );
156
+ return { sessionId: sess, checkpointId: result.checkpointId ?? "", summary };
157
+ }
158
+
122
159
  test("S18: global injected-set skips a foreign checkpoint already injected machine-wide", async () => {
160
+ if (process.env.MEGACOMPACT_PGLITE_DISABLED === "true") { return; } // skip when WASM index is off
123
161
  const indexDir = mkdtempSync(join(tmpdir(), "mc-gi-"));
162
+ const foreignStateDir = mkdtempSync(join(tmpdir(), "mc-gi-foreign-"));
163
+ const selfStateDir = mkdtempSync(join(tmpdir(), "mc-gi-self-"));
164
+ process.env.MEGACOMPACT_VECTOR_INDEX_DIR = mkdtempSync(join(tmpdir(), "mc-gi-vidx-"));
124
165
  try {
166
+ await closeVectorIndex(); // fresh singleton per this index dir
167
+ const seed = await seedForeignRepo(foreignStateDir);
168
+ // Populate the real PGlite index from the foreign store's real checkpoints.
169
+ await rebuildFromSqlite(
170
+ () => [{ repoId: foreignStateDir, stateDir: foreignStateDir }],
171
+ (sd) => {
172
+ // readCheckpoints: yield (sessionId, checkpointId, embedding) for every
173
+ // real checkpoint in this store. Mirrors the production enumerator.
174
+ const store = new VectorStore({ stateDir: sd, dedupSim: 0.9 });
175
+ return vectorList(store, seed.sessionId).map((cp) => ({
176
+ sessionId: seed.sessionId,
177
+ checkpointId: cp.checkpointId,
178
+ embedding: cp.embedding,
179
+ }));
180
+ },
181
+ );
182
+ const pg = await initVectorIndex();
183
+ assert.ok(pg, "PGlite index should initialize (WASM available)");
184
+
125
185
  const sess = "sess_cross";
126
- // A foreign checkpoint already marked injected globally (in this session).
127
- markInjectedGlobal("chkpt_foreign", "/repo/other", sess, indexDir);
128
- assert.equal(wasInjectedGlobal("chkpt_foreign", sess, indexDir), true);
129
- // searchAsync returns the foreign hit; recallAndInlineAsync must skip it
130
- // (globally injected) toInject is empty.
131
- const mockStore = {
132
- searchAsync: async () => [{
133
- checkpoint: { checkpointId: "chkpt_foreign", summary: "foreign work", filesModified: [], dedupStatus: "active" },
134
- score: 0.92,
135
- repoId: "/repo/other",
136
- }],
137
- wasInjected: () => false,
138
- markInjected: () => {},
139
- } as any;
186
+ // Pre-mark the foreign checkpoint as already injected machine-wide.
187
+ markInjectedGlobal(seed.checkpointId, foreignStateDir, sess, indexDir);
188
+ assert.equal(wasInjectedGlobal(seed.checkpointId, sess, indexDir), true);
189
+
190
+ // Self store (different repo) the cross-repo query runs against the index.
191
+ const selfStore = new VectorStore({ stateDir: selfStateDir, dedupSim: 0.9 });
140
192
  const r = await recallAndInlineAsync(
141
- { sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
142
- mockStore,
193
+ { sessionId: sess, query: "foreign repo authentication jwt", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
194
+ selfStore,
143
195
  );
144
196
  assert.equal(r.toInject.length, 0, "globally-injected foreign checkpoint skipped");
145
197
  } finally {
198
+ await closeVectorIndex();
146
199
  closeIndexStore();
200
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
147
201
  rmSync(indexDir, { recursive: true, force: true });
202
+ rmSync(foreignStateDir, { recursive: true, force: true });
203
+ rmSync(selfStateDir, { recursive: true, force: true });
148
204
  }
149
205
  });
150
206
 
151
207
  test("S18: a fresh foreign checkpoint is injected AND recorded globally", async () => {
208
+ if (process.env.MEGACOMPACT_PGLITE_DISABLED === "true") { return; } // skip when WASM index is off
152
209
  const indexDir = mkdtempSync(join(tmpdir(), "mc-gi2-"));
210
+ const foreignStateDir = mkdtempSync(join(tmpdir(), "mc-gi2-foreign-"));
211
+ const selfStateDir = mkdtempSync(join(tmpdir(), "mc-gi2-self-"));
212
+ process.env.MEGACOMPACT_VECTOR_INDEX_DIR = mkdtempSync(join(tmpdir(), "mc-gi2-vidx-"));
153
213
  try {
214
+ await closeVectorIndex(); // fresh singleton per this index dir
215
+ const seed = await seedForeignRepo(foreignStateDir);
216
+ assert.equal(wasInjectedGlobal(seed.checkpointId, "sess_fresh", indexDir), false);
217
+ await rebuildFromSqlite(
218
+ () => [{ repoId: foreignStateDir, stateDir: foreignStateDir }],
219
+ (sd) => {
220
+ const store = new VectorStore({ stateDir: sd, dedupSim: 0.9 });
221
+ return vectorList(store, seed.sessionId).map((cp) => ({
222
+ sessionId: seed.sessionId,
223
+ checkpointId: cp.checkpointId,
224
+ embedding: cp.embedding,
225
+ }));
226
+ },
227
+ );
228
+ const pg = await initVectorIndex();
229
+ assert.ok(pg, "PGlite index should initialize (WASM available)");
230
+
154
231
  const sess = "sess_fresh";
155
- assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), false);
156
- const mockStore = {
157
- searchAsync: async () => [{
158
- checkpoint: { checkpointId: "chkpt_new", summary: "brand new foreign work", filesModified: [], dedupStatus: "active" },
159
- score: 0.93,
160
- repoId: "/repo/alpha",
161
- }],
162
- wasInjected: () => false,
163
- markInjected: () => {},
164
- } as any;
232
+ const selfStore = new VectorStore({ stateDir: selfStateDir, dedupSim: 0.9 });
165
233
  const r = await recallAndInlineAsync(
166
- { sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
167
- mockStore,
234
+ { sessionId: sess, query: "foreign repo authentication jwt", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
235
+ selfStore,
168
236
  );
169
237
  assert.equal(r.toInject.length, 1, "fresh foreign checkpoint injected");
170
- assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), true, "recorded machine-wide");
238
+ assert.equal(wasInjectedGlobal(seed.checkpointId, sess, indexDir), true, "recorded machine-wide");
171
239
  } finally {
240
+ await closeVectorIndex();
172
241
  closeIndexStore();
242
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
173
243
  rmSync(indexDir, { recursive: true, force: true });
244
+ rmSync(foreignStateDir, { recursive: true, force: true });
245
+ rmSync(selfStateDir, { recursive: true, force: true });
174
246
  }
175
247
  });
176
248
 
package/src/recall.ts CHANGED
@@ -15,7 +15,7 @@
15
15
  */
16
16
 
17
17
  import { recall as searchRecall } from "./engine.js";
18
- import type { SearchHit, VectorStore } from "./vectorStore.js";
18
+ import { vectorWasInjected, vectorMarkInjected, type SearchHit, type VectorStore, vectorSearchAsync } from "./vectorStore.js";
19
19
  import { estimateBlockTokens } from "./tokens.js";
20
20
  import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
21
21
 
@@ -107,7 +107,7 @@ export function formatRecallBlock(hits: SearchHit[]): string {
107
107
  */
108
108
  export function recallAndInline(
109
109
  opts: RecallInjectOptions,
110
- store: Pick<VectorStore, "search" | "wasInjected" | "markInjected">,
110
+ store: VectorStore,
111
111
  ): RecallInjectResult {
112
112
  // ── S27 Recall Demotion ─────────────────────────────────────────────
113
113
  //
@@ -134,7 +134,7 @@ export function recallAndInline(
134
134
 
135
135
  const { hits } = searchRecall(
136
136
  { sessionId: opts.sessionId, query: opts.query, limit, skipInjected: false },
137
- store as VectorStore,
137
+ store,
138
138
  );
139
139
 
140
140
  // Precompute live-window embeddings once for inline dedupe (Fix C). Trigram
@@ -152,7 +152,7 @@ export function recallAndInline(
152
152
  let blockTokens = 0;
153
153
 
154
154
  for (const h of hits) {
155
- if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
155
+ if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId)) continue;
156
156
 
157
157
  // Inline dedupe: skip a hit already resident in the live window (Fix C).
158
158
  if (doWindowDedupe && liveEmbeddings.length > 0) {
@@ -168,7 +168,7 @@ export function recallAndInline(
168
168
  parts.push(part);
169
169
  toInject.push(h);
170
170
  blockTokens += partTokens;
171
- store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
171
+ vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
172
172
  }
173
173
 
174
174
  const block = parts.join("\n");
@@ -285,7 +285,7 @@ export async function recallMemoriesAndInline(
285
285
  */
286
286
  export async function recallAndInlineAsync(
287
287
  opts: RecallInjectOptions & { crossRepo?: boolean; repoId?: string },
288
- store: Pick<VectorStore, "searchAsync" | "wasInjected" | "markInjected">,
288
+ store: VectorStore,
289
289
  ): Promise<RecallInjectResult> {
290
290
  const limit = opts.limit ?? 3;
291
291
  const skip = opts.skipInjected ?? true;
@@ -295,7 +295,7 @@ export async function recallAndInlineAsync(
295
295
 
296
296
  let hits: SearchHit[] = [];
297
297
  try {
298
- hits = await store.searchAsync(opts.sessionId, opts.query, limit, {
298
+ hits = await vectorSearchAsync(store, opts.sessionId, opts.query, limit, {
299
299
  crossRepo: opts.crossRepo,
300
300
  repoId: opts.repoId,
301
301
  });
@@ -314,7 +314,7 @@ export async function recallAndInlineAsync(
314
314
  let blockTokens = 0;
315
315
 
316
316
  for (const h of hits) {
317
- if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
317
+ if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId)) continue;
318
318
  // S18: machine-wide injected-set — a foreign checkpoint already injected
319
319
  // (in any session) is never re-injected. Only applies to cross-repo hits
320
320
  // (same-repo hits have no repoId and are handled by the per-session set).
@@ -336,7 +336,7 @@ export async function recallAndInlineAsync(
336
336
  parts.push(part);
337
337
  toInject.push(h);
338
338
  blockTokens += partTokens;
339
- store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
339
+ vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
340
340
  // S18: record the cross-repo injection machine-wide so it's not re-injected
341
341
  // by a later recall (same or different session).
342
342
  if (opts.globalIndexDir && h.repoId) {