pi-mega-compact 0.8.21 → 0.8.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/LICENSE +6 -2
  2. package/README.md +1 -1
  3. package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
  4. package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
  5. package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
  6. package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
  7. package/dist/extensions/dashboard-server/html.js +2 -621
  8. package/dist/extensions/dashboard-server/routes-core.js +62 -0
  9. package/dist/extensions/dashboard-server/routes-game.js +323 -0
  10. package/dist/extensions/dashboard-server/routes-repo.js +170 -0
  11. package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
  12. package/dist/extensions/dashboard-server/routes.js +10 -0
  13. package/dist/extensions/dashboard-server/server.js +26 -623
  14. package/dist/extensions/mega-commands.js +4 -3
  15. package/dist/extensions/mega-events/agent-handlers.js +2 -1
  16. package/dist/extensions/mega-events/compact-handlers.js +26 -0
  17. package/dist/extensions/mega-events/session-handlers.js +2 -1
  18. package/dist/extensions/mega-pipeline/compact.js +3 -2
  19. package/dist/extensions/mega-runtime/state.js +7 -7
  20. package/dist/src/dedup/raptor/multilevel.js +172 -0
  21. package/dist/src/dedup/raptor/multilevel.test.js +203 -0
  22. package/dist/src/dedup/raptor/promote.test.js +5 -5
  23. package/dist/src/dedup/raptor/retrieval.js +1 -1
  24. package/dist/src/dedup/sprint12.test.js +7 -7
  25. package/dist/src/dedup-engine.test.js +29 -29
  26. package/dist/src/e2e.test.js +38 -38
  27. package/dist/src/engine.js +3 -3
  28. package/dist/src/engine.test.js +6 -6
  29. package/dist/src/importance.js +197 -0
  30. package/dist/src/importance.test.js +372 -0
  31. package/dist/src/ratio.bench.test.js +18 -18
  32. package/dist/src/recall.js +6 -5
  33. package/dist/src/recall.test.js +85 -27
  34. package/dist/src/sprint14.test.js +2 -2
  35. package/dist/src/store/migrate.test.js +5 -5
  36. package/dist/src/store/sprint10.test.js +5 -5
  37. package/dist/src/store/sqlite/global-index.js +5 -174
  38. package/dist/src/store/sqlite/global-sessions.js +190 -0
  39. package/dist/src/vector-read.js +168 -0
  40. package/dist/src/vector-search.js +191 -0
  41. package/dist/src/vectorStore.js +10 -297
  42. package/dist/src/vectorStore.test.js +32 -32
  43. package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
  44. package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
  45. package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
  46. package/extensions/dashboard-server/dashboard-client.ts +21 -0
  47. package/extensions/dashboard-server/html.ts +2 -621
  48. package/extensions/dashboard-server/routes-core.ts +113 -0
  49. package/extensions/dashboard-server/routes-game.ts +386 -0
  50. package/extensions/dashboard-server/routes-repo.ts +212 -0
  51. package/extensions/dashboard-server/routes-sessions.ts +195 -0
  52. package/extensions/dashboard-server/routes.ts +13 -0
  53. package/extensions/dashboard-server/server.ts +37 -700
  54. package/extensions/mega-commands.ts +4 -3
  55. package/extensions/mega-events/agent-handlers.ts +2 -1
  56. package/extensions/mega-events/compact-handlers.ts +28 -0
  57. package/extensions/mega-events/session-handlers.ts +2 -1
  58. package/extensions/mega-pipeline/compact.ts +3 -2
  59. package/extensions/mega-runtime/state.ts +7 -7
  60. package/extensions/openclaw-mega-compact.ts +2 -2
  61. package/package.json +2 -2
  62. package/src/dedup/raptor/multilevel.test.ts +278 -0
  63. package/src/dedup/raptor/multilevel.ts +246 -0
  64. package/src/dedup/raptor/promote.test.ts +5 -5
  65. package/src/dedup/raptor/retrieval.ts +1 -1
  66. package/src/dedup/sprint12.test.ts +7 -7
  67. package/src/dedup-engine.test.ts +30 -30
  68. package/src/e2e.test.ts +38 -38
  69. package/src/engine.test.ts +6 -6
  70. package/src/engine.ts +3 -3
  71. package/src/importance.test.ts +538 -0
  72. package/src/importance.ts +312 -0
  73. package/src/ratio.bench.test.ts +18 -18
  74. package/src/recall.test.ts +101 -29
  75. package/src/recall.ts +9 -9
  76. package/src/sprint14.test.ts +2 -2
  77. package/src/store/migrate.test.ts +5 -5
  78. package/src/store/sprint10.test.ts +5 -5
  79. package/src/store/sqlite/global-index.ts +18 -290
  80. package/src/store/sqlite/global-sessions.ts +291 -0
  81. package/src/vector-read.ts +237 -0
  82. package/src/vector-search.ts +231 -0
  83. package/src/vectorStore.test.ts +32 -32
  84. package/src/vectorStore.ts +29 -356
@@ -16,7 +16,7 @@ import fs from "fs";
16
16
  import path from "path";
17
17
  import os from "os";
18
18
  import { compactSession } from "./engine.js";
19
- import { VectorStore } from "./vectorStore.js";
19
+ import { VectorStore, vectorList, vectorStats, vectorWasInjected, vectorMarkInjected, vectorSearch } from "./vectorStore.js";
20
20
  import { extractiveSummarize } from "./extractive.js";
21
21
  import { estimateSessionTokens, estimateMessageTokens } from "./tokens.js";
22
22
  import { autoCompactCheck } from "./compact.js";
@@ -122,7 +122,7 @@ describe("Dedupe Levels", () => {
122
122
  const r2 = compactFull(s, SESS, [makeMsg("user", region)]);
123
123
  assert.equal(r2.deduped, true);
124
124
  assert.equal(r2.checkpointId, r1.checkpointId);
125
- assert.equal(s.list(SESS).length, 1);
125
+ assert.equal(vectorList(s, SESS).length, 1);
126
126
  });
127
127
  it("L0 only: distinct content stored twice creates two checkpoints", () => {
128
128
  const s = makeStore({ L0_ENABLED: true, L1_ENABLED: false, L2_ENABLED: false });
@@ -133,7 +133,7 @@ describe("Dedupe Levels", () => {
133
133
  assert.equal(r1.deduped, false);
134
134
  assert.equal(r2.deduped, false);
135
135
  assert.notEqual(r1.checkpointId, r2.checkpointId);
136
- assert.equal(s.list(SESS).length, 2);
136
+ assert.equal(vectorList(s, SESS).length, 2);
137
137
  });
138
138
  it("L1 only: one-word variants collapse; major rewrites do not", () => {
139
139
  const s = makeStore({ L0_ENABLED: false, L1_ENABLED: true, L2_ENABLED: false });
@@ -144,10 +144,10 @@ describe("Dedupe Levels", () => {
144
144
  assert.equal(r1.deduped, false);
145
145
  const r2 = s.add({ sessionId: SESS, summary: "migration", regionText: variant, timestamp: 2 });
146
146
  assert.equal(r2.deduped, true, "one-word variant should be collapsed by L1");
147
- assert.equal(s.list(SESS).length, 1);
147
+ assert.equal(vectorList(s, SESS).length, 1);
148
148
  const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: rewrite, timestamp: 3 });
149
149
  assert.equal(r3.deduped, false, "major rewrite should not be collapsed by L1");
150
- assert.equal(s.list(SESS).length, 2);
150
+ assert.equal(vectorList(s, SESS).length, 2);
151
151
  });
152
152
  it("L2 only: semantic paraphrases collapse; unrelated topics do not", () => {
153
153
  // Use a lower threshold and longer, lexically-overlapping paraphrase so the
@@ -161,10 +161,10 @@ describe("Dedupe Levels", () => {
161
161
  assert.equal(r1.deduped, false);
162
162
  const r2 = s.add({ sessionId: SESS, summary: "auth paraphrase", regionText: paraphrase, timestamp: 2 });
163
163
  assert.equal(r2.deduped, true, "semantic paraphrase should be collapsed by L2");
164
- assert.equal(s.list(SESS).length, 1);
164
+ assert.equal(vectorList(s, SESS).length, 1);
165
165
  const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: unrelated, timestamp: 3 });
166
166
  assert.equal(r3.deduped, false, "unrelated topic should not be collapsed by L2");
167
- assert.equal(s.list(SESS).length, 2);
167
+ assert.equal(vectorList(s, SESS).length, 2);
168
168
  });
169
169
  it("All tiers disabled: every store.add() with different region text creates a distinct checkpoint", () => {
170
170
  // Even with all dedup tiers disabled, the store still enforces a unique
@@ -178,7 +178,7 @@ describe("Dedupe Levels", () => {
178
178
  assert.equal(r3.deduped, false);
179
179
  assert.notEqual(r1.checkpoint.checkpointId, r2.checkpoint.checkpointId);
180
180
  assert.notEqual(r2.checkpoint.checkpointId, r3.checkpoint.checkpointId);
181
- assert.equal(s.list(SESS).length, 3);
181
+ assert.equal(vectorList(s, SESS).length, 3);
182
182
  });
183
183
  it("Combined L0+L1+L2: layered behavior exact -> near -> semantic", () => {
184
184
  const s = makeStore({ L0_ENABLED: true, L1_ENABLED: true, L2_ENABLED: true });
@@ -202,8 +202,8 @@ describe("Dedupe Levels", () => {
202
202
  if (r4.deduped) {
203
203
  okReason(r4.dedupReason, ["contentSimilarity", "l1MinHash"]);
204
204
  }
205
- assert.ok(s.list(SESS).length >= 1, "layered dedup keeps at least one checkpoint");
206
- assert.ok(s.list(SESS).length <= 4, "layered dedup should not explode to many checkpoints");
205
+ assert.ok(vectorList(s, SESS).length >= 1, "layered dedup keeps at least one checkpoint");
206
+ assert.ok(vectorList(s, SESS).length <= 4, "layered dedup should not explode to many checkpoints");
207
207
  });
208
208
  });
209
209
  function okReason(reason, expected) {
@@ -242,16 +242,16 @@ describe("Compression / Store Stats", () => {
242
242
  compactFull(s, SESS, [makeMsg("user", unique)], 1);
243
243
  const dup = "duplicate topic about payment gateway integration";
244
244
  compactFull(s, SESS, [makeMsg("user", dup)], 1);
245
- const statsBefore = s.stats(SESS);
245
+ const statsBefore = vectorStats(s, SESS);
246
246
  assert.ok(statsBefore.checkpointCount >= 1, "checkpointCount should be positive");
247
247
  assert.ok(statsBefore.totalTokenEstimate >= 0, "totalTokenEstimate should be non-negative");
248
248
  assert.equal(statsBefore.dedupHitRate, 0, "no injections yet => dedupHitRate 0");
249
249
  assert.equal(statsBefore.injectedCount, 0, "no injections yet => injectedCount 0");
250
- const hits = s.search(SESS, "payment gateway", 5);
250
+ const hits = vectorSearch(s, SESS, "payment gateway", 5);
251
251
  assert.ok(hits.length > 0, "should find the stored checkpoint");
252
252
  const cpId = hits[0].checkpoint.checkpointId;
253
- s.markInjected(SESS, cpId);
254
- const statsAfter = s.stats(SESS);
253
+ vectorMarkInjected(s, SESS, cpId);
254
+ const statsAfter = vectorStats(s, SESS);
255
255
  assert.equal(statsAfter.injectedCount, 1, "injectedCount tracks markInjected");
256
256
  if (statsAfter.checkpointCount > 0) {
257
257
  assert.ok(Math.abs(statsAfter.dedupHitRate - 1 / statsAfter.checkpointCount) < 0.001, "dedupHitRate = injected / checkpoints");
@@ -267,10 +267,10 @@ describe("Compression / Store Stats", () => {
267
267
  }
268
268
  compactFull(s, SESS, [makeMsg("user", "unique region for hit-rate measurement variant")], 1);
269
269
  // Mark the first as injected.
270
- const first = s.search(SESS, base, 1)[0]?.checkpoint.checkpointId;
270
+ const first = vectorSearch(s, SESS, base, 1)[0]?.checkpoint.checkpointId;
271
271
  if (first)
272
- s.markInjected(SESS, first);
273
- const stats = s.stats(SESS);
272
+ vectorMarkInjected(s, SESS, first);
273
+ const stats = vectorStats(s, SESS);
274
274
  assert.ok(stats.checkpointCount >= 1);
275
275
  assert.ok(stats.dedupHitRate > 0 || stats.checkpointCount === 1, "hit rate should be positive when there are multiple checkpoint");
276
276
  });
@@ -304,13 +304,13 @@ describe("Recall & Dedup Sentinel", () => {
304
304
  const s = makeStore();
305
305
  const region = "manual sentinel tracking without recallAndInline";
306
306
  compactFull(s, SESS, [makeMsg("user", region)]);
307
- const hits = s.search(SESS, "manual sentinel", 3);
307
+ const hits = vectorSearch(s, SESS, "manual sentinel", 3);
308
308
  assert.ok(hits.length > 0, "search should return checkpoint");
309
309
  const cpId = hits[0].checkpoint.checkpointId;
310
- assert.equal(s.wasInjected(SESS, cpId), false, "not yet injected");
311
- s.markInjected(SESS, cpId);
312
- assert.equal(s.wasInjected(SESS, cpId), true, "markInjected recorded");
313
- const hits2 = s.search(SESS, "manual sentinel", 3).filter((h) => !s.wasInjected(SESS, h.checkpoint.checkpointId));
310
+ assert.equal(vectorWasInjected(s, SESS, cpId), false, "not yet injected");
311
+ vectorMarkInjected(s, SESS, cpId);
312
+ assert.equal(vectorWasInjected(s, SESS, cpId), true, "markInjected recorded");
313
+ const hits2 = vectorSearch(s, SESS, "manual sentinel", 3).filter((h) => !vectorWasInjected(s, SESS, h.checkpoint.checkpointId));
314
314
  assert.equal(hits2.length, 0, "filtered search excludes injected checkpoint");
315
315
  });
316
316
  });
@@ -322,13 +322,13 @@ describe("Edge Cases", () => {
322
322
  const r = compactSession({ sessionId: SESS, messages: [], keepFrom: 0 }, s);
323
323
  assert.equal(r.skipped, true);
324
324
  assert.equal(r.summary, "");
325
- assert.equal(s.list(SESS).length, 0);
325
+ assert.equal(vectorList(s, SESS).length, 0);
326
326
  });
327
327
  it("single message with keepFrom=0 returns skipped", () => {
328
328
  const s = makeStore();
329
329
  const r = compactSession({ sessionId: SESS, messages: [makeMsg("user", "only one message")], keepFrom: 0 }, s);
330
330
  assert.equal(r.skipped, true);
331
- assert.equal(s.list(SESS).length, 0);
331
+ assert.equal(vectorList(s, SESS).length, 0);
332
332
  });
333
333
  it("keepFrom at messages.length compacts all prior messages (verified behavior)", () => {
334
334
  // The engine treats keepFrom as the compactable boundary: messages[0..keepFrom)
@@ -339,7 +339,7 @@ describe("Edge Cases", () => {
339
339
  const r = compactSession({ sessionId: SESS, messages, keepFrom: messages.length }, s);
340
340
  assert.equal(r.skipped, false);
341
341
  assert.ok(r.checkpointId);
342
- assert.equal(s.list(SESS).length, 1);
342
+ assert.equal(vectorList(s, SESS).length, 1);
343
343
  });
344
344
  it("unicode and emoji messages store and retrieve intact", () => {
345
345
  const s = makeStore();
@@ -347,7 +347,7 @@ describe("Edge Cases", () => {
347
347
  "日本語テキスト 日本語テキスト 👍🔥";
348
348
  const r = compactFull(s, SESS, [makeMsg("user", text)], 1);
349
349
  assert.equal(r.skipped, false);
350
- const stored = s.list(SESS)[0];
350
+ const stored = vectorList(s, SESS)[0];
351
351
  assert.ok(stored);
352
352
  const recovered = Buffer.from(stored.compressedOriginal ?? Buffer.alloc(0));
353
353
  assert.ok(recovered.toString("utf-8").includes("🎉"), "emoji recovered from compressedOriginal");
@@ -360,8 +360,8 @@ describe("Edge Cases", () => {
360
360
  const r = compactFull(s, SESS, [makeMsg("user", big)], 1);
361
361
  assert.equal(r.skipped, false);
362
362
  assert.ok(r.checkpointId);
363
- assert.equal(s.list(SESS).length, 1);
364
- const stats = s.stats(SESS);
363
+ assert.equal(vectorList(s, SESS).length, 1);
364
+ const stats = vectorStats(s, SESS);
365
365
  assert.ok(stats.totalTokenEstimate > 0);
366
366
  });
367
367
  it("mixed roles (user/assistant/tool) are included in summary", () => {
@@ -380,7 +380,7 @@ describe("Edge Cases", () => {
380
380
  r.summary.includes("Edit") ||
381
381
  r.summary.includes("user") ||
382
382
  r.summary.includes("assistant"), "summary should reference roles or tools");
383
- assert.equal(s.list(SESS).length, 1);
383
+ assert.equal(vectorList(s, SESS).length, 1);
384
384
  });
385
385
  });
386
386
  // -------------------- 7. Tier Switching (percentage-based) --------------------
@@ -9,7 +9,7 @@ import assert from "node:assert/strict";
9
9
  import { mkdtempSync, rmSync } from "node:fs";
10
10
  import { tmpdir } from "node:os";
11
11
  import { join } from "node:path";
12
- import { VectorStore, computeRegionHash } from "./vectorStore.js";
12
+ import { VectorStore, computeRegionHash, vectorList, vectorSemDedup, vectorWasInjected, vectorMarkInjected, vectorTopSimilar, vectorStats, vectorSearch } from "./vectorStore.js";
13
13
  import { compactSession, recall, supersededCount } from "./engine.js";
14
14
  import { loadDedupConfig } from "./config/dedup.js";
15
15
  import { defaultEmbedder } from "./embedder.js";
@@ -101,8 +101,8 @@ test("1. Full compaction pipeline: SUPERSEDE → COLLAPSE → CLUSTER", () => {
101
101
  // Verify: checkpoint persisted
102
102
  assert.ok(r.checkpointId, "checkpointId should be set");
103
103
  assert.match(r.checkpointId, /^chkpt_001$/);
104
- // Verify: checkpoint is searchable via store.search()
105
- const hits = s.search(SESS, "server.ts memory leak JWT auth", 5);
104
+ // Verify: checkpoint is searchable via vectorSearch(store, )
105
+ const hits = vectorSearch(s, SESS, "server.ts memory leak JWT auth", 5);
106
106
  assert.ok(hits.length > 0, "checkpoint should be searchable");
107
107
  assert.equal(hits[0].checkpoint.checkpointId, r.checkpointId);
108
108
  });
@@ -133,7 +133,7 @@ test("2. Multi-session dedup: identical content in same session detected by L0",
133
133
  assert.equal(rB.deduped, true, "second identical compaction should be deduped by L0");
134
134
  assert.ok(rB.dedupReason === "regionHash" || rB.dedupReason === "contentHash", `dedup reason should be L0 (regionHash or contentHash), got ${rB.dedupReason}`);
135
135
  // Search should still return the checkpoint
136
- const hits = s.search("sess_multi_a", "database connection pooling", 5);
136
+ const hits = vectorSearch(s, "sess_multi_a", "database connection pooling", 5);
137
137
  assert.ok(hits.length > 0, "search returns the deduped result");
138
138
  assert.equal(hits[0].checkpoint.checkpointId, rA.checkpointId);
139
139
  });
@@ -162,7 +162,7 @@ test("3. Near-duplicate detection: L1 MinHash/LSH catches one-word edits that L0
162
162
  // L1 should catch the near-dup
163
163
  assert.equal(r2.deduped, true, "L1 should catch the near-duplicate");
164
164
  assert.equal(r2.reason, "l1MinHash", "dedup reason should be l1MinHash");
165
- assert.equal(s.list(SESS).length, 1, "only one checkpoint stored");
165
+ assert.equal(vectorList(s, SESS).length, 1, "only one checkpoint stored");
166
166
  });
167
167
  test("3b. Negative: L1 does NOT collapse genuinely different content", () => {
168
168
  const s = store({ L2_ENABLED: false });
@@ -180,7 +180,7 @@ test("3b. Negative: L1 does NOT collapse genuinely different content", () => {
180
180
  timestamp: 2,
181
181
  });
182
182
  assert.equal(r2.deduped, false, "distinct content should not be deduped");
183
- assert.equal(s.list(SESS).length, 2, "both checkpoints stored");
183
+ assert.equal(vectorList(s, SESS).length, 2, "both checkpoints stored");
184
184
  });
185
185
  // ---------------------------------------------------------------------------
186
186
  // 4. Semantic Dedup (L2 cosine)
@@ -208,7 +208,7 @@ test("4. Semantic dedup: L2 cosine catches highly similar content that L0/L1 mis
208
208
  // L2 semantic cosine should catch this high-similarity paraphrase
209
209
  assert.equal(r2.deduped, true, "L2 should catch the high-similarity paraphrase");
210
210
  assert.ok(r2.reason === "contentSimilarity" || r2.reason === "l1MinHash", `dedup reason should be L2 contentSimilarity or L1 l1MinHash, got ${r2.reason}`);
211
- assert.equal(s.list(SESS).length, 1, "only one checkpoint after semantic dedup");
211
+ assert.equal(vectorList(s, SESS).length, 1, "only one checkpoint after semantic dedup");
212
212
  });
213
213
  test("4b. MMR diversifies search results", () => {
214
214
  const s = store();
@@ -230,7 +230,7 @@ test("4b. MMR diversifies search results", () => {
230
230
  });
231
231
  }
232
232
  // Search for parser-related content — should return relevant results
233
- const hits = s.search(SESS, "parser tokenization source code files", 4);
233
+ const hits = vectorSearch(s, SESS, "parser tokenization source code files", 4);
234
234
  assert.ok(hits.length >= 1, "should return at least one result");
235
235
  // The top hit should be the parser checkpoint
236
236
  assert.ok(hits[0].checkpoint.summary.includes("topic_0"), "top hit is the parser topic");
@@ -255,20 +255,20 @@ test("5. SemDeDup offline cleanup: redundant rows marked removed, not deleted",
255
255
  { id: "chkpt_003", text: "the frontend uses a virtualized list for rendering large datasets efficiently", tok: 500, ts: 3 },
256
256
  ]);
257
257
  // Run SemDeDup with threshold that catches the near-identical pair
258
- const removed = s.semDedup(SESS, 0.85);
258
+ const removed = vectorSemDedup(s, SESS, 0.85);
259
259
  assert.equal(removed, 1, "one redundant row should be marked removed");
260
260
  // Verify: the lower-tokenEstimate row is the one removed
261
- const all = s.list(SESS);
261
+ const all = vectorList(s, SESS);
262
262
  const dropped = all.find((c) => c.dedupStatus === "removed");
263
263
  assert.ok(dropped, "a row should have dedup_status='removed'");
264
264
  assert.equal(dropped.checkpointId, "chkpt_001", "lower tokenEstimate row removed");
265
265
  assert.equal(dropped.dedupStatus, "removed");
266
266
  // Verify: search excludes removed rows (3 total, 1 removed → 2 active)
267
- const hits = s.search(SESS, "cache parsed ast nodes frontend virtualized list", 10);
267
+ const hits = vectorSearch(s, SESS, "cache parsed ast nodes frontend virtualized list", 10);
268
268
  assert.equal(hits.length, 2, "search excludes the removed row, returns 2 active");
269
269
  assert.ok(hits.every((h) => h.checkpoint.dedupStatus !== "removed"), "no removed rows in search");
270
270
  // Verify: idempotent re-run removes nothing new
271
- const secondRun = s.semDedup(SESS, 0.85);
271
+ const secondRun = vectorSemDedup(s, SESS, 0.85);
272
272
  assert.equal(secondRun, 0, "idempotent re-run removes nothing new");
273
273
  closeStore(dir);
274
274
  });
@@ -293,15 +293,15 @@ test("6. Recall + inline with dedup sentinel: injected checkpoints are skipped",
293
293
  assert.equal(first.hits.length, 1);
294
294
  // Mark it injected
295
295
  const cpId = first.hits[0].checkpoint.checkpointId;
296
- s.markInjected(SESS, cpId);
296
+ vectorMarkInjected(s, SESS, cpId);
297
297
  // Recall again — should skip the injected checkpoint
298
298
  const second = recall({ sessionId: SESS, query: "vectorStore dedup cascade", limit: 5, skipInjected: true }, s);
299
299
  assert.equal(second.newHits.length, 0, "injected checkpoint should be skipped (dedup sentinel)");
300
300
  // Without skipInjected, hits still surface
301
301
  assert.equal(second.hits.length, 1, "hits still surface without skipInjected");
302
302
  // Verify wasInjected
303
- assert.equal(s.wasInjected(SESS, cpId), true, "wasInjected returns true for injected checkpoint");
304
- assert.equal(s.wasInjected(SESS, "chkpt_999"), false, "wasInjected returns false for non-injected");
303
+ assert.equal(vectorWasInjected(s, SESS, cpId), true, "wasInjected returns true for injected checkpoint");
304
+ assert.equal(vectorWasInjected(s, SESS, "chkpt_999"), false, "wasInjected returns false for non-injected");
305
305
  });
306
306
  // ---------------------------------------------------------------------------
307
307
  // 7. topSimilar
@@ -326,11 +326,11 @@ test("7. topSimilar: returns n most cosine-similar checkpoints", () => {
326
326
  });
327
327
  }
328
328
  // Call topSimilar(3)
329
- const hits = s.topSimilar(SESS, 3);
329
+ const hits = vectorTopSimilar(s, SESS, 3);
330
330
  assert.ok(hits.length <= 3, "should respect n limit");
331
331
  assert.ok(hits.length > 0, "should return results");
332
332
  // Verify self-exclusion: the most recent checkpoint should not be in results
333
- const all = s.list(SESS);
333
+ const all = vectorList(s, SESS);
334
334
  const ordered = [...all].sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
335
335
  const current = ordered[ordered.length - 1];
336
336
  assert.ok(!hits.some((h) => h.checkpoint.checkpointId === current.checkpointId), "current checkpoint excluded");
@@ -358,14 +358,14 @@ test("7b. topSimilar respects n limit strictly", () => {
358
358
  timestamp: i + 1,
359
359
  });
360
360
  }
361
- const hits = s.topSimilar(SESS, 2);
361
+ const hits = vectorTopSimilar(s, SESS, 2);
362
362
  assert.equal(hits.length, 2, "n limit respected");
363
363
  });
364
364
  test("7c. topSimilar returns empty for sessions with 0 or 1 checkpoints", () => {
365
365
  const s = store();
366
- assert.deepEqual(s.topSimilar("sess_empty", 5), []);
366
+ assert.deepEqual(vectorTopSimilar(s, "sess_empty", 5), []);
367
367
  s.add({ sessionId: "sess_one", summary: "solo", regionText: "only checkpoint here", timestamp: 1 });
368
- assert.deepEqual(s.topSimilar("sess_one", 5), []);
368
+ assert.deepEqual(vectorTopSimilar(s, "sess_one", 5), []);
369
369
  });
370
370
  // ---------------------------------------------------------------------------
371
371
  // 8. Compression Round-Trip
@@ -390,7 +390,7 @@ test("8. Compression round-trip: decompression produces original content", () =>
390
390
  timestamp: 1,
391
391
  });
392
392
  // Retrieve and verify decompression
393
- const cp = s.list(SESS)[0];
393
+ const cp = vectorList(s, SESS)[0];
394
394
  assert.ok(cp.compressedOriginal instanceof Buffer, "compressedOriginal is a Buffer");
395
395
  const restored = decompressSmart(cp.compressedOriginal).toString("utf-8");
396
396
  assert.equal(restored, originalText, "decompression produces the original content");
@@ -407,7 +407,7 @@ test("8b. Compression round-trip with different content sizes", () => {
407
407
  // Large text (>4KB → gzip level 6 or brotli)
408
408
  const large = "detailed region text with lots of context about the codebase. ".repeat(200);
409
409
  s.add({ sessionId: SESS, summary: "large", regionText: large, timestamp: 3 });
410
- const checkpoints = s.list(SESS);
410
+ const checkpoints = vectorList(s, SESS);
411
411
  assert.equal(checkpoints.length, 3);
412
412
  // Verify each round-trips correctly
413
413
  const texts = [small, medium, large];
@@ -444,7 +444,7 @@ test("9. Store stats and metrics reflect actual state after compactions", () =>
444
444
  const r3 = compactSession({ sessionId: SESS, messages: conv3, keepFrom: 2, timestamp: 3 }, s);
445
445
  assert.equal(r3.deduped, false);
446
446
  // Get stats
447
- const st = s.stats(SESS);
447
+ const st = vectorStats(s, SESS);
448
448
  assert.equal(st.checkpointCount, 3, "three checkpoints stored");
449
449
  assert.ok(st.totalTokenEstimate > 0, "totalTokenEstimate should be positive");
450
450
  assert.equal(st.lastCheckpointId, "chkpt_003", "last checkpoint id correct");
@@ -452,14 +452,14 @@ test("9. Store stats and metrics reflect actual state after compactions", () =>
452
452
  assert.equal(st.injectedCount, 0, "no injections yet");
453
453
  assert.equal(st.dedupHitRate, 0, "dedupHitRate is 0 with no injections");
454
454
  // Mark one injected and re-check
455
- s.markInjected(SESS, "chkpt_001");
456
- const st2 = s.stats(SESS);
455
+ vectorMarkInjected(s, SESS, "chkpt_001");
456
+ const st2 = vectorStats(s, SESS);
457
457
  assert.equal(st2.injectedCount, 1, "one injection tracked");
458
458
  assert.ok(Math.abs(st2.dedupHitRate - 1 / 3) < 1e-9, "dedupHitRate = injected/checkpoints");
459
459
  });
460
460
  test("9b. Stats on empty session returns zeros", () => {
461
461
  const s = store();
462
- const st = s.stats("sess_nothing");
462
+ const st = vectorStats(s, "sess_nothing");
463
463
  assert.equal(st.checkpointCount, 0);
464
464
  assert.equal(st.totalTokenEstimate, 0);
465
465
  assert.equal(st.lastCheckpointId, undefined);
@@ -496,15 +496,15 @@ test("10. Concurrent sessions: no cross-contamination", () => {
496
496
  });
497
497
  // Verify: each session has exactly one checkpoint
498
498
  sessions.forEach((sess) => {
499
- const cps = s.list(sess.id);
499
+ const cps = vectorList(s, sess.id);
500
500
  assert.equal(cps.length, 1, `${sess.id} should have 1 checkpoint`);
501
501
  });
502
502
  // Verify: search in one session doesn't return another session's checkpoints
503
- const hitsA = s.search("sess_concurrent_a", "authentication JWT", 5);
503
+ const hitsA = vectorSearch(s, "sess_concurrent_a", "authentication JWT", 5);
504
504
  assert.ok(hitsA.every((h) => h.checkpoint.sessionId === "sess_concurrent_a"), "search in A returns only A's checkpoints");
505
- const hitsB = s.search("sess_concurrent_b", "database queries indexes", 5);
505
+ const hitsB = vectorSearch(s, "sess_concurrent_b", "database queries indexes", 5);
506
506
  assert.ok(hitsB.every((h) => h.checkpoint.sessionId === "sess_concurrent_b"), "search in B returns only B's checkpoints");
507
- const hitsC = s.search("sess_concurrent_c", "canvas rendering bug", 5);
507
+ const hitsC = vectorSearch(s, "sess_concurrent_c", "canvas rendering bug", 5);
508
508
  assert.ok(hitsC.every((h) => h.checkpoint.sessionId === "sess_concurrent_c"), "search in C returns only C's checkpoints");
509
509
  });
510
510
  // ---------------------------------------------------------------------------
@@ -516,7 +516,7 @@ test("11a. Empty session (no messages) — compactSession should skip", () => {
516
516
  assert.equal(r.skipped, true, "empty session should be skipped");
517
517
  assert.equal(r.summary, "");
518
518
  assert.equal(r.regionHash, "");
519
- assert.equal(s.list("sess_empty_msgs").length, 0);
519
+ assert.equal(vectorList(s, "sess_empty_msgs").length, 0);
520
520
  });
521
521
  test("11b. Single message session — should skip or handle gracefully", () => {
522
522
  const s = store();
@@ -549,7 +549,7 @@ test("11c. Very large region text — should still compact and store", () => {
549
549
  assert.ok(r.checkpointId, "checkpoint created");
550
550
  assert.ok(r.summary.length > 0, "summary produced");
551
551
  // Verify it's searchable
552
- const hits = s.search(SESS, "module feature fix review", 3);
552
+ const hits = vectorSearch(s, SESS, "module feature fix review", 3);
553
553
  assert.ok(hits.length > 0, "large checkpoint is searchable");
554
554
  });
555
555
  test("11d. Unicode and emoji content — should normalize and hash correctly", () => {
@@ -565,7 +565,7 @@ test("11d. Unicode and emoji content — should normalize and hash correctly", (
565
565
  assert.equal(r.skipped, false, "unicode conversation should compact");
566
566
  assert.ok(r.regionHash.length > 0, "regionHash computed for unicode content");
567
567
  // Verify the checkpoint exists and is searchable
568
- const hits = s.search(SESS, "中文 translation 本地化", 3);
568
+ const hits = vectorSearch(s, SESS, "中文 translation 本地化", 3);
569
569
  assert.ok(hits.length > 0, "unicode checkpoint is searchable");
570
570
  // Verify regionHash is deterministic for the same unicode content
571
571
  const r2 = compactSession({ sessionId: SESS, messages, keepFrom: 4, timestamp: 2 }, s);
@@ -588,7 +588,7 @@ test("11e. All messages in preserve-recent window — nothing to compact", () =>
588
588
  timestamp: 1,
589
589
  }, s);
590
590
  assert.equal(r.skipped, true, "should skip when keepFrom=0 (nothing to compact)");
591
- assert.equal(s.list("sess_all_preserved").length, 0, "no checkpoints stored");
591
+ assert.equal(vectorList(s, "sess_all_preserved").length, 0, "no checkpoints stored");
592
592
  });
593
593
  // ---------------------------------------------------------------------------
594
594
  // 12. Feature Flag Toggling
@@ -616,7 +616,7 @@ test("12a. L0_ENABLED=false — L0 does not collapse exact dups", () => {
616
616
  assert.equal(r1.deduped, false);
617
617
  // With L0 off (and L1/L2 off too), near-dup should NOT be collapsed
618
618
  assert.equal(r2.deduped, false, "L0 disabled → near-dup not collapsed");
619
- assert.equal(s.list(SESS).length, 2, "both checkpoints stored when L0 is off");
619
+ assert.equal(vectorList(s, SESS).length, 2, "both checkpoints stored when L0 is off");
620
620
  });
621
621
  test("12b. MARK_ONLY_L1=true — L1 records but does not collapse", () => {
622
622
  // Disable L2 to isolate L1 behavior
@@ -637,9 +637,9 @@ test("12b. MARK_ONLY_L1=true — L1 records but does not collapse", () => {
637
637
  assert.equal(r1.deduped, false);
638
638
  // MARK_ONLY_L1 → L1 detects the near-dup but does NOT collapse
639
639
  assert.equal(r2.deduped, false, "MARK_ONLY_L1 → not collapsed");
640
- assert.equal(s.list(SESS).length, 2, "both checkpoints stored (mark only)");
640
+ assert.equal(vectorList(s, SESS).length, 2, "both checkpoints stored (mark only)");
641
641
  // Both should be active (not removed)
642
- const all = s.list(SESS);
642
+ const all = vectorList(s, SESS);
643
643
  assert.ok(all.every((c) => c.dedupStatus === "active"), "both rows active under MARK_ONLY");
644
644
  });
645
645
  test("12c. L2_ENABLED=false — L2 skipped but L0/L1 still work", () => {
@@ -688,7 +688,7 @@ test("12d. All tiers disabled — nothing deduped, everything stored", () => {
688
688
  });
689
689
  assert.equal(r1.deduped, false);
690
690
  assert.equal(r2.deduped, false, "all tiers off → nothing deduped");
691
- assert.equal(s.list(SESS).length, 2, "both stored");
691
+ assert.equal(vectorList(s, SESS).length, 2, "both stored");
692
692
  });
693
693
  // ---------------------------------------------------------------------------
694
694
  // Cleanup
@@ -15,7 +15,7 @@ import { findSuperseded, supersede } from "./supersede.js";
15
15
  import { summarizeMessages, mergeCompactSummaries, formatCompactSummary } from "./compact.js";
16
16
  import { extractiveSummarize } from "./extractive.js";
17
17
  import { estimateSessionTokens, estimateBlockTokens } from "./tokens.js";
18
- import { computeRegionHash, VectorStore } from "./vectorStore.js";
18
+ import { computeRegionHash, vectorWasInjected, vectorSearch, VectorStore } from "./vectorStore.js";
19
19
  /** Default store used by the convenience `compactSession`. */
20
20
  let defaultStore;
21
21
  export function getDefaultStore(stateDir) {
@@ -125,8 +125,8 @@ export function compactSession(input, store = getDefaultStore()) {
125
125
  * inject (Sprint 4 wires injection); this module only does the deduped search.
126
126
  */
127
127
  export function recall(input, store = getDefaultStore()) {
128
- const hits = store.search(input.sessionId, input.query, input.limit ?? 3);
129
- const newHits = input.skipInjected === false ? hits : hits.filter((h) => !store.wasInjected(input.sessionId, h.checkpoint.checkpointId));
128
+ const hits = vectorSearch(store, input.sessionId, input.query, input.limit ?? 3);
129
+ const newHits = input.skipInjected === false ? hits : hits.filter((h) => !vectorWasInjected(store, input.sessionId, h.checkpoint.checkpointId));
130
130
  return { hits, newHits };
131
131
  }
132
132
  /** Merge a freshly compacted summary into the prior persisted summary text. */
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
3
3
  import { mkdtempSync, rmSync } from "node:fs";
4
4
  import { tmpdir } from "node:os";
5
5
  import { join } from "node:path";
6
- import { VectorStore } from "./vectorStore.js";
6
+ import { VectorStore, vectorMarkInjected, vectorSearch } from "./vectorStore.js";
7
7
  import { compactSession, recall, mergeSummary, supersededCount } from "./engine.js";
8
8
  const baseTmp = mkdtempSync(join(tmpdir(), "mc-engine-"));
9
9
  let counter = 0;
@@ -35,7 +35,7 @@ test("compactSession supersedes then persists a checkpoint", () => {
35
35
  assert.equal(supersededCount(messages.slice(0, 4)), 1);
36
36
  assert.equal(r.compactedFrom, 4);
37
37
  // The persisted checkpoint is searchable.
38
- assert.equal(s.search(SESS, "bug src/server.ts", 5).length, 1);
38
+ assert.equal(vectorSearch(s, SESS, "bug src/server.ts", 5).length, 1);
39
39
  });
40
40
  test("compactSession is idempotent on identical region (dedup sentinel)", () => {
41
41
  const s = store();
@@ -50,20 +50,20 @@ test("compactSession is idempotent on identical region (dedup sentinel)", () =>
50
50
  assert.equal(r1.deduped, false);
51
51
  assert.equal(r2.deduped, true);
52
52
  assert.equal(r1.checkpointId, r2.checkpointId);
53
- assert.equal(s.search(SESS, "parser", 10).length, 1);
53
+ assert.equal(vectorSearch(s, SESS, "parser", 10).length, 1);
54
54
  });
55
55
  test("compactSession skipped when slice is empty", () => {
56
56
  const s = store();
57
57
  const r = compactSession({ sessionId: SESS, messages: [msg("user", "only tail")], keepFrom: 0 }, s);
58
58
  assert.equal(r.skipped, true);
59
- assert.equal(s.search(SESS, "x", 5).length, 0);
59
+ assert.equal(vectorSearch(s, SESS, "x", 5).length, 0);
60
60
  });
61
61
  test("recall drops already-injected checkpoints", () => {
62
62
  const s = store();
63
63
  compactSession({ sessionId: SESS, messages: [msg("user", "investigated src/vectorStore.ts"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
64
64
  const first = recall({ sessionId: SESS, query: "vectorStore", limit: 5, skipInjected: true }, s);
65
65
  assert.equal(first.newHits.length, 1);
66
- s.markInjected(SESS, first.hits[0].checkpoint.checkpointId);
66
+ vectorMarkInjected(s, SESS, first.hits[0].checkpoint.checkpointId);
67
67
  const second = recall({ sessionId: SESS, query: "vectorStore", limit: 5, skipInjected: true }, s);
68
68
  assert.equal(second.newHits.length, 0);
69
69
  // Without the skip flag, both hits still surface.
@@ -100,7 +100,7 @@ test("compactSession with useExtractive produces topicSummary on checkpoint", ()
100
100
  assert.equal(r.skipped, false);
101
101
  assert.ok(r.checkpointId, "checkpoint created");
102
102
  // The checkpoint should have topicSummary populated
103
- const hits = s.search("sess_extr", "auth refactor", 5);
103
+ const hits = vectorSearch(s, "sess_extr", "auth refactor", 5);
104
104
  assert.ok(hits.length > 0);
105
105
  // topicSummary should be present on the stored checkpoint (via extractive path)
106
106
  assert.ok(hits[0].checkpoint.topicSummary, "topicSummary should be populated when useExtractive is true");