pi-mega-compact 0.8.21 → 0.8.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +6 -2
- package/README.md +1 -1
- package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
- package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
- package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
- package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
- package/dist/extensions/dashboard-server/html.js +2 -621
- package/dist/extensions/dashboard-server/routes-core.js +62 -0
- package/dist/extensions/dashboard-server/routes-game.js +323 -0
- package/dist/extensions/dashboard-server/routes-repo.js +170 -0
- package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
- package/dist/extensions/dashboard-server/routes.js +10 -0
- package/dist/extensions/dashboard-server/server.js +26 -623
- package/dist/extensions/mega-commands.js +4 -3
- package/dist/extensions/mega-events/agent-handlers.js +2 -1
- package/dist/extensions/mega-events/compact-handlers.js +26 -0
- package/dist/extensions/mega-events/session-handlers.js +2 -1
- package/dist/extensions/mega-pipeline/compact.js +3 -2
- package/dist/extensions/mega-runtime/state.js +7 -7
- package/dist/src/dedup/raptor/multilevel.js +172 -0
- package/dist/src/dedup/raptor/multilevel.test.js +203 -0
- package/dist/src/dedup/raptor/promote.test.js +5 -5
- package/dist/src/dedup/raptor/retrieval.js +1 -1
- package/dist/src/dedup/sprint12.test.js +7 -7
- package/dist/src/dedup-engine.test.js +29 -29
- package/dist/src/e2e.test.js +38 -38
- package/dist/src/engine.js +3 -3
- package/dist/src/engine.test.js +6 -6
- package/dist/src/importance.js +197 -0
- package/dist/src/importance.test.js +372 -0
- package/dist/src/ratio.bench.test.js +18 -18
- package/dist/src/recall.js +6 -5
- package/dist/src/recall.test.js +85 -27
- package/dist/src/sprint14.test.js +2 -2
- package/dist/src/store/migrate.test.js +5 -5
- package/dist/src/store/sprint10.test.js +5 -5
- package/dist/src/store/sqlite/global-index.js +5 -174
- package/dist/src/store/sqlite/global-sessions.js +190 -0
- package/dist/src/vector-read.js +168 -0
- package/dist/src/vector-search.js +191 -0
- package/dist/src/vectorStore.js +10 -297
- package/dist/src/vectorStore.test.js +32 -32
- package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
- package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
- package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
- package/extensions/dashboard-server/dashboard-client.ts +21 -0
- package/extensions/dashboard-server/html.ts +2 -621
- package/extensions/dashboard-server/routes-core.ts +113 -0
- package/extensions/dashboard-server/routes-game.ts +386 -0
- package/extensions/dashboard-server/routes-repo.ts +212 -0
- package/extensions/dashboard-server/routes-sessions.ts +195 -0
- package/extensions/dashboard-server/routes.ts +13 -0
- package/extensions/dashboard-server/server.ts +37 -700
- package/extensions/mega-commands.ts +4 -3
- package/extensions/mega-events/agent-handlers.ts +2 -1
- package/extensions/mega-events/compact-handlers.ts +28 -0
- package/extensions/mega-events/session-handlers.ts +2 -1
- package/extensions/mega-pipeline/compact.ts +3 -2
- package/extensions/mega-runtime/state.ts +7 -7
- package/extensions/openclaw-mega-compact.ts +2 -2
- package/package.json +2 -2
- package/src/dedup/raptor/multilevel.test.ts +278 -0
- package/src/dedup/raptor/multilevel.ts +246 -0
- package/src/dedup/raptor/promote.test.ts +5 -5
- package/src/dedup/raptor/retrieval.ts +1 -1
- package/src/dedup/sprint12.test.ts +7 -7
- package/src/dedup-engine.test.ts +30 -30
- package/src/e2e.test.ts +38 -38
- package/src/engine.test.ts +6 -6
- package/src/engine.ts +3 -3
- package/src/importance.test.ts +538 -0
- package/src/importance.ts +312 -0
- package/src/ratio.bench.test.ts +18 -18
- package/src/recall.test.ts +101 -29
- package/src/recall.ts +9 -9
- package/src/sprint14.test.ts +2 -2
- package/src/store/migrate.test.ts +5 -5
- package/src/store/sprint10.test.ts +5 -5
- package/src/store/sqlite/global-index.ts +18 -290
- package/src/store/sqlite/global-sessions.ts +291 -0
- package/src/vector-read.ts +237 -0
- package/src/vector-search.ts +231 -0
- package/src/vectorStore.test.ts +32 -32
- package/src/vectorStore.ts +29 -356
|
@@ -16,7 +16,7 @@ import fs from "fs";
|
|
|
16
16
|
import path from "path";
|
|
17
17
|
import os from "os";
|
|
18
18
|
import { compactSession } from "./engine.js";
|
|
19
|
-
import { VectorStore } from "./vectorStore.js";
|
|
19
|
+
import { VectorStore, vectorList, vectorStats, vectorWasInjected, vectorMarkInjected, vectorSearch } from "./vectorStore.js";
|
|
20
20
|
import { extractiveSummarize } from "./extractive.js";
|
|
21
21
|
import { estimateSessionTokens, estimateMessageTokens } from "./tokens.js";
|
|
22
22
|
import { autoCompactCheck } from "./compact.js";
|
|
@@ -122,7 +122,7 @@ describe("Dedupe Levels", () => {
|
|
|
122
122
|
const r2 = compactFull(s, SESS, [makeMsg("user", region)]);
|
|
123
123
|
assert.equal(r2.deduped, true);
|
|
124
124
|
assert.equal(r2.checkpointId, r1.checkpointId);
|
|
125
|
-
assert.equal(s
|
|
125
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
126
126
|
});
|
|
127
127
|
it("L0 only: distinct content stored twice creates two checkpoints", () => {
|
|
128
128
|
const s = makeStore({ L0_ENABLED: true, L1_ENABLED: false, L2_ENABLED: false });
|
|
@@ -133,7 +133,7 @@ describe("Dedupe Levels", () => {
|
|
|
133
133
|
assert.equal(r1.deduped, false);
|
|
134
134
|
assert.equal(r2.deduped, false);
|
|
135
135
|
assert.notEqual(r1.checkpointId, r2.checkpointId);
|
|
136
|
-
assert.equal(s
|
|
136
|
+
assert.equal(vectorList(s, SESS).length, 2);
|
|
137
137
|
});
|
|
138
138
|
it("L1 only: one-word variants collapse; major rewrites do not", () => {
|
|
139
139
|
const s = makeStore({ L0_ENABLED: false, L1_ENABLED: true, L2_ENABLED: false });
|
|
@@ -144,10 +144,10 @@ describe("Dedupe Levels", () => {
|
|
|
144
144
|
assert.equal(r1.deduped, false);
|
|
145
145
|
const r2 = s.add({ sessionId: SESS, summary: "migration", regionText: variant, timestamp: 2 });
|
|
146
146
|
assert.equal(r2.deduped, true, "one-word variant should be collapsed by L1");
|
|
147
|
-
assert.equal(s
|
|
147
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
148
148
|
const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: rewrite, timestamp: 3 });
|
|
149
149
|
assert.equal(r3.deduped, false, "major rewrite should not be collapsed by L1");
|
|
150
|
-
assert.equal(s
|
|
150
|
+
assert.equal(vectorList(s, SESS).length, 2);
|
|
151
151
|
});
|
|
152
152
|
it("L2 only: semantic paraphrases collapse; unrelated topics do not", () => {
|
|
153
153
|
// Use a lower threshold and longer, lexically-overlapping paraphrase so the
|
|
@@ -161,10 +161,10 @@ describe("Dedupe Levels", () => {
|
|
|
161
161
|
assert.equal(r1.deduped, false);
|
|
162
162
|
const r2 = s.add({ sessionId: SESS, summary: "auth paraphrase", regionText: paraphrase, timestamp: 2 });
|
|
163
163
|
assert.equal(r2.deduped, true, "semantic paraphrase should be collapsed by L2");
|
|
164
|
-
assert.equal(s
|
|
164
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
165
165
|
const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: unrelated, timestamp: 3 });
|
|
166
166
|
assert.equal(r3.deduped, false, "unrelated topic should not be collapsed by L2");
|
|
167
|
-
assert.equal(s
|
|
167
|
+
assert.equal(vectorList(s, SESS).length, 2);
|
|
168
168
|
});
|
|
169
169
|
it("All tiers disabled: every store.add() with different region text creates a distinct checkpoint", () => {
|
|
170
170
|
// Even with all dedup tiers disabled, the store still enforces a unique
|
|
@@ -178,7 +178,7 @@ describe("Dedupe Levels", () => {
|
|
|
178
178
|
assert.equal(r3.deduped, false);
|
|
179
179
|
assert.notEqual(r1.checkpoint.checkpointId, r2.checkpoint.checkpointId);
|
|
180
180
|
assert.notEqual(r2.checkpoint.checkpointId, r3.checkpoint.checkpointId);
|
|
181
|
-
assert.equal(s
|
|
181
|
+
assert.equal(vectorList(s, SESS).length, 3);
|
|
182
182
|
});
|
|
183
183
|
it("Combined L0+L1+L2: layered behavior exact -> near -> semantic", () => {
|
|
184
184
|
const s = makeStore({ L0_ENABLED: true, L1_ENABLED: true, L2_ENABLED: true });
|
|
@@ -202,8 +202,8 @@ describe("Dedupe Levels", () => {
|
|
|
202
202
|
if (r4.deduped) {
|
|
203
203
|
okReason(r4.dedupReason, ["contentSimilarity", "l1MinHash"]);
|
|
204
204
|
}
|
|
205
|
-
assert.ok(s
|
|
206
|
-
assert.ok(s
|
|
205
|
+
assert.ok(vectorList(s, SESS).length >= 1, "layered dedup keeps at least one checkpoint");
|
|
206
|
+
assert.ok(vectorList(s, SESS).length <= 4, "layered dedup should not explode to many checkpoints");
|
|
207
207
|
});
|
|
208
208
|
});
|
|
209
209
|
function okReason(reason, expected) {
|
|
@@ -242,16 +242,16 @@ describe("Compression / Store Stats", () => {
|
|
|
242
242
|
compactFull(s, SESS, [makeMsg("user", unique)], 1);
|
|
243
243
|
const dup = "duplicate topic about payment gateway integration";
|
|
244
244
|
compactFull(s, SESS, [makeMsg("user", dup)], 1);
|
|
245
|
-
const statsBefore = s
|
|
245
|
+
const statsBefore = vectorStats(s, SESS);
|
|
246
246
|
assert.ok(statsBefore.checkpointCount >= 1, "checkpointCount should be positive");
|
|
247
247
|
assert.ok(statsBefore.totalTokenEstimate >= 0, "totalTokenEstimate should be non-negative");
|
|
248
248
|
assert.equal(statsBefore.dedupHitRate, 0, "no injections yet => dedupHitRate 0");
|
|
249
249
|
assert.equal(statsBefore.injectedCount, 0, "no injections yet => injectedCount 0");
|
|
250
|
-
const hits = s
|
|
250
|
+
const hits = vectorSearch(s, SESS, "payment gateway", 5);
|
|
251
251
|
assert.ok(hits.length > 0, "should find the stored checkpoint");
|
|
252
252
|
const cpId = hits[0].checkpoint.checkpointId;
|
|
253
|
-
s
|
|
254
|
-
const statsAfter = s
|
|
253
|
+
vectorMarkInjected(s, SESS, cpId);
|
|
254
|
+
const statsAfter = vectorStats(s, SESS);
|
|
255
255
|
assert.equal(statsAfter.injectedCount, 1, "injectedCount tracks markInjected");
|
|
256
256
|
if (statsAfter.checkpointCount > 0) {
|
|
257
257
|
assert.ok(Math.abs(statsAfter.dedupHitRate - 1 / statsAfter.checkpointCount) < 0.001, "dedupHitRate = injected / checkpoints");
|
|
@@ -267,10 +267,10 @@ describe("Compression / Store Stats", () => {
|
|
|
267
267
|
}
|
|
268
268
|
compactFull(s, SESS, [makeMsg("user", "unique region for hit-rate measurement variant")], 1);
|
|
269
269
|
// Mark the first as injected.
|
|
270
|
-
const first = s
|
|
270
|
+
const first = vectorSearch(s, SESS, base, 1)[0]?.checkpoint.checkpointId;
|
|
271
271
|
if (first)
|
|
272
|
-
s
|
|
273
|
-
const stats = s
|
|
272
|
+
vectorMarkInjected(s, SESS, first);
|
|
273
|
+
const stats = vectorStats(s, SESS);
|
|
274
274
|
assert.ok(stats.checkpointCount >= 1);
|
|
275
275
|
assert.ok(stats.dedupHitRate > 0 || stats.checkpointCount === 1, "hit rate should be positive when there are multiple checkpoint");
|
|
276
276
|
});
|
|
@@ -304,13 +304,13 @@ describe("Recall & Dedup Sentinel", () => {
|
|
|
304
304
|
const s = makeStore();
|
|
305
305
|
const region = "manual sentinel tracking without recallAndInline";
|
|
306
306
|
compactFull(s, SESS, [makeMsg("user", region)]);
|
|
307
|
-
const hits = s
|
|
307
|
+
const hits = vectorSearch(s, SESS, "manual sentinel", 3);
|
|
308
308
|
assert.ok(hits.length > 0, "search should return checkpoint");
|
|
309
309
|
const cpId = hits[0].checkpoint.checkpointId;
|
|
310
|
-
assert.equal(s
|
|
311
|
-
s
|
|
312
|
-
assert.equal(s
|
|
313
|
-
const hits2 = s
|
|
310
|
+
assert.equal(vectorWasInjected(s, SESS, cpId), false, "not yet injected");
|
|
311
|
+
vectorMarkInjected(s, SESS, cpId);
|
|
312
|
+
assert.equal(vectorWasInjected(s, SESS, cpId), true, "markInjected recorded");
|
|
313
|
+
const hits2 = vectorSearch(s, SESS, "manual sentinel", 3).filter((h) => !vectorWasInjected(s, SESS, h.checkpoint.checkpointId));
|
|
314
314
|
assert.equal(hits2.length, 0, "filtered search excludes injected checkpoint");
|
|
315
315
|
});
|
|
316
316
|
});
|
|
@@ -322,13 +322,13 @@ describe("Edge Cases", () => {
|
|
|
322
322
|
const r = compactSession({ sessionId: SESS, messages: [], keepFrom: 0 }, s);
|
|
323
323
|
assert.equal(r.skipped, true);
|
|
324
324
|
assert.equal(r.summary, "");
|
|
325
|
-
assert.equal(s
|
|
325
|
+
assert.equal(vectorList(s, SESS).length, 0);
|
|
326
326
|
});
|
|
327
327
|
it("single message with keepFrom=0 returns skipped", () => {
|
|
328
328
|
const s = makeStore();
|
|
329
329
|
const r = compactSession({ sessionId: SESS, messages: [makeMsg("user", "only one message")], keepFrom: 0 }, s);
|
|
330
330
|
assert.equal(r.skipped, true);
|
|
331
|
-
assert.equal(s
|
|
331
|
+
assert.equal(vectorList(s, SESS).length, 0);
|
|
332
332
|
});
|
|
333
333
|
it("keepFrom at messages.length compacts all prior messages (verified behavior)", () => {
|
|
334
334
|
// The engine treats keepFrom as the compactable boundary: messages[0..keepFrom)
|
|
@@ -339,7 +339,7 @@ describe("Edge Cases", () => {
|
|
|
339
339
|
const r = compactSession({ sessionId: SESS, messages, keepFrom: messages.length }, s);
|
|
340
340
|
assert.equal(r.skipped, false);
|
|
341
341
|
assert.ok(r.checkpointId);
|
|
342
|
-
assert.equal(s
|
|
342
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
343
343
|
});
|
|
344
344
|
it("unicode and emoji messages store and retrieve intact", () => {
|
|
345
345
|
const s = makeStore();
|
|
@@ -347,7 +347,7 @@ describe("Edge Cases", () => {
|
|
|
347
347
|
"日本語テキスト 日本語テキスト 👍🔥";
|
|
348
348
|
const r = compactFull(s, SESS, [makeMsg("user", text)], 1);
|
|
349
349
|
assert.equal(r.skipped, false);
|
|
350
|
-
const stored = s
|
|
350
|
+
const stored = vectorList(s, SESS)[0];
|
|
351
351
|
assert.ok(stored);
|
|
352
352
|
const recovered = Buffer.from(stored.compressedOriginal ?? Buffer.alloc(0));
|
|
353
353
|
assert.ok(recovered.toString("utf-8").includes("🎉"), "emoji recovered from compressedOriginal");
|
|
@@ -360,8 +360,8 @@ describe("Edge Cases", () => {
|
|
|
360
360
|
const r = compactFull(s, SESS, [makeMsg("user", big)], 1);
|
|
361
361
|
assert.equal(r.skipped, false);
|
|
362
362
|
assert.ok(r.checkpointId);
|
|
363
|
-
assert.equal(s
|
|
364
|
-
const stats = s
|
|
363
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
364
|
+
const stats = vectorStats(s, SESS);
|
|
365
365
|
assert.ok(stats.totalTokenEstimate > 0);
|
|
366
366
|
});
|
|
367
367
|
it("mixed roles (user/assistant/tool) are included in summary", () => {
|
|
@@ -380,7 +380,7 @@ describe("Edge Cases", () => {
|
|
|
380
380
|
r.summary.includes("Edit") ||
|
|
381
381
|
r.summary.includes("user") ||
|
|
382
382
|
r.summary.includes("assistant"), "summary should reference roles or tools");
|
|
383
|
-
assert.equal(s
|
|
383
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
384
384
|
});
|
|
385
385
|
});
|
|
386
386
|
// -------------------- 7. Tier Switching (percentage-based) --------------------
|
package/dist/src/e2e.test.js
CHANGED
|
@@ -9,7 +9,7 @@ import assert from "node:assert/strict";
|
|
|
9
9
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
10
10
|
import { tmpdir } from "node:os";
|
|
11
11
|
import { join } from "node:path";
|
|
12
|
-
import { VectorStore, computeRegionHash } from "./vectorStore.js";
|
|
12
|
+
import { VectorStore, computeRegionHash, vectorList, vectorSemDedup, vectorWasInjected, vectorMarkInjected, vectorTopSimilar, vectorStats, vectorSearch } from "./vectorStore.js";
|
|
13
13
|
import { compactSession, recall, supersededCount } from "./engine.js";
|
|
14
14
|
import { loadDedupConfig } from "./config/dedup.js";
|
|
15
15
|
import { defaultEmbedder } from "./embedder.js";
|
|
@@ -101,8 +101,8 @@ test("1. Full compaction pipeline: SUPERSEDE → COLLAPSE → CLUSTER", () => {
|
|
|
101
101
|
// Verify: checkpoint persisted
|
|
102
102
|
assert.ok(r.checkpointId, "checkpointId should be set");
|
|
103
103
|
assert.match(r.checkpointId, /^chkpt_001$/);
|
|
104
|
-
// Verify: checkpoint is searchable via store
|
|
105
|
-
const hits = s
|
|
104
|
+
// Verify: checkpoint is searchable via vectorSearch(store, )
|
|
105
|
+
const hits = vectorSearch(s, SESS, "server.ts memory leak JWT auth", 5);
|
|
106
106
|
assert.ok(hits.length > 0, "checkpoint should be searchable");
|
|
107
107
|
assert.equal(hits[0].checkpoint.checkpointId, r.checkpointId);
|
|
108
108
|
});
|
|
@@ -133,7 +133,7 @@ test("2. Multi-session dedup: identical content in same session detected by L0",
|
|
|
133
133
|
assert.equal(rB.deduped, true, "second identical compaction should be deduped by L0");
|
|
134
134
|
assert.ok(rB.dedupReason === "regionHash" || rB.dedupReason === "contentHash", `dedup reason should be L0 (regionHash or contentHash), got ${rB.dedupReason}`);
|
|
135
135
|
// Search should still return the checkpoint
|
|
136
|
-
const hits = s
|
|
136
|
+
const hits = vectorSearch(s, "sess_multi_a", "database connection pooling", 5);
|
|
137
137
|
assert.ok(hits.length > 0, "search returns the deduped result");
|
|
138
138
|
assert.equal(hits[0].checkpoint.checkpointId, rA.checkpointId);
|
|
139
139
|
});
|
|
@@ -162,7 +162,7 @@ test("3. Near-duplicate detection: L1 MinHash/LSH catches one-word edits that L0
|
|
|
162
162
|
// L1 should catch the near-dup
|
|
163
163
|
assert.equal(r2.deduped, true, "L1 should catch the near-duplicate");
|
|
164
164
|
assert.equal(r2.reason, "l1MinHash", "dedup reason should be l1MinHash");
|
|
165
|
-
assert.equal(s
|
|
165
|
+
assert.equal(vectorList(s, SESS).length, 1, "only one checkpoint stored");
|
|
166
166
|
});
|
|
167
167
|
test("3b. Negative: L1 does NOT collapse genuinely different content", () => {
|
|
168
168
|
const s = store({ L2_ENABLED: false });
|
|
@@ -180,7 +180,7 @@ test("3b. Negative: L1 does NOT collapse genuinely different content", () => {
|
|
|
180
180
|
timestamp: 2,
|
|
181
181
|
});
|
|
182
182
|
assert.equal(r2.deduped, false, "distinct content should not be deduped");
|
|
183
|
-
assert.equal(s
|
|
183
|
+
assert.equal(vectorList(s, SESS).length, 2, "both checkpoints stored");
|
|
184
184
|
});
|
|
185
185
|
// ---------------------------------------------------------------------------
|
|
186
186
|
// 4. Semantic Dedup (L2 cosine)
|
|
@@ -208,7 +208,7 @@ test("4. Semantic dedup: L2 cosine catches highly similar content that L0/L1 mis
|
|
|
208
208
|
// L2 semantic cosine should catch this high-similarity paraphrase
|
|
209
209
|
assert.equal(r2.deduped, true, "L2 should catch the high-similarity paraphrase");
|
|
210
210
|
assert.ok(r2.reason === "contentSimilarity" || r2.reason === "l1MinHash", `dedup reason should be L2 contentSimilarity or L1 l1MinHash, got ${r2.reason}`);
|
|
211
|
-
assert.equal(s
|
|
211
|
+
assert.equal(vectorList(s, SESS).length, 1, "only one checkpoint after semantic dedup");
|
|
212
212
|
});
|
|
213
213
|
test("4b. MMR diversifies search results", () => {
|
|
214
214
|
const s = store();
|
|
@@ -230,7 +230,7 @@ test("4b. MMR diversifies search results", () => {
|
|
|
230
230
|
});
|
|
231
231
|
}
|
|
232
232
|
// Search for parser-related content — should return relevant results
|
|
233
|
-
const hits = s
|
|
233
|
+
const hits = vectorSearch(s, SESS, "parser tokenization source code files", 4);
|
|
234
234
|
assert.ok(hits.length >= 1, "should return at least one result");
|
|
235
235
|
// The top hit should be the parser checkpoint
|
|
236
236
|
assert.ok(hits[0].checkpoint.summary.includes("topic_0"), "top hit is the parser topic");
|
|
@@ -255,20 +255,20 @@ test("5. SemDeDup offline cleanup: redundant rows marked removed, not deleted",
|
|
|
255
255
|
{ id: "chkpt_003", text: "the frontend uses a virtualized list for rendering large datasets efficiently", tok: 500, ts: 3 },
|
|
256
256
|
]);
|
|
257
257
|
// Run SemDeDup with threshold that catches the near-identical pair
|
|
258
|
-
const removed = s
|
|
258
|
+
const removed = vectorSemDedup(s, SESS, 0.85);
|
|
259
259
|
assert.equal(removed, 1, "one redundant row should be marked removed");
|
|
260
260
|
// Verify: the lower-tokenEstimate row is the one removed
|
|
261
|
-
const all = s
|
|
261
|
+
const all = vectorList(s, SESS);
|
|
262
262
|
const dropped = all.find((c) => c.dedupStatus === "removed");
|
|
263
263
|
assert.ok(dropped, "a row should have dedup_status='removed'");
|
|
264
264
|
assert.equal(dropped.checkpointId, "chkpt_001", "lower tokenEstimate row removed");
|
|
265
265
|
assert.equal(dropped.dedupStatus, "removed");
|
|
266
266
|
// Verify: search excludes removed rows (3 total, 1 removed → 2 active)
|
|
267
|
-
const hits = s
|
|
267
|
+
const hits = vectorSearch(s, SESS, "cache parsed ast nodes frontend virtualized list", 10);
|
|
268
268
|
assert.equal(hits.length, 2, "search excludes the removed row, returns 2 active");
|
|
269
269
|
assert.ok(hits.every((h) => h.checkpoint.dedupStatus !== "removed"), "no removed rows in search");
|
|
270
270
|
// Verify: idempotent re-run removes nothing new
|
|
271
|
-
const secondRun = s
|
|
271
|
+
const secondRun = vectorSemDedup(s, SESS, 0.85);
|
|
272
272
|
assert.equal(secondRun, 0, "idempotent re-run removes nothing new");
|
|
273
273
|
closeStore(dir);
|
|
274
274
|
});
|
|
@@ -293,15 +293,15 @@ test("6. Recall + inline with dedup sentinel: injected checkpoints are skipped",
|
|
|
293
293
|
assert.equal(first.hits.length, 1);
|
|
294
294
|
// Mark it injected
|
|
295
295
|
const cpId = first.hits[0].checkpoint.checkpointId;
|
|
296
|
-
s
|
|
296
|
+
vectorMarkInjected(s, SESS, cpId);
|
|
297
297
|
// Recall again — should skip the injected checkpoint
|
|
298
298
|
const second = recall({ sessionId: SESS, query: "vectorStore dedup cascade", limit: 5, skipInjected: true }, s);
|
|
299
299
|
assert.equal(second.newHits.length, 0, "injected checkpoint should be skipped (dedup sentinel)");
|
|
300
300
|
// Without skipInjected, hits still surface
|
|
301
301
|
assert.equal(second.hits.length, 1, "hits still surface without skipInjected");
|
|
302
302
|
// Verify wasInjected
|
|
303
|
-
assert.equal(s
|
|
304
|
-
assert.equal(s
|
|
303
|
+
assert.equal(vectorWasInjected(s, SESS, cpId), true, "wasInjected returns true for injected checkpoint");
|
|
304
|
+
assert.equal(vectorWasInjected(s, SESS, "chkpt_999"), false, "wasInjected returns false for non-injected");
|
|
305
305
|
});
|
|
306
306
|
// ---------------------------------------------------------------------------
|
|
307
307
|
// 7. topSimilar
|
|
@@ -326,11 +326,11 @@ test("7. topSimilar: returns n most cosine-similar checkpoints", () => {
|
|
|
326
326
|
});
|
|
327
327
|
}
|
|
328
328
|
// Call topSimilar(3)
|
|
329
|
-
const hits = s
|
|
329
|
+
const hits = vectorTopSimilar(s, SESS, 3);
|
|
330
330
|
assert.ok(hits.length <= 3, "should respect n limit");
|
|
331
331
|
assert.ok(hits.length > 0, "should return results");
|
|
332
332
|
// Verify self-exclusion: the most recent checkpoint should not be in results
|
|
333
|
-
const all = s
|
|
333
|
+
const all = vectorList(s, SESS);
|
|
334
334
|
const ordered = [...all].sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
|
|
335
335
|
const current = ordered[ordered.length - 1];
|
|
336
336
|
assert.ok(!hits.some((h) => h.checkpoint.checkpointId === current.checkpointId), "current checkpoint excluded");
|
|
@@ -358,14 +358,14 @@ test("7b. topSimilar respects n limit strictly", () => {
|
|
|
358
358
|
timestamp: i + 1,
|
|
359
359
|
});
|
|
360
360
|
}
|
|
361
|
-
const hits = s
|
|
361
|
+
const hits = vectorTopSimilar(s, SESS, 2);
|
|
362
362
|
assert.equal(hits.length, 2, "n limit respected");
|
|
363
363
|
});
|
|
364
364
|
test("7c. topSimilar returns empty for sessions with 0 or 1 checkpoints", () => {
|
|
365
365
|
const s = store();
|
|
366
|
-
assert.deepEqual(s
|
|
366
|
+
assert.deepEqual(vectorTopSimilar(s, "sess_empty", 5), []);
|
|
367
367
|
s.add({ sessionId: "sess_one", summary: "solo", regionText: "only checkpoint here", timestamp: 1 });
|
|
368
|
-
assert.deepEqual(s
|
|
368
|
+
assert.deepEqual(vectorTopSimilar(s, "sess_one", 5), []);
|
|
369
369
|
});
|
|
370
370
|
// ---------------------------------------------------------------------------
|
|
371
371
|
// 8. Compression Round-Trip
|
|
@@ -390,7 +390,7 @@ test("8. Compression round-trip: decompression produces original content", () =>
|
|
|
390
390
|
timestamp: 1,
|
|
391
391
|
});
|
|
392
392
|
// Retrieve and verify decompression
|
|
393
|
-
const cp = s
|
|
393
|
+
const cp = vectorList(s, SESS)[0];
|
|
394
394
|
assert.ok(cp.compressedOriginal instanceof Buffer, "compressedOriginal is a Buffer");
|
|
395
395
|
const restored = decompressSmart(cp.compressedOriginal).toString("utf-8");
|
|
396
396
|
assert.equal(restored, originalText, "decompression produces the original content");
|
|
@@ -407,7 +407,7 @@ test("8b. Compression round-trip with different content sizes", () => {
|
|
|
407
407
|
// Large text (>4KB → gzip level 6 or brotli)
|
|
408
408
|
const large = "detailed region text with lots of context about the codebase. ".repeat(200);
|
|
409
409
|
s.add({ sessionId: SESS, summary: "large", regionText: large, timestamp: 3 });
|
|
410
|
-
const checkpoints = s
|
|
410
|
+
const checkpoints = vectorList(s, SESS);
|
|
411
411
|
assert.equal(checkpoints.length, 3);
|
|
412
412
|
// Verify each round-trips correctly
|
|
413
413
|
const texts = [small, medium, large];
|
|
@@ -444,7 +444,7 @@ test("9. Store stats and metrics reflect actual state after compactions", () =>
|
|
|
444
444
|
const r3 = compactSession({ sessionId: SESS, messages: conv3, keepFrom: 2, timestamp: 3 }, s);
|
|
445
445
|
assert.equal(r3.deduped, false);
|
|
446
446
|
// Get stats
|
|
447
|
-
const st = s
|
|
447
|
+
const st = vectorStats(s, SESS);
|
|
448
448
|
assert.equal(st.checkpointCount, 3, "three checkpoints stored");
|
|
449
449
|
assert.ok(st.totalTokenEstimate > 0, "totalTokenEstimate should be positive");
|
|
450
450
|
assert.equal(st.lastCheckpointId, "chkpt_003", "last checkpoint id correct");
|
|
@@ -452,14 +452,14 @@ test("9. Store stats and metrics reflect actual state after compactions", () =>
|
|
|
452
452
|
assert.equal(st.injectedCount, 0, "no injections yet");
|
|
453
453
|
assert.equal(st.dedupHitRate, 0, "dedupHitRate is 0 with no injections");
|
|
454
454
|
// Mark one injected and re-check
|
|
455
|
-
s
|
|
456
|
-
const st2 = s
|
|
455
|
+
vectorMarkInjected(s, SESS, "chkpt_001");
|
|
456
|
+
const st2 = vectorStats(s, SESS);
|
|
457
457
|
assert.equal(st2.injectedCount, 1, "one injection tracked");
|
|
458
458
|
assert.ok(Math.abs(st2.dedupHitRate - 1 / 3) < 1e-9, "dedupHitRate = injected/checkpoints");
|
|
459
459
|
});
|
|
460
460
|
test("9b. Stats on empty session returns zeros", () => {
|
|
461
461
|
const s = store();
|
|
462
|
-
const st = s
|
|
462
|
+
const st = vectorStats(s, "sess_nothing");
|
|
463
463
|
assert.equal(st.checkpointCount, 0);
|
|
464
464
|
assert.equal(st.totalTokenEstimate, 0);
|
|
465
465
|
assert.equal(st.lastCheckpointId, undefined);
|
|
@@ -496,15 +496,15 @@ test("10. Concurrent sessions: no cross-contamination", () => {
|
|
|
496
496
|
});
|
|
497
497
|
// Verify: each session has exactly one checkpoint
|
|
498
498
|
sessions.forEach((sess) => {
|
|
499
|
-
const cps = s
|
|
499
|
+
const cps = vectorList(s, sess.id);
|
|
500
500
|
assert.equal(cps.length, 1, `${sess.id} should have 1 checkpoint`);
|
|
501
501
|
});
|
|
502
502
|
// Verify: search in one session doesn't return another session's checkpoints
|
|
503
|
-
const hitsA = s
|
|
503
|
+
const hitsA = vectorSearch(s, "sess_concurrent_a", "authentication JWT", 5);
|
|
504
504
|
assert.ok(hitsA.every((h) => h.checkpoint.sessionId === "sess_concurrent_a"), "search in A returns only A's checkpoints");
|
|
505
|
-
const hitsB = s
|
|
505
|
+
const hitsB = vectorSearch(s, "sess_concurrent_b", "database queries indexes", 5);
|
|
506
506
|
assert.ok(hitsB.every((h) => h.checkpoint.sessionId === "sess_concurrent_b"), "search in B returns only B's checkpoints");
|
|
507
|
-
const hitsC = s
|
|
507
|
+
const hitsC = vectorSearch(s, "sess_concurrent_c", "canvas rendering bug", 5);
|
|
508
508
|
assert.ok(hitsC.every((h) => h.checkpoint.sessionId === "sess_concurrent_c"), "search in C returns only C's checkpoints");
|
|
509
509
|
});
|
|
510
510
|
// ---------------------------------------------------------------------------
|
|
@@ -516,7 +516,7 @@ test("11a. Empty session (no messages) — compactSession should skip", () => {
|
|
|
516
516
|
assert.equal(r.skipped, true, "empty session should be skipped");
|
|
517
517
|
assert.equal(r.summary, "");
|
|
518
518
|
assert.equal(r.regionHash, "");
|
|
519
|
-
assert.equal(s
|
|
519
|
+
assert.equal(vectorList(s, "sess_empty_msgs").length, 0);
|
|
520
520
|
});
|
|
521
521
|
test("11b. Single message session — should skip or handle gracefully", () => {
|
|
522
522
|
const s = store();
|
|
@@ -549,7 +549,7 @@ test("11c. Very large region text — should still compact and store", () => {
|
|
|
549
549
|
assert.ok(r.checkpointId, "checkpoint created");
|
|
550
550
|
assert.ok(r.summary.length > 0, "summary produced");
|
|
551
551
|
// Verify it's searchable
|
|
552
|
-
const hits = s
|
|
552
|
+
const hits = vectorSearch(s, SESS, "module feature fix review", 3);
|
|
553
553
|
assert.ok(hits.length > 0, "large checkpoint is searchable");
|
|
554
554
|
});
|
|
555
555
|
test("11d. Unicode and emoji content — should normalize and hash correctly", () => {
|
|
@@ -565,7 +565,7 @@ test("11d. Unicode and emoji content — should normalize and hash correctly", (
|
|
|
565
565
|
assert.equal(r.skipped, false, "unicode conversation should compact");
|
|
566
566
|
assert.ok(r.regionHash.length > 0, "regionHash computed for unicode content");
|
|
567
567
|
// Verify the checkpoint exists and is searchable
|
|
568
|
-
const hits = s
|
|
568
|
+
const hits = vectorSearch(s, SESS, "中文 translation 本地化", 3);
|
|
569
569
|
assert.ok(hits.length > 0, "unicode checkpoint is searchable");
|
|
570
570
|
// Verify regionHash is deterministic for the same unicode content
|
|
571
571
|
const r2 = compactSession({ sessionId: SESS, messages, keepFrom: 4, timestamp: 2 }, s);
|
|
@@ -588,7 +588,7 @@ test("11e. All messages in preserve-recent window — nothing to compact", () =>
|
|
|
588
588
|
timestamp: 1,
|
|
589
589
|
}, s);
|
|
590
590
|
assert.equal(r.skipped, true, "should skip when keepFrom=0 (nothing to compact)");
|
|
591
|
-
assert.equal(s
|
|
591
|
+
assert.equal(vectorList(s, "sess_all_preserved").length, 0, "no checkpoints stored");
|
|
592
592
|
});
|
|
593
593
|
// ---------------------------------------------------------------------------
|
|
594
594
|
// 12. Feature Flag Toggling
|
|
@@ -616,7 +616,7 @@ test("12a. L0_ENABLED=false — L0 does not collapse exact dups", () => {
|
|
|
616
616
|
assert.equal(r1.deduped, false);
|
|
617
617
|
// With L0 off (and L1/L2 off too), near-dup should NOT be collapsed
|
|
618
618
|
assert.equal(r2.deduped, false, "L0 disabled → near-dup not collapsed");
|
|
619
|
-
assert.equal(s
|
|
619
|
+
assert.equal(vectorList(s, SESS).length, 2, "both checkpoints stored when L0 is off");
|
|
620
620
|
});
|
|
621
621
|
test("12b. MARK_ONLY_L1=true — L1 records but does not collapse", () => {
|
|
622
622
|
// Disable L2 to isolate L1 behavior
|
|
@@ -637,9 +637,9 @@ test("12b. MARK_ONLY_L1=true — L1 records but does not collapse", () => {
|
|
|
637
637
|
assert.equal(r1.deduped, false);
|
|
638
638
|
// MARK_ONLY_L1 → L1 detects the near-dup but does NOT collapse
|
|
639
639
|
assert.equal(r2.deduped, false, "MARK_ONLY_L1 → not collapsed");
|
|
640
|
-
assert.equal(s
|
|
640
|
+
assert.equal(vectorList(s, SESS).length, 2, "both checkpoints stored (mark only)");
|
|
641
641
|
// Both should be active (not removed)
|
|
642
|
-
const all = s
|
|
642
|
+
const all = vectorList(s, SESS);
|
|
643
643
|
assert.ok(all.every((c) => c.dedupStatus === "active"), "both rows active under MARK_ONLY");
|
|
644
644
|
});
|
|
645
645
|
test("12c. L2_ENABLED=false — L2 skipped but L0/L1 still work", () => {
|
|
@@ -688,7 +688,7 @@ test("12d. All tiers disabled — nothing deduped, everything stored", () => {
|
|
|
688
688
|
});
|
|
689
689
|
assert.equal(r1.deduped, false);
|
|
690
690
|
assert.equal(r2.deduped, false, "all tiers off → nothing deduped");
|
|
691
|
-
assert.equal(s
|
|
691
|
+
assert.equal(vectorList(s, SESS).length, 2, "both stored");
|
|
692
692
|
});
|
|
693
693
|
// ---------------------------------------------------------------------------
|
|
694
694
|
// Cleanup
|
package/dist/src/engine.js
CHANGED
|
@@ -15,7 +15,7 @@ import { findSuperseded, supersede } from "./supersede.js";
|
|
|
15
15
|
import { summarizeMessages, mergeCompactSummaries, formatCompactSummary } from "./compact.js";
|
|
16
16
|
import { extractiveSummarize } from "./extractive.js";
|
|
17
17
|
import { estimateSessionTokens, estimateBlockTokens } from "./tokens.js";
|
|
18
|
-
import { computeRegionHash, VectorStore } from "./vectorStore.js";
|
|
18
|
+
import { computeRegionHash, vectorWasInjected, vectorSearch, VectorStore } from "./vectorStore.js";
|
|
19
19
|
/** Default store used by the convenience `compactSession`. */
|
|
20
20
|
let defaultStore;
|
|
21
21
|
export function getDefaultStore(stateDir) {
|
|
@@ -125,8 +125,8 @@ export function compactSession(input, store = getDefaultStore()) {
|
|
|
125
125
|
* inject (Sprint 4 wires injection); this module only does the deduped search.
|
|
126
126
|
*/
|
|
127
127
|
export function recall(input, store = getDefaultStore()) {
|
|
128
|
-
const hits = store
|
|
129
|
-
const newHits = input.skipInjected === false ? hits : hits.filter((h) => !store
|
|
128
|
+
const hits = vectorSearch(store, input.sessionId, input.query, input.limit ?? 3);
|
|
129
|
+
const newHits = input.skipInjected === false ? hits : hits.filter((h) => !vectorWasInjected(store, input.sessionId, h.checkpoint.checkpointId));
|
|
130
130
|
return { hits, newHits };
|
|
131
131
|
}
|
|
132
132
|
/** Merge a freshly compacted summary into the prior persisted summary text. */
|
package/dist/src/engine.test.js
CHANGED
|
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
|
|
|
3
3
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
|
-
import { VectorStore } from "./vectorStore.js";
|
|
6
|
+
import { VectorStore, vectorMarkInjected, vectorSearch } from "./vectorStore.js";
|
|
7
7
|
import { compactSession, recall, mergeSummary, supersededCount } from "./engine.js";
|
|
8
8
|
const baseTmp = mkdtempSync(join(tmpdir(), "mc-engine-"));
|
|
9
9
|
let counter = 0;
|
|
@@ -35,7 +35,7 @@ test("compactSession supersedes then persists a checkpoint", () => {
|
|
|
35
35
|
assert.equal(supersededCount(messages.slice(0, 4)), 1);
|
|
36
36
|
assert.equal(r.compactedFrom, 4);
|
|
37
37
|
// The persisted checkpoint is searchable.
|
|
38
|
-
assert.equal(s
|
|
38
|
+
assert.equal(vectorSearch(s, SESS, "bug src/server.ts", 5).length, 1);
|
|
39
39
|
});
|
|
40
40
|
test("compactSession is idempotent on identical region (dedup sentinel)", () => {
|
|
41
41
|
const s = store();
|
|
@@ -50,20 +50,20 @@ test("compactSession is idempotent on identical region (dedup sentinel)", () =>
|
|
|
50
50
|
assert.equal(r1.deduped, false);
|
|
51
51
|
assert.equal(r2.deduped, true);
|
|
52
52
|
assert.equal(r1.checkpointId, r2.checkpointId);
|
|
53
|
-
assert.equal(s
|
|
53
|
+
assert.equal(vectorSearch(s, SESS, "parser", 10).length, 1);
|
|
54
54
|
});
|
|
55
55
|
test("compactSession skipped when slice is empty", () => {
|
|
56
56
|
const s = store();
|
|
57
57
|
const r = compactSession({ sessionId: SESS, messages: [msg("user", "only tail")], keepFrom: 0 }, s);
|
|
58
58
|
assert.equal(r.skipped, true);
|
|
59
|
-
assert.equal(s
|
|
59
|
+
assert.equal(vectorSearch(s, SESS, "x", 5).length, 0);
|
|
60
60
|
});
|
|
61
61
|
test("recall drops already-injected checkpoints", () => {
|
|
62
62
|
const s = store();
|
|
63
63
|
compactSession({ sessionId: SESS, messages: [msg("user", "investigated src/vectorStore.ts"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
|
|
64
64
|
const first = recall({ sessionId: SESS, query: "vectorStore", limit: 5, skipInjected: true }, s);
|
|
65
65
|
assert.equal(first.newHits.length, 1);
|
|
66
|
-
s
|
|
66
|
+
vectorMarkInjected(s, SESS, first.hits[0].checkpoint.checkpointId);
|
|
67
67
|
const second = recall({ sessionId: SESS, query: "vectorStore", limit: 5, skipInjected: true }, s);
|
|
68
68
|
assert.equal(second.newHits.length, 0);
|
|
69
69
|
// Without the skip flag, both hits still surface.
|
|
@@ -100,7 +100,7 @@ test("compactSession with useExtractive produces topicSummary on checkpoint", ()
|
|
|
100
100
|
assert.equal(r.skipped, false);
|
|
101
101
|
assert.ok(r.checkpointId, "checkpoint created");
|
|
102
102
|
// The checkpoint should have topicSummary populated
|
|
103
|
-
const hits = s
|
|
103
|
+
const hits = vectorSearch(s, "sess_extr", "auth refactor", 5);
|
|
104
104
|
assert.ok(hits.length > 0);
|
|
105
105
|
// topicSummary should be present on the stored checkpoint (via extractive path)
|
|
106
106
|
assert.ok(hits[0].checkpoint.topicSummary, "topicSummary should be populated when useExtractive is true");
|