pi-mega-compact 0.8.21 → 0.8.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +6 -2
- package/README.md +1 -1
- package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
- package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
- package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
- package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
- package/dist/extensions/dashboard-server/html.js +2 -621
- package/dist/extensions/dashboard-server/routes-core.js +62 -0
- package/dist/extensions/dashboard-server/routes-game.js +323 -0
- package/dist/extensions/dashboard-server/routes-repo.js +170 -0
- package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
- package/dist/extensions/dashboard-server/routes.js +10 -0
- package/dist/extensions/dashboard-server/server.js +26 -623
- package/dist/extensions/mega-commands.js +4 -3
- package/dist/extensions/mega-events/agent-handlers.js +2 -1
- package/dist/extensions/mega-events/compact-handlers.js +26 -0
- package/dist/extensions/mega-events/session-handlers.js +2 -1
- package/dist/extensions/mega-pipeline/compact.js +3 -2
- package/dist/extensions/mega-runtime/state.js +7 -7
- package/dist/src/dedup/raptor/multilevel.js +172 -0
- package/dist/src/dedup/raptor/multilevel.test.js +203 -0
- package/dist/src/dedup/raptor/promote.test.js +5 -5
- package/dist/src/dedup/raptor/retrieval.js +1 -1
- package/dist/src/dedup/sprint12.test.js +7 -7
- package/dist/src/dedup-engine.test.js +29 -29
- package/dist/src/e2e.test.js +38 -38
- package/dist/src/engine.js +3 -3
- package/dist/src/engine.test.js +6 -6
- package/dist/src/importance.js +197 -0
- package/dist/src/importance.test.js +372 -0
- package/dist/src/ratio.bench.test.js +18 -18
- package/dist/src/recall.js +6 -5
- package/dist/src/recall.test.js +85 -27
- package/dist/src/sprint14.test.js +2 -2
- package/dist/src/store/migrate.test.js +5 -5
- package/dist/src/store/sprint10.test.js +5 -5
- package/dist/src/store/sqlite/global-index.js +5 -174
- package/dist/src/store/sqlite/global-sessions.js +190 -0
- package/dist/src/vector-read.js +168 -0
- package/dist/src/vector-search.js +191 -0
- package/dist/src/vectorStore.js +10 -297
- package/dist/src/vectorStore.test.js +32 -32
- package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
- package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
- package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
- package/extensions/dashboard-server/dashboard-client.ts +21 -0
- package/extensions/dashboard-server/html.ts +2 -621
- package/extensions/dashboard-server/routes-core.ts +113 -0
- package/extensions/dashboard-server/routes-game.ts +386 -0
- package/extensions/dashboard-server/routes-repo.ts +212 -0
- package/extensions/dashboard-server/routes-sessions.ts +195 -0
- package/extensions/dashboard-server/routes.ts +13 -0
- package/extensions/dashboard-server/server.ts +37 -700
- package/extensions/mega-commands.ts +4 -3
- package/extensions/mega-events/agent-handlers.ts +2 -1
- package/extensions/mega-events/compact-handlers.ts +28 -0
- package/extensions/mega-events/session-handlers.ts +2 -1
- package/extensions/mega-pipeline/compact.ts +3 -2
- package/extensions/mega-runtime/state.ts +7 -7
- package/extensions/openclaw-mega-compact.ts +2 -2
- package/package.json +2 -2
- package/src/dedup/raptor/multilevel.test.ts +278 -0
- package/src/dedup/raptor/multilevel.ts +246 -0
- package/src/dedup/raptor/promote.test.ts +5 -5
- package/src/dedup/raptor/retrieval.ts +1 -1
- package/src/dedup/sprint12.test.ts +7 -7
- package/src/dedup-engine.test.ts +30 -30
- package/src/e2e.test.ts +38 -38
- package/src/engine.test.ts +6 -6
- package/src/engine.ts +3 -3
- package/src/importance.test.ts +538 -0
- package/src/importance.ts +312 -0
- package/src/ratio.bench.test.ts +18 -18
- package/src/recall.test.ts +101 -29
- package/src/recall.ts +9 -9
- package/src/sprint14.test.ts +2 -2
- package/src/store/migrate.test.ts +5 -5
- package/src/store/sprint10.test.ts +5 -5
- package/src/store/sqlite/global-index.ts +18 -290
- package/src/store/sqlite/global-sessions.ts +291 -0
- package/src/vector-read.ts +237 -0
- package/src/vector-search.ts +231 -0
- package/src/vectorStore.test.ts +32 -32
- package/src/vectorStore.ts +29 -356
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* multilevel.ts — Multi-level RAPTOR retrieval engine (S42A).
|
|
3
|
+
*
|
|
4
|
+
* Upgrades the RAPTOR recall path from flat (leaf-only) to multi-level
|
|
5
|
+
* retrieval across the entire hierarchical tree. Searches ALL levels with
|
|
6
|
+
* configurable level weights, supports leaf expansion for cluster hits,
|
|
7
|
+
* and deduplicates overlapping results.
|
|
8
|
+
*
|
|
9
|
+
* PREVENT-PI-004: pure in-process math (cosine, BFS, extractive). No network.
|
|
10
|
+
* PREVENT-PI-001: produces SearchHit[] that feed into recallAndInline() —
|
|
11
|
+
* affects which checkpoints are recalled, not how messages are dropped.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { Embedder, Vector } from "../../embedder.js";
|
|
15
|
+
import { cosineSimilarity } from "../../embedder.js";
|
|
16
|
+
import { mmrRerank } from "../mmr.js";
|
|
17
|
+
import type { RaptorTree } from "./tree.js";
|
|
18
|
+
import { leafDescendants } from "./retrieval.js";
|
|
19
|
+
|
|
20
|
+
// ── S42A-1: Types ──────────────────────────────────────────────────────────
|
|
21
|
+
|
|
22
|
+
export interface MultilevelRetrieveOptions {
|
|
23
|
+
embedder: Embedder;
|
|
24
|
+
/** Weight per tree level (index 0 = leaves, index 1 = level 1, etc.).
|
|
25
|
+
* Default: [1.0, 0.9, 0.8, 0.7, 0.5]. Capped at tree depth.
|
|
26
|
+
* UNCALIBRATED — requires real-data calibration before stable. */
|
|
27
|
+
levelWeights?: number[];
|
|
28
|
+
/** When true, expand cluster hits to include leaf descendants. Default: true. */
|
|
29
|
+
leafExpansion?: boolean;
|
|
30
|
+
/** Max leaf descendants to fetch per cluster hit. Default: 10. */
|
|
31
|
+
maxLeafExpansion?: number;
|
|
32
|
+
/** Final number of results to return. Default: 5. */
|
|
33
|
+
k?: number;
|
|
34
|
+
/** MMR diversity weight. Default: 0.5. */
|
|
35
|
+
mmrLambda?: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface MultilevelHit {
|
|
39
|
+
nodeId: string;
|
|
40
|
+
level: number;
|
|
41
|
+
/** Weighted score after level weighting. */
|
|
42
|
+
score: number;
|
|
43
|
+
/** Raw cosine similarity before level weighting. */
|
|
44
|
+
rawScore: number;
|
|
45
|
+
isLeaf: boolean;
|
|
46
|
+
/** Leaf ids covered by this node (for leaf expansion). */
|
|
47
|
+
leafIds: string[];
|
|
48
|
+
summary: string;
|
|
49
|
+
embedding: Vector;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const DEFAULT_LEVEL_WEIGHTS = [1.0, 0.9, 0.8, 0.7, 0.5];
|
|
53
|
+
|
|
54
|
+
// ── S42A-2: Level-weighted scoring ─────────────────────────────────────────
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Score all RAPTOR tree nodes by cosine similarity to the query, then apply
|
|
58
|
+
* level-specific weights. Returns hits sorted by weighted score descending.
|
|
59
|
+
*
|
|
60
|
+
* Level weights: leaves (level 0) get weight 1.0, level 1 gets 0.9, etc.
|
|
61
|
+
* This ensures detailed leaves score highest while still surfacing higher-level
|
|
62
|
+
* summaries when they're highly relevant.
|
|
63
|
+
*/
|
|
64
|
+
export function scoreTreeLevels(
|
|
65
|
+
query: string,
|
|
66
|
+
tree: RaptorTree,
|
|
67
|
+
opts: Pick<MultilevelRetrieveOptions, "embedder" | "levelWeights">,
|
|
68
|
+
): MultilevelHit[] {
|
|
69
|
+
const { embedder } = opts;
|
|
70
|
+
const weights = opts.levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
|
|
71
|
+
const qv = embedder.embed(query);
|
|
72
|
+
const hits: MultilevelHit[] = [];
|
|
73
|
+
|
|
74
|
+
// 1. Score all internal (summary) nodes.
|
|
75
|
+
for (const node of tree.nodes.values()) {
|
|
76
|
+
const rawScore = cosineSimilarity(qv, node.embedding);
|
|
77
|
+
const levelWeight = weights[Math.min(node.level, weights.length - 1)];
|
|
78
|
+
hits.push({
|
|
79
|
+
nodeId: node.id,
|
|
80
|
+
level: node.level,
|
|
81
|
+
score: rawScore * levelWeight,
|
|
82
|
+
rawScore,
|
|
83
|
+
isLeaf: false,
|
|
84
|
+
leafIds: node.children,
|
|
85
|
+
summary: node.summary,
|
|
86
|
+
embedding: node.embedding,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// 2. Score leaf nodes. Leaf ids are not in tree.nodes — they are children
|
|
91
|
+
// referenced by internal nodes. Each leaf's embedding is the level-0
|
|
92
|
+
// parent node that wraps it (same approach as stagedExpansion:95–102).
|
|
93
|
+
const seenLeaves = new Set<string>();
|
|
94
|
+
for (const node of tree.nodes.values()) {
|
|
95
|
+
for (const leafId of node.children) {
|
|
96
|
+
if (seenLeaves.has(leafId) || tree.nodes.has(leafId)) continue;
|
|
97
|
+
seenLeaves.add(leafId);
|
|
98
|
+
const rawScore = cosineSimilarity(qv, node.embedding);
|
|
99
|
+
const leafWeight = weights[0];
|
|
100
|
+
hits.push({
|
|
101
|
+
nodeId: leafId,
|
|
102
|
+
level: 0,
|
|
103
|
+
score: rawScore * leafWeight,
|
|
104
|
+
rawScore,
|
|
105
|
+
isLeaf: true,
|
|
106
|
+
leafIds: [leafId],
|
|
107
|
+
summary: "", // leaves have no summary — they are raw checkpoint ids
|
|
108
|
+
embedding: node.embedding,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
hits.sort((a, b) => b.score - a.score);
|
|
114
|
+
return hits;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ── S42A-3: Leaf expansion ─────────────────────────────────────────────────
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Given a set of cluster-level hits, expand each one to include its leaf
|
|
121
|
+
* descendants. Deduplicates: if a leaf is already present as a direct hit,
|
|
122
|
+
* it is not duplicated. Returns the merged set (original hits + expanded leaves).
|
|
123
|
+
*/
|
|
124
|
+
export function expandLeafDescendants(
|
|
125
|
+
hits: MultilevelHit[],
|
|
126
|
+
tree: RaptorTree,
|
|
127
|
+
maxPerCluster: number,
|
|
128
|
+
_embedder: Embedder,
|
|
129
|
+
queryVector: Vector,
|
|
130
|
+
levelWeights?: number[],
|
|
131
|
+
): MultilevelHit[] {
|
|
132
|
+
const weights = levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
|
|
133
|
+
const existingIds = new Set(hits.map((h) => h.nodeId));
|
|
134
|
+
const expanded: MultilevelHit[] = [];
|
|
135
|
+
|
|
136
|
+
for (const hit of hits) {
|
|
137
|
+
if (hit.isLeaf) {
|
|
138
|
+
expanded.push(hit);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Get all leaf descendants for this cluster node.
|
|
143
|
+
const node = tree.nodes.get(hit.nodeId);
|
|
144
|
+
if (!node) {
|
|
145
|
+
expanded.push(hit);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const rawLeafIds = leafDescendants(node, tree);
|
|
150
|
+
|
|
151
|
+
// Sort by cosine similarity to query, cap at maxPerCluster.
|
|
152
|
+
const leafHits: MultilevelHit[] = rawLeafIds
|
|
153
|
+
.map((lid) => {
|
|
154
|
+
// Leaf embedding = its nearest internal parent's embedding.
|
|
155
|
+
const parent = [...tree.nodes.values()].find((n) =>
|
|
156
|
+
n.children.includes(lid),
|
|
157
|
+
);
|
|
158
|
+
const sim = parent
|
|
159
|
+
? cosineSimilarity(queryVector, parent.embedding)
|
|
160
|
+
: 0;
|
|
161
|
+
return { lid, sim, parent };
|
|
162
|
+
})
|
|
163
|
+
.sort((a, b) => b.sim - a.sim)
|
|
164
|
+
.slice(0, maxPerCluster)
|
|
165
|
+
.filter((l) => !existingIds.has(l.lid))
|
|
166
|
+
.map((l) => {
|
|
167
|
+
existingIds.add(l.lid);
|
|
168
|
+
const rawScore = l.sim;
|
|
169
|
+
return {
|
|
170
|
+
nodeId: l.lid,
|
|
171
|
+
level: 0,
|
|
172
|
+
score: rawScore * weights[0],
|
|
173
|
+
rawScore,
|
|
174
|
+
isLeaf: true,
|
|
175
|
+
leafIds: [l.lid],
|
|
176
|
+
summary: "",
|
|
177
|
+
embedding: l.parent?.embedding ?? hit.embedding,
|
|
178
|
+
} as MultilevelHit;
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
expanded.push(hit, ...leafHits);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
return expanded;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// ── S42A-4: Result dedup ───────────────────────────────────────────────────
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Deduplicate hits: if both a cluster node and its leaf children appear in
|
|
191
|
+
* results, remove the cluster hit (leaves provide more specific context).
|
|
192
|
+
* If no leaves are in the set, keep the cluster hit (it provides the abstract view).
|
|
193
|
+
*/
|
|
194
|
+
export function deduplicateMultilevelHits(hits: MultilevelHit[]): MultilevelHit[] {
|
|
195
|
+
const leafIds = new Set(hits.filter((h) => h.isLeaf).map((h) => h.nodeId));
|
|
196
|
+
return hits.filter((h) => {
|
|
197
|
+
if (h.isLeaf) return true;
|
|
198
|
+
// Cluster hit: keep only if none of its leaf children are present.
|
|
199
|
+
return !h.leafIds.some((lid) => leafIds.has(lid));
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// ── S42A-5: Top-level pipeline ─────────────────────────────────────────────
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Full multi-level retrieval pipeline: score → expand → dedup → MMR → top-K.
|
|
207
|
+
* Drop-in replacement for `stagedExpansion()` in the RAPTOR recall path.
|
|
208
|
+
*/
|
|
209
|
+
export function multilevelRetrieval(
|
|
210
|
+
query: string,
|
|
211
|
+
tree: RaptorTree,
|
|
212
|
+
opts: MultilevelRetrieveOptions,
|
|
213
|
+
): MultilevelHit[] {
|
|
214
|
+
if (!tree.rootId) return [];
|
|
215
|
+
|
|
216
|
+
const { embedder } = opts;
|
|
217
|
+
const weights = opts.levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
|
|
218
|
+
const leafExp = opts.leafExpansion !== false; // default true
|
|
219
|
+
const maxLeafExp = opts.maxLeafExpansion ?? 10;
|
|
220
|
+
const k = opts.k ?? 5;
|
|
221
|
+
const lambda = opts.mmrLambda ?? 0.5;
|
|
222
|
+
|
|
223
|
+
const qv = embedder.embed(query);
|
|
224
|
+
|
|
225
|
+
// 1. Score all nodes with level weights.
|
|
226
|
+
const scored = scoreTreeLevels(query, tree, { embedder, levelWeights: weights });
|
|
227
|
+
|
|
228
|
+
// 2. Top-N candidates for MMR diversity window.
|
|
229
|
+
const topN = scored.slice(0, k * 3);
|
|
230
|
+
|
|
231
|
+
// 3. Leaf expansion (optional).
|
|
232
|
+
const expanded = leafExp
|
|
233
|
+
? expandLeafDescendants(topN, tree, maxLeafExp, embedder, qv, weights)
|
|
234
|
+
: topN;
|
|
235
|
+
|
|
236
|
+
// 4. Dedup: remove cluster hits when leaf children are present.
|
|
237
|
+
const deduped = deduplicateMultilevelHits(expanded);
|
|
238
|
+
|
|
239
|
+
// 5. MMR rerank to k.
|
|
240
|
+
const mmrItems = deduped.map((h) => ({
|
|
241
|
+
item: h,
|
|
242
|
+
vector: h.embedding as Vector,
|
|
243
|
+
relevance: h.score,
|
|
244
|
+
}));
|
|
245
|
+
return mmrRerank(mmrItems, k, lambda);
|
|
246
|
+
}
|
|
@@ -12,7 +12,7 @@ import assert from "node:assert/strict";
|
|
|
12
12
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
13
13
|
import { tmpdir } from "node:os";
|
|
14
14
|
import { join } from "node:path";
|
|
15
|
-
import { VectorStore } from "../../vectorStore.js";
|
|
15
|
+
import { VectorStore, vectorList, vectorSearch } from "../../vectorStore.js";
|
|
16
16
|
import { runRaptor } from "./index.js";
|
|
17
17
|
import { compactSession } from "../../engine.js";
|
|
18
18
|
import { Logger } from "../../log.js";
|
|
@@ -44,11 +44,11 @@ test("Fix D: vectorStore.search serves a persisted RAPTOR tree (broader recall)"
|
|
|
44
44
|
|
|
45
45
|
// No tree yet → flat search only, returns hits, no RAPTOR coverage.
|
|
46
46
|
assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree initially");
|
|
47
|
-
const flat = s
|
|
47
|
+
const flat = vectorSearch(s, SESS, "alpha wire bootstrap", 3);
|
|
48
48
|
assert.ok(flat.length > 0, "flat search returns hits");
|
|
49
49
|
|
|
50
50
|
// Build + persist a RAPTOR tree for the session (mirrors runCompact refresh).
|
|
51
|
-
const all = s
|
|
51
|
+
const all = vectorList(s, SESS);
|
|
52
52
|
const leaves = all.map((cp) => ({
|
|
53
53
|
id: cp.checkpointId,
|
|
54
54
|
messages: [],
|
|
@@ -60,7 +60,7 @@ test("Fix D: vectorStore.search serves a persisted RAPTOR tree (broader recall)"
|
|
|
60
60
|
|
|
61
61
|
// With the tree live + RAPTOR_ENABLED, search still returns hits and now
|
|
62
62
|
// exercises the RAPTOR-served path without regression.
|
|
63
|
-
const withTree = s
|
|
63
|
+
const withTree = vectorSearch(s, SESS, "alpha wire bootstrap", 3);
|
|
64
64
|
assert.ok(withTree.length > 0, "search returns hits with RAPTOR promoted");
|
|
65
65
|
// Every returned hit is a real checkpoint in the session.
|
|
66
66
|
for (const h of withTree) {
|
|
@@ -72,7 +72,7 @@ test("Fix D: search still works for a session with <2 leaves (no tree)", () => {
|
|
|
72
72
|
const stateDir = join(baseTmp, `run-${counter++}`);
|
|
73
73
|
const s = new VectorStore({ dedupSim: 0.9, stateDir, config: raptorConfig() });
|
|
74
74
|
compactSession({ sessionId: SESS, messages: [msg("only one topic here"), msg("ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
|
|
75
|
-
const r = s
|
|
75
|
+
const r = vectorSearch(s, SESS, "only one topic", 3);
|
|
76
76
|
assert.ok(r.length > 0, "single-checkpoint search still works (no tree)");
|
|
77
77
|
assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree built for <2 leaves");
|
|
78
78
|
});
|
|
@@ -33,7 +33,7 @@ function isLeafId(id: string, tree: RaptorTree): boolean {
|
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
/** All leaf (raw) ids reachable beneath a node via BFS. */
|
|
36
|
-
function leafDescendants(node: RaptorNode, tree: RaptorTree): string[] {
|
|
36
|
+
export function leafDescendants(node: RaptorNode, tree: RaptorTree): string[] {
|
|
37
37
|
const out: string[] = [];
|
|
38
38
|
const queue = [node];
|
|
39
39
|
while (queue.length) {
|
|
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
|
|
|
3
3
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
|
-
import { VectorStore, L2_ENABLED } from "../vectorStore.js";
|
|
6
|
+
import { VectorStore, L2_ENABLED, vectorSemDedup, vectorList, vectorSearch } from "../vectorStore.js";
|
|
7
7
|
import { mmrRerank } from "./mmr.js";
|
|
8
8
|
import { topK } from "./topk.js";
|
|
9
9
|
import { cosineSimilarity, defaultEmbedder } from "../embedder.js";
|
|
@@ -79,7 +79,7 @@ test("L2_ENABLED defaults true; search still returns hits", () => {
|
|
|
79
79
|
assert.equal(L2_ENABLED, true);
|
|
80
80
|
const s = store();
|
|
81
81
|
s.add({ sessionId: "sess_l2", summary: "investigated the parser", regionText: "investigated src/parser.ts and added a tokenizer", timestamp: 1 });
|
|
82
|
-
const hits = s
|
|
82
|
+
const hits = vectorSearch(s, "sess_l2", "src/parser.ts tokenizer", 3);
|
|
83
83
|
assert.ok(hits.length >= 1);
|
|
84
84
|
});
|
|
85
85
|
|
|
@@ -122,14 +122,14 @@ test("semDedup marks redundant near-identical rows 'removed' and search excludes
|
|
|
122
122
|
{ id: "chkpt_001", text: "the cache stores parsed ast nodes for fast lookup", tok: 100 },
|
|
123
123
|
{ id: "chkpt_002", text: "the cache stores parsed ast nodes for fast lookup and reuse", tok: 900 },
|
|
124
124
|
]);
|
|
125
|
-
const removed = s
|
|
125
|
+
const removed = vectorSemDedup(s,"sess_sd", 0.85);
|
|
126
126
|
assert.equal(removed, 1);
|
|
127
|
-
const st = s
|
|
127
|
+
const st = vectorList(s,"sess_sd");
|
|
128
128
|
const dropped = st.find((c) => c.dedupStatus === "removed");
|
|
129
129
|
assert.ok(dropped);
|
|
130
130
|
assert.equal(dropped.checkpointId, "chkpt_001"); // lower tokenEstimate removed
|
|
131
131
|
// Search excludes the removed row (only one active remains).
|
|
132
|
-
const hits = s
|
|
132
|
+
const hits = vectorSearch(s, "sess_sd", "cache parsed ast nodes", 5);
|
|
133
133
|
assert.equal(hits.length, 1);
|
|
134
134
|
});
|
|
135
135
|
|
|
@@ -140,8 +140,8 @@ test("semDedup is idempotent (re-run removes nothing new)", () => {
|
|
|
140
140
|
{ id: "chkpt_001", text: "identical region text for the dedup job now", tok: 100 },
|
|
141
141
|
{ id: "chkpt_002", text: "identical region text for the dedup job right now", tok: 200 },
|
|
142
142
|
]);
|
|
143
|
-
const first = s
|
|
144
|
-
const second = s
|
|
143
|
+
const first = vectorSemDedup(s,"sess_sd2", 0.85);
|
|
144
|
+
const second = vectorSemDedup(s,"sess_sd2", 0.85);
|
|
145
145
|
assert.equal(first, 1);
|
|
146
146
|
assert.equal(second, 0);
|
|
147
147
|
});
|
package/src/dedup-engine.test.ts
CHANGED
|
@@ -17,7 +17,7 @@ import fs from "fs";
|
|
|
17
17
|
import path from "path";
|
|
18
18
|
import os from "os";
|
|
19
19
|
import { compactSession } from "./engine.js";
|
|
20
|
-
import { VectorStore } from "./vectorStore.js";
|
|
20
|
+
import { VectorStore, vectorList, vectorStats, vectorWasInjected, vectorMarkInjected, vectorSearch } from "./vectorStore.js";
|
|
21
21
|
import { extractiveSummarize } from "./extractive.js";
|
|
22
22
|
import { estimateSessionTokens, estimateMessageTokens } from "./tokens.js";
|
|
23
23
|
import { autoCompactCheck } from "./compact.js";
|
|
@@ -181,7 +181,7 @@ describe("Dedupe Levels", () => {
|
|
|
181
181
|
const r2 = compactFull(s, SESS, [makeMsg("user", region)]);
|
|
182
182
|
assert.equal(r2.deduped, true);
|
|
183
183
|
assert.equal(r2.checkpointId, r1.checkpointId);
|
|
184
|
-
assert.equal(s
|
|
184
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
185
185
|
});
|
|
186
186
|
|
|
187
187
|
it("L0 only: distinct content stored twice creates two checkpoints", () => {
|
|
@@ -194,7 +194,7 @@ describe("Dedupe Levels", () => {
|
|
|
194
194
|
assert.equal(r1.deduped, false);
|
|
195
195
|
assert.equal(r2.deduped, false);
|
|
196
196
|
assert.notEqual(r1.checkpointId, r2.checkpointId);
|
|
197
|
-
assert.equal(s
|
|
197
|
+
assert.equal(vectorList(s,SESS).length, 2);
|
|
198
198
|
});
|
|
199
199
|
|
|
200
200
|
it("L1 only: one-word variants collapse; major rewrites do not", () => {
|
|
@@ -209,11 +209,11 @@ describe("Dedupe Levels", () => {
|
|
|
209
209
|
|
|
210
210
|
const r2 = s.add({ sessionId: SESS, summary: "migration", regionText: variant, timestamp: 2 });
|
|
211
211
|
assert.equal(r2.deduped, true, "one-word variant should be collapsed by L1");
|
|
212
|
-
assert.equal(s
|
|
212
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
213
213
|
|
|
214
214
|
const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: rewrite, timestamp: 3 });
|
|
215
215
|
assert.equal(r3.deduped, false, "major rewrite should not be collapsed by L1");
|
|
216
|
-
assert.equal(s
|
|
216
|
+
assert.equal(vectorList(s,SESS).length, 2);
|
|
217
217
|
});
|
|
218
218
|
|
|
219
219
|
it("L2 only: semantic paraphrases collapse; unrelated topics do not", () => {
|
|
@@ -233,11 +233,11 @@ describe("Dedupe Levels", () => {
|
|
|
233
233
|
|
|
234
234
|
const r2 = s.add({ sessionId: SESS, summary: "auth paraphrase", regionText: paraphrase, timestamp: 2 });
|
|
235
235
|
assert.equal(r2.deduped, true, "semantic paraphrase should be collapsed by L2");
|
|
236
|
-
assert.equal(s
|
|
236
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
237
237
|
|
|
238
238
|
const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: unrelated, timestamp: 3 });
|
|
239
239
|
assert.equal(r3.deduped, false, "unrelated topic should not be collapsed by L2");
|
|
240
|
-
assert.equal(s
|
|
240
|
+
assert.equal(vectorList(s,SESS).length, 2);
|
|
241
241
|
});
|
|
242
242
|
|
|
243
243
|
it("All tiers disabled: every store.add() with different region text creates a distinct checkpoint", () => {
|
|
@@ -254,7 +254,7 @@ describe("Dedupe Levels", () => {
|
|
|
254
254
|
assert.equal(r3.deduped, false);
|
|
255
255
|
assert.notEqual(r1.checkpoint.checkpointId, r2.checkpoint.checkpointId);
|
|
256
256
|
assert.notEqual(r2.checkpoint.checkpointId, r3.checkpoint.checkpointId);
|
|
257
|
-
assert.equal(s
|
|
257
|
+
assert.equal(vectorList(s,SESS).length, 3);
|
|
258
258
|
});
|
|
259
259
|
|
|
260
260
|
it("Combined L0+L1+L2: layered behavior exact -> near -> semantic", () => {
|
|
@@ -284,8 +284,8 @@ describe("Dedupe Levels", () => {
|
|
|
284
284
|
okReason(r4.dedupReason, ["contentSimilarity", "l1MinHash"]);
|
|
285
285
|
}
|
|
286
286
|
|
|
287
|
-
assert.ok(s
|
|
288
|
-
assert.ok(s
|
|
287
|
+
assert.ok(vectorList(s,SESS).length >= 1, "layered dedup keeps at least one checkpoint");
|
|
288
|
+
assert.ok(vectorList(s,SESS).length <= 4, "layered dedup should not explode to many checkpoints");
|
|
289
289
|
});
|
|
290
290
|
});
|
|
291
291
|
|
|
@@ -354,18 +354,18 @@ describe("Compression / Store Stats", () => {
|
|
|
354
354
|
const dup = "duplicate topic about payment gateway integration";
|
|
355
355
|
compactFull(s, SESS, [makeMsg("user", dup)], 1);
|
|
356
356
|
|
|
357
|
-
const statsBefore = s
|
|
357
|
+
const statsBefore = vectorStats(s,SESS);
|
|
358
358
|
assert.ok(statsBefore.checkpointCount >= 1, "checkpointCount should be positive");
|
|
359
359
|
assert.ok(statsBefore.totalTokenEstimate >= 0, "totalTokenEstimate should be non-negative");
|
|
360
360
|
assert.equal(statsBefore.dedupHitRate, 0, "no injections yet => dedupHitRate 0");
|
|
361
361
|
assert.equal(statsBefore.injectedCount, 0, "no injections yet => injectedCount 0");
|
|
362
362
|
|
|
363
|
-
const hits = s
|
|
363
|
+
const hits = vectorSearch(s, SESS, "payment gateway", 5);
|
|
364
364
|
assert.ok(hits.length > 0, "should find the stored checkpoint");
|
|
365
365
|
const cpId = hits[0].checkpoint.checkpointId;
|
|
366
366
|
|
|
367
|
-
s
|
|
368
|
-
const statsAfter = s
|
|
367
|
+
vectorMarkInjected(s,SESS, cpId);
|
|
368
|
+
const statsAfter = vectorStats(s,SESS);
|
|
369
369
|
assert.equal(statsAfter.injectedCount, 1, "injectedCount tracks markInjected");
|
|
370
370
|
if (statsAfter.checkpointCount > 0) {
|
|
371
371
|
assert.ok(
|
|
@@ -387,10 +387,10 @@ describe("Compression / Store Stats", () => {
|
|
|
387
387
|
compactFull(s, SESS, [makeMsg("user", "unique region for hit-rate measurement variant")], 1);
|
|
388
388
|
|
|
389
389
|
// Mark the first as injected.
|
|
390
|
-
const first = s
|
|
391
|
-
if (first) s
|
|
390
|
+
const first = vectorSearch(s, SESS, base, 1)[0]?.checkpoint.checkpointId;
|
|
391
|
+
if (first) vectorMarkInjected(s,SESS, first);
|
|
392
392
|
|
|
393
|
-
const stats = s
|
|
393
|
+
const stats = vectorStats(s,SESS);
|
|
394
394
|
assert.ok(stats.checkpointCount >= 1);
|
|
395
395
|
assert.ok(
|
|
396
396
|
stats.dedupHitRate > 0 || stats.checkpointCount === 1,
|
|
@@ -444,16 +444,16 @@ describe("Recall & Dedup Sentinel", () => {
|
|
|
444
444
|
const region = "manual sentinel tracking without recallAndInline";
|
|
445
445
|
compactFull(s, SESS, [makeMsg("user", region)]);
|
|
446
446
|
|
|
447
|
-
const hits = s
|
|
447
|
+
const hits = vectorSearch(s, SESS, "manual sentinel", 3);
|
|
448
448
|
assert.ok(hits.length > 0, "search should return checkpoint");
|
|
449
449
|
const cpId = hits[0].checkpoint.checkpointId;
|
|
450
|
-
assert.equal(s
|
|
450
|
+
assert.equal(vectorWasInjected(s,SESS, cpId), false, "not yet injected");
|
|
451
451
|
|
|
452
|
-
s
|
|
453
|
-
assert.equal(s
|
|
452
|
+
vectorMarkInjected(s,SESS, cpId);
|
|
453
|
+
assert.equal(vectorWasInjected(s,SESS, cpId), true, "markInjected recorded");
|
|
454
454
|
|
|
455
|
-
const hits2 = s
|
|
456
|
-
(h) => !s
|
|
455
|
+
const hits2 = vectorSearch(s, SESS, "manual sentinel", 3).filter(
|
|
456
|
+
(h) => !vectorWasInjected(s,SESS, h.checkpoint.checkpointId),
|
|
457
457
|
);
|
|
458
458
|
assert.equal(hits2.length, 0, "filtered search excludes injected checkpoint");
|
|
459
459
|
});
|
|
@@ -469,7 +469,7 @@ describe("Edge Cases", () => {
|
|
|
469
469
|
const r = compactSession({ sessionId: SESS, messages: [], keepFrom: 0 }, s);
|
|
470
470
|
assert.equal(r.skipped, true);
|
|
471
471
|
assert.equal(r.summary, "");
|
|
472
|
-
assert.equal(s
|
|
472
|
+
assert.equal(vectorList(s,SESS).length, 0);
|
|
473
473
|
});
|
|
474
474
|
|
|
475
475
|
it("single message with keepFrom=0 returns skipped", () => {
|
|
@@ -479,7 +479,7 @@ describe("Edge Cases", () => {
|
|
|
479
479
|
s,
|
|
480
480
|
);
|
|
481
481
|
assert.equal(r.skipped, true);
|
|
482
|
-
assert.equal(s
|
|
482
|
+
assert.equal(vectorList(s,SESS).length, 0);
|
|
483
483
|
});
|
|
484
484
|
|
|
485
485
|
it("keepFrom at messages.length compacts all prior messages (verified behavior)", () => {
|
|
@@ -491,7 +491,7 @@ describe("Edge Cases", () => {
|
|
|
491
491
|
const r = compactSession({ sessionId: SESS, messages, keepFrom: messages.length }, s);
|
|
492
492
|
assert.equal(r.skipped, false);
|
|
493
493
|
assert.ok(r.checkpointId);
|
|
494
|
-
assert.equal(s
|
|
494
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
495
495
|
});
|
|
496
496
|
|
|
497
497
|
it("unicode and emoji messages store and retrieve intact", () => {
|
|
@@ -501,7 +501,7 @@ describe("Edge Cases", () => {
|
|
|
501
501
|
"日本語テキスト 日本語テキスト 👍🔥";
|
|
502
502
|
const r = compactFull(s, SESS, [makeMsg("user", text)], 1);
|
|
503
503
|
assert.equal(r.skipped, false);
|
|
504
|
-
const stored = s
|
|
504
|
+
const stored = vectorList(s,SESS)[0];
|
|
505
505
|
assert.ok(stored);
|
|
506
506
|
const recovered = Buffer.from(stored.compressedOriginal ?? Buffer.alloc(0));
|
|
507
507
|
assert.ok(
|
|
@@ -518,8 +518,8 @@ describe("Edge Cases", () => {
|
|
|
518
518
|
const r = compactFull(s, SESS, [makeMsg("user", big)], 1);
|
|
519
519
|
assert.equal(r.skipped, false);
|
|
520
520
|
assert.ok(r.checkpointId);
|
|
521
|
-
assert.equal(s
|
|
522
|
-
const stats = s
|
|
521
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
522
|
+
const stats = vectorStats(s,SESS);
|
|
523
523
|
assert.ok(stats.totalTokenEstimate > 0);
|
|
524
524
|
});
|
|
525
525
|
|
|
@@ -542,7 +542,7 @@ describe("Edge Cases", () => {
|
|
|
542
542
|
r.summary.includes("assistant"),
|
|
543
543
|
"summary should reference roles or tools",
|
|
544
544
|
);
|
|
545
|
-
assert.equal(s
|
|
545
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
546
546
|
});
|
|
547
547
|
});
|
|
548
548
|
|