pi-mega-compact 0.8.21 → 0.8.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +6 -2
- package/README.md +1 -1
- package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
- package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
- package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
- package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
- package/dist/extensions/dashboard-server/html.js +2 -621
- package/dist/extensions/dashboard-server/routes-core.js +62 -0
- package/dist/extensions/dashboard-server/routes-game.js +323 -0
- package/dist/extensions/dashboard-server/routes-repo.js +170 -0
- package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
- package/dist/extensions/dashboard-server/routes.js +10 -0
- package/dist/extensions/dashboard-server/server.js +26 -623
- package/dist/extensions/mega-commands.js +4 -3
- package/dist/extensions/mega-events/agent-handlers.js +2 -1
- package/dist/extensions/mega-events/compact-handlers.js +26 -0
- package/dist/extensions/mega-events/session-handlers.js +2 -1
- package/dist/extensions/mega-pipeline/compact.js +3 -2
- package/dist/extensions/mega-runtime/state.js +7 -7
- package/dist/src/dedup/raptor/multilevel.js +172 -0
- package/dist/src/dedup/raptor/multilevel.test.js +203 -0
- package/dist/src/dedup/raptor/promote.test.js +5 -5
- package/dist/src/dedup/raptor/retrieval.js +1 -1
- package/dist/src/dedup/sprint12.test.js +7 -7
- package/dist/src/dedup-engine.test.js +29 -29
- package/dist/src/e2e.test.js +38 -38
- package/dist/src/engine.js +3 -3
- package/dist/src/engine.test.js +6 -6
- package/dist/src/importance.js +197 -0
- package/dist/src/importance.test.js +372 -0
- package/dist/src/ratio.bench.test.js +18 -18
- package/dist/src/recall.js +6 -5
- package/dist/src/recall.test.js +85 -27
- package/dist/src/sprint14.test.js +2 -2
- package/dist/src/store/migrate.test.js +5 -5
- package/dist/src/store/sprint10.test.js +5 -5
- package/dist/src/store/sqlite/global-index.js +5 -174
- package/dist/src/store/sqlite/global-sessions.js +190 -0
- package/dist/src/vector-read.js +168 -0
- package/dist/src/vector-search.js +191 -0
- package/dist/src/vectorStore.js +10 -297
- package/dist/src/vectorStore.test.js +32 -32
- package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
- package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
- package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
- package/extensions/dashboard-server/dashboard-client.ts +21 -0
- package/extensions/dashboard-server/html.ts +2 -621
- package/extensions/dashboard-server/routes-core.ts +113 -0
- package/extensions/dashboard-server/routes-game.ts +386 -0
- package/extensions/dashboard-server/routes-repo.ts +212 -0
- package/extensions/dashboard-server/routes-sessions.ts +195 -0
- package/extensions/dashboard-server/routes.ts +13 -0
- package/extensions/dashboard-server/server.ts +37 -700
- package/extensions/mega-commands.ts +4 -3
- package/extensions/mega-events/agent-handlers.ts +2 -1
- package/extensions/mega-events/compact-handlers.ts +28 -0
- package/extensions/mega-events/session-handlers.ts +2 -1
- package/extensions/mega-pipeline/compact.ts +3 -2
- package/extensions/mega-runtime/state.ts +7 -7
- package/extensions/openclaw-mega-compact.ts +2 -2
- package/package.json +2 -2
- package/src/dedup/raptor/multilevel.test.ts +278 -0
- package/src/dedup/raptor/multilevel.ts +246 -0
- package/src/dedup/raptor/promote.test.ts +5 -5
- package/src/dedup/raptor/retrieval.ts +1 -1
- package/src/dedup/sprint12.test.ts +7 -7
- package/src/dedup-engine.test.ts +30 -30
- package/src/e2e.test.ts +38 -38
- package/src/engine.test.ts +6 -6
- package/src/engine.ts +3 -3
- package/src/importance.test.ts +538 -0
- package/src/importance.ts +312 -0
- package/src/ratio.bench.test.ts +18 -18
- package/src/recall.test.ts +101 -29
- package/src/recall.ts +9 -9
- package/src/sprint14.test.ts +2 -2
- package/src/store/migrate.test.ts +5 -5
- package/src/store/sprint10.test.ts +5 -5
- package/src/store/sqlite/global-index.ts +18 -290
- package/src/store/sqlite/global-sessions.ts +291 -0
- package/src/vector-read.ts +237 -0
- package/src/vector-search.ts +231 -0
- package/src/vectorStore.test.ts +32 -32
- package/src/vectorStore.ts +29 -356
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-search.ts — free functions for VectorStore search operations.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from vectorStore.ts (PR0 split) to bring it under 500 lines.
|
|
5
|
+
* All functions take `store: VectorStore` as first param and access store
|
|
6
|
+
* fields/props directly via type-cast to access private fields (acceptable
|
|
7
|
+
* since these fields were effectively public via the original methods).
|
|
8
|
+
*
|
|
9
|
+
* Call sites in src/ and extensions/ are rewritten from `store.search(...)`
|
|
10
|
+
* → `vectorSearch(store, ...)` and `store.searchAsync(...)` →
|
|
11
|
+
* `vectorSearchAsync(store, ...)`.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { cosineSimilarity } from "./embedder.js";
|
|
15
|
+
import { normalizeSessionId } from "./store.js";
|
|
16
|
+
import { mmrRerank, type MmrItem } from "./dedup/mmr.js";
|
|
17
|
+
import { topK } from "./dedup/topk.js";
|
|
18
|
+
import {
|
|
19
|
+
listCheckpoints,
|
|
20
|
+
getCheckpoint,
|
|
21
|
+
maxCheckpointTimestamp,
|
|
22
|
+
} from "./store/sqlite.js";
|
|
23
|
+
import {
|
|
24
|
+
initVectorIndex,
|
|
25
|
+
searchAsync as vectorIndexSearch,
|
|
26
|
+
type VectorIndexHit,
|
|
27
|
+
} from "./store/vectorIndex.js";
|
|
28
|
+
import { rehydrateRaptorTree, isShadowMode } from "./dedup/raptor/index.js";
|
|
29
|
+
import { stagedExpansion } from "./dedup/raptor/retrieval.js";
|
|
30
|
+
import type { SearchHit, VectorStore } from "./vectorStore.js";
|
|
31
|
+
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
// raptorSearchHits — internal helper (NOT exported)
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
|
|
38
|
+
* return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
|
|
39
|
+
* exists (small sessions — flat search remains the path). Best-effort/non-fatal.
|
|
40
|
+
*/
|
|
41
|
+
function raptorSearchHits(store: VectorStore, sid: string, query: string, k: number): SearchHit[] {
|
|
42
|
+
const t0 = Date.now();
|
|
43
|
+
try {
|
|
44
|
+
const stateDir = store.stateDir;
|
|
45
|
+
const cfg = store.cfg;
|
|
46
|
+
const embedder = store.embedder;
|
|
47
|
+
const record = store.record;
|
|
48
|
+
|
|
49
|
+
// S25 gate (a): honor the shadow contract at SERVE time. The tree is still
|
|
50
|
+
// built + persisted (logging-only) but NOT merged into recall while
|
|
51
|
+
// RAPTOR_SHADOW_MODE is anything other than "false".
|
|
52
|
+
if (isShadowMode()) return [];
|
|
53
|
+
const tree = rehydrateRaptorTree(sid, stateDir);
|
|
54
|
+
if (!tree || !tree.rootId) return [];
|
|
55
|
+
// S25 gate (b): freshness + fallback guards. Skip a tree built before the
|
|
56
|
+
// newest checkpoint (stale → may reference trimmed/deduped leaves) or one
|
|
57
|
+
// whose root is a budget-exhausted extractive fallback (level 99).
|
|
58
|
+
if (tree.timedOut) return [];
|
|
59
|
+
const maxTs = maxCheckpointTimestamp(sid, stateDir);
|
|
60
|
+
if (tree.builtAt && tree.builtAt < maxTs) return [];
|
|
61
|
+
const leafIds = stagedExpansion(query, tree, {
|
|
62
|
+
embedder,
|
|
63
|
+
k,
|
|
64
|
+
topM: cfg.RAPTOR_CLUSTERS_PER_LEVEL,
|
|
65
|
+
mmrLambda: cfg.MMR_LAMBDA,
|
|
66
|
+
});
|
|
67
|
+
if (leafIds.length === 0) return [];
|
|
68
|
+
const all = listCheckpoints(sid, stateDir).filter(
|
|
69
|
+
(cp) => cp.dedupStatus !== "removed",
|
|
70
|
+
);
|
|
71
|
+
const qv = embedder.embed(query);
|
|
72
|
+
const hits: SearchHit[] = [];
|
|
73
|
+
for (const id of leafIds) {
|
|
74
|
+
const cp = all.find((c) => c.checkpointId === id);
|
|
75
|
+
if (cp) hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
|
|
76
|
+
}
|
|
77
|
+
// S25 monitoring: emit a raptor_serve decision so canary.ts can track
|
|
78
|
+
// p95 latency + the tier's live traffic (non-fatal, best-effort).
|
|
79
|
+
record("RAPTOR", hits.length > 0 ? "new" : "mark_only", `leaves=${leafIds.length}`, Date.now() - t0);
|
|
80
|
+
return hits;
|
|
81
|
+
} catch {
|
|
82
|
+
return [];
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
// vectorSearch
|
|
88
|
+
// ---------------------------------------------------------------------------
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Semantic search within a session's checkpoints. Returns top-K by cosine
|
|
92
|
+
* similarity, diversified via MMR (QA #10) so a cluster of near-identical
|
|
93
|
+
* hits yields at most a few distinct-relevance results.
|
|
94
|
+
*
|
|
95
|
+
* Heap-based top-K (QA #4, O(N log k)) replaces the old full sort; MMR then
|
|
96
|
+
* reranks the candidate window for diversity.
|
|
97
|
+
*/
|
|
98
|
+
export function vectorSearch(
|
|
99
|
+
store: VectorStore,
|
|
100
|
+
sessionId: string,
|
|
101
|
+
query: string,
|
|
102
|
+
k = 3,
|
|
103
|
+
): SearchHit[] {
|
|
104
|
+
const stateDir = store.stateDir;
|
|
105
|
+
const cfg = store.cfg;
|
|
106
|
+
const embedder = store.embedder;
|
|
107
|
+
|
|
108
|
+
const sid = normalizeSessionId(sessionId);
|
|
109
|
+
const checkpoints = listCheckpoints(sid, stateDir).filter(
|
|
110
|
+
(cp) => cp.dedupStatus !== "removed", // SemDeDup: exclude removed rows
|
|
111
|
+
);
|
|
112
|
+
if (checkpoints.length === 0) return [];
|
|
113
|
+
const qv = embedder.embed(query);
|
|
114
|
+
|
|
115
|
+
const scored: SearchHit[] = checkpoints.map((cp) => ({
|
|
116
|
+
checkpoint: cp,
|
|
117
|
+
score: cosineSimilarity(qv, cp.embedding),
|
|
118
|
+
}));
|
|
119
|
+
|
|
120
|
+
// Heap top-K over a widened window (2k) so MMR has diverse candidates.
|
|
121
|
+
const window = topK(
|
|
122
|
+
scored.map((h) => ({ item: h, score: h.score })),
|
|
123
|
+
Math.max(k * 2, k),
|
|
124
|
+
).map((s) => s.item);
|
|
125
|
+
// MMR (QA #10) is part of the L2 semantic tier: skip it when L2 is disabled
|
|
126
|
+
// (Sprint 14 flag), returning the plain relevance-ranked window instead.
|
|
127
|
+
if (!cfg.L2_ENABLED) return window.slice(0, k);
|
|
128
|
+
|
|
129
|
+
// Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
|
|
130
|
+
// merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
|
|
131
|
+
// RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
|
|
132
|
+
// O(n) flat leaves, tightening the block at read time.
|
|
133
|
+
if (cfg.RAPTOR_ENABLED) {
|
|
134
|
+
const rh = raptorSearchHits(store, sid, query, k);
|
|
135
|
+
if (rh.length > 0) {
|
|
136
|
+
const merged: SearchHit[] = [...window];
|
|
137
|
+
for (const h of rh) {
|
|
138
|
+
if (!merged.some((m) => m.checkpoint.checkpointId === h.checkpoint.checkpointId)) {
|
|
139
|
+
merged.push(h);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
const mmrItems: MmrItem<SearchHit>[] = merged.map((h) => ({
|
|
143
|
+
item: h,
|
|
144
|
+
vector: h.checkpoint.embedding,
|
|
145
|
+
relevance: h.score,
|
|
146
|
+
}));
|
|
147
|
+
return mmrRerank(mmrItems, k, cfg.MMR_LAMBDA);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const mmrItems: MmrItem<SearchHit>[] = window.map((h) => ({
|
|
152
|
+
item: h,
|
|
153
|
+
vector: h.checkpoint.embedding,
|
|
154
|
+
relevance: h.score,
|
|
155
|
+
}));
|
|
156
|
+
const ranked = mmrRerank(mmrItems, k, cfg.MMR_LAMBDA);
|
|
157
|
+
return ranked;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ---------------------------------------------------------------------------
|
|
161
|
+
// vectorSearchAsync
|
|
162
|
+
// ---------------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
export interface VectorSearchOptions {
|
|
165
|
+
/** Scope to a specific repo (omit for cross-repo NN). */
|
|
166
|
+
repoId?: string;
|
|
167
|
+
/** Include hits from all repos (cross-repo NN). Mutually exclusive with repoId. */
|
|
168
|
+
crossRepo?: boolean;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Slice 2: async cross-repo (or single-repo) recall via the PGlite/HNSW index.
|
|
173
|
+
*
|
|
174
|
+
* This is the ONLY async recall surface and is a BONUS path — the synchronous
|
|
175
|
+
* `vectorSearch` above remains the default. `opts.repoId` scopes to one repo;
|
|
176
|
+
* omit it for cross-repo nearest-neighbor recall (the headline capability the
|
|
177
|
+
* sync per-session scan cannot provide).
|
|
178
|
+
*
|
|
179
|
+
* Best-effort: if the index is disabled/empty/failing, we fall back to the
|
|
180
|
+
* synchronous per-session `vectorSearch` for THIS repo so callers always get
|
|
181
|
+
* a sensible result. Hydrates each hit's StoredCheckpoint from the authoritative
|
|
182
|
+
* node:sqlite store (the hit's repoId doubles as that repo's stateDir), then
|
|
183
|
+
* MMR-dedupes the merged set.
|
|
184
|
+
*/
|
|
185
|
+
export async function vectorSearchAsync(
|
|
186
|
+
store: VectorStore,
|
|
187
|
+
sessionId: string,
|
|
188
|
+
query: string,
|
|
189
|
+
k = 3,
|
|
190
|
+
opts: VectorSearchOptions = {},
|
|
191
|
+
): Promise<SearchHit[]> {
|
|
192
|
+
const cfg = store.cfg;
|
|
193
|
+
const embedder = store.embedder;
|
|
194
|
+
|
|
195
|
+
const sid = normalizeSessionId(sessionId);
|
|
196
|
+
const qv = embedder.embed(query);
|
|
197
|
+
// repoId filter: explicit opts.repoId wins; else this repo unless crossRepo.
|
|
198
|
+
const selfRepo = store.repoId;
|
|
199
|
+
const repoId = opts.repoId ?? (opts.crossRepo ? undefined : selfRepo);
|
|
200
|
+
let indexHits: VectorIndexHit[] = [];
|
|
201
|
+
try {
|
|
202
|
+
await initVectorIndex();
|
|
203
|
+
indexHits = await vectorIndexSearch(qv, { k: Math.max(k * 2, k), repoId });
|
|
204
|
+
} catch {
|
|
205
|
+
indexHits = [];
|
|
206
|
+
}
|
|
207
|
+
if (indexHits.length === 0) {
|
|
208
|
+
// Index empty/unavailable → synchronous per-session fallback (this repo).
|
|
209
|
+
return vectorSearch(store, sid, query, k);
|
|
210
|
+
}
|
|
211
|
+
// Hydrate each index hit from the authoritative node:sqlite store. repoId is
|
|
212
|
+
// that repo's stateDir, so cross-repo hits resolve against their own store.
|
|
213
|
+
// Tag cross-repo hits with their source repoId so the recall block can label
|
|
214
|
+
// them ("from repo <name>"); same-repo hits stay unlabeled.
|
|
215
|
+
const hydrated: SearchHit[] = [];
|
|
216
|
+
for (const h of indexHits) {
|
|
217
|
+
const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
|
|
218
|
+
if (cp && cp.dedupStatus !== "removed") {
|
|
219
|
+
const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
|
|
220
|
+
hydrated.push({ checkpoint: cp, score: h.score, repoId: crossRepo ? h.repoId : undefined });
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
if (hydrated.length === 0) return vectorSearch(store, sid, query, k);
|
|
224
|
+
// MMR-dedupe the merged candidate set for diversity (mirrors sync search).
|
|
225
|
+
const mmrItems: MmrItem<SearchHit>[] = hydrated.map((h) => ({
|
|
226
|
+
item: h,
|
|
227
|
+
vector: h.checkpoint.embedding,
|
|
228
|
+
relevance: h.score,
|
|
229
|
+
}));
|
|
230
|
+
return mmrRerank(mmrItems, k, cfg.MMR_LAMBDA);
|
|
231
|
+
}
|
package/src/vectorStore.test.ts
CHANGED
|
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
|
|
|
3
3
|
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
|
-
import { VectorStore, computeRegionHash } from "./vectorStore.js";
|
|
6
|
+
import { VectorStore, computeRegionHash, vectorStats, vectorDedupe, vectorWasInjected, vectorMarkInjected, vectorList, vectorTopSimilar, vectorRepoStats, vectorSearch } from "./vectorStore.js";
|
|
7
7
|
import {
|
|
8
8
|
TrigramEmbedder,
|
|
9
9
|
cosineSimilarity,
|
|
@@ -57,7 +57,7 @@ test("add then search returns the planted checkpoint top-1", () => {
|
|
|
57
57
|
tokenEstimate: 1200,
|
|
58
58
|
timestamp: 1000,
|
|
59
59
|
});
|
|
60
|
-
const hits = s
|
|
60
|
+
const hits = vectorSearch(s, "sess_abc", "src/compact.ts truncate helper", 3);
|
|
61
61
|
assert.equal(hits.length, 1);
|
|
62
62
|
assert.ok(hits[0].score > 0.5);
|
|
63
63
|
assert.equal(hits[0].checkpoint.summary.includes("src/compact.ts"), true);
|
|
@@ -81,7 +81,7 @@ test("dedup by regionHash: identical region is not double-stored", () => {
|
|
|
81
81
|
assert.equal(r1.deduped, false);
|
|
82
82
|
assert.equal(r2.deduped, true);
|
|
83
83
|
assert.equal(r1.checkpoint.checkpointId, r2.checkpoint.checkpointId);
|
|
84
|
-
assert.equal(s
|
|
84
|
+
assert.equal(vectorSearch(s, "sess_dup", "anything", 10).length, 1);
|
|
85
85
|
});
|
|
86
86
|
|
|
87
87
|
test("dedup cascade: summaryHash catches same-topic incremental compactions", () => {
|
|
@@ -115,7 +115,7 @@ test("dedup cascade: summaryHash dedup still stores only one checkpoint", () =>
|
|
|
115
115
|
const ts = "some topic summary that is identical";
|
|
116
116
|
s.add({ sessionId: "sess_sh2", summary, topicSummary: ts, regionText: "region a", timestamp: 1 });
|
|
117
117
|
s.add({ sessionId: "sess_sh2", summary, topicSummary: ts, regionText: "region b", timestamp: 2 });
|
|
118
|
-
assert.equal(s
|
|
118
|
+
assert.equal(vectorStats(s,"sess_sh2").checkpointCount, 1);
|
|
119
119
|
});
|
|
120
120
|
|
|
121
121
|
test("dedupe() sentinel returns true for a stored region", () => {
|
|
@@ -128,8 +128,8 @@ test("dedupe() sentinel returns true for a stored region", () => {
|
|
|
128
128
|
regionText: region,
|
|
129
129
|
timestamp: 1,
|
|
130
130
|
});
|
|
131
|
-
assert.equal(s
|
|
132
|
-
assert.equal(s
|
|
131
|
+
assert.equal(vectorDedupe(s,"sess_sent", hash), true);
|
|
132
|
+
assert.equal(vectorDedupe(s,"sess_sent", "deadbeef"), false);
|
|
133
133
|
});
|
|
134
134
|
|
|
135
135
|
test("near-duplicate collapse keeps only the top of a near-identical pair", () => {
|
|
@@ -148,7 +148,7 @@ test("near-duplicate collapse keeps only the top of a near-identical pair", () =
|
|
|
148
148
|
"user investigated src/compact.ts and added a truncate helper for the summaries",
|
|
149
149
|
timestamp: 2,
|
|
150
150
|
});
|
|
151
|
-
const hits = s
|
|
151
|
+
const hits = vectorSearch(s,
|
|
152
152
|
"sess_nd",
|
|
153
153
|
"user investigated src/compact.ts and added a truncate helper for summaries",
|
|
154
154
|
5,
|
|
@@ -164,9 +164,9 @@ test("markInjected / wasInjected track injection", () => {
|
|
|
164
164
|
regionText: "inject region",
|
|
165
165
|
timestamp: 1,
|
|
166
166
|
});
|
|
167
|
-
assert.equal(s
|
|
168
|
-
s
|
|
169
|
-
assert.equal(s
|
|
167
|
+
assert.equal(vectorWasInjected(s,"sess_inj", r.checkpoint.checkpointId), false);
|
|
168
|
+
vectorMarkInjected(s,"sess_inj", r.checkpoint.checkpointId);
|
|
169
|
+
assert.equal(vectorWasInjected(s,"sess_inj", r.checkpoint.checkpointId), true);
|
|
170
170
|
});
|
|
171
171
|
|
|
172
172
|
test("normalizeSessionId handles null, prefixed, and uuid forms", () => {
|
|
@@ -207,7 +207,7 @@ test("checkpoints survive a fresh store instance (on-disk)", () => {
|
|
|
207
207
|
timestamp: 1,
|
|
208
208
|
});
|
|
209
209
|
const s2 = new VectorStore({ dedupSim: 0.9, stateDir: dir }); // new instance, same disk state
|
|
210
|
-
const hits = s2
|
|
210
|
+
const hits = vectorSearch(s2, "sess_persist", "persist region text", 3);
|
|
211
211
|
assert.equal(hits.length, 1);
|
|
212
212
|
assert.equal(hits[0].checkpoint.summary, "persisted");
|
|
213
213
|
});
|
|
@@ -218,7 +218,7 @@ test("corrupt checkpoint file falls back to empty (no throw)", () => {
|
|
|
218
218
|
const file = join(dir, "sess_corrupt.checkpoints.json.gz");
|
|
219
219
|
writeFileSync(file, Buffer.from("not a gzip"));
|
|
220
220
|
const s = new VectorStore({ dedupSim: 0.9, stateDir: dir });
|
|
221
|
-
assert.equal(s
|
|
221
|
+
assert.equal(vectorSearch(s, "sess_corrupt", "q", 3).length, 0);
|
|
222
222
|
});
|
|
223
223
|
|
|
224
224
|
test("stats reports counts, last checkpoint, and dedup rate", () => {
|
|
@@ -237,15 +237,15 @@ test("stats reports counts, last checkpoint, and dedup rate", () => {
|
|
|
237
237
|
tokenEstimate: 700,
|
|
238
238
|
timestamp: 2,
|
|
239
239
|
});
|
|
240
|
-
const st1 = s
|
|
240
|
+
const st1 = vectorStats(s,"sess_stats");
|
|
241
241
|
assert.equal(st1.checkpointCount, 2);
|
|
242
242
|
assert.equal(st1.lastCheckpointId, "chkpt_002");
|
|
243
243
|
assert.equal(st1.totalTokenEstimate, 1200);
|
|
244
244
|
assert.equal(st1.injectedCount, 0);
|
|
245
245
|
assert.equal(st1.dedupHitRate, 0);
|
|
246
246
|
|
|
247
|
-
s
|
|
248
|
-
const st2 = s
|
|
247
|
+
vectorMarkInjected(s,"sess_stats", "chkpt_001");
|
|
248
|
+
const st2 = vectorStats(s,"sess_stats");
|
|
249
249
|
assert.equal(st2.injectedCount, 1);
|
|
250
250
|
assert.ok(Math.abs(st2.dedupHitRate - 0.5) < 1e-9);
|
|
251
251
|
});
|
|
@@ -257,7 +257,7 @@ test("tokensSaved = original − stored per session; deduped add saves the whole
|
|
|
257
257
|
// cp2: orig 3000, stored 700 → saved 2300
|
|
258
258
|
s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
|
|
259
259
|
s.add({ sessionId: "sess_saved", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
|
|
260
|
-
const st = s
|
|
260
|
+
const st = vectorStats(s,"sess_saved");
|
|
261
261
|
assert.equal(st.totalTokenEstimate, 1200, "Σ stored summaries");
|
|
262
262
|
assert.equal(st.originalTokens, 5000, "Σ original region tokens");
|
|
263
263
|
assert.equal(st.tokensSaved, 3800, "per-session saved = Σ(original − stored) = 1500 + 2300");
|
|
@@ -269,14 +269,14 @@ test("tokensSaved = original − stored per session; deduped add saves the whole
|
|
|
269
269
|
// dedupCollapsed bumps, and no new checkpoint row is created.
|
|
270
270
|
const deduped = s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
|
|
271
271
|
assert.ok(deduped.deduped, "identical region should dedup");
|
|
272
|
-
const st3 = s
|
|
272
|
+
const st3 = vectorStats(s,"sess_saved");
|
|
273
273
|
// Per-session DB sum only covers stored rows (deduped adds create no row), so
|
|
274
274
|
// the per-session figure is unchanged; the deduped save lands in the repo meta.
|
|
275
275
|
assert.equal(st3.tokensSaved, 3800, "per-session DB sum unchanged by deduped add");
|
|
276
276
|
assert.equal(st3.dedupCollapsed, 1, "deduped collapse counted");
|
|
277
277
|
assert.equal(st3.dedupAttempts, 3);
|
|
278
278
|
// Repo cumulative counter DID capture the deduped region's full original size.
|
|
279
|
-
assert.equal(s
|
|
279
|
+
assert.equal(vectorRepoStats(s).tokensSaved, 3800 + 2000, "repo saved includes deduped original");
|
|
280
280
|
});
|
|
281
281
|
|
|
282
282
|
test("repoStats aggregates every session + counts deduped original tokens", () => {
|
|
@@ -286,7 +286,7 @@ test("repoStats aggregates every session + counts deduped original tokens", () =
|
|
|
286
286
|
a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
|
|
287
287
|
b.add({ sessionId: "sess_b", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
|
|
288
288
|
|
|
289
|
-
const repo = a
|
|
289
|
+
const repo = vectorRepoStats(a);
|
|
290
290
|
assert.equal(repo.checkpointCount, 2, "checkpoints across both sessions");
|
|
291
291
|
assert.equal(repo.sessionCount, 2, "two distinct sessions");
|
|
292
292
|
assert.equal(repo.totalTokenEstimate, 1200, "Σ stored");
|
|
@@ -297,7 +297,7 @@ test("repoStats aggregates every session + counts deduped original tokens", () =
|
|
|
297
297
|
// A deduped add into sess_a: whole original region saved, no new row.
|
|
298
298
|
const deduped = a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
|
|
299
299
|
assert.ok(deduped.deduped);
|
|
300
|
-
const repo2 = a
|
|
300
|
+
const repo2 = vectorRepoStats(a);
|
|
301
301
|
assert.equal(repo2.tokensSaved, 3800 + 2000, "deduped collapse adds full original region to repo saved");
|
|
302
302
|
assert.equal(repo2.dedupCollapsed, 1);
|
|
303
303
|
assert.equal(repo2.checkpointCount, 2, "still two stored checkpoints");
|
|
@@ -329,7 +329,7 @@ test("whitespace-variant region is deduplicated", () => {
|
|
|
329
329
|
timestamp: 2,
|
|
330
330
|
});
|
|
331
331
|
assert.equal(r2.deduped, true, "whitespace-variant should be deduplicated");
|
|
332
|
-
assert.equal(s
|
|
332
|
+
assert.equal(vectorStats(s,"sess_ws").checkpointCount, 1, "only one checkpoint stored");
|
|
333
333
|
});
|
|
334
334
|
|
|
335
335
|
test("topSimilar returns n most similar checkpoints to the current (most recent)", () => {
|
|
@@ -354,7 +354,7 @@ test("topSimilar returns n most similar checkpoints to the current (most recent)
|
|
|
354
354
|
regionText: "fix the buffer overflow in src/compact.ts by adding a bounds check before truncate",
|
|
355
355
|
timestamp: 3,
|
|
356
356
|
});
|
|
357
|
-
const hits = s
|
|
357
|
+
const hits = vectorTopSimilar(s,"sess_top", 10);
|
|
358
358
|
assert.equal(hits.length, 2); // two other checkpoints
|
|
359
359
|
// The compact.ts checkpoint should rank above the guitar checkpoint
|
|
360
360
|
assert.equal(hits[0].checkpoint.summary, "compact");
|
|
@@ -376,21 +376,21 @@ test("topSimilar excludes the current checkpoint itself", () => {
|
|
|
376
376
|
regionText: "alpha region text two",
|
|
377
377
|
timestamp: 2,
|
|
378
378
|
});
|
|
379
|
-
const hits = s
|
|
379
|
+
const hits = vectorTopSimilar(s,"sess_self", 5);
|
|
380
380
|
assert.equal(hits.length, 1);
|
|
381
381
|
assert.notEqual(hits[0].checkpoint.checkpointId, "chkpt_002"); // not the current
|
|
382
382
|
});
|
|
383
383
|
|
|
384
384
|
test("topSimilar returns empty for sessions with 0 or 1 checkpoints", () => {
|
|
385
385
|
const s = store();
|
|
386
|
-
assert.deepEqual(s
|
|
386
|
+
assert.deepEqual(vectorTopSimilar(s,"sess_none", 5), []);
|
|
387
387
|
s.add({
|
|
388
388
|
sessionId: "sess_one",
|
|
389
389
|
summary: "solo",
|
|
390
390
|
regionText: "only checkpoint",
|
|
391
391
|
timestamp: 1,
|
|
392
392
|
});
|
|
393
|
-
assert.deepEqual(s
|
|
393
|
+
assert.deepEqual(vectorTopSimilar(s,"sess_one", 5), []);
|
|
394
394
|
});
|
|
395
395
|
|
|
396
396
|
test("topSimilar respects the n limit", () => {
|
|
@@ -410,13 +410,13 @@ test("topSimilar respects the n limit", () => {
|
|
|
410
410
|
timestamp: i + 1,
|
|
411
411
|
});
|
|
412
412
|
}
|
|
413
|
-
const hits = s
|
|
413
|
+
const hits = vectorTopSimilar(s,"sess_limit", 2);
|
|
414
414
|
assert.equal(hits.length, 2);
|
|
415
415
|
});
|
|
416
416
|
|
|
417
417
|
test("stats on empty session returns zeros and nulls", () => {
|
|
418
418
|
const s = store();
|
|
419
|
-
const st = s
|
|
419
|
+
const st = vectorStats(s,"sess_empty");
|
|
420
420
|
assert.equal(st.checkpointCount, 0);
|
|
421
421
|
assert.equal(st.lastCheckpointId, undefined);
|
|
422
422
|
assert.equal(st.totalTokenEstimate, 0);
|
|
@@ -442,7 +442,7 @@ test("L0 content-hash dedup: identical content under different regionText collap
|
|
|
442
442
|
assert.equal(r1.deduped, false);
|
|
443
443
|
assert.equal(r2.deduped, true);
|
|
444
444
|
assert.equal(r2.reason, "contentHash");
|
|
445
|
-
assert.equal(s
|
|
445
|
+
assert.equal(vectorList(s,"sess_l0").length, 1);
|
|
446
446
|
});
|
|
447
447
|
|
|
448
448
|
test("L0 content-hash dedup stores both hash fields and bumps timestamp on hit", () => {
|
|
@@ -457,7 +457,7 @@ test("L0 content-hash dedup stores both hash fields and bumps timestamp on hit",
|
|
|
457
457
|
});
|
|
458
458
|
assert.equal(r2.deduped, true);
|
|
459
459
|
assert.equal(r2.reason, "contentHash");
|
|
460
|
-
const cp = s
|
|
460
|
+
const cp = vectorList(s,"sess_l0ts")[0];
|
|
461
461
|
assert.equal(cp.contentHash?.length, 64);
|
|
462
462
|
assert.equal(cp.contentHash2?.length, 64);
|
|
463
463
|
assert.equal(cp.contentHashVersion, 1);
|
|
@@ -468,7 +468,7 @@ test("compressed_original roundtrips through versioned compression", () => {
|
|
|
468
468
|
const s = store();
|
|
469
469
|
const raw = "raw region text preserved for audit and replay";
|
|
470
470
|
s.add({ sessionId: "sess_co", summary: "x", regionText: raw, timestamp: 1 });
|
|
471
|
-
const cp = s
|
|
471
|
+
const cp = vectorList(s,"sess_co")[0];
|
|
472
472
|
assert.ok(cp.compressedOriginal instanceof Buffer);
|
|
473
473
|
const restored = decompressSmart(cp.compressedOriginal as Buffer).toString("utf-8");
|
|
474
474
|
assert.equal(restored, raw);
|
|
@@ -517,7 +517,7 @@ test("L1 catches a one-word-diff near-duplicate that L0 misses", () => {
|
|
|
517
517
|
assert.equal(r1.deduped, false);
|
|
518
518
|
assert.equal(r2.deduped, true);
|
|
519
519
|
assert.equal(r2.reason, "l1MinHash");
|
|
520
|
-
assert.equal(s
|
|
520
|
+
assert.equal(vectorList(s,"sess_l1").length, 1);
|
|
521
521
|
});
|
|
522
522
|
|
|
523
523
|
test("L1 does NOT falsely dedup genuinely different content", () => {
|
|
@@ -525,7 +525,7 @@ test("L1 does NOT falsely dedup genuinely different content", () => {
|
|
|
525
525
|
s.add({ sessionId: "sess_l1b", summary: "a", regionText: "the database migration added three indexes", timestamp: 1 });
|
|
526
526
|
const r2 = s.add({ sessionId: "sess_l1b", summary: "b", regionText: "the frontend added a dark mode toggle", timestamp: 2 });
|
|
527
527
|
assert.equal(r2.deduped, false);
|
|
528
|
-
assert.equal(s
|
|
528
|
+
assert.equal(vectorList(s,"sess_l1b").length, 2);
|
|
529
529
|
});
|
|
530
530
|
|
|
531
531
|
test("cleanup", () => {
|