pi-mega-compact 0.4.24 → 0.4.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/openclaw-mega-compact.js +291 -0
- package/dist/src/minilm.js +92 -0
- package/dist/src/recall.js +55 -0
- package/dist/src/store/sqlite.js +8 -0
- package/dist/src/store/vectorIndex.js +210 -0
- package/dist/src/store/vectorIndex.test.js +99 -0
- package/dist/src/vectorStore.js +66 -1
- package/dist/src/wordpiece.js +129 -0
- package/package.json +3 -1
- package/src/recall.ts +63 -0
- package/src/store/sqlite.ts +13 -0
- package/src/store/vectorIndex.test.ts +116 -0
- package/src/store/vectorIndex.ts +243 -0
- package/src/vectorStore.ts +77 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vectorIndex.test.ts — Slice 2 async PGlite/HNSW vector index.
|
|
3
|
+
*
|
|
4
|
+
* Proves: cross-repo nearest-neighbor recall, repoId scoping, the dimension
|
|
5
|
+
* guard (non-512 vectors skipped, never corrupt the index), and graceful
|
|
6
|
+
* degradation when the index is disabled (kill-switch) — all without touching
|
|
7
|
+
* the synchronous node:sqlite store.
|
|
8
|
+
*
|
|
9
|
+
* The index is a WASM Postgres (PGlite) — fully local, zero network
|
|
10
|
+
* (PREVENT-PI-004). Each test isolates state via MEGACOMPACT_VECTOR_INDEX_DIR.
|
|
11
|
+
*/
|
|
12
|
+
import { test } from "node:test";
|
|
13
|
+
import assert from "node:assert/strict";
|
|
14
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
15
|
+
import { tmpdir } from "node:os";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { EMBEDDING_DIM, initVectorIndex, upsertEmbedding, searchAsync, closeVectorIndex, isVectorIndexDisabled, } from "./vectorIndex.js";
|
|
18
|
+
/** A 512-dim unit-ish vector with a single spike at `idx` (deterministic NN). */
|
|
19
|
+
function spikeVec(idx, magnitude = 1) {
|
|
20
|
+
const v = new Array(EMBEDDING_DIM).fill(0);
|
|
21
|
+
v[idx % EMBEDDING_DIM] = magnitude;
|
|
22
|
+
return v;
|
|
23
|
+
}
|
|
24
|
+
function isolateIndexDir() {
|
|
25
|
+
const dir = mkdtempSync(join(tmpdir(), "mc-vecidx-"));
|
|
26
|
+
process.env.MEGACOMPACT_VECTOR_INDEX_DIR = dir;
|
|
27
|
+
return dir;
|
|
28
|
+
}
|
|
29
|
+
test("cross-repo HNSW nearest-neighbor recall across repos + repoId scoping", async () => {
|
|
30
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
31
|
+
const dir = isolateIndexDir();
|
|
32
|
+
try {
|
|
33
|
+
await closeVectorIndex(); // ensure a fresh singleton for this dir
|
|
34
|
+
const pg = await initVectorIndex();
|
|
35
|
+
assert.ok(pg, "index should initialize (PGlite WASM available)");
|
|
36
|
+
// repoA: two checkpoints; repoB: one checkpoint. Distinct spike directions.
|
|
37
|
+
await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_001", spikeVec(0));
|
|
38
|
+
await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_002", spikeVec(5));
|
|
39
|
+
await upsertEmbedding("/repoB/.pi/mega-compact", "sessB", "chkpt_001", spikeVec(0));
|
|
40
|
+
// Cross-repo query near spike(0): nearest are the two spike(0) rows, one per repo.
|
|
41
|
+
const cross = await searchAsync(spikeVec(0), { k: 2 });
|
|
42
|
+
assert.equal(cross.length, 2, "cross-repo returns two nearest");
|
|
43
|
+
const repos = new Set(cross.map((h) => h.repoId));
|
|
44
|
+
assert.ok(repos.has("/repoA/.pi/mega-compact"), "hit from repoA");
|
|
45
|
+
assert.ok(repos.has("/repoB/.pi/mega-compact"), "hit from repoB");
|
|
46
|
+
assert.ok(cross[0].score > 0.99, "top hit is near-identical (cosine ~1)");
|
|
47
|
+
// Scoped to repoA only: excludes repoB even though repoB has an identical vec.
|
|
48
|
+
const scoped = await searchAsync(spikeVec(0), { k: 5, repoId: "/repoA/.pi/mega-compact" });
|
|
49
|
+
assert.ok(scoped.length >= 1, "scoped returns repoA hits");
|
|
50
|
+
assert.ok(scoped.every((h) => h.repoId === "/repoA/.pi/mega-compact"), "repoId filter excludes other repos");
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
await closeVectorIndex();
|
|
54
|
+
rmSync(dir, { recursive: true, force: true });
|
|
55
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
test("dimension guard: non-512 vectors are skipped, never corrupt the index", async () => {
|
|
59
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
60
|
+
const dir = isolateIndexDir();
|
|
61
|
+
try {
|
|
62
|
+
await closeVectorIndex();
|
|
63
|
+
await initVectorIndex();
|
|
64
|
+
// Wrong-dimension vector (BYO embedder mismatch) must be silently skipped.
|
|
65
|
+
await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_001", [1, 2, 3]);
|
|
66
|
+
const hits = await searchAsync(spikeVec(0), { k: 5 });
|
|
67
|
+
assert.equal(hits.length, 0, "no rows stored for a mismatched-dim vector");
|
|
68
|
+
// A correct-dim vector still stores fine afterward (index not corrupted).
|
|
69
|
+
await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_002", spikeVec(3));
|
|
70
|
+
const ok = await searchAsync(spikeVec(3), { k: 1 });
|
|
71
|
+
assert.equal(ok.length, 1, "valid vector stored after a skipped one");
|
|
72
|
+
assert.equal(ok[0].checkpointId, "chkpt_002");
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
await closeVectorIndex();
|
|
76
|
+
rmSync(dir, { recursive: true, force: true });
|
|
77
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
test("kill-switch: MEGACOMPACT_PGLITE_DISABLED disables the index gracefully", async () => {
|
|
81
|
+
const dir = isolateIndexDir();
|
|
82
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
|
|
83
|
+
try {
|
|
84
|
+
await closeVectorIndex();
|
|
85
|
+
assert.equal(isVectorIndexDisabled(), true, "kill-switch reported disabled");
|
|
86
|
+
const pg = await initVectorIndex();
|
|
87
|
+
assert.equal(pg, undefined, "init returns undefined when disabled");
|
|
88
|
+
// Upsert + search are no-ops that never throw and return empty.
|
|
89
|
+
await upsertEmbedding("/repoD/.pi/mega-compact", "sessD", "chkpt_001", spikeVec(0));
|
|
90
|
+
const hits = await searchAsync(spikeVec(0), { k: 3 });
|
|
91
|
+
assert.deepEqual(hits, [], "search returns [] when disabled");
|
|
92
|
+
}
|
|
93
|
+
finally {
|
|
94
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
95
|
+
await closeVectorIndex();
|
|
96
|
+
rmSync(dir, { recursive: true, force: true });
|
|
97
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
98
|
+
}
|
|
99
|
+
});
|
package/dist/src/vectorStore.js
CHANGED
|
@@ -19,7 +19,8 @@ import { isNearDuplicate } from "./dedup/l1-verify.js";
|
|
|
19
19
|
import { mmrRerank } from "./dedup/mmr.js";
|
|
20
20
|
import { topK } from "./dedup/topk.js";
|
|
21
21
|
import { openBloom, saveBloom } from "./store/bloom.js";
|
|
22
|
-
import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
|
|
22
|
+
import { listCheckpoints, nextCheckpointId, upsertCheckpoint, getCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
|
|
23
|
+
import { upsertEmbedding as indexUpsertEmbedding, initVectorIndex, searchAsync as vectorIndexSearch, } from "./store/vectorIndex.js";
|
|
23
24
|
import { rehydrateRaptorTree } from "./dedup/raptor/index.js";
|
|
24
25
|
import { stagedExpansion } from "./dedup/raptor/retrieval.js";
|
|
25
26
|
import { migrateJsonToSqlite } from "./store/migrate.js";
|
|
@@ -39,9 +40,17 @@ export class VectorStore {
|
|
|
39
40
|
cfg;
|
|
40
41
|
/** Optional monitoring target (Sprint 14). Undefined → no monitoring. */
|
|
41
42
|
eventsPath;
|
|
43
|
+
/**
|
|
44
|
+
* Repo key for the async PGlite vector index (Slice 2). We use the stateDir
|
|
45
|
+
* itself as the repo id — it is already unique per repo and available here
|
|
46
|
+
* without crossing into the pi-runtime layer (src/ stays pi-agnostic). The
|
|
47
|
+
* global index keys on repoId so recall can span repos.
|
|
48
|
+
*/
|
|
49
|
+
repoId;
|
|
42
50
|
constructor(opts = {}) {
|
|
43
51
|
this.embedder = opts.embedder ?? defaultEmbedder();
|
|
44
52
|
this.stateDir = opts.stateDir ?? getStateDir();
|
|
53
|
+
this.repoId = opts.repoId ?? this.stateDir;
|
|
45
54
|
// Sprint 14: all tier flags/thresholds flow from the single config source
|
|
46
55
|
// (DedupConfig). The legacy opts.dedupSim / opts.l2Enabled remain accepted
|
|
47
56
|
// for backward-compat callers but flags are authoritative via `cfg`.
|
|
@@ -283,6 +292,12 @@ export class VectorStore {
|
|
|
283
292
|
// Cumulative store-wide dedup accounting (attempt, not collapsed).
|
|
284
293
|
bumpDedupStats(false, this.stateDir);
|
|
285
294
|
onTier?.({ tier: "new", status: "stored" });
|
|
295
|
+
// Slice 2: best-effort, fire-and-forget mirror of this new checkpoint into
|
|
296
|
+
// the async global PGlite/HNSW index. NEVER awaited — must not block or
|
|
297
|
+
// throw into the synchronous add() path. On failure the index degrades to
|
|
298
|
+
// the sync scan (handled inside vectorIndex). The node:sqlite store remains
|
|
299
|
+
// authoritative; the index is rebuildable from it at any time.
|
|
300
|
+
void indexUpsertEmbedding(this.repoId, sessionId, checkpointId, checkpoint.embedding);
|
|
286
301
|
return { checkpoint, deduped: false };
|
|
287
302
|
}
|
|
288
303
|
/**
|
|
@@ -370,6 +385,56 @@ export class VectorStore {
|
|
|
370
385
|
const ranked = mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
|
|
371
386
|
return ranked;
|
|
372
387
|
}
|
|
388
|
+
/**
|
|
389
|
+
* Slice 2: async cross-repo (or single-repo) recall via the PGlite/HNSW index.
|
|
390
|
+
*
|
|
391
|
+
* This is the ONLY async recall surface and is a BONUS path — the synchronous
|
|
392
|
+
* `search()` above remains the default. `opts.repoId` scopes to one repo; omit
|
|
393
|
+
* it for cross-repo nearest-neighbor recall (the headline capability the sync
|
|
394
|
+
* per-session scan cannot provide).
|
|
395
|
+
*
|
|
396
|
+
* Best-effort: if the index is disabled/empty/failing, we fall back to the
|
|
397
|
+
* synchronous per-session `search()` for THIS repo so callers always get a
|
|
398
|
+
* sensible result. Hydrates each hit's StoredCheckpoint from the authoritative
|
|
399
|
+
* node:sqlite store (the hit's repoId doubles as that repo's stateDir), then
|
|
400
|
+
* MMR-dedupes the merged set.
|
|
401
|
+
*/
|
|
402
|
+
async searchAsync(sessionId, query, k = 3, opts = {}) {
|
|
403
|
+
const sid = normalizeSessionId(sessionId);
|
|
404
|
+
const qv = this.embedder.embed(query);
|
|
405
|
+
// repoId filter: explicit opts.repoId wins; else this repo unless crossRepo.
|
|
406
|
+
const repoId = opts.repoId ?? (opts.crossRepo ? undefined : this.repoId);
|
|
407
|
+
let indexHits = [];
|
|
408
|
+
try {
|
|
409
|
+
await initVectorIndex();
|
|
410
|
+
indexHits = await vectorIndexSearch(qv, { k: Math.max(k * 2, k), repoId });
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
indexHits = [];
|
|
414
|
+
}
|
|
415
|
+
if (indexHits.length === 0) {
|
|
416
|
+
// Index empty/unavailable → synchronous per-session fallback (this repo).
|
|
417
|
+
return this.search(sid, query, k);
|
|
418
|
+
}
|
|
419
|
+
// Hydrate each index hit from the authoritative node:sqlite store. repoId is
|
|
420
|
+
// that repo's stateDir, so cross-repo hits resolve against their own store.
|
|
421
|
+
const hydrated = [];
|
|
422
|
+
for (const h of indexHits) {
|
|
423
|
+
const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
|
|
424
|
+
if (cp && cp.dedupStatus !== "removed") {
|
|
425
|
+
hydrated.push({ checkpoint: cp, score: h.score });
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
if (hydrated.length === 0)
|
|
429
|
+
return this.search(sid, query, k);
|
|
430
|
+
// MMR-dedupe the merged candidate set for diversity (mirrors sync search).
|
|
431
|
+
const mmrItems = hydrated.map((h) => ({
|
|
432
|
+
item: h,
|
|
433
|
+
vector: h.checkpoint.embedding,
|
|
434
|
+
relevance: h.score,
|
|
435
|
+
}));
|
|
436
|
+
return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
|
|
437
|
+
}
|
|
373
438
|
/**
|
|
374
439
|
* Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
|
|
375
440
|
* return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* wordpiece.ts — a self-contained WordPiece tokenizer for BERT/MiniLM.
|
|
3
|
+
*
|
|
4
|
+
* Loads the canonical `vocab.txt` (bert-base-uncased, ~30K tokens) from disk and
|
|
5
|
+
* implements the standard uncased BERT preprocessing + greedy longest-match
|
|
6
|
+
* WordPiece segmentation. No native dependency, no network — the vocab file is a
|
|
7
|
+
* local artifact fetched once by scripts/setup-minilm.mjs (PREVENT-PI-004).
|
|
8
|
+
*
|
|
9
|
+
* This mirrors HuggingFace `BertTokenizer` closely enough for sentence-embedding
|
|
10
|
+
* use: lowercase, strip accents, split on whitespace + punctuation, then
|
|
11
|
+
* WordPiece each token with the `##` continuation convention. Special tokens
|
|
12
|
+
* [CLS]/[SEP] are added by the caller's encode().
|
|
13
|
+
*/
|
|
14
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
15
|
+
const UNK = "[UNK]";
|
|
16
|
+
const CLS = "[CLS]";
|
|
17
|
+
const SEP = "[SEP]";
|
|
18
|
+
const PAD = "[PAD]";
|
|
19
|
+
const MAX_INPUT_CHARS_PER_WORD = 200;
|
|
20
|
+
export class WordPieceTokenizer {
|
|
21
|
+
vocab;
|
|
22
|
+
clsId;
|
|
23
|
+
sepId;
|
|
24
|
+
padId;
|
|
25
|
+
unkId;
|
|
26
|
+
constructor(vocab) {
|
|
27
|
+
this.vocab = vocab;
|
|
28
|
+
this.clsId = vocab.get(CLS) ?? 101;
|
|
29
|
+
this.sepId = vocab.get(SEP) ?? 102;
|
|
30
|
+
this.padId = vocab.get(PAD) ?? 0;
|
|
31
|
+
this.unkId = vocab.get(UNK) ?? 100;
|
|
32
|
+
}
|
|
33
|
+
/** Build a tokenizer from a vocab.txt file (one token per line, index = line). */
|
|
34
|
+
static fromVocabFile(path) {
|
|
35
|
+
if (!existsSync(path)) {
|
|
36
|
+
throw new Error(`WordPiece vocab not found at ${path}. Run: node scripts/setup-minilm.mjs`);
|
|
37
|
+
}
|
|
38
|
+
const lines = readFileSync(path, "utf-8").split("\n");
|
|
39
|
+
const vocab = new Map();
|
|
40
|
+
for (let i = 0; i < lines.length; i++) {
|
|
41
|
+
const tok = lines[i].replace(/\r$/, "");
|
|
42
|
+
if (tok.length > 0 || i < lines.length - 1)
|
|
43
|
+
vocab.set(tok, i);
|
|
44
|
+
}
|
|
45
|
+
return new WordPieceTokenizer(vocab);
|
|
46
|
+
}
|
|
47
|
+
/** Uncased BERT basic tokenization: lowercase, strip accents, split on ws+punct. */
|
|
48
|
+
basicTokenize(text) {
|
|
49
|
+
// NFD + strip combining marks (accent removal), then lowercase.
|
|
50
|
+
const cleaned = text
|
|
51
|
+
.normalize("NFD")
|
|
52
|
+
.replace(/[̀-ͯ]/g, "")
|
|
53
|
+
.toLowerCase();
|
|
54
|
+
const tokens = [];
|
|
55
|
+
let buf = "";
|
|
56
|
+
const flush = () => {
|
|
57
|
+
if (buf.length > 0) {
|
|
58
|
+
tokens.push(buf);
|
|
59
|
+
buf = "";
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
for (const ch of cleaned) {
|
|
63
|
+
if (/\s/.test(ch)) {
|
|
64
|
+
flush();
|
|
65
|
+
}
|
|
66
|
+
else if (/[!-/:-@[-`{-~¡-¿]/.test(ch)) {
|
|
67
|
+
// Punctuation becomes its own token.
|
|
68
|
+
flush();
|
|
69
|
+
tokens.push(ch);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
buf += ch;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
flush();
|
|
76
|
+
return tokens;
|
|
77
|
+
}
|
|
78
|
+
/** Greedy longest-match WordPiece for a single word. */
|
|
79
|
+
wordpiece(word) {
|
|
80
|
+
if (word.length > MAX_INPUT_CHARS_PER_WORD)
|
|
81
|
+
return [UNK];
|
|
82
|
+
const pieces = [];
|
|
83
|
+
let start = 0;
|
|
84
|
+
while (start < word.length) {
|
|
85
|
+
let end = word.length;
|
|
86
|
+
let cur = null;
|
|
87
|
+
while (start < end) {
|
|
88
|
+
let sub = word.slice(start, end);
|
|
89
|
+
if (start > 0)
|
|
90
|
+
sub = "##" + sub;
|
|
91
|
+
if (this.vocab.has(sub)) {
|
|
92
|
+
cur = sub;
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
end--;
|
|
96
|
+
}
|
|
97
|
+
if (cur === null)
|
|
98
|
+
return [UNK]; // any unmatchable piece → whole word is UNK
|
|
99
|
+
pieces.push(cur);
|
|
100
|
+
start = end;
|
|
101
|
+
}
|
|
102
|
+
return pieces;
|
|
103
|
+
}
|
|
104
|
+
/** Tokenize text into WordPiece token strings (no special tokens). */
|
|
105
|
+
tokenize(text) {
|
|
106
|
+
const out = [];
|
|
107
|
+
for (const word of this.basicTokenize(text)) {
|
|
108
|
+
for (const piece of this.wordpiece(word))
|
|
109
|
+
out.push(piece);
|
|
110
|
+
}
|
|
111
|
+
return out;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Encode text into model inputs with [CLS]…[SEP], truncated to `maxLen`.
|
|
115
|
+
* attention_mask is all 1s (no padding for single-sequence inference).
|
|
116
|
+
*/
|
|
117
|
+
encode(text, maxLen = 256) {
|
|
118
|
+
const pieces = this.tokenize(text).slice(0, Math.max(0, maxLen - 2));
|
|
119
|
+
const inputIds = [this.clsId];
|
|
120
|
+
for (const p of pieces)
|
|
121
|
+
inputIds.push(this.vocab.get(p) ?? this.unkId);
|
|
122
|
+
inputIds.push(this.sepId);
|
|
123
|
+
return {
|
|
124
|
+
inputIds,
|
|
125
|
+
attentionMask: inputIds.map(() => 1),
|
|
126
|
+
tokenTypeIds: inputIds.map(() => 0),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.25",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-2-Clause",
|
|
@@ -57,6 +57,8 @@
|
|
|
57
57
|
"typescript": "^5.4.0"
|
|
58
58
|
},
|
|
59
59
|
"dependencies": {
|
|
60
|
+
"@electric-sql/pglite": "^0.5.4",
|
|
61
|
+
"@electric-sql/pglite-pgvector": "^0.0.5",
|
|
60
62
|
"@mongodb-js/zstd": "^7.0.0"
|
|
61
63
|
}
|
|
62
64
|
}
|
package/src/recall.ts
CHANGED
|
@@ -137,3 +137,66 @@ export function recallAndInline(
|
|
|
137
137
|
empty: toInject.length === 0,
|
|
138
138
|
};
|
|
139
139
|
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Slice 2 async cross-repo recall. Same dedup/bound/inline contract as
|
|
143
|
+
* `recallAndInline`, but backed by `VectorStore.searchAsync` so it can recall
|
|
144
|
+
* across repos (HNSW NN over the global PGlite index) when `opts.crossRepo` is
|
|
145
|
+
* set. The synchronous `recallAndInline` is unchanged and remains the default
|
|
146
|
+
* per-session path. Inline-window dedupe + token cap (Fix C) apply here too.
|
|
147
|
+
*
|
|
148
|
+
* `store` must provide `searchAsync` (the live VectorStore does). Errors fall
|
|
149
|
+
* back to an empty result — recall is a bonus, never a hard dependency.
|
|
150
|
+
*/
|
|
151
|
+
export async function recallAndInlineAsync(
|
|
152
|
+
opts: RecallInjectOptions & { crossRepo?: boolean; repoId?: string },
|
|
153
|
+
store: Pick<VectorStore, "searchAsync" | "wasInjected" | "markInjected">,
|
|
154
|
+
): Promise<RecallInjectResult> {
|
|
155
|
+
const limit = opts.limit ?? 3;
|
|
156
|
+
const skip = opts.skipInjected ?? true;
|
|
157
|
+
const maxTokens = opts.recallMaxTokens ?? 0;
|
|
158
|
+
const doWindowDedupe = opts.windowDedupe ?? false;
|
|
159
|
+
const dedupSim = opts.dedupSim ?? 0.9;
|
|
160
|
+
|
|
161
|
+
let hits: SearchHit[] = [];
|
|
162
|
+
try {
|
|
163
|
+
hits = await store.searchAsync(opts.sessionId, opts.query, limit, {
|
|
164
|
+
crossRepo: opts.crossRepo,
|
|
165
|
+
repoId: opts.repoId,
|
|
166
|
+
});
|
|
167
|
+
} catch {
|
|
168
|
+
hits = [];
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
let liveEmbeddings: number[][] = [];
|
|
172
|
+
if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
|
|
173
|
+
const embedder = defaultEmbedder();
|
|
174
|
+
liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const toInject: SearchHit[] = [];
|
|
178
|
+
const parts: string[] = [];
|
|
179
|
+
let blockTokens = 0;
|
|
180
|
+
|
|
181
|
+
for (const h of hits) {
|
|
182
|
+
if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
|
|
183
|
+
if (doWindowDedupe && liveEmbeddings.length > 0) {
|
|
184
|
+
const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
|
|
185
|
+
if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim)) continue;
|
|
186
|
+
}
|
|
187
|
+
const part = formatRecallBlock([h]);
|
|
188
|
+
const partTokens = estimateBlockTokens(part);
|
|
189
|
+
if (maxTokens > 0 && blockTokens + partTokens > maxTokens) break;
|
|
190
|
+
parts.push(part);
|
|
191
|
+
toInject.push(h);
|
|
192
|
+
blockTokens += partTokens;
|
|
193
|
+
store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const block = parts.join("\n");
|
|
197
|
+
const report = toInject.map(
|
|
198
|
+
(h) => ` • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`,
|
|
199
|
+
);
|
|
200
|
+
|
|
201
|
+
return { toInject, report, block, empty: toInject.length === 0 };
|
|
202
|
+
}
|
package/src/store/sqlite.ts
CHANGED
|
@@ -872,6 +872,19 @@ export function hasCheckpoint(sessionId: string, checkpointId: string, stateDir:
|
|
|
872
872
|
return row !== undefined;
|
|
873
873
|
}
|
|
874
874
|
|
|
875
|
+
/** Fetch a single checkpoint by (session, id), or undefined if absent. */
|
|
876
|
+
export function getCheckpoint(
|
|
877
|
+
sessionId: string,
|
|
878
|
+
checkpointId: string,
|
|
879
|
+
stateDir: string = getStateDir(),
|
|
880
|
+
): StoredCheckpoint | undefined {
|
|
881
|
+
const db = openStore(stateDir);
|
|
882
|
+
const row = db
|
|
883
|
+
.prepare("SELECT * FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
|
|
884
|
+
.get(normalizeSessionId(sessionId), checkpointId) as any;
|
|
885
|
+
return row ? rowToCheckpoint(row) : undefined;
|
|
886
|
+
}
|
|
887
|
+
|
|
875
888
|
/** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
|
|
876
889
|
export function setDedupStatus(
|
|
877
890
|
checkpointId: string,
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vectorIndex.test.ts — Slice 2 async PGlite/HNSW vector index.
|
|
3
|
+
*
|
|
4
|
+
* Proves: cross-repo nearest-neighbor recall, repoId scoping, the dimension
|
|
5
|
+
* guard (non-512 vectors skipped, never corrupt the index), and graceful
|
|
6
|
+
* degradation when the index is disabled (kill-switch) — all without touching
|
|
7
|
+
* the synchronous node:sqlite store.
|
|
8
|
+
*
|
|
9
|
+
* The index is a WASM Postgres (PGlite) — fully local, zero network
|
|
10
|
+
* (PREVENT-PI-004). Each test isolates state via MEGACOMPACT_VECTOR_INDEX_DIR.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { test } from "node:test";
|
|
14
|
+
import assert from "node:assert/strict";
|
|
15
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
16
|
+
import { tmpdir } from "node:os";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
import {
|
|
19
|
+
EMBEDDING_DIM,
|
|
20
|
+
initVectorIndex,
|
|
21
|
+
upsertEmbedding,
|
|
22
|
+
searchAsync,
|
|
23
|
+
closeVectorIndex,
|
|
24
|
+
isVectorIndexDisabled,
|
|
25
|
+
} from "./vectorIndex.js";
|
|
26
|
+
|
|
27
|
+
/** A 512-dim unit-ish vector with a single spike at `idx` (deterministic NN). */
|
|
28
|
+
function spikeVec(idx: number, magnitude = 1): number[] {
|
|
29
|
+
const v = new Array<number>(EMBEDDING_DIM).fill(0);
|
|
30
|
+
v[idx % EMBEDDING_DIM] = magnitude;
|
|
31
|
+
return v;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function isolateIndexDir(): string {
|
|
35
|
+
const dir = mkdtempSync(join(tmpdir(), "mc-vecidx-"));
|
|
36
|
+
process.env.MEGACOMPACT_VECTOR_INDEX_DIR = dir;
|
|
37
|
+
return dir;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
test("cross-repo HNSW nearest-neighbor recall across repos + repoId scoping", async () => {
|
|
41
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
42
|
+
const dir = isolateIndexDir();
|
|
43
|
+
try {
|
|
44
|
+
await closeVectorIndex(); // ensure a fresh singleton for this dir
|
|
45
|
+
const pg = await initVectorIndex();
|
|
46
|
+
assert.ok(pg, "index should initialize (PGlite WASM available)");
|
|
47
|
+
|
|
48
|
+
// repoA: two checkpoints; repoB: one checkpoint. Distinct spike directions.
|
|
49
|
+
await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_001", spikeVec(0));
|
|
50
|
+
await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_002", spikeVec(5));
|
|
51
|
+
await upsertEmbedding("/repoB/.pi/mega-compact", "sessB", "chkpt_001", spikeVec(0));
|
|
52
|
+
|
|
53
|
+
// Cross-repo query near spike(0): nearest are the two spike(0) rows, one per repo.
|
|
54
|
+
const cross = await searchAsync(spikeVec(0), { k: 2 });
|
|
55
|
+
assert.equal(cross.length, 2, "cross-repo returns two nearest");
|
|
56
|
+
const repos = new Set(cross.map((h) => h.repoId));
|
|
57
|
+
assert.ok(repos.has("/repoA/.pi/mega-compact"), "hit from repoA");
|
|
58
|
+
assert.ok(repos.has("/repoB/.pi/mega-compact"), "hit from repoB");
|
|
59
|
+
assert.ok(cross[0].score > 0.99, "top hit is near-identical (cosine ~1)");
|
|
60
|
+
|
|
61
|
+
// Scoped to repoA only: excludes repoB even though repoB has an identical vec.
|
|
62
|
+
const scoped = await searchAsync(spikeVec(0), { k: 5, repoId: "/repoA/.pi/mega-compact" });
|
|
63
|
+
assert.ok(scoped.length >= 1, "scoped returns repoA hits");
|
|
64
|
+
assert.ok(
|
|
65
|
+
scoped.every((h) => h.repoId === "/repoA/.pi/mega-compact"),
|
|
66
|
+
"repoId filter excludes other repos",
|
|
67
|
+
);
|
|
68
|
+
} finally {
|
|
69
|
+
await closeVectorIndex();
|
|
70
|
+
rmSync(dir, { recursive: true, force: true });
|
|
71
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("dimension guard: non-512 vectors are skipped, never corrupt the index", async () => {
|
|
76
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
77
|
+
const dir = isolateIndexDir();
|
|
78
|
+
try {
|
|
79
|
+
await closeVectorIndex();
|
|
80
|
+
await initVectorIndex();
|
|
81
|
+
// Wrong-dimension vector (BYO embedder mismatch) must be silently skipped.
|
|
82
|
+
await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_001", [1, 2, 3]);
|
|
83
|
+
const hits = await searchAsync(spikeVec(0), { k: 5 });
|
|
84
|
+
assert.equal(hits.length, 0, "no rows stored for a mismatched-dim vector");
|
|
85
|
+
|
|
86
|
+
// A correct-dim vector still stores fine afterward (index not corrupted).
|
|
87
|
+
await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_002", spikeVec(3));
|
|
88
|
+
const ok = await searchAsync(spikeVec(3), { k: 1 });
|
|
89
|
+
assert.equal(ok.length, 1, "valid vector stored after a skipped one");
|
|
90
|
+
assert.equal(ok[0].checkpointId, "chkpt_002");
|
|
91
|
+
} finally {
|
|
92
|
+
await closeVectorIndex();
|
|
93
|
+
rmSync(dir, { recursive: true, force: true });
|
|
94
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("kill-switch: MEGACOMPACT_PGLITE_DISABLED disables the index gracefully", async () => {
|
|
99
|
+
const dir = isolateIndexDir();
|
|
100
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
|
|
101
|
+
try {
|
|
102
|
+
await closeVectorIndex();
|
|
103
|
+
assert.equal(isVectorIndexDisabled(), true, "kill-switch reported disabled");
|
|
104
|
+
const pg = await initVectorIndex();
|
|
105
|
+
assert.equal(pg, undefined, "init returns undefined when disabled");
|
|
106
|
+
// Upsert + search are no-ops that never throw and return empty.
|
|
107
|
+
await upsertEmbedding("/repoD/.pi/mega-compact", "sessD", "chkpt_001", spikeVec(0));
|
|
108
|
+
const hits = await searchAsync(spikeVec(0), { k: 3 });
|
|
109
|
+
assert.deepEqual(hits, [], "search returns [] when disabled");
|
|
110
|
+
} finally {
|
|
111
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
112
|
+
await closeVectorIndex();
|
|
113
|
+
rmSync(dir, { recursive: true, force: true });
|
|
114
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
115
|
+
}
|
|
116
|
+
});
|