pi-mega-compact 0.4.24 → 0.4.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,99 @@
1
+ /**
2
+ * vectorIndex.test.ts — Slice 2 async PGlite/HNSW vector index.
3
+ *
4
+ * Proves: cross-repo nearest-neighbor recall, repoId scoping, the dimension
5
+ * guard (non-512 vectors skipped, never corrupt the index), and graceful
6
+ * degradation when the index is disabled (kill-switch) — all without touching
7
+ * the synchronous node:sqlite store.
8
+ *
9
+ * The index is a WASM Postgres (PGlite) — fully local, zero network
10
+ * (PREVENT-PI-004). Each test isolates state via MEGACOMPACT_VECTOR_INDEX_DIR.
11
+ */
12
+ import { test } from "node:test";
13
+ import assert from "node:assert/strict";
14
+ import { mkdtempSync, rmSync } from "node:fs";
15
+ import { tmpdir } from "node:os";
16
+ import { join } from "node:path";
17
+ import { EMBEDDING_DIM, initVectorIndex, upsertEmbedding, searchAsync, closeVectorIndex, isVectorIndexDisabled, } from "./vectorIndex.js";
18
+ /** A 512-dim unit-ish vector with a single spike at `idx` (deterministic NN). */
19
+ function spikeVec(idx, magnitude = 1) {
20
+ const v = new Array(EMBEDDING_DIM).fill(0);
21
+ v[idx % EMBEDDING_DIM] = magnitude;
22
+ return v;
23
+ }
24
+ function isolateIndexDir() {
25
+ const dir = mkdtempSync(join(tmpdir(), "mc-vecidx-"));
26
+ process.env.MEGACOMPACT_VECTOR_INDEX_DIR = dir;
27
+ return dir;
28
+ }
29
+ test("cross-repo HNSW nearest-neighbor recall across repos + repoId scoping", async () => {
30
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
31
+ const dir = isolateIndexDir();
32
+ try {
33
+ await closeVectorIndex(); // ensure a fresh singleton for this dir
34
+ const pg = await initVectorIndex();
35
+ assert.ok(pg, "index should initialize (PGlite WASM available)");
36
+ // repoA: two checkpoints; repoB: one checkpoint. Distinct spike directions.
37
+ await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_001", spikeVec(0));
38
+ await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_002", spikeVec(5));
39
+ await upsertEmbedding("/repoB/.pi/mega-compact", "sessB", "chkpt_001", spikeVec(0));
40
+ // Cross-repo query near spike(0): nearest are the two spike(0) rows, one per repo.
41
+ const cross = await searchAsync(spikeVec(0), { k: 2 });
42
+ assert.equal(cross.length, 2, "cross-repo returns two nearest");
43
+ const repos = new Set(cross.map((h) => h.repoId));
44
+ assert.ok(repos.has("/repoA/.pi/mega-compact"), "hit from repoA");
45
+ assert.ok(repos.has("/repoB/.pi/mega-compact"), "hit from repoB");
46
+ assert.ok(cross[0].score > 0.99, "top hit is near-identical (cosine ~1)");
47
+ // Scoped to repoA only: excludes repoB even though repoB has an identical vec.
48
+ const scoped = await searchAsync(spikeVec(0), { k: 5, repoId: "/repoA/.pi/mega-compact" });
49
+ assert.ok(scoped.length >= 1, "scoped returns repoA hits");
50
+ assert.ok(scoped.every((h) => h.repoId === "/repoA/.pi/mega-compact"), "repoId filter excludes other repos");
51
+ }
52
+ finally {
53
+ await closeVectorIndex();
54
+ rmSync(dir, { recursive: true, force: true });
55
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
56
+ }
57
+ });
58
+ test("dimension guard: non-512 vectors are skipped, never corrupt the index", async () => {
59
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
60
+ const dir = isolateIndexDir();
61
+ try {
62
+ await closeVectorIndex();
63
+ await initVectorIndex();
64
+ // Wrong-dimension vector (BYO embedder mismatch) must be silently skipped.
65
+ await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_001", [1, 2, 3]);
66
+ const hits = await searchAsync(spikeVec(0), { k: 5 });
67
+ assert.equal(hits.length, 0, "no rows stored for a mismatched-dim vector");
68
+ // A correct-dim vector still stores fine afterward (index not corrupted).
69
+ await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_002", spikeVec(3));
70
+ const ok = await searchAsync(spikeVec(3), { k: 1 });
71
+ assert.equal(ok.length, 1, "valid vector stored after a skipped one");
72
+ assert.equal(ok[0].checkpointId, "chkpt_002");
73
+ }
74
+ finally {
75
+ await closeVectorIndex();
76
+ rmSync(dir, { recursive: true, force: true });
77
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
78
+ }
79
+ });
80
+ test("kill-switch: MEGACOMPACT_PGLITE_DISABLED disables the index gracefully", async () => {
81
+ const dir = isolateIndexDir();
82
+ process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
83
+ try {
84
+ await closeVectorIndex();
85
+ assert.equal(isVectorIndexDisabled(), true, "kill-switch reported disabled");
86
+ const pg = await initVectorIndex();
87
+ assert.equal(pg, undefined, "init returns undefined when disabled");
88
+ // Upsert + search are no-ops that never throw and return empty.
89
+ await upsertEmbedding("/repoD/.pi/mega-compact", "sessD", "chkpt_001", spikeVec(0));
90
+ const hits = await searchAsync(spikeVec(0), { k: 3 });
91
+ assert.deepEqual(hits, [], "search returns [] when disabled");
92
+ }
93
+ finally {
94
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
95
+ await closeVectorIndex();
96
+ rmSync(dir, { recursive: true, force: true });
97
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
98
+ }
99
+ });
@@ -19,7 +19,8 @@ import { isNearDuplicate } from "./dedup/l1-verify.js";
19
19
  import { mmrRerank } from "./dedup/mmr.js";
20
20
  import { topK } from "./dedup/topk.js";
21
21
  import { openBloom, saveBloom } from "./store/bloom.js";
22
- import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
22
+ import { listCheckpoints, nextCheckpointId, upsertCheckpoint, getCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
23
+ import { upsertEmbedding as indexUpsertEmbedding, initVectorIndex, searchAsync as vectorIndexSearch, } from "./store/vectorIndex.js";
23
24
  import { rehydrateRaptorTree } from "./dedup/raptor/index.js";
24
25
  import { stagedExpansion } from "./dedup/raptor/retrieval.js";
25
26
  import { migrateJsonToSqlite } from "./store/migrate.js";
@@ -39,9 +40,17 @@ export class VectorStore {
39
40
  cfg;
40
41
  /** Optional monitoring target (Sprint 14). Undefined → no monitoring. */
41
42
  eventsPath;
43
+ /**
44
+ * Repo key for the async PGlite vector index (Slice 2). We use the stateDir
45
+ * itself as the repo id — it is already unique per repo and available here
46
+ * without crossing into the pi-runtime layer (src/ stays pi-agnostic). The
47
+ * global index keys on repoId so recall can span repos.
48
+ */
49
+ repoId;
42
50
  constructor(opts = {}) {
43
51
  this.embedder = opts.embedder ?? defaultEmbedder();
44
52
  this.stateDir = opts.stateDir ?? getStateDir();
53
+ this.repoId = opts.repoId ?? this.stateDir;
45
54
  // Sprint 14: all tier flags/thresholds flow from the single config source
46
55
  // (DedupConfig). The legacy opts.dedupSim / opts.l2Enabled remain accepted
47
56
  // for backward-compat callers but flags are authoritative via `cfg`.
@@ -283,6 +292,12 @@ export class VectorStore {
283
292
  // Cumulative store-wide dedup accounting (attempt, not collapsed).
284
293
  bumpDedupStats(false, this.stateDir);
285
294
  onTier?.({ tier: "new", status: "stored" });
295
+ // Slice 2: best-effort, fire-and-forget mirror of this new checkpoint into
296
+ // the async global PGlite/HNSW index. NEVER awaited — must not block or
297
+ // throw into the synchronous add() path. On failure the index degrades to
298
+ // the sync scan (handled inside vectorIndex). The node:sqlite store remains
299
+ // authoritative; the index is rebuildable from it at any time.
300
+ void indexUpsertEmbedding(this.repoId, sessionId, checkpointId, checkpoint.embedding);
286
301
  return { checkpoint, deduped: false };
287
302
  }
288
303
  /**
@@ -370,6 +385,56 @@ export class VectorStore {
370
385
  const ranked = mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
371
386
  return ranked;
372
387
  }
388
+ /**
389
+ * Slice 2: async cross-repo (or single-repo) recall via the PGlite/HNSW index.
390
+ *
391
+ * This is the ONLY async recall surface and is a BONUS path — the synchronous
392
+ * `search()` above remains the default. `opts.repoId` scopes to one repo; omit
393
+ * it for cross-repo nearest-neighbor recall (the headline capability the sync
394
+ * per-session scan cannot provide).
395
+ *
396
+ * Best-effort: if the index is disabled/empty/failing, we fall back to the
397
+ * synchronous per-session `search()` for THIS repo so callers always get a
398
+ * sensible result. Hydrates each hit's StoredCheckpoint from the authoritative
399
+ * node:sqlite store (the hit's repoId doubles as that repo's stateDir), then
400
+ * MMR-dedupes the merged set.
401
+ */
402
+ async searchAsync(sessionId, query, k = 3, opts = {}) {
403
+ const sid = normalizeSessionId(sessionId);
404
+ const qv = this.embedder.embed(query);
405
+ // repoId filter: explicit opts.repoId wins; else this repo unless crossRepo.
406
+ const repoId = opts.repoId ?? (opts.crossRepo ? undefined : this.repoId);
407
+ let indexHits = [];
408
+ try {
409
+ await initVectorIndex();
410
+ indexHits = await vectorIndexSearch(qv, { k: Math.max(k * 2, k), repoId });
411
+ }
412
+ catch {
413
+ indexHits = [];
414
+ }
415
+ if (indexHits.length === 0) {
416
+ // Index empty/unavailable → synchronous per-session fallback (this repo).
417
+ return this.search(sid, query, k);
418
+ }
419
+ // Hydrate each index hit from the authoritative node:sqlite store. repoId is
420
+ // that repo's stateDir, so cross-repo hits resolve against their own store.
421
+ const hydrated = [];
422
+ for (const h of indexHits) {
423
+ const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
424
+ if (cp && cp.dedupStatus !== "removed") {
425
+ hydrated.push({ checkpoint: cp, score: h.score });
426
+ }
427
+ }
428
+ if (hydrated.length === 0)
429
+ return this.search(sid, query, k);
430
+ // MMR-dedupe the merged candidate set for diversity (mirrors sync search).
431
+ const mmrItems = hydrated.map((h) => ({
432
+ item: h,
433
+ vector: h.checkpoint.embedding,
434
+ relevance: h.score,
435
+ }));
436
+ return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
437
+ }
373
438
  /**
374
439
  * Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
375
440
  * return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
@@ -0,0 +1,129 @@
1
+ /**
2
+ * wordpiece.ts — a self-contained WordPiece tokenizer for BERT/MiniLM.
3
+ *
4
+ * Loads the canonical `vocab.txt` (bert-base-uncased, ~30K tokens) from disk and
5
+ * implements the standard uncased BERT preprocessing + greedy longest-match
6
+ * WordPiece segmentation. No native dependency, no network — the vocab file is a
7
+ * local artifact fetched once by scripts/setup-minilm.mjs (PREVENT-PI-004).
8
+ *
9
+ * This mirrors HuggingFace `BertTokenizer` closely enough for sentence-embedding
10
+ * use: lowercase, strip accents, split on whitespace + punctuation, then
11
+ * WordPiece each token with the `##` continuation convention. Special tokens
12
+ * [CLS]/[SEP] are added by the caller's encode().
13
+ */
14
+ import { readFileSync, existsSync } from "node:fs";
15
+ const UNK = "[UNK]";
16
+ const CLS = "[CLS]";
17
+ const SEP = "[SEP]";
18
+ const PAD = "[PAD]";
19
+ const MAX_INPUT_CHARS_PER_WORD = 200;
20
+ export class WordPieceTokenizer {
21
+ vocab;
22
+ clsId;
23
+ sepId;
24
+ padId;
25
+ unkId;
26
+ constructor(vocab) {
27
+ this.vocab = vocab;
28
+ this.clsId = vocab.get(CLS) ?? 101;
29
+ this.sepId = vocab.get(SEP) ?? 102;
30
+ this.padId = vocab.get(PAD) ?? 0;
31
+ this.unkId = vocab.get(UNK) ?? 100;
32
+ }
33
+ /** Build a tokenizer from a vocab.txt file (one token per line, index = line). */
34
+ static fromVocabFile(path) {
35
+ if (!existsSync(path)) {
36
+ throw new Error(`WordPiece vocab not found at ${path}. Run: node scripts/setup-minilm.mjs`);
37
+ }
38
+ const lines = readFileSync(path, "utf-8").split("\n");
39
+ const vocab = new Map();
40
+ for (let i = 0; i < lines.length; i++) {
41
+ const tok = lines[i].replace(/\r$/, "");
42
+ if (tok.length > 0 || i < lines.length - 1)
43
+ vocab.set(tok, i);
44
+ }
45
+ return new WordPieceTokenizer(vocab);
46
+ }
47
+ /** Uncased BERT basic tokenization: lowercase, strip accents, split on ws+punct. */
48
+ basicTokenize(text) {
49
+ // NFD + strip combining marks (accent removal), then lowercase.
50
+ const cleaned = text
51
+ .normalize("NFD")
52
+ .replace(/[̀-ͯ]/g, "")
53
+ .toLowerCase();
54
+ const tokens = [];
55
+ let buf = "";
56
+ const flush = () => {
57
+ if (buf.length > 0) {
58
+ tokens.push(buf);
59
+ buf = "";
60
+ }
61
+ };
62
+ for (const ch of cleaned) {
63
+ if (/\s/.test(ch)) {
64
+ flush();
65
+ }
66
+ else if (/[!-/:-@[-`{-~¡-¿]/.test(ch)) {
67
+ // Punctuation becomes its own token.
68
+ flush();
69
+ tokens.push(ch);
70
+ }
71
+ else {
72
+ buf += ch;
73
+ }
74
+ }
75
+ flush();
76
+ return tokens;
77
+ }
78
+ /** Greedy longest-match WordPiece for a single word. */
79
+ wordpiece(word) {
80
+ if (word.length > MAX_INPUT_CHARS_PER_WORD)
81
+ return [UNK];
82
+ const pieces = [];
83
+ let start = 0;
84
+ while (start < word.length) {
85
+ let end = word.length;
86
+ let cur = null;
87
+ while (start < end) {
88
+ let sub = word.slice(start, end);
89
+ if (start > 0)
90
+ sub = "##" + sub;
91
+ if (this.vocab.has(sub)) {
92
+ cur = sub;
93
+ break;
94
+ }
95
+ end--;
96
+ }
97
+ if (cur === null)
98
+ return [UNK]; // any unmatchable piece → whole word is UNK
99
+ pieces.push(cur);
100
+ start = end;
101
+ }
102
+ return pieces;
103
+ }
104
+ /** Tokenize text into WordPiece token strings (no special tokens). */
105
+ tokenize(text) {
106
+ const out = [];
107
+ for (const word of this.basicTokenize(text)) {
108
+ for (const piece of this.wordpiece(word))
109
+ out.push(piece);
110
+ }
111
+ return out;
112
+ }
113
+ /**
114
+ * Encode text into model inputs with [CLS]…[SEP], truncated to `maxLen`.
115
+ * attention_mask is all 1s (no padding for single-sequence inference).
116
+ */
117
+ encode(text, maxLen = 256) {
118
+ const pieces = this.tokenize(text).slice(0, Math.max(0, maxLen - 2));
119
+ const inputIds = [this.clsId];
120
+ for (const p of pieces)
121
+ inputIds.push(this.vocab.get(p) ?? this.unkId);
122
+ inputIds.push(this.sepId);
123
+ return {
124
+ inputIds,
125
+ attentionMask: inputIds.map(() => 1),
126
+ tokenTypeIds: inputIds.map(() => 0),
127
+ };
128
+ }
129
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.4.24",
3
+ "version": "0.4.25",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-2-Clause",
@@ -57,6 +57,8 @@
57
57
  "typescript": "^5.4.0"
58
58
  },
59
59
  "dependencies": {
60
+ "@electric-sql/pglite": "^0.5.4",
61
+ "@electric-sql/pglite-pgvector": "^0.0.5",
60
62
  "@mongodb-js/zstd": "^7.0.0"
61
63
  }
62
64
  }
package/src/recall.ts CHANGED
@@ -137,3 +137,66 @@ export function recallAndInline(
137
137
  empty: toInject.length === 0,
138
138
  };
139
139
  }
140
+
141
+ /**
142
+ * Slice 2 async cross-repo recall. Same dedup/bound/inline contract as
143
+ * `recallAndInline`, but backed by `VectorStore.searchAsync` so it can recall
144
+ * across repos (HNSW NN over the global PGlite index) when `opts.crossRepo` is
145
+ * set. The synchronous `recallAndInline` is unchanged and remains the default
146
+ * per-session path. Inline-window dedupe + token cap (Fix C) apply here too.
147
+ *
148
+ * `store` must provide `searchAsync` (the live VectorStore does). Errors fall
149
+ * back to an empty result — recall is a bonus, never a hard dependency.
150
+ */
151
+ export async function recallAndInlineAsync(
152
+ opts: RecallInjectOptions & { crossRepo?: boolean; repoId?: string },
153
+ store: Pick<VectorStore, "searchAsync" | "wasInjected" | "markInjected">,
154
+ ): Promise<RecallInjectResult> {
155
+ const limit = opts.limit ?? 3;
156
+ const skip = opts.skipInjected ?? true;
157
+ const maxTokens = opts.recallMaxTokens ?? 0;
158
+ const doWindowDedupe = opts.windowDedupe ?? false;
159
+ const dedupSim = opts.dedupSim ?? 0.9;
160
+
161
+ let hits: SearchHit[] = [];
162
+ try {
163
+ hits = await store.searchAsync(opts.sessionId, opts.query, limit, {
164
+ crossRepo: opts.crossRepo,
165
+ repoId: opts.repoId,
166
+ });
167
+ } catch {
168
+ hits = [];
169
+ }
170
+
171
+ let liveEmbeddings: number[][] = [];
172
+ if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
173
+ const embedder = defaultEmbedder();
174
+ liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
175
+ }
176
+
177
+ const toInject: SearchHit[] = [];
178
+ const parts: string[] = [];
179
+ let blockTokens = 0;
180
+
181
+ for (const h of hits) {
182
+ if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
183
+ if (doWindowDedupe && liveEmbeddings.length > 0) {
184
+ const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
185
+ if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim)) continue;
186
+ }
187
+ const part = formatRecallBlock([h]);
188
+ const partTokens = estimateBlockTokens(part);
189
+ if (maxTokens > 0 && blockTokens + partTokens > maxTokens) break;
190
+ parts.push(part);
191
+ toInject.push(h);
192
+ blockTokens += partTokens;
193
+ store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
194
+ }
195
+
196
+ const block = parts.join("\n");
197
+ const report = toInject.map(
198
+ (h) => ` • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`,
199
+ );
200
+
201
+ return { toInject, report, block, empty: toInject.length === 0 };
202
+ }
@@ -872,6 +872,19 @@ export function hasCheckpoint(sessionId: string, checkpointId: string, stateDir:
872
872
  return row !== undefined;
873
873
  }
874
874
 
875
+ /** Fetch a single checkpoint by (session, id), or undefined if absent. */
876
+ export function getCheckpoint(
877
+ sessionId: string,
878
+ checkpointId: string,
879
+ stateDir: string = getStateDir(),
880
+ ): StoredCheckpoint | undefined {
881
+ const db = openStore(stateDir);
882
+ const row = db
883
+ .prepare("SELECT * FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
884
+ .get(normalizeSessionId(sessionId), checkpointId) as any;
885
+ return row ? rowToCheckpoint(row) : undefined;
886
+ }
887
+
875
888
  /** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
876
889
  export function setDedupStatus(
877
890
  checkpointId: string,
@@ -0,0 +1,116 @@
1
+ /**
2
+ * vectorIndex.test.ts — Slice 2 async PGlite/HNSW vector index.
3
+ *
4
+ * Proves: cross-repo nearest-neighbor recall, repoId scoping, the dimension
5
+ * guard (non-512 vectors skipped, never corrupt the index), and graceful
6
+ * degradation when the index is disabled (kill-switch) — all without touching
7
+ * the synchronous node:sqlite store.
8
+ *
9
+ * The index is a WASM Postgres (PGlite) — fully local, zero network
10
+ * (PREVENT-PI-004). Each test isolates state via MEGACOMPACT_VECTOR_INDEX_DIR.
11
+ */
12
+
13
+ import { test } from "node:test";
14
+ import assert from "node:assert/strict";
15
+ import { mkdtempSync, rmSync } from "node:fs";
16
+ import { tmpdir } from "node:os";
17
+ import { join } from "node:path";
18
+ import {
19
+ EMBEDDING_DIM,
20
+ initVectorIndex,
21
+ upsertEmbedding,
22
+ searchAsync,
23
+ closeVectorIndex,
24
+ isVectorIndexDisabled,
25
+ } from "./vectorIndex.js";
26
+
27
+ /** A 512-dim unit-ish vector with a single spike at `idx` (deterministic NN). */
28
+ function spikeVec(idx: number, magnitude = 1): number[] {
29
+ const v = new Array<number>(EMBEDDING_DIM).fill(0);
30
+ v[idx % EMBEDDING_DIM] = magnitude;
31
+ return v;
32
+ }
33
+
34
+ function isolateIndexDir(): string {
35
+ const dir = mkdtempSync(join(tmpdir(), "mc-vecidx-"));
36
+ process.env.MEGACOMPACT_VECTOR_INDEX_DIR = dir;
37
+ return dir;
38
+ }
39
+
40
+ test("cross-repo HNSW nearest-neighbor recall across repos + repoId scoping", async () => {
41
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
42
+ const dir = isolateIndexDir();
43
+ try {
44
+ await closeVectorIndex(); // ensure a fresh singleton for this dir
45
+ const pg = await initVectorIndex();
46
+ assert.ok(pg, "index should initialize (PGlite WASM available)");
47
+
48
+ // repoA: two checkpoints; repoB: one checkpoint. Distinct spike directions.
49
+ await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_001", spikeVec(0));
50
+ await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_002", spikeVec(5));
51
+ await upsertEmbedding("/repoB/.pi/mega-compact", "sessB", "chkpt_001", spikeVec(0));
52
+
53
+ // Cross-repo query near spike(0): nearest are the two spike(0) rows, one per repo.
54
+ const cross = await searchAsync(spikeVec(0), { k: 2 });
55
+ assert.equal(cross.length, 2, "cross-repo returns two nearest");
56
+ const repos = new Set(cross.map((h) => h.repoId));
57
+ assert.ok(repos.has("/repoA/.pi/mega-compact"), "hit from repoA");
58
+ assert.ok(repos.has("/repoB/.pi/mega-compact"), "hit from repoB");
59
+ assert.ok(cross[0].score > 0.99, "top hit is near-identical (cosine ~1)");
60
+
61
+ // Scoped to repoA only: excludes repoB even though repoB has an identical vec.
62
+ const scoped = await searchAsync(spikeVec(0), { k: 5, repoId: "/repoA/.pi/mega-compact" });
63
+ assert.ok(scoped.length >= 1, "scoped returns repoA hits");
64
+ assert.ok(
65
+ scoped.every((h) => h.repoId === "/repoA/.pi/mega-compact"),
66
+ "repoId filter excludes other repos",
67
+ );
68
+ } finally {
69
+ await closeVectorIndex();
70
+ rmSync(dir, { recursive: true, force: true });
71
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
72
+ }
73
+ });
74
+
75
+ test("dimension guard: non-512 vectors are skipped, never corrupt the index", async () => {
76
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
77
+ const dir = isolateIndexDir();
78
+ try {
79
+ await closeVectorIndex();
80
+ await initVectorIndex();
81
+ // Wrong-dimension vector (BYO embedder mismatch) must be silently skipped.
82
+ await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_001", [1, 2, 3]);
83
+ const hits = await searchAsync(spikeVec(0), { k: 5 });
84
+ assert.equal(hits.length, 0, "no rows stored for a mismatched-dim vector");
85
+
86
+ // A correct-dim vector still stores fine afterward (index not corrupted).
87
+ await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_002", spikeVec(3));
88
+ const ok = await searchAsync(spikeVec(3), { k: 1 });
89
+ assert.equal(ok.length, 1, "valid vector stored after a skipped one");
90
+ assert.equal(ok[0].checkpointId, "chkpt_002");
91
+ } finally {
92
+ await closeVectorIndex();
93
+ rmSync(dir, { recursive: true, force: true });
94
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
95
+ }
96
+ });
97
+
98
+ test("kill-switch: MEGACOMPACT_PGLITE_DISABLED disables the index gracefully", async () => {
99
+ const dir = isolateIndexDir();
100
+ process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
101
+ try {
102
+ await closeVectorIndex();
103
+ assert.equal(isVectorIndexDisabled(), true, "kill-switch reported disabled");
104
+ const pg = await initVectorIndex();
105
+ assert.equal(pg, undefined, "init returns undefined when disabled");
106
+ // Upsert + search are no-ops that never throw and return empty.
107
+ await upsertEmbedding("/repoD/.pi/mega-compact", "sessD", "chkpt_001", spikeVec(0));
108
+ const hits = await searchAsync(spikeVec(0), { k: 3 });
109
+ assert.deepEqual(hits, [], "search returns [] when disabled");
110
+ } finally {
111
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
112
+ await closeVectorIndex();
113
+ rmSync(dir, { recursive: true, force: true });
114
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
115
+ }
116
+ });