pi-mega-compact 0.6.1 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,8 +22,39 @@ export function computeLiveTrimCut(view, opts) {
22
22
  return null; // nothing safe to cut — keep everything this call
23
23
  const recent = view.slice(cut);
24
24
  const userCount = recent.filter((m) => m.role === "user").length;
25
- if (userCount < opts.anchorUserMessages)
26
- return null;
25
+ // ANCHOR FLOOR (PREVENT-PI-001): the recent window must keep at least
26
+ // `anchorUserMessages` user messages. The original compactedFrom can land on a
27
+ // run that starts with fewer than that (e.g. the preserved region begins on a
28
+ // tool pair, or the session's tail is tool-heavy). Instead of bailing out and
29
+ // skipping the live trim entirely this call (which left the model fed a
30
+ // 150k-context window during long team runs), walk `cut` backward until the
31
+ // preserved run contains enough user messages — bounded by the boundary-safe
32
+ // constraint so we never split a tool pair. Falls back to null only when the
33
+ // whole view can't satisfy the floor (tiny sessions) — the next context event
34
+ // retries.
35
+ if (userCount < opts.anchorUserMessages) {
36
+ let c = cut;
37
+ while (c > 1) {
38
+ c--;
39
+ if (!isBoundarySafe(view, c))
40
+ continue;
41
+ const recentNow = view.slice(c);
42
+ const usersNow = recentNow.filter((m) => m.role === "user").length;
43
+ if (usersNow >= opts.anchorUserMessages) {
44
+ cut = c;
45
+ break;
46
+ }
47
+ }
48
+ if (cut > 1) {
49
+ const finalRecent = view.slice(cut);
50
+ if (finalRecent.filter((m) => m.role === "user").length < opts.anchorUserMessages) {
51
+ return null; // cannot satisfy the floor without dropping too much — retry next call
52
+ }
53
+ }
54
+ else {
55
+ return null;
56
+ }
57
+ }
27
58
  return cut;
28
59
  }
29
60
  /** The formatted compacted-region summary as a user-role engine message. */
@@ -1,9 +1,44 @@
1
1
  import { addMemory, listMemories, replaceMemory, removeMemory, } from "./store/sqlite.js";
2
+ import { defaultEmbedder } from "./embedder.js";
3
+ import { upsertMemoryEmbedding } from "./store/memoryIndex.js";
4
+ import { execSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: read-only `git rev-parse` to scope the memory index per-repo
5
+ /** Resolve the current repo's git root (mirrors extensions/mega-config.ts but
6
+ * kept local so src/ stays pi-agnostic — no extension-layer import). */
7
+ function resolveRepoRootLocal(cwd) {
8
+ try {
9
+ const out = execSync("git rev-parse --show-toplevel", {
10
+ cwd,
11
+ encoding: "utf-8",
12
+ stdio: ["ignore", "pipe", "ignore"],
13
+ }).trim();
14
+ return out || undefined;
15
+ }
16
+ catch {
17
+ return undefined;
18
+ }
19
+ }
2
20
  /** Find a memory row whose content exactly matches (case-insensitive). */
3
21
  function findByContent(memories, content) {
4
22
  const norm = content.trim().toLowerCase();
5
23
  return memories.find((m) => m.content.trim().toLowerCase() === norm);
6
24
  }
25
+ /**
26
+ * Fire-and-forget mirror of a memory write into the cross-repo PGlite index
27
+ * (S24 optional memory-RAG mirror). Best-effort + non-fatal: never blocks the
28
+ * SQLite write and degrades to the same-repo scan if the index is disabled or
29
+ * fails. `repoId` is the resolved git root so the memory is findable from other
30
+ * repos; falls back to the state dir when outside git.
31
+ */
32
+ function indexMemoryWrite(stateDir, memoryId, content) {
33
+ const repoId = resolveRepoRootLocal(stateDir) ?? stateDir;
34
+ try {
35
+ const vec = defaultEmbedder().embed(content);
36
+ void upsertMemoryEmbedding(repoId, memoryId, content, vec);
37
+ }
38
+ catch {
39
+ /* non-fatal — embedding/index failure must never break the SQLite write */
40
+ }
41
+ }
7
42
  /**
8
43
  * Apply add/replace/remove ops to the memories table. Replaces are matched by
9
44
  * existing content; removes by content. Idempotent: an add that already exists
@@ -19,7 +54,7 @@ export async function applyMemoryOps(ops, stateDir) {
19
54
  // Skip if an identical memory already exists.
20
55
  if (findByContent(existing, op.memory.content))
21
56
  continue;
22
- addMemory({
57
+ const id = addMemory({
23
58
  kind: op.memory.category,
24
59
  content: op.memory.content,
25
60
  tags: [],
@@ -27,6 +62,8 @@ export async function applyMemoryOps(ops, stateDir) {
27
62
  target: op.memory.target,
28
63
  sourceTurn: op.memory.sourceTurn,
29
64
  }, repo, stateDir);
65
+ // S24: mirror into the cross-repo index (fire-and-forget; non-fatal).
66
+ indexMemoryWrite(stateDir, id, op.memory.content);
30
67
  }
31
68
  else if (op.op === "replace") {
32
69
  const match = findByContent(existing, op.targetContent);
@@ -37,16 +74,19 @@ export async function applyMemoryOps(ops, stateDir) {
37
74
  category: op.memory.category,
38
75
  sourceTurn: op.memory.sourceTurn,
39
76
  }, stateDir);
77
+ // S24: re-mirror under the same memory id (fire-and-forget; non-fatal).
78
+ indexMemoryWrite(stateDir, match.id, op.memory.content);
40
79
  }
41
80
  else {
42
81
  // Target missing (e.g. earlier in-conversation contradiction) → add.
43
- addMemory({
82
+ const id = addMemory({
44
83
  kind: op.memory.category,
45
84
  content: op.memory.content,
46
85
  tags: [],
47
86
  category: op.memory.category,
48
87
  sourceTurn: op.memory.sourceTurn,
49
88
  }, repo, stateDir);
89
+ indexMemoryWrite(stateDir, id, op.memory.content);
50
90
  }
51
91
  }
52
92
  else {
@@ -58,3 +58,51 @@ export async function recallMemories(query, stateDir, opts = {}) {
58
58
  }
59
59
  return top;
60
60
  }
61
+ /**
62
+ * Cross-repo memory recall (S24): augments the same-repo `recallMemories` with
63
+ * HNSW NN over the global PGlite `memory_index` (other repos' memories). Content
64
+ * is read inline from the index hit (the recall process can't open other repos'
65
+ * SQLite dirs), so no other-repo db access is required. Returns hits sorted by
66
+ * descending cosine, above `crossRepoCosine`. De-duped by content against
67
+ * `sameRepoContent` so we never surface a memory the same-repo scan already has.
68
+ * Non-fatal: any index failure returns []. Best-effort + PREVENT-PI-004 (local
69
+ * WASM only).
70
+ */
71
+ export async function recallMemoriesCrossRepo(query, stateDir, opts = {}) {
72
+ const embedder = opts.embedder ?? defaultEmbedder();
73
+ const queryVec = embedder.embed(query);
74
+ const { searchMemoriesAsync } = await import("./store/memoryIndex.js");
75
+ const k = opts.limit ?? 5;
76
+ const floor = opts.crossRepoCosine ?? 0.3;
77
+ const hits = await searchMemoriesAsync(queryVec, { k });
78
+ if (!hits.length)
79
+ return [];
80
+ // Mark same-repo content as already-covered so we don't duplicate it.
81
+ const sameRepo = new Set(listMemories(opts.repo ?? null, 1000, stateDir).map((m) => m.content.trim().toLowerCase()));
82
+ const out = [];
83
+ for (const h of hits) {
84
+ if (h.score < floor)
85
+ continue;
86
+ if (sameRepo.has(h.content.trim().toLowerCase()))
87
+ continue;
88
+ out.push({
89
+ memory: {
90
+ id: h.memoryId,
91
+ repo: h.repoId,
92
+ kind: "note",
93
+ content: h.content,
94
+ tags: [],
95
+ createdAt: 0,
96
+ lastRecalledAt: null,
97
+ category: null,
98
+ target: null,
99
+ lastReferenced: null,
100
+ sourceTurn: null,
101
+ },
102
+ score: h.score,
103
+ repoId: h.repoId,
104
+ });
105
+ }
106
+ out.sort((a, b) => b.score - a.score);
107
+ return out;
108
+ }
@@ -90,3 +90,55 @@ test("recallMemories: fresher reference beats older at equal similarity", async
90
90
  test("cleanup memrec", () => {
91
91
  rmSync(baseTmp, { recursive: true, force: true });
92
92
  });
93
+ // ---- S24: cross-repo memory recall (PGlite mirror) ---------------------------
94
+ test("recallMemoriesAndInline: surfaces a memory saved in ANOTHER repo via cross-repo index", async () => {
95
+ // Isolate the global PGlite index to a temp dir shared by both "repos".
96
+ process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index");
97
+ const repoA = join(baseTmp, "repo-a");
98
+ const repoB = join(baseTmp, "repo-b");
99
+ try {
100
+ // repoA owns a decision about the store backend.
101
+ const { applyMemoryOps } = await import("./memoryOps.js");
102
+ await applyMemoryOps([{ op: "add", memory: { content: "we standardized on node:sqlite for the store backend", category: "decision", sourceTurn: 0 } }], repoA);
103
+ // repoB is a fresh session with NO local memory about the store backend.
104
+ const { recallMemoriesAndInline } = await import("./recall.js");
105
+ const res = await recallMemoriesAndInline({
106
+ query: "what store backend do we use?",
107
+ stateDir: repoB,
108
+ limit: 5,
109
+ crossRepo: true,
110
+ crossRepoCosine: 0.3,
111
+ });
112
+ assert.ok(!res.empty, "cross-repo recall found the other repo's memory");
113
+ assert.ok(/node:sqlite/.test(res.block), "the node:sqlite decision was recalled from repo A");
114
+ assert.ok(res.report.some((r) => /from /.test(r)), "report labels the memory as cross-repo");
115
+ }
116
+ finally {
117
+ const { closeMemoryIndex } = await import("./store/memoryIndex.js");
118
+ await closeMemoryIndex();
119
+ delete process.env.MEGACOMPACT_INDEX_DIR;
120
+ }
121
+ });
122
+ test("recallMemoriesAndInline: cross-repo disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
123
+ process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index-off");
124
+ process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
125
+ const repoA = join(baseTmp, "repo-a2");
126
+ const repoB = join(baseTmp, "repo-b2");
127
+ try {
128
+ const { applyMemoryOps } = await import("./memoryOps.js");
129
+ await applyMemoryOps([{ op: "add", memory: { content: "we standardized on node:sqlite for the store backend", category: "decision", sourceTurn: 0 } }], repoA);
130
+ const { recallMemoriesAndInline } = await import("./recall.js");
131
+ const res = await recallMemoriesAndInline({
132
+ query: "what store backend do we use?",
133
+ stateDir: repoB,
134
+ limit: 5,
135
+ crossRepo: true,
136
+ });
137
+ // Index disabled → no cross-repo hit; repoB has no local memory → empty.
138
+ assert.equal(res.empty, true, "cross-repo recall degrades to empty when disabled");
139
+ }
140
+ finally {
141
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
142
+ delete process.env.MEGACOMPACT_INDEX_DIR;
143
+ }
144
+ });
@@ -105,27 +105,53 @@ export function formatMemoryRecallBlock(hits) {
105
105
  export async function recallMemoriesAndInline(opts) {
106
106
  const limit = opts.limit ?? 5;
107
107
  const maxTokens = opts.recallMaxTokens ?? 0;
108
- const { recallMemories } = await import("./memoryRecall.js");
108
+ const { recallMemories, recallMemoriesCrossRepo } = await import("./memoryRecall.js");
109
109
  const hits = await recallMemories(opts.query, opts.stateDir, {
110
110
  topK: limit,
111
111
  minSimilarity: opts.minSimilarity ?? 0.2,
112
112
  });
113
- if (hits.length === 0)
113
+ // S24 cross-repo augmentation: if same-repo recall is thin, pull additional
114
+ // memories from OTHER repos via the PGlite HNSW index. Non-fatal: a failure
115
+ // degrades to the same-repo hits only.
116
+ const crossHits = [];
117
+ if (opts.crossRepo && hits.length < limit) {
118
+ try {
119
+ const x = await recallMemoriesCrossRepo(opts.query, opts.stateDir, {
120
+ repo: null,
121
+ limit: limit - hits.length,
122
+ crossRepoCosine: opts.crossRepoCosine ?? 0.3,
123
+ });
124
+ for (const h of x)
125
+ crossHits.push(h);
126
+ }
127
+ catch {
128
+ /* non-fatal — cross-repo failure → same-repo only */
129
+ }
130
+ }
131
+ if (hits.length === 0 && crossHits.length === 0)
114
132
  return { empty: true, block: "", report: [] };
115
133
  // Same incremental token cap pattern as checkpoint recall.
116
134
  const parts = [];
117
135
  const report = [];
118
136
  let blockTokens = 0;
119
- for (const h of hits) {
120
- const part = formatMemoryRecallBlock([
121
- { content: h.memory.content, category: h.memory.category, score: h.score },
122
- ]);
137
+ const pushHit = (content, category, score, label) => {
138
+ const part = formatMemoryRecallBlock([{ content, category, score }]);
123
139
  const partTokens = estimateBlockTokens(part);
124
140
  if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
125
- break;
141
+ return false;
126
142
  parts.push(part);
127
- report.push(` • memory#${h.memory.id} (${(h.score * 100).toFixed(0)}%): ${h.memory.content.slice(0, 60).replace(/\n/g, " ")}…`);
143
+ report.push(` • ${label} (${(score * 100).toFixed(0)}%): ${content.slice(0, 60).replace(/\n/g, " ")}…`);
128
144
  blockTokens += partTokens;
145
+ return true;
146
+ };
147
+ for (const h of hits) {
148
+ if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id}`))
149
+ break;
150
+ }
151
+ for (const h of crossHits) {
152
+ const repoLabel = h.repoId.split(/[\\/]/).filter(Boolean).pop() ?? h.repoId;
153
+ if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id} (from ${repoLabel})`))
154
+ break;
129
155
  }
130
156
  return { empty: parts.length === 0, block: parts.join("\n"), report };
131
157
  }
@@ -0,0 +1,230 @@
1
+ /**
2
+ * memoryIndex.ts — cross-repo async vector index for durable memories (S24).
3
+ *
4
+ * A REDUNDANT, additive, ASYNC index layered over the authoritative node:sqlite
5
+ * `memories` table. The same-repo linear cosine scan over the in-repo memories
6
+ * (src/memoryRecall.ts) stays the DEFAULT recall path; this global PGlite index
7
+ * exists only to provide real cross-repo nearest-neighbor memory recall — so a
8
+ * decision you saved in repo A can be inlined as RAG context when you start a
9
+ * session in repo B. It is best-effort and non-fatal: any init/write failure
10
+ * degrades to the same-repo scan and must NEVER break memory write, recall, or
11
+ * extension load.
12
+ *
13
+ * PREVENT-PI-004: PGlite is WASM Postgres — fully local, zero network. Memory
14
+ * remains AUTHORITATIVE in SQLite; this index only holds (repo_id, memory_id,
15
+ * content, embedding) for NN lookup and is rebuilt from SQLite at any time.
16
+ *
17
+ * Topology mirrors vectorIndex.ts (Slice 2): ONE global PGlite DB, `repo_id` is
18
+ * a first-class column. `searchMemoriesAsync(q, k, {repoId?})` → omit repoId for
19
+ * cross-repo NN, pass repoId to scope to a single repo. Hit content is stored
20
+ * inline because the recall process cannot open every other repo's SQLite dir.
21
+ */
22
+ import { homedir } from "node:os";
23
+ import { join } from "node:path";
24
+ import { mkdirSync, rmSync, existsSync } from "node:fs";
25
+ /** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
26
+ export const MEMORY_INDEX_DIM = 512;
27
+ let db;
28
+ let initPromise;
29
+ let disabled = false;
30
+ let warned = false;
31
+ /** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
32
+ let pgliteMod;
33
+ let pgliteLoadFailed = false;
34
+ function indexDir() {
35
+ const override = process.env.MEGACOMPACT_INDEX_DIR;
36
+ if (override && override.trim() !== "")
37
+ return join(override, "memory");
38
+ try {
39
+ return join(homedir(), ".pi", "mega-compact-vector", "memory");
40
+ }
41
+ catch {
42
+ return join("/tmp", ".mega-compact-vector", "memory");
43
+ }
44
+ }
45
+ function logWarn(msg) {
46
+ // Never throw — degradation is the whole point. One warning per process.
47
+ if (warned)
48
+ return;
49
+ warned = true;
50
+ try {
51
+ console.warn(`[mega-compact:memoryIndex] ${msg} (falling back to same-repo scan)`);
52
+ }
53
+ catch {
54
+ /* ignore */
55
+ }
56
+ }
57
+ /** Honor the emergency kill-switch (shared with the checkpoint index). */
58
+ export function isMemoryIndexDisabled() {
59
+ return (disabled ||
60
+ process.env.MEGACOMPACT_PGLITE_DISABLED === "true" ||
61
+ process.env.MEGACOMPACT_PGLITE_DISABLED === "1");
62
+ }
63
+ /**
64
+ * Lazily open + schema-init the global PGlite DB. Idempotent and safe to call
65
+ * from many places. Returns undefined when disabled/unavailable so callers can
66
+ * fall back to the synchronous scan. Never throws.
67
+ */
68
+ export function initMemoryIndex() {
69
+ if (isMemoryIndexDisabled())
70
+ return Promise.resolve(undefined);
71
+ if (db)
72
+ return Promise.resolve(db);
73
+ if (initPromise)
74
+ return initPromise;
75
+ initPromise = openPgLite(/* retryOnCorrupt */ true);
76
+ return initPromise;
77
+ }
78
+ /**
79
+ * Lazily load the PGlite module + pgvector extension via dynamic import. Caches
80
+ * success and permanent failure. Returns undefined (once, then forever) when the
81
+ * package is missing/broken so callers fall back to the same-repo scan. Never throws.
82
+ */
83
+ async function loadPgLite() {
84
+ if (pgliteMod)
85
+ return pgliteMod;
86
+ if (pgliteLoadFailed)
87
+ return undefined;
88
+ try {
89
+ const [pglitePkg, pgvectorPkg] = await Promise.all([
90
+ import("@electric-sql/pglite"),
91
+ import("@electric-sql/pglite-pgvector"),
92
+ ]);
93
+ pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
94
+ return pgliteMod;
95
+ }
96
+ catch (err) {
97
+ pgliteLoadFailed = true;
98
+ logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
99
+ return undefined;
100
+ }
101
+ }
102
+ /**
103
+ * Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level abort
104
+ * (typically from a corrupted/torn data dir) triggers a delete + one retry.
105
+ */
106
+ async function openPgLite(retryOnCorrupt) {
107
+ try {
108
+ const mod = await loadPgLite();
109
+ if (!mod)
110
+ return undefined;
111
+ const dir = indexDir();
112
+ mkdirSync(dir, { recursive: true });
113
+ const pg = await new mod.PGlite({
114
+ dataDir: dir,
115
+ extensions: { vector: mod.vector },
116
+ });
117
+ await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
118
+ await pg.exec(`
119
+ CREATE TABLE IF NOT EXISTS memory_index (
120
+ repo_id TEXT NOT NULL,
121
+ memory_id INTEGER NOT NULL,
122
+ content TEXT NOT NULL,
123
+ embedding vector(${MEMORY_INDEX_DIM}) NOT NULL,
124
+ PRIMARY KEY (repo_id, memory_id)
125
+ );
126
+ `);
127
+ await pg.exec("CREATE INDEX IF NOT EXISTS memory_index_hnsw ON memory_index USING hnsw (embedding vector_cosine_ops);");
128
+ db = pg;
129
+ return pg;
130
+ }
131
+ catch (err) {
132
+ const msg = err instanceof Error ? err.message : String(err);
133
+ if (retryOnCorrupt && (msg.includes("Aborted") || msg.includes("RuntimeError"))) {
134
+ try {
135
+ const dir = indexDir();
136
+ if (existsSync(dir))
137
+ rmSync(dir, { recursive: true, force: true });
138
+ initPromise = undefined;
139
+ return openPgLite(/* retryOnCorrupt */ false);
140
+ }
141
+ catch {
142
+ /* self-heal failed — fall through to disable */
143
+ }
144
+ }
145
+ disabled = true;
146
+ logWarn(`init failed: ${msg}`);
147
+ return undefined;
148
+ }
149
+ }
150
+ function toVectorLiteral(v) {
151
+ const parts = v.map((x) => (Number.isFinite(x) ? x : 0));
152
+ return `[${parts.join(",")}]`;
153
+ }
154
+ /**
155
+ * Best-effort upsert of one memory embedding into the global index.
156
+ * Dimension-mismatched vectors (e.g. a BYO embedder with dim ≠ 512) are skipped.
157
+ * Fire-and-forget: callers must NOT await this on the sync write path. Never
158
+ * throws. `content` is stored inline so cross-repo recall can read it directly.
159
+ */
160
+ export async function upsertMemoryEmbedding(repoId, memoryId, content, embedding) {
161
+ if (isMemoryIndexDisabled())
162
+ return;
163
+ if (!embedding || embedding.length !== MEMORY_INDEX_DIM)
164
+ return;
165
+ try {
166
+ const pg = await initMemoryIndex();
167
+ if (!pg)
168
+ return;
169
+ const lit = toVectorLiteral(embedding);
170
+ await pg.query(`INSERT INTO memory_index (repo_id, memory_id, content, embedding)
171
+ VALUES ($1, $2, $3, $4::vector)
172
+ ON CONFLICT (repo_id, memory_id)
173
+ DO UPDATE SET content = EXCLUDED.content, embedding = EXCLUDED.embedding;`, [repoId, memoryId, content, lit]);
174
+ }
175
+ catch (err) {
176
+ disabled = true;
177
+ logWarn(`upsert failed: ${err instanceof Error ? err.message : String(err)}`);
178
+ }
179
+ }
180
+ /**
181
+ * Cross-repo (or single-repo) HNSW nearest-neighbor memory search. Returns hits
182
+ * sorted by descending similarity. Never throws — on any failure returns [].
183
+ */
184
+ export async function searchMemoriesAsync(query, opts = {}) {
185
+ if (isMemoryIndexDisabled() || !query || query.length !== MEMORY_INDEX_DIM)
186
+ return [];
187
+ const k = opts.k ?? 5;
188
+ const repoId = opts.repoId;
189
+ try {
190
+ const pg = await initMemoryIndex();
191
+ if (!pg)
192
+ return [];
193
+ const lit = toVectorLiteral(query);
194
+ const params = [lit, k];
195
+ let sql = "SELECT repo_id, memory_id, content, 1 - (embedding <=> $1::vector) AS score " +
196
+ "FROM memory_index";
197
+ if (repoId) {
198
+ sql += " WHERE repo_id = $3";
199
+ params.push(repoId);
200
+ }
201
+ sql += " ORDER BY embedding <=> $1::vector LIMIT $2";
202
+ const res = await pg.query(sql, params);
203
+ return res.rows.map((r) => ({
204
+ repoId: r.repo_id,
205
+ memoryId: Number(r.memory_id),
206
+ content: r.content,
207
+ score: r.score,
208
+ }));
209
+ }
210
+ catch (err) {
211
+ disabled = true;
212
+ logWarn(`search failed: ${err instanceof Error ? err.message : String(err)}`);
213
+ return [];
214
+ }
215
+ }
216
+ /** Close the index (test teardown / shutdown). Safe to call when unopened. */
217
+ export async function closeMemoryIndex() {
218
+ if (db) {
219
+ try {
220
+ await db.close();
221
+ }
222
+ catch {
223
+ /* ignore */
224
+ }
225
+ }
226
+ db = undefined;
227
+ initPromise = undefined;
228
+ disabled = false;
229
+ warned = false;
230
+ }
@@ -0,0 +1,51 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { mkdtempSync, rmSync } from "node:fs";
4
+ import { tmpdir } from "node:os";
5
+ import { join } from "node:path";
6
+ import { defaultEmbedder } from "../embedder.js";
7
+ import { upsertMemoryEmbedding, searchMemoriesAsync, initMemoryIndex, closeMemoryIndex, isMemoryIndexDisabled, } from "./memoryIndex.js";
8
+ const baseTmp = mkdtempSync(join(tmpdir(), "mc-memidx-"));
9
+ test("memoryIndex: disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
10
+ process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
11
+ try {
12
+ assert.equal(isMemoryIndexDisabled(), true, "kill-switch honored");
13
+ const hits = await searchMemoriesAsync(defaultEmbedder().embed("anything"), { k: 3 });
14
+ assert.deepEqual(hits, [], "search returns [] when disabled");
15
+ }
16
+ finally {
17
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
18
+ }
19
+ });
20
+ test("memoryIndex: cross-repo upsert + NN search returns the right repo's memory", async () => {
21
+ // Isolate the global PGlite dir so concurrent test runs don't collide.
22
+ process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
23
+ const repoA = "/tmp/repo-a";
24
+ const repoB = "/tmp/repo-b";
25
+ try {
26
+ await initMemoryIndex();
27
+ // Two memories in different repos, with clearly distinct content so their
28
+ // trigram embeddings separate.
29
+ const vecA = defaultEmbedder().embed("We standardized on node:sqlite for the store backend");
30
+ const vecB = defaultEmbedder().embed("The deployment target is a raspberry pi in the closet");
31
+ await upsertMemoryEmbedding(repoA, 1, "We standardized on node:sqlite for the store backend", vecA);
32
+ await upsertMemoryEmbedding(repoB, 7, "The deployment target is a raspberry pi in the closet", vecB);
33
+ // Query close to A's content → top hit should be A's memory, not B's.
34
+ const q = defaultEmbedder().embed("standardized node:sqlite store backend choice");
35
+ const hits = await searchMemoriesAsync(q, { k: 3 });
36
+ assert.ok(hits.length >= 1, "at least one hit");
37
+ assert.equal(hits[0].repoId, repoA, "nearest neighbor is repo A");
38
+ assert.equal(hits[0].memoryId, 1, "correct memory id");
39
+ assert.ok(hits[0].score > 0.5, "high cosine for the matching memory");
40
+ // Scope to repoB only → A must not appear.
41
+ const scoped = await searchMemoriesAsync(q, { k: 3, repoId: repoB });
42
+ assert.ok(scoped.every((h) => h.repoId === repoB), "scoped search stays within repoB");
43
+ }
44
+ finally {
45
+ await closeMemoryIndex();
46
+ delete process.env.MEGACOMPACT_INDEX_DIR;
47
+ }
48
+ });
49
+ test("memoryIndex: cleanup", () => {
50
+ rmSync(baseTmp, { recursive: true, force: true });
51
+ });
@@ -18,17 +18,15 @@
18
18
  import { homedir } from "node:os";
19
19
  import { join } from "node:path";
20
20
  import { mkdirSync, rmSync, existsSync } from "node:fs";
21
- // PGlite + pgvector are script-free WASM (no native build) → survive pi's
22
- // install-script block. Imported lazily so a missing/broken package degrades
23
- // gracefully instead of crashing module load.
24
- import { PGlite } from "@electric-sql/pglite";
25
- import { vector } from "@electric-sql/pglite-pgvector";
26
21
  /** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
27
22
  export const EMBEDDING_DIM = 512;
28
23
  let db;
29
24
  let initPromise;
30
25
  let disabled = false;
31
26
  let warned = false;
27
+ /** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
28
+ let pgliteMod;
29
+ let pgliteLoadFailed = false;
32
30
  function indexDir() {
33
31
  const override = process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
34
32
  if (override && override.trim() !== "")
@@ -73,6 +71,30 @@ export function initVectorIndex() {
73
71
  initPromise = openPgLite(/* retryOnCorrupt */ true);
74
72
  return initPromise;
75
73
  }
74
+ /**
75
+ * Lazily load the PGlite module + pgvector extension via dynamic import. Caches
76
+ * success and permanent failure. Returns undefined (once, then forever) when the
77
+ * package is missing/broken so callers fall back to the sync scan. Never throws.
78
+ */
79
+ async function loadPgLite() {
80
+ if (pgliteMod)
81
+ return pgliteMod;
82
+ if (pgliteLoadFailed)
83
+ return undefined;
84
+ try {
85
+ const [pglitePkg, pgvectorPkg] = await Promise.all([
86
+ import("@electric-sql/pglite"),
87
+ import("@electric-sql/pglite-pgvector"),
88
+ ]);
89
+ pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
90
+ return pgliteMod;
91
+ }
92
+ catch (err) {
93
+ pgliteLoadFailed = true;
94
+ logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
95
+ return undefined;
96
+ }
97
+ }
76
98
  /**
77
99
  * Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level
78
100
  * abort (typically from a corrupted/torn data dir) triggers a delete + one
@@ -80,11 +102,14 @@ export function initVectorIndex() {
80
102
  */
81
103
  async function openPgLite(retryOnCorrupt) {
82
104
  try {
105
+ const mod = await loadPgLite();
106
+ if (!mod)
107
+ return undefined;
83
108
  const dir = indexDir();
84
109
  mkdirSync(dir, { recursive: true });
85
- const pg = await new PGlite({
110
+ const pg = await new mod.PGlite({
86
111
  dataDir: dir,
87
- extensions: { vector },
112
+ extensions: { vector: mod.vector },
88
113
  });
89
114
  await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
90
115
  await pg.exec(`