pi-mega-compact 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,44 @@
1
1
  import { addMemory, listMemories, replaceMemory, removeMemory, } from "./store/sqlite.js";
2
+ import { defaultEmbedder } from "./embedder.js";
3
+ import { upsertMemoryEmbedding } from "./store/memoryIndex.js";
4
+ import { execSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: read-only `git rev-parse` to scope the memory index per-repo
5
+ /** Resolve the current repo's git root (mirrors extensions/mega-config.ts but
6
+ * kept local so src/ stays pi-agnostic — no extension-layer import). */
7
+ function resolveRepoRootLocal(cwd) {
8
+ try {
9
+ const out = execSync("git rev-parse --show-toplevel", {
10
+ cwd,
11
+ encoding: "utf-8",
12
+ stdio: ["ignore", "pipe", "ignore"],
13
+ }).trim();
14
+ return out || undefined;
15
+ }
16
+ catch {
17
+ return undefined;
18
+ }
19
+ }
2
20
  /** Find a memory row whose content exactly matches (case-insensitive). */
3
21
  function findByContent(memories, content) {
4
22
  const norm = content.trim().toLowerCase();
5
23
  return memories.find((m) => m.content.trim().toLowerCase() === norm);
6
24
  }
25
+ /**
26
+ * Fire-and-forget mirror of a memory write into the cross-repo PGlite index
27
+ * (S24 optional memory-RAG mirror). Best-effort + non-fatal: never blocks the
28
+ * SQLite write and degrades to the same-repo scan if the index is disabled or
29
+ * fails. `repoId` is the resolved git root so the memory is findable from other
30
+ * repos; falls back to the state dir when outside git.
31
+ */
32
+ function indexMemoryWrite(stateDir, memoryId, content) {
33
+ const repoId = resolveRepoRootLocal(stateDir) ?? stateDir;
34
+ try {
35
+ const vec = defaultEmbedder().embed(content);
36
+ void upsertMemoryEmbedding(repoId, memoryId, content, vec);
37
+ }
38
+ catch {
39
+ /* non-fatal — embedding/index failure must never break the SQLite write */
40
+ }
41
+ }
7
42
  /**
8
43
  * Apply add/replace/remove ops to the memories table. Replaces are matched by
9
44
  * existing content; removes by content. Idempotent: an add that already exists
@@ -19,7 +54,7 @@ export async function applyMemoryOps(ops, stateDir) {
19
54
  // Skip if an identical memory already exists.
20
55
  if (findByContent(existing, op.memory.content))
21
56
  continue;
22
- addMemory({
57
+ const id = addMemory({
23
58
  kind: op.memory.category,
24
59
  content: op.memory.content,
25
60
  tags: [],
@@ -27,6 +62,8 @@ export async function applyMemoryOps(ops, stateDir) {
27
62
  target: op.memory.target,
28
63
  sourceTurn: op.memory.sourceTurn,
29
64
  }, repo, stateDir);
65
+ // S24: mirror into the cross-repo index (fire-and-forget; non-fatal).
66
+ indexMemoryWrite(stateDir, id, op.memory.content);
30
67
  }
31
68
  else if (op.op === "replace") {
32
69
  const match = findByContent(existing, op.targetContent);
@@ -37,16 +74,19 @@ export async function applyMemoryOps(ops, stateDir) {
37
74
  category: op.memory.category,
38
75
  sourceTurn: op.memory.sourceTurn,
39
76
  }, stateDir);
77
+ // S24: re-mirror under the same memory id (fire-and-forget; non-fatal).
78
+ indexMemoryWrite(stateDir, match.id, op.memory.content);
40
79
  }
41
80
  else {
42
81
  // Target missing (e.g. earlier in-conversation contradiction) → add.
43
- addMemory({
82
+ const id = addMemory({
44
83
  kind: op.memory.category,
45
84
  content: op.memory.content,
46
85
  tags: [],
47
86
  category: op.memory.category,
48
87
  sourceTurn: op.memory.sourceTurn,
49
88
  }, repo, stateDir);
89
+ indexMemoryWrite(stateDir, id, op.memory.content);
50
90
  }
51
91
  }
52
92
  else {
@@ -4,7 +4,7 @@ import { mkdtempSync, rmSync } from "node:fs";
4
4
  import { tmpdir } from "node:os";
5
5
  import { join } from "node:path";
6
6
  import { applyMemoryOps } from "./memoryOps.js";
7
- import { addMemory, listMemories, replaceMemory, referenceMemory, MEMORY_MAX_CHARS, MEMORY_MAX_ROWS, } from "./store/sqlite.js";
7
+ import { addMemory, listMemories, replaceMemory, referenceMemory, MEMORY_MAX_CHARS, } from "./store/sqlite.js";
8
8
  const baseTmp = mkdtempSync(join(tmpdir(), "mc-memops-"));
9
9
  test("applyMemoryOps: ADD inserts a new memory", async () => {
10
10
  const dir = join(baseTmp, "add");
@@ -58,32 +58,55 @@ test("S24: replaceMemory also truncates oversized content", () => {
58
58
  assert.ok(row.content.endsWith("…[truncated]"), "marker appended");
59
59
  });
60
60
  test("S24: addMemory evicts LRU rows past MEMORY_MAX_ROWS per repo", () => {
61
- const dir = join(baseTmp, "lru");
62
- const n = MEMORY_MAX_ROWS;
63
- const seeds = n - 2;
64
- for (let i = 0; i < seeds; i++)
65
- addMemory({ content: `seed-${i}`, category: "note" }, null, dir);
66
- const keep1 = addMemory({ content: "keep-recent-1", category: "note" }, null, dir);
67
- const keep2 = addMemory({ content: "keep-recent-2", category: "note" }, null, dir);
68
- // Mark the two as referenced so the LRU eviction spares them (they get a
69
- // higher last_referenced than the un-referenced seeds).
70
- assert.ok(referenceMemory(keep1, dir), "reference keep1");
71
- assert.ok(referenceMemory(keep2, dir), "reference keep2");
72
- // Insert 3 more 3 over the cap across the inserts. The two referenced rows
73
- // must survive; only un-referenced (oldest) seeds should be evicted.
74
- addMemory({ content: "new-1", category: "note" }, null, dir);
75
- addMemory({ content: "new-2", category: "note" }, null, dir);
76
- addMemory({ content: "new-3", category: "note" }, null, dir);
77
- const rows = listMemories(null, 1000, dir);
78
- assert.equal(rows.length, n, "row count clamped to MEMORY_MAX_ROWS");
79
- assert.ok(rows.some((m) => /keep-recent-1/.test(m.content)), "referenced row survived");
80
- assert.ok(rows.some((m) => /keep-recent-2/.test(m.content)), "referenced row survived");
81
- assert.ok(rows.some((m) => /new-3/.test(m.content)), "newest row present");
82
- const seedRows = rows.filter((m) => /seed-/.test(m.content));
83
- // 3 rows were evicted (the inserts pushed 3 past the cap); all evicted rows
84
- // must be un-referenced seeds — the referenced rows survived above.
85
- assert.equal(seedRows.length, seeds - 3, "exactly 3 oldest un-referenced seeds evicted");
86
- assert.ok(!seedRows.some((m) => /seed-0/.test(m.content)), "oldest un-referenced seed evicted");
61
+ // Use a small env cap for a fast, deterministic LRU check (the production
62
+ // default is 500; this exercises the same code path).
63
+ process.env.MEGACOMPACT_MEMORY_MAX_ROWS = "10";
64
+ try {
65
+ const dir = join(baseTmp, "lru");
66
+ const n = 10;
67
+ const seeds = n - 2;
68
+ for (let i = 0; i < seeds; i++)
69
+ addMemory({ content: `seed-${i}`, category: "note" }, null, dir);
70
+ const keep1 = addMemory({ content: "keep-recent-1", category: "note" }, null, dir);
71
+ const keep2 = addMemory({ content: "keep-recent-2", category: "note" }, null, dir);
72
+ // Mark the two as referenced so the LRU eviction spares them (they get a
73
+ // higher last_referenced than the un-referenced seeds).
74
+ assert.ok(referenceMemory(keep1, dir), "reference keep1");
75
+ assert.ok(referenceMemory(keep2, dir), "reference keep2");
76
+ // Insert 3 more 3 over the cap across the inserts. The two referenced rows
77
+ // must survive; only un-referenced (oldest) seeds should be evicted.
78
+ addMemory({ content: "new-1", category: "note" }, null, dir);
79
+ addMemory({ content: "new-2", category: "note" }, null, dir);
80
+ addMemory({ content: "new-3", category: "note" }, null, dir);
81
+ const rows = listMemories(null, 1000, dir);
82
+ assert.equal(rows.length, n, "row count clamped to MEMORY_MAX_ROWS");
83
+ assert.ok(rows.some((m) => /keep-recent-1/.test(m.content)), "referenced row survived");
84
+ assert.ok(rows.some((m) => /keep-recent-2/.test(m.content)), "referenced row survived");
85
+ assert.ok(rows.some((m) => /new-3/.test(m.content)), "newest row present");
86
+ const seedRows = rows.filter((m) => /seed-/.test(m.content));
87
+ // 3 rows were evicted (the inserts pushed 3 past the cap); all evicted rows
88
+ // must be un-referenced seeds — the referenced rows survived above.
89
+ assert.equal(seedRows.length, seeds - 3, "exactly 3 oldest un-referenced seeds evicted");
90
+ assert.ok(!seedRows.some((m) => /seed-0/.test(m.content)), "oldest un-referenced seed evicted");
91
+ }
92
+ finally {
93
+ delete process.env.MEGACOMPACT_MEMORY_MAX_ROWS;
94
+ }
95
+ });
96
+ test("S24: MEGACOMPACT_MEMORY_MAX_CHARS env override truncates content", () => {
97
+ process.env.MEGACOMPACT_MEMORY_MAX_CHARS = "50";
98
+ try {
99
+ const dir = join(baseTmp, "cap-env");
100
+ const id = addMemory({ content: "x".repeat(500), category: "note" }, null, dir);
101
+ const rows = listMemories(null, 50, dir);
102
+ const row = rows.find((m) => m.id === id);
103
+ assert.ok(row, "row present");
104
+ assert.equal(row.content.length, 50 + "…[truncated]".length, "truncated to env cap + marker");
105
+ assert.ok(row.content.endsWith("…[truncated]"), "marker appended");
106
+ }
107
+ finally {
108
+ delete process.env.MEGACOMPACT_MEMORY_MAX_CHARS;
109
+ }
87
110
  });
88
111
  test("cleanup memops", () => {
89
112
  rmSync(baseTmp, { recursive: true, force: true });
@@ -58,3 +58,51 @@ export async function recallMemories(query, stateDir, opts = {}) {
58
58
  }
59
59
  return top;
60
60
  }
61
+ /**
62
+ * Cross-repo memory recall (S24): augments the same-repo `recallMemories` with
63
+ * HNSW NN over the global PGlite `memory_index` (other repos' memories). Content
64
+ * is read inline from the index hit (the recall process can't open other repos'
65
+ * SQLite dirs), so no other-repo db access is required. Returns hits sorted by
66
+ * descending cosine, above `crossRepoCosine`. De-duped by content against
67
+ * `sameRepoContent` so we never surface a memory the same-repo scan already has.
68
+ * Non-fatal: any index failure returns []. Best-effort + PREVENT-PI-004 (local
69
+ * WASM only).
70
+ */
71
+ export async function recallMemoriesCrossRepo(query, stateDir, opts = {}) {
72
+ const embedder = opts.embedder ?? defaultEmbedder();
73
+ const queryVec = embedder.embed(query);
74
+ const { searchMemoriesAsync } = await import("./store/memoryIndex.js");
75
+ const k = opts.limit ?? 5;
76
+ const floor = opts.crossRepoCosine ?? 0.3;
77
+ const hits = await searchMemoriesAsync(queryVec, { k });
78
+ if (!hits.length)
79
+ return [];
80
+ // Mark same-repo content as already-covered so we don't duplicate it.
81
+ const sameRepo = new Set(listMemories(opts.repo ?? null, 1000, stateDir).map((m) => m.content.trim().toLowerCase()));
82
+ const out = [];
83
+ for (const h of hits) {
84
+ if (h.score < floor)
85
+ continue;
86
+ if (sameRepo.has(h.content.trim().toLowerCase()))
87
+ continue;
88
+ out.push({
89
+ memory: {
90
+ id: h.memoryId,
91
+ repo: h.repoId,
92
+ kind: "note",
93
+ content: h.content,
94
+ tags: [],
95
+ createdAt: 0,
96
+ lastRecalledAt: null,
97
+ category: null,
98
+ target: null,
99
+ lastReferenced: null,
100
+ sourceTurn: null,
101
+ },
102
+ score: h.score,
103
+ repoId: h.repoId,
104
+ });
105
+ }
106
+ out.sort((a, b) => b.score - a.score);
107
+ return out;
108
+ }
@@ -90,3 +90,55 @@ test("recallMemories: fresher reference beats older at equal similarity", async
90
90
  test("cleanup memrec", () => {
91
91
  rmSync(baseTmp, { recursive: true, force: true });
92
92
  });
93
+ // ---- S24: cross-repo memory recall (PGlite mirror) ---------------------------
94
+ test("recallMemoriesAndInline: surfaces a memory saved in ANOTHER repo via cross-repo index", async () => {
95
+ // Isolate the global PGlite index to a temp dir shared by both "repos".
96
+ process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index");
97
+ const repoA = join(baseTmp, "repo-a");
98
+ const repoB = join(baseTmp, "repo-b");
99
+ try {
100
+ // repoA owns a decision about the store backend.
101
+ const { applyMemoryOps } = await import("./memoryOps.js");
102
+ await applyMemoryOps([{ op: "add", memory: { content: "we standardized on node:sqlite for the store backend", category: "decision", sourceTurn: 0 } }], repoA);
103
+ // repoB is a fresh session with NO local memory about the store backend.
104
+ const { recallMemoriesAndInline } = await import("./recall.js");
105
+ const res = await recallMemoriesAndInline({
106
+ query: "what store backend do we use?",
107
+ stateDir: repoB,
108
+ limit: 5,
109
+ crossRepo: true,
110
+ crossRepoCosine: 0.3,
111
+ });
112
+ assert.ok(!res.empty, "cross-repo recall found the other repo's memory");
113
+ assert.ok(/node:sqlite/.test(res.block), "the node:sqlite decision was recalled from repo A");
114
+ assert.ok(res.report.some((r) => /from /.test(r)), "report labels the memory as cross-repo");
115
+ }
116
+ finally {
117
+ const { closeMemoryIndex } = await import("./store/memoryIndex.js");
118
+ await closeMemoryIndex();
119
+ delete process.env.MEGACOMPACT_INDEX_DIR;
120
+ }
121
+ });
122
+ test("recallMemoriesAndInline: cross-repo disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
123
+ process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index-off");
124
+ process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
125
+ const repoA = join(baseTmp, "repo-a2");
126
+ const repoB = join(baseTmp, "repo-b2");
127
+ try {
128
+ const { applyMemoryOps } = await import("./memoryOps.js");
129
+ await applyMemoryOps([{ op: "add", memory: { content: "we standardized on node:sqlite for the store backend", category: "decision", sourceTurn: 0 } }], repoA);
130
+ const { recallMemoriesAndInline } = await import("./recall.js");
131
+ const res = await recallMemoriesAndInline({
132
+ query: "what store backend do we use?",
133
+ stateDir: repoB,
134
+ limit: 5,
135
+ crossRepo: true,
136
+ });
137
+ // Index disabled → no cross-repo hit; repoB has no local memory → empty.
138
+ assert.equal(res.empty, true, "cross-repo recall degrades to empty when disabled");
139
+ }
140
+ finally {
141
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
142
+ delete process.env.MEGACOMPACT_INDEX_DIR;
143
+ }
144
+ });
@@ -105,27 +105,53 @@ export function formatMemoryRecallBlock(hits) {
105
105
  export async function recallMemoriesAndInline(opts) {
106
106
  const limit = opts.limit ?? 5;
107
107
  const maxTokens = opts.recallMaxTokens ?? 0;
108
- const { recallMemories } = await import("./memoryRecall.js");
108
+ const { recallMemories, recallMemoriesCrossRepo } = await import("./memoryRecall.js");
109
109
  const hits = await recallMemories(opts.query, opts.stateDir, {
110
110
  topK: limit,
111
111
  minSimilarity: opts.minSimilarity ?? 0.2,
112
112
  });
113
- if (hits.length === 0)
113
+ // S24 cross-repo augmentation: if same-repo recall is thin, pull additional
114
+ // memories from OTHER repos via the PGlite HNSW index. Non-fatal: a failure
115
+ // degrades to the same-repo hits only.
116
+ const crossHits = [];
117
+ if (opts.crossRepo && hits.length < limit) {
118
+ try {
119
+ const x = await recallMemoriesCrossRepo(opts.query, opts.stateDir, {
120
+ repo: null,
121
+ limit: limit - hits.length,
122
+ crossRepoCosine: opts.crossRepoCosine ?? 0.3,
123
+ });
124
+ for (const h of x)
125
+ crossHits.push(h);
126
+ }
127
+ catch {
128
+ /* non-fatal — cross-repo failure → same-repo only */
129
+ }
130
+ }
131
+ if (hits.length === 0 && crossHits.length === 0)
114
132
  return { empty: true, block: "", report: [] };
115
133
  // Same incremental token cap pattern as checkpoint recall.
116
134
  const parts = [];
117
135
  const report = [];
118
136
  let blockTokens = 0;
119
- for (const h of hits) {
120
- const part = formatMemoryRecallBlock([
121
- { content: h.memory.content, category: h.memory.category, score: h.score },
122
- ]);
137
+ const pushHit = (content, category, score, label) => {
138
+ const part = formatMemoryRecallBlock([{ content, category, score }]);
123
139
  const partTokens = estimateBlockTokens(part);
124
140
  if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
125
- break;
141
+ return false;
126
142
  parts.push(part);
127
- report.push(` • memory#${h.memory.id} (${(h.score * 100).toFixed(0)}%): ${h.memory.content.slice(0, 60).replace(/\n/g, " ")}…`);
143
+ report.push(` • ${label} (${(score * 100).toFixed(0)}%): ${content.slice(0, 60).replace(/\n/g, " ")}…`);
128
144
  blockTokens += partTokens;
145
+ return true;
146
+ };
147
+ for (const h of hits) {
148
+ if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id}`))
149
+ break;
150
+ }
151
+ for (const h of crossHits) {
152
+ const repoLabel = h.repoId.split(/[\\/]/).filter(Boolean).pop() ?? h.repoId;
153
+ if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id} (from ${repoLabel})`))
154
+ break;
129
155
  }
130
156
  return { empty: parts.length === 0, block: parts.join("\n"), report };
131
157
  }
@@ -0,0 +1,205 @@
1
+ /**
2
+ * memoryIndex.ts — cross-repo async vector index for durable memories (S24).
3
+ *
4
+ * A REDUNDANT, additive, ASYNC index layered over the authoritative node:sqlite
5
+ * `memories` table. The same-repo linear cosine scan over the in-repo memories
6
+ * (src/memoryRecall.ts) stays the DEFAULT recall path; this global PGlite index
7
+ * exists only to provide real cross-repo nearest-neighbor memory recall — so a
8
+ * decision you saved in repo A can be inlined as RAG context when you start a
9
+ * session in repo B. It is best-effort and non-fatal: any init/write failure
10
+ * degrades to the same-repo scan and must NEVER break memory write, recall, or
11
+ * extension load.
12
+ *
13
+ * PREVENT-PI-004: PGlite is WASM Postgres — fully local, zero network. Memory
14
+ * remains AUTHORITATIVE in SQLite; this index only holds (repo_id, memory_id,
15
+ * content, embedding) for NN lookup and is rebuilt from SQLite at any time.
16
+ *
17
+ * Topology mirrors vectorIndex.ts (Slice 2): ONE global PGlite DB, `repo_id` is
18
+ * a first-class column. `searchMemoriesAsync(q, k, {repoId?})` → omit repoId for
19
+ * cross-repo NN, pass repoId to scope to a single repo. Hit content is stored
20
+ * inline because the recall process cannot open every other repo's SQLite dir.
21
+ */
22
+ import { homedir } from "node:os";
23
+ import { join } from "node:path";
24
+ import { mkdirSync, rmSync, existsSync } from "node:fs";
25
+ // PGlite + pgvector are script-free WASM (no native build) → survive pi's
26
+ // install-script block. Imported lazily so a missing/broken package degrades
27
+ // gracefully instead of crashing module load.
28
+ import { PGlite } from "@electric-sql/pglite";
29
+ import { vector } from "@electric-sql/pglite-pgvector";
30
+ /** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
31
+ export const MEMORY_INDEX_DIM = 512;
32
+ let db;
33
+ let initPromise;
34
+ let disabled = false;
35
+ let warned = false;
36
+ function indexDir() {
37
+ const override = process.env.MEGACOMPACT_INDEX_DIR;
38
+ if (override && override.trim() !== "")
39
+ return join(override, "memory");
40
+ try {
41
+ return join(homedir(), ".pi", "mega-compact-vector", "memory");
42
+ }
43
+ catch {
44
+ return join("/tmp", ".mega-compact-vector", "memory");
45
+ }
46
+ }
47
+ function logWarn(msg) {
48
+ // Never throw — degradation is the whole point. One warning per process.
49
+ if (warned)
50
+ return;
51
+ warned = true;
52
+ try {
53
+ console.warn(`[mega-compact:memoryIndex] ${msg} (falling back to same-repo scan)`);
54
+ }
55
+ catch {
56
+ /* ignore */
57
+ }
58
+ }
59
+ /** Honor the emergency kill-switch (shared with the checkpoint index). */
60
+ export function isMemoryIndexDisabled() {
61
+ return (disabled ||
62
+ process.env.MEGACOMPACT_PGLITE_DISABLED === "true" ||
63
+ process.env.MEGACOMPACT_PGLITE_DISABLED === "1");
64
+ }
65
+ /**
66
+ * Lazily open + schema-init the global PGlite DB. Idempotent and safe to call
67
+ * from many places. Returns undefined when disabled/unavailable so callers can
68
+ * fall back to the synchronous scan. Never throws.
69
+ */
70
+ export function initMemoryIndex() {
71
+ if (isMemoryIndexDisabled())
72
+ return Promise.resolve(undefined);
73
+ if (db)
74
+ return Promise.resolve(db);
75
+ if (initPromise)
76
+ return initPromise;
77
+ initPromise = openPgLite(/* retryOnCorrupt */ true);
78
+ return initPromise;
79
+ }
80
+ /**
81
+ * Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level abort
82
+ * (typically from a corrupted/torn data dir) triggers a delete + one retry.
83
+ */
84
+ async function openPgLite(retryOnCorrupt) {
85
+ try {
86
+ const dir = indexDir();
87
+ mkdirSync(dir, { recursive: true });
88
+ const pg = await new PGlite({
89
+ dataDir: dir,
90
+ extensions: { vector },
91
+ });
92
+ await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
93
+ await pg.exec(`
94
+ CREATE TABLE IF NOT EXISTS memory_index (
95
+ repo_id TEXT NOT NULL,
96
+ memory_id INTEGER NOT NULL,
97
+ content TEXT NOT NULL,
98
+ embedding vector(${MEMORY_INDEX_DIM}) NOT NULL,
99
+ PRIMARY KEY (repo_id, memory_id)
100
+ );
101
+ `);
102
+ await pg.exec("CREATE INDEX IF NOT EXISTS memory_index_hnsw ON memory_index USING hnsw (embedding vector_cosine_ops);");
103
+ db = pg;
104
+ return pg;
105
+ }
106
+ catch (err) {
107
+ const msg = err instanceof Error ? err.message : String(err);
108
+ if (retryOnCorrupt && (msg.includes("Aborted") || msg.includes("RuntimeError"))) {
109
+ try {
110
+ const dir = indexDir();
111
+ if (existsSync(dir))
112
+ rmSync(dir, { recursive: true, force: true });
113
+ initPromise = undefined;
114
+ return openPgLite(/* retryOnCorrupt */ false);
115
+ }
116
+ catch {
117
+ /* self-heal failed — fall through to disable */
118
+ }
119
+ }
120
+ disabled = true;
121
+ logWarn(`init failed: ${msg}`);
122
+ return undefined;
123
+ }
124
+ }
125
+ function toVectorLiteral(v) {
126
+ const parts = v.map((x) => (Number.isFinite(x) ? x : 0));
127
+ return `[${parts.join(",")}]`;
128
+ }
129
+ /**
130
+ * Best-effort upsert of one memory embedding into the global index.
131
+ * Dimension-mismatched vectors (e.g. a BYO embedder with dim ≠ 512) are skipped.
132
+ * Fire-and-forget: callers must NOT await this on the sync write path. Never
133
+ * throws. `content` is stored inline so cross-repo recall can read it directly.
134
+ */
135
+ export async function upsertMemoryEmbedding(repoId, memoryId, content, embedding) {
136
+ if (isMemoryIndexDisabled())
137
+ return;
138
+ if (!embedding || embedding.length !== MEMORY_INDEX_DIM)
139
+ return;
140
+ try {
141
+ const pg = await initMemoryIndex();
142
+ if (!pg)
143
+ return;
144
+ const lit = toVectorLiteral(embedding);
145
+ await pg.query(`INSERT INTO memory_index (repo_id, memory_id, content, embedding)
146
+ VALUES ($1, $2, $3, $4::vector)
147
+ ON CONFLICT (repo_id, memory_id)
148
+ DO UPDATE SET content = EXCLUDED.content, embedding = EXCLUDED.embedding;`, [repoId, memoryId, content, lit]);
149
+ }
150
+ catch (err) {
151
+ disabled = true;
152
+ logWarn(`upsert failed: ${err instanceof Error ? err.message : String(err)}`);
153
+ }
154
+ }
155
+ /**
156
+ * Cross-repo (or single-repo) HNSW nearest-neighbor memory search. Returns hits
157
+ * sorted by descending similarity. Never throws — on any failure returns [].
158
+ */
159
+ export async function searchMemoriesAsync(query, opts = {}) {
160
+ if (isMemoryIndexDisabled() || !query || query.length !== MEMORY_INDEX_DIM)
161
+ return [];
162
+ const k = opts.k ?? 5;
163
+ const repoId = opts.repoId;
164
+ try {
165
+ const pg = await initMemoryIndex();
166
+ if (!pg)
167
+ return [];
168
+ const lit = toVectorLiteral(query);
169
+ const params = [lit, k];
170
+ let sql = "SELECT repo_id, memory_id, content, 1 - (embedding <=> $1::vector) AS score " +
171
+ "FROM memory_index";
172
+ if (repoId) {
173
+ sql += " WHERE repo_id = $3";
174
+ params.push(repoId);
175
+ }
176
+ sql += " ORDER BY embedding <=> $1::vector LIMIT $2";
177
+ const res = await pg.query(sql, params);
178
+ return res.rows.map((r) => ({
179
+ repoId: r.repo_id,
180
+ memoryId: Number(r.memory_id),
181
+ content: r.content,
182
+ score: r.score,
183
+ }));
184
+ }
185
+ catch (err) {
186
+ disabled = true;
187
+ logWarn(`search failed: ${err instanceof Error ? err.message : String(err)}`);
188
+ return [];
189
+ }
190
+ }
191
+ /** Close the index (test teardown / shutdown). Safe to call when unopened. */
192
+ export async function closeMemoryIndex() {
193
+ if (db) {
194
+ try {
195
+ await db.close();
196
+ }
197
+ catch {
198
+ /* ignore */
199
+ }
200
+ }
201
+ db = undefined;
202
+ initPromise = undefined;
203
+ disabled = false;
204
+ warned = false;
205
+ }
@@ -0,0 +1,51 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { mkdtempSync, rmSync } from "node:fs";
4
+ import { tmpdir } from "node:os";
5
+ import { join } from "node:path";
6
+ import { defaultEmbedder } from "../embedder.js";
7
+ import { upsertMemoryEmbedding, searchMemoriesAsync, initMemoryIndex, closeMemoryIndex, isMemoryIndexDisabled, } from "./memoryIndex.js";
8
+ const baseTmp = mkdtempSync(join(tmpdir(), "mc-memidx-"));
9
+ test("memoryIndex: disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
10
+ process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
11
+ try {
12
+ assert.equal(isMemoryIndexDisabled(), true, "kill-switch honored");
13
+ const hits = await searchMemoriesAsync(defaultEmbedder().embed("anything"), { k: 3 });
14
+ assert.deepEqual(hits, [], "search returns [] when disabled");
15
+ }
16
+ finally {
17
+ delete process.env.MEGACOMPACT_PGLITE_DISABLED;
18
+ }
19
+ });
20
+ test("memoryIndex: cross-repo upsert + NN search returns the right repo's memory", async () => {
21
+ // Isolate the global PGlite dir so concurrent test runs don't collide.
22
+ process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
23
+ const repoA = "/tmp/repo-a";
24
+ const repoB = "/tmp/repo-b";
25
+ try {
26
+ await initMemoryIndex();
27
+ // Two memories in different repos, with clearly distinct content so their
28
+ // trigram embeddings separate.
29
+ const vecA = defaultEmbedder().embed("We standardized on node:sqlite for the store backend");
30
+ const vecB = defaultEmbedder().embed("The deployment target is a raspberry pi in the closet");
31
+ await upsertMemoryEmbedding(repoA, 1, "We standardized on node:sqlite for the store backend", vecA);
32
+ await upsertMemoryEmbedding(repoB, 7, "The deployment target is a raspberry pi in the closet", vecB);
33
+ // Query close to A's content → top hit should be A's memory, not B's.
34
+ const q = defaultEmbedder().embed("standardized node:sqlite store backend choice");
35
+ const hits = await searchMemoriesAsync(q, { k: 3 });
36
+ assert.ok(hits.length >= 1, "at least one hit");
37
+ assert.equal(hits[0].repoId, repoA, "nearest neighbor is repo A");
38
+ assert.equal(hits[0].memoryId, 1, "correct memory id");
39
+ assert.ok(hits[0].score > 0.5, "high cosine for the matching memory");
40
+ // Scope to repoB only → A must not appear.
41
+ const scoped = await searchMemoriesAsync(q, { k: 3, repoId: repoB });
42
+ assert.ok(scoped.every((h) => h.repoId === repoB), "scoped search stays within repoB");
43
+ }
44
+ finally {
45
+ await closeMemoryIndex();
46
+ delete process.env.MEGACOMPACT_INDEX_DIR;
47
+ }
48
+ });
49
+ test("memoryIndex: cleanup", () => {
50
+ rmSync(baseTmp, { recursive: true, force: true });
51
+ });
@@ -581,14 +581,32 @@ export function addLesson(sessionId, repo, lesson, stateDir = getStateDir()) {
581
581
  // file-backed memory caps a single entry at ~5k chars). We truncate content at
582
582
  // MEMORY_MAX_CHARS and evict the least-recently-referenced rows past
583
583
  // MEMORY_MAX_ROWS per repo via LRU. Both are SQLite-only (PREVENT-PI-004): no
584
- // file-backed memory is written anywhere.
584
+ // file-backed memory is written anywhere. Defaults are overridable via env
585
+ // (MEGACOMPACT_MEMORY_MAX_CHARS / MEGACOMPACT_MEMORY_MAX_ROWS).
585
586
  export const MEMORY_MAX_CHARS = 4000;
586
- export const MEMORY_MAX_ROWS = 200;
587
+ export const MEMORY_MAX_ROWS = 500;
588
+ /** Read an env override as a positive int, falling back to `fallback`. */
589
+ function envInt(name, fallback) {
590
+ const v = process.env[name];
591
+ if (v == null || v === "")
592
+ return fallback;
593
+ const n = Number(v);
594
+ return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
595
+ }
596
+ /** Effective per-entry char cap (env-overridable, default MEMORY_MAX_CHARS). */
597
+ export function memoryMaxChars() {
598
+ return envInt("MEGACOMPACT_MEMORY_MAX_CHARS", MEMORY_MAX_CHARS);
599
+ }
600
+ /** Effective per-repo row cap (env-overridable, default MEMORY_MAX_ROWS). */
601
+ export function memoryMaxRows() {
602
+ return envInt("MEGACOMPACT_MEMORY_MAX_ROWS", MEMORY_MAX_ROWS);
603
+ }
587
604
  /** Truncate memory content to the per-entry cap, preserving a trailing marker. */
588
605
  function capMemoryContent(content) {
589
- if (content.length <= MEMORY_MAX_CHARS)
606
+ const cap = memoryMaxChars();
607
+ if (content.length <= cap)
590
608
  return content;
591
- return content.slice(0, MEMORY_MAX_CHARS) + "…[truncated]";
609
+ return content.slice(0, cap) + "…[truncated]";
592
610
  }
593
611
  /**
594
612
  * Evict the least-recently-referenced rows for a repo past MEMORY_MAX_ROWS.
@@ -598,6 +616,7 @@ function capMemoryContent(content) {
598
616
  */
599
617
  function evictMemoryLru(repo, stateDir) {
600
618
  const db = openStore(stateDir);
619
+ const maxRows = memoryMaxRows();
601
620
  // SQLite `= NULL` is never true, so the null-repo scope (memories are
602
621
  // stateDir-scoped when repo is null — the applyMemoryOps path) needs `IS NULL`.
603
622
  const where = repo == null ? "repo IS NULL" : "repo = ?";
@@ -605,7 +624,7 @@ function evictMemoryLru(repo, stateDir) {
605
624
  ? db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get()
606
625
  : db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get(repo);
607
626
  const count = countRow.n;
608
- const over = count - MEMORY_MAX_ROWS;
627
+ const over = count - maxRows;
609
628
  if (over <= 0)
610
629
  return;
611
630
  // Delete the `over` least-recently-used rows. ORDER BY the LRU key ASC, id ASC