pi-mega-compact 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -23
- package/dist/extensions/conflict-scan.test.js +115 -0
- package/dist/extensions/mega-conflict-cmds.js +17 -0
- package/dist/extensions/mega-events.js +102 -6
- package/dist/extensions/mega-runtime.js +20 -0
- package/dist/extensions/mega-teamrun.test.js +143 -0
- package/dist/extensions/mega-trim.js +33 -2
- package/dist/src/memoryOps.js +42 -2
- package/dist/src/memoryRecall.js +48 -0
- package/dist/src/memoryRecall.test.js +52 -0
- package/dist/src/recall.js +34 -8
- package/dist/src/store/memoryIndex.js +230 -0
- package/dist/src/store/memoryIndex.test.js +51 -0
- package/dist/src/store/vectorIndex.js +32 -7
- package/extensions/mega-conflict-cmds.ts +15 -0
- package/extensions/mega-events.ts +96 -6
- package/extensions/mega-runtime.ts +21 -0
- package/extensions/mega-teamrun.test.ts +164 -0
- package/extensions/mega-trim.ts +28 -1
- package/package.json +1 -1
- package/src/memoryOps.ts +42 -2
- package/src/memoryRecall.test.ts +58 -0
- package/src/memoryRecall.ts +52 -0
- package/src/recall.ts +35 -10
- package/src/store/memoryIndex.test.ts +61 -0
- package/src/store/memoryIndex.ts +269 -0
- package/src/store/vectorIndex.ts +40 -6
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
|
@@ -22,8 +22,39 @@ export function computeLiveTrimCut(view, opts) {
|
|
|
22
22
|
return null; // nothing safe to cut — keep everything this call
|
|
23
23
|
const recent = view.slice(cut);
|
|
24
24
|
const userCount = recent.filter((m) => m.role === "user").length;
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
// ANCHOR FLOOR (PREVENT-PI-001): the recent window must keep at least
|
|
26
|
+
// `anchorUserMessages` user messages. The original compactedFrom can land on a
|
|
27
|
+
// run that starts with fewer than that (e.g. the preserved region begins on a
|
|
28
|
+
// tool pair, or the session's tail is tool-heavy). Instead of bailing out and
|
|
29
|
+
// skipping the live trim entirely this call (which left the model fed a
|
|
30
|
+
// 150k-context window during long team runs), walk `cut` backward until the
|
|
31
|
+
// preserved run contains enough user messages — bounded by the boundary-safe
|
|
32
|
+
// constraint so we never split a tool pair. Falls back to null only when the
|
|
33
|
+
// whole view can't satisfy the floor (tiny sessions) — the next context event
|
|
34
|
+
// retries.
|
|
35
|
+
if (userCount < opts.anchorUserMessages) {
|
|
36
|
+
let c = cut;
|
|
37
|
+
while (c > 1) {
|
|
38
|
+
c--;
|
|
39
|
+
if (!isBoundarySafe(view, c))
|
|
40
|
+
continue;
|
|
41
|
+
const recentNow = view.slice(c);
|
|
42
|
+
const usersNow = recentNow.filter((m) => m.role === "user").length;
|
|
43
|
+
if (usersNow >= opts.anchorUserMessages) {
|
|
44
|
+
cut = c;
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (cut > 1) {
|
|
49
|
+
const finalRecent = view.slice(cut);
|
|
50
|
+
if (finalRecent.filter((m) => m.role === "user").length < opts.anchorUserMessages) {
|
|
51
|
+
return null; // cannot satisfy the floor without dropping too much — retry next call
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
27
58
|
return cut;
|
|
28
59
|
}
|
|
29
60
|
/** The formatted compacted-region summary as a user-role engine message. */
|
package/dist/src/memoryOps.js
CHANGED
|
@@ -1,9 +1,44 @@
|
|
|
1
1
|
import { addMemory, listMemories, replaceMemory, removeMemory, } from "./store/sqlite.js";
|
|
2
|
+
import { defaultEmbedder } from "./embedder.js";
|
|
3
|
+
import { upsertMemoryEmbedding } from "./store/memoryIndex.js";
|
|
4
|
+
import { execSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: read-only `git rev-parse` to scope the memory index per-repo
|
|
5
|
+
/** Resolve the current repo's git root (mirrors extensions/mega-config.ts but
|
|
6
|
+
* kept local so src/ stays pi-agnostic — no extension-layer import). */
|
|
7
|
+
function resolveRepoRootLocal(cwd) {
|
|
8
|
+
try {
|
|
9
|
+
const out = execSync("git rev-parse --show-toplevel", {
|
|
10
|
+
cwd,
|
|
11
|
+
encoding: "utf-8",
|
|
12
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
13
|
+
}).trim();
|
|
14
|
+
return out || undefined;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
2
20
|
/** Find a memory row whose content exactly matches (case-insensitive). */
|
|
3
21
|
function findByContent(memories, content) {
|
|
4
22
|
const norm = content.trim().toLowerCase();
|
|
5
23
|
return memories.find((m) => m.content.trim().toLowerCase() === norm);
|
|
6
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* Fire-and-forget mirror of a memory write into the cross-repo PGlite index
|
|
27
|
+
* (S24 optional memory-RAG mirror). Best-effort + non-fatal: never blocks the
|
|
28
|
+
* SQLite write and degrades to the same-repo scan if the index is disabled or
|
|
29
|
+
* fails. `repoId` is the resolved git root so the memory is findable from other
|
|
30
|
+
* repos; falls back to the state dir when outside git.
|
|
31
|
+
*/
|
|
32
|
+
function indexMemoryWrite(stateDir, memoryId, content) {
|
|
33
|
+
const repoId = resolveRepoRootLocal(stateDir) ?? stateDir;
|
|
34
|
+
try {
|
|
35
|
+
const vec = defaultEmbedder().embed(content);
|
|
36
|
+
void upsertMemoryEmbedding(repoId, memoryId, content, vec);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
/* non-fatal — embedding/index failure must never break the SQLite write */
|
|
40
|
+
}
|
|
41
|
+
}
|
|
7
42
|
/**
|
|
8
43
|
* Apply add/replace/remove ops to the memories table. Replaces are matched by
|
|
9
44
|
* existing content; removes by content. Idempotent: an add that already exists
|
|
@@ -19,7 +54,7 @@ export async function applyMemoryOps(ops, stateDir) {
|
|
|
19
54
|
// Skip if an identical memory already exists.
|
|
20
55
|
if (findByContent(existing, op.memory.content))
|
|
21
56
|
continue;
|
|
22
|
-
addMemory({
|
|
57
|
+
const id = addMemory({
|
|
23
58
|
kind: op.memory.category,
|
|
24
59
|
content: op.memory.content,
|
|
25
60
|
tags: [],
|
|
@@ -27,6 +62,8 @@ export async function applyMemoryOps(ops, stateDir) {
|
|
|
27
62
|
target: op.memory.target,
|
|
28
63
|
sourceTurn: op.memory.sourceTurn,
|
|
29
64
|
}, repo, stateDir);
|
|
65
|
+
// S24: mirror into the cross-repo index (fire-and-forget; non-fatal).
|
|
66
|
+
indexMemoryWrite(stateDir, id, op.memory.content);
|
|
30
67
|
}
|
|
31
68
|
else if (op.op === "replace") {
|
|
32
69
|
const match = findByContent(existing, op.targetContent);
|
|
@@ -37,16 +74,19 @@ export async function applyMemoryOps(ops, stateDir) {
|
|
|
37
74
|
category: op.memory.category,
|
|
38
75
|
sourceTurn: op.memory.sourceTurn,
|
|
39
76
|
}, stateDir);
|
|
77
|
+
// S24: re-mirror under the same memory id (fire-and-forget; non-fatal).
|
|
78
|
+
indexMemoryWrite(stateDir, match.id, op.memory.content);
|
|
40
79
|
}
|
|
41
80
|
else {
|
|
42
81
|
// Target missing (e.g. earlier in-conversation contradiction) → add.
|
|
43
|
-
addMemory({
|
|
82
|
+
const id = addMemory({
|
|
44
83
|
kind: op.memory.category,
|
|
45
84
|
content: op.memory.content,
|
|
46
85
|
tags: [],
|
|
47
86
|
category: op.memory.category,
|
|
48
87
|
sourceTurn: op.memory.sourceTurn,
|
|
49
88
|
}, repo, stateDir);
|
|
89
|
+
indexMemoryWrite(stateDir, id, op.memory.content);
|
|
50
90
|
}
|
|
51
91
|
}
|
|
52
92
|
else {
|
package/dist/src/memoryRecall.js
CHANGED
|
@@ -58,3 +58,51 @@ export async function recallMemories(query, stateDir, opts = {}) {
|
|
|
58
58
|
}
|
|
59
59
|
return top;
|
|
60
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Cross-repo memory recall (S24): augments the same-repo `recallMemories` with
|
|
63
|
+
* HNSW NN over the global PGlite `memory_index` (other repos' memories). Content
|
|
64
|
+
* is read inline from the index hit (the recall process can't open other repos'
|
|
65
|
+
* SQLite dirs), so no other-repo db access is required. Returns hits sorted by
|
|
66
|
+
* descending cosine, above `crossRepoCosine`. De-duped by content against
|
|
67
|
+
* `sameRepoContent` so we never surface a memory the same-repo scan already has.
|
|
68
|
+
* Non-fatal: any index failure returns []. Best-effort + PREVENT-PI-004 (local
|
|
69
|
+
* WASM only).
|
|
70
|
+
*/
|
|
71
|
+
export async function recallMemoriesCrossRepo(query, stateDir, opts = {}) {
|
|
72
|
+
const embedder = opts.embedder ?? defaultEmbedder();
|
|
73
|
+
const queryVec = embedder.embed(query);
|
|
74
|
+
const { searchMemoriesAsync } = await import("./store/memoryIndex.js");
|
|
75
|
+
const k = opts.limit ?? 5;
|
|
76
|
+
const floor = opts.crossRepoCosine ?? 0.3;
|
|
77
|
+
const hits = await searchMemoriesAsync(queryVec, { k });
|
|
78
|
+
if (!hits.length)
|
|
79
|
+
return [];
|
|
80
|
+
// Mark same-repo content as already-covered so we don't duplicate it.
|
|
81
|
+
const sameRepo = new Set(listMemories(opts.repo ?? null, 1000, stateDir).map((m) => m.content.trim().toLowerCase()));
|
|
82
|
+
const out = [];
|
|
83
|
+
for (const h of hits) {
|
|
84
|
+
if (h.score < floor)
|
|
85
|
+
continue;
|
|
86
|
+
if (sameRepo.has(h.content.trim().toLowerCase()))
|
|
87
|
+
continue;
|
|
88
|
+
out.push({
|
|
89
|
+
memory: {
|
|
90
|
+
id: h.memoryId,
|
|
91
|
+
repo: h.repoId,
|
|
92
|
+
kind: "note",
|
|
93
|
+
content: h.content,
|
|
94
|
+
tags: [],
|
|
95
|
+
createdAt: 0,
|
|
96
|
+
lastRecalledAt: null,
|
|
97
|
+
category: null,
|
|
98
|
+
target: null,
|
|
99
|
+
lastReferenced: null,
|
|
100
|
+
sourceTurn: null,
|
|
101
|
+
},
|
|
102
|
+
score: h.score,
|
|
103
|
+
repoId: h.repoId,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
out.sort((a, b) => b.score - a.score);
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
@@ -90,3 +90,55 @@ test("recallMemories: fresher reference beats older at equal similarity", async
|
|
|
90
90
|
test("cleanup memrec", () => {
|
|
91
91
|
rmSync(baseTmp, { recursive: true, force: true });
|
|
92
92
|
});
|
|
93
|
+
// ---- S24: cross-repo memory recall (PGlite mirror) ---------------------------
|
|
94
|
+
test("recallMemoriesAndInline: surfaces a memory saved in ANOTHER repo via cross-repo index", async () => {
|
|
95
|
+
// Isolate the global PGlite index to a temp dir shared by both "repos".
|
|
96
|
+
process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index");
|
|
97
|
+
const repoA = join(baseTmp, "repo-a");
|
|
98
|
+
const repoB = join(baseTmp, "repo-b");
|
|
99
|
+
try {
|
|
100
|
+
// repoA owns a decision about the store backend.
|
|
101
|
+
const { applyMemoryOps } = await import("./memoryOps.js");
|
|
102
|
+
await applyMemoryOps([{ op: "add", memory: { content: "we standardized on node:sqlite for the store backend", category: "decision", sourceTurn: 0 } }], repoA);
|
|
103
|
+
// repoB is a fresh session with NO local memory about the store backend.
|
|
104
|
+
const { recallMemoriesAndInline } = await import("./recall.js");
|
|
105
|
+
const res = await recallMemoriesAndInline({
|
|
106
|
+
query: "what store backend do we use?",
|
|
107
|
+
stateDir: repoB,
|
|
108
|
+
limit: 5,
|
|
109
|
+
crossRepo: true,
|
|
110
|
+
crossRepoCosine: 0.3,
|
|
111
|
+
});
|
|
112
|
+
assert.ok(!res.empty, "cross-repo recall found the other repo's memory");
|
|
113
|
+
assert.ok(/node:sqlite/.test(res.block), "the node:sqlite decision was recalled from repo A");
|
|
114
|
+
assert.ok(res.report.some((r) => /from /.test(r)), "report labels the memory as cross-repo");
|
|
115
|
+
}
|
|
116
|
+
finally {
|
|
117
|
+
const { closeMemoryIndex } = await import("./store/memoryIndex.js");
|
|
118
|
+
await closeMemoryIndex();
|
|
119
|
+
delete process.env.MEGACOMPACT_INDEX_DIR;
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
test("recallMemoriesAndInline: cross-repo disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
|
|
123
|
+
process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index-off");
|
|
124
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
|
|
125
|
+
const repoA = join(baseTmp, "repo-a2");
|
|
126
|
+
const repoB = join(baseTmp, "repo-b2");
|
|
127
|
+
try {
|
|
128
|
+
const { applyMemoryOps } = await import("./memoryOps.js");
|
|
129
|
+
await applyMemoryOps([{ op: "add", memory: { content: "we standardized on node:sqlite for the store backend", category: "decision", sourceTurn: 0 } }], repoA);
|
|
130
|
+
const { recallMemoriesAndInline } = await import("./recall.js");
|
|
131
|
+
const res = await recallMemoriesAndInline({
|
|
132
|
+
query: "what store backend do we use?",
|
|
133
|
+
stateDir: repoB,
|
|
134
|
+
limit: 5,
|
|
135
|
+
crossRepo: true,
|
|
136
|
+
});
|
|
137
|
+
// Index disabled → no cross-repo hit; repoB has no local memory → empty.
|
|
138
|
+
assert.equal(res.empty, true, "cross-repo recall degrades to empty when disabled");
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
142
|
+
delete process.env.MEGACOMPACT_INDEX_DIR;
|
|
143
|
+
}
|
|
144
|
+
});
|
package/dist/src/recall.js
CHANGED
|
@@ -105,27 +105,53 @@ export function formatMemoryRecallBlock(hits) {
|
|
|
105
105
|
export async function recallMemoriesAndInline(opts) {
|
|
106
106
|
const limit = opts.limit ?? 5;
|
|
107
107
|
const maxTokens = opts.recallMaxTokens ?? 0;
|
|
108
|
-
const { recallMemories } = await import("./memoryRecall.js");
|
|
108
|
+
const { recallMemories, recallMemoriesCrossRepo } = await import("./memoryRecall.js");
|
|
109
109
|
const hits = await recallMemories(opts.query, opts.stateDir, {
|
|
110
110
|
topK: limit,
|
|
111
111
|
minSimilarity: opts.minSimilarity ?? 0.2,
|
|
112
112
|
});
|
|
113
|
-
if
|
|
113
|
+
// S24 cross-repo augmentation: if same-repo recall is thin, pull additional
|
|
114
|
+
// memories from OTHER repos via the PGlite HNSW index. Non-fatal: a failure
|
|
115
|
+
// degrades to the same-repo hits only.
|
|
116
|
+
const crossHits = [];
|
|
117
|
+
if (opts.crossRepo && hits.length < limit) {
|
|
118
|
+
try {
|
|
119
|
+
const x = await recallMemoriesCrossRepo(opts.query, opts.stateDir, {
|
|
120
|
+
repo: null,
|
|
121
|
+
limit: limit - hits.length,
|
|
122
|
+
crossRepoCosine: opts.crossRepoCosine ?? 0.3,
|
|
123
|
+
});
|
|
124
|
+
for (const h of x)
|
|
125
|
+
crossHits.push(h);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
/* non-fatal — cross-repo failure → same-repo only */
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
if (hits.length === 0 && crossHits.length === 0)
|
|
114
132
|
return { empty: true, block: "", report: [] };
|
|
115
133
|
// Same incremental token cap pattern as checkpoint recall.
|
|
116
134
|
const parts = [];
|
|
117
135
|
const report = [];
|
|
118
136
|
let blockTokens = 0;
|
|
119
|
-
|
|
120
|
-
const part = formatMemoryRecallBlock([
|
|
121
|
-
{ content: h.memory.content, category: h.memory.category, score: h.score },
|
|
122
|
-
]);
|
|
137
|
+
const pushHit = (content, category, score, label) => {
|
|
138
|
+
const part = formatMemoryRecallBlock([{ content, category, score }]);
|
|
123
139
|
const partTokens = estimateBlockTokens(part);
|
|
124
140
|
if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
|
|
125
|
-
|
|
141
|
+
return false;
|
|
126
142
|
parts.push(part);
|
|
127
|
-
report.push(` •
|
|
143
|
+
report.push(` • ${label} (${(score * 100).toFixed(0)}%): ${content.slice(0, 60).replace(/\n/g, " ")}…`);
|
|
128
144
|
blockTokens += partTokens;
|
|
145
|
+
return true;
|
|
146
|
+
};
|
|
147
|
+
for (const h of hits) {
|
|
148
|
+
if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id}`))
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
for (const h of crossHits) {
|
|
152
|
+
const repoLabel = h.repoId.split(/[\\/]/).filter(Boolean).pop() ?? h.repoId;
|
|
153
|
+
if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id} (from ${repoLabel})`))
|
|
154
|
+
break;
|
|
129
155
|
}
|
|
130
156
|
return { empty: parts.length === 0, block: parts.join("\n"), report };
|
|
131
157
|
}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* memoryIndex.ts — cross-repo async vector index for durable memories (S24).
|
|
3
|
+
*
|
|
4
|
+
* A REDUNDANT, additive, ASYNC index layered over the authoritative node:sqlite
|
|
5
|
+
* `memories` table. The same-repo linear cosine scan over the in-repo memories
|
|
6
|
+
* (src/memoryRecall.ts) stays the DEFAULT recall path; this global PGlite index
|
|
7
|
+
* exists only to provide real cross-repo nearest-neighbor memory recall — so a
|
|
8
|
+
* decision you saved in repo A can be inlined as RAG context when you start a
|
|
9
|
+
* session in repo B. It is best-effort and non-fatal: any init/write failure
|
|
10
|
+
* degrades to the same-repo scan and must NEVER break memory write, recall, or
|
|
11
|
+
* extension load.
|
|
12
|
+
*
|
|
13
|
+
* PREVENT-PI-004: PGlite is WASM Postgres — fully local, zero network. Memory
|
|
14
|
+
* remains AUTHORITATIVE in SQLite; this index only holds (repo_id, memory_id,
|
|
15
|
+
* content, embedding) for NN lookup and is rebuilt from SQLite at any time.
|
|
16
|
+
*
|
|
17
|
+
* Topology mirrors vectorIndex.ts (Slice 2): ONE global PGlite DB, `repo_id` is
|
|
18
|
+
* a first-class column. `searchMemoriesAsync(q, k, {repoId?})` → omit repoId for
|
|
19
|
+
* cross-repo NN, pass repoId to scope to a single repo. Hit content is stored
|
|
20
|
+
* inline because the recall process cannot open every other repo's SQLite dir.
|
|
21
|
+
*/
|
|
22
|
+
import { homedir } from "node:os";
|
|
23
|
+
import { join } from "node:path";
|
|
24
|
+
import { mkdirSync, rmSync, existsSync } from "node:fs";
|
|
25
|
+
/** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
|
|
26
|
+
export const MEMORY_INDEX_DIM = 512;
|
|
27
|
+
let db;
|
|
28
|
+
let initPromise;
|
|
29
|
+
let disabled = false;
|
|
30
|
+
let warned = false;
|
|
31
|
+
/** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
|
|
32
|
+
let pgliteMod;
|
|
33
|
+
let pgliteLoadFailed = false;
|
|
34
|
+
function indexDir() {
|
|
35
|
+
const override = process.env.MEGACOMPACT_INDEX_DIR;
|
|
36
|
+
if (override && override.trim() !== "")
|
|
37
|
+
return join(override, "memory");
|
|
38
|
+
try {
|
|
39
|
+
return join(homedir(), ".pi", "mega-compact-vector", "memory");
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return join("/tmp", ".mega-compact-vector", "memory");
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function logWarn(msg) {
|
|
46
|
+
// Never throw — degradation is the whole point. One warning per process.
|
|
47
|
+
if (warned)
|
|
48
|
+
return;
|
|
49
|
+
warned = true;
|
|
50
|
+
try {
|
|
51
|
+
console.warn(`[mega-compact:memoryIndex] ${msg} (falling back to same-repo scan)`);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
/* ignore */
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** Honor the emergency kill-switch (shared with the checkpoint index). */
|
|
58
|
+
export function isMemoryIndexDisabled() {
|
|
59
|
+
return (disabled ||
|
|
60
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED === "true" ||
|
|
61
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED === "1");
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Lazily open + schema-init the global PGlite DB. Idempotent and safe to call
|
|
65
|
+
* from many places. Returns undefined when disabled/unavailable so callers can
|
|
66
|
+
* fall back to the synchronous scan. Never throws.
|
|
67
|
+
*/
|
|
68
|
+
export function initMemoryIndex() {
|
|
69
|
+
if (isMemoryIndexDisabled())
|
|
70
|
+
return Promise.resolve(undefined);
|
|
71
|
+
if (db)
|
|
72
|
+
return Promise.resolve(db);
|
|
73
|
+
if (initPromise)
|
|
74
|
+
return initPromise;
|
|
75
|
+
initPromise = openPgLite(/* retryOnCorrupt */ true);
|
|
76
|
+
return initPromise;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Lazily load the PGlite module + pgvector extension via dynamic import. Caches
|
|
80
|
+
* success and permanent failure. Returns undefined (once, then forever) when the
|
|
81
|
+
* package is missing/broken so callers fall back to the same-repo scan. Never throws.
|
|
82
|
+
*/
|
|
83
|
+
async function loadPgLite() {
|
|
84
|
+
if (pgliteMod)
|
|
85
|
+
return pgliteMod;
|
|
86
|
+
if (pgliteLoadFailed)
|
|
87
|
+
return undefined;
|
|
88
|
+
try {
|
|
89
|
+
const [pglitePkg, pgvectorPkg] = await Promise.all([
|
|
90
|
+
import("@electric-sql/pglite"),
|
|
91
|
+
import("@electric-sql/pglite-pgvector"),
|
|
92
|
+
]);
|
|
93
|
+
pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
|
|
94
|
+
return pgliteMod;
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
pgliteLoadFailed = true;
|
|
98
|
+
logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level abort
|
|
104
|
+
* (typically from a corrupted/torn data dir) triggers a delete + one retry.
|
|
105
|
+
*/
|
|
106
|
+
async function openPgLite(retryOnCorrupt) {
|
|
107
|
+
try {
|
|
108
|
+
const mod = await loadPgLite();
|
|
109
|
+
if (!mod)
|
|
110
|
+
return undefined;
|
|
111
|
+
const dir = indexDir();
|
|
112
|
+
mkdirSync(dir, { recursive: true });
|
|
113
|
+
const pg = await new mod.PGlite({
|
|
114
|
+
dataDir: dir,
|
|
115
|
+
extensions: { vector: mod.vector },
|
|
116
|
+
});
|
|
117
|
+
await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
|
|
118
|
+
await pg.exec(`
|
|
119
|
+
CREATE TABLE IF NOT EXISTS memory_index (
|
|
120
|
+
repo_id TEXT NOT NULL,
|
|
121
|
+
memory_id INTEGER NOT NULL,
|
|
122
|
+
content TEXT NOT NULL,
|
|
123
|
+
embedding vector(${MEMORY_INDEX_DIM}) NOT NULL,
|
|
124
|
+
PRIMARY KEY (repo_id, memory_id)
|
|
125
|
+
);
|
|
126
|
+
`);
|
|
127
|
+
await pg.exec("CREATE INDEX IF NOT EXISTS memory_index_hnsw ON memory_index USING hnsw (embedding vector_cosine_ops);");
|
|
128
|
+
db = pg;
|
|
129
|
+
return pg;
|
|
130
|
+
}
|
|
131
|
+
catch (err) {
|
|
132
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
133
|
+
if (retryOnCorrupt && (msg.includes("Aborted") || msg.includes("RuntimeError"))) {
|
|
134
|
+
try {
|
|
135
|
+
const dir = indexDir();
|
|
136
|
+
if (existsSync(dir))
|
|
137
|
+
rmSync(dir, { recursive: true, force: true });
|
|
138
|
+
initPromise = undefined;
|
|
139
|
+
return openPgLite(/* retryOnCorrupt */ false);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
/* self-heal failed — fall through to disable */
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
disabled = true;
|
|
146
|
+
logWarn(`init failed: ${msg}`);
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
function toVectorLiteral(v) {
|
|
151
|
+
const parts = v.map((x) => (Number.isFinite(x) ? x : 0));
|
|
152
|
+
return `[${parts.join(",")}]`;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Best-effort upsert of one memory embedding into the global index.
|
|
156
|
+
* Dimension-mismatched vectors (e.g. a BYO embedder with dim ≠ 512) are skipped.
|
|
157
|
+
* Fire-and-forget: callers must NOT await this on the sync write path. Never
|
|
158
|
+
* throws. `content` is stored inline so cross-repo recall can read it directly.
|
|
159
|
+
*/
|
|
160
|
+
export async function upsertMemoryEmbedding(repoId, memoryId, content, embedding) {
|
|
161
|
+
if (isMemoryIndexDisabled())
|
|
162
|
+
return;
|
|
163
|
+
if (!embedding || embedding.length !== MEMORY_INDEX_DIM)
|
|
164
|
+
return;
|
|
165
|
+
try {
|
|
166
|
+
const pg = await initMemoryIndex();
|
|
167
|
+
if (!pg)
|
|
168
|
+
return;
|
|
169
|
+
const lit = toVectorLiteral(embedding);
|
|
170
|
+
await pg.query(`INSERT INTO memory_index (repo_id, memory_id, content, embedding)
|
|
171
|
+
VALUES ($1, $2, $3, $4::vector)
|
|
172
|
+
ON CONFLICT (repo_id, memory_id)
|
|
173
|
+
DO UPDATE SET content = EXCLUDED.content, embedding = EXCLUDED.embedding;`, [repoId, memoryId, content, lit]);
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
disabled = true;
|
|
177
|
+
logWarn(`upsert failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Cross-repo (or single-repo) HNSW nearest-neighbor memory search. Returns hits
|
|
182
|
+
* sorted by descending similarity. Never throws — on any failure returns [].
|
|
183
|
+
*/
|
|
184
|
+
export async function searchMemoriesAsync(query, opts = {}) {
|
|
185
|
+
if (isMemoryIndexDisabled() || !query || query.length !== MEMORY_INDEX_DIM)
|
|
186
|
+
return [];
|
|
187
|
+
const k = opts.k ?? 5;
|
|
188
|
+
const repoId = opts.repoId;
|
|
189
|
+
try {
|
|
190
|
+
const pg = await initMemoryIndex();
|
|
191
|
+
if (!pg)
|
|
192
|
+
return [];
|
|
193
|
+
const lit = toVectorLiteral(query);
|
|
194
|
+
const params = [lit, k];
|
|
195
|
+
let sql = "SELECT repo_id, memory_id, content, 1 - (embedding <=> $1::vector) AS score " +
|
|
196
|
+
"FROM memory_index";
|
|
197
|
+
if (repoId) {
|
|
198
|
+
sql += " WHERE repo_id = $3";
|
|
199
|
+
params.push(repoId);
|
|
200
|
+
}
|
|
201
|
+
sql += " ORDER BY embedding <=> $1::vector LIMIT $2";
|
|
202
|
+
const res = await pg.query(sql, params);
|
|
203
|
+
return res.rows.map((r) => ({
|
|
204
|
+
repoId: r.repo_id,
|
|
205
|
+
memoryId: Number(r.memory_id),
|
|
206
|
+
content: r.content,
|
|
207
|
+
score: r.score,
|
|
208
|
+
}));
|
|
209
|
+
}
|
|
210
|
+
catch (err) {
|
|
211
|
+
disabled = true;
|
|
212
|
+
logWarn(`search failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
213
|
+
return [];
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
/** Close the index (test teardown / shutdown). Safe to call when unopened. */
|
|
217
|
+
export async function closeMemoryIndex() {
|
|
218
|
+
if (db) {
|
|
219
|
+
try {
|
|
220
|
+
await db.close();
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
/* ignore */
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
db = undefined;
|
|
227
|
+
initPromise = undefined;
|
|
228
|
+
disabled = false;
|
|
229
|
+
warned = false;
|
|
230
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { defaultEmbedder } from "../embedder.js";
|
|
7
|
+
import { upsertMemoryEmbedding, searchMemoriesAsync, initMemoryIndex, closeMemoryIndex, isMemoryIndexDisabled, } from "./memoryIndex.js";
|
|
8
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-memidx-"));
|
|
9
|
+
test("memoryIndex: disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
|
|
10
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
|
|
11
|
+
try {
|
|
12
|
+
assert.equal(isMemoryIndexDisabled(), true, "kill-switch honored");
|
|
13
|
+
const hits = await searchMemoriesAsync(defaultEmbedder().embed("anything"), { k: 3 });
|
|
14
|
+
assert.deepEqual(hits, [], "search returns [] when disabled");
|
|
15
|
+
}
|
|
16
|
+
finally {
|
|
17
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
test("memoryIndex: cross-repo upsert + NN search returns the right repo's memory", async () => {
|
|
21
|
+
// Isolate the global PGlite dir so concurrent test runs don't collide.
|
|
22
|
+
process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
|
|
23
|
+
const repoA = "/tmp/repo-a";
|
|
24
|
+
const repoB = "/tmp/repo-b";
|
|
25
|
+
try {
|
|
26
|
+
await initMemoryIndex();
|
|
27
|
+
// Two memories in different repos, with clearly distinct content so their
|
|
28
|
+
// trigram embeddings separate.
|
|
29
|
+
const vecA = defaultEmbedder().embed("We standardized on node:sqlite for the store backend");
|
|
30
|
+
const vecB = defaultEmbedder().embed("The deployment target is a raspberry pi in the closet");
|
|
31
|
+
await upsertMemoryEmbedding(repoA, 1, "We standardized on node:sqlite for the store backend", vecA);
|
|
32
|
+
await upsertMemoryEmbedding(repoB, 7, "The deployment target is a raspberry pi in the closet", vecB);
|
|
33
|
+
// Query close to A's content → top hit should be A's memory, not B's.
|
|
34
|
+
const q = defaultEmbedder().embed("standardized node:sqlite store backend choice");
|
|
35
|
+
const hits = await searchMemoriesAsync(q, { k: 3 });
|
|
36
|
+
assert.ok(hits.length >= 1, "at least one hit");
|
|
37
|
+
assert.equal(hits[0].repoId, repoA, "nearest neighbor is repo A");
|
|
38
|
+
assert.equal(hits[0].memoryId, 1, "correct memory id");
|
|
39
|
+
assert.ok(hits[0].score > 0.5, "high cosine for the matching memory");
|
|
40
|
+
// Scope to repoB only → A must not appear.
|
|
41
|
+
const scoped = await searchMemoriesAsync(q, { k: 3, repoId: repoB });
|
|
42
|
+
assert.ok(scoped.every((h) => h.repoId === repoB), "scoped search stays within repoB");
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
await closeMemoryIndex();
|
|
46
|
+
delete process.env.MEGACOMPACT_INDEX_DIR;
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
test("memoryIndex: cleanup", () => {
|
|
50
|
+
rmSync(baseTmp, { recursive: true, force: true });
|
|
51
|
+
});
|
|
@@ -18,17 +18,15 @@
|
|
|
18
18
|
import { homedir } from "node:os";
|
|
19
19
|
import { join } from "node:path";
|
|
20
20
|
import { mkdirSync, rmSync, existsSync } from "node:fs";
|
|
21
|
-
// PGlite + pgvector are script-free WASM (no native build) → survive pi's
|
|
22
|
-
// install-script block. Imported lazily so a missing/broken package degrades
|
|
23
|
-
// gracefully instead of crashing module load.
|
|
24
|
-
import { PGlite } from "@electric-sql/pglite";
|
|
25
|
-
import { vector } from "@electric-sql/pglite-pgvector";
|
|
26
21
|
/** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
|
|
27
22
|
export const EMBEDDING_DIM = 512;
|
|
28
23
|
let db;
|
|
29
24
|
let initPromise;
|
|
30
25
|
let disabled = false;
|
|
31
26
|
let warned = false;
|
|
27
|
+
/** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
|
|
28
|
+
let pgliteMod;
|
|
29
|
+
let pgliteLoadFailed = false;
|
|
32
30
|
function indexDir() {
|
|
33
31
|
const override = process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
34
32
|
if (override && override.trim() !== "")
|
|
@@ -73,6 +71,30 @@ export function initVectorIndex() {
|
|
|
73
71
|
initPromise = openPgLite(/* retryOnCorrupt */ true);
|
|
74
72
|
return initPromise;
|
|
75
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Lazily load the PGlite module + pgvector extension via dynamic import. Caches
|
|
76
|
+
* success and permanent failure. Returns undefined (once, then forever) when the
|
|
77
|
+
* package is missing/broken so callers fall back to the sync scan. Never throws.
|
|
78
|
+
*/
|
|
79
|
+
async function loadPgLite() {
|
|
80
|
+
if (pgliteMod)
|
|
81
|
+
return pgliteMod;
|
|
82
|
+
if (pgliteLoadFailed)
|
|
83
|
+
return undefined;
|
|
84
|
+
try {
|
|
85
|
+
const [pglitePkg, pgvectorPkg] = await Promise.all([
|
|
86
|
+
import("@electric-sql/pglite"),
|
|
87
|
+
import("@electric-sql/pglite-pgvector"),
|
|
88
|
+
]);
|
|
89
|
+
pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
|
|
90
|
+
return pgliteMod;
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
pgliteLoadFailed = true;
|
|
94
|
+
logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
95
|
+
return undefined;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
76
98
|
/**
|
|
77
99
|
* Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level
|
|
78
100
|
* abort (typically from a corrupted/torn data dir) triggers a delete + one
|
|
@@ -80,11 +102,14 @@ export function initVectorIndex() {
|
|
|
80
102
|
*/
|
|
81
103
|
async function openPgLite(retryOnCorrupt) {
|
|
82
104
|
try {
|
|
105
|
+
const mod = await loadPgLite();
|
|
106
|
+
if (!mod)
|
|
107
|
+
return undefined;
|
|
83
108
|
const dir = indexDir();
|
|
84
109
|
mkdirSync(dir, { recursive: true });
|
|
85
|
-
const pg = await new PGlite({
|
|
110
|
+
const pg = await new mod.PGlite({
|
|
86
111
|
dataDir: dir,
|
|
87
|
-
extensions: { vector },
|
|
112
|
+
extensions: { vector: mod.vector },
|
|
88
113
|
});
|
|
89
114
|
await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
|
|
90
115
|
await pg.exec(`
|