pi-mega-compact 0.4.28 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -2
- package/dist/extensions/dashboard-server.js +66 -3
- package/dist/extensions/dashboard-server.test.js +95 -3
- package/dist/extensions/mega-commands.js +25 -9
- package/dist/extensions/mega-compact.test.js +133 -31
- package/dist/extensions/mega-config.js +5 -0
- package/dist/extensions/mega-conflict-cmds.js +79 -0
- package/dist/extensions/mega-dashboard-cmds.js +6 -4
- package/dist/extensions/mega-events.js +144 -27
- package/dist/extensions/mega-pipeline.js +84 -1
- package/dist/extensions/mega-runtime.js +35 -2
- package/dist/extensions/mega-trim.js +48 -0
- package/dist/extensions/mega-trim.test.js +58 -0
- package/dist/src/config/dedup.js +1 -0
- package/dist/src/driftDetection.js +103 -0
- package/dist/src/driftDetection.test.js +87 -0
- package/dist/src/memory.js +147 -0
- package/dist/src/memory.test.js +41 -0
- package/dist/src/memoryConsolidate.test.js +38 -0
- package/dist/src/memoryOps.js +58 -0
- package/dist/src/memoryOps.test.js +41 -0
- package/dist/src/memoryRecall.js +60 -0
- package/dist/src/memoryRecall.test.js +92 -0
- package/dist/src/recall.js +70 -1
- package/dist/src/recall.test.js +69 -1
- package/dist/src/store/sqlite.js +127 -11
- package/dist/src/vectorStore.js +6 -1
- package/extensions/dashboard-server.test.ts +115 -3
- package/extensions/dashboard-server.ts +69 -4
- package/extensions/mega-commands.ts +24 -9
- package/extensions/mega-compact.test.ts +134 -31
- package/extensions/mega-config.ts +22 -0
- package/extensions/mega-conflict-cmds.ts +81 -0
- package/extensions/mega-dashboard-cmds.ts +6 -4
- package/extensions/mega-events.ts +139 -28
- package/extensions/mega-pipeline.ts +94 -1
- package/extensions/mega-runtime.ts +35 -2
- package/extensions/mega-trim.test.ts +64 -0
- package/extensions/mega-trim.ts +75 -0
- package/extensions/openclaw-mega-compact.ts +24 -9
- package/package.json +2 -2
- package/src/config/dedup.ts +2 -0
- package/src/driftDetection.test.ts +100 -0
- package/src/driftDetection.ts +136 -0
- package/src/memory.test.ts +46 -0
- package/src/memory.ts +164 -0
- package/src/memoryConsolidate.test.ts +47 -0
- package/src/memoryOps.test.ts +53 -0
- package/src/memoryOps.ts +75 -0
- package/src/memoryRecall.test.ts +100 -0
- package/src/memoryRecall.ts +83 -0
- package/src/recall.test.ts +77 -1
- package/src/recall.ts +94 -1
- package/src/store/sqlite.ts +188 -11
- package/src/store.ts +3 -0
- package/src/vectorStore.ts +10 -1
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { reviewConversation } from "./memory.js";
|
|
4
|
+
test("reviewConversation: yields an ADD op for a stated decision", () => {
|
|
5
|
+
const msgs = [
|
|
6
|
+
{ role: "user", text: "we use node:sqlite as the store" },
|
|
7
|
+
{ role: "assistant", text: "got it, node:sqlite is the source of truth" },
|
|
8
|
+
];
|
|
9
|
+
const ops = reviewConversation(msgs);
|
|
10
|
+
assert.ok(ops.some((o) => o.op === "add" && /sqlite|store/i.test(o.memory.content)), "adds a decision memory");
|
|
11
|
+
});
|
|
12
|
+
test("reviewConversation: REPLACE when a later message contradicts an earlier one", () => {
|
|
13
|
+
const msgs = [
|
|
14
|
+
{ role: "user", text: "the threshold is 50k" },
|
|
15
|
+
{ role: "assistant", text: "ok 50k threshold" },
|
|
16
|
+
{ role: "user", text: "actually raise the threshold to 100k" },
|
|
17
|
+
];
|
|
18
|
+
const ops = reviewConversation(msgs);
|
|
19
|
+
assert.ok(ops.some((o) => o.op === "replace"), "replaces the superseded value");
|
|
20
|
+
});
|
|
21
|
+
test("reviewConversation: no ops on pure smalltalk (no durable fact)", () => {
|
|
22
|
+
const msgs = [{ role: "user", text: "hi" }, { role: "assistant", text: "hey" }];
|
|
23
|
+
assert.equal(reviewConversation(msgs).length, 0);
|
|
24
|
+
});
|
|
25
|
+
test("reviewConversation: emits REMOVE when a user asks to drop an existing memory", () => {
|
|
26
|
+
const existing = [{ content: "we use node:sqlite as the store" }];
|
|
27
|
+
// Plain drop statement — no "switch to …" phrasing so we don't accidentally
|
|
28
|
+
// match DECISION_PATTERNS and route into the replace branch instead.
|
|
29
|
+
const msgs = [
|
|
30
|
+
{ role: "user", text: "stop using node:sqlite for the store — drop it from memory" },
|
|
31
|
+
{ role: "assistant", text: "ok dropped" },
|
|
32
|
+
];
|
|
33
|
+
const ops = reviewConversation(msgs, existing);
|
|
34
|
+
assert.ok(ops.some((o) => o.op === "remove" && /sqlite/i.test(o.content)), "emits a remove op targeting the old memory");
|
|
35
|
+
});
|
|
36
|
+
test("reviewConversation: REMOVE requires topic overlap (no accidental drop)", () => {
|
|
37
|
+
const existing = [{ content: "the timezone is America/Los_Angeles" }];
|
|
38
|
+
const msgs = [{ role: "user", text: "drop it" }];
|
|
39
|
+
const ops = reviewConversation(msgs, existing);
|
|
40
|
+
assert.equal(ops.filter((o) => o.op === "remove").length, 0, "vague 'drop it' with no topic overlap does not remove anything");
|
|
41
|
+
});
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { consolidateMemories } from "./memory.js";
|
|
7
|
+
import { addMemory, listMemories } from "./store/sqlite.js";
|
|
8
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-consolidate-"));
|
|
9
|
+
test("consolidateMemories: merges near-duplicate memories (one kept, one removed)", async () => {
|
|
10
|
+
const dir = join(baseTmp, "merge");
|
|
11
|
+
// Two near-identical memories — same topic, minor wording drift.
|
|
12
|
+
addMemory({ content: "we use node:sqlite as the store", category: "decision" }, null, dir);
|
|
13
|
+
addMemory({ content: "we use node:sqlite for our store", category: "decision" }, null, dir);
|
|
14
|
+
const merged = await consolidateMemories(dir);
|
|
15
|
+
assert.equal(merged, 1, "one merge was performed");
|
|
16
|
+
const rows = listMemories(null, 50, dir);
|
|
17
|
+
// Exactly one of the two near-dupes survives.
|
|
18
|
+
assert.equal(rows.length, 1, "exactly one memory remains after merge");
|
|
19
|
+
assert.match(rows[0].content, /node:sqlite.*store/i, "survivor mentions node:sqlite + store");
|
|
20
|
+
});
|
|
21
|
+
test("consolidateMemories: leaves unrelated memories alone", async () => {
|
|
22
|
+
const dir = join(baseTmp, "noop");
|
|
23
|
+
addMemory({ content: "we use node:sqlite as the store", category: "decision" }, null, dir);
|
|
24
|
+
addMemory({ content: "the gpt-5 release is on hold pending benchmark results", category: "note" }, null, dir);
|
|
25
|
+
const merged = await consolidateMemories(dir);
|
|
26
|
+
assert.equal(merged, 0, "no merges on unrelated memories");
|
|
27
|
+
const rows = listMemories(null, 50, dir);
|
|
28
|
+
assert.equal(rows.length, 2, "both memories survive");
|
|
29
|
+
});
|
|
30
|
+
test("consolidateMemories: empty store returns 0 and changes nothing", async () => {
|
|
31
|
+
const dir = join(baseTmp, "empty");
|
|
32
|
+
const merged = await consolidateMemories(dir);
|
|
33
|
+
assert.equal(merged, 0, "no merges on empty store");
|
|
34
|
+
assert.equal(listMemories(null, 50, dir).length, 0, "store still empty");
|
|
35
|
+
});
|
|
36
|
+
test("cleanup memops", () => {
|
|
37
|
+
rmSync(baseTmp, { recursive: true, force: true });
|
|
38
|
+
});
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { addMemory, listMemories, replaceMemory, removeMemory, } from "./store/sqlite.js";
|
|
2
|
+
/** Find a memory row whose content exactly matches (case-insensitive). */
|
|
3
|
+
function findByContent(memories, content) {
|
|
4
|
+
const norm = content.trim().toLowerCase();
|
|
5
|
+
return memories.find((m) => m.content.trim().toLowerCase() === norm);
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Apply add/replace/remove ops to the memories table. Replaces are matched by
|
|
9
|
+
* existing content; removes by content. Idempotent: an add that already exists
|
|
10
|
+
* is a no-op, a replace of a missing memory degrades to an add.
|
|
11
|
+
*/
|
|
12
|
+
export async function applyMemoryOps(ops, stateDir) {
|
|
13
|
+
if (!ops.length)
|
|
14
|
+
return;
|
|
15
|
+
const repo = null; // memories are repo-scoped by stateDir, not by the repo arg here
|
|
16
|
+
const existing = listMemories(repo, 1000, stateDir);
|
|
17
|
+
for (const op of ops) {
|
|
18
|
+
if (op.op === "add") {
|
|
19
|
+
// Skip if an identical memory already exists.
|
|
20
|
+
if (findByContent(existing, op.memory.content))
|
|
21
|
+
continue;
|
|
22
|
+
addMemory({
|
|
23
|
+
kind: op.memory.category,
|
|
24
|
+
content: op.memory.content,
|
|
25
|
+
tags: [],
|
|
26
|
+
category: op.memory.category,
|
|
27
|
+
target: op.memory.target,
|
|
28
|
+
sourceTurn: op.memory.sourceTurn,
|
|
29
|
+
}, repo, stateDir);
|
|
30
|
+
}
|
|
31
|
+
else if (op.op === "replace") {
|
|
32
|
+
const match = findByContent(existing, op.targetContent);
|
|
33
|
+
if (match) {
|
|
34
|
+
replaceMemory(match.id, {
|
|
35
|
+
kind: op.memory.category,
|
|
36
|
+
content: op.memory.content,
|
|
37
|
+
category: op.memory.category,
|
|
38
|
+
sourceTurn: op.memory.sourceTurn,
|
|
39
|
+
}, stateDir);
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
// Target missing (e.g. earlier in-conversation contradiction) → add.
|
|
43
|
+
addMemory({
|
|
44
|
+
kind: op.memory.category,
|
|
45
|
+
content: op.memory.content,
|
|
46
|
+
tags: [],
|
|
47
|
+
category: op.memory.category,
|
|
48
|
+
sourceTurn: op.memory.sourceTurn,
|
|
49
|
+
}, repo, stateDir);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
const match = findByContent(existing, op.content);
|
|
54
|
+
if (match)
|
|
55
|
+
removeMemory(match.id, stateDir);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { applyMemoryOps } from "./memoryOps.js";
|
|
7
|
+
import { addMemory, listMemories } from "./store/sqlite.js";
|
|
8
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-memops-"));
|
|
9
|
+
test("applyMemoryOps: ADD inserts a new memory", async () => {
|
|
10
|
+
const dir = join(baseTmp, "add");
|
|
11
|
+
await applyMemoryOps([{ op: "add", memory: { content: "we use node:sqlite as the store", category: "decision", sourceTurn: 0 } }], dir);
|
|
12
|
+
const rows = listMemories(null, 50, dir);
|
|
13
|
+
assert.ok(rows.some((m) => /node:sqlite/.test(m.content)), "added memory present");
|
|
14
|
+
assert.equal(rows[0].category, "decision", "category persisted");
|
|
15
|
+
});
|
|
16
|
+
test("applyMemoryOps: ADD is idempotent (no duplicate)", async () => {
|
|
17
|
+
const dir = join(baseTmp, "dup");
|
|
18
|
+
const op = { op: "add", memory: { content: "threshold is 50k", category: "decision", sourceTurn: 0 } };
|
|
19
|
+
await applyMemoryOps([op], dir);
|
|
20
|
+
await applyMemoryOps([op], dir);
|
|
21
|
+
const rows = listMemories(null, 50, dir);
|
|
22
|
+
assert.equal(rows.filter((m) => /threshold is 50k/.test(m.content)).length, 1, "no duplicate");
|
|
23
|
+
});
|
|
24
|
+
test("applyMemoryOps: REPLACE updates the matching memory", async () => {
|
|
25
|
+
const dir = join(baseTmp, "replace");
|
|
26
|
+
addMemory({ content: "the threshold is 50k", category: "decision" }, null, dir);
|
|
27
|
+
await applyMemoryOps([{ op: "replace", targetContent: "the threshold is 50k", memory: { content: "the threshold is 100k", category: "decision", sourceTurn: 2 } }], dir);
|
|
28
|
+
const rows = listMemories(null, 50, dir);
|
|
29
|
+
assert.ok(rows.some((m) => /100k/.test(m.content)), "replaced content present");
|
|
30
|
+
assert.ok(!rows.some((m) => /50k/.test(m.content)), "old content gone");
|
|
31
|
+
});
|
|
32
|
+
test("applyMemoryOps: REMOVE deletes the matching memory", async () => {
|
|
33
|
+
const dir = join(baseTmp, "remove");
|
|
34
|
+
addMemory({ content: "obsolete note", category: "note" }, null, dir);
|
|
35
|
+
await applyMemoryOps([{ op: "remove", content: "obsolete note" }], dir);
|
|
36
|
+
const rows = listMemories(null, 50, dir);
|
|
37
|
+
assert.ok(!rows.some((m) => /obsolete note/.test(m.content)), "removed");
|
|
38
|
+
});
|
|
39
|
+
test("cleanup memops", () => {
|
|
40
|
+
rmSync(baseTmp, { recursive: true, force: true });
|
|
41
|
+
});
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* memoryRecall.ts — semantic recall over the durable memories table (S21).
|
|
3
|
+
* Embeds a query with the same local embedder used by RAPTOR, ranks every
|
|
4
|
+
* memory in the current repo's SQLite by cosine similarity combined with a
|
|
5
|
+
* category-weighted + recency-boosted score, and returns the top-k. Side
|
|
6
|
+
* effect: marks returned memories as referenced (last_referenced) so drift
|
|
7
|
+
* can be measured. PREVENT-PI-004 — embedder is the same local one used
|
|
8
|
+
* everywhere else; no remote calls are introduced here.
|
|
9
|
+
* @module
|
|
10
|
+
*/
|
|
11
|
+
import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
|
|
12
|
+
import { listMemories, referenceMemory } from "./store/sqlite.js";
|
|
13
|
+
/** Default category weights — `decision` wins ties on tied cosine. */
|
|
14
|
+
export const DEFAULT_CATEGORY_WEIGHTS = {
|
|
15
|
+
decision: 1.10,
|
|
16
|
+
preference: 1.05,
|
|
17
|
+
fact: 1.0,
|
|
18
|
+
};
|
|
19
|
+
const DEFAULT_RECENCY_WEIGHT = 0.05;
|
|
20
|
+
/**
|
|
21
|
+
* Rank memories by a blended score of cosine similarity, category weight, and
|
|
22
|
+
* recency of last_referenced (with createdAt as a fallback). Returns the
|
|
23
|
+
* top-k above minSimilarity, sorted by blended score descending. Empty on
|
|
24
|
+
* no matches.
|
|
25
|
+
*/
|
|
26
|
+
export async function recallMemories(query, stateDir, opts = {}) {
|
|
27
|
+
const topK = opts.topK ?? 10;
|
|
28
|
+
const minSimilarity = opts.minSimilarity ?? 0.2;
|
|
29
|
+
const markReferenced = opts.markReferenced ?? true;
|
|
30
|
+
const embedder = opts.embedder ?? defaultEmbedder();
|
|
31
|
+
const repo = opts.repo === undefined ? null : opts.repo;
|
|
32
|
+
const categoryWeights = { ...DEFAULT_CATEGORY_WEIGHTS, ...(opts.categoryWeights ?? {}) };
|
|
33
|
+
const recencyWeight = opts.recencyWeight ?? DEFAULT_RECENCY_WEIGHT;
|
|
34
|
+
const queryVec = embedder.embed(query);
|
|
35
|
+
const memories = listMemories(repo, 1000, stateDir);
|
|
36
|
+
if (!memories.length || !query.trim())
|
|
37
|
+
return [];
|
|
38
|
+
const nowSec = Math.floor(Date.now() / 1000);
|
|
39
|
+
const scored = [];
|
|
40
|
+
for (const mem of memories) {
|
|
41
|
+
const vec = embedder.embed(mem.content);
|
|
42
|
+
const sim = cosineSimilarity(queryVec, vec);
|
|
43
|
+
if (sim < minSimilarity)
|
|
44
|
+
continue;
|
|
45
|
+
const categoryW = categoryWeights[mem.category ?? "fact"] ?? 1.0;
|
|
46
|
+
const lastTouch = mem.lastReferenced ?? mem.createdAt ?? nowSec;
|
|
47
|
+
// log(1 + daysSince) grows slowly — half-life-style decay.
|
|
48
|
+
const daysSince = Math.max(0, (nowSec - lastTouch) / 86_400);
|
|
49
|
+
const recencyBoost = Math.log1p(daysSince) * recencyWeight;
|
|
50
|
+
const finalScore = sim * categoryW * (1 + recencyBoost);
|
|
51
|
+
scored.push({ memory: mem, score: finalScore });
|
|
52
|
+
}
|
|
53
|
+
scored.sort((a, b) => b.score - a.score);
|
|
54
|
+
const top = scored.slice(0, topK);
|
|
55
|
+
if (markReferenced) {
|
|
56
|
+
for (const hit of top)
|
|
57
|
+
referenceMemory(hit.memory.id, stateDir);
|
|
58
|
+
}
|
|
59
|
+
return top;
|
|
60
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { recallMemories } from "./memoryRecall.js";
|
|
7
|
+
import { addMemory, getMemory } from "./store/sqlite.js";
|
|
8
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-memrec-"));
|
|
9
|
+
function biGramEmbedder() {
|
|
10
|
+
// Deterministic test embedder — bigger dim + binary encoding so two semantically
|
|
11
|
+
// related strings have higher cosine than unrelated ones.
|
|
12
|
+
const dim = 64;
|
|
13
|
+
return {
|
|
14
|
+
dim,
|
|
15
|
+
embed(text) {
|
|
16
|
+
const v = new Array(dim).fill(0);
|
|
17
|
+
const norm = text.toLowerCase().replace(/[^a-z0-9 ]/g, "").trim();
|
|
18
|
+
for (let i = 0; i < norm.length - 1; i++) {
|
|
19
|
+
const idx = ((norm.charCodeAt(i) * 31 + norm.charCodeAt(i + 1)) >>> 0) % dim;
|
|
20
|
+
v[idx] = 1;
|
|
21
|
+
}
|
|
22
|
+
return v;
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
test("recallMemories: ranks relevant memory above unrelated", async () => {
|
|
27
|
+
const dir = join(baseTmp, "rank");
|
|
28
|
+
addMemory({ content: "we use sqlite for the durable store", category: "decision" }, null, dir);
|
|
29
|
+
addMemory({ content: "the threshold is 100 thousand tokens", category: "decision" }, null, dir);
|
|
30
|
+
addMemory({ content: "katz says hi", category: "note" }, null, dir);
|
|
31
|
+
const hits = await recallMemories("what store do we use?", dir, {
|
|
32
|
+
embedder: biGramEmbedder(),
|
|
33
|
+
topK: 3,
|
|
34
|
+
minSimilarity: 0.0,
|
|
35
|
+
});
|
|
36
|
+
assert.ok(hits.length >= 2, "finds relevant");
|
|
37
|
+
assert.ok(/sqlite/.test(hits[0].memory.content), "top hit is sqlite one");
|
|
38
|
+
assert.ok(!/katz/.test(hits[0].memory.content), "unrelated not on top");
|
|
39
|
+
});
|
|
40
|
+
test("recallMemories: marks referenced hits (last_referenced updated)", async () => {
|
|
41
|
+
const dir = join(baseTmp, "ref");
|
|
42
|
+
addMemory({ content: "policy is local-only", category: "rule" }, null, dir);
|
|
43
|
+
const hits = await recallMemories("local-only policy", dir, { embedder: biGramEmbedder() });
|
|
44
|
+
assert.ok(hits.length >= 1);
|
|
45
|
+
const fresh = getMemory(hits[0].memory.id, dir);
|
|
46
|
+
assert.ok(fresh && fresh.lastReferenced && fresh.lastReferenced > 0, "lastReferenced set");
|
|
47
|
+
});
|
|
48
|
+
test("recallMemories: empty store returns []", async () => {
|
|
49
|
+
const dir = join(baseTmp, "empty");
|
|
50
|
+
const hits = await recallMemories("anything", dir, { embedder: biGramEmbedder() });
|
|
51
|
+
assert.deepEqual(hits, []);
|
|
52
|
+
});
|
|
53
|
+
test("recallMemories: decision category beats fact category at equal similarity", async () => {
|
|
54
|
+
const dir = join(baseTmp, "category");
|
|
55
|
+
// Two memories that share many bigrams with the query — both will have very
|
|
56
|
+
// similar cosine. The decision category should win because of categoryWeight.
|
|
57
|
+
addMemory({ content: "we use redis for cache" }, null, dir);
|
|
58
|
+
const aId = addMemory({ content: "we use redis as primary cache key", category: "fact" }, null, dir);
|
|
59
|
+
const dId = addMemory({ content: "we use redis as primary cache layer", category: "decision" }, null, dir);
|
|
60
|
+
const hits = await recallMemories("redis cache layer", dir, {
|
|
61
|
+
embedder: biGramEmbedder(),
|
|
62
|
+
topK: 3,
|
|
63
|
+
minSimilarity: 0.0,
|
|
64
|
+
});
|
|
65
|
+
assert.ok(hits.length >= 2);
|
|
66
|
+
assert.equal(hits[0].memory.id, dId, "decision-tagged memory outranks fact-tagged");
|
|
67
|
+
assert.notEqual(hits[0].memory.id, aId, "top is the decision row, not the unknown-category row");
|
|
68
|
+
});
|
|
69
|
+
test("recallMemories: fresher reference beats older at equal similarity", async () => {
|
|
70
|
+
const dir = join(baseTmp, "recency");
|
|
71
|
+
const aId = addMemory({ content: "we use redis for cache", category: "fact" }, null, dir);
|
|
72
|
+
const bId = addMemory({ content: "we use redis for cache layer", category: "fact" }, null, dir);
|
|
73
|
+
// Backdate aId so its last_referenced is older; bId stays fresh.
|
|
74
|
+
const { openStore, closeStore } = await import("./store/sqlite.js");
|
|
75
|
+
const db = openStore(dir);
|
|
76
|
+
const longAgo = Math.floor(Date.now() / 1000) - 30 * 86_400; // 30d
|
|
77
|
+
db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(longAgo, aId);
|
|
78
|
+
// closeStore evicts the cached handle — calling db.close() directly would
|
|
79
|
+
// leave a stale closed DB in the openStore cache and break subsequent
|
|
80
|
+
// callers (the "database is not open" failure under parallel runs).
|
|
81
|
+
closeStore(dir);
|
|
82
|
+
const hits = await recallMemories("redis cache layer", dir, {
|
|
83
|
+
embedder: biGramEmbedder(),
|
|
84
|
+
topK: 3,
|
|
85
|
+
minSimilarity: 0.0,
|
|
86
|
+
});
|
|
87
|
+
assert.ok(hits.length >= 2);
|
|
88
|
+
assert.equal(hits[0].memory.id, bId, "freshly-referenced memory outranks 30-day-old one");
|
|
89
|
+
});
|
|
90
|
+
test("cleanup memrec", () => {
|
|
91
|
+
rmSync(baseTmp, { recursive: true, force: true });
|
|
92
|
+
});
|
package/dist/src/recall.js
CHANGED
|
@@ -22,7 +22,11 @@ export function formatRecallBlock(hits) {
|
|
|
22
22
|
return "";
|
|
23
23
|
const parts = hits.map((h, i) => {
|
|
24
24
|
const score = (h.score * 100).toFixed(0);
|
|
25
|
-
|
|
25
|
+
// S17: label a cross-repo hit with its source repo (the repoId doubles as
|
|
26
|
+
// that repo's stateDir, so the last path segment is the repo's display
|
|
27
|
+
// name). Same-repo hits (no repoId) stay unlabeled.
|
|
28
|
+
const repoName = h.repoId ? ` (from repo ${h.repoId.split("/").filter(Boolean).pop() ?? h.repoId})` : "";
|
|
29
|
+
return (`### Recalled context [${i + 1}] (relevance ${score}%)${repoName}\n` +
|
|
26
30
|
`${h.checkpoint.summary.trim()}\n` +
|
|
27
31
|
(h.checkpoint.filesModified.length
|
|
28
32
|
? `Key files: ${h.checkpoint.filesModified.join(", ")}.\n`
|
|
@@ -84,6 +88,47 @@ export function recallAndInline(opts, store) {
|
|
|
84
88
|
empty: toInject.length === 0,
|
|
85
89
|
};
|
|
86
90
|
}
|
|
91
|
+
/** Format one memory hit for the recall block. Category + score for traceability. */
|
|
92
|
+
export function formatMemoryRecallBlock(hits) {
|
|
93
|
+
if (hits.length === 0)
|
|
94
|
+
return "";
|
|
95
|
+
const parts = hits.map((h, i) => {
|
|
96
|
+
const pct = (h.score * 100).toFixed(0);
|
|
97
|
+
const cat = h.category ? `[${h.category}] ` : "";
|
|
98
|
+
return `### Recalled memory [${i + 1}] (relevance ${pct}%)\n${cat}${h.content.trim()}`;
|
|
99
|
+
});
|
|
100
|
+
return ("The following facts about this project were saved from earlier turns " +
|
|
101
|
+
"and are relevant to the current request. Treat them as established:\n\n" +
|
|
102
|
+
parts.join("\n"));
|
|
103
|
+
}
|
|
104
|
+
/** Recall top-k durable memories, format into a token-capped block. */
|
|
105
|
+
export async function recallMemoriesAndInline(opts) {
|
|
106
|
+
const limit = opts.limit ?? 5;
|
|
107
|
+
const maxTokens = opts.recallMaxTokens ?? 0;
|
|
108
|
+
const { recallMemories } = await import("./memoryRecall.js");
|
|
109
|
+
const hits = await recallMemories(opts.query, opts.stateDir, {
|
|
110
|
+
topK: limit,
|
|
111
|
+
minSimilarity: opts.minSimilarity ?? 0.2,
|
|
112
|
+
});
|
|
113
|
+
if (hits.length === 0)
|
|
114
|
+
return { empty: true, block: "", report: [] };
|
|
115
|
+
// Same incremental token cap pattern as checkpoint recall.
|
|
116
|
+
const parts = [];
|
|
117
|
+
const report = [];
|
|
118
|
+
let blockTokens = 0;
|
|
119
|
+
for (const h of hits) {
|
|
120
|
+
const part = formatMemoryRecallBlock([
|
|
121
|
+
{ content: h.memory.content, category: h.memory.category, score: h.score },
|
|
122
|
+
]);
|
|
123
|
+
const partTokens = estimateBlockTokens(part);
|
|
124
|
+
if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
|
|
125
|
+
break;
|
|
126
|
+
parts.push(part);
|
|
127
|
+
report.push(` • memory#${h.memory.id} (${(h.score * 100).toFixed(0)}%): ${h.memory.content.slice(0, 60).replace(/\n/g, " ")}…`);
|
|
128
|
+
blockTokens += partTokens;
|
|
129
|
+
}
|
|
130
|
+
return { empty: parts.length === 0, block: parts.join("\n"), report };
|
|
131
|
+
}
|
|
87
132
|
/**
|
|
88
133
|
* Slice 2 async cross-repo recall. Same dedup/bound/inline contract as
|
|
89
134
|
* `recallAndInline`, but backed by `VectorStore.searchAsync` so it can recall
|
|
@@ -121,6 +166,19 @@ export async function recallAndInlineAsync(opts, store) {
|
|
|
121
166
|
for (const h of hits) {
|
|
122
167
|
if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId))
|
|
123
168
|
continue;
|
|
169
|
+
// S18: machine-wide injected-set — a foreign checkpoint already injected
|
|
170
|
+
// (in any session) is never re-injected. Only applies to cross-repo hits
|
|
171
|
+
// (same-repo hits have no repoId and are handled by the per-session set).
|
|
172
|
+
if (opts.globalIndexDir && h.repoId) {
|
|
173
|
+
try {
|
|
174
|
+
const { wasInjectedGlobal } = await import("./store/sqlite.js");
|
|
175
|
+
if (wasInjectedGlobal(h.checkpoint.checkpointId, opts.sessionId, opts.globalIndexDir))
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
/* non-fatal: degrade to per-session injected-set only */
|
|
180
|
+
}
|
|
181
|
+
}
|
|
124
182
|
if (doWindowDedupe && liveEmbeddings.length > 0) {
|
|
125
183
|
const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
|
|
126
184
|
if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim))
|
|
@@ -134,6 +192,17 @@ export async function recallAndInlineAsync(opts, store) {
|
|
|
134
192
|
toInject.push(h);
|
|
135
193
|
blockTokens += partTokens;
|
|
136
194
|
store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
|
|
195
|
+
// S18: record the cross-repo injection machine-wide so it's not re-injected
|
|
196
|
+
// by a later recall (same or different session).
|
|
197
|
+
if (opts.globalIndexDir && h.repoId) {
|
|
198
|
+
try {
|
|
199
|
+
const { markInjectedGlobal } = await import("./store/sqlite.js");
|
|
200
|
+
markInjectedGlobal(h.checkpoint.checkpointId, h.repoId, opts.sessionId, opts.globalIndexDir);
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
/* non-fatal */
|
|
204
|
+
}
|
|
205
|
+
}
|
|
137
206
|
}
|
|
138
207
|
const block = parts.join("\n");
|
|
139
208
|
const report = toInject.map((h) => ` • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
|
package/dist/src/recall.test.js
CHANGED
|
@@ -5,7 +5,8 @@ import { tmpdir } from "node:os";
|
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { VectorStore } from "./vectorStore.js";
|
|
7
7
|
import { compactSession } from "./engine.js";
|
|
8
|
-
import { recallAndInline, formatRecallBlock } from "./recall.js";
|
|
8
|
+
import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "./recall.js";
|
|
9
|
+
import { markInjectedGlobal, wasInjectedGlobal, closeIndexStore } from "./store/sqlite.js";
|
|
9
10
|
const baseTmp = mkdtempSync(join(tmpdir(), "mc-recall-"));
|
|
10
11
|
let counter = 0;
|
|
11
12
|
function store() {
|
|
@@ -39,6 +40,24 @@ test("recallAndInline skipInjected=false re-returns hits", () => {
|
|
|
39
40
|
test("formatRecallBlock is empty for no hits", () => {
|
|
40
41
|
assert.equal(formatRecallBlock([]), "");
|
|
41
42
|
});
|
|
43
|
+
test("formatRecallBlock (S17): labels a cross-repo hit with its source repo", () => {
|
|
44
|
+
const hit = {
|
|
45
|
+
checkpoint: { checkpointId: "chkpt_x", summary: "did thing Y", filesModified: ["a.ts"] },
|
|
46
|
+
score: 0.91,
|
|
47
|
+
repoId: "/home/u/rad-gateway",
|
|
48
|
+
};
|
|
49
|
+
const block = formatRecallBlock([hit]);
|
|
50
|
+
assert.ok(block.includes("from repo"), "labels cross-repo source");
|
|
51
|
+
assert.ok(block.includes("rad-gateway"), "includes the repo display name");
|
|
52
|
+
});
|
|
53
|
+
test("formatRecallBlock (S17): omits the label for same-repo hits (no repoId)", () => {
|
|
54
|
+
const hit = {
|
|
55
|
+
checkpoint: { checkpointId: "c1", summary: "s", filesModified: [] },
|
|
56
|
+
score: 0.9,
|
|
57
|
+
};
|
|
58
|
+
const block = formatRecallBlock([hit]);
|
|
59
|
+
assert.ok(!block.includes("from repo"), "no source label for same-repo hits");
|
|
60
|
+
});
|
|
42
61
|
test("recallAndInline empty when store has nothing for query", () => {
|
|
43
62
|
const s = store();
|
|
44
63
|
const r = recallAndInline({ sessionId: SESS, query: "no such topic exists here", limit: 5, source: "command" }, s);
|
|
@@ -73,6 +92,55 @@ test("Fix C: inline dedupe drops a hit already resident in the live window", ()
|
|
|
73
92
|
assert.ok(rDedup.toInject.length <= rNoDedup.toInject.length, "dedupe never adds hits");
|
|
74
93
|
assert.ok(rDedup.toInject.length < rNoDedup.toInject.length, "inline dedupe dropped a resident hit");
|
|
75
94
|
});
|
|
95
|
+
test("S18: global injected-set skips a foreign checkpoint already injected machine-wide", async () => {
|
|
96
|
+
const indexDir = mkdtempSync(join(tmpdir(), "mc-gi-"));
|
|
97
|
+
try {
|
|
98
|
+
const sess = "sess_cross";
|
|
99
|
+
// A foreign checkpoint already marked injected globally (in this session).
|
|
100
|
+
markInjectedGlobal("chkpt_foreign", "/repo/other", sess, indexDir);
|
|
101
|
+
assert.equal(wasInjectedGlobal("chkpt_foreign", sess, indexDir), true);
|
|
102
|
+
// searchAsync returns the foreign hit; recallAndInlineAsync must skip it
|
|
103
|
+
// (globally injected) → toInject is empty.
|
|
104
|
+
const mockStore = {
|
|
105
|
+
searchAsync: async () => [{
|
|
106
|
+
checkpoint: { checkpointId: "chkpt_foreign", summary: "foreign work", filesModified: [], dedupStatus: "active" },
|
|
107
|
+
score: 0.92,
|
|
108
|
+
repoId: "/repo/other",
|
|
109
|
+
}],
|
|
110
|
+
wasInjected: () => false,
|
|
111
|
+
markInjected: () => { },
|
|
112
|
+
};
|
|
113
|
+
const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, mockStore);
|
|
114
|
+
assert.equal(r.toInject.length, 0, "globally-injected foreign checkpoint skipped");
|
|
115
|
+
}
|
|
116
|
+
finally {
|
|
117
|
+
closeIndexStore();
|
|
118
|
+
rmSync(indexDir, { recursive: true, force: true });
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
test("S18: a fresh foreign checkpoint is injected AND recorded globally", async () => {
|
|
122
|
+
const indexDir = mkdtempSync(join(tmpdir(), "mc-gi2-"));
|
|
123
|
+
try {
|
|
124
|
+
const sess = "sess_fresh";
|
|
125
|
+
assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), false);
|
|
126
|
+
const mockStore = {
|
|
127
|
+
searchAsync: async () => [{
|
|
128
|
+
checkpoint: { checkpointId: "chkpt_new", summary: "brand new foreign work", filesModified: [], dedupStatus: "active" },
|
|
129
|
+
score: 0.93,
|
|
130
|
+
repoId: "/repo/alpha",
|
|
131
|
+
}],
|
|
132
|
+
wasInjected: () => false,
|
|
133
|
+
markInjected: () => { },
|
|
134
|
+
};
|
|
135
|
+
const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, mockStore);
|
|
136
|
+
assert.equal(r.toInject.length, 1, "fresh foreign checkpoint injected");
|
|
137
|
+
assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), true, "recorded machine-wide");
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
closeIndexStore();
|
|
141
|
+
rmSync(indexDir, { recursive: true, force: true });
|
|
142
|
+
}
|
|
143
|
+
});
|
|
76
144
|
test("cleanup", () => {
|
|
77
145
|
rmSync(baseTmp, { recursive: true, force: true });
|
|
78
146
|
});
|