pi-mega-compact 0.4.28 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +47 -2
  2. package/dist/extensions/dashboard-server.js +58 -2
  3. package/dist/extensions/dashboard-server.test.js +95 -3
  4. package/dist/extensions/mega-commands.js +25 -9
  5. package/dist/extensions/mega-compact.test.js +133 -31
  6. package/dist/extensions/mega-config.js +5 -0
  7. package/dist/extensions/mega-conflict-cmds.js +79 -0
  8. package/dist/extensions/mega-dashboard-cmds.js +6 -4
  9. package/dist/extensions/mega-events.js +144 -27
  10. package/dist/extensions/mega-pipeline.js +84 -1
  11. package/dist/extensions/mega-runtime.js +14 -0
  12. package/dist/extensions/mega-trim.js +48 -0
  13. package/dist/extensions/mega-trim.test.js +58 -0
  14. package/dist/src/config/dedup.js +1 -0
  15. package/dist/src/driftDetection.js +103 -0
  16. package/dist/src/driftDetection.test.js +87 -0
  17. package/dist/src/memory.js +147 -0
  18. package/dist/src/memory.test.js +41 -0
  19. package/dist/src/memoryConsolidate.test.js +38 -0
  20. package/dist/src/memoryOps.js +58 -0
  21. package/dist/src/memoryOps.test.js +41 -0
  22. package/dist/src/memoryRecall.js +60 -0
  23. package/dist/src/memoryRecall.test.js +92 -0
  24. package/dist/src/recall.js +70 -1
  25. package/dist/src/recall.test.js +69 -1
  26. package/dist/src/store/sqlite.js +127 -11
  27. package/dist/src/vectorStore.js +6 -1
  28. package/extensions/dashboard-server.test.ts +115 -3
  29. package/extensions/dashboard-server.ts +63 -2
  30. package/extensions/mega-commands.ts +24 -9
  31. package/extensions/mega-compact.test.ts +134 -31
  32. package/extensions/mega-config.ts +22 -0
  33. package/extensions/mega-conflict-cmds.ts +81 -0
  34. package/extensions/mega-dashboard-cmds.ts +6 -4
  35. package/extensions/mega-events.ts +139 -28
  36. package/extensions/mega-pipeline.ts +94 -1
  37. package/extensions/mega-runtime.ts +15 -0
  38. package/extensions/mega-trim.test.ts +64 -0
  39. package/extensions/mega-trim.ts +75 -0
  40. package/extensions/openclaw-mega-compact.ts +24 -9
  41. package/package.json +2 -2
  42. package/src/config/dedup.ts +2 -0
  43. package/src/driftDetection.test.ts +100 -0
  44. package/src/driftDetection.ts +136 -0
  45. package/src/memory.test.ts +46 -0
  46. package/src/memory.ts +164 -0
  47. package/src/memoryConsolidate.test.ts +47 -0
  48. package/src/memoryOps.test.ts +53 -0
  49. package/src/memoryOps.ts +75 -0
  50. package/src/memoryRecall.test.ts +100 -0
  51. package/src/memoryRecall.ts +83 -0
  52. package/src/recall.test.ts +77 -1
  53. package/src/recall.ts +94 -1
  54. package/src/store/sqlite.ts +188 -11
  55. package/src/store.ts +3 -0
  56. package/src/vectorStore.ts +10 -1
  57. package/dist/extensions/openclaw-mega-compact.js +0 -291
  58. package/dist/src/minilm.js +0 -92
  59. package/dist/src/wordpiece.js +0 -129
@@ -0,0 +1,92 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { mkdtempSync, rmSync } from "node:fs";
4
+ import { tmpdir } from "node:os";
5
+ import { join } from "node:path";
6
+ import { recallMemories } from "./memoryRecall.js";
7
+ import { addMemory, getMemory } from "./store/sqlite.js";
8
+ const baseTmp = mkdtempSync(join(tmpdir(), "mc-memrec-"));
9
+ function biGramEmbedder() {
10
+ // Deterministic test embedder — bigger dim + binary encoding so two semantically
11
+ // related strings have higher cosine than unrelated ones.
12
+ const dim = 64;
13
+ return {
14
+ dim,
15
+ embed(text) {
16
+ const v = new Array(dim).fill(0);
17
+ const norm = text.toLowerCase().replace(/[^a-z0-9 ]/g, "").trim();
18
+ for (let i = 0; i < norm.length - 1; i++) {
19
+ const idx = ((norm.charCodeAt(i) * 31 + norm.charCodeAt(i + 1)) >>> 0) % dim;
20
+ v[idx] = 1;
21
+ }
22
+ return v;
23
+ },
24
+ };
25
+ }
26
+ test("recallMemories: ranks relevant memory above unrelated", async () => {
27
+ const dir = join(baseTmp, "rank");
28
+ addMemory({ content: "we use sqlite for the durable store", category: "decision" }, null, dir);
29
+ addMemory({ content: "the threshold is 100 thousand tokens", category: "decision" }, null, dir);
30
+ addMemory({ content: "katz says hi", category: "note" }, null, dir);
31
+ const hits = await recallMemories("what store do we use?", dir, {
32
+ embedder: biGramEmbedder(),
33
+ topK: 3,
34
+ minSimilarity: 0.0,
35
+ });
36
+ assert.ok(hits.length >= 2, "finds relevant");
37
+ assert.ok(/sqlite/.test(hits[0].memory.content), "top hit is sqlite one");
38
+ assert.ok(!/katz/.test(hits[0].memory.content), "unrelated not on top");
39
+ });
40
+ test("recallMemories: marks referenced hits (last_referenced updated)", async () => {
41
+ const dir = join(baseTmp, "ref");
42
+ addMemory({ content: "policy is local-only", category: "rule" }, null, dir);
43
+ const hits = await recallMemories("local-only policy", dir, { embedder: biGramEmbedder() });
44
+ assert.ok(hits.length >= 1);
45
+ const fresh = getMemory(hits[0].memory.id, dir);
46
+ assert.ok(fresh && fresh.lastReferenced && fresh.lastReferenced > 0, "lastReferenced set");
47
+ });
48
+ test("recallMemories: empty store returns []", async () => {
49
+ const dir = join(baseTmp, "empty");
50
+ const hits = await recallMemories("anything", dir, { embedder: biGramEmbedder() });
51
+ assert.deepEqual(hits, []);
52
+ });
53
+ test("recallMemories: decision category beats fact category at equal similarity", async () => {
54
+ const dir = join(baseTmp, "category");
55
+ // Two memories that share many bigrams with the query — both will have very
56
+ // similar cosine. The decision category should win because of categoryWeight.
57
+ addMemory({ content: "we use redis for cache" }, null, dir);
58
+ const aId = addMemory({ content: "we use redis as primary cache key", category: "fact" }, null, dir);
59
+ const dId = addMemory({ content: "we use redis as primary cache layer", category: "decision" }, null, dir);
60
+ const hits = await recallMemories("redis cache layer", dir, {
61
+ embedder: biGramEmbedder(),
62
+ topK: 3,
63
+ minSimilarity: 0.0,
64
+ });
65
+ assert.ok(hits.length >= 2);
66
+ assert.equal(hits[0].memory.id, dId, "decision-tagged memory outranks fact-tagged");
67
+ assert.notEqual(hits[0].memory.id, aId, "top is the decision row, not the unknown-category row");
68
+ });
69
+ test("recallMemories: fresher reference beats older at equal similarity", async () => {
70
+ const dir = join(baseTmp, "recency");
71
+ const aId = addMemory({ content: "we use redis for cache", category: "fact" }, null, dir);
72
+ const bId = addMemory({ content: "we use redis for cache layer", category: "fact" }, null, dir);
73
+ // Backdate aId so its last_referenced is older; bId stays fresh.
74
+ const { openStore, closeStore } = await import("./store/sqlite.js");
75
+ const db = openStore(dir);
76
+ const longAgo = Math.floor(Date.now() / 1000) - 30 * 86_400; // 30d
77
+ db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(longAgo, aId);
78
+ // closeStore evicts the cached handle — calling db.close() directly would
79
+ // leave a stale closed DB in the openStore cache and break subsequent
80
+ // callers (the "database is not open" failure under parallel runs).
81
+ closeStore(dir);
82
+ const hits = await recallMemories("redis cache layer", dir, {
83
+ embedder: biGramEmbedder(),
84
+ topK: 3,
85
+ minSimilarity: 0.0,
86
+ });
87
+ assert.ok(hits.length >= 2);
88
+ assert.equal(hits[0].memory.id, bId, "freshly-referenced memory outranks 30-day-old one");
89
+ });
90
+ test("cleanup memrec", () => {
91
+ rmSync(baseTmp, { recursive: true, force: true });
92
+ });
@@ -22,7 +22,11 @@ export function formatRecallBlock(hits) {
22
22
  return "";
23
23
  const parts = hits.map((h, i) => {
24
24
  const score = (h.score * 100).toFixed(0);
25
- return (`### Recalled context [${i + 1}] (relevance ${score}%)\n` +
25
+ // S17: label a cross-repo hit with its source repo (the repoId doubles as
26
+ // that repo's stateDir, so the last path segment is the repo's display
27
+ // name). Same-repo hits (no repoId) stay unlabeled.
28
+ const repoName = h.repoId ? ` (from repo ${h.repoId.split("/").filter(Boolean).pop() ?? h.repoId})` : "";
29
+ return (`### Recalled context [${i + 1}] (relevance ${score}%)${repoName}\n` +
26
30
  `${h.checkpoint.summary.trim()}\n` +
27
31
  (h.checkpoint.filesModified.length
28
32
  ? `Key files: ${h.checkpoint.filesModified.join(", ")}.\n`
@@ -84,6 +88,47 @@ export function recallAndInline(opts, store) {
84
88
  empty: toInject.length === 0,
85
89
  };
86
90
  }
91
+ /** Format one memory hit for the recall block. Category + score for traceability. */
92
+ export function formatMemoryRecallBlock(hits) {
93
+ if (hits.length === 0)
94
+ return "";
95
+ const parts = hits.map((h, i) => {
96
+ const pct = (h.score * 100).toFixed(0);
97
+ const cat = h.category ? `[${h.category}] ` : "";
98
+ return `### Recalled memory [${i + 1}] (relevance ${pct}%)\n${cat}${h.content.trim()}`;
99
+ });
100
+ return ("The following facts about this project were saved from earlier turns " +
101
+ "and are relevant to the current request. Treat them as established:\n\n" +
102
+ parts.join("\n"));
103
+ }
104
+ /** Recall top-k durable memories, format into a token-capped block. */
105
+ export async function recallMemoriesAndInline(opts) {
106
+ const limit = opts.limit ?? 5;
107
+ const maxTokens = opts.recallMaxTokens ?? 0;
108
+ const { recallMemories } = await import("./memoryRecall.js");
109
+ const hits = await recallMemories(opts.query, opts.stateDir, {
110
+ topK: limit,
111
+ minSimilarity: opts.minSimilarity ?? 0.2,
112
+ });
113
+ if (hits.length === 0)
114
+ return { empty: true, block: "", report: [] };
115
+ // Same incremental token cap pattern as checkpoint recall.
116
+ const parts = [];
117
+ const report = [];
118
+ let blockTokens = 0;
119
+ for (const h of hits) {
120
+ const part = formatMemoryRecallBlock([
121
+ { content: h.memory.content, category: h.memory.category, score: h.score },
122
+ ]);
123
+ const partTokens = estimateBlockTokens(part);
124
+ if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
125
+ break;
126
+ parts.push(part);
127
+ report.push(` • memory#${h.memory.id} (${(h.score * 100).toFixed(0)}%): ${h.memory.content.slice(0, 60).replace(/\n/g, " ")}…`);
128
+ blockTokens += partTokens;
129
+ }
130
+ return { empty: parts.length === 0, block: parts.join("\n"), report };
131
+ }
87
132
  /**
88
133
  * Slice 2 async cross-repo recall. Same dedup/bound/inline contract as
89
134
  * `recallAndInline`, but backed by `VectorStore.searchAsync` so it can recall
@@ -121,6 +166,19 @@ export async function recallAndInlineAsync(opts, store) {
121
166
  for (const h of hits) {
122
167
  if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId))
123
168
  continue;
169
+ // S18: machine-wide injected-set — a foreign checkpoint already injected
170
+ // (in any session) is never re-injected. Only applies to cross-repo hits
171
+ // (same-repo hits have no repoId and are handled by the per-session set).
172
+ if (opts.globalIndexDir && h.repoId) {
173
+ try {
174
+ const { wasInjectedGlobal } = await import("./store/sqlite.js");
175
+ if (wasInjectedGlobal(h.checkpoint.checkpointId, opts.sessionId, opts.globalIndexDir))
176
+ continue;
177
+ }
178
+ catch {
179
+ /* non-fatal: degrade to per-session injected-set only */
180
+ }
181
+ }
124
182
  if (doWindowDedupe && liveEmbeddings.length > 0) {
125
183
  const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
126
184
  if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim))
@@ -134,6 +192,17 @@ export async function recallAndInlineAsync(opts, store) {
134
192
  toInject.push(h);
135
193
  blockTokens += partTokens;
136
194
  store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
195
+ // S18: record the cross-repo injection machine-wide so it's not re-injected
196
+ // by a later recall (same or different session).
197
+ if (opts.globalIndexDir && h.repoId) {
198
+ try {
199
+ const { markInjectedGlobal } = await import("./store/sqlite.js");
200
+ markInjectedGlobal(h.checkpoint.checkpointId, h.repoId, opts.sessionId, opts.globalIndexDir);
201
+ }
202
+ catch {
203
+ /* non-fatal */
204
+ }
205
+ }
137
206
  }
138
207
  const block = parts.join("\n");
139
208
  const report = toInject.map((h) => ` • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
@@ -5,7 +5,8 @@ import { tmpdir } from "node:os";
5
5
  import { join } from "node:path";
6
6
  import { VectorStore } from "./vectorStore.js";
7
7
  import { compactSession } from "./engine.js";
8
- import { recallAndInline, formatRecallBlock } from "./recall.js";
8
+ import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "./recall.js";
9
+ import { markInjectedGlobal, wasInjectedGlobal, closeIndexStore } from "./store/sqlite.js";
9
10
  const baseTmp = mkdtempSync(join(tmpdir(), "mc-recall-"));
10
11
  let counter = 0;
11
12
  function store() {
@@ -39,6 +40,24 @@ test("recallAndInline skipInjected=false re-returns hits", () => {
39
40
  test("formatRecallBlock is empty for no hits", () => {
40
41
  assert.equal(formatRecallBlock([]), "");
41
42
  });
43
+ test("formatRecallBlock (S17): labels a cross-repo hit with its source repo", () => {
44
+ const hit = {
45
+ checkpoint: { checkpointId: "chkpt_x", summary: "did thing Y", filesModified: ["a.ts"] },
46
+ score: 0.91,
47
+ repoId: "/home/u/rad-gateway",
48
+ };
49
+ const block = formatRecallBlock([hit]);
50
+ assert.ok(block.includes("from repo"), "labels cross-repo source");
51
+ assert.ok(block.includes("rad-gateway"), "includes the repo display name");
52
+ });
53
+ test("formatRecallBlock (S17): omits the label for same-repo hits (no repoId)", () => {
54
+ const hit = {
55
+ checkpoint: { checkpointId: "c1", summary: "s", filesModified: [] },
56
+ score: 0.9,
57
+ };
58
+ const block = formatRecallBlock([hit]);
59
+ assert.ok(!block.includes("from repo"), "no source label for same-repo hits");
60
+ });
42
61
  test("recallAndInline empty when store has nothing for query", () => {
43
62
  const s = store();
44
63
  const r = recallAndInline({ sessionId: SESS, query: "no such topic exists here", limit: 5, source: "command" }, s);
@@ -73,6 +92,55 @@ test("Fix C: inline dedupe drops a hit already resident in the live window", ()
73
92
  assert.ok(rDedup.toInject.length <= rNoDedup.toInject.length, "dedupe never adds hits");
74
93
  assert.ok(rDedup.toInject.length < rNoDedup.toInject.length, "inline dedupe dropped a resident hit");
75
94
  });
95
+ test("S18: global injected-set skips a foreign checkpoint already injected machine-wide", async () => {
96
+ const indexDir = mkdtempSync(join(tmpdir(), "mc-gi-"));
97
+ try {
98
+ const sess = "sess_cross";
99
+ // A foreign checkpoint already marked injected globally (in this session).
100
+ markInjectedGlobal("chkpt_foreign", "/repo/other", sess, indexDir);
101
+ assert.equal(wasInjectedGlobal("chkpt_foreign", sess, indexDir), true);
102
+ // searchAsync returns the foreign hit; recallAndInlineAsync must skip it
103
+ // (globally injected) → toInject is empty.
104
+ const mockStore = {
105
+ searchAsync: async () => [{
106
+ checkpoint: { checkpointId: "chkpt_foreign", summary: "foreign work", filesModified: [], dedupStatus: "active" },
107
+ score: 0.92,
108
+ repoId: "/repo/other",
109
+ }],
110
+ wasInjected: () => false,
111
+ markInjected: () => { },
112
+ };
113
+ const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, mockStore);
114
+ assert.equal(r.toInject.length, 0, "globally-injected foreign checkpoint skipped");
115
+ }
116
+ finally {
117
+ closeIndexStore();
118
+ rmSync(indexDir, { recursive: true, force: true });
119
+ }
120
+ });
121
+ test("S18: a fresh foreign checkpoint is injected AND recorded globally", async () => {
122
+ const indexDir = mkdtempSync(join(tmpdir(), "mc-gi2-"));
123
+ try {
124
+ const sess = "sess_fresh";
125
+ assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), false);
126
+ const mockStore = {
127
+ searchAsync: async () => [{
128
+ checkpoint: { checkpointId: "chkpt_new", summary: "brand new foreign work", filesModified: [], dedupStatus: "active" },
129
+ score: 0.93,
130
+ repoId: "/repo/alpha",
131
+ }],
132
+ wasInjected: () => false,
133
+ markInjected: () => { },
134
+ };
135
+ const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, mockStore);
136
+ assert.equal(r.toInject.length, 1, "fresh foreign checkpoint injected");
137
+ assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), true, "recorded machine-wide");
138
+ }
139
+ finally {
140
+ closeIndexStore();
141
+ rmSync(indexDir, { recursive: true, force: true });
142
+ }
143
+ });
76
144
  test("cleanup", () => {
77
145
  rmSync(baseTmp, { recursive: true, force: true });
78
146
  });
@@ -53,8 +53,18 @@ const cache = new Map();
53
53
  /** Open (or reuse) the SQLite store for a state dir. */
54
54
  export function openStore(stateDir = getStateDir()) {
55
55
  const existing = cache.get(stateDir);
56
- if (existing)
57
- return existing;
56
+ if (existing) {
57
+ // A closed handle in the cache (e.g. a test calling db.close() directly
58
+ // instead of closeStore) would surface as "database is not open" on the
59
+ // next reuse. Detect and evict so callers never see a dead handle.
60
+ try {
61
+ existing.prepare("SELECT 1");
62
+ return existing;
63
+ }
64
+ catch {
65
+ cache.delete(stateDir);
66
+ }
67
+ }
58
68
  if (!existsSync(stateDir))
59
69
  mkdirSync(stateDir, { recursive: true });
60
70
  const db = new DatabaseSync(join(stateDir, "sqlite.db"));
@@ -118,6 +128,19 @@ export function openIndexStore(indexDir = getIndexDir()) {
118
128
  model_captured_at INTEGER
119
129
  );
120
130
  CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
131
+ -- S18: machine-wide injected-set. A foreign checkpoint injected in repo A is
132
+ -- recorded here so repo B's recall never re-injects it. Keyed by checkpoint
133
+ -- + session (a checkpoint may be injected once per session); repo_id is the
134
+ -- source repo (the foreign repo's stateDir) for tracking/source labels.
135
+ -- PRAMETERIZED queries (PREVENT-002); local node:sqlite (PREVENT-PI-004).
136
+ CREATE TABLE IF NOT EXISTS injected_global (
137
+ checkpoint_id TEXT NOT NULL,
138
+ repo_id TEXT NOT NULL,
139
+ session_id TEXT NOT NULL,
140
+ injected_at INTEGER NOT NULL,
141
+ PRIMARY KEY (checkpoint_id, session_id)
142
+ );
143
+ CREATE INDEX IF NOT EXISTS idx_injected_global_cid ON injected_global(checkpoint_id);
121
144
  `);
122
145
  indexCache = iddb;
123
146
  indexCacheDir = indexDir;
@@ -133,25 +156,41 @@ export function upsertRepoRegistry(row, indexDir = getIndexDir()) {
133
156
  const now = Date.now();
134
157
  db.prepare(`INSERT INTO repo_registry
135
158
  (repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
136
- checkpoint_count, tokens_saved, compressed_original_bytes)
137
- VALUES (@repo_root, @display_name, @state_dir, @now, @now, @last_compacted_at,
138
- @checkpoint_count, @tokens_saved, @compressed_original_bytes)
159
+ checkpoint_count, tokens_saved, compressed_original_bytes,
160
+ provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
161
+ VALUES (@repo_root, @display_name, @state_dir, @first_seen, @last_seen, @last_compacted_at,
162
+ @checkpoint_count, @tokens_saved, @compressed_original_bytes,
163
+ @provider, @provider_name, @model_name, @input_rate, @output_rate, @model_captured_at)
139
164
  ON CONFLICT(repo_root) DO UPDATE SET
140
165
  display_name = excluded.display_name,
141
166
  state_dir = excluded.state_dir,
142
- last_seen = excluded.last_seen,
167
+ last_seen = COALESCE(excluded.last_seen, @now),
143
168
  last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
144
169
  checkpoint_count = excluded.checkpoint_count,
145
170
  tokens_saved = excluded.tokens_saved,
146
- compressed_original_bytes = excluded.compressed_original_bytes`).run({
171
+ compressed_original_bytes = excluded.compressed_original_bytes,
172
+ provider = COALESCE(excluded.provider, repo_registry.provider),
173
+ provider_name = COALESCE(excluded.provider_name, repo_registry.provider_name),
174
+ model_name = COALESCE(excluded.model_name, repo_registry.model_name),
175
+ input_rate = COALESCE(excluded.input_rate, repo_registry.input_rate),
176
+ output_rate = COALESCE(excluded.output_rate, repo_registry.output_rate),
177
+ model_captured_at = COALESCE(excluded.model_captured_at, repo_registry.model_captured_at)`).run({
147
178
  repo_root: row.repoRoot,
148
179
  display_name: row.displayName,
149
180
  state_dir: row.stateDir,
150
181
  now,
182
+ first_seen: row.firstSeen ?? null,
183
+ last_seen: row.lastSeen ?? null,
151
184
  last_compacted_at: row.lastCompactedAt ?? null,
152
185
  checkpoint_count: row.checkpointCount,
153
186
  tokens_saved: row.tokensSaved,
154
187
  compressed_original_bytes: row.compressedOriginalBytes,
188
+ provider: row.provider ?? null,
189
+ provider_name: row.providerName ?? null,
190
+ model_name: row.modelName ?? null,
191
+ input_rate: row.inputRate ?? null,
192
+ output_rate: row.outputRate ?? null,
193
+ model_captured_at: row.modelCapturedAt ?? null,
155
194
  });
156
195
  }
157
196
  /**
@@ -225,6 +264,34 @@ export function closeIndexStore() {
225
264
  indexCacheDir = undefined;
226
265
  }
227
266
  }
267
+ // ---------------------------------------------------------------------------
268
+ // S18: machine-wide injected-set (cross-repo dedup markers)
269
+ //
270
+ // A foreign checkpoint injected in repo A is recorded here so repo B's recall
271
+ // never re-injects it (a stronger, machine-wide version of the per-session
272
+ // injected-set in the local store). Keyed by (checkpoint_id, session_id); the
273
+ // session_id here is the RECEIVING session, so the same foreign checkpoint can
274
+ // be injected into different sessions but never twice into the same one.
275
+ // PRAMETERIZED queries (PREVENT-002); local node:sqlite + WAL (PREVENT-PI-004),
276
+ // multi-process safe.
277
+ // ---------------------------------------------------------------------------
278
+ /** Record that a (foreign) checkpoint was injected into `sessionId`. Idempotent. */
279
+ export function markInjectedGlobal(checkpointId, repoId, sessionId, indexDir = getIndexDir()) {
280
+ const db = openIndexStore(indexDir);
281
+ db.prepare("INSERT OR IGNORE INTO injected_global (checkpoint_id, repo_id, session_id, injected_at) VALUES ($cid, $rid, $sid, $ts)").run({ $cid: checkpointId, $rid: repoId, $sid: sessionId, $ts: Date.now() });
282
+ }
283
+ /** True when a checkpoint was already injected into `sessionId` (machine-wide). */
284
+ export function wasInjectedGlobal(checkpointId, sessionId, indexDir = getIndexDir()) {
285
+ const db = openIndexStore(indexDir);
286
+ const row = db.prepare("SELECT 1 FROM injected_global WHERE checkpoint_id = $cid AND session_id = $sid LIMIT 1").get({ $cid: checkpointId, $sid: sessionId });
287
+ return row !== undefined;
288
+ }
289
+ /** Count of cross-repo injections recorded (for /mega-status stats). */
290
+ export function countInjectedGlobal(indexDir = getIndexDir()) {
291
+ const db = openIndexStore(indexDir);
292
+ const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get();
293
+ return row?.n ?? 0;
294
+ }
228
295
  function initSchema(db) {
229
296
  db.exec(`
230
297
  CREATE TABLE IF NOT EXISTS context_chunks (
@@ -374,7 +441,12 @@ function initSchema(db) {
374
441
  content TEXT NOT NULL,
375
442
  tags TEXT, -- JSON array of strings
376
443
  created_at INTEGER,
377
- last_recalled_at INTEGER
444
+ last_recalled_at INTEGER,
445
+ -- S20 memory-RAG extension (auto-review add/replace/remove ops).
446
+ category TEXT, -- typed bucket, e.g. decision | fact | preference
447
+ target TEXT, -- optional subject/scope this memory targets
448
+ last_referenced INTEGER, -- last time memory was referenced by recall (epoch s)
449
+ source_turn INTEGER -- conversation turn that produced this memory
378
450
  );
379
451
  CREATE INDEX IF NOT EXISTS idx_memories_repo ON memories(repo);
380
452
 
@@ -391,6 +463,12 @@ function initSchema(db) {
391
463
  // databases created by an older version — otherwise repoStats()/upsert crash
392
464
  // with "no such column" and the extension fails to load. Additive only.
393
465
  ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
466
+ // S20 memory-RAG extension: additive columns for auto-review ops. Idempotent —
467
+ // only alters DBs created by an older version that lack these columns.
468
+ ensureColumn(db, "memories", "category", "TEXT");
469
+ ensureColumn(db, "memories", "target", "TEXT");
470
+ ensureColumn(db, "memories", "last_referenced", "INTEGER");
471
+ ensureColumn(db, "memories", "source_turn", "INTEGER");
394
472
  const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get();
395
473
  if (!v) {
396
474
  db.prepare("INSERT INTO meta(key, value) VALUES(?, ?)").run("schema_version", String(SCHEMA_VERSION));
@@ -500,9 +578,9 @@ export function addMemory(memory, repo, stateDir = getStateDir()) {
500
578
  const db = openStore(stateDir);
501
579
  const now = Math.floor(Date.now() / 1000);
502
580
  const res = db
503
- .prepare(`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at)
504
- VALUES(?, ?, ?, ?, ?, NULL)`)
505
- .run(repo ?? null, memory.kind ?? "note", memory.content, JSON.stringify(memory.tags ?? []), now);
581
+ .prepare(`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at, category, target, source_turn)
582
+ VALUES(?, ?, ?, ?, ?, NULL, ?, ?, ?)`)
583
+ .run(repo ?? null, memory.kind ?? "note", memory.content, JSON.stringify(memory.tags ?? []), now, memory.category ?? null, memory.target ?? null, memory.sourceTurn ?? null);
506
584
  return Number(res.lastInsertRowid);
507
585
  }
508
586
  /** List recent memories for a repo (or all repos when repo is null). */
@@ -529,6 +607,40 @@ export function recallMemory(id, stateDir = getStateDir()) {
529
607
  const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
530
608
  return res.changes > 0;
531
609
  }
610
+ /** Mark a memory as referenced (updates last_referenced). Returns true if found. */
611
+ export function referenceMemory(id, stateDir = getStateDir()) {
612
+ const db = openStore(stateDir);
613
+ const now = Math.floor(Date.now() / 1000);
614
+ const res = db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(now, id);
615
+ return res.changes > 0;
616
+ }
617
+ /** Replace a memory's mutable fields by id. Returns true if a row was updated. */
618
+ export function replaceMemory(id, patch, stateDir = getStateDir()) {
619
+ const db = openStore(stateDir);
620
+ const res = db
621
+ .prepare(`UPDATE memories
622
+ SET kind = COALESCE(?, kind),
623
+ content = COALESCE(?, content),
624
+ tags = COALESCE(?, tags),
625
+ category = COALESCE(?, category),
626
+ target = COALESCE(?, target),
627
+ source_turn = COALESCE(?, source_turn)
628
+ WHERE id = ?`)
629
+ .run(patch.kind ?? null, patch.content ?? null, patch.tags ? JSON.stringify(patch.tags) : null, "category" in patch ? (patch.category ?? null) : null, "target" in patch ? (patch.target ?? null) : null, "sourceTurn" in patch ? (patch.sourceTurn ?? null) : null, id);
630
+ return res.changes > 0;
631
+ }
632
+ /** Remove a memory by id. Returns true if a row was deleted. */
633
+ export function removeMemory(id, stateDir = getStateDir()) {
634
+ const db = openStore(stateDir);
635
+ const res = db.prepare("DELETE FROM memories WHERE id = ?").run(id);
636
+ return res.changes > 0;
637
+ }
638
+ /** Look up a single memory by id (or undefined). */
639
+ export function getMemory(id, stateDir = getStateDir()) {
640
+ const db = openStore(stateDir);
641
+ const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
642
+ return row ? mapMemoryRow(row) : undefined;
643
+ }
532
644
  function mapMemoryRow(row) {
533
645
  return {
534
646
  id: row.id,
@@ -538,6 +650,10 @@ function mapMemoryRow(row) {
538
650
  tags: row.tags ? JSON.parse(row.tags) : [],
539
651
  createdAt: row.created_at ?? 0,
540
652
  lastRecalledAt: row.last_recalled_at ?? null,
653
+ category: row.category ?? null,
654
+ target: row.target ?? null,
655
+ lastReferenced: row.last_referenced ?? null,
656
+ sourceTurn: row.source_turn ?? null,
541
657
  };
542
658
  }
543
659
  /**
@@ -240,6 +240,7 @@ export class VectorStore {
240
240
  const checkpoint = {
241
241
  checkpointId,
242
242
  sessionId,
243
+ repoId: this.repoId,
243
244
  summary: input.summary,
244
245
  topicSummary: input.topicSummary,
245
246
  summaryHash,
@@ -412,11 +413,15 @@ export class VectorStore {
412
413
  }
413
414
  // Hydrate each index hit from the authoritative node:sqlite store. repoId is
414
415
  // that repo's stateDir, so cross-repo hits resolve against their own store.
416
+ // Tag cross-repo hits with their source repoId so the recall block can label
417
+ // them ("from repo <name>"); same-repo hits stay unlabeled.
418
+ const selfRepo = this.repoId;
415
419
  const hydrated = [];
416
420
  for (const h of indexHits) {
417
421
  const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
418
422
  if (cp && cp.dedupStatus !== "removed") {
419
- hydrated.push({ checkpoint: cp, score: h.score });
423
+ const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
424
+ hydrated.push({ checkpoint: cp, score: h.score, repoId: crossRepo ? h.repoId : undefined });
420
425
  }
421
426
  }
422
427
  if (hydrated.length === 0)
@@ -124,6 +124,110 @@ describe("port.pid file", () => {
124
124
  });
125
125
  });
126
126
 
127
+ // ---------------------------------------------------------------------------
128
+ // Multi-repo dashboard (S19 / Phase 5b) — launch the real server subprocess,
129
+ // seed the machine-wide repo_registry, and assert /api/index returns every repo
130
+ // plus the aggregate summary the Summary + All-repos tabs render.
131
+ // ---------------------------------------------------------------------------
132
+
133
+ describe("multi-repo /api/index (S19)", () => {
134
+ test("lists all repos from the global index with an aggregate summary", async () => {
135
+ const dir = mkdtempSync(join(tmpdir(), "dash-index-"));
136
+ const indexDir = mkdtempSync(join(tmpdir(), "index-"));
137
+ // The server reads MEGACOMPACT_INDEX_DIR for the machine-wide registry.
138
+ process.env.MEGACOMPACT_INDEX_DIR = indexDir;
139
+ process.env.MEGACOMPACT_DASHBOARD_PORT = "19321"; // private base, non-colliding
140
+
141
+ const { upsertRepoRegistry } = await import("../src/store/sqlite.js");
142
+ upsertRepoRegistry(
143
+ { repoRoot: "/home/u/repoA", displayName: "repoA", stateDir: dir, checkpointCount: 3, tokensSaved: 1000, compressedOriginalBytes: 0 },
144
+ indexDir,
145
+ );
146
+ upsertRepoRegistry(
147
+ { repoRoot: "/home/u/repoB", displayName: "repoB", stateDir: dir, checkpointCount: 5, tokensSaved: 2000, compressedOriginalBytes: 0 },
148
+ indexDir,
149
+ );
150
+
151
+ const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
152
+ try {
153
+ await waitFor(async () => {
154
+ try {
155
+ const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
156
+ const res = await fetch(`http://localhost:${raw.port}/api/version`);
157
+ return res.ok;
158
+ } catch {
159
+ return false;
160
+ }
161
+ });
162
+ const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
163
+ const idx = (await fetch(`http://localhost:${raw.port}/api/index`).then((r) => r.json())) as {
164
+ summary: { totalRepos: number; totalCheckpoints: number; totalTokensSaved: number };
165
+ repos: { repoRoot: string; displayName: string; checkpointCount: number; tokensSaved: number }[];
166
+ };
167
+ const names = idx.repos.map((r) => r.displayName).sort();
168
+ assert.deepEqual(names, ["repoA", "repoB"], "both repos from the global index");
169
+ assert.equal(idx.summary.totalRepos, 2, "repo count");
170
+ assert.equal(idx.summary.totalCheckpoints, 8, "3 + 5 checkpoints");
171
+ assert.equal(idx.summary.totalTokensSaved, 3000, "1000 + 2000 tokens saved");
172
+ } finally {
173
+ child.kill("SIGTERM");
174
+ delete process.env.MEGACOMPACT_INDEX_DIR;
175
+ delete process.env.MEGACOMPACT_DASHBOARD_PORT;
176
+ rmSync(dir, { recursive: true, force: true });
177
+ rmSync(indexDir, { recursive: true, force: true });
178
+ }
179
+ });
180
+
181
+ test("/api/repos filters by ?active=Nh and /api/summary counts activeRepos", async () => {
182
+ const dir = mkdtempSync(join(tmpdir(), "dash-active-"));
183
+ const indexDir = mkdtempSync(join(tmpdir(), "index-active-"));
184
+ process.env.MEGACOMPACT_INDEX_DIR = indexDir;
185
+ process.env.MEGACOMPACT_DASHBOARD_PORT = "19322";
186
+
187
+ const { upsertRepoRegistry } = await import("../src/store/sqlite.js");
188
+ // Fresh repo, last_seen = now
189
+ upsertRepoRegistry(
190
+ { repoRoot: "/home/u/fresh", displayName: "fresh", stateDir: dir, checkpointCount: 1, tokensSaved: 100, compressedOriginalBytes: 0, lastSeen: Math.floor(Date.now() / 1000) },
191
+ indexDir,
192
+ );
193
+ // Stale repo, last_seen = 90 days ago — must be filtered out by ?active=24h.
194
+ const longAgo = Math.floor(Date.now() / 1000) - 90 * 86_400;
195
+ upsertRepoRegistry(
196
+ { repoRoot: "/home/u/stale", displayName: "stale", stateDir: dir, checkpointCount: 2, tokensSaved: 200, compressedOriginalBytes: 0, lastSeen: longAgo },
197
+ indexDir,
198
+ );
199
+
200
+ const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
201
+ try {
202
+ await waitFor(async () => {
203
+ try {
204
+ const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
205
+ const res = await fetch(`http://localhost:${raw.port}/api/version`);
206
+ return res.ok;
207
+ } catch { return false; }
208
+ });
209
+ const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
210
+
211
+ const allRepos = (await fetch(`http://localhost:${raw.port}/api/repos`).then((r) => r.json())) as { repos: { displayName: string }[]; count: number };
212
+ assert.equal(allRepos.count, 2, "unfiltered list has both repos");
213
+
214
+ const activeRepos = (await fetch(`http://localhost:${raw.port}/api/repos?active=24h`).then((r) => r.json())) as { repos: { displayName: string }[]; count: number };
215
+ assert.equal(activeRepos.count, 1, "active=24h drops the 90-day-old repo");
216
+ assert.equal(activeRepos.repos[0].displayName, "fresh");
217
+
218
+ const summary = (await fetch(`http://localhost:${raw.port}/api/summary`).then((r) => r.json())) as { activeRepos: number; totalRepos: number };
219
+ assert.equal(summary.activeRepos, 1, "summary counts only fresh repo as active");
220
+ assert.equal(summary.totalRepos, 2, "summary counts both repos total");
221
+ } finally {
222
+ child.kill("SIGTERM");
223
+ delete process.env.MEGACOMPACT_INDEX_DIR;
224
+ delete process.env.MEGACOMPACT_DASHBOARD_PORT;
225
+ rmSync(dir, { recursive: true, force: true });
226
+ rmSync(indexDir, { recursive: true, force: true });
227
+ }
228
+ });
229
+ });
230
+
127
231
  // ---------------------------------------------------------------------------
128
232
  // Lifecycle integration — launch the compiled server as a real subprocess
129
233
  // (the same way the /dashboard command spawns it) and assert the two failure
@@ -136,6 +240,12 @@ describe("port.pid file", () => {
136
240
 
137
241
  const SERVER_ENTRY = new URL("./dashboard-server.js", import.meta.url).pathname;
138
242
 
243
+ // Tests run in parallel across files and a killed run can leave a server bound
244
+ // to 9320. Use a private, non-colliding base so this file never races the
245
+ // mega-compact.test.js dashboard tests (which scan a DIFFERENT base) and never
246
+ // collides with a leftover production server on 9320.
247
+ process.env.MEGACOMPACT_DASHBOARD_PORT = "19320";
248
+
139
249
  function waitFor(cond: () => boolean | Promise<boolean>, timeoutMs = 6000): Promise<void> {
140
250
  const start = Date.now();
141
251
  return new Promise((resolve, reject) => {
@@ -151,8 +261,10 @@ function waitFor(cond: () => boolean | Promise<boolean>, timeoutMs = 6000): Prom
151
261
  describe("server lifecycle", () => {
152
262
  test("drops a stale port.pid and binds a fresh port", async () => {
153
263
  const dir = mkdtempSync(join(tmpdir(), "dash-stale-"));
154
- // A marker claiming a port where nothing is listening.
155
- writeFileSync(join(dir, "port.pid"), JSON.stringify({ port: 9325, pid: 999999 }));
264
+ // A marker claiming a port where nothing is listening — use the test's own
265
+ // private base + 5 so the dead port is inside the server's scan range.
266
+ const deadPort = 19325;
267
+ writeFileSync(join(dir, "port.pid"), JSON.stringify({ port: deadPort, pid: 999999 }));
156
268
 
157
269
  const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
158
270
  try {
@@ -169,7 +281,7 @@ describe("server lifecycle", () => {
169
281
  });
170
282
  const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
171
283
  assert.equal(typeof raw.port, "number");
172
- assert.notEqual(raw.port, 9325, "should not reuse the dead port from the stale marker");
284
+ assert.notEqual(raw.port, deadPort, "should not reuse the dead port from the stale marker");
173
285
  // And a real server must answer on it.
174
286
  const res = await fetch(`http://localhost:${raw.port}/api/version`);
175
287
  assert.equal(res.ok, true);