pi-mega-compact 0.4.28 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -2
- package/dist/extensions/dashboard-server.js +66 -3
- package/dist/extensions/dashboard-server.test.js +95 -3
- package/dist/extensions/mega-commands.js +25 -9
- package/dist/extensions/mega-compact.test.js +133 -31
- package/dist/extensions/mega-config.js +5 -0
- package/dist/extensions/mega-conflict-cmds.js +79 -0
- package/dist/extensions/mega-dashboard-cmds.js +6 -4
- package/dist/extensions/mega-events.js +144 -27
- package/dist/extensions/mega-pipeline.js +84 -1
- package/dist/extensions/mega-runtime.js +35 -2
- package/dist/extensions/mega-trim.js +48 -0
- package/dist/extensions/mega-trim.test.js +58 -0
- package/dist/src/config/dedup.js +1 -0
- package/dist/src/driftDetection.js +103 -0
- package/dist/src/driftDetection.test.js +87 -0
- package/dist/src/memory.js +147 -0
- package/dist/src/memory.test.js +41 -0
- package/dist/src/memoryConsolidate.test.js +38 -0
- package/dist/src/memoryOps.js +58 -0
- package/dist/src/memoryOps.test.js +41 -0
- package/dist/src/memoryRecall.js +60 -0
- package/dist/src/memoryRecall.test.js +92 -0
- package/dist/src/recall.js +70 -1
- package/dist/src/recall.test.js +69 -1
- package/dist/src/store/sqlite.js +127 -11
- package/dist/src/vectorStore.js +6 -1
- package/extensions/dashboard-server.test.ts +115 -3
- package/extensions/dashboard-server.ts +69 -4
- package/extensions/mega-commands.ts +24 -9
- package/extensions/mega-compact.test.ts +134 -31
- package/extensions/mega-config.ts +22 -0
- package/extensions/mega-conflict-cmds.ts +81 -0
- package/extensions/mega-dashboard-cmds.ts +6 -4
- package/extensions/mega-events.ts +139 -28
- package/extensions/mega-pipeline.ts +94 -1
- package/extensions/mega-runtime.ts +35 -2
- package/extensions/mega-trim.test.ts +64 -0
- package/extensions/mega-trim.ts +75 -0
- package/extensions/openclaw-mega-compact.ts +24 -9
- package/package.json +2 -2
- package/src/config/dedup.ts +2 -0
- package/src/driftDetection.test.ts +100 -0
- package/src/driftDetection.ts +136 -0
- package/src/memory.test.ts +46 -0
- package/src/memory.ts +164 -0
- package/src/memoryConsolidate.test.ts +47 -0
- package/src/memoryOps.test.ts +53 -0
- package/src/memoryOps.ts +75 -0
- package/src/memoryRecall.test.ts +100 -0
- package/src/memoryRecall.ts +83 -0
- package/src/recall.test.ts +77 -1
- package/src/recall.ts +94 -1
- package/src/store/sqlite.ts +188 -11
- package/src/store.ts +3 -0
- package/src/vectorStore.ts +10 -1
package/dist/src/store/sqlite.js
CHANGED
|
@@ -53,8 +53,18 @@ const cache = new Map();
|
|
|
53
53
|
/** Open (or reuse) the SQLite store for a state dir. */
|
|
54
54
|
export function openStore(stateDir = getStateDir()) {
|
|
55
55
|
const existing = cache.get(stateDir);
|
|
56
|
-
if (existing)
|
|
57
|
-
|
|
56
|
+
if (existing) {
|
|
57
|
+
// A closed handle in the cache (e.g. a test calling db.close() directly
|
|
58
|
+
// instead of closeStore) would surface as "database is not open" on the
|
|
59
|
+
// next reuse. Detect and evict so callers never see a dead handle.
|
|
60
|
+
try {
|
|
61
|
+
existing.prepare("SELECT 1");
|
|
62
|
+
return existing;
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
cache.delete(stateDir);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
58
68
|
if (!existsSync(stateDir))
|
|
59
69
|
mkdirSync(stateDir, { recursive: true });
|
|
60
70
|
const db = new DatabaseSync(join(stateDir, "sqlite.db"));
|
|
@@ -118,6 +128,19 @@ export function openIndexStore(indexDir = getIndexDir()) {
|
|
|
118
128
|
model_captured_at INTEGER
|
|
119
129
|
);
|
|
120
130
|
CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
|
|
131
|
+
-- S18: machine-wide injected-set. A foreign checkpoint injected in repo A is
|
|
132
|
+
-- recorded here so repo B's recall never re-injects it. Keyed by checkpoint
|
|
133
|
+
-- + session (a checkpoint may be injected once per session); repo_id is the
|
|
134
|
+
-- source repo (the foreign repo's stateDir) for tracking/source labels.
|
|
135
|
+
-- PRAMETERIZED queries (PREVENT-002); local node:sqlite (PREVENT-PI-004).
|
|
136
|
+
CREATE TABLE IF NOT EXISTS injected_global (
|
|
137
|
+
checkpoint_id TEXT NOT NULL,
|
|
138
|
+
repo_id TEXT NOT NULL,
|
|
139
|
+
session_id TEXT NOT NULL,
|
|
140
|
+
injected_at INTEGER NOT NULL,
|
|
141
|
+
PRIMARY KEY (checkpoint_id, session_id)
|
|
142
|
+
);
|
|
143
|
+
CREATE INDEX IF NOT EXISTS idx_injected_global_cid ON injected_global(checkpoint_id);
|
|
121
144
|
`);
|
|
122
145
|
indexCache = iddb;
|
|
123
146
|
indexCacheDir = indexDir;
|
|
@@ -133,25 +156,41 @@ export function upsertRepoRegistry(row, indexDir = getIndexDir()) {
|
|
|
133
156
|
const now = Date.now();
|
|
134
157
|
db.prepare(`INSERT INTO repo_registry
|
|
135
158
|
(repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
|
|
136
|
-
checkpoint_count, tokens_saved, compressed_original_bytes
|
|
137
|
-
|
|
138
|
-
|
|
159
|
+
checkpoint_count, tokens_saved, compressed_original_bytes,
|
|
160
|
+
provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
|
|
161
|
+
VALUES (@repo_root, @display_name, @state_dir, @first_seen, @last_seen, @last_compacted_at,
|
|
162
|
+
@checkpoint_count, @tokens_saved, @compressed_original_bytes,
|
|
163
|
+
@provider, @provider_name, @model_name, @input_rate, @output_rate, @model_captured_at)
|
|
139
164
|
ON CONFLICT(repo_root) DO UPDATE SET
|
|
140
165
|
display_name = excluded.display_name,
|
|
141
166
|
state_dir = excluded.state_dir,
|
|
142
|
-
last_seen = excluded.last_seen,
|
|
167
|
+
last_seen = COALESCE(excluded.last_seen, @now),
|
|
143
168
|
last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
|
|
144
169
|
checkpoint_count = excluded.checkpoint_count,
|
|
145
170
|
tokens_saved = excluded.tokens_saved,
|
|
146
|
-
compressed_original_bytes = excluded.compressed_original_bytes
|
|
171
|
+
compressed_original_bytes = excluded.compressed_original_bytes,
|
|
172
|
+
provider = COALESCE(excluded.provider, repo_registry.provider),
|
|
173
|
+
provider_name = COALESCE(excluded.provider_name, repo_registry.provider_name),
|
|
174
|
+
model_name = COALESCE(excluded.model_name, repo_registry.model_name),
|
|
175
|
+
input_rate = COALESCE(excluded.input_rate, repo_registry.input_rate),
|
|
176
|
+
output_rate = COALESCE(excluded.output_rate, repo_registry.output_rate),
|
|
177
|
+
model_captured_at = COALESCE(excluded.model_captured_at, repo_registry.model_captured_at)`).run({
|
|
147
178
|
repo_root: row.repoRoot,
|
|
148
179
|
display_name: row.displayName,
|
|
149
180
|
state_dir: row.stateDir,
|
|
150
181
|
now,
|
|
182
|
+
first_seen: row.firstSeen ?? null,
|
|
183
|
+
last_seen: row.lastSeen ?? null,
|
|
151
184
|
last_compacted_at: row.lastCompactedAt ?? null,
|
|
152
185
|
checkpoint_count: row.checkpointCount,
|
|
153
186
|
tokens_saved: row.tokensSaved,
|
|
154
187
|
compressed_original_bytes: row.compressedOriginalBytes,
|
|
188
|
+
provider: row.provider ?? null,
|
|
189
|
+
provider_name: row.providerName ?? null,
|
|
190
|
+
model_name: row.modelName ?? null,
|
|
191
|
+
input_rate: row.inputRate ?? null,
|
|
192
|
+
output_rate: row.outputRate ?? null,
|
|
193
|
+
model_captured_at: row.modelCapturedAt ?? null,
|
|
155
194
|
});
|
|
156
195
|
}
|
|
157
196
|
/**
|
|
@@ -225,6 +264,34 @@ export function closeIndexStore() {
|
|
|
225
264
|
indexCacheDir = undefined;
|
|
226
265
|
}
|
|
227
266
|
}
|
|
267
|
+
// ---------------------------------------------------------------------------
|
|
268
|
+
// S18: machine-wide injected-set (cross-repo dedup markers)
|
|
269
|
+
//
|
|
270
|
+
// A foreign checkpoint injected in repo A is recorded here so repo B's recall
|
|
271
|
+
// never re-injects it (a stronger, machine-wide version of the per-session
|
|
272
|
+
// injected-set in the local store). Keyed by (checkpoint_id, session_id); the
|
|
273
|
+
// session_id here is the RECEIVING session, so the same foreign checkpoint can
|
|
274
|
+
// be injected into different sessions but never twice into the same one.
|
|
275
|
+
// PRAMETERIZED queries (PREVENT-002); local node:sqlite + WAL (PREVENT-PI-004),
|
|
276
|
+
// multi-process safe.
|
|
277
|
+
// ---------------------------------------------------------------------------
|
|
278
|
+
/** Record that a (foreign) checkpoint was injected into `sessionId`. Idempotent. */
|
|
279
|
+
export function markInjectedGlobal(checkpointId, repoId, sessionId, indexDir = getIndexDir()) {
|
|
280
|
+
const db = openIndexStore(indexDir);
|
|
281
|
+
db.prepare("INSERT OR IGNORE INTO injected_global (checkpoint_id, repo_id, session_id, injected_at) VALUES ($cid, $rid, $sid, $ts)").run({ $cid: checkpointId, $rid: repoId, $sid: sessionId, $ts: Date.now() });
|
|
282
|
+
}
|
|
283
|
+
/** True when a checkpoint was already injected into `sessionId` (machine-wide). */
|
|
284
|
+
export function wasInjectedGlobal(checkpointId, sessionId, indexDir = getIndexDir()) {
|
|
285
|
+
const db = openIndexStore(indexDir);
|
|
286
|
+
const row = db.prepare("SELECT 1 FROM injected_global WHERE checkpoint_id = $cid AND session_id = $sid LIMIT 1").get({ $cid: checkpointId, $sid: sessionId });
|
|
287
|
+
return row !== undefined;
|
|
288
|
+
}
|
|
289
|
+
/** Count of cross-repo injections recorded (for /mega-status stats). */
|
|
290
|
+
export function countInjectedGlobal(indexDir = getIndexDir()) {
|
|
291
|
+
const db = openIndexStore(indexDir);
|
|
292
|
+
const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get();
|
|
293
|
+
return row?.n ?? 0;
|
|
294
|
+
}
|
|
228
295
|
function initSchema(db) {
|
|
229
296
|
db.exec(`
|
|
230
297
|
CREATE TABLE IF NOT EXISTS context_chunks (
|
|
@@ -374,7 +441,12 @@ function initSchema(db) {
|
|
|
374
441
|
content TEXT NOT NULL,
|
|
375
442
|
tags TEXT, -- JSON array of strings
|
|
376
443
|
created_at INTEGER,
|
|
377
|
-
last_recalled_at INTEGER
|
|
444
|
+
last_recalled_at INTEGER,
|
|
445
|
+
-- S20 memory-RAG extension (auto-review add/replace/remove ops).
|
|
446
|
+
category TEXT, -- typed bucket, e.g. decision | fact | preference
|
|
447
|
+
target TEXT, -- optional subject/scope this memory targets
|
|
448
|
+
last_referenced INTEGER, -- last time memory was referenced by recall (epoch s)
|
|
449
|
+
source_turn INTEGER -- conversation turn that produced this memory
|
|
378
450
|
);
|
|
379
451
|
CREATE INDEX IF NOT EXISTS idx_memories_repo ON memories(repo);
|
|
380
452
|
|
|
@@ -391,6 +463,12 @@ function initSchema(db) {
|
|
|
391
463
|
// databases created by an older version — otherwise repoStats()/upsert crash
|
|
392
464
|
// with "no such column" and the extension fails to load. Additive only.
|
|
393
465
|
ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
|
|
466
|
+
// S20 memory-RAG extension: additive columns for auto-review ops. Idempotent —
|
|
467
|
+
// only alters DBs created by an older version that lack these columns.
|
|
468
|
+
ensureColumn(db, "memories", "category", "TEXT");
|
|
469
|
+
ensureColumn(db, "memories", "target", "TEXT");
|
|
470
|
+
ensureColumn(db, "memories", "last_referenced", "INTEGER");
|
|
471
|
+
ensureColumn(db, "memories", "source_turn", "INTEGER");
|
|
394
472
|
const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get();
|
|
395
473
|
if (!v) {
|
|
396
474
|
db.prepare("INSERT INTO meta(key, value) VALUES(?, ?)").run("schema_version", String(SCHEMA_VERSION));
|
|
@@ -500,9 +578,9 @@ export function addMemory(memory, repo, stateDir = getStateDir()) {
|
|
|
500
578
|
const db = openStore(stateDir);
|
|
501
579
|
const now = Math.floor(Date.now() / 1000);
|
|
502
580
|
const res = db
|
|
503
|
-
.prepare(`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at)
|
|
504
|
-
VALUES(?, ?, ?, ?, ?, NULL)`)
|
|
505
|
-
.run(repo ?? null, memory.kind ?? "note", memory.content, JSON.stringify(memory.tags ?? []), now);
|
|
581
|
+
.prepare(`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at, category, target, source_turn)
|
|
582
|
+
VALUES(?, ?, ?, ?, ?, NULL, ?, ?, ?)`)
|
|
583
|
+
.run(repo ?? null, memory.kind ?? "note", memory.content, JSON.stringify(memory.tags ?? []), now, memory.category ?? null, memory.target ?? null, memory.sourceTurn ?? null);
|
|
506
584
|
return Number(res.lastInsertRowid);
|
|
507
585
|
}
|
|
508
586
|
/** List recent memories for a repo (or all repos when repo is null). */
|
|
@@ -529,6 +607,40 @@ export function recallMemory(id, stateDir = getStateDir()) {
|
|
|
529
607
|
const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
|
|
530
608
|
return res.changes > 0;
|
|
531
609
|
}
|
|
610
|
+
/** Mark a memory as referenced (updates last_referenced). Returns true if found. */
|
|
611
|
+
export function referenceMemory(id, stateDir = getStateDir()) {
|
|
612
|
+
const db = openStore(stateDir);
|
|
613
|
+
const now = Math.floor(Date.now() / 1000);
|
|
614
|
+
const res = db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(now, id);
|
|
615
|
+
return res.changes > 0;
|
|
616
|
+
}
|
|
617
|
+
/** Replace a memory's mutable fields by id. Returns true if a row was updated. */
|
|
618
|
+
export function replaceMemory(id, patch, stateDir = getStateDir()) {
|
|
619
|
+
const db = openStore(stateDir);
|
|
620
|
+
const res = db
|
|
621
|
+
.prepare(`UPDATE memories
|
|
622
|
+
SET kind = COALESCE(?, kind),
|
|
623
|
+
content = COALESCE(?, content),
|
|
624
|
+
tags = COALESCE(?, tags),
|
|
625
|
+
category = COALESCE(?, category),
|
|
626
|
+
target = COALESCE(?, target),
|
|
627
|
+
source_turn = COALESCE(?, source_turn)
|
|
628
|
+
WHERE id = ?`)
|
|
629
|
+
.run(patch.kind ?? null, patch.content ?? null, patch.tags ? JSON.stringify(patch.tags) : null, "category" in patch ? (patch.category ?? null) : null, "target" in patch ? (patch.target ?? null) : null, "sourceTurn" in patch ? (patch.sourceTurn ?? null) : null, id);
|
|
630
|
+
return res.changes > 0;
|
|
631
|
+
}
|
|
632
|
+
/** Remove a memory by id. Returns true if a row was deleted. */
|
|
633
|
+
export function removeMemory(id, stateDir = getStateDir()) {
|
|
634
|
+
const db = openStore(stateDir);
|
|
635
|
+
const res = db.prepare("DELETE FROM memories WHERE id = ?").run(id);
|
|
636
|
+
return res.changes > 0;
|
|
637
|
+
}
|
|
638
|
+
/** Look up a single memory by id (or undefined). */
|
|
639
|
+
export function getMemory(id, stateDir = getStateDir()) {
|
|
640
|
+
const db = openStore(stateDir);
|
|
641
|
+
const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
|
|
642
|
+
return row ? mapMemoryRow(row) : undefined;
|
|
643
|
+
}
|
|
532
644
|
function mapMemoryRow(row) {
|
|
533
645
|
return {
|
|
534
646
|
id: row.id,
|
|
@@ -538,6 +650,10 @@ function mapMemoryRow(row) {
|
|
|
538
650
|
tags: row.tags ? JSON.parse(row.tags) : [],
|
|
539
651
|
createdAt: row.created_at ?? 0,
|
|
540
652
|
lastRecalledAt: row.last_recalled_at ?? null,
|
|
653
|
+
category: row.category ?? null,
|
|
654
|
+
target: row.target ?? null,
|
|
655
|
+
lastReferenced: row.last_referenced ?? null,
|
|
656
|
+
sourceTurn: row.source_turn ?? null,
|
|
541
657
|
};
|
|
542
658
|
}
|
|
543
659
|
/**
|
package/dist/src/vectorStore.js
CHANGED
|
@@ -240,6 +240,7 @@ export class VectorStore {
|
|
|
240
240
|
const checkpoint = {
|
|
241
241
|
checkpointId,
|
|
242
242
|
sessionId,
|
|
243
|
+
repoId: this.repoId,
|
|
243
244
|
summary: input.summary,
|
|
244
245
|
topicSummary: input.topicSummary,
|
|
245
246
|
summaryHash,
|
|
@@ -412,11 +413,15 @@ export class VectorStore {
|
|
|
412
413
|
}
|
|
413
414
|
// Hydrate each index hit from the authoritative node:sqlite store. repoId is
|
|
414
415
|
// that repo's stateDir, so cross-repo hits resolve against their own store.
|
|
416
|
+
// Tag cross-repo hits with their source repoId so the recall block can label
|
|
417
|
+
// them ("from repo <name>"); same-repo hits stay unlabeled.
|
|
418
|
+
const selfRepo = this.repoId;
|
|
415
419
|
const hydrated = [];
|
|
416
420
|
for (const h of indexHits) {
|
|
417
421
|
const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
|
|
418
422
|
if (cp && cp.dedupStatus !== "removed") {
|
|
419
|
-
|
|
423
|
+
const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
|
|
424
|
+
hydrated.push({ checkpoint: cp, score: h.score, repoId: crossRepo ? h.repoId : undefined });
|
|
420
425
|
}
|
|
421
426
|
}
|
|
422
427
|
if (hydrated.length === 0)
|
|
@@ -124,6 +124,110 @@ describe("port.pid file", () => {
|
|
|
124
124
|
});
|
|
125
125
|
});
|
|
126
126
|
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
// Multi-repo dashboard (S19 / Phase 5b) — launch the real server subprocess,
|
|
129
|
+
// seed the machine-wide repo_registry, and assert /api/index returns every repo
|
|
130
|
+
// plus the aggregate summary the Summary + All-repos tabs render.
|
|
131
|
+
// ---------------------------------------------------------------------------
|
|
132
|
+
|
|
133
|
+
describe("multi-repo /api/index (S19)", () => {
|
|
134
|
+
test("lists all repos from the global index with an aggregate summary", async () => {
|
|
135
|
+
const dir = mkdtempSync(join(tmpdir(), "dash-index-"));
|
|
136
|
+
const indexDir = mkdtempSync(join(tmpdir(), "index-"));
|
|
137
|
+
// The server reads MEGACOMPACT_INDEX_DIR for the machine-wide registry.
|
|
138
|
+
process.env.MEGACOMPACT_INDEX_DIR = indexDir;
|
|
139
|
+
process.env.MEGACOMPACT_DASHBOARD_PORT = "19321"; // private base, non-colliding
|
|
140
|
+
|
|
141
|
+
const { upsertRepoRegistry } = await import("../src/store/sqlite.js");
|
|
142
|
+
upsertRepoRegistry(
|
|
143
|
+
{ repoRoot: "/home/u/repoA", displayName: "repoA", stateDir: dir, checkpointCount: 3, tokensSaved: 1000, compressedOriginalBytes: 0 },
|
|
144
|
+
indexDir,
|
|
145
|
+
);
|
|
146
|
+
upsertRepoRegistry(
|
|
147
|
+
{ repoRoot: "/home/u/repoB", displayName: "repoB", stateDir: dir, checkpointCount: 5, tokensSaved: 2000, compressedOriginalBytes: 0 },
|
|
148
|
+
indexDir,
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
|
|
152
|
+
try {
|
|
153
|
+
await waitFor(async () => {
|
|
154
|
+
try {
|
|
155
|
+
const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
|
|
156
|
+
const res = await fetch(`http://localhost:${raw.port}/api/version`);
|
|
157
|
+
return res.ok;
|
|
158
|
+
} catch {
|
|
159
|
+
return false;
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
|
|
163
|
+
const idx = (await fetch(`http://localhost:${raw.port}/api/index`).then((r) => r.json())) as {
|
|
164
|
+
summary: { totalRepos: number; totalCheckpoints: number; totalTokensSaved: number };
|
|
165
|
+
repos: { repoRoot: string; displayName: string; checkpointCount: number; tokensSaved: number }[];
|
|
166
|
+
};
|
|
167
|
+
const names = idx.repos.map((r) => r.displayName).sort();
|
|
168
|
+
assert.deepEqual(names, ["repoA", "repoB"], "both repos from the global index");
|
|
169
|
+
assert.equal(idx.summary.totalRepos, 2, "repo count");
|
|
170
|
+
assert.equal(idx.summary.totalCheckpoints, 8, "3 + 5 checkpoints");
|
|
171
|
+
assert.equal(idx.summary.totalTokensSaved, 3000, "1000 + 2000 tokens saved");
|
|
172
|
+
} finally {
|
|
173
|
+
child.kill("SIGTERM");
|
|
174
|
+
delete process.env.MEGACOMPACT_INDEX_DIR;
|
|
175
|
+
delete process.env.MEGACOMPACT_DASHBOARD_PORT;
|
|
176
|
+
rmSync(dir, { recursive: true, force: true });
|
|
177
|
+
rmSync(indexDir, { recursive: true, force: true });
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test("/api/repos filters by ?active=Nh and /api/summary counts activeRepos", async () => {
|
|
182
|
+
const dir = mkdtempSync(join(tmpdir(), "dash-active-"));
|
|
183
|
+
const indexDir = mkdtempSync(join(tmpdir(), "index-active-"));
|
|
184
|
+
process.env.MEGACOMPACT_INDEX_DIR = indexDir;
|
|
185
|
+
process.env.MEGACOMPACT_DASHBOARD_PORT = "19322";
|
|
186
|
+
|
|
187
|
+
const { upsertRepoRegistry } = await import("../src/store/sqlite.js");
|
|
188
|
+
// Fresh repo, last_seen = now
|
|
189
|
+
upsertRepoRegistry(
|
|
190
|
+
{ repoRoot: "/home/u/fresh", displayName: "fresh", stateDir: dir, checkpointCount: 1, tokensSaved: 100, compressedOriginalBytes: 0, lastSeen: Math.floor(Date.now() / 1000) },
|
|
191
|
+
indexDir,
|
|
192
|
+
);
|
|
193
|
+
// Stale repo, last_seen = 90 days ago — must be filtered out by ?active=24h.
|
|
194
|
+
const longAgo = Math.floor(Date.now() / 1000) - 90 * 86_400;
|
|
195
|
+
upsertRepoRegistry(
|
|
196
|
+
{ repoRoot: "/home/u/stale", displayName: "stale", stateDir: dir, checkpointCount: 2, tokensSaved: 200, compressedOriginalBytes: 0, lastSeen: longAgo },
|
|
197
|
+
indexDir,
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
|
|
201
|
+
try {
|
|
202
|
+
await waitFor(async () => {
|
|
203
|
+
try {
|
|
204
|
+
const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
|
|
205
|
+
const res = await fetch(`http://localhost:${raw.port}/api/version`);
|
|
206
|
+
return res.ok;
|
|
207
|
+
} catch { return false; }
|
|
208
|
+
});
|
|
209
|
+
const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
|
|
210
|
+
|
|
211
|
+
const allRepos = (await fetch(`http://localhost:${raw.port}/api/repos`).then((r) => r.json())) as { repos: { displayName: string }[]; count: number };
|
|
212
|
+
assert.equal(allRepos.count, 2, "unfiltered list has both repos");
|
|
213
|
+
|
|
214
|
+
const activeRepos = (await fetch(`http://localhost:${raw.port}/api/repos?active=24h`).then((r) => r.json())) as { repos: { displayName: string }[]; count: number };
|
|
215
|
+
assert.equal(activeRepos.count, 1, "active=24h drops the 90-day-old repo");
|
|
216
|
+
assert.equal(activeRepos.repos[0].displayName, "fresh");
|
|
217
|
+
|
|
218
|
+
const summary = (await fetch(`http://localhost:${raw.port}/api/summary`).then((r) => r.json())) as { activeRepos: number; totalRepos: number };
|
|
219
|
+
assert.equal(summary.activeRepos, 1, "summary counts only fresh repo as active");
|
|
220
|
+
assert.equal(summary.totalRepos, 2, "summary counts both repos total");
|
|
221
|
+
} finally {
|
|
222
|
+
child.kill("SIGTERM");
|
|
223
|
+
delete process.env.MEGACOMPACT_INDEX_DIR;
|
|
224
|
+
delete process.env.MEGACOMPACT_DASHBOARD_PORT;
|
|
225
|
+
rmSync(dir, { recursive: true, force: true });
|
|
226
|
+
rmSync(indexDir, { recursive: true, force: true });
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
});
|
|
230
|
+
|
|
127
231
|
// ---------------------------------------------------------------------------
|
|
128
232
|
// Lifecycle integration — launch the compiled server as a real subprocess
|
|
129
233
|
// (the same way the /dashboard command spawns it) and assert the two failure
|
|
@@ -136,6 +240,12 @@ describe("port.pid file", () => {
|
|
|
136
240
|
|
|
137
241
|
const SERVER_ENTRY = new URL("./dashboard-server.js", import.meta.url).pathname;
|
|
138
242
|
|
|
243
|
+
// Tests run in parallel across files and a killed run can leave a server bound
|
|
244
|
+
// to 9320. Use a private, non-colliding base so this file never races the
|
|
245
|
+
// mega-compact.test.js dashboard tests (which scan a DIFFERENT base) and never
|
|
246
|
+
// collides with a leftover production server on 9320.
|
|
247
|
+
process.env.MEGACOMPACT_DASHBOARD_PORT = "19320";
|
|
248
|
+
|
|
139
249
|
function waitFor(cond: () => boolean | Promise<boolean>, timeoutMs = 6000): Promise<void> {
|
|
140
250
|
const start = Date.now();
|
|
141
251
|
return new Promise((resolve, reject) => {
|
|
@@ -151,8 +261,10 @@ function waitFor(cond: () => boolean | Promise<boolean>, timeoutMs = 6000): Prom
|
|
|
151
261
|
describe("server lifecycle", () => {
|
|
152
262
|
test("drops a stale port.pid and binds a fresh port", async () => {
|
|
153
263
|
const dir = mkdtempSync(join(tmpdir(), "dash-stale-"));
|
|
154
|
-
// A marker claiming a port where nothing is listening
|
|
155
|
-
|
|
264
|
+
// A marker claiming a port where nothing is listening — use the test's own
|
|
265
|
+
// private base + 5 so the dead port is inside the server's scan range.
|
|
266
|
+
const deadPort = 19325;
|
|
267
|
+
writeFileSync(join(dir, "port.pid"), JSON.stringify({ port: deadPort, pid: 999999 }));
|
|
156
268
|
|
|
157
269
|
const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
|
|
158
270
|
try {
|
|
@@ -169,7 +281,7 @@ describe("server lifecycle", () => {
|
|
|
169
281
|
});
|
|
170
282
|
const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
|
|
171
283
|
assert.equal(typeof raw.port, "number");
|
|
172
|
-
assert.notEqual(raw.port,
|
|
284
|
+
assert.notEqual(raw.port, deadPort, "should not reuse the dead port from the stale marker");
|
|
173
285
|
// And a real server must answer on it.
|
|
174
286
|
const res = await fetch(`http://localhost:${raw.port}/api/version`);
|
|
175
287
|
assert.equal(res.ok, true);
|
|
@@ -16,6 +16,7 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync,
|
|
|
16
16
|
import { homedir } from "node:os";
|
|
17
17
|
import { join, dirname } from "node:path";
|
|
18
18
|
import { fileURLToPath } from "node:url";
|
|
19
|
+
import { createRequire } from "node:module";
|
|
19
20
|
import { DatabaseSync } from "node:sqlite";
|
|
20
21
|
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
@@ -130,6 +131,9 @@ function readIndex(): { updatedAt: string; summary: unknown; repos: unknown[] }
|
|
|
130
131
|
// Types
|
|
131
132
|
// ---------------------------------------------------------------------------
|
|
132
133
|
|
|
134
|
+
/** Package version of this extension, surfaced in the dashboard header. */
|
|
135
|
+
let dashboardServerVersion = "0.0.0";
|
|
136
|
+
|
|
133
137
|
interface Snapshot {
|
|
134
138
|
version: number;
|
|
135
139
|
updatedAt: string | null;
|
|
@@ -252,6 +256,7 @@ function dashboardHtml(tierName: string): string {
|
|
|
252
256
|
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; background: #0d1117; color: #c9d1d9; padding: 24px; line-height: 1.5; }
|
|
253
257
|
h1 { font-size: 20px; font-weight: 600; margin-bottom: 20px; display: flex; align-items: center; gap: 10px; color: #f0f6fc; }
|
|
254
258
|
h1 .tier { background: #1f6feb; color: #fff; font-size: 11px; font-weight: 700; padding: 2px 8px; border-radius: 10px; text-transform: uppercase; letter-spacing: .5px; }
|
|
259
|
+
h1 .version-pill { background: #30363d; color: #8b949e; font-size: 11px; font-weight: 600; padding: 2px 8px; border-radius: 10px; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }
|
|
255
260
|
.grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin-bottom: 20px; }
|
|
256
261
|
.card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; }
|
|
257
262
|
.card.safe { border-color: #238636; }
|
|
@@ -338,7 +343,7 @@ function dashboardHtml(tierName: string): string {
|
|
|
338
343
|
|
|
339
344
|
<div class="offline-banner" id="offline-banner">Dashboard data unavailable — waiting for a pi session to write snapshot...</div>
|
|
340
345
|
|
|
341
|
-
<h1><span>mega-compact</span><span class="tier">${tierName}</span><span class="model-pill" id="hdr-model">—</span></h1>
|
|
346
|
+
<h1><span>mega-compact</span><span class="tier">${tierName}</span><span class="version-pill">v${dashboardServerVersion}</span><span class="model-pill" id="hdr-model">—</span></h1>
|
|
342
347
|
|
|
343
348
|
<nav class="tabs">
|
|
344
349
|
<button class="tab active" data-tab="current">Current repo</button>
|
|
@@ -771,9 +776,16 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
771
776
|
for (const p of candidates) {
|
|
772
777
|
if (!existsSync(p)) continue;
|
|
773
778
|
const pkg = JSON.parse(readFileSync(p, "utf-8"));
|
|
774
|
-
if (pkg.version) { SERVER_VERSION = pkg.version; break; }
|
|
779
|
+
if (pkg.version) { SERVER_VERSION = pkg.version; dashboardServerVersion = pkg.version; break; }
|
|
775
780
|
}
|
|
776
781
|
} catch { /* non-fatal */ }
|
|
782
|
+
|
|
783
|
+
// Lazy-loaded via require so the dashboard stays cheap to boot and we don't
|
|
784
|
+
// need a top-level await in the handler.
|
|
785
|
+
const driftReq = createRequire(import.meta.url);
|
|
786
|
+
const detectCrossRepoDrift = (idxDir: string) =>
|
|
787
|
+
(driftReq("../src/driftDetection.js") as typeof import("../src/driftDetection.js"))
|
|
788
|
+
.detectCrossRepoDrift(idxDir);
|
|
777
789
|
const portFile = join(stateDir, "port.pid");
|
|
778
790
|
const snapshotPath = join(stateDir, "dashboard.json");
|
|
779
791
|
const eventsPath = join(stateDir, "events.log");
|
|
@@ -858,6 +870,54 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
858
870
|
return;
|
|
859
871
|
}
|
|
860
872
|
|
|
873
|
+
// /api/repos — registry list. Optional `?active=24h` filters to repos
|
|
874
|
+
// seen within the last N hours (default: all). The dashboard uses this to
|
|
875
|
+
// drive its "active vs archived" badge without refetching /api/index.
|
|
876
|
+
if (req.url?.startsWith("/api/repos")) {
|
|
877
|
+
const url = new URL(req.url, "http://x");
|
|
878
|
+
const activeParam = url.searchParams.get("active");
|
|
879
|
+
const idx = readIndex() ?? { updatedAt: null, summary: null, repos: [] };
|
|
880
|
+
let repos = (idx.repos ?? []) as IndexRepo[];
|
|
881
|
+
if (activeParam) {
|
|
882
|
+
const m = /^(\d+)h$/.exec(activeParam);
|
|
883
|
+
if (m) {
|
|
884
|
+
const cutoffSec = Math.floor(Date.now() / 1000) - Number(m[1]) * 3600;
|
|
885
|
+
repos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
889
|
+
res.end(JSON.stringify({ updatedAt: idx.updatedAt, repos, count: repos.length }));
|
|
890
|
+
return;
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
// /api/summary — header tiles without the full repo list (keeps payload
|
|
894
|
+
// small for embed scenarios). activeRepos mirrors the /api/repos?active=24h
|
|
895
|
+
// count so the dashboard can render the active badge alongside totals.
|
|
896
|
+
if (req.url?.startsWith("/api/summary")) {
|
|
897
|
+
const idx = readIndex() ?? { updatedAt: null, summary: null, repos: [] };
|
|
898
|
+
const repos = (idx.repos ?? []) as IndexRepo[];
|
|
899
|
+
const cutoffSec = Math.floor(Date.now() / 1000) - 24 * 3600;
|
|
900
|
+
const activeRepos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec).length;
|
|
901
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
902
|
+
res.end(JSON.stringify({
|
|
903
|
+
updatedAt: idx.updatedAt,
|
|
904
|
+
summary: idx.summary,
|
|
905
|
+
activeRepos,
|
|
906
|
+
totalRepos: repos.length,
|
|
907
|
+
}));
|
|
908
|
+
return;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
// /api/drift — R4: cross-repo drift report over repo_registry. Flags stale
|
|
912
|
+
// repos (>30d idle), compaction lag (active but >24h since last
|
|
913
|
+
// compaction), and recent model churn. Read-only.
|
|
914
|
+
if (req.url?.startsWith("/api/drift")) {
|
|
915
|
+
const report = detectCrossRepoDrift(getIndexDir());
|
|
916
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
917
|
+
res.end(JSON.stringify(report));
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
|
|
861
921
|
if (req.url === "/api/events") {
|
|
862
922
|
res.writeHead(200, {
|
|
863
923
|
"Content-Type": "text/event-stream",
|
|
@@ -924,8 +984,13 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
924
984
|
res.end(dashboardHtml(tier));
|
|
925
985
|
});
|
|
926
986
|
|
|
927
|
-
|
|
928
|
-
|
|
987
|
+
// Bind base + range are env-configurable so tests can use a private,
|
|
988
|
+
// non-colliding range (parallel runs / leftover servers from killed runs
|
|
989
|
+
// would otherwise EADDRINUSE on the machine-global 9320 range). Default
|
|
990
|
+
// MEGACOMPACT_DASHBOARD_PORT=9320 (10-port range 9320–9329) preserves the
|
|
991
|
+
// production behavior.
|
|
992
|
+
const TARGET_PORT = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
|
|
993
|
+
const PORT_RANGE = 10; // TARGET_PORT..TARGET_PORT+9
|
|
929
994
|
|
|
930
995
|
return new Promise((resolve, reject) => {
|
|
931
996
|
function tryPort(port: number) {
|
|
@@ -9,11 +9,11 @@
|
|
|
9
9
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
10
10
|
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
11
11
|
import { normalizeSessionId } from "../src/store.js";
|
|
12
|
-
import { listCheckpoints, latestModelSnapshot } from "../src/store/sqlite.js";
|
|
12
|
+
import { listCheckpoints, latestModelSnapshot, countInjectedGlobal, listRepoRegistry } from "../src/store/sqlite.js";
|
|
13
13
|
import { decompressSmart } from "../src/store/compression.js";
|
|
14
14
|
import { loadMetrics, fpRate, p95 } from "../src/monitoring.js";
|
|
15
15
|
import { MegaRuntime, C, recentUserQuery } from "./mega-runtime.js";
|
|
16
|
-
import { runCompact, doRecall } from "./mega-pipeline.js";
|
|
16
|
+
import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
|
|
17
17
|
import { setTier, COMPACT_TIERS, type MegaConfig, type CompactTier } from "./mega-config.js";
|
|
18
18
|
|
|
19
19
|
/** Resolve a checkpoint by id (or "recent"/"last") from this session's store. */
|
|
@@ -47,16 +47,21 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
47
47
|
});
|
|
48
48
|
|
|
49
49
|
pi.registerCommand("mega-recall", {
|
|
50
|
-
description: "Recall relevant compacted context from the vector store and inline it.",
|
|
50
|
+
description: "Recall relevant compacted context from the vector store and inline it. Use --cross-repo to search all repos.",
|
|
51
51
|
handler: async (args: string, ctx: ExtensionContext) => {
|
|
52
|
-
|
|
52
|
+
// S17: --cross-repo (or --cross repo) runs the async path over every repo's
|
|
53
|
+
// PGlite HNSW index (stricter cosine floor + source labels).
|
|
54
|
+
const crossRepo = /\-\-cross[\- ]repo\b/.test(args);
|
|
55
|
+
const query = args.replace(/--cross[\- ]repo\b/, "").trim() || recentUserQuery(ctx);
|
|
53
56
|
if (!query) {
|
|
54
57
|
ctx.ui.notify("[mega-compact] /mega-recall needs a query or a prior user message.");
|
|
55
58
|
return;
|
|
56
59
|
}
|
|
57
|
-
const r =
|
|
60
|
+
const r = crossRepo
|
|
61
|
+
? await doRecallAsync(runtime, config, ctx, query, "command", { crossRepo: true })
|
|
62
|
+
: doRecall(runtime, config, ctx, query, "command");
|
|
58
63
|
if (r.empty) {
|
|
59
|
-
runtime.logger.info("recall-empty", { query });
|
|
64
|
+
runtime.logger.info("recall-empty", { query, crossRepo });
|
|
60
65
|
ctx.ui.notify(`[mega-compact] recall found nothing new for "${query}".`);
|
|
61
66
|
return;
|
|
62
67
|
}
|
|
@@ -64,10 +69,10 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
64
69
|
// injection). Report what was selected now for immediate feedback.
|
|
65
70
|
runtime.pendingRecallBlock = r.block;
|
|
66
71
|
const list = r.report.map((l) => l).join("\n");
|
|
67
|
-
runtime.logger.info("recall", { query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
|
|
68
|
-
runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt`);
|
|
72
|
+
runtime.logger.info("recall", { query, crossRepo, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
|
|
73
|
+
runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt${crossRepo ? " (cross-repo)" : ""}`);
|
|
69
74
|
ctx.ui.notify(
|
|
70
|
-
`[mega-compact] recall staged ${r.toInject.length} checkpoint(s) for "${query}":\n${list}\n` +
|
|
75
|
+
`[mega-compact] recall staged ${r.toInject.length} checkpoint(s) for "${query}"${crossRepo ? " (cross-repo)" : ""}:\n${list}\n` +
|
|
71
76
|
`(injected at the next turn via system prompt)`,
|
|
72
77
|
);
|
|
73
78
|
},
|
|
@@ -111,6 +116,15 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
111
116
|
const p95L2 = p95(m.latency.L2 ?? []);
|
|
112
117
|
const relPct = (st.dedupHitRate * 100).toFixed(0);
|
|
113
118
|
const qualityStr = `recall ${relPct}% relevant · FP ${(fp * 100).toFixed(1)}% · L2 p95 ${p95L2.toFixed(0)}ms`;
|
|
119
|
+
// S18: cross-repo stats from the machine-wide index (best-effort; the
|
|
120
|
+
// index dir may be unset → 0/empty, never throws).
|
|
121
|
+
let crossRepoInjections = 0;
|
|
122
|
+
let repoCount = 0;
|
|
123
|
+
try {
|
|
124
|
+
crossRepoInjections = countInjectedGlobal(process.env.MEGACOMPACT_INDEX_DIR);
|
|
125
|
+
repoCount = listRepoRegistry(process.env.MEGACOMPACT_INDEX_DIR).length;
|
|
126
|
+
} catch { /* non-fatal */ }
|
|
127
|
+
const crossRepoStr = `${crossRepoInjections} cross-repo injections recorded · ${repoCount} repos indexed`;
|
|
114
128
|
ctx.ui.notify(
|
|
115
129
|
`[mega-compact] pct=${pct} tokens=${tokens} tier=${config.tier} fastGate=${config.fastGatePct}% ` +
|
|
116
130
|
`threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
|
|
@@ -126,6 +140,7 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
126
140
|
`[mega-compact] 💰 ${costStr}\n` +
|
|
127
141
|
`[mega-compact] 🤖 model: ${modelStr}\n` +
|
|
128
142
|
`[mega-compact] 🎯 ${qualityStr}\n` +
|
|
143
|
+
`[mega-compact] 🌐 ${crossRepoStr}\n` +
|
|
129
144
|
`[mega-compact] stateDir=${runtime.currentStateDir}`,
|
|
130
145
|
);
|
|
131
146
|
},
|