pi-mega-compact 0.4.27 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -2
- package/dist/extensions/dashboard-server.js +58 -2
- package/dist/extensions/dashboard-server.test.js +95 -3
- package/dist/extensions/mega-commands.js +25 -9
- package/dist/extensions/mega-compact.test.js +161 -29
- package/dist/extensions/mega-config.js +5 -0
- package/dist/extensions/mega-conflict-cmds.js +79 -0
- package/dist/extensions/mega-dashboard-cmds.js +6 -4
- package/dist/extensions/mega-events.js +145 -20
- package/dist/extensions/mega-pipeline.js +179 -1
- package/dist/extensions/mega-runtime.js +14 -0
- package/dist/extensions/mega-trim.js +48 -0
- package/dist/extensions/mega-trim.test.js +58 -0
- package/dist/src/config/dedup.js +1 -0
- package/dist/src/driftDetection.js +103 -0
- package/dist/src/driftDetection.test.js +87 -0
- package/dist/src/memory.js +147 -0
- package/dist/src/memory.test.js +41 -0
- package/dist/src/memoryConsolidate.test.js +38 -0
- package/dist/src/memoryOps.js +58 -0
- package/dist/src/memoryOps.test.js +41 -0
- package/dist/src/memoryRecall.js +60 -0
- package/dist/src/memoryRecall.test.js +92 -0
- package/dist/src/recall.js +70 -1
- package/dist/src/recall.test.js +69 -1
- package/dist/src/store/sqlite.js +127 -11
- package/dist/src/vectorStore.js +6 -1
- package/extensions/dashboard-server.test.ts +115 -3
- package/extensions/dashboard-server.ts +63 -2
- package/extensions/mega-commands.ts +24 -9
- package/extensions/mega-compact.test.ts +162 -29
- package/extensions/mega-config.ts +22 -0
- package/extensions/mega-conflict-cmds.ts +81 -0
- package/extensions/mega-dashboard-cmds.ts +6 -4
- package/extensions/mega-events.ts +139 -20
- package/extensions/mega-pipeline.ts +179 -1
- package/extensions/mega-runtime.ts +15 -0
- package/extensions/mega-trim.test.ts +64 -0
- package/extensions/mega-trim.ts +75 -0
- package/extensions/openclaw-mega-compact.ts +24 -9
- package/package.json +2 -2
- package/src/config/dedup.ts +2 -0
- package/src/driftDetection.test.ts +100 -0
- package/src/driftDetection.ts +136 -0
- package/src/memory.test.ts +46 -0
- package/src/memory.ts +164 -0
- package/src/memoryConsolidate.test.ts +47 -0
- package/src/memoryOps.test.ts +53 -0
- package/src/memoryOps.ts +75 -0
- package/src/memoryRecall.test.ts +100 -0
- package/src/memoryRecall.ts +83 -0
- package/src/recall.test.ts +77 -1
- package/src/recall.ts +94 -1
- package/src/store/sqlite.ts +188 -11
- package/src/store.ts +3 -0
- package/src/vectorStore.ts +10 -1
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
|
@@ -16,6 +16,7 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync,
|
|
|
16
16
|
import { homedir } from "node:os";
|
|
17
17
|
import { join, dirname } from "node:path";
|
|
18
18
|
import { fileURLToPath } from "node:url";
|
|
19
|
+
import { createRequire } from "node:module";
|
|
19
20
|
import { DatabaseSync } from "node:sqlite";
|
|
20
21
|
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
@@ -774,6 +775,13 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
774
775
|
if (pkg.version) { SERVER_VERSION = pkg.version; break; }
|
|
775
776
|
}
|
|
776
777
|
} catch { /* non-fatal */ }
|
|
778
|
+
|
|
779
|
+
// Lazy-loaded via require so the dashboard stays cheap to boot and we don't
|
|
780
|
+
// need a top-level await in the handler.
|
|
781
|
+
const driftReq = createRequire(import.meta.url);
|
|
782
|
+
const detectCrossRepoDrift = (idxDir: string) =>
|
|
783
|
+
(driftReq("../src/driftDetection.js") as typeof import("../src/driftDetection.js"))
|
|
784
|
+
.detectCrossRepoDrift(idxDir);
|
|
777
785
|
const portFile = join(stateDir, "port.pid");
|
|
778
786
|
const snapshotPath = join(stateDir, "dashboard.json");
|
|
779
787
|
const eventsPath = join(stateDir, "events.log");
|
|
@@ -858,6 +866,54 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
858
866
|
return;
|
|
859
867
|
}
|
|
860
868
|
|
|
869
|
+
// /api/repos — registry list. Optional `?active=24h` filters to repos
|
|
870
|
+
// seen within the last N hours (default: all). The dashboard uses this to
|
|
871
|
+
// drive its "active vs archived" badge without refetching /api/index.
|
|
872
|
+
if (req.url?.startsWith("/api/repos")) {
|
|
873
|
+
const url = new URL(req.url, "http://x");
|
|
874
|
+
const activeParam = url.searchParams.get("active");
|
|
875
|
+
const idx = readIndex() ?? { updatedAt: null, summary: null, repos: [] };
|
|
876
|
+
let repos = (idx.repos ?? []) as IndexRepo[];
|
|
877
|
+
if (activeParam) {
|
|
878
|
+
const m = /^(\d+)h$/.exec(activeParam);
|
|
879
|
+
if (m) {
|
|
880
|
+
const cutoffSec = Math.floor(Date.now() / 1000) - Number(m[1]) * 3600;
|
|
881
|
+
repos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec);
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
885
|
+
res.end(JSON.stringify({ updatedAt: idx.updatedAt, repos, count: repos.length }));
|
|
886
|
+
return;
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
// /api/summary — header tiles without the full repo list (keeps payload
|
|
890
|
+
// small for embed scenarios). activeRepos mirrors the /api/repos?active=24h
|
|
891
|
+
// count so the dashboard can render the active badge alongside totals.
|
|
892
|
+
if (req.url?.startsWith("/api/summary")) {
|
|
893
|
+
const idx = readIndex() ?? { updatedAt: null, summary: null, repos: [] };
|
|
894
|
+
const repos = (idx.repos ?? []) as IndexRepo[];
|
|
895
|
+
const cutoffSec = Math.floor(Date.now() / 1000) - 24 * 3600;
|
|
896
|
+
const activeRepos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec).length;
|
|
897
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
898
|
+
res.end(JSON.stringify({
|
|
899
|
+
updatedAt: idx.updatedAt,
|
|
900
|
+
summary: idx.summary,
|
|
901
|
+
activeRepos,
|
|
902
|
+
totalRepos: repos.length,
|
|
903
|
+
}));
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
// /api/drift — R4: cross-repo drift report over repo_registry. Flags stale
|
|
908
|
+
// repos (>30d idle), compaction lag (active but >24h since last
|
|
909
|
+
// compaction), and recent model churn. Read-only.
|
|
910
|
+
if (req.url?.startsWith("/api/drift")) {
|
|
911
|
+
const report = detectCrossRepoDrift(getIndexDir());
|
|
912
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
913
|
+
res.end(JSON.stringify(report));
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
|
|
861
917
|
if (req.url === "/api/events") {
|
|
862
918
|
res.writeHead(200, {
|
|
863
919
|
"Content-Type": "text/event-stream",
|
|
@@ -924,8 +980,13 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
924
980
|
res.end(dashboardHtml(tier));
|
|
925
981
|
});
|
|
926
982
|
|
|
927
|
-
|
|
928
|
-
|
|
983
|
+
// Bind base + range are env-configurable so tests can use a private,
|
|
984
|
+
// non-colliding range (parallel runs / leftover servers from killed runs
|
|
985
|
+
// would otherwise EADDRINUSE on the machine-global 9320 range). Default
|
|
986
|
+
// MEGACOMPACT_DASHBOARD_PORT=9320 (10-port range 9320–9329) preserves the
|
|
987
|
+
// production behavior.
|
|
988
|
+
const TARGET_PORT = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
|
|
989
|
+
const PORT_RANGE = 10; // TARGET_PORT..TARGET_PORT+9
|
|
929
990
|
|
|
930
991
|
return new Promise((resolve, reject) => {
|
|
931
992
|
function tryPort(port: number) {
|
|
@@ -9,11 +9,11 @@
|
|
|
9
9
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
10
10
|
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
11
11
|
import { normalizeSessionId } from "../src/store.js";
|
|
12
|
-
import { listCheckpoints, latestModelSnapshot } from "../src/store/sqlite.js";
|
|
12
|
+
import { listCheckpoints, latestModelSnapshot, countInjectedGlobal, listRepoRegistry } from "../src/store/sqlite.js";
|
|
13
13
|
import { decompressSmart } from "../src/store/compression.js";
|
|
14
14
|
import { loadMetrics, fpRate, p95 } from "../src/monitoring.js";
|
|
15
15
|
import { MegaRuntime, C, recentUserQuery } from "./mega-runtime.js";
|
|
16
|
-
import { runCompact, doRecall } from "./mega-pipeline.js";
|
|
16
|
+
import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
|
|
17
17
|
import { setTier, COMPACT_TIERS, type MegaConfig, type CompactTier } from "./mega-config.js";
|
|
18
18
|
|
|
19
19
|
/** Resolve a checkpoint by id (or "recent"/"last") from this session's store. */
|
|
@@ -47,16 +47,21 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
47
47
|
});
|
|
48
48
|
|
|
49
49
|
pi.registerCommand("mega-recall", {
|
|
50
|
-
description: "Recall relevant compacted context from the vector store and inline it.",
|
|
50
|
+
description: "Recall relevant compacted context from the vector store and inline it. Use --cross-repo to search all repos.",
|
|
51
51
|
handler: async (args: string, ctx: ExtensionContext) => {
|
|
52
|
-
|
|
52
|
+
// S17: --cross-repo (or --cross repo) runs the async path over every repo's
|
|
53
|
+
// PGlite HNSW index (stricter cosine floor + source labels).
|
|
54
|
+
const crossRepo = /\-\-cross[\- ]repo\b/.test(args);
|
|
55
|
+
const query = args.replace(/--cross[\- ]repo\b/, "").trim() || recentUserQuery(ctx);
|
|
53
56
|
if (!query) {
|
|
54
57
|
ctx.ui.notify("[mega-compact] /mega-recall needs a query or a prior user message.");
|
|
55
58
|
return;
|
|
56
59
|
}
|
|
57
|
-
const r =
|
|
60
|
+
const r = crossRepo
|
|
61
|
+
? await doRecallAsync(runtime, config, ctx, query, "command", { crossRepo: true })
|
|
62
|
+
: doRecall(runtime, config, ctx, query, "command");
|
|
58
63
|
if (r.empty) {
|
|
59
|
-
runtime.logger.info("recall-empty", { query });
|
|
64
|
+
runtime.logger.info("recall-empty", { query, crossRepo });
|
|
60
65
|
ctx.ui.notify(`[mega-compact] recall found nothing new for "${query}".`);
|
|
61
66
|
return;
|
|
62
67
|
}
|
|
@@ -64,10 +69,10 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
64
69
|
// injection). Report what was selected now for immediate feedback.
|
|
65
70
|
runtime.pendingRecallBlock = r.block;
|
|
66
71
|
const list = r.report.map((l) => l).join("\n");
|
|
67
|
-
runtime.logger.info("recall", { query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
|
|
68
|
-
runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt`);
|
|
72
|
+
runtime.logger.info("recall", { query, crossRepo, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
|
|
73
|
+
runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt${crossRepo ? " (cross-repo)" : ""}`);
|
|
69
74
|
ctx.ui.notify(
|
|
70
|
-
`[mega-compact] recall staged ${r.toInject.length} checkpoint(s) for "${query}":\n${list}\n` +
|
|
75
|
+
`[mega-compact] recall staged ${r.toInject.length} checkpoint(s) for "${query}"${crossRepo ? " (cross-repo)" : ""}:\n${list}\n` +
|
|
71
76
|
`(injected at the next turn via system prompt)`,
|
|
72
77
|
);
|
|
73
78
|
},
|
|
@@ -111,6 +116,15 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
111
116
|
const p95L2 = p95(m.latency.L2 ?? []);
|
|
112
117
|
const relPct = (st.dedupHitRate * 100).toFixed(0);
|
|
113
118
|
const qualityStr = `recall ${relPct}% relevant · FP ${(fp * 100).toFixed(1)}% · L2 p95 ${p95L2.toFixed(0)}ms`;
|
|
119
|
+
// S18: cross-repo stats from the machine-wide index (best-effort; the
|
|
120
|
+
// index dir may be unset → 0/empty, never throws).
|
|
121
|
+
let crossRepoInjections = 0;
|
|
122
|
+
let repoCount = 0;
|
|
123
|
+
try {
|
|
124
|
+
crossRepoInjections = countInjectedGlobal(process.env.MEGACOMPACT_INDEX_DIR);
|
|
125
|
+
repoCount = listRepoRegistry(process.env.MEGACOMPACT_INDEX_DIR).length;
|
|
126
|
+
} catch { /* non-fatal */ }
|
|
127
|
+
const crossRepoStr = `${crossRepoInjections} cross-repo injections recorded · ${repoCount} repos indexed`;
|
|
114
128
|
ctx.ui.notify(
|
|
115
129
|
`[mega-compact] pct=${pct} tokens=${tokens} tier=${config.tier} fastGate=${config.fastGatePct}% ` +
|
|
116
130
|
`threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
|
|
@@ -126,6 +140,7 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
126
140
|
`[mega-compact] 💰 ${costStr}\n` +
|
|
127
141
|
`[mega-compact] 🤖 model: ${modelStr}\n` +
|
|
128
142
|
`[mega-compact] 🎯 ${qualityStr}\n` +
|
|
143
|
+
`[mega-compact] 🌐 ${crossRepoStr}\n` +
|
|
129
144
|
`[mega-compact] stateDir=${runtime.currentStateDir}`,
|
|
130
145
|
);
|
|
131
146
|
},
|
|
@@ -18,6 +18,7 @@ import { mkdtempSync, rmSync } from "node:fs";
|
|
|
18
18
|
import { tmpdir } from "node:os";
|
|
19
19
|
import { join } from "node:path";
|
|
20
20
|
import { createRequire } from "node:module";
|
|
21
|
+
import { closeVectorIndex } from "../src/store/vectorIndex.js";
|
|
21
22
|
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
22
23
|
|
|
23
24
|
const require = createRequire(import.meta.url);
|
|
@@ -83,6 +84,9 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
|
|
|
83
84
|
const sessionManager = {
|
|
84
85
|
getSessionId: () => "sess_ext_001",
|
|
85
86
|
getEntries: () => session.map(toEntry),
|
|
87
|
+
// Faithful mock: getBranch() returns the current branch's entries, which
|
|
88
|
+
// piCompactWouldNoop() reads to predict whether ctx.compact() would no-op.
|
|
89
|
+
getBranch: () => session.map(toEntry),
|
|
86
90
|
};
|
|
87
91
|
|
|
88
92
|
function makeCtx(over: Partial<any> = {}) {
|
|
@@ -174,22 +178,134 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
|
|
|
174
178
|
};
|
|
175
179
|
}
|
|
176
180
|
|
|
177
|
-
test("auto-trigger: past threshold persists a chkpt and starts a durable trim", async () => {
|
|
181
|
+
test("auto-trigger (legacy): past threshold persists a chkpt and starts a durable trim via ctx.compact", async () => {
|
|
178
182
|
const h = harness();
|
|
179
183
|
const messages = h.session;
|
|
184
|
+
// The mock session is tiny (~100 tokens). piCompactWouldNoop() would skip
|
|
185
|
+
// ctx.compact() for a transcript under pi's keepRecentTokens budget — so
|
|
186
|
+
// lower the floor to 0 to simulate a transcript large enough that pi WOULD
|
|
187
|
+
// compact (the positive path this test exercises).
|
|
188
|
+
// S16: this is the LEGACY path — the default no longer calls ctx.compact()
|
|
189
|
+
// (it returns a live-trimmed view instead). Set the legacy flag to exercise
|
|
190
|
+
// the v0.4.28 ctx.compact durable-trim flow this test asserts.
|
|
191
|
+
process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR = "0";
|
|
192
|
+
process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM = "true";
|
|
193
|
+
try {
|
|
194
|
+
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
195
|
+
const res = await h.fire("context", { type: "context", messages }, ctx);
|
|
196
|
+
// L1->L4 ran: a checkpoint was persisted to the SQLite store + a marker entry written.
|
|
197
|
+
const { listCheckpoints } = await import("../src/store/sqlite.js");
|
|
198
|
+
assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "checkpoint persisted to local vector db");
|
|
199
|
+
assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel appended");
|
|
200
|
+
// The legacy context handler triggers pi's compaction flow (ctx.compact),
|
|
201
|
+
// which calls our session_before_compact handler to supply the DURABLE trim.
|
|
202
|
+
assert.equal(res, undefined, "legacy context handler returns nothing (no local drop)");
|
|
203
|
+
assert.equal(h.compactCalls.length, 1, "ctx.compact() called to start durable trim (legacy path)");
|
|
204
|
+
// The durable trim was supplied (summary + firstKeptEntryId from pi's prep).
|
|
205
|
+
assert.ok(h.compactCalls[0] !== undefined, "compaction flow executed");
|
|
206
|
+
} finally {
|
|
207
|
+
delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
208
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
test("auto-trigger: skips ctx.compact() when pi would no-op (session too small, legacy path)", async () => {
|
|
213
|
+
const h = harness();
|
|
214
|
+
const messages = h.session;
|
|
215
|
+
// Default floor (20000): the tiny mock transcript is below pi's
|
|
216
|
+
// keepRecentTokens budget, so piCompactWouldNoop() must skip ctx.compact()
|
|
217
|
+
// rather than surface pi's "Nothing to compact (session too small)" throw.
|
|
218
|
+
// S16: exercised under the legacy flag (the default path never calls ctx.compact).
|
|
219
|
+
delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
220
|
+
process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM = "true";
|
|
221
|
+
try {
|
|
222
|
+
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
223
|
+
const res = await h.fire("context", { type: "context", messages }, ctx);
|
|
224
|
+
assert.equal(res, undefined, "legacy context handler returns nothing (no local drop)");
|
|
225
|
+
assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called — pi would no-op");
|
|
226
|
+
// Our recall checkpoint still persisted (Path A) — the durable trim is the
|
|
227
|
+
// only thing skipped; recall is independent of it.
|
|
228
|
+
const { listCheckpoints } = await import("../src/store/sqlite.js");
|
|
229
|
+
assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "recall checkpoint still persisted");
|
|
230
|
+
assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel still appended");
|
|
231
|
+
} finally {
|
|
232
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
233
|
+
}
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
test("auto-trigger (S16): trims the live view and does NOT call ctx.compact()", async () => {
|
|
237
|
+
const h = harness();
|
|
238
|
+
const messages = h.session;
|
|
239
|
+
// S16 default: live context-event trim. No legacy flag. Lower the anchor floor
|
|
240
|
+
// so the trimmed recent window (4 messages, 2 user) clears the anchor check
|
|
241
|
+
// and the live trim actually fires — mirrors how the legacy test lowers the
|
|
242
|
+
// durable floor to exercise its positive path.
|
|
243
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
244
|
+
delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
245
|
+
process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES = "1";
|
|
246
|
+
try {
|
|
247
|
+
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
248
|
+
const res = await h.fire("context", { type: "context", messages }, ctx);
|
|
249
|
+
// S16: context handler returns a TRIMMED messages array (live trim), not undefined.
|
|
250
|
+
assert.ok(res && typeof res === "object", "context handler returns a result object (live trim)");
|
|
251
|
+
assert.ok(Array.isArray((res as any).messages), "result has a trimmed messages array");
|
|
252
|
+
// The trimmed view starts with the compacted summary (user-role) + is shorter.
|
|
253
|
+
assert.ok((res as any).messages.length < messages.length, "trimmed view is shorter than the full session");
|
|
254
|
+
// S16: ctx.compact() is NEVER called (it would stop the agent).
|
|
255
|
+
assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called — compact-and-continue");
|
|
256
|
+
// The recall checkpoint is still persisted (the durable value).
|
|
257
|
+
const { listCheckpoints } = await import("../src/store/sqlite.js");
|
|
258
|
+
assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "recall checkpoint persisted under live trim");
|
|
259
|
+
} finally {
|
|
260
|
+
delete process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
test("auto-trigger (S16): does not trim when below the anchor floor (returns undefined, no ctx.compact)", async () => {
|
|
265
|
+
const h = harness();
|
|
266
|
+
// A session so short that buildLiveTrimmedView's anchor floor can't hold — the
|
|
267
|
+
// live trim skips this call (returns undefined, the next context event retries).
|
|
268
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
269
|
+
delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
270
|
+
const shortSession = [h.session[0], h.session[1]]; // one user + one assistant
|
|
180
271
|
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
181
|
-
const res = await h.fire("context", { type: "context", messages }, ctx);
|
|
182
|
-
//
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
272
|
+
const res = await h.fire("context", { type: "context", messages: shortSession }, ctx);
|
|
273
|
+
// Either it skipped (undefined) or trimmed safely — but it must never call ctx.compact.
|
|
274
|
+
assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called under live trim (short session)");
|
|
275
|
+
if (res === undefined) {
|
|
276
|
+
// skipped path is fine
|
|
277
|
+
assert.ok(true, "below anchor floor → no trim this call (retries next event)");
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
test("auto-trigger (S16): sendUserMessage resume nudge fires only when idle + queued + not already nudged", async () => {
|
|
282
|
+
const h = harness();
|
|
283
|
+
// No queued messages → the nudge must NOT fire (the guard prevents busy-loops).
|
|
284
|
+
// We assert the extension did not throw and did not push a spurious resume.
|
|
285
|
+
const ctx = h.ctx({ isIdle: () => true, hasPendingMessages: () => false });
|
|
286
|
+
await h.fire("agent_end", { type: "agent_end", messages: [] }, ctx);
|
|
287
|
+
// No throw + no spurious nudge side-effect is the contract; appended stays
|
|
288
|
+
// free of any auto "continue" marker when there is no queued work.
|
|
289
|
+
assert.equal(h.appended.some((a) => a.t && /continue/i.test(String(a.d ?? ""))), false, "no spurious continue when no queued work");
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
test("auto-trigger (S16): durable trim still happens via pi native auto-compaction (session_before_compact)", async () => {
|
|
293
|
+
const h = harness();
|
|
294
|
+
// pi's native auto-compaction fires at agent-end with reason "threshold" (the
|
|
295
|
+
// CONTINUING path). Our session_before_compact handler must still supply the
|
|
296
|
+
// durable trim summary — independent of the live context-event trim.
|
|
297
|
+
const prep = {
|
|
298
|
+
firstKeptEntryId: "e2",
|
|
299
|
+
messagesToSummarize: h.session.slice(0, 4),
|
|
300
|
+
tokensBefore: 500,
|
|
301
|
+
};
|
|
302
|
+
const res = await h.fire("session_before_compact", {
|
|
303
|
+
type: "session_before_compact", reason: "threshold", willRetry: false,
|
|
304
|
+
signal: undefined, preparation: prep,
|
|
305
|
+
} as any, h.ctx());
|
|
306
|
+
assert.ok(res?.compaction, "we supply a durable compaction result to pi's native path");
|
|
307
|
+
assert.ok(res.compaction.firstKeptEntryId === "e2", "reuses pi's boundary (PREVENT-PI-002)");
|
|
308
|
+
assert.ok(res.compaction.summary.length > 0, "summary is non-empty");
|
|
193
309
|
});
|
|
194
310
|
|
|
195
311
|
test("session_before_compact supplies our durable trim (not pi's summary)", async () => {
|
|
@@ -331,10 +447,17 @@ test("explicit MEGACOMPACT_THRESHOLD_TOKENS overrides the tier", async () => {
|
|
|
331
447
|
|
|
332
448
|
// ---- /dashboard commands ----------------------------------------------------
|
|
333
449
|
test("/dashboard-status reports no server when pid file missing", async () => {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
450
|
+
// Private base so this asserts "no server" on a range nothing else uses,
|
|
451
|
+
// not the machine-global 9320 family (which may hold a leftover/production server).
|
|
452
|
+
process.env.MEGACOMPACT_DASHBOARD_PORT = "49320";
|
|
453
|
+
try {
|
|
454
|
+
const h = harness();
|
|
455
|
+
const ctx = h.ctx();
|
|
456
|
+
await h.commands["mega-dashboard-status"].handler("", ctx);
|
|
457
|
+
assert.ok(h.notifies.some((n) => n.includes("not running")), "reports no server running");
|
|
458
|
+
} finally {
|
|
459
|
+
delete process.env.MEGACOMPACT_DASHBOARD_PORT;
|
|
460
|
+
}
|
|
338
461
|
});
|
|
339
462
|
|
|
340
463
|
test("/dashboard-stop reports no server when pid file missing", async () => {
|
|
@@ -345,20 +468,23 @@ test("/dashboard-stop reports no server when pid file missing", async () => {
|
|
|
345
468
|
});
|
|
346
469
|
|
|
347
470
|
test("/dashboard skips server spawn when already running", async () => {
|
|
471
|
+
// Use a private dashboard port base for THIS test's harness + fake server so
|
|
472
|
+
// it never races the (parallel, hard-coded-9320) dashboard-server.test.js or
|
|
473
|
+
// a leftover production server. Set BEFORE harness() so registerDashboardCommands
|
|
474
|
+
// reads our base for findLivePort().
|
|
475
|
+
process.env.MEGACOMPACT_DASHBOARD_PORT = "29320";
|
|
348
476
|
const h = harness();
|
|
349
477
|
const confirms: boolean[] = [];
|
|
350
|
-
|
|
351
|
-
// (9320–9329) — isServerRunning() probes those ports, not the port.pid value.
|
|
478
|
+
const livPort = 29320; // inside the harness's private scan range (29320–29329)
|
|
352
479
|
const { createServer } = await import("node:http");
|
|
353
480
|
const server = createServer((_req, res) => {
|
|
354
481
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
355
482
|
res.end(JSON.stringify({ updatedAt: new Date().toISOString(), tier: "test", version: 1, config: {}, session: {}, context: {}, trigger: {}, store: {} }));
|
|
356
483
|
});
|
|
357
|
-
await new Promise<void>((r) => server.listen(
|
|
358
|
-
const addr = server.address() as any;
|
|
484
|
+
await new Promise<void>((r) => server.listen(livPort, "127.0.0.1", r));
|
|
359
485
|
const { join: j } = await import("node:path");
|
|
360
486
|
const { writeFileSync: wf } = await import("node:fs");
|
|
361
|
-
wf(j(h.stateDir, "port.pid"), JSON.stringify({ port:
|
|
487
|
+
wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: livPort, pid: process.pid }));
|
|
362
488
|
|
|
363
489
|
const ctx = h.ctx({
|
|
364
490
|
ui: {
|
|
@@ -375,12 +501,15 @@ test("/dashboard skips server spawn when already running", async () => {
|
|
|
375
501
|
assert.ok(confirms.length > 0, "confirm dialog was shown");
|
|
376
502
|
|
|
377
503
|
await new Promise<void>((r) => server.close(() => r()));
|
|
504
|
+
delete process.env.MEGACOMPACT_DASHBOARD_PORT;
|
|
378
505
|
});
|
|
379
506
|
|
|
380
507
|
test("/dashboard-status reports running after dashboard start", async () => {
|
|
508
|
+
// Private dashboard port base for this harness — never collides with the
|
|
509
|
+
// parallel dashboard-server.test.js (9320 family) or a leftover server.
|
|
510
|
+
process.env.MEGACOMPACT_DASHBOARD_PORT = "39320";
|
|
381
511
|
const h = harness();
|
|
382
|
-
|
|
383
|
-
// (9320–9329) or isServerRunning() won't detect it.
|
|
512
|
+
const livPort = 39320;
|
|
384
513
|
const { createServer } = await import("node:http");
|
|
385
514
|
const { join: j } = await import("node:path");
|
|
386
515
|
const { writeFileSync: wf } = await import("node:fs");
|
|
@@ -388,15 +517,15 @@ test("/dashboard-status reports running after dashboard start", async () => {
|
|
|
388
517
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
389
518
|
res.end(JSON.stringify({ updatedAt: new Date().toISOString(), tier: "test" }));
|
|
390
519
|
});
|
|
391
|
-
await new Promise<void>((r) => server.listen(
|
|
392
|
-
|
|
393
|
-
wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: addr.port, pid: process.pid }));
|
|
520
|
+
await new Promise<void>((r) => server.listen(livPort, "127.0.0.1", r));
|
|
521
|
+
wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: livPort, pid: process.pid }));
|
|
394
522
|
|
|
395
523
|
const ctx = h.ctx();
|
|
396
524
|
await h.commands["mega-dashboard-status"].handler("", ctx);
|
|
397
|
-
assert.ok(h.notifies.some((n) => n.includes("running") && n.includes(String(
|
|
525
|
+
assert.ok(h.notifies.some((n) => n.includes("running") && n.includes(String(livPort))), "reports running with port");
|
|
398
526
|
|
|
399
527
|
await new Promise<void>((r) => server.close(() => r()));
|
|
528
|
+
delete process.env.MEGACOMPACT_DASHBOARD_PORT;
|
|
400
529
|
});
|
|
401
530
|
|
|
402
531
|
test("state snapshot writes dashboard.json after compaction", async () => {
|
|
@@ -443,6 +572,10 @@ test("events.log receives compaction events", async () => {
|
|
|
443
572
|
}
|
|
444
573
|
});
|
|
445
574
|
|
|
446
|
-
test("cleanup", () => {
|
|
575
|
+
test("cleanup", async () => {
|
|
576
|
+
// Terminate the global PGlite cross-repo index (WASM worker thread) so the
|
|
577
|
+
// test process can exit. Without this, node --test never returns even though
|
|
578
|
+
// every test passed — the leaked worker keeps the event loop alive.
|
|
579
|
+
await closeVectorIndex();
|
|
447
580
|
rmSync(baseTmp, { recursive: true, force: true });
|
|
448
581
|
});
|
|
@@ -44,6 +44,23 @@ export interface MegaConfig {
|
|
|
44
44
|
/** RAPTOR hierarchical recall enabled (Fix D). Drives both live recall and
|
|
45
45
|
* the durable-trim summary source (root summary). */
|
|
46
46
|
raptorEnabled: boolean;
|
|
47
|
+
/** Legacy v0.4.28 behavior: auto-trigger calls ctx.compact() (which STOPS
|
|
48
|
+
* the agent). Default false — the S16 redesign uses the live context-event
|
|
49
|
+
* trim + pi native auto-compaction instead (compact and continue). Kept for
|
|
50
|
+
* one release as rollback. */
|
|
51
|
+
legacyDurableTrim: boolean;
|
|
52
|
+
/** Cross-repo recall enabled (S17). Resume + /mega-recall --cross-repo can
|
|
53
|
+
* pull checkpoints from OTHER repos via the PGlite HNSW index. Default true. */
|
|
54
|
+
crossRepoEnabled: boolean;
|
|
55
|
+
/** Stricter cosine floor for cross-repo hits (S17). Default 0.90 (trigram) /
|
|
56
|
+
* tighter than same-repo so only genuinely-relevant cross-repo context is
|
|
57
|
+
* injected. */
|
|
58
|
+
crossRepoCosine: number;
|
|
59
|
+
/** Memory-RAG auto-review enabled (S20). Every memoryReviewInterval turns the
|
|
60
|
+
* conversation is auto-reviewed into durable add/replace/remove memories. */
|
|
61
|
+
memoryAutoReview: boolean;
|
|
62
|
+
/** Turn cadence for the auto-review scan (S20). Default 10. */
|
|
63
|
+
memoryReviewInterval: number;
|
|
47
64
|
/** Token ceiling for the re-injected recall block (Fix C). Recall stops
|
|
48
65
|
* adding checkpoints once the block would exceed this — bounds read-path
|
|
49
66
|
* token cost so it can never net-inflate the window. */
|
|
@@ -104,6 +121,11 @@ export function loadConfig(): MegaConfig {
|
|
|
104
121
|
autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
|
|
105
122
|
dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
|
|
106
123
|
raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
|
|
124
|
+
legacyDurableTrim: envBool("MEGACOMPACT_LEGACY_DURABLE_TRIM", false),
|
|
125
|
+
crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
|
|
126
|
+
crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
|
|
127
|
+
memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
|
|
128
|
+
memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
|
|
107
129
|
recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
|
|
108
130
|
windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
|
|
109
131
|
debug: envBool("MEGACOMPACT_DEBUG", false),
|
|
@@ -126,4 +126,85 @@ export function registerConflictCommands(pi: ExtensionAPI, runtime: MegaRuntime)
|
|
|
126
126
|
for (const m of all) ctx.ui.notify(memoryLine(m));
|
|
127
127
|
},
|
|
128
128
|
});
|
|
129
|
+
|
|
130
|
+
// Shortform aliases — `m save "..."`, `m status`, `m list`, `m search <q>`,
|
|
131
|
+
// `m recall <id>`. Delegates to the same SQLite store so there's one source
|
|
132
|
+
// of truth. The /mega-memory command remains the canonical form.
|
|
133
|
+
pi.registerCommand("m", {
|
|
134
|
+
description: "Shortform alias for /mega-memory. Usage: /m save <text> | list | search <q> | recall <id> | status",
|
|
135
|
+
handler: async (args: string, ctx: ExtensionContext) => {
|
|
136
|
+
const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
|
|
137
|
+
const parts = args.trim().split(/\s+/);
|
|
138
|
+
const sub = parts[0]?.toLowerCase() ?? "list";
|
|
139
|
+
|
|
140
|
+
if (sub === "save") {
|
|
141
|
+
// Strip leading "save" so /m save "#foo bar" works the same as the
|
|
142
|
+
// canonical form. Then strip a balanced outer quote pair if the user
|
|
143
|
+
// wrote /m save "..." — common when the text contains spaces.
|
|
144
|
+
let text = args.trim().slice(4).trim();
|
|
145
|
+
const mq = text.match(/^["“](.*)["”]$/s);
|
|
146
|
+
if (mq) text = mq[1].trim();
|
|
147
|
+
if (!text) {
|
|
148
|
+
ctx.ui.notify('[/m] usage: /m save "<text>" or /m save <text>');
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
152
|
+
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
153
|
+
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
154
|
+
ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
if (sub === "search") {
|
|
159
|
+
const q = parts.slice(1).join(" ").trim();
|
|
160
|
+
if (!q) {
|
|
161
|
+
ctx.ui.notify("[/m] usage: /m search <query>");
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const hits = searchMemories(q, repo, 50, runtime.currentStateDir);
|
|
165
|
+
if (!hits.length) {
|
|
166
|
+
ctx.ui.notify("[/m] no memories match.");
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
for (const mem of hits) ctx.ui.notify(memoryLine(mem));
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (sub === "recall") {
|
|
174
|
+
const id = Number(parts[1]);
|
|
175
|
+
if (!Number.isFinite(id) || parts[1] === undefined) {
|
|
176
|
+
ctx.ui.notify("[/m] usage: /m recall <id>");
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
if (recallMemory(id, runtime.currentStateDir)) {
|
|
180
|
+
const found = listMemories(repo, 1000, runtime.currentStateDir).find((mem) => mem.id === id);
|
|
181
|
+
ctx.ui.notify(found ? `[/m] ${memoryLine(found)}` : `[/m] recalled #${id}`);
|
|
182
|
+
} else {
|
|
183
|
+
ctx.ui.notify(`[/m] #${id} not found.`);
|
|
184
|
+
}
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if (sub === "status") {
|
|
189
|
+
const all = listMemories(repo, 1000, runtime.currentStateDir);
|
|
190
|
+
const byKind = all.reduce<Record<string, number>>((acc, m) => {
|
|
191
|
+
acc[m.kind] = (acc[m.kind] ?? 0) + 1;
|
|
192
|
+
return acc;
|
|
193
|
+
}, {});
|
|
194
|
+
const kinds = Object.entries(byKind).map(([k, n]) => `${k}=${n}`).join(", ");
|
|
195
|
+
const head = `[/m] ${all.length} memory record(s) in ${repo.split(/[\\/]/).pop() ?? repo}`;
|
|
196
|
+
ctx.ui.notify(kinds ? `${head} (${kinds})` : head);
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// default: list
|
|
201
|
+
const all = listMemories(repo, 50, runtime.currentStateDir);
|
|
202
|
+
if (!all.length) {
|
|
203
|
+
ctx.ui.notify("[/m] no saved memories yet. Use /m save <text>.");
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
ctx.ui.notify(`[/m] ${all.length} memory record(s):`);
|
|
207
|
+
for (const mem of all) ctx.ui.notify(memoryLine(mem));
|
|
208
|
+
},
|
|
209
|
+
});
|
|
129
210
|
}
|
|
@@ -23,11 +23,13 @@ export function registerDashboardCommands(pi: ExtensionAPI, runtime: MegaRuntime
|
|
|
23
23
|
// the shipped compiled dist/extensions/dashboard-server.js).
|
|
24
24
|
let dashboardNeedsStrip = false;
|
|
25
25
|
|
|
26
|
-
// The dashboard server binds
|
|
27
|
-
// in dashboard-server.js
|
|
28
|
-
// readiness even when port.pid landed in a different
|
|
26
|
+
// The dashboard server binds a 10-port range starting at MEGACOMPACT_DASHBOARD_PORT
|
|
27
|
+
// (default 9320) — see TARGET_PORT/PORT_RANGE in dashboard-server.js. Probe each for
|
|
28
|
+
// a live /api/snapshot so we detect readiness even when port.pid landed in a different
|
|
29
|
+
// state dir than we poll. Configurable so tests can use a private, non-colliding range.
|
|
30
|
+
const DASH_BASE = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
|
|
29
31
|
async function findLivePort(): Promise<number | null> {
|
|
30
|
-
for (let port =
|
|
32
|
+
for (let port = DASH_BASE; port <= DASH_BASE + 9; port++) {
|
|
31
33
|
try {
|
|
32
34
|
const res = await fetch(`http://localhost:${port}/api/snapshot`, { signal: AbortSignal.timeout(800) }); // guardrails-allow PREVENT-PI-004: localhost liveness probe of the dashboard server this extension spawned
|
|
33
35
|
if (res.ok) return port;
|