pi-mega-compact 0.4.28 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -2
- package/dist/extensions/dashboard-server.js +58 -2
- package/dist/extensions/dashboard-server.test.js +95 -3
- package/dist/extensions/mega-commands.js +25 -9
- package/dist/extensions/mega-compact.test.js +133 -31
- package/dist/extensions/mega-config.js +5 -0
- package/dist/extensions/mega-conflict-cmds.js +79 -0
- package/dist/extensions/mega-dashboard-cmds.js +6 -4
- package/dist/extensions/mega-events.js +144 -27
- package/dist/extensions/mega-pipeline.js +84 -1
- package/dist/extensions/mega-runtime.js +14 -0
- package/dist/extensions/mega-trim.js +48 -0
- package/dist/extensions/mega-trim.test.js +58 -0
- package/dist/src/config/dedup.js +1 -0
- package/dist/src/driftDetection.js +103 -0
- package/dist/src/driftDetection.test.js +87 -0
- package/dist/src/memory.js +147 -0
- package/dist/src/memory.test.js +41 -0
- package/dist/src/memoryConsolidate.test.js +38 -0
- package/dist/src/memoryOps.js +58 -0
- package/dist/src/memoryOps.test.js +41 -0
- package/dist/src/memoryRecall.js +60 -0
- package/dist/src/memoryRecall.test.js +92 -0
- package/dist/src/recall.js +70 -1
- package/dist/src/recall.test.js +69 -1
- package/dist/src/store/sqlite.js +127 -11
- package/dist/src/vectorStore.js +6 -1
- package/extensions/dashboard-server.test.ts +115 -3
- package/extensions/dashboard-server.ts +63 -2
- package/extensions/mega-commands.ts +24 -9
- package/extensions/mega-compact.test.ts +134 -31
- package/extensions/mega-config.ts +22 -0
- package/extensions/mega-conflict-cmds.ts +81 -0
- package/extensions/mega-dashboard-cmds.ts +6 -4
- package/extensions/mega-events.ts +139 -28
- package/extensions/mega-pipeline.ts +94 -1
- package/extensions/mega-runtime.ts +15 -0
- package/extensions/mega-trim.test.ts +64 -0
- package/extensions/mega-trim.ts +75 -0
- package/extensions/openclaw-mega-compact.ts +24 -9
- package/package.json +2 -2
- package/src/config/dedup.ts +2 -0
- package/src/driftDetection.test.ts +100 -0
- package/src/driftDetection.ts +136 -0
- package/src/memory.test.ts +46 -0
- package/src/memory.ts +164 -0
- package/src/memoryConsolidate.test.ts +47 -0
- package/src/memoryOps.test.ts +53 -0
- package/src/memoryOps.ts +75 -0
- package/src/memoryRecall.test.ts +100 -0
- package/src/memoryRecall.ts +83 -0
- package/src/recall.test.ts +77 -1
- package/src/recall.ts +94 -1
- package/src/store/sqlite.ts +188 -11
- package/src/store.ts +3 -0
- package/src/vectorStore.ts +10 -1
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
|
@@ -16,6 +16,7 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync,
|
|
|
16
16
|
import { homedir } from "node:os";
|
|
17
17
|
import { join, dirname } from "node:path";
|
|
18
18
|
import { fileURLToPath } from "node:url";
|
|
19
|
+
import { createRequire } from "node:module";
|
|
19
20
|
import { DatabaseSync } from "node:sqlite";
|
|
20
21
|
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
@@ -774,6 +775,13 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
774
775
|
if (pkg.version) { SERVER_VERSION = pkg.version; break; }
|
|
775
776
|
}
|
|
776
777
|
} catch { /* non-fatal */ }
|
|
778
|
+
|
|
779
|
+
// Lazy-loaded via require so the dashboard stays cheap to boot and we don't
|
|
780
|
+
// need a top-level await in the handler.
|
|
781
|
+
const driftReq = createRequire(import.meta.url);
|
|
782
|
+
const detectCrossRepoDrift = (idxDir: string) =>
|
|
783
|
+
(driftReq("../src/driftDetection.js") as typeof import("../src/driftDetection.js"))
|
|
784
|
+
.detectCrossRepoDrift(idxDir);
|
|
777
785
|
const portFile = join(stateDir, "port.pid");
|
|
778
786
|
const snapshotPath = join(stateDir, "dashboard.json");
|
|
779
787
|
const eventsPath = join(stateDir, "events.log");
|
|
@@ -858,6 +866,54 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
858
866
|
return;
|
|
859
867
|
}
|
|
860
868
|
|
|
869
|
+
// /api/repos — registry list. Optional `?active=24h` filters to repos
|
|
870
|
+
// seen within the last N hours (default: all). The dashboard uses this to
|
|
871
|
+
// drive its "active vs archived" badge without refetching /api/index.
|
|
872
|
+
if (req.url?.startsWith("/api/repos")) {
|
|
873
|
+
const url = new URL(req.url, "http://x");
|
|
874
|
+
const activeParam = url.searchParams.get("active");
|
|
875
|
+
const idx = readIndex() ?? { updatedAt: null, summary: null, repos: [] };
|
|
876
|
+
let repos = (idx.repos ?? []) as IndexRepo[];
|
|
877
|
+
if (activeParam) {
|
|
878
|
+
const m = /^(\d+)h$/.exec(activeParam);
|
|
879
|
+
if (m) {
|
|
880
|
+
const cutoffSec = Math.floor(Date.now() / 1000) - Number(m[1]) * 3600;
|
|
881
|
+
repos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec);
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
885
|
+
res.end(JSON.stringify({ updatedAt: idx.updatedAt, repos, count: repos.length }));
|
|
886
|
+
return;
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
// /api/summary — header tiles without the full repo list (keeps payload
|
|
890
|
+
// small for embed scenarios). activeRepos mirrors the /api/repos?active=24h
|
|
891
|
+
// count so the dashboard can render the active badge alongside totals.
|
|
892
|
+
if (req.url?.startsWith("/api/summary")) {
|
|
893
|
+
const idx = readIndex() ?? { updatedAt: null, summary: null, repos: [] };
|
|
894
|
+
const repos = (idx.repos ?? []) as IndexRepo[];
|
|
895
|
+
const cutoffSec = Math.floor(Date.now() / 1000) - 24 * 3600;
|
|
896
|
+
const activeRepos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec).length;
|
|
897
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
898
|
+
res.end(JSON.stringify({
|
|
899
|
+
updatedAt: idx.updatedAt,
|
|
900
|
+
summary: idx.summary,
|
|
901
|
+
activeRepos,
|
|
902
|
+
totalRepos: repos.length,
|
|
903
|
+
}));
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
// /api/drift — R4: cross-repo drift report over repo_registry. Flags stale
|
|
908
|
+
// repos (>30d idle), compaction lag (active but >24h since last
|
|
909
|
+
// compaction), and recent model churn. Read-only.
|
|
910
|
+
if (req.url?.startsWith("/api/drift")) {
|
|
911
|
+
const report = detectCrossRepoDrift(getIndexDir());
|
|
912
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
913
|
+
res.end(JSON.stringify(report));
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
|
|
861
917
|
if (req.url === "/api/events") {
|
|
862
918
|
res.writeHead(200, {
|
|
863
919
|
"Content-Type": "text/event-stream",
|
|
@@ -924,8 +980,13 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
924
980
|
res.end(dashboardHtml(tier));
|
|
925
981
|
});
|
|
926
982
|
|
|
927
|
-
|
|
928
|
-
|
|
983
|
+
// Bind base + range are env-configurable so tests can use a private,
|
|
984
|
+
// non-colliding range (parallel runs / leftover servers from killed runs
|
|
985
|
+
// would otherwise EADDRINUSE on the machine-global 9320 range). Default
|
|
986
|
+
// MEGACOMPACT_DASHBOARD_PORT=9320 (10-port range 9320–9329) preserves the
|
|
987
|
+
// production behavior.
|
|
988
|
+
const TARGET_PORT = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
|
|
989
|
+
const PORT_RANGE = 10; // TARGET_PORT..TARGET_PORT+9
|
|
929
990
|
|
|
930
991
|
return new Promise((resolve, reject) => {
|
|
931
992
|
function tryPort(port: number) {
|
|
@@ -9,11 +9,11 @@
|
|
|
9
9
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
10
10
|
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
11
11
|
import { normalizeSessionId } from "../src/store.js";
|
|
12
|
-
import { listCheckpoints, latestModelSnapshot } from "../src/store/sqlite.js";
|
|
12
|
+
import { listCheckpoints, latestModelSnapshot, countInjectedGlobal, listRepoRegistry } from "../src/store/sqlite.js";
|
|
13
13
|
import { decompressSmart } from "../src/store/compression.js";
|
|
14
14
|
import { loadMetrics, fpRate, p95 } from "../src/monitoring.js";
|
|
15
15
|
import { MegaRuntime, C, recentUserQuery } from "./mega-runtime.js";
|
|
16
|
-
import { runCompact, doRecall } from "./mega-pipeline.js";
|
|
16
|
+
import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
|
|
17
17
|
import { setTier, COMPACT_TIERS, type MegaConfig, type CompactTier } from "./mega-config.js";
|
|
18
18
|
|
|
19
19
|
/** Resolve a checkpoint by id (or "recent"/"last") from this session's store. */
|
|
@@ -47,16 +47,21 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
47
47
|
});
|
|
48
48
|
|
|
49
49
|
pi.registerCommand("mega-recall", {
|
|
50
|
-
description: "Recall relevant compacted context from the vector store and inline it.",
|
|
50
|
+
description: "Recall relevant compacted context from the vector store and inline it. Use --cross-repo to search all repos.",
|
|
51
51
|
handler: async (args: string, ctx: ExtensionContext) => {
|
|
52
|
-
|
|
52
|
+
// S17: --cross-repo (or --cross repo) runs the async path over every repo's
|
|
53
|
+
// PGlite HNSW index (stricter cosine floor + source labels).
|
|
54
|
+
const crossRepo = /\-\-cross[\- ]repo\b/.test(args);
|
|
55
|
+
const query = args.replace(/--cross[\- ]repo\b/, "").trim() || recentUserQuery(ctx);
|
|
53
56
|
if (!query) {
|
|
54
57
|
ctx.ui.notify("[mega-compact] /mega-recall needs a query or a prior user message.");
|
|
55
58
|
return;
|
|
56
59
|
}
|
|
57
|
-
const r =
|
|
60
|
+
const r = crossRepo
|
|
61
|
+
? await doRecallAsync(runtime, config, ctx, query, "command", { crossRepo: true })
|
|
62
|
+
: doRecall(runtime, config, ctx, query, "command");
|
|
58
63
|
if (r.empty) {
|
|
59
|
-
runtime.logger.info("recall-empty", { query });
|
|
64
|
+
runtime.logger.info("recall-empty", { query, crossRepo });
|
|
60
65
|
ctx.ui.notify(`[mega-compact] recall found nothing new for "${query}".`);
|
|
61
66
|
return;
|
|
62
67
|
}
|
|
@@ -64,10 +69,10 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
64
69
|
// injection). Report what was selected now for immediate feedback.
|
|
65
70
|
runtime.pendingRecallBlock = r.block;
|
|
66
71
|
const list = r.report.map((l) => l).join("\n");
|
|
67
|
-
runtime.logger.info("recall", { query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
|
|
68
|
-
runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt`);
|
|
72
|
+
runtime.logger.info("recall", { query, crossRepo, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
|
|
73
|
+
runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt${crossRepo ? " (cross-repo)" : ""}`);
|
|
69
74
|
ctx.ui.notify(
|
|
70
|
-
`[mega-compact] recall staged ${r.toInject.length} checkpoint(s) for "${query}":\n${list}\n` +
|
|
75
|
+
`[mega-compact] recall staged ${r.toInject.length} checkpoint(s) for "${query}"${crossRepo ? " (cross-repo)" : ""}:\n${list}\n` +
|
|
71
76
|
`(injected at the next turn via system prompt)`,
|
|
72
77
|
);
|
|
73
78
|
},
|
|
@@ -111,6 +116,15 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
111
116
|
const p95L2 = p95(m.latency.L2 ?? []);
|
|
112
117
|
const relPct = (st.dedupHitRate * 100).toFixed(0);
|
|
113
118
|
const qualityStr = `recall ${relPct}% relevant · FP ${(fp * 100).toFixed(1)}% · L2 p95 ${p95L2.toFixed(0)}ms`;
|
|
119
|
+
// S18: cross-repo stats from the machine-wide index (best-effort; the
|
|
120
|
+
// index dir may be unset → 0/empty, never throws).
|
|
121
|
+
let crossRepoInjections = 0;
|
|
122
|
+
let repoCount = 0;
|
|
123
|
+
try {
|
|
124
|
+
crossRepoInjections = countInjectedGlobal(process.env.MEGACOMPACT_INDEX_DIR);
|
|
125
|
+
repoCount = listRepoRegistry(process.env.MEGACOMPACT_INDEX_DIR).length;
|
|
126
|
+
} catch { /* non-fatal */ }
|
|
127
|
+
const crossRepoStr = `${crossRepoInjections} cross-repo injections recorded · ${repoCount} repos indexed`;
|
|
114
128
|
ctx.ui.notify(
|
|
115
129
|
`[mega-compact] pct=${pct} tokens=${tokens} tier=${config.tier} fastGate=${config.fastGatePct}% ` +
|
|
116
130
|
`threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
|
|
@@ -126,6 +140,7 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
126
140
|
`[mega-compact] 💰 ${costStr}\n` +
|
|
127
141
|
`[mega-compact] 🤖 model: ${modelStr}\n` +
|
|
128
142
|
`[mega-compact] 🎯 ${qualityStr}\n` +
|
|
143
|
+
`[mega-compact] 🌐 ${crossRepoStr}\n` +
|
|
129
144
|
`[mega-compact] stateDir=${runtime.currentStateDir}`,
|
|
130
145
|
);
|
|
131
146
|
},
|
|
@@ -18,6 +18,7 @@ import { mkdtempSync, rmSync } from "node:fs";
|
|
|
18
18
|
import { tmpdir } from "node:os";
|
|
19
19
|
import { join } from "node:path";
|
|
20
20
|
import { createRequire } from "node:module";
|
|
21
|
+
import { closeVectorIndex } from "../src/store/vectorIndex.js";
|
|
21
22
|
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
22
23
|
|
|
23
24
|
const require = createRequire(import.meta.url);
|
|
@@ -177,14 +178,18 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
|
|
|
177
178
|
};
|
|
178
179
|
}
|
|
179
180
|
|
|
180
|
-
test("auto-trigger: past threshold persists a chkpt and starts a durable trim", async () => {
|
|
181
|
+
test("auto-trigger (legacy): past threshold persists a chkpt and starts a durable trim via ctx.compact", async () => {
|
|
181
182
|
const h = harness();
|
|
182
183
|
const messages = h.session;
|
|
183
184
|
// The mock session is tiny (~100 tokens). piCompactWouldNoop() would skip
|
|
184
185
|
// ctx.compact() for a transcript under pi's keepRecentTokens budget — so
|
|
185
186
|
// lower the floor to 0 to simulate a transcript large enough that pi WOULD
|
|
186
187
|
// compact (the positive path this test exercises).
|
|
188
|
+
// S16: this is the LEGACY path — the default no longer calls ctx.compact()
|
|
189
|
+
// (it returns a live-trimmed view instead). Set the legacy flag to exercise
|
|
190
|
+
// the v0.4.28 ctx.compact durable-trim flow this test asserts.
|
|
187
191
|
process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR = "0";
|
|
192
|
+
process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM = "true";
|
|
188
193
|
try {
|
|
189
194
|
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
190
195
|
const res = await h.fire("context", { type: "context", messages }, ctx);
|
|
@@ -192,34 +197,115 @@ test("auto-trigger: past threshold persists a chkpt and starts a durable trim",
|
|
|
192
197
|
const { listCheckpoints } = await import("../src/store/sqlite.js");
|
|
193
198
|
assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "checkpoint persisted to local vector db");
|
|
194
199
|
assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel appended");
|
|
195
|
-
// The context handler
|
|
196
|
-
//
|
|
197
|
-
|
|
198
|
-
assert.equal(
|
|
199
|
-
assert.equal(h.compactCalls.length, 1, "ctx.compact() called to start durable trim");
|
|
200
|
+
// The legacy context handler triggers pi's compaction flow (ctx.compact),
|
|
201
|
+
// which calls our session_before_compact handler to supply the DURABLE trim.
|
|
202
|
+
assert.equal(res, undefined, "legacy context handler returns nothing (no local drop)");
|
|
203
|
+
assert.equal(h.compactCalls.length, 1, "ctx.compact() called to start durable trim (legacy path)");
|
|
200
204
|
// The durable trim was supplied (summary + firstKeptEntryId from pi's prep).
|
|
201
205
|
assert.ok(h.compactCalls[0] !== undefined, "compaction flow executed");
|
|
202
206
|
} finally {
|
|
203
207
|
delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
208
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
204
209
|
}
|
|
205
210
|
});
|
|
206
211
|
|
|
207
|
-
test("auto-trigger: skips ctx.compact() when pi would no-op (session too small)", async () => {
|
|
212
|
+
test("auto-trigger: skips ctx.compact() when pi would no-op (session too small, legacy path)", async () => {
|
|
208
213
|
const h = harness();
|
|
209
214
|
const messages = h.session;
|
|
210
215
|
// Default floor (20000): the tiny mock transcript is below pi's
|
|
211
216
|
// keepRecentTokens budget, so piCompactWouldNoop() must skip ctx.compact()
|
|
212
217
|
// rather than surface pi's "Nothing to compact (session too small)" throw.
|
|
218
|
+
// S16: exercised under the legacy flag (the default path never calls ctx.compact).
|
|
213
219
|
delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
220
|
+
process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM = "true";
|
|
221
|
+
try {
|
|
222
|
+
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
223
|
+
const res = await h.fire("context", { type: "context", messages }, ctx);
|
|
224
|
+
assert.equal(res, undefined, "legacy context handler returns nothing (no local drop)");
|
|
225
|
+
assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called — pi would no-op");
|
|
226
|
+
// Our recall checkpoint still persisted (Path A) — the durable trim is the
|
|
227
|
+
// only thing skipped; recall is independent of it.
|
|
228
|
+
const { listCheckpoints } = await import("../src/store/sqlite.js");
|
|
229
|
+
assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "recall checkpoint still persisted");
|
|
230
|
+
assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel still appended");
|
|
231
|
+
} finally {
|
|
232
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
233
|
+
}
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
test("auto-trigger (S16): trims the live view and does NOT call ctx.compact()", async () => {
|
|
237
|
+
const h = harness();
|
|
238
|
+
const messages = h.session;
|
|
239
|
+
// S16 default: live context-event trim. No legacy flag. Lower the anchor floor
|
|
240
|
+
// so the trimmed recent window (4 messages, 2 user) clears the anchor check
|
|
241
|
+
// and the live trim actually fires — mirrors how the legacy test lowers the
|
|
242
|
+
// durable floor to exercise its positive path.
|
|
243
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
244
|
+
delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
245
|
+
process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES = "1";
|
|
246
|
+
try {
|
|
247
|
+
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
248
|
+
const res = await h.fire("context", { type: "context", messages }, ctx);
|
|
249
|
+
// S16: context handler returns a TRIMMED messages array (live trim), not undefined.
|
|
250
|
+
assert.ok(res && typeof res === "object", "context handler returns a result object (live trim)");
|
|
251
|
+
assert.ok(Array.isArray((res as any).messages), "result has a trimmed messages array");
|
|
252
|
+
// The trimmed view starts with the compacted summary (user-role) + is shorter.
|
|
253
|
+
assert.ok((res as any).messages.length < messages.length, "trimmed view is shorter than the full session");
|
|
254
|
+
// S16: ctx.compact() is NEVER called (it would stop the agent).
|
|
255
|
+
assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called — compact-and-continue");
|
|
256
|
+
// The recall checkpoint is still persisted (the durable value).
|
|
257
|
+
const { listCheckpoints } = await import("../src/store/sqlite.js");
|
|
258
|
+
assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "recall checkpoint persisted under live trim");
|
|
259
|
+
} finally {
|
|
260
|
+
delete process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
test("auto-trigger (S16): does not trim when below the anchor floor (returns undefined, no ctx.compact)", async () => {
|
|
265
|
+
const h = harness();
|
|
266
|
+
// A session so short that buildLiveTrimmedView's anchor floor can't hold — the
|
|
267
|
+
// live trim skips this call (returns undefined, the next context event retries).
|
|
268
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
269
|
+
delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
270
|
+
const shortSession = [h.session[0], h.session[1]]; // one user + one assistant
|
|
214
271
|
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
215
|
-
const res = await h.fire("context", { type: "context", messages }, ctx);
|
|
216
|
-
|
|
217
|
-
assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
272
|
+
const res = await h.fire("context", { type: "context", messages: shortSession }, ctx);
|
|
273
|
+
// Either it skipped (undefined) or trimmed safely — but it must never call ctx.compact.
|
|
274
|
+
assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called under live trim (short session)");
|
|
275
|
+
if (res === undefined) {
|
|
276
|
+
// skipped path is fine
|
|
277
|
+
assert.ok(true, "below anchor floor → no trim this call (retries next event)");
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
test("auto-trigger (S16): sendUserMessage resume nudge fires only when idle + queued + not already nudged", async () => {
|
|
282
|
+
const h = harness();
|
|
283
|
+
// No queued messages → the nudge must NOT fire (the guard prevents busy-loops).
|
|
284
|
+
// We assert the extension did not throw and did not push a spurious resume.
|
|
285
|
+
const ctx = h.ctx({ isIdle: () => true, hasPendingMessages: () => false });
|
|
286
|
+
await h.fire("agent_end", { type: "agent_end", messages: [] }, ctx);
|
|
287
|
+
// No throw + no spurious nudge side-effect is the contract; appended stays
|
|
288
|
+
// free of any auto "continue" marker when there is no queued work.
|
|
289
|
+
assert.equal(h.appended.some((a) => a.t && /continue/i.test(String(a.d ?? ""))), false, "no spurious continue when no queued work");
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
test("auto-trigger (S16): durable trim still happens via pi native auto-compaction (session_before_compact)", async () => {
|
|
293
|
+
const h = harness();
|
|
294
|
+
// pi's native auto-compaction fires at agent-end with reason "threshold" (the
|
|
295
|
+
// CONTINUING path). Our session_before_compact handler must still supply the
|
|
296
|
+
// durable trim summary — independent of the live context-event trim.
|
|
297
|
+
const prep = {
|
|
298
|
+
firstKeptEntryId: "e2",
|
|
299
|
+
messagesToSummarize: h.session.slice(0, 4),
|
|
300
|
+
tokensBefore: 500,
|
|
301
|
+
};
|
|
302
|
+
const res = await h.fire("session_before_compact", {
|
|
303
|
+
type: "session_before_compact", reason: "threshold", willRetry: false,
|
|
304
|
+
signal: undefined, preparation: prep,
|
|
305
|
+
} as any, h.ctx());
|
|
306
|
+
assert.ok(res?.compaction, "we supply a durable compaction result to pi's native path");
|
|
307
|
+
assert.ok(res.compaction.firstKeptEntryId === "e2", "reuses pi's boundary (PREVENT-PI-002)");
|
|
308
|
+
assert.ok(res.compaction.summary.length > 0, "summary is non-empty");
|
|
223
309
|
});
|
|
224
310
|
|
|
225
311
|
test("session_before_compact supplies our durable trim (not pi's summary)", async () => {
|
|
@@ -361,10 +447,17 @@ test("explicit MEGACOMPACT_THRESHOLD_TOKENS overrides the tier", async () => {
|
|
|
361
447
|
|
|
362
448
|
// ---- /dashboard commands ----------------------------------------------------
|
|
363
449
|
test("/dashboard-status reports no server when pid file missing", async () => {
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
450
|
+
// Private base so this asserts "no server" on a range nothing else uses,
|
|
451
|
+
// not the machine-global 9320 family (which may hold a leftover/production server).
|
|
452
|
+
process.env.MEGACOMPACT_DASHBOARD_PORT = "49320";
|
|
453
|
+
try {
|
|
454
|
+
const h = harness();
|
|
455
|
+
const ctx = h.ctx();
|
|
456
|
+
await h.commands["mega-dashboard-status"].handler("", ctx);
|
|
457
|
+
assert.ok(h.notifies.some((n) => n.includes("not running")), "reports no server running");
|
|
458
|
+
} finally {
|
|
459
|
+
delete process.env.MEGACOMPACT_DASHBOARD_PORT;
|
|
460
|
+
}
|
|
368
461
|
});
|
|
369
462
|
|
|
370
463
|
test("/dashboard-stop reports no server when pid file missing", async () => {
|
|
@@ -375,20 +468,23 @@ test("/dashboard-stop reports no server when pid file missing", async () => {
|
|
|
375
468
|
});
|
|
376
469
|
|
|
377
470
|
test("/dashboard skips server spawn when already running", async () => {
|
|
471
|
+
// Use a private dashboard port base for THIS test's harness + fake server so
|
|
472
|
+
// it never races the (parallel, hard-coded-9320) dashboard-server.test.js or
|
|
473
|
+
// a leftover production server. Set BEFORE harness() so registerDashboardCommands
|
|
474
|
+
// reads our base for findLivePort().
|
|
475
|
+
process.env.MEGACOMPACT_DASHBOARD_PORT = "29320";
|
|
378
476
|
const h = harness();
|
|
379
477
|
const confirms: boolean[] = [];
|
|
380
|
-
|
|
381
|
-
// (9320–9329) — isServerRunning() probes those ports, not the port.pid value.
|
|
478
|
+
const livPort = 29320; // inside the harness's private scan range (29320–29329)
|
|
382
479
|
const { createServer } = await import("node:http");
|
|
383
480
|
const server = createServer((_req, res) => {
|
|
384
481
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
385
482
|
res.end(JSON.stringify({ updatedAt: new Date().toISOString(), tier: "test", version: 1, config: {}, session: {}, context: {}, trigger: {}, store: {} }));
|
|
386
483
|
});
|
|
387
|
-
await new Promise<void>((r) => server.listen(
|
|
388
|
-
const addr = server.address() as any;
|
|
484
|
+
await new Promise<void>((r) => server.listen(livPort, "127.0.0.1", r));
|
|
389
485
|
const { join: j } = await import("node:path");
|
|
390
486
|
const { writeFileSync: wf } = await import("node:fs");
|
|
391
|
-
wf(j(h.stateDir, "port.pid"), JSON.stringify({ port:
|
|
487
|
+
wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: livPort, pid: process.pid }));
|
|
392
488
|
|
|
393
489
|
const ctx = h.ctx({
|
|
394
490
|
ui: {
|
|
@@ -405,12 +501,15 @@ test("/dashboard skips server spawn when already running", async () => {
|
|
|
405
501
|
assert.ok(confirms.length > 0, "confirm dialog was shown");
|
|
406
502
|
|
|
407
503
|
await new Promise<void>((r) => server.close(() => r()));
|
|
504
|
+
delete process.env.MEGACOMPACT_DASHBOARD_PORT;
|
|
408
505
|
});
|
|
409
506
|
|
|
410
507
|
test("/dashboard-status reports running after dashboard start", async () => {
|
|
508
|
+
// Private dashboard port base for this harness — never collides with the
|
|
509
|
+
// parallel dashboard-server.test.js (9320 family) or a leftover server.
|
|
510
|
+
process.env.MEGACOMPACT_DASHBOARD_PORT = "39320";
|
|
411
511
|
const h = harness();
|
|
412
|
-
|
|
413
|
-
// (9320–9329) or isServerRunning() won't detect it.
|
|
512
|
+
const livPort = 39320;
|
|
414
513
|
const { createServer } = await import("node:http");
|
|
415
514
|
const { join: j } = await import("node:path");
|
|
416
515
|
const { writeFileSync: wf } = await import("node:fs");
|
|
@@ -418,15 +517,15 @@ test("/dashboard-status reports running after dashboard start", async () => {
|
|
|
418
517
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
419
518
|
res.end(JSON.stringify({ updatedAt: new Date().toISOString(), tier: "test" }));
|
|
420
519
|
});
|
|
421
|
-
await new Promise<void>((r) => server.listen(
|
|
422
|
-
|
|
423
|
-
wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: addr.port, pid: process.pid }));
|
|
520
|
+
await new Promise<void>((r) => server.listen(livPort, "127.0.0.1", r));
|
|
521
|
+
wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: livPort, pid: process.pid }));
|
|
424
522
|
|
|
425
523
|
const ctx = h.ctx();
|
|
426
524
|
await h.commands["mega-dashboard-status"].handler("", ctx);
|
|
427
|
-
assert.ok(h.notifies.some((n) => n.includes("running") && n.includes(String(
|
|
525
|
+
assert.ok(h.notifies.some((n) => n.includes("running") && n.includes(String(livPort))), "reports running with port");
|
|
428
526
|
|
|
429
527
|
await new Promise<void>((r) => server.close(() => r()));
|
|
528
|
+
delete process.env.MEGACOMPACT_DASHBOARD_PORT;
|
|
430
529
|
});
|
|
431
530
|
|
|
432
531
|
test("state snapshot writes dashboard.json after compaction", async () => {
|
|
@@ -473,6 +572,10 @@ test("events.log receives compaction events", async () => {
|
|
|
473
572
|
}
|
|
474
573
|
});
|
|
475
574
|
|
|
476
|
-
test("cleanup", () => {
|
|
575
|
+
test("cleanup", async () => {
|
|
576
|
+
// Terminate the global PGlite cross-repo index (WASM worker thread) so the
|
|
577
|
+
// test process can exit. Without this, node --test never returns even though
|
|
578
|
+
// every test passed — the leaked worker keeps the event loop alive.
|
|
579
|
+
await closeVectorIndex();
|
|
477
580
|
rmSync(baseTmp, { recursive: true, force: true });
|
|
478
581
|
});
|
|
@@ -44,6 +44,23 @@ export interface MegaConfig {
|
|
|
44
44
|
/** RAPTOR hierarchical recall enabled (Fix D). Drives both live recall and
|
|
45
45
|
* the durable-trim summary source (root summary). */
|
|
46
46
|
raptorEnabled: boolean;
|
|
47
|
+
/** Legacy v0.4.28 behavior: auto-trigger calls ctx.compact() (which STOPS
|
|
48
|
+
* the agent). Default false — the S16 redesign uses the live context-event
|
|
49
|
+
* trim + pi native auto-compaction instead (compact and continue). Kept for
|
|
50
|
+
* one release as rollback. */
|
|
51
|
+
legacyDurableTrim: boolean;
|
|
52
|
+
/** Cross-repo recall enabled (S17). Resume + /mega-recall --cross-repo can
|
|
53
|
+
* pull checkpoints from OTHER repos via the PGlite HNSW index. Default true. */
|
|
54
|
+
crossRepoEnabled: boolean;
|
|
55
|
+
/** Stricter cosine floor for cross-repo hits (S17). Default 0.90 (trigram) /
|
|
56
|
+
* tighter than same-repo so only genuinely-relevant cross-repo context is
|
|
57
|
+
* injected. */
|
|
58
|
+
crossRepoCosine: number;
|
|
59
|
+
/** Memory-RAG auto-review enabled (S20). Every memoryReviewInterval turns the
|
|
60
|
+
* conversation is auto-reviewed into durable add/replace/remove memories. */
|
|
61
|
+
memoryAutoReview: boolean;
|
|
62
|
+
/** Turn cadence for the auto-review scan (S20). Default 10. */
|
|
63
|
+
memoryReviewInterval: number;
|
|
47
64
|
/** Token ceiling for the re-injected recall block (Fix C). Recall stops
|
|
48
65
|
* adding checkpoints once the block would exceed this — bounds read-path
|
|
49
66
|
* token cost so it can never net-inflate the window. */
|
|
@@ -104,6 +121,11 @@ export function loadConfig(): MegaConfig {
|
|
|
104
121
|
autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
|
|
105
122
|
dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
|
|
106
123
|
raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
|
|
124
|
+
legacyDurableTrim: envBool("MEGACOMPACT_LEGACY_DURABLE_TRIM", false),
|
|
125
|
+
crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
|
|
126
|
+
crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
|
|
127
|
+
memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
|
|
128
|
+
memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
|
|
107
129
|
recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
|
|
108
130
|
windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
|
|
109
131
|
debug: envBool("MEGACOMPACT_DEBUG", false),
|
|
@@ -126,4 +126,85 @@ export function registerConflictCommands(pi: ExtensionAPI, runtime: MegaRuntime)
|
|
|
126
126
|
for (const m of all) ctx.ui.notify(memoryLine(m));
|
|
127
127
|
},
|
|
128
128
|
});
|
|
129
|
+
|
|
130
|
+
// Shortform aliases — `m save "..."`, `m status`, `m list`, `m search <q>`,
|
|
131
|
+
// `m recall <id>`. Delegates to the same SQLite store so there's one source
|
|
132
|
+
// of truth. The /mega-memory command remains the canonical form.
|
|
133
|
+
pi.registerCommand("m", {
|
|
134
|
+
description: "Shortform alias for /mega-memory. Usage: /m save <text> | list | search <q> | recall <id> | status",
|
|
135
|
+
handler: async (args: string, ctx: ExtensionContext) => {
|
|
136
|
+
const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
|
|
137
|
+
const parts = args.trim().split(/\s+/);
|
|
138
|
+
const sub = parts[0]?.toLowerCase() ?? "list";
|
|
139
|
+
|
|
140
|
+
if (sub === "save") {
|
|
141
|
+
// Strip leading "save" so /m save "#foo bar" works the same as the
|
|
142
|
+
// canonical form. Then strip a balanced outer quote pair if the user
|
|
143
|
+
// wrote /m save "..." — common when the text contains spaces.
|
|
144
|
+
let text = args.trim().slice(4).trim();
|
|
145
|
+
const mq = text.match(/^["“](.*)["”]$/s);
|
|
146
|
+
if (mq) text = mq[1].trim();
|
|
147
|
+
if (!text) {
|
|
148
|
+
ctx.ui.notify('[/m] usage: /m save "<text>" or /m save <text>');
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
152
|
+
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
153
|
+
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
154
|
+
ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
if (sub === "search") {
|
|
159
|
+
const q = parts.slice(1).join(" ").trim();
|
|
160
|
+
if (!q) {
|
|
161
|
+
ctx.ui.notify("[/m] usage: /m search <query>");
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const hits = searchMemories(q, repo, 50, runtime.currentStateDir);
|
|
165
|
+
if (!hits.length) {
|
|
166
|
+
ctx.ui.notify("[/m] no memories match.");
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
for (const mem of hits) ctx.ui.notify(memoryLine(mem));
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (sub === "recall") {
|
|
174
|
+
const id = Number(parts[1]);
|
|
175
|
+
if (!Number.isFinite(id) || parts[1] === undefined) {
|
|
176
|
+
ctx.ui.notify("[/m] usage: /m recall <id>");
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
if (recallMemory(id, runtime.currentStateDir)) {
|
|
180
|
+
const found = listMemories(repo, 1000, runtime.currentStateDir).find((mem) => mem.id === id);
|
|
181
|
+
ctx.ui.notify(found ? `[/m] ${memoryLine(found)}` : `[/m] recalled #${id}`);
|
|
182
|
+
} else {
|
|
183
|
+
ctx.ui.notify(`[/m] #${id} not found.`);
|
|
184
|
+
}
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if (sub === "status") {
|
|
189
|
+
const all = listMemories(repo, 1000, runtime.currentStateDir);
|
|
190
|
+
const byKind = all.reduce<Record<string, number>>((acc, m) => {
|
|
191
|
+
acc[m.kind] = (acc[m.kind] ?? 0) + 1;
|
|
192
|
+
return acc;
|
|
193
|
+
}, {});
|
|
194
|
+
const kinds = Object.entries(byKind).map(([k, n]) => `${k}=${n}`).join(", ");
|
|
195
|
+
const head = `[/m] ${all.length} memory record(s) in ${repo.split(/[\\/]/).pop() ?? repo}`;
|
|
196
|
+
ctx.ui.notify(kinds ? `${head} (${kinds})` : head);
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// default: list
|
|
201
|
+
const all = listMemories(repo, 50, runtime.currentStateDir);
|
|
202
|
+
if (!all.length) {
|
|
203
|
+
ctx.ui.notify("[/m] no saved memories yet. Use /m save <text>.");
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
ctx.ui.notify(`[/m] ${all.length} memory record(s):`);
|
|
207
|
+
for (const mem of all) ctx.ui.notify(memoryLine(mem));
|
|
208
|
+
},
|
|
209
|
+
});
|
|
129
210
|
}
|
|
@@ -23,11 +23,13 @@ export function registerDashboardCommands(pi: ExtensionAPI, runtime: MegaRuntime
|
|
|
23
23
|
// the shipped compiled dist/extensions/dashboard-server.js).
|
|
24
24
|
let dashboardNeedsStrip = false;
|
|
25
25
|
|
|
26
|
-
// The dashboard server binds
|
|
27
|
-
// in dashboard-server.js
|
|
28
|
-
// readiness even when port.pid landed in a different
|
|
26
|
+
// The dashboard server binds a 10-port range starting at MEGACOMPACT_DASHBOARD_PORT
|
|
27
|
+
// (default 9320) — see TARGET_PORT/PORT_RANGE in dashboard-server.js. Probe each for
|
|
28
|
+
// a live /api/snapshot so we detect readiness even when port.pid landed in a different
|
|
29
|
+
// state dir than we poll. Configurable so tests can use a private, non-colliding range.
|
|
30
|
+
const DASH_BASE = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
|
|
29
31
|
async function findLivePort(): Promise<number | null> {
|
|
30
|
-
for (let port =
|
|
32
|
+
for (let port = DASH_BASE; port <= DASH_BASE + 9; port++) {
|
|
31
33
|
try {
|
|
32
34
|
const res = await fetch(`http://localhost:${port}/api/snapshot`, { signal: AbortSignal.timeout(800) }); // guardrails-allow PREVENT-PI-004: localhost liveness probe of the dashboard server this extension spawned
|
|
33
35
|
if (res.ok) return port;
|