pi-mega-compact 0.4.20 → 0.4.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/conflict-scan.js +201 -0
- package/dist/extensions/dashboard-server.js +3 -3
- package/dist/extensions/mega-compact-driver.js +79 -0
- package/dist/extensions/mega-compact.js +2 -0
- package/dist/extensions/mega-compact.test.js +54 -18
- package/dist/extensions/mega-config.js +10 -0
- package/dist/extensions/mega-conflict-cmds.js +121 -0
- package/dist/extensions/mega-events.js +45 -23
- package/dist/extensions/mega-pipeline.js +80 -8
- package/dist/extensions/mega-runtime.js +14 -20
- package/dist/src/config/dedup.js +4 -1
- package/dist/src/config.js +21 -0
- package/dist/src/dedup/raptor/index.js +28 -6
- package/dist/src/dedup/raptor/promote.test.js +69 -0
- package/dist/src/engine.js +1 -0
- package/dist/src/recall.js +30 -4
- package/dist/src/recall.test.js +28 -0
- package/dist/src/store/backfill.js +5 -6
- package/dist/src/store/compression.js +47 -7
- package/dist/src/store/compression.test.js +48 -0
- package/dist/src/store/sqlite.js +123 -41
- package/dist/src/store.test.js +19 -0
- package/dist/src/vectorStore.js +56 -1
- package/extensions/DASHBOARD.md +3 -3
- package/extensions/conflict-scan.ts +209 -0
- package/extensions/dashboard-server.ts +4 -4
- package/extensions/mega-compact-driver.ts +105 -0
- package/extensions/mega-compact.test.ts +65 -18
- package/extensions/mega-compact.ts +2 -0
- package/extensions/mega-config.ts +25 -0
- package/extensions/mega-conflict-cmds.ts +129 -0
- package/extensions/mega-events.ts +43 -24
- package/extensions/mega-pipeline.ts +86 -9
- package/extensions/mega-runtime.ts +14 -18
- package/package.json +6 -7
- package/src/config/dedup.ts +4 -1
- package/src/config.ts +26 -0
- package/src/dedup/raptor/index.ts +42 -7
- package/src/dedup/raptor/promote.test.ts +82 -0
- package/src/engine.ts +5 -0
- package/src/recall.test.ts +44 -0
- package/src/recall.ts +43 -4
- package/src/store/backfill.ts +10 -11
- package/src/store/compression.test.ts +58 -0
- package/src/store/compression.ts +48 -7
- package/src/store/sqlite.ts +156 -49
- package/src/store.test.ts +22 -0
- package/src/vectorStore.ts +63 -1
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* conflict-scan.ts — detect other installed pi extensions that overlap with
|
|
3
|
+
* pi-mega-compact's two owned responsibilities:
|
|
4
|
+
*
|
|
5
|
+
* 1. Conversation auto-compaction (we hook session_before_compact).
|
|
6
|
+
* 2. Durable "save to memory" (we now keep a `memories` table in our SQLite).
|
|
7
|
+
*
|
|
8
|
+
* This is a DETECT-AND-WARN scanner only. pi has no pre-load / veto hook — one
|
|
9
|
+
* extension cannot block another from loading — so we inspect the installed
|
|
10
|
+
* package set at startup and on demand, then report overlaps. No config is
|
|
11
|
+
* mutated. (See memory `pi-memory-mcp-review` for the original conflict pattern.)
|
|
12
|
+
*
|
|
13
|
+
* Pi-agnostic: reads package.json + greps source. No pi runtime types, so it is
|
|
14
|
+
* unit-testable against a fixture node_modules tree.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
18
|
+
import { join, dirname } from "node:path";
|
|
19
|
+
import { fileURLToPath } from "node:url";
|
|
20
|
+
|
|
21
|
+
export type ConflictKind = "compaction" | "memory" | "tool-output";
|
|
22
|
+
export type ConflictSeverity = "high" | "info";
|
|
23
|
+
|
|
24
|
+
export interface ConflictHit {
|
|
25
|
+
package: string;
|
|
26
|
+
severity: ConflictSeverity;
|
|
27
|
+
kind: ConflictKind;
|
|
28
|
+
evidence: string[];
|
|
29
|
+
/** One-line recommended action for the user. */
|
|
30
|
+
recommendation: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface ConflictReport {
|
|
34
|
+
scanned: string[];
|
|
35
|
+
conflicts: ConflictHit[];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Marker sets. A package is flagged when its source matches a marker in a
|
|
39
|
+
// category. File-grep (not AST) keeps this dependency-free and fast.
|
|
40
|
+
const MARKERS = {
|
|
41
|
+
// Directly competes with our conversation compaction.
|
|
42
|
+
compaction: [
|
|
43
|
+
"session_before_compact",
|
|
44
|
+
"session_compact",
|
|
45
|
+
"compactSession",
|
|
46
|
+
"autoCompact",
|
|
47
|
+
"auto_compact",
|
|
48
|
+
],
|
|
49
|
+
// Saves durable memory to its own store — the takeover target.
|
|
50
|
+
memory: [
|
|
51
|
+
"MEMORY_TOOL",
|
|
52
|
+
"learn-memory",
|
|
53
|
+
"saveMemory",
|
|
54
|
+
"memoryPolicy",
|
|
55
|
+
"wal_checkpoint",
|
|
56
|
+
"store/db.ts",
|
|
57
|
+
"memoryTool",
|
|
58
|
+
],
|
|
59
|
+
// Tool-output shaping (compact/summarize tool results) — overlap, not a rival.
|
|
60
|
+
toolOutput: [
|
|
61
|
+
"tool_result",
|
|
62
|
+
"ToolResult",
|
|
63
|
+
],
|
|
64
|
+
} as const;
|
|
65
|
+
|
|
66
|
+
/** Resolve the node_modules dir that contains this package (or env override). */
|
|
67
|
+
export function resolveExtensionRoot(selfDir: string = dirname(fileURLToPath(import.meta.url))): string | null {
|
|
68
|
+
const override = process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
69
|
+
if (override && override.trim() !== "") return override;
|
|
70
|
+
// selfDir is <root>/extensions or <root>/dist/extensions. Walk up to the
|
|
71
|
+
// node_modules that holds pi-mega-compact.
|
|
72
|
+
let dir = selfDir;
|
|
73
|
+
for (let i = 0; i < 6; i++) {
|
|
74
|
+
const candidate = join(dir, "node_modules");
|
|
75
|
+
if (existsSync(candidate) && existsSync(join(candidate, "pi-mega-compact"))) return candidate;
|
|
76
|
+
const parent = dirname(dir);
|
|
77
|
+
if (parent === dir) break;
|
|
78
|
+
dir = parent;
|
|
79
|
+
}
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Recursively collect source-ish files under a package, capped to avoid scans. */
|
|
84
|
+
function collectFiles(root: string, max = 400): string[] {
|
|
85
|
+
const out: string[] = [];
|
|
86
|
+
const walk = (dir: string): void => {
|
|
87
|
+
if (out.length >= max) return;
|
|
88
|
+
let entries: string[];
|
|
89
|
+
try {
|
|
90
|
+
entries = readdirSync(dir);
|
|
91
|
+
} catch {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
for (const e of entries) {
|
|
95
|
+
if (out.length >= max) return;
|
|
96
|
+
const full = join(dir, e);
|
|
97
|
+
let st;
|
|
98
|
+
try {
|
|
99
|
+
st = statSync(full);
|
|
100
|
+
} catch {
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (st.isDirectory()) {
|
|
104
|
+
if (e === "node_modules" || e === ".git") continue;
|
|
105
|
+
walk(full);
|
|
106
|
+
} else if (/\.(ts|js|mjs|cjs|json|md)$/.test(e)) {
|
|
107
|
+
out.push(full);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
walk(root);
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Grep a package's source for any marker in `keys`; return matched markers. */
|
|
116
|
+
function matchMarkers(pkgDir: string, keys: readonly string[]): string[] {
|
|
117
|
+
const found = new Set<string>();
|
|
118
|
+
let files: string[];
|
|
119
|
+
try {
|
|
120
|
+
files = collectFiles(pkgDir);
|
|
121
|
+
} catch {
|
|
122
|
+
return [];
|
|
123
|
+
}
|
|
124
|
+
for (const f of files) {
|
|
125
|
+
let text: string;
|
|
126
|
+
try {
|
|
127
|
+
text = readFileSync(f, "utf-8");
|
|
128
|
+
} catch {
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
for (const m of keys) {
|
|
132
|
+
if (text.includes(m)) found.add(m);
|
|
133
|
+
}
|
|
134
|
+
if (found.size === keys.length) break;
|
|
135
|
+
}
|
|
136
|
+
return [...found];
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Scan installed extensions for overlaps with pi-mega-compact.
|
|
141
|
+
* @param selfName package name to skip (defaults to this package's name).
|
|
142
|
+
*/
|
|
143
|
+
export function detectConflicts(selfName = "pi-mega-compact"): ConflictReport {
|
|
144
|
+
const root = resolveExtensionRoot();
|
|
145
|
+
const scanned: string[] = [];
|
|
146
|
+
const conflicts: ConflictHit[] = [];
|
|
147
|
+
if (!root || !existsSync(root)) return { scanned, conflicts };
|
|
148
|
+
|
|
149
|
+
let entries: string[];
|
|
150
|
+
try {
|
|
151
|
+
entries = readdirSync(root);
|
|
152
|
+
} catch {
|
|
153
|
+
return { scanned, conflicts };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
for (const name of entries) {
|
|
157
|
+
const pkgDir = join(root, name);
|
|
158
|
+
if (!statSync(pkgDir).isDirectory()) continue;
|
|
159
|
+
const pkgJson = join(pkgDir, "package.json");
|
|
160
|
+
if (!existsSync(pkgJson)) continue;
|
|
161
|
+
let pkg: { name?: string; pi?: { extensions?: string[] } };
|
|
162
|
+
try {
|
|
163
|
+
pkg = JSON.parse(readFileSync(pkgJson, "utf-8"));
|
|
164
|
+
} catch {
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
const pkgName = pkg.name ?? name;
|
|
168
|
+
if (pkgName === selfName) continue;
|
|
169
|
+
// Only consider packages that declare pi extensions.
|
|
170
|
+
if (!pkg.pi || !Array.isArray(pkg.pi.extensions) || pkg.pi.extensions.length === 0) continue;
|
|
171
|
+
scanned.push(pkgName);
|
|
172
|
+
|
|
173
|
+
const memHits = matchMarkers(pkgDir, MARKERS.memory);
|
|
174
|
+
const compHits = matchMarkers(pkgDir, MARKERS.compaction);
|
|
175
|
+
const toolHits = matchMarkers(pkgDir, MARKERS.toolOutput);
|
|
176
|
+
|
|
177
|
+
if (compHits.length > 0) {
|
|
178
|
+
conflicts.push({
|
|
179
|
+
package: pkgName,
|
|
180
|
+
severity: "high",
|
|
181
|
+
kind: "compaction",
|
|
182
|
+
evidence: compHits,
|
|
183
|
+
recommendation: "Disabling recommended — competes with pi-mega-compact's conversation compaction.",
|
|
184
|
+
});
|
|
185
|
+
continue; // compaction is the dominant conflict; don't double-flag.
|
|
186
|
+
}
|
|
187
|
+
if (memHits.length > 0) {
|
|
188
|
+
conflicts.push({
|
|
189
|
+
package: pkgName,
|
|
190
|
+
severity: "high",
|
|
191
|
+
kind: "memory",
|
|
192
|
+
evidence: memHits,
|
|
193
|
+
recommendation: "pi-mega-compact now owns save-to-memory (/mega-memory, its own SQLite). Disable this to avoid duplicate memory stores.",
|
|
194
|
+
});
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
if (toolHits.length > 0) {
|
|
198
|
+
conflicts.push({
|
|
199
|
+
package: pkgName,
|
|
200
|
+
severity: "info",
|
|
201
|
+
kind: "tool-output",
|
|
202
|
+
evidence: toolHits,
|
|
203
|
+
recommendation: "Shapes tool output (summarize/compact tool results). Generally compatible; no action needed.",
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
return { scanned, conflicts };
|
|
209
|
+
}
|
|
@@ -16,7 +16,7 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync }
|
|
|
16
16
|
import { homedir } from "node:os";
|
|
17
17
|
import { join, dirname } from "node:path";
|
|
18
18
|
import { fileURLToPath } from "node:url";
|
|
19
|
-
import
|
|
19
|
+
import { DatabaseSync } from "node:sqlite";
|
|
20
20
|
|
|
21
21
|
// --- Multi-repo index (Phase 5b) ------------------------------------------------
|
|
22
22
|
// The extension writes a machine-wide repo registry into a single SQLite DB
|
|
@@ -54,11 +54,11 @@ interface IndexRepo {
|
|
|
54
54
|
function readIndex(): { updatedAt: string; summary: unknown; repos: unknown[] } | null {
|
|
55
55
|
const indexPath = join(getIndexDir(), "index.sqlite");
|
|
56
56
|
if (!existsSync(indexPath)) return null;
|
|
57
|
-
let db:
|
|
57
|
+
let db: DatabaseSync | undefined;
|
|
58
58
|
try {
|
|
59
59
|
// Read-only + immutable WAL so a concurrent writer's WAL never blocks us.
|
|
60
|
-
db = new
|
|
61
|
-
db.
|
|
60
|
+
db = new DatabaseSync(indexPath, { readOnly: true });
|
|
61
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
62
62
|
const rows = db
|
|
63
63
|
.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC")
|
|
64
64
|
.all() as Record<string, unknown>[];
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mega-compact-driver.ts — the durable-trim driver (Fix B).
|
|
3
|
+
*
|
|
4
|
+
* The read-path token-growth bug: the old design cancelled pi's native
|
|
5
|
+
* compaction (`{ cancel: true }`) and did its own ephemeral `context`-hook
|
|
6
|
+
* drop. That drop only affected the outgoing request — the on-disk transcript
|
|
7
|
+
* was never trimmed (the session manager is read-only for extensions). So on
|
|
8
|
+
* resume pi reloaded the FULL transcript and we ADDED a recall block on top →
|
|
9
|
+
* more tokens than before compaction.
|
|
10
|
+
*
|
|
11
|
+
* The fix: on `session_before_compact` we RUN the Trident pipeline to produce a
|
|
12
|
+
* genuinely compressed summary, then RETURN it as a `CompactionResult`. pi
|
|
13
|
+
* durably writes our summary into a `compactionSummary` entry AND truncates the
|
|
14
|
+
* transcript from `firstKeptEntryId`. After that, resume reloads the already-
|
|
15
|
+
* trimmed transcript (summary baked in) — no additive re-injection, no token
|
|
16
|
+
* growth.
|
|
17
|
+
*
|
|
18
|
+
* We reuse pi's `preparation.firstKeptEntryId` (pi already computed the cut
|
|
19
|
+
* honoring the anchor-floor + tool-pair guards — PREVENT-PI-002) rather than
|
|
20
|
+
* recomputing it, so we cannot hand pi a boundary that splits a tool pair.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { SessionBeforeCompactEvent } from "@earendil-works/pi-coding-agent";
|
|
24
|
+
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
25
|
+
import { compactSession } from "../src/engine.js";
|
|
26
|
+
import { toEngineMessages } from "../src/adapt.js";
|
|
27
|
+
import { estimateBlockTokens, estimateSessionTokens } from "../src/tokens.js";
|
|
28
|
+
import type { MegaRuntime } from "./mega-runtime.js";
|
|
29
|
+
import type { MegaConfig } from "./mega-config.js";
|
|
30
|
+
import { recallRaptorRootSummary } from "../src/dedup/raptor/index.js";
|
|
31
|
+
|
|
32
|
+
export interface NativeCompactionResult {
|
|
33
|
+
/** Our trimmed summary + the pi entry to keep from (durable trim). */
|
|
34
|
+
compaction: {
|
|
35
|
+
summary: string;
|
|
36
|
+
firstKeptEntryId: string;
|
|
37
|
+
tokensBefore: number;
|
|
38
|
+
estimatedTokensAfter: number;
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Build our durable compaction result from pi's pre-computed preparation.
|
|
44
|
+
*
|
|
45
|
+
* Returns undefined when there is nothing to summarize (pi will then run its
|
|
46
|
+
* own native compaction, or skip). Never throws for "empty" — best-effort.
|
|
47
|
+
*/
|
|
48
|
+
export function driveNativeCompaction(
|
|
49
|
+
event: SessionBeforeCompactEvent,
|
|
50
|
+
runtime: MegaRuntime,
|
|
51
|
+
config: MegaConfig,
|
|
52
|
+
): NativeCompactionResult | undefined {
|
|
53
|
+
const prep = event.preparation;
|
|
54
|
+
if (!prep) return undefined;
|
|
55
|
+
|
|
56
|
+
const sid = runtime.rt.sessionId;
|
|
57
|
+
const messagesToSummarize: AgentMessage[] = prep.messagesToSummarize ?? [];
|
|
58
|
+
if (messagesToSummarize.length === 0) return undefined;
|
|
59
|
+
|
|
60
|
+
const engineView = toEngineMessages(messagesToSummarize);
|
|
61
|
+
// We don't drop anything here — pi keeps from prep.firstKeptEntryId. We only
|
|
62
|
+
// summarize the region pi is about to discard.
|
|
63
|
+
const keepFrom = engineView.length;
|
|
64
|
+
|
|
65
|
+
const result = compactSession(
|
|
66
|
+
{
|
|
67
|
+
sessionId: sid,
|
|
68
|
+
messages: engineView,
|
|
69
|
+
keepFrom,
|
|
70
|
+
timestamp: Date.now(),
|
|
71
|
+
useExtractiveSummary: true,
|
|
72
|
+
},
|
|
73
|
+
runtime.store,
|
|
74
|
+
);
|
|
75
|
+
if (result.skipped) return undefined;
|
|
76
|
+
|
|
77
|
+
// Prefer the RAPTOR root summary when the tree is built + enabled (Fix D):
|
|
78
|
+
// it is a session-level compressed summary, broader than one slice's. Fall
|
|
79
|
+
// back to the extractive topicSummary of this slice.
|
|
80
|
+
let summary = result.summary;
|
|
81
|
+
if (config.raptorEnabled) {
|
|
82
|
+
const root = recallRaptorRootSummary(sid, runtime.currentStateDir);
|
|
83
|
+
if (root) summary = root;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const tokensBefore = prep.tokensBefore ?? estimateSessionTokens(engineView);
|
|
87
|
+
const summaryTokens = estimateBlockTokens(summary);
|
|
88
|
+
// pi keeps the tail from firstKeptEntryId; our summary replaces the discarded
|
|
89
|
+
// region. Honest saved = discarded-region tokens − our summary tokens.
|
|
90
|
+
const savedTokens = Math.max(0, tokensBefore - summaryTokens);
|
|
91
|
+
|
|
92
|
+
runtime.rt.lastCompactedFrom = keepFrom;
|
|
93
|
+
runtime.rt.lastCompactedTokens = tokensBefore;
|
|
94
|
+
runtime.rt.tokensSaved += savedTokens;
|
|
95
|
+
runtime.rt.persistedThisSession = true;
|
|
96
|
+
|
|
97
|
+
return {
|
|
98
|
+
compaction: {
|
|
99
|
+
summary,
|
|
100
|
+
firstKeptEntryId: prep.firstKeptEntryId,
|
|
101
|
+
tokensBefore,
|
|
102
|
+
estimatedTokensAfter: summaryTokens,
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
|
@@ -45,6 +45,7 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
|
|
|
45
45
|
let statusKey: string | undefined;
|
|
46
46
|
let statusText: string | undefined;
|
|
47
47
|
const notifies: string[] = [];
|
|
48
|
+
const compactCalls: any[] = [];
|
|
48
49
|
|
|
49
50
|
// Minimal AgentMessage factory for the session we project into the extension.
|
|
50
51
|
function msg(role: string, text: string, toolName?: string): AgentMessage {
|
|
@@ -107,7 +108,30 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
|
|
|
107
108
|
hasPendingMessages: () => false,
|
|
108
109
|
shutdown: () => {},
|
|
109
110
|
getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }),
|
|
110
|
-
|
|
111
|
+
// Faithful mock: ctx.compact() starts pi's flow, which fires the
|
|
112
|
+
// session_before_compact handler (where WE supply the durable trim).
|
|
113
|
+
compact: (opts?: any) => {
|
|
114
|
+
compactCalls.push(opts);
|
|
115
|
+
if (handlers["session_before_compact"]) {
|
|
116
|
+
return handlers["session_before_compact"](
|
|
117
|
+
{
|
|
118
|
+
type: "session_before_compact",
|
|
119
|
+
reason: "threshold",
|
|
120
|
+
willRetry: false,
|
|
121
|
+
signal: undefined,
|
|
122
|
+
// pi computed the cut honoring anchor floor + tool-pair (PREVENT-PI-002);
|
|
123
|
+
// our handler reuses it as firstKeptEntryId.
|
|
124
|
+
preparation: {
|
|
125
|
+
firstKeptEntryId: "e2",
|
|
126
|
+
messagesToSummarize: session.slice(0, 2),
|
|
127
|
+
tokensBefore: 500,
|
|
128
|
+
},
|
|
129
|
+
} as any,
|
|
130
|
+
makeCtx(),
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
return undefined;
|
|
134
|
+
},
|
|
111
135
|
getSystemPrompt: () => "system base",
|
|
112
136
|
...over,
|
|
113
137
|
} as any;
|
|
@@ -143,14 +167,14 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
|
|
|
143
167
|
mod.default(pi);
|
|
144
168
|
|
|
145
169
|
return {
|
|
146
|
-
stateDir, handlers, commands, appended, get status() { return { statusKey, statusText }; }, notifies,
|
|
170
|
+
stateDir, handlers, commands, appended, get status() { return { statusKey, statusText }; }, notifies, compactCalls,
|
|
147
171
|
fire: (ev: string, event: any, ctx: any) => handlers[ev](event, ctx),
|
|
148
172
|
ctx: makeCtx,
|
|
149
173
|
session,
|
|
150
174
|
};
|
|
151
175
|
}
|
|
152
176
|
|
|
153
|
-
test("auto-trigger: past threshold persists a chkpt and
|
|
177
|
+
test("auto-trigger: past threshold persists a chkpt and starts a durable trim", async () => {
|
|
154
178
|
const h = harness();
|
|
155
179
|
const messages = h.session;
|
|
156
180
|
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
@@ -159,27 +183,50 @@ test("auto-trigger: past threshold persists a chkpt and drops context", async ()
|
|
|
159
183
|
const { listCheckpoints } = await import("../src/store/sqlite.js");
|
|
160
184
|
assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "checkpoint persisted to local vector db");
|
|
161
185
|
assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel appended");
|
|
162
|
-
//
|
|
163
|
-
|
|
164
|
-
|
|
186
|
+
// The context handler no longer drops messages itself (that was ephemeral —
|
|
187
|
+
// the read-path token-growth bug). It triggers pi's compaction flow, which
|
|
188
|
+
// calls our session_before_compact handler to supply the DURABLE trim.
|
|
189
|
+
assert.equal(res, undefined, "context handler returns nothing (no local drop)");
|
|
190
|
+
assert.equal(h.compactCalls.length, 1, "ctx.compact() called to start durable trim");
|
|
191
|
+
// The durable trim was supplied (summary + firstKeptEntryId from pi's prep).
|
|
192
|
+
assert.ok(h.compactCalls[0] !== undefined, "compaction flow executed");
|
|
165
193
|
});
|
|
166
194
|
|
|
167
|
-
test("session_before_compact
|
|
195
|
+
test("session_before_compact supplies our durable trim (not pi's summary)", async () => {
|
|
168
196
|
const h = harness();
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
197
|
+
// pi fires session_before_compact with its own computed preparation.
|
|
198
|
+
const res = await h.fire(
|
|
199
|
+
"session_before_compact",
|
|
200
|
+
{
|
|
201
|
+
type: "session_before_compact",
|
|
202
|
+
reason: "overflow",
|
|
203
|
+
willRetry: true,
|
|
204
|
+
preparation: { firstKeptEntryId: "e2", messagesToSummarize: h.session.slice(0, 2), tokensBefore: 500 },
|
|
205
|
+
signal: undefined,
|
|
206
|
+
} as any,
|
|
207
|
+
h.ctx(),
|
|
208
|
+
);
|
|
209
|
+
assert.ok(res && res.compaction, "returns a compaction result");
|
|
210
|
+
assert.equal(res.compaction.firstKeptEntryId, "e2", "reuses pi's cut boundary (PREVENT-PI-002 safe)");
|
|
211
|
+
assert.ok(typeof res.compaction.summary === "string" && res.compaction.summary.length > 0, "our summary supplied");
|
|
212
|
+
assert.ok(res.compaction.tokensBefore >= 0, "tokensBefore reported");
|
|
175
213
|
});
|
|
176
214
|
|
|
177
|
-
test("session_before_compact
|
|
215
|
+
test("session_before_compact falls back to pi when nothing to summarize", async () => {
|
|
178
216
|
const h = harness();
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
217
|
+
// Empty preparation → no messages to summarize → return {} so pi compacts natively.
|
|
218
|
+
const res = await h.fire(
|
|
219
|
+
"session_before_compact",
|
|
220
|
+
{
|
|
221
|
+
type: "session_before_compact",
|
|
222
|
+
reason: "threshold",
|
|
223
|
+
willRetry: false,
|
|
224
|
+
preparation: { firstKeptEntryId: "e0", messagesToSummarize: [], tokensBefore: 0 },
|
|
225
|
+
signal: undefined,
|
|
226
|
+
} as any,
|
|
227
|
+
h.ctx(),
|
|
228
|
+
);
|
|
229
|
+
assert.deepEqual(res, {}, "no compaction supplied → pi runs its own");
|
|
183
230
|
});
|
|
184
231
|
|
|
185
232
|
test("resume auto-inline stages recall into the system prompt", async () => {
|
|
@@ -31,6 +31,7 @@ import { MegaRuntime } from "./mega-runtime.js";
|
|
|
31
31
|
import { registerEventHandlers } from "./mega-events.js";
|
|
32
32
|
import { registerCommands } from "./mega-commands.js";
|
|
33
33
|
import { registerDashboardCommands } from "./mega-dashboard-cmds.js";
|
|
34
|
+
import { registerConflictCommands } from "./mega-conflict-cmds.js";
|
|
34
35
|
|
|
35
36
|
export default function (pi: ExtensionAPI) {
|
|
36
37
|
const config = loadConfig();
|
|
@@ -38,4 +39,5 @@ export default function (pi: ExtensionAPI) {
|
|
|
38
39
|
registerEventHandlers(pi, runtime, config);
|
|
39
40
|
registerCommands(pi, runtime, config);
|
|
40
41
|
registerDashboardCommands(pi, runtime);
|
|
42
|
+
registerConflictCommands(pi, runtime);
|
|
41
43
|
}
|
|
@@ -34,10 +34,24 @@ export interface MegaConfig {
|
|
|
34
34
|
fastGatePct: number;
|
|
35
35
|
anchorUserMessages: number;
|
|
36
36
|
preserveRecent: number;
|
|
37
|
+
/** High-pressure floor for preserveRecent — when context is near the limit
|
|
38
|
+
* we compact deeper, but never below this (keeps recent turns for coherence). */
|
|
39
|
+
preserveRecentMin: number;
|
|
37
40
|
auto: boolean;
|
|
38
41
|
autoInline: boolean;
|
|
39
42
|
autoInlineK: number;
|
|
40
43
|
dedupSim: number;
|
|
44
|
+
/** RAPTOR hierarchical recall enabled (Fix D). Drives both live recall and
|
|
45
|
+
* the durable-trim summary source (root summary). */
|
|
46
|
+
raptorEnabled: boolean;
|
|
47
|
+
/** Token ceiling for the re-injected recall block (Fix C). Recall stops
|
|
48
|
+
* adding checkpoints once the block would exceed this — bounds read-path
|
|
49
|
+
* token cost so it can never net-inflate the window. */
|
|
50
|
+
recallMaxTokens: number;
|
|
51
|
+
/** Inline-dedupe recalled checkpoints against the live window (Fix C): drop
|
|
52
|
+
* a hit whose summary is ≥ dedupSim similar to a live message — "dedupe on
|
|
53
|
+
* inline/read" so we never re-inject context already resident. */
|
|
54
|
+
windowDedupe: boolean;
|
|
41
55
|
debug: boolean;
|
|
42
56
|
}
|
|
43
57
|
|
|
@@ -65,6 +79,13 @@ function resolveThreshold(): { tier: CompactTier | "custom"; thresholdTokens: nu
|
|
|
65
79
|
return { tier, thresholdTokens: COMPACT_TIERS[tier] };
|
|
66
80
|
}
|
|
67
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Pressure helpers for adaptive compression (Fix E) live in src/config.ts
|
|
84
|
+
* (pi-agnostic) so unit tests can import them without the pi runtime. Re-export
|
|
85
|
+
* here so the extension has one import surface.
|
|
86
|
+
*/
|
|
87
|
+
export { pressureFromPct, preserveRecentForPressure } from "../src/config.js";
|
|
88
|
+
|
|
68
89
|
/** Build the resolved config from env + defaults. */
|
|
69
90
|
export function loadConfig(): MegaConfig {
|
|
70
91
|
const { tier, thresholdTokens } = resolveThreshold();
|
|
@@ -77,10 +98,14 @@ export function loadConfig(): MegaConfig {
|
|
|
77
98
|
thresholdTokens,
|
|
78
99
|
anchorUserMessages: envFlag("MEGACOMPACT_ANCHOR_USER_MESSAGES", 3),
|
|
79
100
|
preserveRecent: envFlag("MEGACOMPACT_PRESERVE_RECENT", 4),
|
|
101
|
+
preserveRecentMin: envFlag("MEGACOMPACT_PRESERVE_RECENT_MIN", 2),
|
|
80
102
|
auto: envBool("MEGACOMPACT_AUTO", true),
|
|
81
103
|
autoInline: envBool("MEGACOMPACT_AUTO_INLINE", true),
|
|
82
104
|
autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
|
|
83
105
|
dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
|
|
106
|
+
raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
|
|
107
|
+
recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
|
|
108
|
+
windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
|
|
84
109
|
debug: envBool("MEGACOMPACT_DEBUG", false),
|
|
85
110
|
};
|
|
86
111
|
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mega-conflict-cmds.ts — extension conflict validator + save-to-memory command.
|
|
3
|
+
*
|
|
4
|
+
* Detects other installed extensions that overlap with pi-mega-compact
|
|
5
|
+
* (conversation compaction, or save-to-memory) and WARNs — pi has no pre-load
|
|
6
|
+
* veto hook, so this is detect-and-warn only. Also registers /mega-memory, our
|
|
7
|
+
* own durable memory store in SQLite (the takeover of memory extensions).
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
11
|
+
import { detectConflicts, type ConflictReport } from "./conflict-scan.js";
|
|
12
|
+
import { addMemory, listMemories, searchMemories, recallMemory, type MemoryRecord } from "../src/store/sqlite.js";
|
|
13
|
+
import { resolveRepoRoot } from "./mega-config.js";
|
|
14
|
+
import { MegaRuntime } from "./mega-runtime.js";
|
|
15
|
+
|
|
16
|
+
/** Run the conflict scan and format a human-readable report. */
|
|
17
|
+
export function validateExtensions(): { report: ConflictReport; lines: string[] } {
|
|
18
|
+
const report = detectConflicts();
|
|
19
|
+
const lines: string[] = [];
|
|
20
|
+
if (report.conflicts.length === 0) {
|
|
21
|
+
lines.push(`[mega-compact] conflict check: ${report.scanned.length} extensions scanned, no overlaps.`);
|
|
22
|
+
return { report, lines };
|
|
23
|
+
}
|
|
24
|
+
const high = report.conflicts.filter((c) => c.severity === "high");
|
|
25
|
+
lines.push(`[mega-compact] conflict check: ${report.scanned.length} scanned, ${report.conflicts.length} overlap(s), ${high.length} high-severity.`);
|
|
26
|
+
for (const c of report.conflicts) {
|
|
27
|
+
const tag = c.severity === "high" ? "⚠ HIGH" : "ℹ info";
|
|
28
|
+
lines.push(` ${tag} ${c.package} — ${c.kind} — ${c.recommendation}`);
|
|
29
|
+
}
|
|
30
|
+
return { report, lines };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Run the scan at activation and surface a one-line warning if needed. */
|
|
34
|
+
export function runLoadTimeConflictCheck(): void {
|
|
35
|
+
try {
|
|
36
|
+
const { lines } = validateExtensions();
|
|
37
|
+
const high = lines.filter((l) => l.includes("⚠ HIGH"));
|
|
38
|
+
if (high.length > 0) {
|
|
39
|
+
// Non-fatal: just inform on stderr; the dashboard/commands carry details.
|
|
40
|
+
console.warn(lines.join("\n"));
|
|
41
|
+
}
|
|
42
|
+
} catch {
|
|
43
|
+
/* best-effort; never block session load */
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function memoryLine(m: MemoryRecord): string {
|
|
48
|
+
const tags = m.tags.length ? ` [${m.tags.join(", ")}]` : "";
|
|
49
|
+
const snap = m.content.length > 80 ? m.content.slice(0, 77) + "…" : m.content;
|
|
50
|
+
return `#${m.id} (${m.kind})${tags}: ${snap}`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Register conflict-check + memory commands. */
|
|
54
|
+
export function registerConflictCommands(pi: ExtensionAPI, runtime: MegaRuntime): void {
|
|
55
|
+
runLoadTimeConflictCheck();
|
|
56
|
+
|
|
57
|
+
pi.registerCommand("mega-compat-check", {
|
|
58
|
+
description: "Scan installed extensions for overlaps with pi-mega-compact (compaction / save-to-memory) and warn.",
|
|
59
|
+
handler: async (_args: string, ctx: ExtensionContext) => {
|
|
60
|
+
const { lines } = validateExtensions();
|
|
61
|
+
for (const l of lines) ctx.ui.notify(l);
|
|
62
|
+
if (lines.length === 1) {
|
|
63
|
+
ctx.ui.notify("[mega-compact] You own compaction + memory; no conflicting extensions detected.");
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
pi.registerCommand("mega-memory", {
|
|
69
|
+
description: "Save and recall durable memory in pi-mega-compact's SQLite store. Usage: /mega-memory save <text> | list | search <q> | recall <id>",
|
|
70
|
+
handler: async (args: string, ctx: ExtensionContext) => {
|
|
71
|
+
const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
|
|
72
|
+
const parts = args.trim().split(/\s+/);
|
|
73
|
+
const sub = parts[0]?.toLowerCase() ?? "list";
|
|
74
|
+
|
|
75
|
+
if (sub === "save") {
|
|
76
|
+
const text = args.trim().slice(4).trim();
|
|
77
|
+
if (!text) {
|
|
78
|
+
ctx.ui.notify("[mega-memory] usage: /mega-memory save <text>");
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
// Optional "#tag #tag" parsing from the tail.
|
|
82
|
+
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
83
|
+
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
84
|
+
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
85
|
+
ctx.ui.notify(`[mega-memory] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (sub === "search") {
|
|
90
|
+
const q = parts.slice(1).join(" ").trim();
|
|
91
|
+
if (!q) {
|
|
92
|
+
ctx.ui.notify("[mega-memory] usage: /mega-memory search <query>");
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
const hits = searchMemories(q, repo, 50, runtime.currentStateDir);
|
|
96
|
+
if (!hits.length) {
|
|
97
|
+
ctx.ui.notify("[mega-memory] no memories match.");
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
for (const m of hits) ctx.ui.notify(memoryLine(m));
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (sub === "recall") {
|
|
105
|
+
const id = Number(parts[1]);
|
|
106
|
+
if (!Number.isFinite(id) || parts[1] === undefined) {
|
|
107
|
+
ctx.ui.notify("[mega-memory] usage: /mega-memory recall <id>");
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
if (recallMemory(id, runtime.currentStateDir)) {
|
|
111
|
+
const found = listMemories(repo, 1000, runtime.currentStateDir).find((m) => m.id === id);
|
|
112
|
+
ctx.ui.notify(found ? `[mega-memory] ${memoryLine(found)}` : `[mega-memory] recalled #${id}`);
|
|
113
|
+
} else {
|
|
114
|
+
ctx.ui.notify(`[mega-memory] #${id} not found.`);
|
|
115
|
+
}
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// default: list
|
|
120
|
+
const all = listMemories(repo, 50, runtime.currentStateDir);
|
|
121
|
+
if (!all.length) {
|
|
122
|
+
ctx.ui.notify("[mega-memory] no saved memories yet. Use /mega-memory save <text>.");
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
ctx.ui.notify(`[mega-memory] ${all.length} saved to ${repo.split(/[\\/]/).pop()}:`);
|
|
126
|
+
for (const m of all) ctx.ui.notify(memoryLine(m));
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
}
|