pi-mega-compact 0.6.7 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -8
- package/dist/extensions/dashboard-server.js +112 -14
- package/dist/extensions/mega-commands.js +12 -1
- package/dist/extensions/mega-compact.test.js +286 -51
- package/dist/extensions/mega-config.js +67 -5
- package/dist/extensions/mega-events.js +151 -27
- package/dist/extensions/mega-runtime.js +163 -32
- package/dist/extensions/openclaw-mega-compact.js +291 -0
- package/dist/src/compact.js +5 -1
- package/dist/src/dedup-engine.test.js +63 -38
- package/dist/src/minilm.js +92 -0
- package/dist/src/supersede.js +8 -5
- package/dist/src/wordpiece.js +129 -0
- package/extensions/dashboard-server.ts +125 -14
- package/extensions/mega-commands.ts +12 -1
- package/extensions/mega-compact.test.ts +947 -516
- package/extensions/mega-config.ts +84 -6
- package/extensions/mega-dashboard.ts +11 -0
- package/extensions/mega-events.ts +558 -360
- package/extensions/mega-runtime.ts +168 -32
- package/package.json +1 -1
- package/src/compact.ts +5 -1
- package/src/dedup-engine.test.ts +103 -42
- package/src/supersede.ts +8 -5
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* openclaw-mega-compact — OpenClaw plugin adapter for the pi-mega-compact engine.
|
|
3
|
+
*
|
|
4
|
+
* Wires the pi-agnostic Trident engine (src/) into OpenClaw's plugin lifecycle:
|
|
5
|
+
* - Registers a CompactionProvider that replaces the built-in summarizeInStages.
|
|
6
|
+
* - Exposes `mega_status` and `mega_recall` tools for on-demand inspection.
|
|
7
|
+
* - Hooks into `before_compaction` / `after_compaction` for diagnostics.
|
|
8
|
+
*
|
|
9
|
+
* Design constraints:
|
|
10
|
+
* - NO imports from `@earendil-works/pi-coding-agent` or pi-agent-core.
|
|
11
|
+
* - The engine core (src/) is pi-agnostic; this file is the sole OpenClaw boundary.
|
|
12
|
+
* - No network at runtime — everything is local (stores + extractive summarizer).
|
|
13
|
+
*/
|
|
14
|
+
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
15
|
+
import { compactSession, setDefaultStore, } from "../src/engine.js";
|
|
16
|
+
import { recallAndInline } from "../src/recall.js";
|
|
17
|
+
import { VectorStore } from "../src/vectorStore.js";
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
// Constants
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
const PLUGIN_ID = "mega-compact";
|
|
22
|
+
const PLUGIN_LABEL = "Mega Compact (Trident)";
|
|
23
|
+
/** Default state directory for vector store persistence. */
|
|
24
|
+
const STATE_DIR = process.env.MEGA_COMPACT_STATE_DIR ?? undefined;
|
|
25
|
+
/** Minimum messages before we bother compacting. */
|
|
26
|
+
const MIN_MESSAGES_FOR_COMPACT = 6;
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
// Message conversion — OpenClaw unknown[] → EngineMessage[]
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
/**
|
|
31
|
+
* Best-effort conversion from OpenClaw's opaque message array to our
|
|
32
|
+
* EngineMessage shape. OpenClaw messages are typed as `unknown[]` so we
|
|
33
|
+
* handle whatever shape comes through gracefully.
|
|
34
|
+
*/
|
|
35
|
+
function toEngineMessages(messages) {
|
|
36
|
+
return messages.map((msg) => {
|
|
37
|
+
if (!msg || typeof msg !== "object") {
|
|
38
|
+
// Primitive fallback — treat as custom text.
|
|
39
|
+
return {
|
|
40
|
+
role: "custom",
|
|
41
|
+
text: String(msg ?? ""),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
const m = msg;
|
|
45
|
+
const role = typeof m.role === "string" ? m.role : "custom";
|
|
46
|
+
// Normalize role to one of our four engine roles.
|
|
47
|
+
let engineRole;
|
|
48
|
+
switch (role) {
|
|
49
|
+
case "user":
|
|
50
|
+
engineRole = "user";
|
|
51
|
+
break;
|
|
52
|
+
case "assistant":
|
|
53
|
+
engineRole = "assistant";
|
|
54
|
+
break;
|
|
55
|
+
case "tool":
|
|
56
|
+
case "function":
|
|
57
|
+
engineRole = "tool";
|
|
58
|
+
break;
|
|
59
|
+
default:
|
|
60
|
+
engineRole = "custom";
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
// Extract text content from common message shapes.
|
|
64
|
+
const text = typeof m.content === "string"
|
|
65
|
+
? m.content
|
|
66
|
+
: typeof m.text === "string"
|
|
67
|
+
? m.text
|
|
68
|
+
: Array.isArray(m.content)
|
|
69
|
+
? m.content
|
|
70
|
+
.filter((part) => part.type === "text" && typeof part.text === "string")
|
|
71
|
+
.map((part) => part.text)
|
|
72
|
+
.join("\n")
|
|
73
|
+
: "";
|
|
74
|
+
// Preserve tool metadata when present.
|
|
75
|
+
const toolName = typeof m.name === "string"
|
|
76
|
+
? m.name
|
|
77
|
+
: typeof m.toolName === "string"
|
|
78
|
+
? m.toolName
|
|
79
|
+
: undefined;
|
|
80
|
+
const input = typeof m.input === "string"
|
|
81
|
+
? m.input
|
|
82
|
+
: typeof m.arguments === "string"
|
|
83
|
+
? m.arguments
|
|
84
|
+
: m.arguments !== undefined
|
|
85
|
+
? JSON.stringify(m.arguments)
|
|
86
|
+
: undefined;
|
|
87
|
+
const output = typeof m.output === "string"
|
|
88
|
+
? m.output
|
|
89
|
+
: engineRole === "tool" && typeof m.content === "string"
|
|
90
|
+
? m.content
|
|
91
|
+
: undefined;
|
|
92
|
+
return { role: engineRole, text, toolName, input, output };
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
// Compaction provider
|
|
97
|
+
// ---------------------------------------------------------------------------
|
|
98
|
+
function createCompactionProvider(store) {
|
|
99
|
+
return {
|
|
100
|
+
id: PLUGIN_ID,
|
|
101
|
+
label: PLUGIN_LABEL,
|
|
102
|
+
async summarize({ messages, signal, compressionRatio, }) {
|
|
103
|
+
// Abort check — bail early if the caller cancelled.
|
|
104
|
+
if (signal?.aborted) {
|
|
105
|
+
throw new DOMException("Aborted", "AbortError");
|
|
106
|
+
}
|
|
107
|
+
const engineMessages = toEngineMessages(messages);
|
|
108
|
+
// Nothing meaningful to compact.
|
|
109
|
+
if (engineMessages.length < MIN_MESSAGES_FOR_COMPACT) {
|
|
110
|
+
return "";
|
|
111
|
+
}
|
|
112
|
+
// Map compression ratio → keepFrom boundary.
|
|
113
|
+
// compressionRatio=0.5 means "compact the oldest 50%".
|
|
114
|
+
// Default to compacting the oldest half if not specified.
|
|
115
|
+
const ratio = compressionRatio ?? 0.5;
|
|
116
|
+
const keepFrom = Math.max(MIN_MESSAGES_FOR_COMPACT, Math.floor(engineMessages.length * (1 - ratio)));
|
|
117
|
+
// Abort check after conversion (conversion is cheap but check anyway).
|
|
118
|
+
if (signal?.aborted) {
|
|
119
|
+
throw new DOMException("Aborted", "AbortError");
|
|
120
|
+
}
|
|
121
|
+
const sessionId = `openclaw-${Date.now()}`;
|
|
122
|
+
const input = {
|
|
123
|
+
sessionId,
|
|
124
|
+
messages: engineMessages,
|
|
125
|
+
keepFrom,
|
|
126
|
+
};
|
|
127
|
+
const result = compactSession(input, store);
|
|
128
|
+
if (result.skipped) {
|
|
129
|
+
return "";
|
|
130
|
+
}
|
|
131
|
+
return result.summary;
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
// ---------------------------------------------------------------------------
|
|
136
|
+
// Plugin entry
|
|
137
|
+
// ---------------------------------------------------------------------------
|
|
138
|
+
export default definePluginEntry({
|
|
139
|
+
id: PLUGIN_ID,
|
|
140
|
+
name: "Mega Compact",
|
|
141
|
+
description: "Layered, local, vector-backed context compressor (Trident engine) for OpenClaw compaction.",
|
|
142
|
+
register(api) {
|
|
143
|
+
const logger = api.logger;
|
|
144
|
+
// Resolve state directory — prefer plugin config override.
|
|
145
|
+
const pluginCfg = (api.pluginConfig ?? {});
|
|
146
|
+
const stateDir = typeof pluginCfg.stateDir === "string" && pluginCfg.stateDir.length > 0
|
|
147
|
+
? pluginCfg.stateDir
|
|
148
|
+
: STATE_DIR;
|
|
149
|
+
// Initialize vector store.
|
|
150
|
+
let store;
|
|
151
|
+
try {
|
|
152
|
+
store = new VectorStore({ stateDir });
|
|
153
|
+
setDefaultStore(store);
|
|
154
|
+
logger.info?.(`${PLUGIN_ID}: vector store initialized (stateDir=${stateDir ?? "default"})`);
|
|
155
|
+
}
|
|
156
|
+
catch (err) {
|
|
157
|
+
logger.error?.(`${PLUGIN_ID}: failed to init vector store:`, err);
|
|
158
|
+
return; // Hard bail — no point registering if store is broken.
|
|
159
|
+
}
|
|
160
|
+
// -----------------------------------------------------------------------
|
|
161
|
+
// Register compaction provider
|
|
162
|
+
// -----------------------------------------------------------------------
|
|
163
|
+
const provider = createCompactionProvider(store);
|
|
164
|
+
api.registerCompactionProvider(provider);
|
|
165
|
+
logger.info?.(`${PLUGIN_ID}: registered compaction provider "${provider.id}"`);
|
|
166
|
+
// -----------------------------------------------------------------------
|
|
167
|
+
// Hooks — before / after compaction diagnostics
|
|
168
|
+
// -----------------------------------------------------------------------
|
|
169
|
+
api.registerHook({
|
|
170
|
+
event: "before_compaction",
|
|
171
|
+
handler: async (ctx) => {
|
|
172
|
+
const msgCount = Array.isArray(ctx?.messages) ? ctx.messages.length : 0;
|
|
173
|
+
logger.info?.(`${PLUGIN_ID}: before_compaction — ${msgCount} messages in scope`);
|
|
174
|
+
},
|
|
175
|
+
});
|
|
176
|
+
api.registerHook({
|
|
177
|
+
event: "after_compaction",
|
|
178
|
+
handler: async (ctx) => {
|
|
179
|
+
const summaryLen = typeof ctx?.summary === "string" ? ctx.summary.length : 0;
|
|
180
|
+
logger.info?.(`${PLUGIN_ID}: after_compaction — summary ${summaryLen} chars`);
|
|
181
|
+
},
|
|
182
|
+
});
|
|
183
|
+
// -----------------------------------------------------------------------
|
|
184
|
+
// Tool: mega_status
|
|
185
|
+
// -----------------------------------------------------------------------
|
|
186
|
+
api.registerTool({
|
|
187
|
+
name: "mega_status",
|
|
188
|
+
description: "Show the current status of the mega-compact engine: vector store stats, checkpoint count, and recent compaction activity.",
|
|
189
|
+
parameters: {
|
|
190
|
+
type: "object",
|
|
191
|
+
properties: {
|
|
192
|
+
sessionId: {
|
|
193
|
+
type: "string",
|
|
194
|
+
description: "Optional session ID to scope stats to.",
|
|
195
|
+
},
|
|
196
|
+
},
|
|
197
|
+
additionalProperties: false,
|
|
198
|
+
},
|
|
199
|
+
handler: async (args) => {
|
|
200
|
+
const sessionId = args?.sessionId ?? "global";
|
|
201
|
+
try {
|
|
202
|
+
const stats = store.stats(sessionId);
|
|
203
|
+
const parts = [
|
|
204
|
+
`**Mega Compact Status**`,
|
|
205
|
+
`Session: ${sessionId}`,
|
|
206
|
+
`Checkpoints: ${stats.checkpointCount}`,
|
|
207
|
+
`Total tokens saved: ${stats.totalTokenEstimate}`,
|
|
208
|
+
`Last checkpoint: ${stats.lastCheckpointId ?? "—"}`,
|
|
209
|
+
`Injected count: ${stats.injectedCount}`,
|
|
210
|
+
`Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
|
|
211
|
+
];
|
|
212
|
+
if (stats.lastSummary) {
|
|
213
|
+
parts.push(`\nLast summary (truncated):\n ${stats.lastSummary.slice(0, 120).replace(/\n/g, " ")}…`);
|
|
214
|
+
}
|
|
215
|
+
return { content: [{ type: "text", text: parts.join("\n") }] };
|
|
216
|
+
}
|
|
217
|
+
catch (err) {
|
|
218
|
+
return {
|
|
219
|
+
content: [{ type: "text", text: `Error reading mega-compact status: ${err}` }],
|
|
220
|
+
isError: true,
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
},
|
|
224
|
+
});
|
|
225
|
+
// -----------------------------------------------------------------------
|
|
226
|
+
// Tool: mega_recall
|
|
227
|
+
// -----------------------------------------------------------------------
|
|
228
|
+
api.registerTool({
|
|
229
|
+
name: "mega_recall",
|
|
230
|
+
description: "Recall and inline relevant context from the mega-compact vector store for the current session.",
|
|
231
|
+
parameters: {
|
|
232
|
+
type: "object",
|
|
233
|
+
properties: {
|
|
234
|
+
sessionId: {
|
|
235
|
+
type: "string",
|
|
236
|
+
description: "Session ID to recall context for.",
|
|
237
|
+
},
|
|
238
|
+
query: {
|
|
239
|
+
type: "string",
|
|
240
|
+
description: "Natural language query for relevant context.",
|
|
241
|
+
},
|
|
242
|
+
limit: {
|
|
243
|
+
type: "number",
|
|
244
|
+
description: "Max checkpoints to recall (default 3).",
|
|
245
|
+
},
|
|
246
|
+
},
|
|
247
|
+
required: ["sessionId", "query"],
|
|
248
|
+
additionalProperties: false,
|
|
249
|
+
},
|
|
250
|
+
handler: async (args) => {
|
|
251
|
+
const { sessionId, query, limit } = args;
|
|
252
|
+
if (!sessionId || !query) {
|
|
253
|
+
return {
|
|
254
|
+
content: [{ type: "text", text: "Both `sessionId` and `query` are required." }],
|
|
255
|
+
isError: true,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
try {
|
|
259
|
+
const result = recallAndInline({ sessionId, query, limit: limit ?? 3, source: "command", skipInjected: false }, store);
|
|
260
|
+
if (result.toInject.length === 0) {
|
|
261
|
+
return {
|
|
262
|
+
content: [{ type: "text", text: "No relevant context found in the mega-compact store." }],
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
const parts = [
|
|
266
|
+
`**Recalled ${result.toInject.length} checkpoint(s):**`,
|
|
267
|
+
...result.report,
|
|
268
|
+
"",
|
|
269
|
+
"---",
|
|
270
|
+
result.block,
|
|
271
|
+
];
|
|
272
|
+
return { content: [{ type: "text", text: parts.join("\n") }] };
|
|
273
|
+
}
|
|
274
|
+
catch (err) {
|
|
275
|
+
return {
|
|
276
|
+
content: [{ type: "text", text: `Error during mega-recall: ${err}` }],
|
|
277
|
+
isError: true,
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
},
|
|
281
|
+
});
|
|
282
|
+
// -----------------------------------------------------------------------
|
|
283
|
+
// Cleanup on shutdown
|
|
284
|
+
// -----------------------------------------------------------------------
|
|
285
|
+
api.on("shutdown", () => {
|
|
286
|
+
logger.info?.(`${PLUGIN_ID}: shutting down — clearing default store`);
|
|
287
|
+
setDefaultStore(undefined);
|
|
288
|
+
});
|
|
289
|
+
logger.info?.(`${PLUGIN_ID}: plugin registered (tools: mega_status, mega_recall)`);
|
|
290
|
+
},
|
|
291
|
+
});
|
package/dist/src/compact.js
CHANGED
|
@@ -16,7 +16,11 @@ function truncate(s, max) {
|
|
|
16
16
|
return s.length <= max ? s : `${s.slice(0, max)}…`;
|
|
17
17
|
}
|
|
18
18
|
function firstText(m) {
|
|
19
|
-
|
|
19
|
+
// PREVENT crash: pi can hand us a message with text: undefined (pure
|
|
20
|
+
// tool-call/tool-result). Guard the trim so the legacy summarizeMessages
|
|
21
|
+
// path can't throw the same undefined-text crash the extractive path did.
|
|
22
|
+
const raw = m.text ?? "";
|
|
23
|
+
const t = raw.trim();
|
|
20
24
|
return t.length > 0 ? t : undefined;
|
|
21
25
|
}
|
|
22
26
|
/** Heuristic: does this text look like chatty filler we can collapse? */
|
|
@@ -21,6 +21,11 @@ import { extractiveSummarize } from "./extractive.js";
|
|
|
21
21
|
import { estimateSessionTokens, estimateMessageTokens } from "./tokens.js";
|
|
22
22
|
import { autoCompactCheck } from "./compact.js";
|
|
23
23
|
import { loadDedupConfig } from "./config/dedup.js";
|
|
24
|
+
// Real percentage-based threshold config. Replaces the previous LOCAL replica of
|
|
25
|
+
// COMPACT_TIERS + resolveThresholdFromEnv that asserted the OLD static token
|
|
26
|
+
// amounts — importing the live source of truth keeps tests in sync with the
|
|
27
|
+
// source (thresholds are tierPct × the model's context window, not fixed tokens).
|
|
28
|
+
import { TIER_PCT, effectiveThresholdTokens, loadConfig } from "../extensions/mega-config.js";
|
|
24
29
|
// recallAndInline may or may not be exported; import safely.
|
|
25
30
|
import * as recallMod from "./recall.js";
|
|
26
31
|
const recallAndInline = recallMod.recallAndInline;
|
|
@@ -378,43 +383,58 @@ describe("Edge Cases", () => {
|
|
|
378
383
|
assert.equal(s.list(SESS).length, 1);
|
|
379
384
|
});
|
|
380
385
|
});
|
|
381
|
-
// -------------------- 7. Tier Switching --------------------
|
|
382
|
-
|
|
383
|
-
|
|
386
|
+
// -------------------- 7. Tier Switching (percentage-based) --------------------
|
|
387
|
+
// Replaces the previous LOCAL replica of COMPACT_TIERS + resolveThresholdFromEnv
|
|
388
|
+
// that asserted the OLD static token amounts. We now import the REAL config
|
|
389
|
+
// helpers from extensions/mega-config.js so the tests track the live source of
|
|
390
|
+
// truth: thresholds are tierPct × the model's context window (not fixed tokens).
|
|
391
|
+
describe("Tier Switching — percentage-based thresholds", () => {
|
|
392
|
+
// Documented tierPct fractions (single source of truth in mega-config.ts).
|
|
393
|
+
it("each named tier carries the documented tierPct fraction", () => {
|
|
394
|
+
assert.equal(TIER_PCT.low, 0.5);
|
|
395
|
+
assert.equal(TIER_PCT.medium, 0.6);
|
|
396
|
+
assert.equal(TIER_PCT.high, 0.7);
|
|
397
|
+
assert.equal(TIER_PCT.ultra, 0.7);
|
|
398
|
+
assert.equal(TIER_PCT.mega, 0.75);
|
|
399
|
+
});
|
|
400
|
+
// Boot fallback threshold (sane gate before the first context event supplies a
|
|
401
|
+
// window): round(tierPct × 200_000). Resolved through the REAL loadConfig().
|
|
402
|
+
it("MEGACOMPACT_TIER env resolves to the boot fallback threshold via real config", () => {
|
|
384
403
|
const tiers = [
|
|
385
|
-
["low",
|
|
386
|
-
["medium",
|
|
387
|
-
["high",
|
|
388
|
-
["ultra",
|
|
389
|
-
["mega",
|
|
404
|
+
["low", 100_000], // 0.50 × 200_000
|
|
405
|
+
["medium", 120_000], // 0.60 × 200_000
|
|
406
|
+
["high", 140_000], // 0.70 × 200_000
|
|
407
|
+
["ultra", 140_000], // 0.70 × 200_000
|
|
408
|
+
["mega", 150_000], // 0.75 × 200_000
|
|
390
409
|
];
|
|
391
|
-
for (const [tier,
|
|
410
|
+
for (const [tier, expectedBoot] of tiers) {
|
|
392
411
|
const original = process.env.MEGACOMPACT_TIER;
|
|
412
|
+
delete process.env.MEGACOMPACT_THRESHOLD_TOKENS;
|
|
393
413
|
process.env.MEGACOMPACT_TIER = tier;
|
|
394
414
|
try {
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
assert.equal(
|
|
415
|
+
const cfg = loadConfig();
|
|
416
|
+
assert.equal(cfg.tier, tier, `tier ${tier} should resolve`);
|
|
417
|
+
assert.equal(cfg.tierPct, TIER_PCT[tier], `tier ${tier} tierPct`);
|
|
418
|
+
assert.equal(cfg.thresholdTokens, expectedBoot, `tier ${tier} boot fallback threshold should be ${expectedBoot}`);
|
|
399
419
|
}
|
|
400
420
|
finally {
|
|
401
|
-
if (original === undefined)
|
|
421
|
+
if (original === undefined)
|
|
402
422
|
delete process.env.MEGACOMPACT_TIER;
|
|
403
|
-
|
|
404
|
-
else {
|
|
423
|
+
else
|
|
405
424
|
process.env.MEGACOMPACT_TIER = original;
|
|
406
|
-
}
|
|
407
425
|
}
|
|
408
426
|
}
|
|
409
427
|
});
|
|
410
|
-
it("explicit MEGACOMPACT_THRESHOLD_TOKENS overrides tier", () => {
|
|
428
|
+
it("explicit MEGACOMPACT_THRESHOLD_TOKENS overrides tier (custom stays absolute)", () => {
|
|
411
429
|
const originalTier = process.env.MEGACOMPACT_TIER;
|
|
412
430
|
const originalThreshold = process.env.MEGACOMPACT_THRESHOLD_TOKENS;
|
|
413
|
-
process.env.MEGACOMPACT_TIER
|
|
431
|
+
delete process.env.MEGACOMPACT_TIER;
|
|
414
432
|
process.env.MEGACOMPACT_THRESHOLD_TOKENS = "123456";
|
|
415
433
|
try {
|
|
416
|
-
const
|
|
417
|
-
assert.equal(
|
|
434
|
+
const cfg = loadConfig();
|
|
435
|
+
assert.equal(cfg.tier, "custom", "explicit token threshold → custom tier");
|
|
436
|
+
assert.equal(cfg.tierPct, null, "custom tier has no tierPct (stays absolute)");
|
|
437
|
+
assert.equal(cfg.thresholdTokens, 123_456, "explicit token threshold should win");
|
|
418
438
|
}
|
|
419
439
|
finally {
|
|
420
440
|
if (originalTier === undefined)
|
|
@@ -428,20 +448,25 @@ describe("Tier Switching", () => {
|
|
|
428
448
|
}
|
|
429
449
|
});
|
|
430
450
|
});
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
451
|
+
describe("effectiveThresholdTokens — tierPct × model window", () => {
|
|
452
|
+
// The real compaction fire point. Tiered → scales with the window so it always
|
|
453
|
+
// fires BELOW pi's native ~80% auto-compact for any model size. Custom (null
|
|
454
|
+
// tierPct) → absolute explicitThreshold, never percent-scaled.
|
|
455
|
+
it("scales tierPct × window for a 200k model", () => {
|
|
456
|
+
assert.equal(effectiveThresholdTokens({ tierPct: TIER_PCT.low, fallbackThreshold: 100_000, window: 200_000 }), 100_000);
|
|
457
|
+
assert.equal(effectiveThresholdTokens({ tierPct: TIER_PCT.mega, fallbackThreshold: 150_000, window: 200_000 }), 150_000);
|
|
458
|
+
});
|
|
459
|
+
it("scales tierPct × window for a 1M model", () => {
|
|
460
|
+
assert.equal(effectiveThresholdTokens({ tierPct: TIER_PCT.low, fallbackThreshold: 500_000, window: 1_000_000 }), 500_000);
|
|
461
|
+
assert.equal(effectiveThresholdTokens({ tierPct: TIER_PCT.mega, fallbackThreshold: 750_000, window: 1_000_000 }), 750_000);
|
|
462
|
+
});
|
|
463
|
+
it("falls back to the boot threshold when the window is 0/unknown", () => {
|
|
464
|
+
assert.equal(effectiveThresholdTokens({ tierPct: TIER_PCT.mega, fallbackThreshold: 150_000, window: 0 }), 150_000);
|
|
465
|
+
assert.equal(effectiveThresholdTokens({ tierPct: TIER_PCT.low, fallbackThreshold: 100_000, window: -5 }), 100_000);
|
|
466
|
+
});
|
|
467
|
+
it("custom (tierPct null) stays an absolute threshold regardless of window", () => {
|
|
468
|
+
assert.equal(effectiveThresholdTokens({ tierPct: null, fallbackThreshold: 100_000, window: 200_000, explicitThreshold: 123456 }), 123456, "explicit absolute wins (200k window)");
|
|
469
|
+
assert.equal(effectiveThresholdTokens({ tierPct: null, fallbackThreshold: 100_000, window: 1_000_000, explicitThreshold: 123456 }), 123456, "explicit absolute wins (1M window)");
|
|
470
|
+
assert.equal(effectiveThresholdTokens({ tierPct: null, fallbackThreshold: 100_000, window: 200_000 }), 100_000, "no explicit → boot fallback");
|
|
471
|
+
});
|
|
472
|
+
});
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* minilm.ts — local MiniLM (all-MiniLM-L6-v2) sentence embedder (Sprint 12).
|
|
3
|
+
*
|
|
4
|
+
* Implements the `Embedder` interface so it drops into the existing VectorStore
|
|
5
|
+
* dedup cascade and search with no call-site changes. Inference is 100% local:
|
|
6
|
+
* the ONNX model + WordPiece vocab are on-disk artifacts fetched once by
|
|
7
|
+
* scripts/setup-minilm.mjs. There is NO network call at runtime (PREVENT-PI-004).
|
|
8
|
+
*
|
|
9
|
+
* Inputs (dynamic): input_ids, attention_mask, token_type_ids (int64).
|
|
10
|
+
* Output: last_hidden_state (batch, seq, 384). We mean-pool over non-padded
|
|
11
|
+
* tokens (attention_mask == 1) and L2-normalize → 384-dim unit vector.
|
|
12
|
+
*
|
|
13
|
+
* The ONNX session + tokenizer are loaded LAZILY on first embed() so the default
|
|
14
|
+
* TrigramEmbedder path (and its zero native-init cost) is untouched unless
|
|
15
|
+
* MEGACOMPACT_EMBEDDER=minilm is selected.
|
|
16
|
+
*/
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
import { homedir } from "node:os";
|
|
19
|
+
import { existsSync } from "node:fs";
|
|
20
|
+
import { l2Normalize, awaitSync } from "./embedder.js";
|
|
21
|
+
import { WordPieceTokenizer } from "./wordpiece.js";
|
|
22
|
+
export const MINILM_DIM = 384;
|
|
23
|
+
export const MINILM_MAX_LEN = 256;
|
|
24
|
+
/** Resolve the model directory: MEGACOMPACT_MINILM_DIR > ./models/minilm > ~/.pi … */
|
|
25
|
+
function resolveModelDir() {
|
|
26
|
+
if (process.env.MEGACOMPACT_MINILM_DIR)
|
|
27
|
+
return process.env.MEGACOMPACT_MINILM_DIR;
|
|
28
|
+
// Repo-local vendored path (gitignored).
|
|
29
|
+
const local = join(process.cwd(), "models", "minilm");
|
|
30
|
+
if (existsSync(local))
|
|
31
|
+
return local;
|
|
32
|
+
return join(homedir(), ".pi", "agent", "extensions", "mega-compact", "models", "minilm");
|
|
33
|
+
}
|
|
34
|
+
export class MiniLMEmbedder {
|
|
35
|
+
dim = MINILM_DIM;
|
|
36
|
+
session = null;
|
|
37
|
+
tokenizer = null;
|
|
38
|
+
modelDir;
|
|
39
|
+
loadPromise = null;
|
|
40
|
+
constructor(modelDir = resolveModelDir()) {
|
|
41
|
+
this.modelDir = modelDir;
|
|
42
|
+
}
|
|
43
|
+
async ensureLoaded() {
|
|
44
|
+
if (this.session && this.tokenizer)
|
|
45
|
+
return;
|
|
46
|
+
if (this.loadPromise)
|
|
47
|
+
return this.loadPromise;
|
|
48
|
+
this.loadPromise = (async () => {
|
|
49
|
+
const ort = await import("onnxruntime-node");
|
|
50
|
+
const modelPath = join(this.modelDir, "model_quantized.onnx");
|
|
51
|
+
const vocabPath = join(this.modelDir, "vocab.txt");
|
|
52
|
+
if (!existsSync(modelPath) || !existsSync(vocabPath)) {
|
|
53
|
+
throw new Error(`MiniLM artifacts missing in ${this.modelDir}. Run: node scripts/setup-minilm.mjs`);
|
|
54
|
+
}
|
|
55
|
+
// 1 thread is plenty for a single short-region embed and bounds CPU.
|
|
56
|
+
this.session = await ort.InferenceSession.create(modelPath, {
|
|
57
|
+
executionProviders: ["cpu"],
|
|
58
|
+
graphOptimizationLevel: "all",
|
|
59
|
+
});
|
|
60
|
+
this.tokenizer = WordPieceTokenizer.fromVocabFile(vocabPath);
|
|
61
|
+
})();
|
|
62
|
+
return this.loadPromise;
|
|
63
|
+
}
|
|
64
|
+
embed(text) {
|
|
65
|
+
awaitSync(this.ensureLoaded());
|
|
66
|
+
const enc = this.tokenizer.encode(text, MINILM_MAX_LEN);
|
|
67
|
+
const n = enc.inputIds.length;
|
|
68
|
+
const BigInt64 = (arr) => arr.map((x) => BigInt(x));
|
|
69
|
+
const ort = awaitSync(import("onnxruntime-node"));
|
|
70
|
+
const tensors = {
|
|
71
|
+
input_ids: new ort.Tensor("int64", BigInt64(enc.inputIds), [1, n]),
|
|
72
|
+
attention_mask: new ort.Tensor("int64", BigInt64(enc.attentionMask), [1, n]),
|
|
73
|
+
token_type_ids: new ort.Tensor("int64", BigInt64(enc.tokenTypeIds), [1, n]),
|
|
74
|
+
};
|
|
75
|
+
const out = awaitSync(this.session.run(tensors));
|
|
76
|
+
const hidden = out.last_hidden_state.data;
|
|
77
|
+
// hidden shape: [1, n, 384]. Mean-pool over non-padded positions.
|
|
78
|
+
const pooled = new Array(MINILM_DIM).fill(0);
|
|
79
|
+
let count = 0;
|
|
80
|
+
for (let i = 0; i < n; i++) {
|
|
81
|
+
if (enc.attentionMask[i] === 0)
|
|
82
|
+
continue;
|
|
83
|
+
const base = i * MINILM_DIM;
|
|
84
|
+
for (let d = 0; d < MINILM_DIM; d++)
|
|
85
|
+
pooled[d] += hidden[base + d];
|
|
86
|
+
count++;
|
|
87
|
+
}
|
|
88
|
+
if (count === 0)
|
|
89
|
+
return l2Normalize(new Array(MINILM_DIM).fill(0));
|
|
90
|
+
return l2Normalize(pooled.map((x) => x / count));
|
|
91
|
+
}
|
|
92
|
+
}
|
package/dist/src/supersede.js
CHANGED
|
@@ -9,13 +9,16 @@
|
|
|
9
9
|
import { extractFileCandidates } from "./compact.js";
|
|
10
10
|
/** Classify a message's relationship to a file path. */
|
|
11
11
|
function fileOps(msg) {
|
|
12
|
-
// msg.text may be undefined for pure tool-call/result
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
|
|
12
|
+
// PREVENT crash: msg.text may be undefined for pure tool-call/result
|
|
13
|
+
// messages. extractFileCandidates guards the split, but if it returned a
|
|
14
|
+
// hit we'd still call .toLowerCase() on the raw (possibly-undefined) text.
|
|
15
|
+
// Coerce once so both extractFileCandidates and the write-detection regex
|
|
16
|
+
// are safe, and the early return still skips empty messages.
|
|
17
|
+
const text = msg.text ?? "";
|
|
18
|
+
const paths = extractFileCandidates(text);
|
|
16
19
|
if (paths.length === 0)
|
|
17
20
|
return [];
|
|
18
|
-
const low =
|
|
21
|
+
const low = text.toLowerCase();
|
|
19
22
|
const isWrite = /\b(write|edit|create|save|append|overwrite|update|patch|modify)\b/.test(low);
|
|
20
23
|
return paths.map((p) => ({ path: p, op: isWrite ? "write" : "read" }));
|
|
21
24
|
}
|