pi-mega-compact 0.20.84 → 0.20.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +7 -0
  2. package/dist/extensions/mega-config.js +26 -1
  3. package/dist/extensions/mega-events/context-handler/gateCheck.js +41 -2
  4. package/dist/extensions/mega-events/context-handler/thrashGuard.js +186 -0
  5. package/dist/extensions/mega-events/context-handler.js +33 -1
  6. package/dist/extensions/mega-pipeline/compact/noop.js +104 -0
  7. package/dist/extensions/mega-pipeline/compact/run.js +268 -0
  8. package/dist/extensions/mega-pipeline/compact/vote.js +72 -0
  9. package/dist/extensions/mega-pipeline/compact.js +12 -343
  10. package/dist/extensions/mega-runtime/pressure-getters.js +12 -0
  11. package/dist/src/failback/compact.js +109 -0
  12. package/dist/src/store/sqlite/meta.js +32 -0
  13. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +21 -0
  14. package/extensions/mega-config-types.ts +8 -0
  15. package/extensions/mega-config.ts +26 -1
  16. package/extensions/mega-events/context-handler/gateCheck.ts +48 -2
  17. package/extensions/mega-events/context-handler/thrashGuard.ts +228 -0
  18. package/extensions/mega-events/context-handler.ts +36 -1
  19. package/extensions/mega-pipeline/compact/noop.ts +96 -0
  20. package/extensions/mega-pipeline/compact/run.ts +322 -0
  21. package/extensions/mega-pipeline/compact/vote.ts +85 -0
  22. package/extensions/mega-pipeline/compact.ts +12 -385
  23. package/extensions/mega-runtime/pressure-getters.ts +14 -0
  24. package/package.json +1 -1
  25. package/src/failback/compact.ts +122 -0
  26. package/src/failback/types.ts +44 -0
  27. package/src/store/sqlite/meta.ts +36 -0
@@ -1,345 +1,14 @@
1
1
  /**
2
- * compact.ts — full compaction pipeline (Trident) + pi no-op prediction.
3
- *
4
- * `runCompact` runs the full Trident pipeline (fast-gate aside) and persists a
5
- * checkpoint. `piCompactWouldNoop` predicts whether pi's `ctx.compact()` would
6
- * throw a no-op error. Both mutate the shared MegaRuntime (token accounting,
7
- * ticker, status, events) and are driven by the event + command handlers in
8
- * mega-events.ts / mega-commands.ts.
9
- */
10
- import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
11
- import { compactSession } from "../../src/engine.js";
12
- import { normalizeSessionId } from "../../src/store.js";
13
- import { repoKey } from "../../src/store/repoKey.js";
14
- import { estimateBlockTokens } from "../../src/tokens.js";
15
- import { touchSession, logDaily, incCompactCount, incCacheHitTokens } from "../../src/store/sqlite.js";
16
- import { consolidateMemories } from "../../src/memory.js";
17
- import { C, MARKER_TYPE, } from "../mega-runtime.js";
18
- import { resolveRepoRoot, preserveRecentForPressure } from "../mega-config.js";
19
- import { runRaptor } from "../../src/dedup/raptor/index.js";
20
- import { isRaptorTreeFresh } from "../../src/dedup/raptor/buildHistory.js";
21
- import { loadDedupConfig } from "../../src/config/dedup.js";
22
- import { upsertEmbedding as indexUpsertEmbedding } from "../../src/store/vectorIndex.js";
23
- import { runMemoryReview } from "./memory-review.js";
24
- import { vectorList } from "../../src/vectorStore.js";
25
- /** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
26
- export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
27
- runtime.bindRepo(ctx.cwd);
28
- const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
29
- runtime.resetRuntime(sid);
30
- runtime.rt.sessionId = sid;
31
- const view = runtime.engineView(messages);
32
- // keepFrom deepens with context pressure (Fix E): under high pressure we
33
- // compact more of the session, down to the preserveRecentMin floor.
34
- const preserve = preserveRecentForPressure(opts.compressionPressure ?? 0, config.preserveRecent, config.preserveRecentMin);
35
- const keepFrom = opts.keepFrom ?? Math.max(0, view.length - preserve);
36
- // For very small sessions (fewer messages than preserveRecent), allow
37
- // compacting everything except the last message — the user explicitly
38
- // requested compaction, so don't refuse it just because the session is short.
39
- if (keepFrom <= 0) {
40
- if (view.length <= 1)
41
- return { skipped: true };
42
- // Use the fallback: compact everything except the last message
43
- const fallbackKeepFrom = view.length - 1;
44
- return doCompact(view, fallbackKeepFrom, opts, sid, config, pi, ctx, runtime);
45
- }
46
- return doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime);
47
- }
48
- function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
49
- runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
50
- runtime.setEffect?.("pulse", "accent", 1500); // v0.8.3: ambient border pulse during compaction
51
- // S21.2: reset the per-compaction memory-op counter so the post-compact
52
- // consolidate pass only fires when memory rows actually changed during the
53
- // compaction window (turn_end → auto-review may have written some).
54
- runtime.memoriesTouchedThisCompaction = 0;
55
- const result = compactSession({
56
- sessionId: sid,
57
- messages: view,
58
- keepFrom,
59
- summary: opts.summary,
60
- timestamp: Date.now(),
61
- onTier: runtime.makeTierCallback(ctx),
62
- compressionPressure: opts.compressionPressure,
63
- }, runtime.store);
64
- runtime.pulsing = false;
65
- if (result.skipped)
66
- return { skipped: true };
67
- if (!result.deduped) {
68
- runtime.rt.persistedThisSession = true;
69
- runtime.rt.lastCheckpointId = result.checkpointId;
70
- }
71
- runtime.rt.lastCompactedFrom = result.compactedFrom;
72
- runtime.rt.lastCompactedTokens = result.tokenEstimate;
73
- runtime.rt.dedupAttempts++;
74
- // Honest "tokens saved" for this session-instance only:
75
- // new checkpoint → original − stored
76
- // deduped onto existing → whole original region (nothing new stored)
77
- // Resets to 0 on session_start (rt is rebuilt) — so a fresh session shows 0
78
- // while the repo's cumulative saved (SQLite meta) keeps the running total.
79
- const saved = result.deduped
80
- ? result.originalTokenEstimate
81
- : Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
82
- runtime.rt.tokensSaved += saved;
83
- runtime.rt.compactCount += 1;
84
- incCompactCount(runtime.currentStateDir);
85
- if (result.deduped) {
86
- runtime.rt.cacheHitTokens += saved;
87
- incCacheHitTokens(saved, runtime.currentStateDir);
88
- }
89
- runtime.rt.lastCompactAt = Date.now();
90
- if (result.deduped)
91
- runtime.rt.dedupSkips++;
92
- // Grow the rolling "saved" goal so the progress bar always has a fresh
93
- // denominator (we don't want it pinned at 100% once we pass an old target).
94
- if (runtime.rt.tokensSaved > runtime.savedGoal)
95
- runtime.savedGoal = Math.ceil((runtime.rt.tokensSaved * 1.25) / 10_000) * 10_000;
96
- // Live toolbar activity: what file/region just got compacted or deduped.
97
- // Rendered via the rotating ticker line (see snapshot); the ring buffer is
98
- // cycled one-per-repaint so the single line scrolls through recent files.
99
- const files = result.filesModified ?? [];
100
- const fileLabel = files.length
101
- ? files.map((f) => f.split("/").pop() ?? f).slice(0, 2).join(", ")
102
- : result.regionHash.slice(0, 8);
103
- runtime.lastActivityAt = Date.now();
104
- // Explain-why line: surfaced while fresh. Pulls the dedup reason (which for
105
- // L2 includes the cosine sim) so the user sees WHY a region was kept/dropped.
106
- runtime.lastWhy = result.deduped
107
- ? `why: deduped@${result.dedupReason ?? "tier"}`
108
- : `why: compacted → ${result.checkpointId}`;
109
- // Recall/activity ticker: record this event in the ring buffer.
110
- const savedK = (saved / 1000).toFixed(1);
111
- runtime.pushTicker(result.deduped
112
- ? `${C.green}♻${C.reset} deduped ${fileLabel} · ${savedK}k saved`
113
- : `${C.cyan}🗜${C.reset} ${result.checkpointId} · +${savedK}k · ${fileLabel}`);
114
- // The per-tier trace has settled into the final outcome — fold it back into
115
- // the activity line and stop showing the live trace.
116
- runtime.tierTrace = undefined;
117
- // Record session activity + a daily-log entry in the per-repo SQLite store
118
- // (foundation for resume-sessions / daily-log features). Best-effort — never
119
- // block a compaction on bookkeeping.
120
- try {
121
- const root = resolveRepoRoot(ctx.cwd);
122
- touchSession(sid, root, runtime.currentStateDir);
123
- logDaily(sid, "compact", result.checkpointId, saved, runtime.currentStateDir);
124
- }
125
- catch {
126
- /* non-fatal: stats bookkeeping only */
127
- }
128
- // S21.2: best-effort consolidation of near-duplicate memories for this repo.
129
- // Runs after the per-repo stats touch so `consolidateMemories` can use the
130
- // same stateDir. Non-fatal — a failed consolidate never blocks a compaction.
131
- // Only runs when new memory ops landed in this pass (otherwise the prior
132
- // compaction's consolidate already had its shot — re-running would just
133
- // touch every row again with no merges).
134
- if (!result.deduped && runtime.memoriesTouchedThisCompaction > 0) {
135
- try {
136
- const root = resolveRepoRoot(ctx.cwd);
137
- void consolidateMemories(runtime.currentStateDir, root).then((n) => {
138
- if (n > 0)
139
- runtime.pushTicker(`${C.green}∫${C.reset} consolidated ${n} memory dup${n === 1 ? "" : "s"}`);
140
- }, () => {
141
- /* swallow: consolidate failures must never surface to the user */
142
- });
143
- }
144
- catch {
145
- /* non-fatal */
146
- }
147
- }
148
- // S24 review-on-compact: when pressure is high, the just-compacted region is
149
- // exactly the context worth remembering, so review it immediately rather than
150
- // waiting for the next turn-cadence tick. Uses the shared runMemoryReview
151
- // helper (fire-and-forget; doCompact is sync). Best-effort + non-fatal. Only
152
- // fires above the `high` band so low-pressure compactions don't pay the cost.
153
- if (!result.deduped && config.memoryAutoReview && runtime.pressureBand !== "low" && runtime.pressureBand !== "medium") {
154
- void runMemoryReview(runtime, view, "pressure");
155
- }
156
- // Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
157
- // skip re-vectorizing an already-compacted region (zero token cost).
158
- // v0.8.6: gate on !result.deduped so the marker ONLY lands when a genuinely
159
- // new checkpoint was created. Without this, every dedup re-fire appended a
160
- // fresh sentinel to the real transcript, bloating it and perturbing the
161
- // provider KV-cache prefix (the alternating cache-miss regression). Matches
162
- // the RAPTOR + vector-index blocks above, which are already !deduped-gated.
163
- if (!result.deduped) {
164
- pi.appendEntry(MARKER_TYPE, {
165
- checkpointId: result.checkpointId,
166
- regionHash: result.regionHash,
167
- tokenEstimate: result.tokenEstimate,
168
- deduped: result.deduped,
169
- });
170
- }
171
- // Fix D: refresh the RAPTOR tree for this session so live recall (search) can
172
- // serve high-level summaries. Best-effort + non-fatal: never block compaction.
173
- // Budget-guarded (RAPTOR_BUDGET_MS) so it can't hang a large session.
174
- if (config.raptorEnabled && !result.deduped) {
175
- try {
176
- const dd = loadDedupConfig();
177
- const all = vectorList(runtime.store, sid);
178
- const leaves = all.map((cp) => ({
179
- id: cp.checkpointId,
180
- messages: [],
181
- sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
182
- embedding: cp.embedding,
183
- }));
184
- if (leaves.length >= 2) {
185
- // S42D: skip the rebuild when the last build is fresh (within
186
- // RAPTOR_FRESHNESS_HOURS) and the checkpoint count hasn't drifted by
187
- // more than 20%. avoids re-clustering on every compaction when the
188
- // tree is still representative. 0 disables (always rebuild).
189
- if (dd.RAPTOR_FRESHNESS_HOURS > 0 &&
190
- isRaptorTreeFresh(sid, runtime.currentStateDir, dd.RAPTOR_FRESHNESS_HOURS, all.length)) {
191
- runtime.logger?.info("raptor_skip_fresh", { sessionId: sid });
192
- }
193
- else {
194
- // S25: stamp the tree with the newest checkpoint epoch so the
195
- // freshness guard in raptorSearchHits can reject stale trees after a
196
- // later compaction adds newer checkpoints.
197
- const builtAt = all.length > 0 ? Math.max(...all.map((c) => c.timestamp)) : Date.now();
198
- runRaptor(leaves, {
199
- stateDir: runtime.currentStateDir,
200
- sessionId: sid,
201
- budgetMs: dd.RAPTOR_BUDGET_MS,
202
- clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
203
- consistencyThreshold: dd.RAPTOR_CONSISTENCY,
204
- logger: runtime.logger,
205
- builtAt: Number.isFinite(builtAt) ? builtAt : Date.now(),
206
- });
207
- }
208
- }
209
- }
210
- catch {
211
- /* non-fatal: tree refresh never blocks a compaction */
212
- }
213
- }
214
- // Slice 2: best-effort mirror of the new checkpoint into the async global
215
- // PGlite/HNSW vector index. Fires once per compaction (not per-add), so the
216
- // shared global dir is never hammered by concurrent test workers.
217
- // Non-fatal: a WASM init failure degrades to the sync scan silently.
218
- if (!result.deduped) {
219
- try {
220
- const all = vectorList(runtime.store, sid);
221
- const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
222
- if (latest?.embedding) {
223
- void indexUpsertEmbedding(repoKey(runtime.currentStateDir), sid, latest.checkpointId, latest.embedding).catch(() => {
224
- /* non-fatal: index refresh never blocks a compaction */
225
- });
226
- }
227
- }
228
- catch {
229
- /* non-fatal: index refresh never blocks a compaction */
230
- }
231
- }
232
- runtime.setStatus(ctx, runtime.rt.persistedThisSession
233
- ? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
234
- : `mega-compact: ready`);
235
- runtime.logger.info("compact", {
236
- sessionId: sid,
237
- checkpointId: result.checkpointId ?? "(deduped)",
238
- deduped: result.deduped,
239
- tokenEstimate: saved,
240
- compactedFrom: result.compactedFrom,
241
- });
242
- runtime.dashboard.event("compact", {
243
- sessionId: sid,
244
- checkpointId: result.checkpointId ?? "(deduped)",
245
- deduped: result.deduped,
246
- tokenEstimate: saved,
247
- compactedFrom: result.compactedFrom,
248
- });
249
- runtime.snapshot(ctx);
250
- return { skipped: false, result, keepFrom, saved };
251
- }
252
- /**
253
- * Predict whether pi's `ctx.compact()` would throw a no-op error — "Already
254
- * compacted" or "Nothing to compact (session too small)" — so the auto-trigger
255
- * can SKIP the call instead of surfacing a hard, user-facing error.
256
- *
257
- * Why we can't intercept or suppress it: pi's public `compact()` computes
258
- * `prepareCompaction()` and throws *before* it emits `session_before_compact`,
259
- * so our handler there never runs on the no-op path. And `ctx.compact()`'s
260
- * `onError` callback fires only AFTER pi has already emitted a `compaction_end`
261
- * event carrying the error message (which the interactive UI renders) — so
262
- * `onError` cannot mute it either. The only robust fix is to not call
263
- * `ctx.compact()` when pi would no-op. (pi's own `_runAutoCompaction` path is
264
- * silent on this same condition; the public path we're forced through is the
265
- * one that throws.)
266
- *
267
- * Skipping is correct, not a compromise: by the time this runs, `runCompact()`
268
- * has already persisted the recall checkpoint (Path A). The durable on-disk
269
- * trim is only useful when pi can actually summarize a region; a transcript
270
- * under pi's `keepRecentTokens` budget is small enough that reloading it on
271
- * resume isn't a token-growth problem, so the durable trim is unnecessary
272
- * there anyway.
273
- *
274
- * Mirrors pi's `prepareCompaction()` return-undefined conditions (compaction.js):
275
- * (1) last entry is a compaction → "Already compacted"
276
- * (2) <2 cut-point messages since the last compaction → nothing to summarize
277
- * (a cut point = any non-toolResult message — user/assistant/bash/custom/
278
- * branchSummary/compactionSummary — matching pi's isCutPointMessage)
279
- * (3) transcript tokens since the last compaction < keepRecentTokens → pi
280
- * keeps everything → nothing to summarize
281
- * `keepRecentTokens` isn't readable from the extension API, so (3) uses the pi
282
- * default (20000) as a conservative floor; raise it via
283
- * `MEGACOMPACT_DURABLE_TRIM_FLOOR` if you raise pi's `compact.keepRecentTokens`.
284
- *
285
- * Best-effort: on any read error returns true (skip) — skipping a durable trim
286
- * is always safe; calling `ctx.compact()` on a no-op throws to the user.
2
+ * compact.ts — delegate-shell for the compaction pipeline (3WF-2 split).
3
+ *
4
+ * This file is now a thin barrel: the real bodies live in `./compact/` so the
5
+ * module stays well under the extensions/ 400-line soft cap. The public API is
6
+ * UNCHANGED every existing importer (via `../mega-pipeline.js`, which does
7
+ * `export * from "./mega-pipeline/compact.js"`) keeps working with ZERO changes
8
+ * at their call sites. Exports preserved exactly:
9
+ * - `RunCompactResult` (type)
10
+ * - `runCompact`
11
+ * - `piCompactWouldNoop`
287
12
  */
288
- export function piCompactWouldNoop(ctx) {
289
- try {
290
- const branch = ctx.sessionManager.getBranch();
291
- if (branch.length === 0)
292
- return true;
293
- // (1) already compacted — pi throws "Already compacted"
294
- if (branch[branch.length - 1].type === "compaction")
295
- return true;
296
- // boundaryStart = index just after the most recent compaction entry (or 0)
297
- let boundaryStart = 0;
298
- for (let i = branch.length - 1; i >= 0; i--) {
299
- if (branch[i].type === "compaction") {
300
- boundaryStart = i + 1;
301
- break;
302
- }
303
- }
304
- let cutPoints = 0;
305
- let tokens = 0;
306
- for (let i = boundaryStart; i < branch.length; i++) {
307
- const e = branch[i];
308
- if (e.type === "compaction")
309
- continue;
310
- let isCut = false;
311
- for (const m of sessionEntryToContextMessages(e)) {
312
- // pi's isCutPointMessage: every role except toolResult
313
- if (m.role !== "toolResult")
314
- isCut = true;
315
- const c = m.content;
316
- const text = typeof c === "string" ? c
317
- : Array.isArray(c)
318
- ? c.map((b) => b?.text ?? "").join(" ")
319
- : "";
320
- if (text)
321
- tokens += estimateBlockTokens(text);
322
- }
323
- if (isCut)
324
- cutPoints++;
325
- }
326
- // (2) need >=2 cut points so the kept cut isn't the first message
327
- if (cutPoints < 2)
328
- return true;
329
- // (3) transcript under pi's keepRecentTokens budget → pi keeps everything
330
- if (tokens < durableTrimFloorTokens())
331
- return true;
332
- return false;
333
- }
334
- catch {
335
- return true; // safe: skip the durable trim rather than risk a user-facing throw
336
- }
337
- }
338
- /** pi's default keepRecentTokens (compaction settings). Override with
339
- * MEGACOMPACT_DURABLE_TRIM_FLOOR if you raise pi's compact.keepRecentTokens. */
340
- function durableTrimFloorTokens() {
341
- const raw = process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
342
- if (raw !== undefined && Number.isFinite(Number(raw)))
343
- return Number(raw);
344
- return 20_000;
345
- }
13
+ export { runCompact } from "./compact/run.js";
14
+ export { piCompactWouldNoop } from "./compact/noop.js";
@@ -51,6 +51,18 @@ export function pressureImpl(self) {
51
51
  * always below pi's native auto-compaction (~80% of window).
52
52
  */
53
53
  export function effectiveThresholdImpl(self) {
54
+ // 3WF-2 threshold invariant: under the umbrella, a tiered config with an
55
+ // UNKNOWN window (lastCtxWindow <= 0) DEFERS — auto-compaction must never
56
+ // substitute a guessed window. Returning +Infinity keeps every downstream
57
+ // `tokens >= threshold` comparison false (gateCheck token path,
58
+ // agent_end durable trigger, live-trim re-compact), so no compaction fires
59
+ // until the provider reports a real window. custom (tierPct null) and
60
+ // umbrella-OFF fall through to the legacy helper (byte-identical).
61
+ if (self.config.threeWayFailback &&
62
+ self.config.tierPct != null &&
63
+ self.lastCtxWindow <= 0) {
64
+ return Number.POSITIVE_INFINITY;
65
+ }
54
66
  return effectiveThresholdTokens({
55
67
  tierPct: self.config.tierPct,
56
68
  fallbackThreshold: self.config.thresholdTokens,
@@ -0,0 +1,109 @@
1
+ /**
2
+ * src/failback/compact.ts — 3WF-2 candidate-veto + vote module (pure, advisory).
3
+ *
4
+ * The production bug this fixes: compaction "succeeded" (a checkpoint was
5
+ * persisted, `saved` grew) while the LIVE WINDOW (`currentTokens`) never
6
+ * shrank — because `saved` is a cumulative SQLite total, not the working-set
7
+ * delta. This module builds competing summary candidates (extractive vs
8
+ * cluster/raptor) and VOTES which one, if any, is worth replacing the
9
+ * supersede-only result. It is purely advisory/observational: it never mutates
10
+ * a checkpoint, never overwrites `result.summary`, and returning `null` means
11
+ * "keep the supersede-only result" — the caller must NOT substitute a summary.
12
+ *
13
+ * Pure: no store mutation, no I/O, no network, no console.*. pi-agnostic
14
+ * (imports only from src/). Designed for the 3WF umbrella flag gate at the
15
+ * extension layer (see extensions/mega-pipeline/compact/vote.ts).
16
+ */
17
+ import { collectRecentUserRequests, summarizeMessages } from "../compact.js";
18
+ import { summarizeCluster } from "../dedup/raptor/summarizer.js";
19
+ import { estimateBlockTokens } from "../tokens.js";
20
+ /**
21
+ * Default floor (tokens of net reduction) below which a candidate vote is
22
+ * REJECTED, returning `null` (keep the supersede-only result).
23
+ *
24
+ * Rationale: a candidate that reduces the region by fewer than 1 token is not
25
+ * meaningfully smaller than the compacted region it would replace — swapping
26
+ * the supersede-only result for it buys nothing and only adds a (possibly
27
+ * less faithful) summary. The floor therefore requires the voted summary to be
28
+ * STRICTLY smaller than the compacted region. Set to 1 (minimally defensible:
29
+ * the summary must actually be smaller). Overridable via `opts.floor`.
30
+ */
31
+ export const DEFAULT_VOTE_FLOOR_TOKENS = 1;
32
+ /** Strip a trailing ellipsis/truncation marker from a needle before containment. */
33
+ function stripEllipsis(s) {
34
+ // collectRecentUserRequests truncates to 160 chars via compact.ts's truncate,
35
+ // which appends the U+2026 ellipsis when it cuts. Drop it for a fair test.
36
+ return s.replace(/…\s*$/u, "").trim();
37
+ }
38
+ /** Normalize for containment: collapse whitespace, lowercase. */
39
+ function normalize(s) {
40
+ return s.replace(/\s+/g, " ").trim().toLowerCase();
41
+ }
42
+ /** True when `summary` contains the content of EVERY recent user request. */
43
+ export function signalPreserved(summary, messages) {
44
+ const requests = collectRecentUserRequests(messages, 3);
45
+ if (requests.length === 0)
46
+ return true; // nothing to preserve
47
+ const haystack = normalize(summary);
48
+ return requests.every((r) => haystack.includes(normalize(stripEllipsis(r))));
49
+ }
50
+ /**
51
+ * Build the two competing candidates (extractive + cluster/raptor) for a
52
+ * compacted message region. Degenerate (empty/whitespace-only) summaries are
53
+ * VETOED — never returned. Both candidates use estimateBlockTokens(summary) for
54
+ * a single consistent token basis so the vote compares like with like (the
55
+ * cluster path's own tokenEstimate is intentionally ignored for fairness).
56
+ */
57
+ export function buildCandidates(messages) {
58
+ const out = [];
59
+ const extractive = summarizeMessages(messages);
60
+ if (extractive.trim().length > 0) {
61
+ out.push({
62
+ source: "extractive",
63
+ summary: extractive,
64
+ tokenEstimate: estimateBlockTokens(extractive),
65
+ signalPreserved: signalPreserved(extractive, messages),
66
+ });
67
+ }
68
+ // summarizeCluster returns deterministic extractive when MEGACOMPACT_RAPTOR_MODEL
69
+ // is unset and the local-only Ollama variant when set — so using it makes the
70
+ // Ollama path an insertion that adds NO new LLM call site for the on-by-default
71
+ // extraction. No behavior change for the default config.
72
+ const cluster = summarizeCluster(messages).summary;
73
+ if (cluster.trim().length > 0) {
74
+ out.push({
75
+ source: "cluster",
76
+ summary: cluster,
77
+ tokenEstimate: estimateBlockTokens(cluster),
78
+ signalPreserved: signalPreserved(cluster, messages),
79
+ });
80
+ }
81
+ return out;
82
+ }
83
+ /**
84
+ * Vote the best candidate. `score = reduction * (signalPreserved ? 1 : 0.5)`
85
+ * where `reduction = tokensBefore - candidate.tokenEstimate`. Ties resolve to
86
+ * the EARLIER (extractive) candidate for determinism. Returns `null` when the
87
+ * winner's score is below `opts.floor` (default DEFAULT_VOTE_FLOOR_TOKENS) —
88
+ * caller MUST keep the supersede-only result and must NOT substitute a summary.
89
+ */
90
+ export function voteCandidate(messages, tokensBefore, opts = {}) {
91
+ const floor = opts.floor ?? DEFAULT_VOTE_FLOOR_TOKENS;
92
+ const candidates = buildCandidates(messages);
93
+ let best = null;
94
+ let bestScore = -Infinity;
95
+ for (const c of candidates) {
96
+ const reduction = tokensBefore - c.tokenEstimate;
97
+ const score = reduction * (c.signalPreserved ? 1 : 0.5);
98
+ // Earlier candidate wins ties (strict > keeps insertion order = extractive first).
99
+ if (score > bestScore) {
100
+ bestScore = score;
101
+ best = c;
102
+ }
103
+ }
104
+ if (best === null)
105
+ return null;
106
+ if (bestScore < floor)
107
+ return null;
108
+ return best;
109
+ }
@@ -36,6 +36,38 @@ export function getMetaNumber(key, stateDir = getStateDir()) {
36
36
  const n = raw == null ? 0 : Number(raw);
37
37
  return Number.isFinite(n) ? n : 0;
38
38
  }
39
+ /**
40
+ * Upsert a single numeric meta key to an absolute value (NOT a cumulative
41
+ * counter). Follows the `addTokensSaved` INSERT-ON-CONFLICT pattern with a
42
+ * fully parameterized query (PREVENT-002: no SQL string concat — `key` and
43
+ * `value` are both bound, never interpolated). The only write is the standard
44
+ * ON CONFLICT upsert of THIS key's own value; no other key is touched, no
45
+ * DELETE is issued.
46
+ *
47
+ * Used by the 3WF-2 ThrashGuard to persist exactly two keys:
48
+ * - `thrasguard.baseline_tokens` — the live-window token count at the moment
49
+ * an ineffective compaction was observed (the baseline the guard re-arms from).
50
+ * - `thrasguard.blocked_until` — the live-window token count below which
51
+ * re-firing is refused (guard active).
52
+ *
53
+ * Non-finite input (NaN / ±Infinity) is rejected: the extension must never
54
+ * persist a non-number into the meta table (getMetaNumber would read it back as
55
+ * 0), so we return early, non-fatal. Best-effort: any store failure is swallowed.
56
+ */
57
+ export function setMetaNumber(key, value, stateDir = getStateDir()) {
58
+ if (key.length === 0)
59
+ return;
60
+ if (!Number.isFinite(value))
61
+ return;
62
+ try {
63
+ const db = openStore(stateDir);
64
+ db.prepare(`INSERT INTO meta(key, value) VALUES(?, ?)
65
+ ON CONFLICT(key) DO UPDATE SET value = ?`).run(key, String(value), String(value));
66
+ }
67
+ catch {
68
+ /* non-fatal: meta writes never break the agent loop */
69
+ }
70
+ }
39
71
  /** Atomically add `delta` to an integer meta counter. */
40
72
  function incMeta(key, delta, stateDir = getStateDir()) {
41
73
  if (!(delta > 0))
@@ -274,6 +274,27 @@ export const SETTINGS: ReadonlyArray<SettingGroup> = [
274
274
  num("MEGACOMPACT_EMBEDDING_CHARS_PER_TOKEN", "Embedding Chars per Token", "Estimated characters per token used for embedder chunking size", 4, 1, 32),
275
275
  ],
276
276
  },
277
+ {
278
+ name: "Compaction",
279
+ settings: [
280
+ num(
281
+ "MEGACOMPACT_THRESHOLD_PCT",
282
+ "Compaction Threshold",
283
+ "Fraction of the actual model context window at which compaction fires — 0.80 fires at 80% used (leaves 20% free). Applies to any model size; a per-model Model Thresholds row overrides it",
284
+ 0.8,
285
+ 0.1,
286
+ 0.95,
287
+ ),
288
+ num(
289
+ "MEGACOMPACT_THRASH_REARM_PCT",
290
+ "Thrash Re-arm %",
291
+ "After an ineffective compaction (live window did not shrink), refuse to re-fire until the live window grows by this fraction of the effective threshold. Default 0.10 (10%)",
292
+ 0.1,
293
+ 0.01,
294
+ 0.5,
295
+ ),
296
+ ],
297
+ },
277
298
  VECTOR_CORTEX_SETTINGS,
278
299
  {
279
300
  name: "Cost API",
@@ -140,6 +140,14 @@ export interface MegaConfig {
140
140
  * session_start never fired, so every session has a staged block (recall hits,
141
141
  * else a provenance floor). Default ON; OFF = byte-identical pre-sprint. */
142
142
  threeWayFailback: boolean;
143
+ /** 3WF-2: ThrashGuard re-arm budget as a FRACTION of `effectiveThreshold`.
144
+ * After an ineffective compaction (live window did not shrink), the guard
145
+ * refuses to re-fire until the live window has grown by at least
146
+ * `rearmPct × effectiveThreshold` tokens past the observed baseline. Default
147
+ * 0.10 (10% of the effective threshold). Env-overridable via
148
+ * MEGACOMPACT_THRASH_REARM_PCT. When the effective threshold is unknown
149
+ * (+Infinity), the guard skips arming (cannot compute N) and logs instead. */
150
+ thrashRearmPct: number;
143
151
  /** A1 PLAN_V2 Phase 2: Message Separation — isolate user/assistant turns
144
152
  * from volatile tool results so the prompt-cache prefix stays stable.
145
153
  * PC-A: positive sprint flag, now default ON; flag-OFF (=0) is byte-identical
@@ -78,12 +78,31 @@ function resolveThreshold(): {
78
78
  }
79
79
  const raw = (process.env.MEGACOMPACT_TIER ?? "low").toLowerCase();
80
80
  const tier = (raw in COMPACT_TIERS ? raw : "low") as CompactTier;
81
- const tierPct = TIER_PCT[tier];
81
+ let tierPct = TIER_PCT[tier];
82
+ // 3WF-2 threshold invariant: under the umbrella, when no named tier is set
83
+ // the fire point is the configurable % of the ACTUAL model window (default
84
+ // 0.80 — "20% free remaining"). Tiered (named preset) keeps its preset pct;
85
+ // both paths still compute the legacy 200k boot fallback below as a display
86
+ // placeholder + the custom-tier absolute companion. Umbrella OFF stays
87
+ // byte-identical to v0.20.83 (default tier=low 0.5).
88
+ const umbrella = envBool("MEGACOMPACT_THREE_WAY_FAILBACK", true);
89
+ if (umbrella && !(process.env.MEGACOMPACT_TIER && process.env.MEGACOMPACT_TIER !== "")) {
90
+ tierPct = clamp(envFlag("MEGACOMPACT_THRESHOLD_PCT", 0.8), 0.1, 0.95);
91
+ }
82
92
  // Boot fallback: sane gate before the first context event provides a window.
93
+ // (NO hardcoded window in the firing path — effectiveThresholdImpl defers
94
+ // when window unknown; this remains only a display placeholder + custom
95
+ // companion under the umbrella.)
83
96
  const thresholdTokens = Math.round(tierPct * 200_000);
84
97
  return { tier, tierPct, thresholdTokens };
85
98
  }
86
99
 
100
+ /** Clamp `n` into [lo, hi]; non-finite → fallback. */
101
+ function clamp(n: number, lo: number, hi: number): number {
102
+ if (!Number.isFinite(n)) return lo;
103
+ return Math.min(hi, Math.max(lo, n));
104
+ }
105
+
87
106
  /**
88
107
  * Pure helper: the real compaction fire point, given the model context window.
89
108
  *
@@ -206,6 +225,12 @@ export function loadConfig(): MegaConfig {
206
225
  // 3WF-1: TriggerGuard — guarantee a staged recall block on every context
207
226
  // event even when session_start never fires. Default ON; OFF = byte-identical.
208
227
  threeWayFailback: envBool("MEGACOMPACT_THREE_WAY_FAILBACK", true),
228
+ // 3WF-2: ThrashGuard re-arm budget as a fraction of effectiveThreshold.
229
+ // 0.10 default (10% of the effective threshold) — see mega-config-types.
230
+ // Clamped to [0.01, 0.5]: below 1% the guard is almost never armed (any
231
+ // growth re-fires, defeating the anti-thrash purpose); above 50% it would
232
+ // suppress legitimate re-fires for half the window. Env-overridable.
233
+ thrashRearmPct: clamp(envFlag("MEGACOMPACT_THRASH_REARM_PCT", 0.1), 0.01, 0.5),
209
234
  // PC-A: positive sprint flag, default ON. =0 byte-identical to the
210
235
  // pre-change OFF state (single gate lives at the call site in tailResult.ts).
211
236
  messageSeparation: envBool("MEGACOMPACT_MESSAGE_SEPARATION", true),