pi-mega-compact 0.21.8 → 0.21.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/dedup/degenerate.js +68 -0
  2. package/dist/extensions/dashboard-server/routes-dedup-attribution.js +8 -1
  3. package/dist/extensions/dashboard-server/routes-rag-settings-compaction.js +42 -0
  4. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +8 -8
  5. package/dist/extensions/mega-config.js +12 -0
  6. package/dist/extensions/mega-events/context-handler/gateCheck.js +51 -1
  7. package/dist/extensions/mega-events/context-handler/headroom.js +128 -0
  8. package/dist/extensions/mega-events/context-handler/liveTrim.js +37 -51
  9. package/dist/extensions/mega-events/context-handler/pipelineRun.js +14 -1
  10. package/dist/extensions/mega-events/context-handler.js +26 -3
  11. package/dist/extensions/mega-pipeline/compact/run.js +14 -3
  12. package/dist/extensions/mega-runtime/dashboard-snapshot.js +1 -0
  13. package/dist/extensions/mega-runtime/runtime-instrumentation.js +1 -0
  14. package/dist/extensions/mega-runtime/runtime-snapshot.js +1 -0
  15. package/dist/src/config/dedup.js +3 -0
  16. package/dist/src/dedup/degenerate.js +68 -0
  17. package/dist/src/extractive-salvage.js +195 -0
  18. package/dist/src/extractive.js +63 -72
  19. package/dist/src/vector-cortex/dedup-attr/rollup.js +5 -0
  20. package/dist/src/vectorStore/add-degenerate.js +25 -0
  21. package/dist/src/vectorStore/add.js +28 -5
  22. package/dist/src/vectorStore/dedup-audit.js +8 -0
  23. package/dist/vector-cortex/dedup-attr/rollup.js +5 -0
  24. package/dist/vectorStore/dedup-audit.js +8 -0
  25. package/extensions/dashboard-server/api-contracts/endpoints/types.ts +2 -0
  26. package/extensions/dashboard-server/routes-dedup-attribution.ts +11 -1
  27. package/extensions/dashboard-server/routes-rag-settings-compaction.ts +91 -0
  28. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +13 -27
  29. package/extensions/mega-config-types.ts +25 -0
  30. package/extensions/mega-config.ts +12 -0
  31. package/extensions/mega-dashboard.ts +4 -1
  32. package/extensions/mega-events/context-handler/gateCheck.ts +67 -0
  33. package/extensions/mega-events/context-handler/headroom.ts +190 -0
  34. package/extensions/mega-events/context-handler/liveTrim.ts +37 -57
  35. package/extensions/mega-events/context-handler/pipelineRun.ts +14 -1
  36. package/extensions/mega-events/context-handler.ts +26 -3
  37. package/extensions/mega-pipeline/compact/run.ts +14 -4
  38. package/extensions/mega-runtime/dashboard-snapshot.ts +3 -0
  39. package/extensions/mega-runtime/runtime-instrumentation.ts +4 -0
  40. package/extensions/mega-runtime/runtime-snapshot.ts +2 -0
  41. package/package.json +1 -1
  42. package/src/config/dedup.ts +15 -0
  43. package/src/dedup/degenerate.ts +125 -0
  44. package/src/extractive-salvage.ts +212 -0
  45. package/src/extractive.ts +70 -75
  46. package/src/vector-cortex/dedup-attr/rollup.ts +4 -0
  47. package/src/vectorStore/add-degenerate.ts +64 -0
  48. package/src/vectorStore/add.ts +29 -5
  49. package/src/vectorStore/dedup-audit.ts +25 -2
@@ -13,12 +13,9 @@
13
13
  import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
14
14
  import type { AgentMessage } from "@earendil-works/pi-agent-core";
15
15
  import type { EngineMessage } from "../../../src/types.js";
16
- import {
17
- estimateBlockTokens,
18
- estimateMessageTokens,
19
- } from "../../../src/tokens.js";
16
+ import { estimateBlockTokens } from "../../../src/tokens.js";
20
17
  import { computeLiveTrimCut, liveTrimSummaryMessage } from "../../mega-trim.js";
21
- import { messageContentText } from "./messageText.js";
18
+ import { applyTailCap } from "./headroom.js";
22
19
  import type { MegaRuntime } from "../../mega-runtime.js";
23
20
  import type { MegaConfig } from "../../mega-config.js";
24
21
  import type { TailResultFn } from "./gateCheck.js";
@@ -136,59 +133,33 @@ export function buildLiveTrimView(
136
133
  // but has NO token cap, so a 2-message tail of two 80K bash outputs sails
137
134
  // right past the window.
138
135
  //
139
- // Cap: when the model context window is known, reserve room for the
140
- // summary + the model's max output tokens + a 10% safety margin, then
141
- // drop oldest preserved messages from the front of `recentRaw` until the
142
- // tail fits. Never drops below the FINAL message (always keep the latest
143
- // turn so the agent can respond). This is a last-resort HARD cap — it
144
- // only fires when the preserved tail alone is oversized, which is rare.
136
+ // v0.21.9: the reserve + front-drop now lives in headroom.ts (single
137
+ // source shared with the gate's pre-fire headroom check and the D.2/D.3
138
+ // replay paths): (a) percent-based reserve — plausible declared maxTokens
139
+ // wins, else clamp(MEGACOMPACT_OUTPUT_RESERVE_PCT, 10–95%) × window — so
140
+ // the math is identical at any window size and a sentinel maxTokens
141
+ // (1e9/1e38) can no longer drive the budget negative and silently
142
+ // disable the cap; (b) budget floor (max(1, …)) so the cap stays active
143
+ // on every window; (c) pair-safe front-drop — the preserved tail never
144
+ // begins on an orphaned toolResult (PREVENT-PI-002).
145
145
  const ctxWindow = runtime.lastCtxWindow;
146
146
  // Reuse the per-model threshold resolved at the gate (single lookup).
147
147
  const modelThreshold = perModelThreshold;
148
- // Reserve room for output tokens. Use the model's reported max output
149
- // when known; fall back to 10% of the window (scales with any model —
150
- // 20K for a 200K window, 100K for a 1M window) so we never let the
151
- // preserved tail eat the model's output budget when maxTokens is unknown.
152
- const maxOutput =
153
- runtime.currentModel?.maxTokens && runtime.currentModel.maxTokens > 0
154
- ? runtime.currentModel.maxTokens
155
- : Math.ceil(ctxWindow * 0.1);
156
- let recent = recentRaw;
157
- if (ctxWindow > 0 && recentRaw.length > 1) {
158
- const summaryTokens = estimateBlockTokens(summaryMsg.text);
159
- // Reserve: summary + max output + per-model safety margin (0-20%).
160
- const safetyMargin = Math.ceil(
161
- ctxWindow * (modelThreshold.safetyMarginPct / 100),
162
- );
163
- const budget = ctxWindow - maxOutput - safetyMargin - summaryTokens;
164
- if (budget > 0) {
165
- // Walk recent from the front, dropping oldest first until the
166
- // remaining tail fits. Use the AgentMessage→engine-text estimate via
167
- // messageContentText (already imported) + estimateMessageTokens.
168
- let tailTokens = 0;
169
- for (let i = recentRaw.length - 1; i >= 0; i--) {
170
- const m = recentRaw[i];
171
- tailTokens += estimateMessageTokens({
172
- text: messageContentText(m),
173
- });
174
- if (tailTokens > budget) {
175
- // Keep from i+1 onward; but never fewer than the final message.
176
- const startIdx = Math.min(i + 1, recentRaw.length - 1);
177
- if (startIdx > 0) {
178
- recent = recentRaw.slice(startIdx);
179
- runtime.logger.warn("live-trim-tail-cap", {
180
- sessionId: runtime.rt.sessionId,
181
- dropped: startIdx,
182
- tailTokens,
183
- safetyMarginPct: modelThreshold.safetyMarginPct,
184
- budget,
185
- ctxWindow,
186
- });
187
- }
188
- break;
189
- }
190
- }
191
- }
148
+ const { recent, dropped } = applyTailCap({
149
+ recentRaw,
150
+ summaryTokens: estimateBlockTokens(summaryMsg.text),
151
+ ctxWindow,
152
+ maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
153
+ outputReservePct: config.outputReservePct,
154
+ safetyMarginPct: modelThreshold.safetyMarginPct,
155
+ });
156
+ if (dropped > 0) {
157
+ runtime.logger.warn("live-trim-tail-cap", {
158
+ sessionId: runtime.rt.sessionId,
159
+ dropped,
160
+ safetyMarginPct: modelThreshold.safetyMarginPct,
161
+ ctxWindow,
162
+ });
192
163
  }
193
164
 
194
165
  // v0.8.6: cache the trim view so subsequent gated calls in this epoch
@@ -199,8 +170,12 @@ export function buildLiveTrimView(
199
170
  // (rt.lastCheckpointId) instead of ran.result.checkpointId, which is
200
171
  // dedup-volatile: on a re-compact that dedups onto a DIFFERENT existing
201
172
  // checkpoint, result.checkpointId is the matched id (engine.ts:188) while
202
- // lastCheckpointId is only updated on a genuinely new checkpoint
203
- // (compact.ts:100-104). Keying on result.checkpointId would make
173
+ // lastCheckpointId was, pre-C1, only updated on a genuinely new checkpoint.
174
+ // C1 (v0.21.10) now stamps lastCheckpointId on the dedup path too (see
175
+ // compact/run.ts) — it means "the checkpoint backing this epoch" — so this
176
+ // key and the D.2/D.3 comparison agree in both directions and the `??`
177
+ // fallbacks below are now only for the truly-no-checkpoint edge case.
178
+ // Keying on result.checkpointId directly would still make
204
179
  // trimCache.checkpointId != rt.lastCheckpointId forever after that
205
180
  // dedup fire, disabling replay for the rest of the epoch (the
206
181
  // alternating cache-miss that 0.8.6 meant to fix). Prefer the stable
@@ -214,6 +189,11 @@ export function buildLiveTrimView(
214
189
  summaryAgentMsg,
215
190
  ctxPct: pct ?? null,
216
191
  ctxTokens: currentTokens,
192
+ // v0.21.9: the D.2/D.3 replay paths re-cap the replayed tail against
193
+ // the CURRENT window (a model switch can change it mid-epoch). The
194
+ // margin % used at fire time is stored alongside so the replay uses
195
+ // the same reserve math as the fire that built the view.
196
+ safetyMarginPct: modelThreshold.safetyMarginPct,
217
197
  };
218
198
  runtime.snapshot(ctx);
219
199
  // DIAG (team-run relief): confirm the live trim actually fires + how big
@@ -21,6 +21,7 @@ import type { MegaConfig } from "../../mega-config.js";
21
21
  import type { TailResultFn } from "./gateCheck.js";
22
22
  import { recordCompactLatency } from "../../mega-runtime/vc-observer.js";
23
23
  import { decideLivePath } from "../../mega-runtime/vector-cortex-live.js";
24
+ import { recapReplayedTail } from "./headroom.js";
24
25
  import { defaultClock, type RolloutEvidence } from "../../../src/vector-cortex/rollout/gate.js";
25
26
  import { VC5C_ENABLED } from "../../../src/config/vector-cortex.js";
26
27
 
@@ -120,7 +121,19 @@ export function invokePipeline(
120
121
  runtime.trimCache.checkpointId === runtime.rt.lastCheckpointId &&
121
122
  runtime.trimCache.cut <= opts.messages.length
122
123
  ) {
123
- const recent = opts.messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized by computeLiveTrimCut (src/boundary.ts); replayed verbatim, transcript only grows within an epoch.
124
+ const recentRaw = opts.messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized by computeLiveTrimCut (src/boundary.ts); replayed verbatim, transcript only grows within an epoch.
125
+ // v0.21.9: RE-CAP the replayed tail against the CURRENT window —
126
+ // the D.3 skip-replay bypasses the fire-time tail cap exactly like
127
+ // D.2; a model switch mid-epoch can shrink the window below what
128
+ // the cached view was built for. No-op when the tail already fits.
129
+ const { recent } = recapReplayedTail({
130
+ recentRaw,
131
+ summaryAgentMsg: runtime.trimCache.summaryAgentMsg,
132
+ ctxWindow: runtime.lastCtxWindow,
133
+ maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
134
+ outputReservePct: config.outputReservePct,
135
+ safetyMarginPct: runtime.trimCache.safetyMarginPct,
136
+ });
124
137
  runtime.diagLiveTrimFires++;
125
138
  runtime.diagLiveTrimReplays++;
126
139
  runtime.snapshot(ctx);
@@ -34,6 +34,7 @@ import {
34
34
  } from "./context-handler/thrashGuard.js";
35
35
  import { invokePipeline } from "./context-handler/pipelineRun.js";
36
36
  import { buildLiveTrimView } from "./context-handler/liveTrim.js";
37
+ import { recapReplayedTail } from "./context-handler/headroom.js";
37
38
 
38
39
  /** Register the context event handler (live-trim auto-trigger). */
39
40
  export function registerContextHandler(
@@ -173,7 +174,23 @@ export function registerContextHandler(
173
174
  : currentTokens - (runtime.trimCache.ctxTokens ?? 0) >=
174
175
  runtime.effectiveThreshold * 0.5;
175
176
  if (!grewEnough) {
176
- const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized once by computeLiveTrimCut (src/boundary.ts) and replayed verbatim; the transcript only grows within an epoch (cache is cleared on durable truncation), so the preserved run still starts on a toolPair-safe index.
177
+ const recentRaw = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized once by computeLiveTrimCut (src/boundary.ts) and replayed verbatim; the transcript only grows within an epoch (cache is cleared on durable truncation), so the preserved run still starts on a toolPair-safe index.
178
+ // v0.21.9: RE-CAP the replayed tail against the CURRENT window.
179
+ // Replay returns the cached view verbatim, which bypasses the
180
+ // fire-time tail cap — a model switch mid-epoch can shrink the
181
+ // window and leave a replayed tail that fit the OLD window
182
+ // overflowing the NEW one. Same reserve math as the fire
183
+ // (margin % stored in the cache at fire time); no-op when the
184
+ // tail already fits. Pair-safe (applyTailCap advances past any
185
+ // leading toolResult its front-drop exposes).
186
+ const { recent } = recapReplayedTail({
187
+ recentRaw,
188
+ summaryAgentMsg: runtime.trimCache.summaryAgentMsg,
189
+ ctxWindow: runtime.lastCtxWindow,
190
+ maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
191
+ outputReservePct: config.outputReservePct,
192
+ safetyMarginPct: runtime.trimCache.safetyMarginPct,
193
+ });
177
194
  runtime.diagLiveTrimFires++; // trim view returned this call (replay counts as a fire)
178
195
  runtime.diagLiveTrimReplays++;
179
196
  runtime.snapshot(ctx);
@@ -194,7 +211,7 @@ export function registerContextHandler(
194
211
  // invokePipeline (the real fire point), so it covers the percent + token
195
212
  // gate paths alike. Umbrella OFF ⇒ never blocks (byte-identical). Returns
196
213
  // the tailed view so a staged recall block still rides along.
197
- if (thrashGuardBlocks(runtime, config, currentTokens)) {
214
+ if (thrashGuardBlocks(runtime, config, currentTokens, gate.headroomExceeded)) {
198
215
  runtime.diagCtxFastGate++;
199
216
  runtime.snapshot(ctx);
200
217
  return tailResult() ?? undefined;
@@ -202,8 +219,14 @@ export function registerContextHandler(
202
219
 
203
220
  // Debounce so we don't fire on every context event past threshold.
204
221
  // (Replay already returned above — only fresh compacts reach this point.)
222
+ // C2 (v0.21.10): EXEMPT headroom-triggered fires, matching the thrash-guard
223
+ // exemption above. pi's own overflow recovery (400 → compact → immediate
224
+ // retry) re-fires a context event <2s after our last fire; debouncing it
225
+ // returned the RAW untrimmed view, so input + output reserve still blew the
226
+ // window → 400 → "recovery failed after one compact-and-retry attempt".
227
+ // An overflowed session is unrecoverable; a re-fire is merely wasteful.
205
228
  const now = Date.now();
206
- if (now < runtime.debounceUntil) {
229
+ if (now < runtime.debounceUntil && !gate.headroomExceeded) {
207
230
  runtime.diagCtxDebounce++;
208
231
  return tailResult() ?? undefined;
209
232
  }
@@ -109,10 +109,20 @@ function doCompact(
109
109
  runtime.pulsing = false;
110
110
 
111
111
  if (result.skipped) return { skipped: true };
112
- if (!result.deduped) {
113
- runtime.rt.persistedThisSession = true;
114
- runtime.rt.lastCheckpointId = result.checkpointId;
115
- }
112
+ // C1 (v0.21.10): lastCheckpointId tracks "the checkpoint backing this epoch",
113
+ // so it is stamped on BOTH paths — a matched-dedup checkpoint backs this epoch
114
+ // just as much as a freshly created one. Previously the dedup path left it
115
+ // undefined, so a runtime session whose every compaction deduped (common after
116
+ // a process restart, when checkpoints persist but `rt` is rebuilt) never set it
117
+ // → liveTrim's trimCache fell back to result.checkpointId (the matched id) →
118
+ // `trimCache.checkpointId === rt.lastCheckpointId` was `"chkpt_001" !== undefined`
119
+ // → the D.2/D.3 replay NEVER matched and the full pipeline re-ran on every
120
+ // context event (liveTrimReplays: 0, "comp lag warn"). A later fire matching a
121
+ // DIFFERENT checkpoint now changes the key once (one cache regeneration), then
122
+ // replays stabilise. `persistedThisSession` keeps its narrower meaning ("we
123
+ // wrote NEW state this session") and stays gated on !deduped.
124
+ if (!result.deduped) runtime.rt.persistedThisSession = true;
125
+ runtime.rt.lastCheckpointId = result.checkpointId;
116
126
  runtime.rt.lastCompactedFrom = result.compactedFrom;
117
127
  runtime.rt.lastCompactedTokens = result.tokenEstimate;
118
128
  runtime.rt.dedupAttempts++;
@@ -49,6 +49,8 @@ export interface SnapshotBuildContext {
49
49
  readonly diagCtxFastGate: number;
50
50
  readonly diagLiveTrimFires: number;
51
51
  readonly diagLiveTrimReplays: number;
52
+ /** v0.21.9: output-headroom gate trips (pre-overflow compaction fires). */
53
+ readonly diagCtxHeadroomTrip: number;
52
54
  readonly errorRetryCount: number;
53
55
  readonly consecutiveErrors: number;
54
56
  readonly ERROR_RETRY_MAX_CONSECUTIVE: number;
@@ -190,6 +192,7 @@ export function buildDashboardSnapshot(ctx: SnapshotBuildContext): DashboardSnap
190
192
  ctxFastGate: ctx.diagCtxFastGate,
191
193
  liveTrimFires: ctx.diagLiveTrimFires,
192
194
  liveTrimReplays: ctx.diagLiveTrimReplays,
195
+ headroomTrips: ctx.diagCtxHeadroomTrip,
193
196
  },
194
197
  retries: {
195
198
  errorRetryCount: ctx.errorRetryCount,
@@ -39,6 +39,7 @@ export class RuntimeInstrumentation {
39
39
  diagCtxCutNull = 0; // computeLiveTrimCut returned null (anchor/boundary)
40
40
  diagCtxThrown = 0; // live-trim try threw (caught)
41
41
  diagCtxOutputErrorTrip = 0; // Phase H: output-error catch tripped a forced compaction
42
+ diagCtxHeadroomTrip = 0; // v0.21.9: output-headroom gate tripped a pre-overflow compaction
42
43
 
43
44
  // Context health instrumentation (v0.12): rolling ring buffers for
44
45
  // drift detection + cache poison Layer 1 hash baseline.
@@ -79,6 +80,9 @@ export class RuntimeInstrumentation {
79
80
  summaryAgentMsg: AgentMessage;
80
81
  ctxPct: number | null;
81
82
  ctxTokens: number | null;
83
+ /** v0.21.9: safety margin % recorded at fire time so the D.2/D.3 replay
84
+ * paths can re-cap the replayed tail with the same reserve math. */
85
+ safetyMarginPct: number;
82
86
  } | null = null;
83
87
  debounceUntil = 0;
84
88
  // S16: debounce for the agent_end resume nudge (avoid busy-loops).
@@ -84,6 +84,7 @@ export interface RuntimeSnapshotContext extends RuntimeHelpersContext {
84
84
  diagCtxFastGate: number;
85
85
  diagLiveTrimFires: number;
86
86
  diagLiveTrimReplays: number;
87
+ diagCtxHeadroomTrip: number;
87
88
 
88
89
  // ── public methods the orchestration calls ──
89
90
  bindRepo(cwd: string | undefined): string;
@@ -177,6 +178,7 @@ export function snapshotImpl(
177
178
  diagCtxFastGate: self.diagCtxFastGate,
178
179
  diagLiveTrimFires: self.diagLiveTrimFires,
179
180
  diagLiveTrimReplays: self.diagLiveTrimReplays,
181
+ diagCtxHeadroomTrip: self.diagCtxHeadroomTrip,
180
182
  errorRetryCount: self.rt.errorRetryCount,
181
183
  consecutiveErrors: self.rt.consecutiveErrors,
182
184
  ERROR_RETRY_MAX_CONSECUTIVE: self.config.maxConsecutiveErrors,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.21.8",
3
+ "version": "0.21.10",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-3-Clause",
@@ -66,6 +66,18 @@ export interface DedupConfigShape {
66
66
  L2_COSINE_CODE: number | null;
67
67
  L2_COSINE_PROSE: number | null;
68
68
  L1_JACCARD: number; // MinHash/LSH near-dup verification
69
+ /**
70
+ * Degenerate-match guard (incident 2026-08-19). ON declines an L1/L2 collapse
71
+ * when the MATCHED stored checkpoint is a content-free skeleton and the
72
+ * incoming candidate is richer, so a degenerate checkpoint can no longer
73
+ * absorb every future compaction forever. OFF is byte-identical to the
74
+ * pre-guard cascade. See src/dedup/degenerate.ts.
75
+ */
76
+ DEDUP_DEGENERATE_GUARD: boolean;
77
+ /** Absolute token floor under which a stored summary counts as degenerate. */
78
+ DEDUP_DEGEN_MIN_TOKENS: number;
79
+ /** Relative floor as a fraction of the summary's original region size. */
80
+ DEDUP_DEGEN_MIN_PCT: number;
69
81
  DEDUP_SIM: number; // legacy content-similarity fallback
70
82
  MMR_LAMBDA: number; // retrieval diversity
71
83
  SEMDEDUP_COSINE: number; // offline SemDeDup pair threshold
@@ -123,6 +135,9 @@ export function loadDedupConfig(): DedupConfigShape {
123
135
  L2_COSINE_CODE: envNumOrNull("MEGACOMPACT_L2_THRESHOLD_CODE"),
124
136
  L2_COSINE_PROSE: envNumOrNull("MEGACOMPACT_L2_THRESHOLD_PROSE"),
125
137
  L1_JACCARD: envNum("MEGACOMPACT_L1_JACCARD", 0.8),
138
+ DEDUP_DEGENERATE_GUARD: envBool("MEGACOMPACT_DEDUP_DEGENERATE_GUARD", true),
139
+ DEDUP_DEGEN_MIN_TOKENS: envNum("MEGACOMPACT_DEDUP_DEGEN_MIN_TOKENS", 48),
140
+ DEDUP_DEGEN_MIN_PCT: envNum("MEGACOMPACT_DEDUP_DEGEN_MIN_PCT", 0.005),
126
141
  DEDUP_SIM: envNum("MEGACOMPACT_DEDUP_SIM", 0.9),
127
142
  MMR_LAMBDA: envNum("MEGACOMPACT_MMR_LAMBDA", 0.5),
128
143
  SEMDEDUP_COSINE: envNum("MEGACOMPACT_SEMDEDUP_COSINE", 0.95),
@@ -0,0 +1,125 @@
1
+ /**
2
+ * degenerate.ts — the degenerate-checkpoint predicate + the L1/L2 collapse guard.
3
+ *
4
+ * WHY THIS EXISTS (incident 2026-08-19). A defect in the summarizer could emit a
5
+ * DEGENERATE summary: a ~30-40 token structural skeleton with no informational
6
+ * content, e.g.
7
+ *
8
+ * "Conversation: 64 messages (5 user, 29 assistant, 27 tool). Tools: bash, edit, read."
9
+ *
10
+ * Once such a checkpoint landed in the store it became an absorbing state. Every
11
+ * later compaction produced a structurally identical skeleton, so L1 MinHash and
12
+ * L2 cosine (0.872 against a 0.85 threshold) matched it EVERY time — each add()
13
+ * returned `deduped: true`, discarded the incoming content, and only bumped the
14
+ * stored skeleton's timestamp. The store could never heal, even after the
15
+ * summarizer was fixed, because a rich new summary that happened to match the
16
+ * skeleton would still be swallowed by it.
17
+ *
18
+ * THE GUARD. When L1 or L2 finds a match, we ask whether the MATCHED (stored)
19
+ * checkpoint is degenerate and the incoming candidate is RICHER. If both hold, we
20
+ * decline the collapse and let the cascade continue as if nothing matched — so a
21
+ * fresh, informative checkpoint is written and the skeleton stops absorbing.
22
+ *
23
+ * Direction matters: only "poor stored ← rich incoming" is unblocked. Two equal
24
+ * skeletons still collapse (dedup is doing its job), and a rich stored
25
+ * checkpoint absorbing a poor incoming one is left alone — that is ordinary
26
+ * dedup, not the pathology.
27
+ *
28
+ * PREVENT-PI-004: pure arithmetic over already-loaded fields. No IO, no network.
29
+ */
30
+ import type { StoredCheckpoint } from "../store.js";
31
+
32
+ /** The tunables the guard reads (thread from DedupConfigShape). */
33
+ export interface DegenerateGuardTunables {
34
+ /** Umbrella flag. OFF ⇒ the guard never fires (byte-identical predecessor). */
35
+ readonly DEDUP_DEGENERATE_GUARD: boolean;
36
+ /** Absolute token floor below which a stored summary is structural, not informational. */
37
+ readonly DEDUP_DEGEN_MIN_TOKENS: number;
38
+ /** Relative floor as a fraction of the ORIGINAL region the summary stands in for. */
39
+ readonly DEDUP_DEGEN_MIN_PCT: number;
40
+ }
41
+
42
+ /** The two fields the predicate scores. Kept structural so tests need no full row. */
43
+ export interface DegenerateSubject {
44
+ tokenEstimate?: number;
45
+ originalTokenEstimate?: number;
46
+ }
47
+
48
+ /**
49
+ * The effective token floor for a checkpoint: the larger of the absolute floor
50
+ * and `MIN_PCT × originalTokenEstimate`.
51
+ *
52
+ * A missing / zero / non-finite `originalTokenEstimate` contributes nothing, so
53
+ * the absolute floor applies alone — direct add() callers and pre-v0.4 rows that
54
+ * never recorded the original region size are judged on absolute size only,
55
+ * never accidentally deemed degenerate by a 0-valued percentage term.
56
+ */
57
+ export function degenerateFloor(
58
+ subject: DegenerateSubject,
59
+ tunables: DegenerateGuardTunables,
60
+ ): number {
61
+ const orig = subject.originalTokenEstimate;
62
+ const relative =
63
+ typeof orig === "number" && Number.isFinite(orig) && orig > 0
64
+ ? orig * tunables.DEDUP_DEGEN_MIN_PCT
65
+ : 0;
66
+ return Math.max(tunables.DEDUP_DEGEN_MIN_TOKENS, relative);
67
+ }
68
+
69
+ /**
70
+ * Is this stored checkpoint a degenerate (content-free) summary?
71
+ *
72
+ * Calibration against the incident data:
73
+ * - skeleton: tokenEstimate 34, original ≈19166 → 34 < max(48, 95.8) → TRUE
74
+ * - normal: tokenEstimate 2000, original 70000 → 2000 > max(48, 350) → FALSE
75
+ *
76
+ * The relative term is what makes this scale: a 34-token summary of a 900-token
77
+ * region is a legitimate 26× compression, while the same 34 tokens standing in
78
+ * for 19k is a skeleton.
79
+ */
80
+ export function isDegenerateCheckpoint(
81
+ subject: DegenerateSubject,
82
+ tunables: DegenerateGuardTunables,
83
+ ): boolean {
84
+ const tokens = subject.tokenEstimate ?? 0;
85
+ return tokens < degenerateFloor(subject, tunables);
86
+ }
87
+
88
+ /**
89
+ * Should an L1/L2 match be DECLINED because it would collapse richer incoming
90
+ * content onto a degenerate stored checkpoint?
91
+ *
92
+ * Returns true only when all four hold:
93
+ * 1. the umbrella flag is ON,
94
+ * 2. the matched (stored) checkpoint is degenerate,
95
+ * 3. the candidate is strictly richer than the match,
96
+ * 4. the candidate's content is not byte-identical to the match's.
97
+ *
98
+ * Condition 3 uses a strict `>`: equal-size skeletons collapsing is harmless and
99
+ * keeps the store from growing one row per compaction while the summarizer is
100
+ * broken. Only a genuine improvement is worth declining a collapse for.
101
+ *
102
+ * Condition 4 is a CORRECTNESS requirement, not a refinement. `context_chunks`
103
+ * carries a partial UNIQUE index on (session_id, content_hash) (schema/core.ts
104
+ * QA #1), so declining a match whose content hash already exists would fall
105
+ * through to an INSERT that throws — inside add(), which sits on the agent loop.
106
+ * It is also the semantically right call: identical bytes are the SAME region,
107
+ * so re-storing them adds no information and heals nothing. Only L0 may own the
108
+ * exact-match case; the guard exists for fuzzy matches on genuinely different
109
+ * text, which is exactly the incident's shape (each compaction produced a
110
+ * *similar but distinct* skeleton).
111
+ */
112
+ export function shouldSkipDegenerateMatch(
113
+ matched: StoredCheckpoint,
114
+ candidate: DegenerateSubject & { contentHash?: string },
115
+ tunables: DegenerateGuardTunables,
116
+ ): boolean {
117
+ if (!tunables.DEDUP_DEGENERATE_GUARD) return false;
118
+ if (!isDegenerateCheckpoint(matched, tunables)) return false;
119
+ if ((candidate.tokenEstimate ?? 0) <= (matched.tokenEstimate ?? 0)) return false;
120
+ // Byte-identical content → not a healing opportunity (and would violate the
121
+ // UNIQUE index). Compared only when both hashes are known.
122
+ const a = candidate.contentHash;
123
+ const b = matched.contentHash;
124
+ return !(a !== undefined && b !== undefined && a === b);
125
+ }