pi-mega-compact 0.21.8 → 0.21.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dedup/degenerate.js +68 -0
- package/dist/extensions/dashboard-server/routes-dedup-attribution.js +8 -1
- package/dist/extensions/dashboard-server/routes-rag-settings-compaction.js +42 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +8 -8
- package/dist/extensions/mega-config.js +12 -0
- package/dist/extensions/mega-events/context-handler/gateCheck.js +51 -1
- package/dist/extensions/mega-events/context-handler/headroom.js +128 -0
- package/dist/extensions/mega-events/context-handler/liveTrim.js +37 -51
- package/dist/extensions/mega-events/context-handler/pipelineRun.js +14 -1
- package/dist/extensions/mega-events/context-handler.js +26 -3
- package/dist/extensions/mega-pipeline/compact/run.js +14 -3
- package/dist/extensions/mega-runtime/dashboard-snapshot.js +1 -0
- package/dist/extensions/mega-runtime/runtime-instrumentation.js +1 -0
- package/dist/extensions/mega-runtime/runtime-snapshot.js +1 -0
- package/dist/src/config/dedup.js +3 -0
- package/dist/src/dedup/degenerate.js +68 -0
- package/dist/src/extractive-salvage.js +195 -0
- package/dist/src/extractive.js +63 -72
- package/dist/src/vector-cortex/dedup-attr/rollup.js +5 -0
- package/dist/src/vectorStore/add-degenerate.js +25 -0
- package/dist/src/vectorStore/add.js +28 -5
- package/dist/src/vectorStore/dedup-audit.js +8 -0
- package/dist/vector-cortex/dedup-attr/rollup.js +5 -0
- package/dist/vectorStore/dedup-audit.js +8 -0
- package/extensions/dashboard-server/api-contracts/endpoints/types.ts +2 -0
- package/extensions/dashboard-server/routes-dedup-attribution.ts +11 -1
- package/extensions/dashboard-server/routes-rag-settings-compaction.ts +91 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +13 -27
- package/extensions/mega-config-types.ts +25 -0
- package/extensions/mega-config.ts +12 -0
- package/extensions/mega-dashboard.ts +4 -1
- package/extensions/mega-events/context-handler/gateCheck.ts +67 -0
- package/extensions/mega-events/context-handler/headroom.ts +190 -0
- package/extensions/mega-events/context-handler/liveTrim.ts +37 -57
- package/extensions/mega-events/context-handler/pipelineRun.ts +14 -1
- package/extensions/mega-events/context-handler.ts +26 -3
- package/extensions/mega-pipeline/compact/run.ts +14 -4
- package/extensions/mega-runtime/dashboard-snapshot.ts +3 -0
- package/extensions/mega-runtime/runtime-instrumentation.ts +4 -0
- package/extensions/mega-runtime/runtime-snapshot.ts +2 -0
- package/package.json +1 -1
- package/src/config/dedup.ts +15 -0
- package/src/dedup/degenerate.ts +125 -0
- package/src/extractive-salvage.ts +212 -0
- package/src/extractive.ts +70 -75
- package/src/vector-cortex/dedup-attr/rollup.ts +4 -0
- package/src/vectorStore/add-degenerate.ts +64 -0
- package/src/vectorStore/add.ts +29 -5
- package/src/vectorStore/dedup-audit.ts +25 -2
|
@@ -13,12 +13,9 @@
|
|
|
13
13
|
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
14
14
|
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
15
15
|
import type { EngineMessage } from "../../../src/types.js";
|
|
16
|
-
import {
|
|
17
|
-
estimateBlockTokens,
|
|
18
|
-
estimateMessageTokens,
|
|
19
|
-
} from "../../../src/tokens.js";
|
|
16
|
+
import { estimateBlockTokens } from "../../../src/tokens.js";
|
|
20
17
|
import { computeLiveTrimCut, liveTrimSummaryMessage } from "../../mega-trim.js";
|
|
21
|
-
import {
|
|
18
|
+
import { applyTailCap } from "./headroom.js";
|
|
22
19
|
import type { MegaRuntime } from "../../mega-runtime.js";
|
|
23
20
|
import type { MegaConfig } from "../../mega-config.js";
|
|
24
21
|
import type { TailResultFn } from "./gateCheck.js";
|
|
@@ -136,59 +133,33 @@ export function buildLiveTrimView(
|
|
|
136
133
|
// but has NO token cap, so a 2-message tail of two 80K bash outputs sails
|
|
137
134
|
// right past the window.
|
|
138
135
|
//
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
//
|
|
142
|
-
//
|
|
143
|
-
//
|
|
144
|
-
//
|
|
136
|
+
// v0.21.9: the reserve + front-drop now lives in headroom.ts (single
|
|
137
|
+
// source shared with the gate's pre-fire headroom check and the D.2/D.3
|
|
138
|
+
// replay paths): (a) percent-based reserve — plausible declared maxTokens
|
|
139
|
+
// wins, else clamp(MEGACOMPACT_OUTPUT_RESERVE_PCT, 10–95%) × window — so
|
|
140
|
+
// the math is identical at any window size and a sentinel maxTokens
|
|
141
|
+
// (1e9/1e38) can no longer drive the budget negative and silently
|
|
142
|
+
// disable the cap; (b) budget floor (max(1, …)) so the cap stays active
|
|
143
|
+
// on every window; (c) pair-safe front-drop — the preserved tail never
|
|
144
|
+
// begins on an orphaned toolResult (PREVENT-PI-002).
|
|
145
145
|
const ctxWindow = runtime.lastCtxWindow;
|
|
146
146
|
// Reuse the per-model threshold resolved at the gate (single lookup).
|
|
147
147
|
const modelThreshold = perModelThreshold;
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
ctxWindow
|
|
162
|
-
);
|
|
163
|
-
const budget = ctxWindow - maxOutput - safetyMargin - summaryTokens;
|
|
164
|
-
if (budget > 0) {
|
|
165
|
-
// Walk recent from the front, dropping oldest first until the
|
|
166
|
-
// remaining tail fits. Use the AgentMessage→engine-text estimate via
|
|
167
|
-
// messageContentText (already imported) + estimateMessageTokens.
|
|
168
|
-
let tailTokens = 0;
|
|
169
|
-
for (let i = recentRaw.length - 1; i >= 0; i--) {
|
|
170
|
-
const m = recentRaw[i];
|
|
171
|
-
tailTokens += estimateMessageTokens({
|
|
172
|
-
text: messageContentText(m),
|
|
173
|
-
});
|
|
174
|
-
if (tailTokens > budget) {
|
|
175
|
-
// Keep from i+1 onward; but never fewer than the final message.
|
|
176
|
-
const startIdx = Math.min(i + 1, recentRaw.length - 1);
|
|
177
|
-
if (startIdx > 0) {
|
|
178
|
-
recent = recentRaw.slice(startIdx);
|
|
179
|
-
runtime.logger.warn("live-trim-tail-cap", {
|
|
180
|
-
sessionId: runtime.rt.sessionId,
|
|
181
|
-
dropped: startIdx,
|
|
182
|
-
tailTokens,
|
|
183
|
-
safetyMarginPct: modelThreshold.safetyMarginPct,
|
|
184
|
-
budget,
|
|
185
|
-
ctxWindow,
|
|
186
|
-
});
|
|
187
|
-
}
|
|
188
|
-
break;
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
}
|
|
148
|
+
const { recent, dropped } = applyTailCap({
|
|
149
|
+
recentRaw,
|
|
150
|
+
summaryTokens: estimateBlockTokens(summaryMsg.text),
|
|
151
|
+
ctxWindow,
|
|
152
|
+
maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
|
|
153
|
+
outputReservePct: config.outputReservePct,
|
|
154
|
+
safetyMarginPct: modelThreshold.safetyMarginPct,
|
|
155
|
+
});
|
|
156
|
+
if (dropped > 0) {
|
|
157
|
+
runtime.logger.warn("live-trim-tail-cap", {
|
|
158
|
+
sessionId: runtime.rt.sessionId,
|
|
159
|
+
dropped,
|
|
160
|
+
safetyMarginPct: modelThreshold.safetyMarginPct,
|
|
161
|
+
ctxWindow,
|
|
162
|
+
});
|
|
192
163
|
}
|
|
193
164
|
|
|
194
165
|
// v0.8.6: cache the trim view so subsequent gated calls in this epoch
|
|
@@ -199,8 +170,12 @@ export function buildLiveTrimView(
|
|
|
199
170
|
// (rt.lastCheckpointId) instead of ran.result.checkpointId, which is
|
|
200
171
|
// dedup-volatile: on a re-compact that dedups onto a DIFFERENT existing
|
|
201
172
|
// checkpoint, result.checkpointId is the matched id (engine.ts:188) while
|
|
202
|
-
// lastCheckpointId
|
|
203
|
-
// (
|
|
173
|
+
// lastCheckpointId was, pre-C1, only updated on a genuinely new checkpoint.
|
|
174
|
+
// C1 (v0.21.10) now stamps lastCheckpointId on the dedup path too (see
|
|
175
|
+
// compact/run.ts) — it means "the checkpoint backing this epoch" — so this
|
|
176
|
+
// key and the D.2/D.3 comparison agree in both directions and the `??`
|
|
177
|
+
// fallbacks below are now only for the truly-no-checkpoint edge case.
|
|
178
|
+
// Keying on result.checkpointId directly would still make
|
|
204
179
|
// trimCache.checkpointId != rt.lastCheckpointId forever after that
|
|
205
180
|
// dedup fire, disabling replay for the rest of the epoch (the
|
|
206
181
|
// alternating cache-miss that 0.8.6 meant to fix). Prefer the stable
|
|
@@ -214,6 +189,11 @@ export function buildLiveTrimView(
|
|
|
214
189
|
summaryAgentMsg,
|
|
215
190
|
ctxPct: pct ?? null,
|
|
216
191
|
ctxTokens: currentTokens,
|
|
192
|
+
// v0.21.9: the D.2/D.3 replay paths re-cap the replayed tail against
|
|
193
|
+
// the CURRENT window (a model switch can change it mid-epoch). The
|
|
194
|
+
// margin % used at fire time is stored alongside so the replay uses
|
|
195
|
+
// the same reserve math as the fire that built the view.
|
|
196
|
+
safetyMarginPct: modelThreshold.safetyMarginPct,
|
|
217
197
|
};
|
|
218
198
|
runtime.snapshot(ctx);
|
|
219
199
|
// DIAG (team-run relief): confirm the live trim actually fires + how big
|
|
@@ -21,6 +21,7 @@ import type { MegaConfig } from "../../mega-config.js";
|
|
|
21
21
|
import type { TailResultFn } from "./gateCheck.js";
|
|
22
22
|
import { recordCompactLatency } from "../../mega-runtime/vc-observer.js";
|
|
23
23
|
import { decideLivePath } from "../../mega-runtime/vector-cortex-live.js";
|
|
24
|
+
import { recapReplayedTail } from "./headroom.js";
|
|
24
25
|
import { defaultClock, type RolloutEvidence } from "../../../src/vector-cortex/rollout/gate.js";
|
|
25
26
|
import { VC5C_ENABLED } from "../../../src/config/vector-cortex.js";
|
|
26
27
|
|
|
@@ -120,7 +121,19 @@ export function invokePipeline(
|
|
|
120
121
|
runtime.trimCache.checkpointId === runtime.rt.lastCheckpointId &&
|
|
121
122
|
runtime.trimCache.cut <= opts.messages.length
|
|
122
123
|
) {
|
|
123
|
-
const
|
|
124
|
+
const recentRaw = opts.messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized by computeLiveTrimCut (src/boundary.ts); replayed verbatim, transcript only grows within an epoch.
|
|
125
|
+
// v0.21.9: RE-CAP the replayed tail against the CURRENT window —
|
|
126
|
+
// the D.3 skip-replay bypasses the fire-time tail cap exactly like
|
|
127
|
+
// D.2; a model switch mid-epoch can shrink the window below what
|
|
128
|
+
// the cached view was built for. No-op when the tail already fits.
|
|
129
|
+
const { recent } = recapReplayedTail({
|
|
130
|
+
recentRaw,
|
|
131
|
+
summaryAgentMsg: runtime.trimCache.summaryAgentMsg,
|
|
132
|
+
ctxWindow: runtime.lastCtxWindow,
|
|
133
|
+
maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
|
|
134
|
+
outputReservePct: config.outputReservePct,
|
|
135
|
+
safetyMarginPct: runtime.trimCache.safetyMarginPct,
|
|
136
|
+
});
|
|
124
137
|
runtime.diagLiveTrimFires++;
|
|
125
138
|
runtime.diagLiveTrimReplays++;
|
|
126
139
|
runtime.snapshot(ctx);
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
} from "./context-handler/thrashGuard.js";
|
|
35
35
|
import { invokePipeline } from "./context-handler/pipelineRun.js";
|
|
36
36
|
import { buildLiveTrimView } from "./context-handler/liveTrim.js";
|
|
37
|
+
import { recapReplayedTail } from "./context-handler/headroom.js";
|
|
37
38
|
|
|
38
39
|
/** Register the context event handler (live-trim auto-trigger). */
|
|
39
40
|
export function registerContextHandler(
|
|
@@ -173,7 +174,23 @@ export function registerContextHandler(
|
|
|
173
174
|
: currentTokens - (runtime.trimCache.ctxTokens ?? 0) >=
|
|
174
175
|
runtime.effectiveThreshold * 0.5;
|
|
175
176
|
if (!grewEnough) {
|
|
176
|
-
const
|
|
177
|
+
const recentRaw = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized once by computeLiveTrimCut (src/boundary.ts) and replayed verbatim; the transcript only grows within an epoch (cache is cleared on durable truncation), so the preserved run still starts on a toolPair-safe index.
|
|
178
|
+
// v0.21.9: RE-CAP the replayed tail against the CURRENT window.
|
|
179
|
+
// Replay returns the cached view verbatim, which bypasses the
|
|
180
|
+
// fire-time tail cap — a model switch mid-epoch can shrink the
|
|
181
|
+
// window and leave a replayed tail that fit the OLD window
|
|
182
|
+
// overflowing the NEW one. Same reserve math as the fire
|
|
183
|
+
// (margin % stored in the cache at fire time); no-op when the
|
|
184
|
+
// tail already fits. Pair-safe (applyTailCap advances past any
|
|
185
|
+
// leading toolResult its front-drop exposes).
|
|
186
|
+
const { recent } = recapReplayedTail({
|
|
187
|
+
recentRaw,
|
|
188
|
+
summaryAgentMsg: runtime.trimCache.summaryAgentMsg,
|
|
189
|
+
ctxWindow: runtime.lastCtxWindow,
|
|
190
|
+
maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
|
|
191
|
+
outputReservePct: config.outputReservePct,
|
|
192
|
+
safetyMarginPct: runtime.trimCache.safetyMarginPct,
|
|
193
|
+
});
|
|
177
194
|
runtime.diagLiveTrimFires++; // trim view returned this call (replay counts as a fire)
|
|
178
195
|
runtime.diagLiveTrimReplays++;
|
|
179
196
|
runtime.snapshot(ctx);
|
|
@@ -194,7 +211,7 @@ export function registerContextHandler(
|
|
|
194
211
|
// invokePipeline (the real fire point), so it covers the percent + token
|
|
195
212
|
// gate paths alike. Umbrella OFF ⇒ never blocks (byte-identical). Returns
|
|
196
213
|
// the tailed view so a staged recall block still rides along.
|
|
197
|
-
if (thrashGuardBlocks(runtime, config, currentTokens)) {
|
|
214
|
+
if (thrashGuardBlocks(runtime, config, currentTokens, gate.headroomExceeded)) {
|
|
198
215
|
runtime.diagCtxFastGate++;
|
|
199
216
|
runtime.snapshot(ctx);
|
|
200
217
|
return tailResult() ?? undefined;
|
|
@@ -202,8 +219,14 @@ export function registerContextHandler(
|
|
|
202
219
|
|
|
203
220
|
// Debounce so we don't fire on every context event past threshold.
|
|
204
221
|
// (Replay already returned above — only fresh compacts reach this point.)
|
|
222
|
+
// C2 (v0.21.10): EXEMPT headroom-triggered fires, matching the thrash-guard
|
|
223
|
+
// exemption above. pi's own overflow recovery (400 → compact → immediate
|
|
224
|
+
// retry) re-fires a context event <2s after our last fire; debouncing it
|
|
225
|
+
// returned the RAW untrimmed view, so input + output reserve still blew the
|
|
226
|
+
// window → 400 → "recovery failed after one compact-and-retry attempt".
|
|
227
|
+
// An overflowed session is unrecoverable; a re-fire is merely wasteful.
|
|
205
228
|
const now = Date.now();
|
|
206
|
-
if (now < runtime.debounceUntil) {
|
|
229
|
+
if (now < runtime.debounceUntil && !gate.headroomExceeded) {
|
|
207
230
|
runtime.diagCtxDebounce++;
|
|
208
231
|
return tailResult() ?? undefined;
|
|
209
232
|
}
|
|
@@ -109,10 +109,20 @@ function doCompact(
|
|
|
109
109
|
runtime.pulsing = false;
|
|
110
110
|
|
|
111
111
|
if (result.skipped) return { skipped: true };
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
112
|
+
// C1 (v0.21.10): lastCheckpointId tracks "the checkpoint backing this epoch",
|
|
113
|
+
// so it is stamped on BOTH paths — a matched-dedup checkpoint backs this epoch
|
|
114
|
+
// just as much as a freshly created one. Previously the dedup path left it
|
|
115
|
+
// undefined, so a runtime session whose every compaction deduped (common after
|
|
116
|
+
// a process restart, when checkpoints persist but `rt` is rebuilt) never set it
|
|
117
|
+
// → liveTrim's trimCache fell back to result.checkpointId (the matched id) →
|
|
118
|
+
// `trimCache.checkpointId === rt.lastCheckpointId` was `"chkpt_001" !== undefined`
|
|
119
|
+
// → the D.2/D.3 replay NEVER matched and the full pipeline re-ran on every
|
|
120
|
+
// context event (liveTrimReplays: 0, "comp lag warn"). A later fire matching a
|
|
121
|
+
// DIFFERENT checkpoint now changes the key once (one cache regeneration), then
|
|
122
|
+
// replays stabilise. `persistedThisSession` keeps its narrower meaning ("we
|
|
123
|
+
// wrote NEW state this session") and stays gated on !deduped.
|
|
124
|
+
if (!result.deduped) runtime.rt.persistedThisSession = true;
|
|
125
|
+
runtime.rt.lastCheckpointId = result.checkpointId;
|
|
116
126
|
runtime.rt.lastCompactedFrom = result.compactedFrom;
|
|
117
127
|
runtime.rt.lastCompactedTokens = result.tokenEstimate;
|
|
118
128
|
runtime.rt.dedupAttempts++;
|
|
@@ -49,6 +49,8 @@ export interface SnapshotBuildContext {
|
|
|
49
49
|
readonly diagCtxFastGate: number;
|
|
50
50
|
readonly diagLiveTrimFires: number;
|
|
51
51
|
readonly diagLiveTrimReplays: number;
|
|
52
|
+
/** v0.21.9: output-headroom gate trips (pre-overflow compaction fires). */
|
|
53
|
+
readonly diagCtxHeadroomTrip: number;
|
|
52
54
|
readonly errorRetryCount: number;
|
|
53
55
|
readonly consecutiveErrors: number;
|
|
54
56
|
readonly ERROR_RETRY_MAX_CONSECUTIVE: number;
|
|
@@ -190,6 +192,7 @@ export function buildDashboardSnapshot(ctx: SnapshotBuildContext): DashboardSnap
|
|
|
190
192
|
ctxFastGate: ctx.diagCtxFastGate,
|
|
191
193
|
liveTrimFires: ctx.diagLiveTrimFires,
|
|
192
194
|
liveTrimReplays: ctx.diagLiveTrimReplays,
|
|
195
|
+
headroomTrips: ctx.diagCtxHeadroomTrip,
|
|
193
196
|
},
|
|
194
197
|
retries: {
|
|
195
198
|
errorRetryCount: ctx.errorRetryCount,
|
|
@@ -39,6 +39,7 @@ export class RuntimeInstrumentation {
|
|
|
39
39
|
diagCtxCutNull = 0; // computeLiveTrimCut returned null (anchor/boundary)
|
|
40
40
|
diagCtxThrown = 0; // live-trim try threw (caught)
|
|
41
41
|
diagCtxOutputErrorTrip = 0; // Phase H: output-error catch tripped a forced compaction
|
|
42
|
+
diagCtxHeadroomTrip = 0; // v0.21.9: output-headroom gate tripped a pre-overflow compaction
|
|
42
43
|
|
|
43
44
|
// Context health instrumentation (v0.12): rolling ring buffers for
|
|
44
45
|
// drift detection + cache poison Layer 1 hash baseline.
|
|
@@ -79,6 +80,9 @@ export class RuntimeInstrumentation {
|
|
|
79
80
|
summaryAgentMsg: AgentMessage;
|
|
80
81
|
ctxPct: number | null;
|
|
81
82
|
ctxTokens: number | null;
|
|
83
|
+
/** v0.21.9: safety margin % recorded at fire time so the D.2/D.3 replay
|
|
84
|
+
* paths can re-cap the replayed tail with the same reserve math. */
|
|
85
|
+
safetyMarginPct: number;
|
|
82
86
|
} | null = null;
|
|
83
87
|
debounceUntil = 0;
|
|
84
88
|
// S16: debounce for the agent_end resume nudge (avoid busy-loops).
|
|
@@ -84,6 +84,7 @@ export interface RuntimeSnapshotContext extends RuntimeHelpersContext {
|
|
|
84
84
|
diagCtxFastGate: number;
|
|
85
85
|
diagLiveTrimFires: number;
|
|
86
86
|
diagLiveTrimReplays: number;
|
|
87
|
+
diagCtxHeadroomTrip: number;
|
|
87
88
|
|
|
88
89
|
// ── public methods the orchestration calls ──
|
|
89
90
|
bindRepo(cwd: string | undefined): string;
|
|
@@ -177,6 +178,7 @@ export function snapshotImpl(
|
|
|
177
178
|
diagCtxFastGate: self.diagCtxFastGate,
|
|
178
179
|
diagLiveTrimFires: self.diagLiveTrimFires,
|
|
179
180
|
diagLiveTrimReplays: self.diagLiveTrimReplays,
|
|
181
|
+
diagCtxHeadroomTrip: self.diagCtxHeadroomTrip,
|
|
180
182
|
errorRetryCount: self.rt.errorRetryCount,
|
|
181
183
|
consecutiveErrors: self.rt.consecutiveErrors,
|
|
182
184
|
ERROR_RETRY_MAX_CONSECUTIVE: self.config.maxConsecutiveErrors,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.21.
|
|
3
|
+
"version": "0.21.10",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|
package/src/config/dedup.ts
CHANGED
|
@@ -66,6 +66,18 @@ export interface DedupConfigShape {
|
|
|
66
66
|
L2_COSINE_CODE: number | null;
|
|
67
67
|
L2_COSINE_PROSE: number | null;
|
|
68
68
|
L1_JACCARD: number; // MinHash/LSH near-dup verification
|
|
69
|
+
/**
|
|
70
|
+
* Degenerate-match guard (incident 2026-08-19). ON declines an L1/L2 collapse
|
|
71
|
+
* when the MATCHED stored checkpoint is a content-free skeleton and the
|
|
72
|
+
* incoming candidate is richer, so a degenerate checkpoint can no longer
|
|
73
|
+
* absorb every future compaction forever. OFF is byte-identical to the
|
|
74
|
+
* pre-guard cascade. See src/dedup/degenerate.ts.
|
|
75
|
+
*/
|
|
76
|
+
DEDUP_DEGENERATE_GUARD: boolean;
|
|
77
|
+
/** Absolute token floor under which a stored summary counts as degenerate. */
|
|
78
|
+
DEDUP_DEGEN_MIN_TOKENS: number;
|
|
79
|
+
/** Relative floor as a fraction of the summary's original region size. */
|
|
80
|
+
DEDUP_DEGEN_MIN_PCT: number;
|
|
69
81
|
DEDUP_SIM: number; // legacy content-similarity fallback
|
|
70
82
|
MMR_LAMBDA: number; // retrieval diversity
|
|
71
83
|
SEMDEDUP_COSINE: number; // offline SemDeDup pair threshold
|
|
@@ -123,6 +135,9 @@ export function loadDedupConfig(): DedupConfigShape {
|
|
|
123
135
|
L2_COSINE_CODE: envNumOrNull("MEGACOMPACT_L2_THRESHOLD_CODE"),
|
|
124
136
|
L2_COSINE_PROSE: envNumOrNull("MEGACOMPACT_L2_THRESHOLD_PROSE"),
|
|
125
137
|
L1_JACCARD: envNum("MEGACOMPACT_L1_JACCARD", 0.8),
|
|
138
|
+
DEDUP_DEGENERATE_GUARD: envBool("MEGACOMPACT_DEDUP_DEGENERATE_GUARD", true),
|
|
139
|
+
DEDUP_DEGEN_MIN_TOKENS: envNum("MEGACOMPACT_DEDUP_DEGEN_MIN_TOKENS", 48),
|
|
140
|
+
DEDUP_DEGEN_MIN_PCT: envNum("MEGACOMPACT_DEDUP_DEGEN_MIN_PCT", 0.005),
|
|
126
141
|
DEDUP_SIM: envNum("MEGACOMPACT_DEDUP_SIM", 0.9),
|
|
127
142
|
MMR_LAMBDA: envNum("MEGACOMPACT_MMR_LAMBDA", 0.5),
|
|
128
143
|
SEMDEDUP_COSINE: envNum("MEGACOMPACT_SEMDEDUP_COSINE", 0.95),
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* degenerate.ts — the degenerate-checkpoint predicate + the L1/L2 collapse guard.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS (incident 2026-08-19). A defect in the summarizer could emit a
|
|
5
|
+
* DEGENERATE summary: a ~30-40 token structural skeleton with no informational
|
|
6
|
+
* content, e.g.
|
|
7
|
+
*
|
|
8
|
+
* "Conversation: 64 messages (5 user, 29 assistant, 27 tool). Tools: bash, edit, read."
|
|
9
|
+
*
|
|
10
|
+
* Once such a checkpoint landed in the store it became an absorbing state. Every
|
|
11
|
+
* later compaction produced a structurally identical skeleton, so L1 MinHash and
|
|
12
|
+
* L2 cosine (0.872 against a 0.85 threshold) matched it EVERY time — each add()
|
|
13
|
+
* returned `deduped: true`, discarded the incoming content, and only bumped the
|
|
14
|
+
* stored skeleton's timestamp. The store could never heal, even after the
|
|
15
|
+
* summarizer was fixed, because a rich new summary that happened to match the
|
|
16
|
+
* skeleton would still be swallowed by it.
|
|
17
|
+
*
|
|
18
|
+
* THE GUARD. When L1 or L2 finds a match, we ask whether the MATCHED (stored)
|
|
19
|
+
* checkpoint is degenerate and the incoming candidate is RICHER. If both hold, we
|
|
20
|
+
* decline the collapse and let the cascade continue as if nothing matched — so a
|
|
21
|
+
* fresh, informative checkpoint is written and the skeleton stops absorbing.
|
|
22
|
+
*
|
|
23
|
+
* Direction matters: only "poor stored ← rich incoming" is unblocked. Two equal
|
|
24
|
+
* skeletons still collapse (dedup is doing its job), and a rich stored
|
|
25
|
+
* checkpoint absorbing a poor incoming one is left alone — that is ordinary
|
|
26
|
+
* dedup, not the pathology.
|
|
27
|
+
*
|
|
28
|
+
* PREVENT-PI-004: pure arithmetic over already-loaded fields. No IO, no network.
|
|
29
|
+
*/
|
|
30
|
+
import type { StoredCheckpoint } from "../store.js";
|
|
31
|
+
|
|
32
|
+
/** The tunables the guard reads (thread from DedupConfigShape). */
|
|
33
|
+
export interface DegenerateGuardTunables {
|
|
34
|
+
/** Umbrella flag. OFF ⇒ the guard never fires (byte-identical predecessor). */
|
|
35
|
+
readonly DEDUP_DEGENERATE_GUARD: boolean;
|
|
36
|
+
/** Absolute token floor below which a stored summary is structural, not informational. */
|
|
37
|
+
readonly DEDUP_DEGEN_MIN_TOKENS: number;
|
|
38
|
+
/** Relative floor as a fraction of the ORIGINAL region the summary stands in for. */
|
|
39
|
+
readonly DEDUP_DEGEN_MIN_PCT: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The two fields the predicate scores. Kept structural so tests need no full row. */
|
|
43
|
+
export interface DegenerateSubject {
|
|
44
|
+
tokenEstimate?: number;
|
|
45
|
+
originalTokenEstimate?: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* The effective token floor for a checkpoint: the larger of the absolute floor
|
|
50
|
+
* and `MIN_PCT × originalTokenEstimate`.
|
|
51
|
+
*
|
|
52
|
+
* A missing / zero / non-finite `originalTokenEstimate` contributes nothing, so
|
|
53
|
+
* the absolute floor applies alone — direct add() callers and pre-v0.4 rows that
|
|
54
|
+
* never recorded the original region size are judged on absolute size only,
|
|
55
|
+
* never accidentally deemed degenerate by a 0-valued percentage term.
|
|
56
|
+
*/
|
|
57
|
+
export function degenerateFloor(
|
|
58
|
+
subject: DegenerateSubject,
|
|
59
|
+
tunables: DegenerateGuardTunables,
|
|
60
|
+
): number {
|
|
61
|
+
const orig = subject.originalTokenEstimate;
|
|
62
|
+
const relative =
|
|
63
|
+
typeof orig === "number" && Number.isFinite(orig) && orig > 0
|
|
64
|
+
? orig * tunables.DEDUP_DEGEN_MIN_PCT
|
|
65
|
+
: 0;
|
|
66
|
+
return Math.max(tunables.DEDUP_DEGEN_MIN_TOKENS, relative);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Is this stored checkpoint a degenerate (content-free) summary?
|
|
71
|
+
*
|
|
72
|
+
* Calibration against the incident data:
|
|
73
|
+
* - skeleton: tokenEstimate 34, original ≈19166 → 34 < max(48, 95.8) → TRUE
|
|
74
|
+
* - normal: tokenEstimate 2000, original 70000 → 2000 > max(48, 350) → FALSE
|
|
75
|
+
*
|
|
76
|
+
* The relative term is what makes this scale: a 34-token summary of a 900-token
|
|
77
|
+
* region is a legitimate 26× compression, while the same 34 tokens standing in
|
|
78
|
+
* for 19k is a skeleton.
|
|
79
|
+
*/
|
|
80
|
+
export function isDegenerateCheckpoint(
|
|
81
|
+
subject: DegenerateSubject,
|
|
82
|
+
tunables: DegenerateGuardTunables,
|
|
83
|
+
): boolean {
|
|
84
|
+
const tokens = subject.tokenEstimate ?? 0;
|
|
85
|
+
return tokens < degenerateFloor(subject, tunables);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Should an L1/L2 match be DECLINED because it would collapse richer incoming
|
|
90
|
+
* content onto a degenerate stored checkpoint?
|
|
91
|
+
*
|
|
92
|
+
* Returns true only when all four hold:
|
|
93
|
+
* 1. the umbrella flag is ON,
|
|
94
|
+
* 2. the matched (stored) checkpoint is degenerate,
|
|
95
|
+
* 3. the candidate is strictly richer than the match,
|
|
96
|
+
* 4. the candidate's content is not byte-identical to the match's.
|
|
97
|
+
*
|
|
98
|
+
* Condition 3 uses a strict `>`: equal-size skeletons collapsing is harmless and
|
|
99
|
+
* keeps the store from growing one row per compaction while the summarizer is
|
|
100
|
+
* broken. Only a genuine improvement is worth declining a collapse for.
|
|
101
|
+
*
|
|
102
|
+
* Condition 4 is a CORRECTNESS requirement, not a refinement. `context_chunks`
|
|
103
|
+
* carries a partial UNIQUE index on (session_id, content_hash) (schema/core.ts
|
|
104
|
+
* QA #1), so declining a match whose content hash already exists would fall
|
|
105
|
+
* through to an INSERT that throws — inside add(), which sits on the agent loop.
|
|
106
|
+
* It is also the semantically right call: identical bytes are the SAME region,
|
|
107
|
+
* so re-storing them adds no information and heals nothing. Only L0 may own the
|
|
108
|
+
* exact-match case; the guard exists for fuzzy matches on genuinely different
|
|
109
|
+
* text, which is exactly the incident's shape (each compaction produced a
|
|
110
|
+
* *similar but distinct* skeleton).
|
|
111
|
+
*/
|
|
112
|
+
export function shouldSkipDegenerateMatch(
|
|
113
|
+
matched: StoredCheckpoint,
|
|
114
|
+
candidate: DegenerateSubject & { contentHash?: string },
|
|
115
|
+
tunables: DegenerateGuardTunables,
|
|
116
|
+
): boolean {
|
|
117
|
+
if (!tunables.DEDUP_DEGENERATE_GUARD) return false;
|
|
118
|
+
if (!isDegenerateCheckpoint(matched, tunables)) return false;
|
|
119
|
+
if ((candidate.tokenEstimate ?? 0) <= (matched.tokenEstimate ?? 0)) return false;
|
|
120
|
+
// Byte-identical content → not a healing opportunity (and would violate the
|
|
121
|
+
// UNIQUE index). Compared only when both hashes are known.
|
|
122
|
+
const a = candidate.contentHash;
|
|
123
|
+
const b = matched.contentHash;
|
|
124
|
+
return !(a !== undefined && b !== undefined && a === b);
|
|
125
|
+
}
|