pi-mega-compact 0.20.85 → 0.20.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/config.js +9 -0
  2. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +2 -0
  3. package/dist/extensions/mega-config.js +12 -0
  4. package/dist/extensions/mega-events/context-handler/gateCheck.js +27 -0
  5. package/dist/extensions/mega-events/context-handler/thrashGuard.js +186 -0
  6. package/dist/extensions/mega-events/context-handler.js +33 -1
  7. package/dist/extensions/mega-pipeline/compact/noop.js +104 -0
  8. package/dist/extensions/mega-pipeline/compact/run.js +268 -0
  9. package/dist/extensions/mega-pipeline/compact/vote.js +72 -0
  10. package/dist/extensions/mega-pipeline/compact.js +12 -343
  11. package/dist/extensions/mega-pipeline/recall/impl.js +258 -0
  12. package/dist/extensions/mega-pipeline/recall.js +6 -253
  13. package/dist/src/config.js +9 -0
  14. package/dist/src/failback/compact.js +109 -0
  15. package/dist/src/recall/readonly.js +39 -0
  16. package/dist/src/recall/recall3wf.fixture.js +67 -0
  17. package/dist/src/recall/validator.js +113 -0
  18. package/dist/src/recall/vote.js +217 -0
  19. package/dist/src/store/sqlite/fts5-search.js +26 -0
  20. package/dist/src/store/sqlite/meta.js +32 -0
  21. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +9 -0
  22. package/extensions/mega-config-types.ts +13 -0
  23. package/extensions/mega-config.ts +12 -0
  24. package/extensions/mega-events/context-handler/gateCheck.ts +30 -0
  25. package/extensions/mega-events/context-handler/thrashGuard.ts +228 -0
  26. package/extensions/mega-events/context-handler.ts +36 -1
  27. package/extensions/mega-pipeline/compact/noop.ts +96 -0
  28. package/extensions/mega-pipeline/compact/run.ts +322 -0
  29. package/extensions/mega-pipeline/compact/vote.ts +85 -0
  30. package/extensions/mega-pipeline/compact.ts +12 -385
  31. package/extensions/mega-pipeline/recall/impl.ts +312 -0
  32. package/extensions/mega-pipeline/recall.ts +10 -306
  33. package/package.json +1 -1
  34. package/src/config.ts +12 -0
  35. package/src/failback/compact.ts +122 -0
  36. package/src/failback/types.ts +72 -0
  37. package/src/recall/readonly.ts +57 -0
  38. package/src/recall/recall3wf.fixture.ts +87 -0
  39. package/src/recall/validator.ts +150 -0
  40. package/src/recall/vote.ts +240 -0
  41. package/src/store/sqlite/fts5-search.ts +40 -0
  42. package/src/store/sqlite/meta.ts +36 -0
@@ -0,0 +1,217 @@
1
+ /**
2
+ * recall/vote.ts — the 3-independent-source recall vote (3WF-3).
3
+ *
4
+ * Three INDEPENDENT, read-only sources name candidate checkpoints:
5
+ * A vector — raw semantic hits (recall/readonly.ts), cosine 0..1 scale.
6
+ * B fts5 — BM25 trigram hits (hydrated to checkpointIds), FTS5 BM25 scale
7
+ * (negative score = better match; ranking order is what matters).
8
+ * C recency — the N freshest checkpoints by timestamp, query-INDEPENDENT.
9
+ * (NOT turn_recall / TurnReader — that would just echo already-
10
+ * injected content, not an independent signal.)
11
+ *
12
+ * Each source is a different score scale, so they are NOT directly comparable.
13
+ * We normalize each source's scores to a 0..1 scale (per-source min-max) BEFORE
14
+ * combining. Averaging raw cosine (0..1) with raw BM25 (arbitrary negative
15
+ * magnitude) or a recency rank would be meaningless — the largest-magnitude
16
+ * scale would always dominate. Normalization makes each source a peer voter.
17
+ *
18
+ * Overlap rule: a checkpoint named by >=2 of 3 distinct sources short-circuits
19
+ * as a winner. Fallback (no 2/3 majority): rank all candidates by the
20
+ * cross-source MEAN of their normalized scores.
21
+ *
22
+ * Non-fatal throughout. Pi-agnostic: no pi runtime imports.
23
+ */
24
+ import { openStore } from "../store/sqlite/utils.js";
25
+ import { fts5SearchScoped, hydrateFts5Hits } from "../store/sqlite/fts5-search.js";
26
+ // The SQLite store is the source of truth (src/store.ts's same-named helper
27
+ // reads the LEGACY gzipped-JSON DR snapshot, which is empty for live sessions —
28
+ // importing it here would silently starve sources B and C). Mirrors the import
29
+ // in vector-search.ts + tieredRouter.ts.
30
+ import { listCheckpoints } from "../store/sqlite.js";
31
+ import { computeContentDigest } from "../dedup/digest.js";
32
+ import { recallRawHits } from "./readonly.js";
33
+ import { Logger } from "../log.js";
34
+ /** Per-source normalization: map raw scores to 0..1 via min-max within source. */
35
+ function normalizeScores(scores) {
36
+ const map = new Map();
37
+ if (scores.length === 0)
38
+ return map;
39
+ const min = Math.min(...scores);
40
+ const max = Math.max(...scores);
41
+ const span = max - min;
42
+ scores.forEach((s, i) => {
43
+ map.set(i, span === 0 ? 1 : (s - min) / span);
44
+ });
45
+ return map;
46
+ }
47
+ /**
48
+ * Run the three-source recall vote. Returns agreement winners + a per-id vote
49
+ * count + the names of sources that produced no winning candidate. Non-fatal:
50
+ * any search failure degrades to the remaining sources (empty winners allowed).
51
+ */
52
+ export function voteRecall(opts, store) {
53
+ const logger = new Logger();
54
+ const limit = opts.limit ?? 3;
55
+ const recencyCount = opts.recencyCount ?? limit;
56
+ // ── Source A: vector (raw hits, cosine 0..1). ────────────────────────────
57
+ const vectorCands = recallRawHits({ sessionId: opts.sessionId, query: opts.query, limit }, store).map((h) => ({
58
+ checkpointId: h.checkpoint.checkpointId,
59
+ score: h.score,
60
+ source: "vector",
61
+ }));
62
+ // ── Source B: fts5 (BM25, hydrated to checkpointIds). ─────────────────────
63
+ // Dedup on L0 content digest so the SAME normalized text under two ids does
64
+ // not double-vote — collapse to one candidate per digest.
65
+ const fts5Cands = (() => {
66
+ try {
67
+ const reader = openStore(store.stateDir);
68
+ const hits = fts5SearchScoped(opts.query, reader, opts.sessionId, limit);
69
+ // FTS5 returns scores ordered best-first (bm25 asc); lower = better.
70
+ // We keep the raw score (negative-is-better) and flip in normalization.
71
+ const hydrated = hydrateFts5Hits(hits, opts.sessionId, store.stateDir);
72
+ // Dedup on checkpointId so the same checkpoint cannot double-count, AND
73
+ // on the L0 CONTENT digest so identical normalized text stored under two
74
+ // different ids collapses to one vote. The digest is taken over the
75
+ // joined `summary` (real content): hashing the id string would be a
76
+ // no-op tier, since ids are unique by definition.
77
+ const seenDigest = new Set();
78
+ const seenId = new Set();
79
+ const out = [];
80
+ for (const h of hydrated) {
81
+ if (seenId.has(h.checkpointId))
82
+ continue;
83
+ seenId.add(h.checkpointId);
84
+ const digest = computeContentDigest(h.summary).contentHash;
85
+ if (seenDigest.has(digest))
86
+ continue;
87
+ seenDigest.add(digest);
88
+ out.push({ checkpointId: h.checkpointId, score: h.score, source: "fts5" });
89
+ }
90
+ return out;
91
+ }
92
+ catch {
93
+ return [];
94
+ }
95
+ })();
96
+ // ── Source C: recency (N freshest checkpoints, query-independent). ────────
97
+ // Timestamp-ordered; the lower the index the fresher. Score = recency rank
98
+ // (fresh = high) so normalization treats newest as best.
99
+ const recencyCands = (() => {
100
+ try {
101
+ const cps = listCheckpoints(opts.sessionId, store.stateDir)
102
+ .filter((c) => c.dedupStatus !== "removed")
103
+ .sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
104
+ .slice(0, recencyCount);
105
+ return cps.map((cp, i) => ({
106
+ checkpointId: cp.checkpointId,
107
+ // Fresher => higher raw score (recency rank). Normalized below.
108
+ score: cps.length - i,
109
+ source: "recency",
110
+ }));
111
+ }
112
+ catch {
113
+ return [];
114
+ }
115
+ })();
116
+ const sources = [
117
+ { name: "vector", cands: vectorCands },
118
+ { name: "fts5", cands: fts5Cands },
119
+ { name: "recency", cands: recencyCands },
120
+ ];
121
+ // Per-source normalization to 0..1 so the three scales are comparable.
122
+ const perSource = sources.map((s) => ({
123
+ name: s.name,
124
+ norm: normalizeScores(s.cands.map((c) => c.score)),
125
+ }));
126
+ // Aggregate: best normalized score per source per checkpointId + vote count.
127
+ const bestScoreBySource = new Map();
128
+ const seenIds = new Set();
129
+ for (const src of sources) {
130
+ const norm = perSource.find((p) => p.name === src.name).norm;
131
+ const bestByCp = new Map();
132
+ src.cands.forEach((c, i) => {
133
+ const n = norm.get(i) ?? 0;
134
+ const prev = bestByCp.get(c.checkpointId);
135
+ if (prev === undefined || n > prev)
136
+ bestByCp.set(c.checkpointId, n);
137
+ seenIds.add(c.checkpointId);
138
+ });
139
+ bestScoreBySource.set(src.name, bestByCp);
140
+ }
141
+ // Vote count = number of DISTINCT sources naming each checkpointId.
142
+ const votes = {};
143
+ const sumByCp = new Map();
144
+ for (const id of seenIds) {
145
+ let count = 0;
146
+ let sum = 0;
147
+ for (const src of sources) {
148
+ const m = bestScoreBySource.get(src.name);
149
+ if (m.has(id)) {
150
+ count++;
151
+ sum += m.get(id);
152
+ }
153
+ }
154
+ votes[id] = count;
155
+ sumByCp.set(id, sum);
156
+ }
157
+ /** Mean normalized score across the sources that named `id`. */
158
+ const meanScore = (id) => (sumByCp.get(id) ?? 0) / (votes[id] ?? 1);
159
+ // Short-circuit: >=2 of 3 distinct sources => winner (agreement). Ranked by
160
+ // vote count first (stronger agreement wins), then by mean normalized score —
161
+ // the validator consumes this list in order and takes the first that passes,
162
+ // so the ordering IS the ranking and must not be Set-insertion order.
163
+ const winners = [];
164
+ const divergent = new Set(sources.map((s) => s.name));
165
+ const agreed = [...seenIds]
166
+ .filter((id) => (votes[id] ?? 0) >= 2)
167
+ .sort((a, b) => (votes[b] ?? 0) - (votes[a] ?? 0) || meanScore(b) - meanScore(a));
168
+ for (const id of agreed) {
169
+ const cand = (() => {
170
+ for (const src of sources) {
171
+ const c = src.cands.find((x) => x.checkpointId === id);
172
+ if (c)
173
+ return c;
174
+ }
175
+ return null;
176
+ })();
177
+ if (cand)
178
+ winners.push(cand);
179
+ }
180
+ if (winners.length === 0) {
181
+ // Fallback (no 2-of-3 agreement): rank by cross-source MEAN normalized score.
182
+ const ranked = [...seenIds].sort((a, b) => meanScore(b) - meanScore(a));
183
+ for (const id of ranked) {
184
+ const cand = (() => {
185
+ for (const src of sources) {
186
+ const c = src.cands.find((x) => x.checkpointId === id);
187
+ if (c)
188
+ return c;
189
+ }
190
+ return null;
191
+ })();
192
+ if (cand)
193
+ winners.push(cand);
194
+ }
195
+ }
196
+ // Divergence = a source that named NONE of the winning checkpoints. This must
197
+ // be computed per SOURCE against the winning ID SET, not from `winner.source`
198
+ // (a winner is one candidate object carrying a single source label, so an id
199
+ // agreed on by all three sources would still credit only one of them).
200
+ const winningIds = new Set(winners.map((w) => w.checkpointId));
201
+ for (const src of sources) {
202
+ if (src.cands.some((c) => winningIds.has(c.checkpointId))) {
203
+ divergent.delete(src.name);
204
+ }
205
+ }
206
+ if (divergent.size > 0) {
207
+ logger.info("recall_vote_divergence", {
208
+ divergentSources: [...divergent],
209
+ winnerCount: winners.length,
210
+ });
211
+ }
212
+ return {
213
+ winners,
214
+ votes,
215
+ divergentSources: [...divergent],
216
+ };
217
+ }
@@ -11,6 +11,7 @@
11
11
  * time automatically — no manual n-gram splitting needed. The BM25 rank from
12
12
  * fts5 ranks results by trigram-overlap density.
13
13
  */
14
+ import { listCheckpoints } from "./checkpoints.js";
14
15
  /**
15
16
  * Search the context_chunks_trgm FTS5 table for checkpoints whose
16
17
  * `normalized_text` matches `query` via trigram similarity.
@@ -66,3 +67,28 @@ export function fts5SearchScoped(query, reader, sessionId, limit = 10) {
66
67
  // No session filter — plain FTS5 search across all sessions.
67
68
  return fts5Search(query, reader, limit);
68
69
  }
70
+ /**
71
+ * Hydrate FTS5 hits (which only carry checkpoint `id` + BM25 `score`, with NO
72
+ * checkpoint object) by joining against the real stored checkpoints. Mirrors the
73
+ * private `hydrateHits` in tieredRouter.ts (listCheckpoints → Map on
74
+ * checkpointId → join, dropping ids with no checkpoint).
75
+ *
76
+ * The joined `summary` is returned alongside the id because the 3WF-3 voter
77
+ * dedups candidates on the L0 CONTENT digest — hashing the id string instead
78
+ * would be a no-op tier (ids are already unique), silently disabling the check.
79
+ *
80
+ * @param hits Raw Fts5Hit rows (id + BM25 score).
81
+ * @param sessionId Session scope used to fetch the checkpoint set.
82
+ * @param stateDir On-disk state dir for the sync store.
83
+ */
84
+ export function hydrateFts5Hits(hits, sessionId, stateDir) {
85
+ const cps = listCheckpoints(sessionId, stateDir);
86
+ const cpMap = new Map(cps.map((cp) => [cp.checkpointId, cp]));
87
+ const out = [];
88
+ for (const h of hits) {
89
+ const cp = cpMap.get(h.id);
90
+ if (cp)
91
+ out.push({ checkpointId: h.id, score: h.score, summary: cp.summary });
92
+ }
93
+ return out;
94
+ }
@@ -36,6 +36,38 @@ export function getMetaNumber(key, stateDir = getStateDir()) {
36
36
  const n = raw == null ? 0 : Number(raw);
37
37
  return Number.isFinite(n) ? n : 0;
38
38
  }
39
+ /**
40
+ * Upsert a single numeric meta key to an absolute value (NOT a cumulative
41
+ * counter). Follows the `addTokensSaved` INSERT-ON-CONFLICT pattern with a
42
+ * fully parameterized query (PREVENT-002: no SQL string concat — `key` and
43
+ * `value` are both bound, never interpolated). The only write is the standard
44
+ * ON CONFLICT upsert of THIS key's own value; no other key is touched, no
45
+ * DELETE is issued.
46
+ *
47
+ * Used by the 3WF-2 ThrashGuard to persist exactly two keys:
48
+ * - `thrasguard.baseline_tokens` — the live-window token count at the moment
49
+ * an ineffective compaction was observed (the baseline the guard re-arms from).
50
+ * - `thrasguard.blocked_until` — the live-window token count below which
51
+ * re-firing is refused (guard active).
52
+ *
53
+ * Non-finite input (NaN / ±Infinity) is rejected: the extension must never
54
+ * persist a non-number into the meta table (getMetaNumber would read it back as
55
+ * 0), so we return early, non-fatal. Best-effort: any store failure is swallowed.
56
+ */
57
+ export function setMetaNumber(key, value, stateDir = getStateDir()) {
58
+ if (key.length === 0)
59
+ return;
60
+ if (!Number.isFinite(value))
61
+ return;
62
+ try {
63
+ const db = openStore(stateDir);
64
+ db.prepare(`INSERT INTO meta(key, value) VALUES(?, ?)
65
+ ON CONFLICT(key) DO UPDATE SET value = ?`).run(key, String(value), String(value));
66
+ }
67
+ catch {
68
+ /* non-fatal: meta writes never break the agent loop */
69
+ }
70
+ }
39
71
  /** Atomically add `delta` to an integer meta counter. */
40
72
  function incMeta(key, delta, stateDir = getStateDir()) {
41
73
  if (!(delta > 0))
@@ -241,6 +241,7 @@ export const SETTINGS: ReadonlyArray<SettingGroup> = [
241
241
  num("MEGACOMPACT_L2_THRESHOLD", "L2 Cosine Threshold", "L2 semantic dedup firing point", 0.85, 0, 1),
242
242
  num("MEGACOMPACT_L1_JACCARD", "L1 Jaccard Threshold", "L1 MinHash near-dup threshold", 0.8, 0, 1),
243
243
  num("MEGACOMPACT_DEDUP_SIM", "Dedup Similarity", "Legacy content-similarity fallback", 0.9, 0, 1),
244
+ num("MEGACOMPACT_RECALL_MIN_COSINE", "Recall Min Cosine (same-repo)", "3WF-3 same-repo floor the 3-source validator applies to the top winner (cross-repo 0.90 stays separate)", 0.12, 0, 1),
244
245
  num("MEGACOMPACT_MMR_LAMBDA", "MMR Lambda", "Maximal Marginal Relevance diversity", 0.5, 0, 1),
245
246
  num("MEGACOMPACT_SEMDEDUP_COSINE", "SemDeDup Cosine", "Offline SemDeDup pair threshold", 0.95, 0, 1),
246
247
  num("MEGACOMPACT_CONSOLIDATE_COSINE", "Consolidate Cosine", "Memory consolidation merge threshold", 0.7, 0, 1),
@@ -285,6 +286,14 @@ export const SETTINGS: ReadonlyArray<SettingGroup> = [
285
286
  0.1,
286
287
  0.95,
287
288
  ),
289
+ num(
290
+ "MEGACOMPACT_THRASH_REARM_PCT",
291
+ "Thrash Re-arm %",
292
+ "After an ineffective compaction (live window did not shrink), refuse to re-fire until the live window grows by this fraction of the effective threshold. Default 0.10 (10%)",
293
+ 0.1,
294
+ 0.01,
295
+ 0.5,
296
+ ),
288
297
  ],
289
298
  },
290
299
  VECTOR_CORTEX_SETTINGS,
@@ -119,6 +119,11 @@ export interface MegaConfig {
119
119
  * tighter than same-repo so only genuinely-relevant cross-repo context is
120
120
  * injected. */
121
121
  crossRepoCosine: number;
122
+ /** Same-repo recall cosine floor (3WF-3). Default 0.12. SEPARATE from
123
+ * `crossRepoCosine` (S17, default 0.90 — stricter, cross-repo only). The
124
+ * 3-source validator applies this to the top winner; hits below it are
125
+ * rejected in favor of the next-ranked candidate or the provenance floor. */
126
+ recallMinCosine: number;
122
127
  /** Memory-RAG auto-review enabled (S20). Every memoryReviewInterval turns the
123
128
  * conversation is auto-reviewed into durable add/replace/remove memories. */
124
129
  memoryAutoReview: boolean;
@@ -140,6 +145,14 @@ export interface MegaConfig {
140
145
  * session_start never fired, so every session has a staged block (recall hits,
141
146
  * else a provenance floor). Default ON; OFF = byte-identical pre-sprint. */
142
147
  threeWayFailback: boolean;
148
+ /** 3WF-2: ThrashGuard re-arm budget as a FRACTION of `effectiveThreshold`.
149
+ * After an ineffective compaction (live window did not shrink), the guard
150
+ * refuses to re-fire until the live window has grown by at least
151
+ * `rearmPct × effectiveThreshold` tokens past the observed baseline. Default
152
+ * 0.10 (10% of the effective threshold). Env-overridable via
153
+ * MEGACOMPACT_THRASH_REARM_PCT. When the effective threshold is unknown
154
+ * (+Infinity), the guard skips arming (cannot compute N) and logs instead. */
155
+ thrashRearmPct: number;
143
156
  /** A1 PLAN_V2 Phase 2: Message Separation — isolate user/assistant turns
144
157
  * from volatile tool results so the prompt-cache prefix stays stable.
145
158
  * PC-A: positive sprint flag, now default ON; flag-OFF (=0) is byte-identical
@@ -217,6 +217,12 @@ export function loadConfig(): MegaConfig {
217
217
  autoWikiEnabled: envBool("MEGACOMPACT_AUTO_WIKI", true),
218
218
  crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
219
219
  crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
220
+ // 3WF-3: SAME-repo recall cosine floor applied by the 3-source validator to
221
+ // the top winner. SEPARATE from crossRepoCosine (S17, default 0.90, stricter
222
+ // and cross-repo only). This same-repo floor is permissive by default (0.12)
223
+ // so recall still surfaces loosely-relevant within-repo context while
224
+ // rejecting effectively-unrelated hits. Mirrors src/config.ts RECALL_MIN_COSINE.
225
+ recallMinCosine: Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12"),
220
226
  memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
221
227
  memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
222
228
  recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
@@ -225,6 +231,12 @@ export function loadConfig(): MegaConfig {
225
231
  // 3WF-1: TriggerGuard — guarantee a staged recall block on every context
226
232
  // event even when session_start never fires. Default ON; OFF = byte-identical.
227
233
  threeWayFailback: envBool("MEGACOMPACT_THREE_WAY_FAILBACK", true),
234
+ // 3WF-2: ThrashGuard re-arm budget as a fraction of effectiveThreshold.
235
+ // 0.10 default (10% of the effective threshold) — see mega-config-types.
236
+ // Clamped to [0.01, 0.5]: below 1% the guard is almost never armed (any
237
+ // growth re-fires, defeating the anti-thrash purpose); above 50% it would
238
+ // suppress legitimate re-fires for half the window. Env-overridable.
239
+ thrashRearmPct: clamp(envFlag("MEGACOMPACT_THRASH_REARM_PCT", 0.1), 0.01, 0.5),
228
240
  // PC-A: positive sprint flag, default ON. =0 byte-identical to the
229
241
  // pre-change OFF state (single gate lives at the call site in tailResult.ts).
230
242
  messageSeparation: envBool("MEGACOMPACT_MESSAGE_SEPARATION", true),
@@ -17,6 +17,7 @@ import {
17
17
  import { autoCompactCheck } from "../../../src/compact.js";
18
18
  import type { MegaRuntime } from "../../mega-runtime.js";
19
19
  import type { MegaConfig } from "../../mega-config.js";
20
+ import { isThrashBlockedFor } from "./thrashGuard.js";
20
21
 
21
22
  /** Tail-injection closure shape produced by buildTailResult (tailResult.ts). */
22
23
  export type TailResultFn = (
@@ -31,6 +32,35 @@ export type GateOutcome =
31
32
  perModelThreshold: { safetyMarginPct: number; firePointPct: number };
32
33
  };
33
34
 
35
+ /**
36
+ * 3WF-2 ThrashGuard consult — refuse to fire a NEW compaction while the guard
37
+ * is armed. After an ineffective compaction (the live window did not shrink),
38
+ * `thrasguard.blocked_until` holds the live-token count the window must exceed
39
+ * before re-firing is allowed.
40
+ *
41
+ * WHY THIS IS NOT INSIDE `evaluateGate`: the fast gate runs BEFORE the cached
42
+ * replay path in context-handler.ts, and REPLAY MUST STAY EXEMPT. A replay is
43
+ * free (no compute, no new checkpoint) and re-stabilises the provider KV-cache
44
+ * prefix — suppressing it would cause the very cache invalidation the D.2/D.3
45
+ * replay design exists to prevent. The guard's job is to stop wasted NEW
46
+ * compaction work, not to withhold an already-computed view. So the consult is
47
+ * called from the handler AFTER the replay block and BEFORE the debounce +
48
+ * `invokePipeline` (the actual fire point), covering the percent branch and the
49
+ * token branch alike since both converge there.
50
+ *
51
+ * Umbrella OFF ⇒ always false (byte-identical to v0.20.83). Non-fatal: a store
52
+ * read error returns false — never refuse compaction on a store fault.
53
+ */
54
+ export function thrashGuardBlocks(
55
+ runtime: MegaRuntime,
56
+ config: MegaConfig,
57
+ currentTokens: number | null | undefined,
58
+ ): boolean {
59
+ if (!config.threeWayFailback) return false;
60
+ if (currentTokens == null) return false;
61
+ return isThrashBlockedFor(runtime, currentTokens, runtime.currentStateDir);
62
+ }
63
+
34
64
  /**
35
65
  * Evaluate whether the current context warrants compaction. Returns a tailed
36
66
  * view ("return") when the gate does not pass, or "proceed" with the resolved
@@ -0,0 +1,228 @@
1
+ /**
2
+ * context-handler/thrashGuard.ts — 3WF-2 ReductionValidator + ThrashGuard.
3
+ *
4
+ * Production bug fixed here: compaction fired 496× freeing 0.0% of the live
5
+ * window. Root cause — correctness was judged by the STORED `saved` metric
6
+ * (a cumulative SQLite total that the dedup made look healthy every fire)
7
+ * while the LIVE context window (`currentTokens`) never shrank. This module
8
+ * judges correctness by the LIVE-WINDOW delta across consecutive `context`
9
+ * events, and after an ineffective compaction persists a meta-backed refusal
10
+ * so the guard will not re-fire until the window has grown meaningfully again.
11
+ *
12
+ * Everything is gated on the umbrella `config.threeWayFailback`
13
+ * (MEGACOMPACT_THREE_WAY_FAILBACK, default ON). Flag OFF ⇒ every entry point
14
+ * is an immediate no-op, so gateCheck + compactSession behave byte-identically
15
+ * to v0.20.83.
16
+ *
17
+ * Non-fatal EVERYWHERE: every store read/write is best-effort, swallowed on
18
+ * failure. Structured JSON logging only (runtime.logger.info with ts + event).
19
+ * No console.*, no network, no mocks.
20
+ */
21
+ import type { MegaRuntime } from "../../mega-runtime.js";
22
+ import type { MegaConfig } from "../../mega-config.js";
23
+ import { getMetaNumber, setMetaNumber } from "../../../src/store/sqlite.js";
24
+ import type { ReductionVerdict } from "../../../src/failback/types.js";
25
+
26
+ /** Meta key holding the live-window baseline (tokens) at the ineffective fire. */
27
+ export const THRASH_BASELINE_KEY = "thrasguard.baseline_tokens";
28
+ /** Meta key holding the live-window token count below which re-firing is blocked. */
29
+ export const THRASH_BLOCKED_KEY = "thrasguard.blocked_until";
30
+
31
+ /**
32
+ * "Meaningful reduction" floor as a FRACTION of `liveBefore`. A compaction is
33
+ * only credited with freeing space when the live window shrank by at least this
34
+ * fraction of its pre-compaction size.
35
+ *
36
+ * Rationale (invented constant, calibrated + configurable-by-design): the model
37
+ * re-reports token counts on every context event with noise on the order of a
38
+ * percent or two, so a sub-1% wobble is not a real reduction — crediting it
39
+ * would suppress the guard on a genuine no-op fire (the exact bug we are
40
+ * fixing). 2% is a defensible "real shrink" threshold: it is well above typical
41
+ * re-estimation noise but low enough that a compaction that freed even a few
42
+ * percent of the window is not punished. A reduction of ≤0 tokens is
43
+ * unconditionally ineffective regardless of this floor.
44
+ */
45
+ const MEANINGFUL_REDUCTION_PCT = 0.02;
46
+
47
+ /** Pure reduction verdict for a live-window bracketing pair. */
48
+ export const ReductionValidator = {
49
+ /**
50
+ * Judge whether the LIVE window actually shrank between two consecutive
51
+ * context events bracketing a compaction.
52
+ * - a reduction of ≤0 tokens ⇒ definitively ineffective.
53
+ * - otherwise effective only when the reduction is ≥ the small positive
54
+ * floor (MEANINGFUL_REDUCTION_PCT of liveBefore), so estimation noise on
55
+ * the model's re-reported token count is not mistaken for a real shrink.
56
+ */
57
+ validateReduction(liveBefore: number, liveAfter: number): ReductionVerdict {
58
+ const reduction = liveBefore - liveAfter;
59
+ const floor = Math.max(1, Math.round(MEANINGFUL_REDUCTION_PCT * liveBefore));
60
+ const effective = Number.isFinite(reduction) && reduction > 0 && reduction >= floor;
61
+ return { effective, liveBefore, liveAfter };
62
+ },
63
+ };
64
+
65
+ /**
66
+ * Arm the ThrashGuard after an ineffective compaction. Persists:
67
+ * - `thrasguard.baseline_tokens` = the live currentTokens at this (post-fire)
68
+ * context event, so re-arm is measured from the window that failed to shrink.
69
+ * - `thrasguard.blocked_until` = baseline + N, where N = `rearmPct ×
70
+ * effectiveThreshold`. Re-firing is refused until the live window grows past
71
+ * `blocked_until`.
72
+ *
73
+ * If `effectiveThreshold` is non-finite (+Infinity — the 3WF-2 invariant when
74
+ * the model window is unknown), N cannot be computed; we MUST NOT persist
75
+ * Infinity/NaN into meta (getMetaNumber would read it back as 0). Skip arming
76
+ * + log instead; the next over-threshold event simply re-fires (pre-sprint
77
+ * behavior) rather than corrupting the guard.
78
+ */
79
+ export function armThrashGuard(
80
+ currentTokens: number,
81
+ rearmPct: number,
82
+ effectiveThreshold: number,
83
+ stateDir: string,
84
+ logger?: { info(event: string, fields?: Record<string, unknown>): void },
85
+ ): void {
86
+ if (!Number.isFinite(currentTokens) || currentTokens <= 0) return;
87
+ if (!Number.isFinite(rearmPct) || rearmPct <= 0) return;
88
+ if (!Number.isFinite(effectiveThreshold)) {
89
+ logger?.info("thrasguard_skip_arm", {
90
+ reason: "nonfinite_effective_threshold",
91
+ currentTokens,
92
+ });
93
+ return;
94
+ }
95
+ try {
96
+ const n = Math.round(rearmPct * effectiveThreshold);
97
+ setMetaNumber(THRASH_BASELINE_KEY, Math.round(currentTokens), stateDir);
98
+ setMetaNumber(THRASH_BLOCKED_KEY, Math.round(currentTokens + n), stateDir);
99
+ logger?.info("thrasguard_armed", {
100
+ baselineTokens: Math.round(currentTokens),
101
+ blockedUntilTokens: Math.round(currentTokens + n),
102
+ rearmTokens: n,
103
+ });
104
+ } catch {
105
+ /* non-fatal: best-effort meta write */
106
+ }
107
+ }
108
+
109
+ /**
110
+ * Consult the ThrashGuard for the current live-window token count. Returns true
111
+ * when compaction must be refused (the window is still below the armed
112
+ * `blocked_until`). A `blocked_until` of 0/absent ⇒ never blocked. When the
113
+ * live tokens have grown past `blocked_until`, the guard no longer blocks
114
+ * (caller re-fires normally). Pure read, best-effort — on any failure returns
115
+ * false (do not refuse compaction on a store error).
116
+ */
117
+ export function isThrashBlocked(currentTokens: number, stateDir: string): boolean {
118
+ try {
119
+ const blockedUntil = getMetaNumber(THRASH_BLOCKED_KEY, stateDir);
120
+ if (blockedUntil <= 0) return false;
121
+ return Number.isFinite(currentTokens) && currentTokens < blockedUntil;
122
+ } catch {
123
+ return false; // non-fatal: never refuse compaction on a read error
124
+ }
125
+ }
126
+
127
+ /**
128
+ * Per-runtime one-shot session state for the live-window delta correlation.
129
+ * Like triggerGuard.ts, keyed by runtime in a WeakMap so it dies with the
130
+ * runtime and a test can pass a thin stub. Holds the live token count observed
131
+ * at the event that FIRED a compaction; consumed on the following context event
132
+ * to judge whether the window actually shrank.
133
+ */
134
+ const sessionBefore = new WeakMap<MegaRuntime, { liveBefore: number }>();
135
+
136
+ /**
137
+ * The live-token count of the event that most recently ARMED the guard, per
138
+ * runtime. The arming happens early in a context event (the live-delta consume
139
+ * point), but the guard consult runs LATER IN THAT SAME EVENT — and since
140
+ * `blocked_until = currentTokens + N`, a naive consult would always find
141
+ * `currentTokens < blocked_until` and swallow the very event that armed it.
142
+ * That is an off-by-one-event error: the guard's contract is to refuse
143
+ * SUBSEQUENT re-fires, not to cancel the compaction that revealed the problem.
144
+ * Recording the arming event's token count lets the consult skip exactly that
145
+ * one event. Cleared once the window grows past it.
146
+ */
147
+ const armedOnEvent = new WeakMap<MegaRuntime, number>();
148
+
149
+ /**
150
+ * Runtime-aware ThrashGuard consult: true when a NEW compaction must be refused.
151
+ *
152
+ * Reads the persisted `thrasguard.blocked_until` (see `isThrashBlocked`) but
153
+ * EXEMPTS the single event that armed the guard — otherwise, because arming sets
154
+ * `blocked_until = currentTokens + N` earlier in the very same context event, the
155
+ * consult would always fire and cancel the compaction that exposed the thrash.
156
+ * The guard exists to refuse SUBSEQUENT re-fires. Once the live window grows past
157
+ * the armed count the exemption is dropped, and normal blocking resumes until the
158
+ * window clears `blocked_until`.
159
+ *
160
+ * Best-effort: any failure returns false (never refuse on a store fault).
161
+ */
162
+ export function isThrashBlockedFor(
163
+ runtime: MegaRuntime,
164
+ currentTokens: number,
165
+ stateDir: string,
166
+ ): boolean {
167
+ try {
168
+ if (!isThrashBlocked(currentTokens, stateDir)) return false;
169
+ const armedAt = armedOnEvent.get(runtime);
170
+ if (armedAt !== undefined && currentTokens === armedAt) {
171
+ // EXACTLY the event that armed the guard (same live-token reading): let it
172
+ // through once, then block normally from the next event onward. An exact
173
+ // match (not <=) is required so a genuinely lower or different live reading
174
+ // on a later event is still blocked.
175
+ armedOnEvent.delete(runtime);
176
+ return false;
177
+ }
178
+ return true;
179
+ } catch {
180
+ return false;
181
+ }
182
+ }
183
+
184
+ /** Record that a compaction fired at `liveBefore` tokens (call on the firing event). */
185
+ export function markCompactionFired(runtime: MegaRuntime, liveBefore: number): void {
186
+ try {
187
+ if (Number.isFinite(liveBefore) && liveBefore > 0) {
188
+ sessionBefore.set(runtime, { liveBefore });
189
+ }
190
+ } catch {
191
+ /* non-fatal */
192
+ }
193
+ }
194
+
195
+ /**
196
+ * Consume a pending live-window delta on a subsequent context event. If a
197
+ * compaction fired on a prior event, compare THIS event's live tokens against
198
+ * that pre-fire baseline; an ineffective reduction arms the guard. The pending
199
+ * marker is consumed exactly once (cleared before any re-arm). No-op when no
200
+ * compaction is pending, when the umbrella flag is OFF, or on any error.
201
+ */
202
+ export function evaluatePendingReduction(
203
+ runtime: MegaRuntime,
204
+ currentTokens: number,
205
+ config: MegaConfig,
206
+ ): void {
207
+ if (!config.threeWayFailback) return;
208
+ const pending = sessionBefore.get(runtime);
209
+ if (pending == null) return;
210
+ try {
211
+ sessionBefore.delete(runtime); // consume once, regardless of verdict
212
+ const verdict = ReductionValidator.validateReduction(pending.liveBefore, currentTokens);
213
+ if (!verdict.effective) {
214
+ // Remember which event armed us so the consult later in THIS SAME event
215
+ // does not swallow it (see armedOnEvent).
216
+ armedOnEvent.set(runtime, currentTokens);
217
+ armThrashGuard(
218
+ currentTokens,
219
+ config.thrashRearmPct,
220
+ runtime.effectiveThreshold,
221
+ runtime.currentStateDir,
222
+ runtime.logger,
223
+ );
224
+ }
225
+ } catch {
226
+ /* non-fatal */
227
+ }
228
+ }