pi-mega-compact 0.20.85 → 0.20.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +9 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +2 -0
- package/dist/extensions/mega-config.js +12 -0
- package/dist/extensions/mega-events/context-handler/gateCheck.js +27 -0
- package/dist/extensions/mega-events/context-handler/thrashGuard.js +186 -0
- package/dist/extensions/mega-events/context-handler.js +33 -1
- package/dist/extensions/mega-pipeline/compact/noop.js +104 -0
- package/dist/extensions/mega-pipeline/compact/run.js +268 -0
- package/dist/extensions/mega-pipeline/compact/vote.js +72 -0
- package/dist/extensions/mega-pipeline/compact.js +12 -343
- package/dist/extensions/mega-pipeline/recall/impl.js +258 -0
- package/dist/extensions/mega-pipeline/recall.js +6 -253
- package/dist/src/config.js +9 -0
- package/dist/src/failback/compact.js +109 -0
- package/dist/src/recall/readonly.js +39 -0
- package/dist/src/recall/recall3wf.fixture.js +67 -0
- package/dist/src/recall/validator.js +113 -0
- package/dist/src/recall/vote.js +217 -0
- package/dist/src/store/sqlite/fts5-search.js +26 -0
- package/dist/src/store/sqlite/meta.js +32 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +9 -0
- package/extensions/mega-config-types.ts +13 -0
- package/extensions/mega-config.ts +12 -0
- package/extensions/mega-events/context-handler/gateCheck.ts +30 -0
- package/extensions/mega-events/context-handler/thrashGuard.ts +228 -0
- package/extensions/mega-events/context-handler.ts +36 -1
- package/extensions/mega-pipeline/compact/noop.ts +96 -0
- package/extensions/mega-pipeline/compact/run.ts +322 -0
- package/extensions/mega-pipeline/compact/vote.ts +85 -0
- package/extensions/mega-pipeline/compact.ts +12 -385
- package/extensions/mega-pipeline/recall/impl.ts +312 -0
- package/extensions/mega-pipeline/recall.ts +10 -306
- package/package.json +1 -1
- package/src/config.ts +12 -0
- package/src/failback/compact.ts +122 -0
- package/src/failback/types.ts +72 -0
- package/src/recall/readonly.ts +57 -0
- package/src/recall/recall3wf.fixture.ts +87 -0
- package/src/recall/validator.ts +150 -0
- package/src/recall/vote.ts +240 -0
- package/src/store/sqlite/fts5-search.ts +40 -0
- package/src/store/sqlite/meta.ts +36 -0
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recall/vote.ts — the 3-independent-source recall vote (3WF-3).
|
|
3
|
+
*
|
|
4
|
+
* Three INDEPENDENT, read-only sources name candidate checkpoints:
|
|
5
|
+
* A vector — raw semantic hits (recall/readonly.ts), cosine 0..1 scale.
|
|
6
|
+
* B fts5 — BM25 trigram hits (hydrated to checkpointIds), FTS5 BM25 scale
|
|
7
|
+
* (negative score = better match; ranking order is what matters).
|
|
8
|
+
* C recency — the N freshest checkpoints by timestamp, query-INDEPENDENT.
|
|
9
|
+
* (NOT turn_recall / TurnReader — that would just echo already-
|
|
10
|
+
* injected content, not an independent signal.)
|
|
11
|
+
*
|
|
12
|
+
* Each source is a different score scale, so they are NOT directly comparable.
|
|
13
|
+
* We normalize each source's scores to a 0..1 scale (per-source min-max) BEFORE
|
|
14
|
+
* combining. Averaging raw cosine (0..1) with raw BM25 (arbitrary negative
|
|
15
|
+
* magnitude) or a recency rank would be meaningless — the largest-magnitude
|
|
16
|
+
* scale would always dominate. Normalization makes each source a peer voter.
|
|
17
|
+
*
|
|
18
|
+
* Overlap rule: a checkpoint named by >=2 of 3 distinct sources short-circuits
|
|
19
|
+
* as a winner. Fallback (no 2/3 majority): rank all candidates by the
|
|
20
|
+
* cross-source MEAN of their normalized scores.
|
|
21
|
+
*
|
|
22
|
+
* Non-fatal throughout. Pi-agnostic: no pi runtime imports.
|
|
23
|
+
*/
|
|
24
|
+
import { openStore } from "../store/sqlite/utils.js";
|
|
25
|
+
import { fts5SearchScoped, hydrateFts5Hits } from "../store/sqlite/fts5-search.js";
|
|
26
|
+
// The SQLite store is the source of truth (src/store.ts's same-named helper
|
|
27
|
+
// reads the LEGACY gzipped-JSON DR snapshot, which is empty for live sessions —
|
|
28
|
+
// importing it here would silently starve sources B and C). Mirrors the import
|
|
29
|
+
// in vector-search.ts + tieredRouter.ts.
|
|
30
|
+
import { listCheckpoints } from "../store/sqlite.js";
|
|
31
|
+
import { computeContentDigest } from "../dedup/digest.js";
|
|
32
|
+
import { recallRawHits } from "./readonly.js";
|
|
33
|
+
import { Logger } from "../log.js";
|
|
34
|
+
/** Per-source normalization: map raw scores to 0..1 via min-max within source. */
|
|
35
|
+
function normalizeScores(scores) {
|
|
36
|
+
const map = new Map();
|
|
37
|
+
if (scores.length === 0)
|
|
38
|
+
return map;
|
|
39
|
+
const min = Math.min(...scores);
|
|
40
|
+
const max = Math.max(...scores);
|
|
41
|
+
const span = max - min;
|
|
42
|
+
scores.forEach((s, i) => {
|
|
43
|
+
map.set(i, span === 0 ? 1 : (s - min) / span);
|
|
44
|
+
});
|
|
45
|
+
return map;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Run the three-source recall vote. Returns agreement winners + a per-id vote
|
|
49
|
+
* count + the names of sources that produced no winning candidate. Non-fatal:
|
|
50
|
+
* any search failure degrades to the remaining sources (empty winners allowed).
|
|
51
|
+
*/
|
|
52
|
+
export function voteRecall(opts, store) {
|
|
53
|
+
const logger = new Logger();
|
|
54
|
+
const limit = opts.limit ?? 3;
|
|
55
|
+
const recencyCount = opts.recencyCount ?? limit;
|
|
56
|
+
// ── Source A: vector (raw hits, cosine 0..1). ────────────────────────────
|
|
57
|
+
const vectorCands = recallRawHits({ sessionId: opts.sessionId, query: opts.query, limit }, store).map((h) => ({
|
|
58
|
+
checkpointId: h.checkpoint.checkpointId,
|
|
59
|
+
score: h.score,
|
|
60
|
+
source: "vector",
|
|
61
|
+
}));
|
|
62
|
+
// ── Source B: fts5 (BM25, hydrated to checkpointIds). ─────────────────────
|
|
63
|
+
// Dedup on L0 content digest so the SAME normalized text under two ids does
|
|
64
|
+
// not double-vote — collapse to one candidate per digest.
|
|
65
|
+
const fts5Cands = (() => {
|
|
66
|
+
try {
|
|
67
|
+
const reader = openStore(store.stateDir);
|
|
68
|
+
const hits = fts5SearchScoped(opts.query, reader, opts.sessionId, limit);
|
|
69
|
+
// FTS5 returns scores ordered best-first (bm25 asc); lower = better.
|
|
70
|
+
// We keep the raw score (negative-is-better) and flip in normalization.
|
|
71
|
+
const hydrated = hydrateFts5Hits(hits, opts.sessionId, store.stateDir);
|
|
72
|
+
// Dedup on checkpointId so the same checkpoint cannot double-count, AND
|
|
73
|
+
// on the L0 CONTENT digest so identical normalized text stored under two
|
|
74
|
+
// different ids collapses to one vote. The digest is taken over the
|
|
75
|
+
// joined `summary` (real content): hashing the id string would be a
|
|
76
|
+
// no-op tier, since ids are unique by definition.
|
|
77
|
+
const seenDigest = new Set();
|
|
78
|
+
const seenId = new Set();
|
|
79
|
+
const out = [];
|
|
80
|
+
for (const h of hydrated) {
|
|
81
|
+
if (seenId.has(h.checkpointId))
|
|
82
|
+
continue;
|
|
83
|
+
seenId.add(h.checkpointId);
|
|
84
|
+
const digest = computeContentDigest(h.summary).contentHash;
|
|
85
|
+
if (seenDigest.has(digest))
|
|
86
|
+
continue;
|
|
87
|
+
seenDigest.add(digest);
|
|
88
|
+
out.push({ checkpointId: h.checkpointId, score: h.score, source: "fts5" });
|
|
89
|
+
}
|
|
90
|
+
return out;
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return [];
|
|
94
|
+
}
|
|
95
|
+
})();
|
|
96
|
+
// ── Source C: recency (N freshest checkpoints, query-independent). ────────
|
|
97
|
+
// Timestamp-ordered; the lower the index the fresher. Score = recency rank
|
|
98
|
+
// (fresh = high) so normalization treats newest as best.
|
|
99
|
+
const recencyCands = (() => {
|
|
100
|
+
try {
|
|
101
|
+
const cps = listCheckpoints(opts.sessionId, store.stateDir)
|
|
102
|
+
.filter((c) => c.dedupStatus !== "removed")
|
|
103
|
+
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
|
|
104
|
+
.slice(0, recencyCount);
|
|
105
|
+
return cps.map((cp, i) => ({
|
|
106
|
+
checkpointId: cp.checkpointId,
|
|
107
|
+
// Fresher => higher raw score (recency rank). Normalized below.
|
|
108
|
+
score: cps.length - i,
|
|
109
|
+
source: "recency",
|
|
110
|
+
}));
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return [];
|
|
114
|
+
}
|
|
115
|
+
})();
|
|
116
|
+
const sources = [
|
|
117
|
+
{ name: "vector", cands: vectorCands },
|
|
118
|
+
{ name: "fts5", cands: fts5Cands },
|
|
119
|
+
{ name: "recency", cands: recencyCands },
|
|
120
|
+
];
|
|
121
|
+
// Per-source normalization to 0..1 so the three scales are comparable.
|
|
122
|
+
const perSource = sources.map((s) => ({
|
|
123
|
+
name: s.name,
|
|
124
|
+
norm: normalizeScores(s.cands.map((c) => c.score)),
|
|
125
|
+
}));
|
|
126
|
+
// Aggregate: best normalized score per source per checkpointId + vote count.
|
|
127
|
+
const bestScoreBySource = new Map();
|
|
128
|
+
const seenIds = new Set();
|
|
129
|
+
for (const src of sources) {
|
|
130
|
+
const norm = perSource.find((p) => p.name === src.name).norm;
|
|
131
|
+
const bestByCp = new Map();
|
|
132
|
+
src.cands.forEach((c, i) => {
|
|
133
|
+
const n = norm.get(i) ?? 0;
|
|
134
|
+
const prev = bestByCp.get(c.checkpointId);
|
|
135
|
+
if (prev === undefined || n > prev)
|
|
136
|
+
bestByCp.set(c.checkpointId, n);
|
|
137
|
+
seenIds.add(c.checkpointId);
|
|
138
|
+
});
|
|
139
|
+
bestScoreBySource.set(src.name, bestByCp);
|
|
140
|
+
}
|
|
141
|
+
// Vote count = number of DISTINCT sources naming each checkpointId.
|
|
142
|
+
const votes = {};
|
|
143
|
+
const sumByCp = new Map();
|
|
144
|
+
for (const id of seenIds) {
|
|
145
|
+
let count = 0;
|
|
146
|
+
let sum = 0;
|
|
147
|
+
for (const src of sources) {
|
|
148
|
+
const m = bestScoreBySource.get(src.name);
|
|
149
|
+
if (m.has(id)) {
|
|
150
|
+
count++;
|
|
151
|
+
sum += m.get(id);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
votes[id] = count;
|
|
155
|
+
sumByCp.set(id, sum);
|
|
156
|
+
}
|
|
157
|
+
/** Mean normalized score across the sources that named `id`. */
|
|
158
|
+
const meanScore = (id) => (sumByCp.get(id) ?? 0) / (votes[id] ?? 1);
|
|
159
|
+
// Short-circuit: >=2 of 3 distinct sources => winner (agreement). Ranked by
|
|
160
|
+
// vote count first (stronger agreement wins), then by mean normalized score —
|
|
161
|
+
// the validator consumes this list in order and takes the first that passes,
|
|
162
|
+
// so the ordering IS the ranking and must not be Set-insertion order.
|
|
163
|
+
const winners = [];
|
|
164
|
+
const divergent = new Set(sources.map((s) => s.name));
|
|
165
|
+
const agreed = [...seenIds]
|
|
166
|
+
.filter((id) => (votes[id] ?? 0) >= 2)
|
|
167
|
+
.sort((a, b) => (votes[b] ?? 0) - (votes[a] ?? 0) || meanScore(b) - meanScore(a));
|
|
168
|
+
for (const id of agreed) {
|
|
169
|
+
const cand = (() => {
|
|
170
|
+
for (const src of sources) {
|
|
171
|
+
const c = src.cands.find((x) => x.checkpointId === id);
|
|
172
|
+
if (c)
|
|
173
|
+
return c;
|
|
174
|
+
}
|
|
175
|
+
return null;
|
|
176
|
+
})();
|
|
177
|
+
if (cand)
|
|
178
|
+
winners.push(cand);
|
|
179
|
+
}
|
|
180
|
+
if (winners.length === 0) {
|
|
181
|
+
// Fallback (no 2-of-3 agreement): rank by cross-source MEAN normalized score.
|
|
182
|
+
const ranked = [...seenIds].sort((a, b) => meanScore(b) - meanScore(a));
|
|
183
|
+
for (const id of ranked) {
|
|
184
|
+
const cand = (() => {
|
|
185
|
+
for (const src of sources) {
|
|
186
|
+
const c = src.cands.find((x) => x.checkpointId === id);
|
|
187
|
+
if (c)
|
|
188
|
+
return c;
|
|
189
|
+
}
|
|
190
|
+
return null;
|
|
191
|
+
})();
|
|
192
|
+
if (cand)
|
|
193
|
+
winners.push(cand);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// Divergence = a source that named NONE of the winning checkpoints. This must
|
|
197
|
+
// be computed per SOURCE against the winning ID SET, not from `winner.source`
|
|
198
|
+
// (a winner is one candidate object carrying a single source label, so an id
|
|
199
|
+
// agreed on by all three sources would still credit only one of them).
|
|
200
|
+
const winningIds = new Set(winners.map((w) => w.checkpointId));
|
|
201
|
+
for (const src of sources) {
|
|
202
|
+
if (src.cands.some((c) => winningIds.has(c.checkpointId))) {
|
|
203
|
+
divergent.delete(src.name);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
if (divergent.size > 0) {
|
|
207
|
+
logger.info("recall_vote_divergence", {
|
|
208
|
+
divergentSources: [...divergent],
|
|
209
|
+
winnerCount: winners.length,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
winners,
|
|
214
|
+
votes,
|
|
215
|
+
divergentSources: [...divergent],
|
|
216
|
+
};
|
|
217
|
+
}
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* time automatically — no manual n-gram splitting needed. The BM25 rank from
|
|
12
12
|
* fts5 ranks results by trigram-overlap density.
|
|
13
13
|
*/
|
|
14
|
+
import { listCheckpoints } from "./checkpoints.js";
|
|
14
15
|
/**
|
|
15
16
|
* Search the context_chunks_trgm FTS5 table for checkpoints whose
|
|
16
17
|
* `normalized_text` matches `query` via trigram similarity.
|
|
@@ -66,3 +67,28 @@ export function fts5SearchScoped(query, reader, sessionId, limit = 10) {
|
|
|
66
67
|
// No session filter — plain FTS5 search across all sessions.
|
|
67
68
|
return fts5Search(query, reader, limit);
|
|
68
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* Hydrate FTS5 hits (which only carry checkpoint `id` + BM25 `score`, with NO
|
|
72
|
+
* checkpoint object) by joining against the real stored checkpoints. Mirrors the
|
|
73
|
+
* private `hydrateHits` in tieredRouter.ts (listCheckpoints → Map on
|
|
74
|
+
* checkpointId → join, dropping ids with no checkpoint).
|
|
75
|
+
*
|
|
76
|
+
* The joined `summary` is returned alongside the id because the 3WF-3 voter
|
|
77
|
+
* dedups candidates on the L0 CONTENT digest — hashing the id string instead
|
|
78
|
+
* would be a no-op tier (ids are already unique), silently disabling the check.
|
|
79
|
+
*
|
|
80
|
+
* @param hits Raw Fts5Hit rows (id + BM25 score).
|
|
81
|
+
* @param sessionId Session scope used to fetch the checkpoint set.
|
|
82
|
+
* @param stateDir On-disk state dir for the sync store.
|
|
83
|
+
*/
|
|
84
|
+
export function hydrateFts5Hits(hits, sessionId, stateDir) {
|
|
85
|
+
const cps = listCheckpoints(sessionId, stateDir);
|
|
86
|
+
const cpMap = new Map(cps.map((cp) => [cp.checkpointId, cp]));
|
|
87
|
+
const out = [];
|
|
88
|
+
for (const h of hits) {
|
|
89
|
+
const cp = cpMap.get(h.id);
|
|
90
|
+
if (cp)
|
|
91
|
+
out.push({ checkpointId: h.id, score: h.score, summary: cp.summary });
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
@@ -36,6 +36,38 @@ export function getMetaNumber(key, stateDir = getStateDir()) {
|
|
|
36
36
|
const n = raw == null ? 0 : Number(raw);
|
|
37
37
|
return Number.isFinite(n) ? n : 0;
|
|
38
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* Upsert a single numeric meta key to an absolute value (NOT a cumulative
|
|
41
|
+
* counter). Follows the `addTokensSaved` INSERT-ON-CONFLICT pattern with a
|
|
42
|
+
* fully parameterized query (PREVENT-002: no SQL string concat — `key` and
|
|
43
|
+
* `value` are both bound, never interpolated). The only write is the standard
|
|
44
|
+
* ON CONFLICT upsert of THIS key's own value; no other key is touched, no
|
|
45
|
+
* DELETE is issued.
|
|
46
|
+
*
|
|
47
|
+
* Used by the 3WF-2 ThrashGuard to persist exactly two keys:
|
|
48
|
+
* - `thrasguard.baseline_tokens` — the live-window token count at the moment
|
|
49
|
+
* an ineffective compaction was observed (the baseline the guard re-arms from).
|
|
50
|
+
* - `thrasguard.blocked_until` — the live-window token count below which
|
|
51
|
+
* re-firing is refused (guard active).
|
|
52
|
+
*
|
|
53
|
+
* Non-finite input (NaN / ±Infinity) is rejected: the extension must never
|
|
54
|
+
* persist a non-number into the meta table (getMetaNumber would read it back as
|
|
55
|
+
* 0), so we return early, non-fatal. Best-effort: any store failure is swallowed.
|
|
56
|
+
*/
|
|
57
|
+
export function setMetaNumber(key, value, stateDir = getStateDir()) {
|
|
58
|
+
if (key.length === 0)
|
|
59
|
+
return;
|
|
60
|
+
if (!Number.isFinite(value))
|
|
61
|
+
return;
|
|
62
|
+
try {
|
|
63
|
+
const db = openStore(stateDir);
|
|
64
|
+
db.prepare(`INSERT INTO meta(key, value) VALUES(?, ?)
|
|
65
|
+
ON CONFLICT(key) DO UPDATE SET value = ?`).run(key, String(value), String(value));
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
/* non-fatal: meta writes never break the agent loop */
|
|
69
|
+
}
|
|
70
|
+
}
|
|
39
71
|
/** Atomically add `delta` to an integer meta counter. */
|
|
40
72
|
function incMeta(key, delta, stateDir = getStateDir()) {
|
|
41
73
|
if (!(delta > 0))
|
|
@@ -241,6 +241,7 @@ export const SETTINGS: ReadonlyArray<SettingGroup> = [
|
|
|
241
241
|
num("MEGACOMPACT_L2_THRESHOLD", "L2 Cosine Threshold", "L2 semantic dedup firing point", 0.85, 0, 1),
|
|
242
242
|
num("MEGACOMPACT_L1_JACCARD", "L1 Jaccard Threshold", "L1 MinHash near-dup threshold", 0.8, 0, 1),
|
|
243
243
|
num("MEGACOMPACT_DEDUP_SIM", "Dedup Similarity", "Legacy content-similarity fallback", 0.9, 0, 1),
|
|
244
|
+
num("MEGACOMPACT_RECALL_MIN_COSINE", "Recall Min Cosine (same-repo)", "3WF-3 same-repo floor the 3-source validator applies to the top winner (cross-repo 0.90 stays separate)", 0.12, 0, 1),
|
|
244
245
|
num("MEGACOMPACT_MMR_LAMBDA", "MMR Lambda", "Maximal Marginal Relevance diversity", 0.5, 0, 1),
|
|
245
246
|
num("MEGACOMPACT_SEMDEDUP_COSINE", "SemDeDup Cosine", "Offline SemDeDup pair threshold", 0.95, 0, 1),
|
|
246
247
|
num("MEGACOMPACT_CONSOLIDATE_COSINE", "Consolidate Cosine", "Memory consolidation merge threshold", 0.7, 0, 1),
|
|
@@ -285,6 +286,14 @@ export const SETTINGS: ReadonlyArray<SettingGroup> = [
|
|
|
285
286
|
0.1,
|
|
286
287
|
0.95,
|
|
287
288
|
),
|
|
289
|
+
num(
|
|
290
|
+
"MEGACOMPACT_THRASH_REARM_PCT",
|
|
291
|
+
"Thrash Re-arm %",
|
|
292
|
+
"After an ineffective compaction (live window did not shrink), refuse to re-fire until the live window grows by this fraction of the effective threshold. Default 0.10 (10%)",
|
|
293
|
+
0.1,
|
|
294
|
+
0.01,
|
|
295
|
+
0.5,
|
|
296
|
+
),
|
|
288
297
|
],
|
|
289
298
|
},
|
|
290
299
|
VECTOR_CORTEX_SETTINGS,
|
|
@@ -119,6 +119,11 @@ export interface MegaConfig {
|
|
|
119
119
|
* tighter than same-repo so only genuinely-relevant cross-repo context is
|
|
120
120
|
* injected. */
|
|
121
121
|
crossRepoCosine: number;
|
|
122
|
+
/** Same-repo recall cosine floor (3WF-3). Default 0.12. SEPARATE from
|
|
123
|
+
* `crossRepoCosine` (S17, default 0.90 — stricter, cross-repo only). The
|
|
124
|
+
* 3-source validator applies this to the top winner; hits below it are
|
|
125
|
+
* rejected in favor of the next-ranked candidate or the provenance floor. */
|
|
126
|
+
recallMinCosine: number;
|
|
122
127
|
/** Memory-RAG auto-review enabled (S20). Every memoryReviewInterval turns the
|
|
123
128
|
* conversation is auto-reviewed into durable add/replace/remove memories. */
|
|
124
129
|
memoryAutoReview: boolean;
|
|
@@ -140,6 +145,14 @@ export interface MegaConfig {
|
|
|
140
145
|
* session_start never fired, so every session has a staged block (recall hits,
|
|
141
146
|
* else a provenance floor). Default ON; OFF = byte-identical pre-sprint. */
|
|
142
147
|
threeWayFailback: boolean;
|
|
148
|
+
/** 3WF-2: ThrashGuard re-arm budget as a FRACTION of `effectiveThreshold`.
|
|
149
|
+
* After an ineffective compaction (live window did not shrink), the guard
|
|
150
|
+
* refuses to re-fire until the live window has grown by at least
|
|
151
|
+
* `rearmPct × effectiveThreshold` tokens past the observed baseline. Default
|
|
152
|
+
* 0.10 (10% of the effective threshold). Env-overridable via
|
|
153
|
+
* MEGACOMPACT_THRASH_REARM_PCT. When the effective threshold is unknown
|
|
154
|
+
* (+Infinity), the guard skips arming (cannot compute N) and logs instead. */
|
|
155
|
+
thrashRearmPct: number;
|
|
143
156
|
/** A1 PLAN_V2 Phase 2: Message Separation — isolate user/assistant turns
|
|
144
157
|
* from volatile tool results so the prompt-cache prefix stays stable.
|
|
145
158
|
* PC-A: positive sprint flag, now default ON; flag-OFF (=0) is byte-identical
|
|
@@ -217,6 +217,12 @@ export function loadConfig(): MegaConfig {
|
|
|
217
217
|
autoWikiEnabled: envBool("MEGACOMPACT_AUTO_WIKI", true),
|
|
218
218
|
crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
|
|
219
219
|
crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
|
|
220
|
+
// 3WF-3: SAME-repo recall cosine floor applied by the 3-source validator to
|
|
221
|
+
// the top winner. SEPARATE from crossRepoCosine (S17, default 0.90, stricter
|
|
222
|
+
// and cross-repo only). This same-repo floor is permissive by default (0.12)
|
|
223
|
+
// so recall still surfaces loosely-relevant within-repo context while
|
|
224
|
+
// rejecting effectively-unrelated hits. Mirrors src/config.ts RECALL_MIN_COSINE.
|
|
225
|
+
recallMinCosine: Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12"),
|
|
220
226
|
memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
|
|
221
227
|
memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
|
|
222
228
|
recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
|
|
@@ -225,6 +231,12 @@ export function loadConfig(): MegaConfig {
|
|
|
225
231
|
// 3WF-1: TriggerGuard — guarantee a staged recall block on every context
|
|
226
232
|
// event even when session_start never fires. Default ON; OFF = byte-identical.
|
|
227
233
|
threeWayFailback: envBool("MEGACOMPACT_THREE_WAY_FAILBACK", true),
|
|
234
|
+
// 3WF-2: ThrashGuard re-arm budget as a fraction of effectiveThreshold.
|
|
235
|
+
// 0.10 default (10% of the effective threshold) — see mega-config-types.
|
|
236
|
+
// Clamped to [0.01, 0.5]: below 1% the guard is almost never armed (any
|
|
237
|
+
// growth re-fires, defeating the anti-thrash purpose); above 50% it would
|
|
238
|
+
// suppress legitimate re-fires for half the window. Env-overridable.
|
|
239
|
+
thrashRearmPct: clamp(envFlag("MEGACOMPACT_THRASH_REARM_PCT", 0.1), 0.01, 0.5),
|
|
228
240
|
// PC-A: positive sprint flag, default ON. =0 byte-identical to the
|
|
229
241
|
// pre-change OFF state (single gate lives at the call site in tailResult.ts).
|
|
230
242
|
messageSeparation: envBool("MEGACOMPACT_MESSAGE_SEPARATION", true),
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
import { autoCompactCheck } from "../../../src/compact.js";
|
|
18
18
|
import type { MegaRuntime } from "../../mega-runtime.js";
|
|
19
19
|
import type { MegaConfig } from "../../mega-config.js";
|
|
20
|
+
import { isThrashBlockedFor } from "./thrashGuard.js";
|
|
20
21
|
|
|
21
22
|
/** Tail-injection closure shape produced by buildTailResult (tailResult.ts). */
|
|
22
23
|
export type TailResultFn = (
|
|
@@ -31,6 +32,35 @@ export type GateOutcome =
|
|
|
31
32
|
perModelThreshold: { safetyMarginPct: number; firePointPct: number };
|
|
32
33
|
};
|
|
33
34
|
|
|
35
|
+
/**
|
|
36
|
+
* 3WF-2 ThrashGuard consult — refuse to fire a NEW compaction while the guard
|
|
37
|
+
* is armed. After an ineffective compaction (the live window did not shrink),
|
|
38
|
+
* `thrasguard.blocked_until` holds the live-token count the window must exceed
|
|
39
|
+
* before re-firing is allowed.
|
|
40
|
+
*
|
|
41
|
+
* WHY THIS IS NOT INSIDE `evaluateGate`: the fast gate runs BEFORE the cached
|
|
42
|
+
* replay path in context-handler.ts, and REPLAY MUST STAY EXEMPT. A replay is
|
|
43
|
+
* free (no compute, no new checkpoint) and re-stabilises the provider KV-cache
|
|
44
|
+
* prefix — suppressing it would cause the very cache invalidation the D.2/D.3
|
|
45
|
+
* replay design exists to prevent. The guard's job is to stop wasted NEW
|
|
46
|
+
* compaction work, not to withhold an already-computed view. So the consult is
|
|
47
|
+
* called from the handler AFTER the replay block and BEFORE the debounce +
|
|
48
|
+
* `invokePipeline` (the actual fire point), covering the percent branch and the
|
|
49
|
+
* token branch alike since both converge there.
|
|
50
|
+
*
|
|
51
|
+
* Umbrella OFF ⇒ always false (byte-identical to v0.20.83). Non-fatal: a store
|
|
52
|
+
* read error returns false — never refuse compaction on a store fault.
|
|
53
|
+
*/
|
|
54
|
+
export function thrashGuardBlocks(
|
|
55
|
+
runtime: MegaRuntime,
|
|
56
|
+
config: MegaConfig,
|
|
57
|
+
currentTokens: number | null | undefined,
|
|
58
|
+
): boolean {
|
|
59
|
+
if (!config.threeWayFailback) return false;
|
|
60
|
+
if (currentTokens == null) return false;
|
|
61
|
+
return isThrashBlockedFor(runtime, currentTokens, runtime.currentStateDir);
|
|
62
|
+
}
|
|
63
|
+
|
|
34
64
|
/**
|
|
35
65
|
* Evaluate whether the current context warrants compaction. Returns a tailed
|
|
36
66
|
* view ("return") when the gate does not pass, or "proceed" with the resolved
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* context-handler/thrashGuard.ts — 3WF-2 ReductionValidator + ThrashGuard.
|
|
3
|
+
*
|
|
4
|
+
* Production bug fixed here: compaction fired 496× freeing 0.0% of the live
|
|
5
|
+
* window. Root cause — correctness was judged by the STORED `saved` metric
|
|
6
|
+
* (a cumulative SQLite total that the dedup made look healthy every fire)
|
|
7
|
+
* while the LIVE context window (`currentTokens`) never shrank. This module
|
|
8
|
+
* judges correctness by the LIVE-WINDOW delta across consecutive `context`
|
|
9
|
+
* events, and after an ineffective compaction persists a meta-backed refusal
|
|
10
|
+
* so the guard will not re-fire until the window has grown meaningfully again.
|
|
11
|
+
*
|
|
12
|
+
* Everything is gated on the umbrella `config.threeWayFailback`
|
|
13
|
+
* (MEGACOMPACT_THREE_WAY_FAILBACK, default ON). Flag OFF ⇒ every entry point
|
|
14
|
+
* is an immediate no-op, so gateCheck + compactSession behave byte-identically
|
|
15
|
+
* to v0.20.83.
|
|
16
|
+
*
|
|
17
|
+
* Non-fatal EVERYWHERE: every store read/write is best-effort, swallowed on
|
|
18
|
+
* failure. Structured JSON logging only (runtime.logger.info with ts + event).
|
|
19
|
+
* No console.*, no network, no mocks.
|
|
20
|
+
*/
|
|
21
|
+
import type { MegaRuntime } from "../../mega-runtime.js";
|
|
22
|
+
import type { MegaConfig } from "../../mega-config.js";
|
|
23
|
+
import { getMetaNumber, setMetaNumber } from "../../../src/store/sqlite.js";
|
|
24
|
+
import type { ReductionVerdict } from "../../../src/failback/types.js";
|
|
25
|
+
|
|
26
|
+
/** Meta key holding the live-window baseline (tokens) at the ineffective fire. */
|
|
27
|
+
export const THRASH_BASELINE_KEY = "thrasguard.baseline_tokens";
|
|
28
|
+
/** Meta key holding the live-window token count below which re-firing is blocked. */
|
|
29
|
+
export const THRASH_BLOCKED_KEY = "thrasguard.blocked_until";
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* "Meaningful reduction" floor as a FRACTION of `liveBefore`. A compaction is
|
|
33
|
+
* only credited with freeing space when the live window shrank by at least this
|
|
34
|
+
* fraction of its pre-compaction size.
|
|
35
|
+
*
|
|
36
|
+
* Rationale (invented constant, calibrated + configurable-by-design): the model
|
|
37
|
+
* re-reports token counts on every context event with noise on the order of a
|
|
38
|
+
* percent or two, so a sub-1% wobble is not a real reduction — crediting it
|
|
39
|
+
* would suppress the guard on a genuine no-op fire (the exact bug we are
|
|
40
|
+
* fixing). 2% is a defensible "real shrink" threshold: it is well above typical
|
|
41
|
+
* re-estimation noise but low enough that a compaction that freed even a few
|
|
42
|
+
* percent of the window is not punished. A reduction of ≤0 tokens is
|
|
43
|
+
* unconditionally ineffective regardless of this floor.
|
|
44
|
+
*/
|
|
45
|
+
const MEANINGFUL_REDUCTION_PCT = 0.02;
|
|
46
|
+
|
|
47
|
+
/** Pure reduction verdict for a live-window bracketing pair. */
|
|
48
|
+
export const ReductionValidator = {
|
|
49
|
+
/**
|
|
50
|
+
* Judge whether the LIVE window actually shrank between two consecutive
|
|
51
|
+
* context events bracketing a compaction.
|
|
52
|
+
* - a reduction of ≤0 tokens ⇒ definitively ineffective.
|
|
53
|
+
* - otherwise effective only when the reduction is ≥ the small positive
|
|
54
|
+
* floor (MEANINGFUL_REDUCTION_PCT of liveBefore), so estimation noise on
|
|
55
|
+
* the model's re-reported token count is not mistaken for a real shrink.
|
|
56
|
+
*/
|
|
57
|
+
validateReduction(liveBefore: number, liveAfter: number): ReductionVerdict {
|
|
58
|
+
const reduction = liveBefore - liveAfter;
|
|
59
|
+
const floor = Math.max(1, Math.round(MEANINGFUL_REDUCTION_PCT * liveBefore));
|
|
60
|
+
const effective = Number.isFinite(reduction) && reduction > 0 && reduction >= floor;
|
|
61
|
+
return { effective, liveBefore, liveAfter };
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Arm the ThrashGuard after an ineffective compaction. Persists:
|
|
67
|
+
* - `thrasguard.baseline_tokens` = the live currentTokens at this (post-fire)
|
|
68
|
+
* context event, so re-arm is measured from the window that failed to shrink.
|
|
69
|
+
* - `thrasguard.blocked_until` = baseline + N, where N = `rearmPct ×
|
|
70
|
+
* effectiveThreshold`. Re-firing is refused until the live window grows past
|
|
71
|
+
* `blocked_until`.
|
|
72
|
+
*
|
|
73
|
+
* If `effectiveThreshold` is non-finite (+Infinity — the 3WF-2 invariant when
|
|
74
|
+
* the model window is unknown), N cannot be computed; we MUST NOT persist
|
|
75
|
+
* Infinity/NaN into meta (getMetaNumber would read it back as 0). Skip arming
|
|
76
|
+
* + log instead; the next over-threshold event simply re-fires (pre-sprint
|
|
77
|
+
* behavior) rather than corrupting the guard.
|
|
78
|
+
*/
|
|
79
|
+
export function armThrashGuard(
|
|
80
|
+
currentTokens: number,
|
|
81
|
+
rearmPct: number,
|
|
82
|
+
effectiveThreshold: number,
|
|
83
|
+
stateDir: string,
|
|
84
|
+
logger?: { info(event: string, fields?: Record<string, unknown>): void },
|
|
85
|
+
): void {
|
|
86
|
+
if (!Number.isFinite(currentTokens) || currentTokens <= 0) return;
|
|
87
|
+
if (!Number.isFinite(rearmPct) || rearmPct <= 0) return;
|
|
88
|
+
if (!Number.isFinite(effectiveThreshold)) {
|
|
89
|
+
logger?.info("thrasguard_skip_arm", {
|
|
90
|
+
reason: "nonfinite_effective_threshold",
|
|
91
|
+
currentTokens,
|
|
92
|
+
});
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
try {
|
|
96
|
+
const n = Math.round(rearmPct * effectiveThreshold);
|
|
97
|
+
setMetaNumber(THRASH_BASELINE_KEY, Math.round(currentTokens), stateDir);
|
|
98
|
+
setMetaNumber(THRASH_BLOCKED_KEY, Math.round(currentTokens + n), stateDir);
|
|
99
|
+
logger?.info("thrasguard_armed", {
|
|
100
|
+
baselineTokens: Math.round(currentTokens),
|
|
101
|
+
blockedUntilTokens: Math.round(currentTokens + n),
|
|
102
|
+
rearmTokens: n,
|
|
103
|
+
});
|
|
104
|
+
} catch {
|
|
105
|
+
/* non-fatal: best-effort meta write */
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Consult the ThrashGuard for the current live-window token count. Returns true
|
|
111
|
+
* when compaction must be refused (the window is still below the armed
|
|
112
|
+
* `blocked_until`). A `blocked_until` of 0/absent ⇒ never blocked. When the
|
|
113
|
+
* live tokens have grown past `blocked_until`, the guard no longer blocks
|
|
114
|
+
* (caller re-fires normally). Pure read, best-effort — on any failure returns
|
|
115
|
+
* false (do not refuse compaction on a store error).
|
|
116
|
+
*/
|
|
117
|
+
export function isThrashBlocked(currentTokens: number, stateDir: string): boolean {
|
|
118
|
+
try {
|
|
119
|
+
const blockedUntil = getMetaNumber(THRASH_BLOCKED_KEY, stateDir);
|
|
120
|
+
if (blockedUntil <= 0) return false;
|
|
121
|
+
return Number.isFinite(currentTokens) && currentTokens < blockedUntil;
|
|
122
|
+
} catch {
|
|
123
|
+
return false; // non-fatal: never refuse compaction on a read error
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Per-runtime one-shot session state for the live-window delta correlation.
|
|
129
|
+
* Like triggerGuard.ts, keyed by runtime in a WeakMap so it dies with the
|
|
130
|
+
* runtime and a test can pass a thin stub. Holds the live token count observed
|
|
131
|
+
* at the event that FIRED a compaction; consumed on the following context event
|
|
132
|
+
* to judge whether the window actually shrank.
|
|
133
|
+
*/
|
|
134
|
+
const sessionBefore = new WeakMap<MegaRuntime, { liveBefore: number }>();
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The live-token count of the event that most recently ARMED the guard, per
|
|
138
|
+
* runtime. The arming happens early in a context event (the live-delta consume
|
|
139
|
+
* point), but the guard consult runs LATER IN THAT SAME EVENT — and since
|
|
140
|
+
* `blocked_until = currentTokens + N`, a naive consult would always find
|
|
141
|
+
* `currentTokens < blocked_until` and swallow the very event that armed it.
|
|
142
|
+
* That is an off-by-one-event error: the guard's contract is to refuse
|
|
143
|
+
* SUBSEQUENT re-fires, not to cancel the compaction that revealed the problem.
|
|
144
|
+
* Recording the arming event's token count lets the consult skip exactly that
|
|
145
|
+
* one event. Cleared once the window grows past it.
|
|
146
|
+
*/
|
|
147
|
+
const armedOnEvent = new WeakMap<MegaRuntime, number>();
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Runtime-aware ThrashGuard consult: true when a NEW compaction must be refused.
|
|
151
|
+
*
|
|
152
|
+
* Reads the persisted `thrasguard.blocked_until` (see `isThrashBlocked`) but
|
|
153
|
+
* EXEMPTS the single event that armed the guard — otherwise, because arming sets
|
|
154
|
+
* `blocked_until = currentTokens + N` earlier in the very same context event, the
|
|
155
|
+
* consult would always fire and cancel the compaction that exposed the thrash.
|
|
156
|
+
* The guard exists to refuse SUBSEQUENT re-fires. Once the live window grows past
|
|
157
|
+
* the armed count the exemption is dropped, and normal blocking resumes until the
|
|
158
|
+
* window clears `blocked_until`.
|
|
159
|
+
*
|
|
160
|
+
* Best-effort: any failure returns false (never refuse on a store fault).
|
|
161
|
+
*/
|
|
162
|
+
export function isThrashBlockedFor(
|
|
163
|
+
runtime: MegaRuntime,
|
|
164
|
+
currentTokens: number,
|
|
165
|
+
stateDir: string,
|
|
166
|
+
): boolean {
|
|
167
|
+
try {
|
|
168
|
+
if (!isThrashBlocked(currentTokens, stateDir)) return false;
|
|
169
|
+
const armedAt = armedOnEvent.get(runtime);
|
|
170
|
+
if (armedAt !== undefined && currentTokens === armedAt) {
|
|
171
|
+
// EXACTLY the event that armed the guard (same live-token reading): let it
|
|
172
|
+
// through once, then block normally from the next event onward. An exact
|
|
173
|
+
// match (not <=) is required so a genuinely lower or different live reading
|
|
174
|
+
// on a later event is still blocked.
|
|
175
|
+
armedOnEvent.delete(runtime);
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
178
|
+
return true;
|
|
179
|
+
} catch {
|
|
180
|
+
return false;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Record that a compaction fired at `liveBefore` tokens (call on the firing event). */
|
|
185
|
+
export function markCompactionFired(runtime: MegaRuntime, liveBefore: number): void {
|
|
186
|
+
try {
|
|
187
|
+
if (Number.isFinite(liveBefore) && liveBefore > 0) {
|
|
188
|
+
sessionBefore.set(runtime, { liveBefore });
|
|
189
|
+
}
|
|
190
|
+
} catch {
|
|
191
|
+
/* non-fatal */
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Consume a pending live-window delta on a subsequent context event. If a
|
|
197
|
+
* compaction fired on a prior event, compare THIS event's live tokens against
|
|
198
|
+
* that pre-fire baseline; an ineffective reduction arms the guard. The pending
|
|
199
|
+
* marker is consumed exactly once (cleared before any re-arm). No-op when no
|
|
200
|
+
* compaction is pending, when the umbrella flag is OFF, or on any error.
|
|
201
|
+
*/
|
|
202
|
+
export function evaluatePendingReduction(
|
|
203
|
+
runtime: MegaRuntime,
|
|
204
|
+
currentTokens: number,
|
|
205
|
+
config: MegaConfig,
|
|
206
|
+
): void {
|
|
207
|
+
if (!config.threeWayFailback) return;
|
|
208
|
+
const pending = sessionBefore.get(runtime);
|
|
209
|
+
if (pending == null) return;
|
|
210
|
+
try {
|
|
211
|
+
sessionBefore.delete(runtime); // consume once, regardless of verdict
|
|
212
|
+
const verdict = ReductionValidator.validateReduction(pending.liveBefore, currentTokens);
|
|
213
|
+
if (!verdict.effective) {
|
|
214
|
+
// Remember which event armed us so the consult later in THIS SAME event
|
|
215
|
+
// does not swallow it (see armedOnEvent).
|
|
216
|
+
armedOnEvent.set(runtime, currentTokens);
|
|
217
|
+
armThrashGuard(
|
|
218
|
+
currentTokens,
|
|
219
|
+
config.thrashRearmPct,
|
|
220
|
+
runtime.effectiveThreshold,
|
|
221
|
+
runtime.currentStateDir,
|
|
222
|
+
runtime.logger,
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
} catch {
|
|
226
|
+
/* non-fatal */
|
|
227
|
+
}
|
|
228
|
+
}
|