pi-mega-compact 0.21.8 → 0.21.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/dedup/degenerate.js +68 -0
  2. package/dist/extensions/dashboard-server/routes-dedup-attribution.js +8 -1
  3. package/dist/extensions/dashboard-server/routes-rag-settings-compaction.js +42 -0
  4. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +8 -8
  5. package/dist/extensions/mega-config.js +12 -0
  6. package/dist/extensions/mega-events/context-handler/gateCheck.js +51 -1
  7. package/dist/extensions/mega-events/context-handler/headroom.js +128 -0
  8. package/dist/extensions/mega-events/context-handler/liveTrim.js +37 -51
  9. package/dist/extensions/mega-events/context-handler/pipelineRun.js +14 -1
  10. package/dist/extensions/mega-events/context-handler.js +26 -3
  11. package/dist/extensions/mega-pipeline/compact/run.js +14 -3
  12. package/dist/extensions/mega-runtime/dashboard-snapshot.js +1 -0
  13. package/dist/extensions/mega-runtime/runtime-instrumentation.js +1 -0
  14. package/dist/extensions/mega-runtime/runtime-snapshot.js +1 -0
  15. package/dist/src/config/dedup.js +3 -0
  16. package/dist/src/dedup/degenerate.js +68 -0
  17. package/dist/src/extractive-salvage.js +195 -0
  18. package/dist/src/extractive.js +63 -72
  19. package/dist/src/vector-cortex/dedup-attr/rollup.js +5 -0
  20. package/dist/src/vectorStore/add-degenerate.js +25 -0
  21. package/dist/src/vectorStore/add.js +28 -5
  22. package/dist/src/vectorStore/dedup-audit.js +8 -0
  23. package/dist/vector-cortex/dedup-attr/rollup.js +5 -0
  24. package/dist/vectorStore/dedup-audit.js +8 -0
  25. package/extensions/dashboard-server/api-contracts/endpoints/types.ts +2 -0
  26. package/extensions/dashboard-server/routes-dedup-attribution.ts +11 -1
  27. package/extensions/dashboard-server/routes-rag-settings-compaction.ts +91 -0
  28. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +13 -27
  29. package/extensions/mega-config-types.ts +25 -0
  30. package/extensions/mega-config.ts +12 -0
  31. package/extensions/mega-dashboard.ts +4 -1
  32. package/extensions/mega-events/context-handler/gateCheck.ts +67 -0
  33. package/extensions/mega-events/context-handler/headroom.ts +190 -0
  34. package/extensions/mega-events/context-handler/liveTrim.ts +37 -57
  35. package/extensions/mega-events/context-handler/pipelineRun.ts +14 -1
  36. package/extensions/mega-events/context-handler.ts +26 -3
  37. package/extensions/mega-pipeline/compact/run.ts +14 -4
  38. package/extensions/mega-runtime/dashboard-snapshot.ts +3 -0
  39. package/extensions/mega-runtime/runtime-instrumentation.ts +4 -0
  40. package/extensions/mega-runtime/runtime-snapshot.ts +2 -0
  41. package/package.json +1 -1
  42. package/src/config/dedup.ts +15 -0
  43. package/src/dedup/degenerate.ts +125 -0
  44. package/src/extractive-salvage.ts +212 -0
  45. package/src/extractive.ts +70 -75
  46. package/src/vector-cortex/dedup-attr/rollup.ts +4 -0
  47. package/src/vectorStore/add-degenerate.ts +64 -0
  48. package/src/vectorStore/add.ts +29 -5
  49. package/src/vectorStore/dedup-audit.ts +25 -2
@@ -0,0 +1,212 @@
1
+ /**
2
+ * extractive-salvage.ts — file-path policy + skeleton salvage for extractive.ts.
3
+ *
4
+ * Extracted from extractive.ts (A1/A2 sprint) to keep that file under the
5
+ * 300-line src/ soft limit. Pure functions only — no I/O, no logging.
6
+ *
7
+ * DESIGN (A1): blocklist, not allowlist. The old `INTERESTING_EXT` allowlist
8
+ * (rs/ts/tsx/js/json/md) silently produced content-free summaries for every
9
+ * other language — a .go/.py/.c project got a 34-token skeleton. An allowlist
10
+ * has to be *right* about ~40 ecosystems to be useful and fails closed (drops
11
+ * real work) when it is wrong; a NOISE blocklist only has to be right about the
12
+ * small, stable set of binary/generated/asset extensions and fails open (an
13
+ * unknown extension is surfaced, which is the safe direction for a summary).
14
+ */
15
+
16
+ import type { EngineMessage } from "./types.js";
17
+
18
+ // ---- File path policy ------------------------------------------------------
19
+
20
+ /**
21
+ * Generic extension capture: any 1–6 char ALPHABETIC extension. Path
22
+ * character class is unchanged from the original FILE_PATH_RE so existing
23
+ * matching behaviour (quotes/backticks/whitespace as delimiters) is preserved.
24
+ * Alphabetic-only is deliberate: every real source/config extension is alpha
25
+ * (c..tsx, tsconfig.json), while an alphanumeric class matches version strings
26
+ * ("GLM-4.7", "v0.21.9", "Node 18.2") as "files" and spams Key files.
27
+ */
28
+ export const FILE_PATH_RE = /(?:^|\s)([^\s"`']+\.([A-Za-z]{1,6}))\b/g;
29
+
30
+ /** Same policy, single-match, for inferCurrentWork (also excludes ':'). */
31
+ export const CURRENT_WORK_PATH_RE = /(?:^|\s)([^\s"`':]+\.([A-Za-z]{1,6}))\b/m;
32
+
33
+ /**
34
+ * Binary / generated / asset / vendored extensions that carry no summary value.
35
+ * Deliberately small and stable. Note `md`, `json`, `toml`, `yaml`, `sql`, `css`
36
+ * and `html` are NOT noise — they are hand-edited source in most repos.
37
+ */
38
+ export const NOISE_EXT = new Set([
39
+ // lockfiles & logs
40
+ "lock", "log", "sum",
41
+ // images & media
42
+ "png", "jpg", "jpeg", "gif", "webp", "svg", "ico", "bmp", "tiff",
43
+ "mp3", "mp4", "mov", "wav", "webm", "avi",
44
+ // fonts
45
+ "woff", "woff2", "ttf", "otf", "eot",
46
+ // archives & binaries
47
+ "zip", "gz", "tgz", "bz2", "xz", "7z", "rar", "tar",
48
+ "exe", "dll", "so", "dylib", "bin", "o", "a", "obj", "class", "pyc", "pyo",
49
+ "wasm", "node", "jar", "war", "deb", "rpm", "dmg", "iso", "img",
50
+ // generated / build artifacts
51
+ "map", "min", "lockb", "snap", "cache", "tmp", "temp", "swp", "bak", "orig",
52
+ // data blobs & databases
53
+ "pdf", "db", "sqlite", "sqlite3", "pack", "idx", "pem", "key", "crt",
54
+ ]);
55
+
56
+ /** Directory fragments whose files are never "key files" for a summary. */
57
+ const NOISE_DIR_RE = /(?:^|\/)(?:node_modules|\.git|dist|build|coverage|vendor|target|__pycache__|\.venv|venv)(?:\/|$)/;
58
+
59
+ /**
60
+ * TLD-shaped extensions: `example.com`, `github.com`, `npm.cmd` are domains and
61
+ * launchers, not code. (QA lens 1 finding, 2026-08-19.) Conservative list only —
62
+ * no real source extension lives here (.go/.rs/.ts stay interesting).
63
+ */
64
+ const TLD_EXT = new Set([
65
+ "com", "org", "net", "io", "gov", "edu", "biz", "info", "dev", "app",
66
+ "page", "xyz", "site", "online", "cloud", "me", "co", "us", "uk", "de",
67
+ "fr", "jp", "cn", "nl", "se", "eu", "int", "mil", "cmd",
68
+ ]);
69
+
70
+ /** True when a matched path is worth surfacing in a summary. */
71
+ export function isInterestingPath(filePath: string, ext: string): boolean {
72
+ const lowerExt = ext.toLowerCase();
73
+ if (NOISE_EXT.has(lowerExt)) return false;
74
+ if (TLD_EXT.has(lowerExt)) return false; // bare domains, not code
75
+ if (/^(?:https?|ftp):\/\//i.test(filePath) || filePath.toLowerCase().startsWith("www.")) return false;
76
+ if (NOISE_DIR_RE.test(filePath)) return false;
77
+ // `app.min.js` / `bundle.min.css` style double extensions.
78
+ if (/\.min\.[A-Za-z0-9]{1,6}$/.test(filePath)) return false;
79
+ if (/\.map$/.test(filePath)) return false;
80
+ // Prose abbreviations ("e.g", "i.e", "U.S"): a single-char base with no slash
81
+ // and no digit is never a filename. ("a.c" is rare collateral; "q1.py" and
82
+ // "src/a.ts" survive — digit / slash both exempt.)
83
+ const base = filePath.slice(0, filePath.lastIndexOf("."));
84
+ if (!filePath.includes("/") && !/\d/.test(filePath) && base.length <= 1) return false;
85
+ return true;
86
+ }
87
+
88
+ // ---- Path collection -------------------------------------------------------
89
+
90
+ const MAX_KEY_FILES = 5;
91
+ const MAX_FILES = 10;
92
+ const FRESHNESS_WINDOW = 10;
93
+
94
+ /** All interesting file paths mentioned in a blob of text. */
95
+ export function extractFilePaths(text: string): string[] {
96
+ const paths: string[] = [];
97
+ for (const m of text.matchAll(FILE_PATH_RE)) {
98
+ if (isInterestingPath(m[1], m[2])) paths.push(m[1]);
99
+ }
100
+ return paths;
101
+ }
102
+
103
+ /** Most-mentioned paths within the recency window (behaviour unchanged). */
104
+ export function collectKeyFiles(messages: EngineMessage[]): string[] {
105
+ const recent = messages.slice(-FRESHNESS_WINDOW);
106
+ const pathFreq = new Map<string, number>();
107
+ for (const m of recent) {
108
+ for (const p of extractFilePaths(m.text)) {
109
+ pathFreq.set(p, (pathFreq.get(p) ?? 0) + 1);
110
+ }
111
+ }
112
+ return [...pathFreq.entries()]
113
+ .sort((a, b) => b[1] - a[1])
114
+ .slice(0, MAX_KEY_FILES)
115
+ .map(([p]) => p);
116
+ }
117
+
118
+ /** Paths written/edited by tools (extension-agnostic; behaviour unchanged). */
119
+ export function extractFilesModified(tools: EngineMessage[]): string[] {
120
+ const files = new Set<string>();
121
+ for (const m of tools) {
122
+ if (!m.toolName) continue;
123
+ const name = m.toolName.toLowerCase();
124
+ if (name === "write" || name === "edit" || name === "notebookedit") {
125
+ const input = m.input ?? m.text;
126
+ const pathMatch = input.match(/["']?(\/[^\s"']+\.\w+)["']?/);
127
+ if (pathMatch) files.add(pathMatch[1]);
128
+ }
129
+ if (name === "bash") {
130
+ const cmd = m.input ?? m.text;
131
+ if (cmd.includes("git add") || cmd.includes("git commit") || cmd.includes("git diff")) {
132
+ for (const p of extractFilePaths(cmd)) files.add(p);
133
+ }
134
+ }
135
+ }
136
+ return [...files].slice(0, MAX_FILES);
137
+ }
138
+
139
+ // ---- Placeholder user turns (A2b) ------------------------------------------
140
+
141
+ /**
142
+ * Content-free user turns. Extremely common in resumed sessions; taking them
143
+ * verbatim as "User requests" is what produced the three "• resume" bullets.
144
+ */
145
+ const PLACEHOLDER_RE =
146
+ /^(?:resume|continue|go on|go ahead|proceed|next|yes|yeah|yep|y|ok|okay|k|sure|thanks|thank you|ty|done|please continue|carry on)\W*$/i;
147
+
148
+ export function isPlaceholderRequest(text: string): boolean {
149
+ return PLACEHOLDER_RE.test(text.trim());
150
+ }
151
+
152
+ // ---- Skeleton salvage (A2c) ------------------------------------------------
153
+
154
+ const MAX_SALVAGE_LINES = 5;
155
+ const SALVAGE_LINE_LEN = 120;
156
+
157
+ /**
158
+ * A "skeleton" summary is the scope line and nothing else — no files, no current
159
+ * work, no decisions, no pending items, and no *substantive* user request. That
160
+ * is ~34 tokens of zero information and is what breaks a resumed session.
161
+ *
162
+ * NOTE: `recentUser` is treated as empty when it holds only placeholders. The
163
+ * incident summary DID have three "• resume" bullets, so a plain
164
+ * `recentUser.length === 0` test would never have fired on the very case this
165
+ * salvage exists for.
166
+ */
167
+ export function isSkeletonSummary(parts: {
168
+ recentUser: string[];
169
+ keyFiles: string[];
170
+ currentWork: string | undefined;
171
+ decisions: string[];
172
+ pending: string[];
173
+ }): boolean {
174
+ const hasRealRequest = parts.recentUser.some((r) => !isPlaceholderRequest(r));
175
+ return (
176
+ !hasRealRequest &&
177
+ parts.keyFiles.length === 0 &&
178
+ !parts.currentWork &&
179
+ parts.decisions.length === 0 &&
180
+ parts.pending.length === 0
181
+ );
182
+ }
183
+
184
+ /**
185
+ * Last-resort content: the first meaningful line of the most recent assistant
186
+ * and tool messages. Deterministic (pure scan, newest-first, then re-ordered
187
+ * oldest-first for reading). Returns [] when there is genuinely nothing.
188
+ */
189
+ export function buildSalvageDigest(messages: EngineMessage[]): string[] {
190
+ const out: string[] = [];
191
+ const seen = new Set<string>();
192
+ for (let i = messages.length - 1; i >= 0 && out.length < MAX_SALVAGE_LINES; i--) {
193
+ const m = messages[i];
194
+ if (m.role !== "assistant" && m.role !== "tool") continue;
195
+ const raw = m.text || m.output || m.input || "";
196
+ const line = raw
197
+ .split("\n")
198
+ .map((l) => l.trim())
199
+ .find((l) => l.length > 0);
200
+ if (!line) continue;
201
+ const label = m.role === "tool" ? `${m.toolName ?? "tool"}: ` : "";
202
+ const entry = truncateLine(`${label}${line}`, SALVAGE_LINE_LEN);
203
+ if (seen.has(entry)) continue;
204
+ seen.add(entry);
205
+ out.push(entry);
206
+ }
207
+ return out.reverse();
208
+ }
209
+
210
+ function truncateLine(s: string, maxLen: number): string {
211
+ return s.length <= maxLen ? s : s.slice(0, maxLen - 1) + "…";
212
+ }
package/src/extractive.ts CHANGED
@@ -10,14 +10,24 @@
10
10
 
11
11
  import type { EngineMessage } from "./types.js";
12
12
  import { estimateBlockTokens } from "./tokens.js";
13
+ import {
14
+ CURRENT_WORK_PATH_RE,
15
+ isInterestingPath,
16
+ isPlaceholderRequest,
17
+ isSkeletonSummary,
18
+ buildSalvageDigest,
19
+ collectKeyFiles,
20
+ extractFilesModified,
21
+ } from "./extractive-salvage.js";
13
22
 
14
23
  // ---- Limits ----------------------------------------------------------------
15
24
 
16
25
  const MAX_RECENT_USER = 3;
17
26
  const MAX_DECISIONS = 5;
18
- const MAX_FILES = 10;
19
27
  const MAX_PENDING = 5;
20
28
  const MAX_TOPIC_LINES = 12;
29
+ /** Cap for the merged keyFiles ∪ filesModified "Key files" line (A2a). */
30
+ const MAX_SUMMARY_FILES = 8;
21
31
 
22
32
  // ---- Truncation helper -----------------------------------------------------
23
33
 
@@ -57,9 +67,27 @@ function buildTopicSummary(
57
67
  currentWork: string | undefined,
58
68
  keyFiles: string[],
59
69
  pending: string[],
70
+ filesModified: string[],
71
+ decisions: string[],
60
72
  ): string {
61
73
  const lines: string[] = [];
62
74
 
75
+ // A2a: files captured from write/edit tool inputs are extracted
76
+ // extension-agnostically but never reached the summary. Fold them in so work
77
+ // outside the recency window (and outside the path regex) is still reported.
78
+ // Drop an absolute path when a kept relative path already names the SAME file.
79
+ // Only MULTI-COMPONENT relative paths fold ("engine/mesh.go" absorbs
80
+ // "/proj/engine/mesh.go"); a bare basename ("mesh.go") is never folded into an
81
+ // absolute path, since "/proj/other/x/mesh.go" may be a genuinely different
82
+ // file (QA lens 1 finding: the naive endsWith dropped different directories).
83
+ const combined = [...keyFiles, ...filesModified];
84
+ const allFiles = [...new Set(combined)].filter((p) => {
85
+ if (!p.startsWith("/")) return true;
86
+ return !combined.some(
87
+ (r) => r !== p && !r.startsWith("/") && r.includes("/") && p.endsWith("/" + r),
88
+ );
89
+ }).slice(0, MAX_SUMMARY_FILES);
90
+
63
91
  // Scope line
64
92
  const users = messages.filter((m) => m.role === "user");
65
93
  const assistants = messages.filter((m) => m.role === "assistant");
@@ -79,8 +107,8 @@ function buildTopicSummary(
79
107
  // Current work
80
108
  if (currentWork) lines.push(`Current work: ${currentWork}`);
81
109
 
82
- // Key files
83
- if (keyFiles.length) lines.push(`Key files: ${keyFiles.join(", ")}.`);
110
+ // Key files (keyFiles ∪ filesModified)
111
+ if (allFiles.length) lines.push(`Key files: ${allFiles.join(", ")}.`);
84
112
 
85
113
  // Pending work
86
114
  if (pending.length) {
@@ -88,42 +116,52 @@ function buildTopicSummary(
88
116
  for (const p of pending) lines.push(` • ${p}`);
89
117
  }
90
118
 
119
+ // A2c: a scope-line-only summary carries zero information and strands a
120
+ // resumed session. Salvage the tail of the conversation instead. The line cap
121
+ // is raised ONLY here: the salvage block is bounded at 5 lines + 1 header, and
122
+ // a skeleton by definition contributed just the 1 scope line, so the worst
123
+ // case is 7 lines — still well under the normal 12-line budget.
124
+ const skeleton = isSkeletonSummary({ recentUser, keyFiles: allFiles, currentWork, decisions, pending });
125
+ if (skeleton) {
126
+ const digest = buildSalvageDigest(messages);
127
+ if (digest.length) {
128
+ lines.push("Recent activity:");
129
+ for (const d of digest) lines.push(` • ${d}`);
130
+ }
131
+ return lines.join("\n");
132
+ }
133
+
91
134
  // Cap total length
92
135
  return lines.slice(0, MAX_TOPIC_LINES).join("\n");
93
136
  }
94
137
 
95
- // ---- File path extraction --------------------------------------------------
96
-
97
- const INTERESTING_EXT = new Set(["rs", "ts", "tsx", "js", "json", "md"]);
98
- const FILE_PATH_RE = /(?:^|\s)([^\s"`']+\.(rs|ts|tsx|js|json|md|py|sh|sql|toml|yaml|yml|css|html))\b/g;
99
-
100
- function extractFilePaths(text: string): string[] {
101
- const paths: string[] = [];
102
- for (const m of text.matchAll(FILE_PATH_RE)) {
103
- const filePath = m[1];
104
- const ext = m[2];
105
- const basename = filePath.split("/").pop() ?? filePath;
106
- if (basename === "node_modules" || filePath.includes("node_modules/")) continue;
107
- if (INTERESTING_EXT.has(ext)) paths.push(filePath);
108
- }
109
- return paths;
110
- }
111
-
112
- // ---- Recent user requests (existing logic, kept) ---------------------------
138
+ // ---- Recent user requests --------------------------------------------------
113
139
 
140
+ /**
141
+ * A2b: skip content-free "resume"/"continue" turns and look further back to
142
+ * fill the quota, so a resumed session surfaces its real requests. Falls back
143
+ * to the placeholders when EVERY user turn is one (an honest "• resume" beats
144
+ * an empty section).
145
+ */
114
146
  function collectRecentUserRequests(
115
147
  messages: EngineMessage[],
116
148
  limit: number,
117
149
  ): string[] {
118
- const requests: string[] = [];
119
- for (let i = messages.length - 1; i >= 0 && requests.length < limit; i--) {
120
- if (messages[i].role === "user") {
121
- let snippet = messages[i].text.split("\n").slice(0, 3).join(" ");
122
- snippet = snippet.replace(/^.+\nProcessed\$?\s*/i, "").replace(/\n/g, " ");
123
- requests.push(truncate(snippet, 200));
150
+ const substantive: string[] = [];
151
+ const placeholders: string[] = [];
152
+ for (let i = messages.length - 1; i >= 0 && substantive.length < limit; i--) {
153
+ if (messages[i].role !== "user") continue;
154
+ let snippet = messages[i].text.split("\n").slice(0, 3).join(" ");
155
+ snippet = snippet.replace(/^.+\nProcessed\$?\s*/i, "").replace(/\n/g, " ");
156
+ const cleaned = truncate(snippet, 200);
157
+ if (!cleaned.trim()) continue;
158
+ if (isPlaceholderRequest(cleaned)) {
159
+ if (placeholders.length < limit) placeholders.push(cleaned);
160
+ continue;
124
161
  }
162
+ substantive.push(cleaned);
125
163
  }
126
- return requests.reverse();
164
+ return (substantive.length ? substantive : placeholders).reverse();
127
165
  }
128
166
 
129
167
  // ---- Pending work (existing logic, kept) -----------------------------------
@@ -149,10 +187,9 @@ function inferCurrentWork(messages: EngineMessage[]): string | undefined {
149
187
  for (let i = messages.length - 1; i >= 0; i--) {
150
188
  const m = messages[i];
151
189
  if (m.role !== "assistant") continue;
152
- const path = m.text.match(
153
- /(?:^|\s)([^\s"`':]+\.(rs|ts|tsx|js|json|md|py|toml|yaml|yml|sql))\b/m,
154
- );
155
- if (path) {
190
+ // A1: same language-agnostic policy as extractFilePaths.
191
+ const path = m.text.match(CURRENT_WORK_PATH_RE);
192
+ if (path && isInterestingPath(path[1], path[2])) {
156
193
  const line = m.text.split("\n").slice(0, 2).join(" ");
157
194
  return truncate(line, 200);
158
195
  }
@@ -190,29 +227,6 @@ function extractDecisions(messages: EngineMessage[]): string[] {
190
227
  return [...new Set(decisions)];
191
228
  }
192
229
 
193
- // ---- Files modified --------------------------------------------------------
194
-
195
- function extractFilesModified(tools: EngineMessage[]): string[] {
196
- const files = new Set<string>();
197
- for (const m of tools) {
198
- if (!m.toolName) continue;
199
- const name = m.toolName.toLowerCase();
200
- if (name === "write" || name === "edit" || name === "notebookedit") {
201
- // Extract file path from input payload
202
- const input = m.input ?? m.text;
203
- const pathMatch = input.match(/["']?(\/[^\s"']+\.\w+)["']?/);
204
- if (pathMatch) files.add(pathMatch[1]);
205
- }
206
- if (name === "bash") {
207
- const cmd = m.input ?? m.text;
208
- if (cmd.includes("git add") || cmd.includes("git commit") || cmd.includes("git diff")) {
209
- for (const p of extractFilePaths(cmd)) files.add(p);
210
- }
211
- }
212
- }
213
- return [...files].slice(0, MAX_FILES);
214
- }
215
-
216
230
  // ---- Public API ------------------------------------------------------------
217
231
 
218
232
  /**
@@ -243,29 +257,10 @@ export function extractiveSummarize(messages: EngineMessage[]): ExtractiveSummar
243
257
  const filesModified = extractFilesModified(toolMsgs);
244
258
 
245
259
  const topicSummary = buildTopicSummary(
246
- safe, tools, recentUser, currentWork, keyFiles, pending,
260
+ safe, tools, recentUser, currentWork, keyFiles, pending, filesModified, keyDecisions,
247
261
  );
248
262
 
249
263
  const tokenEstimate = estimateBlockTokens(topicSummary);
250
264
 
251
265
  return { topicSummary, keyDecisions, nextSteps: pending, filesModified, tokenEstimate };
252
266
  }
253
-
254
- // ---- Key files (existing logic from compact.ts, moved here) ----------------
255
-
256
- const MAX_KEY_FILES = 5;
257
- const FRESHNESS_WINDOW = 10;
258
-
259
- function collectKeyFiles(messages: EngineMessage[]): string[] {
260
- const recent = messages.slice(-FRESHNESS_WINDOW);
261
- const pathFreq = new Map<string, number>();
262
- for (const m of recent) {
263
- for (const p of extractFilePaths(m.text)) {
264
- pathFreq.set(p, (pathFreq.get(p) ?? 0) + 1);
265
- }
266
- }
267
- return [...pathFreq.entries()]
268
- .sort((a, b) => b[1] - a[1])
269
- .slice(0, MAX_KEY_FILES)
270
- .map(([p]) => p);
271
- }
@@ -91,6 +91,10 @@ export function computeDedupTierRollup(
91
91
  if (ev.type !== "dedup_audit") continue;
92
92
  const ts = Date.parse(ev.ts);
93
93
  if (Number.isNaN(ts) || ts < windowStart || ts > windowEnd) continue;
94
+ // "skipped" (degenerate-match guard declined a collapse) is a decision, but
95
+ // NOT a tier catch — it is attributed to no tier below. Excluding it from the
96
+ // denominator keeps l0Share + l1Share + l2Share summing to 1 over the window.
97
+ if (ev.status === "skipped") continue;
94
98
  total += 1;
95
99
  switch (ev.tier) {
96
100
  case "L0":
@@ -0,0 +1,64 @@
1
+ /**
2
+ * add-degenerate.ts — the L1/L2 degenerate-match decliner bound to one add().
3
+ *
4
+ * Split out of add.ts to keep it under the 300-line src/ soft limit (the same
5
+ * sibling-helper pattern as add-l0.ts / add-l1.ts). The PREDICATE lives in
6
+ * src/dedup/degenerate.ts alongside the other dedup primitives; what lives here
7
+ * is only the cascade-side glue: the telemetry the decision emits and the
8
+ * boolean the two call sites branch on.
9
+ *
10
+ * See src/dedup/degenerate.ts for the incident (2026-08-19) this guards against.
11
+ */
12
+ import type { StoredCheckpoint } from "../store.js";
13
+ import { shouldSkipDegenerateMatch } from "../dedup/degenerate.js";
14
+ import type { DegenerateGuardTunables } from "../dedup/degenerate.js";
15
+ import type { DedupAuditRecorder } from "./dedup-audit.js";
16
+ import type { VectorStore } from "./class.js";
17
+ import type { AddInput } from "./types.js";
18
+
19
+ /** Declines one fuzzy-tier match; true ⇒ treat the match as if it never happened. */
20
+ export type DegenerateDecliner = (
21
+ tier: "L1" | "L2",
22
+ matched: StoredCheckpoint,
23
+ similarity?: number,
24
+ ) => boolean;
25
+
26
+ /**
27
+ * Build the decliner for one add() cascade.
28
+ *
29
+ * Returns a predicate the L1/L2 call sites use as a one-line guard. When it
30
+ * returns true it has ALREADY recorded the declined decision (monitoring event +
31
+ * `skipped` audit line + the live `onTier` detail), so the caller only has to
32
+ * fall through. When the umbrella flag is off it always returns false and emits
33
+ * nothing — byte-identical to the pre-guard cascade.
34
+ */
35
+ export function degenerateDecliner(args: {
36
+ store: VectorStore;
37
+ input: AddInput;
38
+ /** This candidate's content hash — blocks declining an exact-content match. */
39
+ contentHash: string;
40
+ cfg: DegenerateGuardTunables;
41
+ audit: DedupAuditRecorder;
42
+ /** Cascade start time, for the latency field on the monitoring event. */
43
+ t0: number;
44
+ }): DegenerateDecliner {
45
+ const { store, input, contentHash, cfg, audit, t0 } = args;
46
+ const candidate = { ...input, contentHash };
47
+ return (tier, matched, similarity) => {
48
+ if (!shouldSkipDegenerateMatch(matched, candidate, cfg)) return false;
49
+ // Reported as `mark_only`: a tier matched but policy declined to collapse —
50
+ // exactly the existing MARK_ONLY shape, with a distinct reason string so the
51
+ // dashboard can tell a guard decline from an operator-configured MARK_ONLY.
52
+ store.record(
53
+ tier,
54
+ "mark_only",
55
+ "degenerateGuard",
56
+ Date.now() - t0,
57
+ similarity,
58
+ matched.checkpointId,
59
+ );
60
+ audit.skipped(tier, matched.checkpointId, "degenerateGuard", similarity);
61
+ input.onTier?.({ tier, status: "passed", detail: "degenerateGuard" });
62
+ return true;
63
+ };
64
+ }
@@ -38,6 +38,7 @@ import {
38
38
  bumpDedupStats,
39
39
  } from "../store/sqlite.js";
40
40
  import { computeRegionHash } from "./hash.js";
41
+ import { degenerateDecliner } from "./add-degenerate.js";
41
42
  import { runL0Tier, computeSummaryHash } from "./add-l0.js";
42
43
  import { findL1Duplicate } from "./add-l1.js";
43
44
  import { dedupAuditRecorder } from "./dedup-audit.js";
@@ -86,9 +87,23 @@ export function addCheckpoint(store: VectorStore, input: AddInput): AddResult {
86
87
  // Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
87
88
  // and which tier.
88
89
  let markOnly: "L0" | "L1" | "L2" | null = null;
90
+ // Content digest for this candidate (also consumed by the L0 tier below).
91
+ const digest = computeContentDigest(input.regionText);
92
+ // Degenerate-match guard (incident 2026-08-19): declines a fuzzy-tier collapse
93
+ // onto a content-free skeleton when the incoming region is richer, so the
94
+ // skeleton stops absorbing every future compaction. Returns true (having
95
+ // already recorded the decision) ⇒ the caller treats the match as a non-match.
96
+ // Flag-off ⇒ always false ⇒ byte-identical predecessor. See add-degenerate.ts.
97
+ const declineDegenerate = degenerateDecliner({
98
+ store,
99
+ input,
100
+ contentHash: digest.contentHash,
101
+ cfg,
102
+ audit,
103
+ t0,
104
+ });
89
105
 
90
106
  // L0 exact-match tier (contentHash / regionHash / summaryHash) — see add-l0.ts.
91
- const digest = computeContentDigest(input.regionText);
92
107
  const bloom = openBloom(store.stateDir);
93
108
  const summaryHash = computeSummaryHash(input.topicSummary);
94
109
  const l0 = runL0Tier({
@@ -113,7 +128,10 @@ export function addCheckpoint(store: VectorStore, input: AddInput): AddResult {
113
128
  onTier?.({ tier: "L1", status: "scanning" });
114
129
  if (cfg.L1_ENABLED) {
115
130
  const l1 = findL1Duplicate(store, sessionId, input.regionText, all);
116
- if (l1 && !cfg.MARK_ONLY_L1) {
131
+ // Guard first: a declined match must not collapse and must not be recorded
132
+ // as a MARK_ONLY hit either — it is a non-match for the rest of the cascade.
133
+ const l1Declined = l1 !== undefined && declineDegenerate("L1", l1);
134
+ if (l1 && !l1Declined && !cfg.MARK_ONLY_L1) {
117
135
  l1.timestamp = input.timestamp;
118
136
  upsertCheckpoint(l1, store.stateDir);
119
137
  bumpDedupStats(true, store.stateDir);
@@ -124,7 +142,7 @@ export function addCheckpoint(store: VectorStore, input: AddInput): AddResult {
124
142
  onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
125
143
  return r;
126
144
  }
127
- if (l1 && cfg.MARK_ONLY_L1) markOnly = "L1";
145
+ if (l1 && !l1Declined && cfg.MARK_ONLY_L1) markOnly = "L1";
128
146
  }
129
147
  onTier?.({ tier: "L1", status: "passed" });
130
148
 
@@ -152,7 +170,13 @@ export function addCheckpoint(store: VectorStore, input: AddInput): AddResult {
152
170
  },
153
171
  { checkpoint: all[0], sim: -1 },
154
172
  );
155
- if (!timedOut && nearest.sim >= simThreshold) {
173
+ // A declined match falls through to the "store a fresh checkpoint" path; the
174
+ // guard already audited the decision, so the near-miss emit below is skipped.
175
+ const l2Declined =
176
+ !timedOut &&
177
+ nearest.sim >= simThreshold &&
178
+ declineDegenerate("L2", nearest.checkpoint, nearest.sim);
179
+ if (!timedOut && !l2Declined && nearest.sim >= simThreshold) {
156
180
  if (!cfg.MARK_ONLY_L2) {
157
181
  // Near-identical — update timestamp on existing checkpoint
158
182
  nearest.checkpoint.timestamp = input.timestamp;
@@ -190,7 +214,7 @@ export function addCheckpoint(store: VectorStore, input: AddInput): AddResult {
190
214
  // Near-miss: how close did we come to collapsing? Only emitted when the
191
215
  // scan actually completed and scored a candidate — a timed-out scan has no
192
216
  // honest best to report.
193
- if (!timedOut && nearest.sim >= 0) {
217
+ if (!timedOut && !l2Declined && nearest.sim >= 0) {
194
218
  audit.passed("L2", nearest.checkpoint.checkpointId, nearest.sim);
195
219
  }
196
220
  }
@@ -51,8 +51,8 @@ export interface DedupAuditEvent {
51
51
  sessionId: string;
52
52
  /** Which layer produced the decision ("new" = nothing collapsed). */
53
53
  tier: "L0" | "L1" | "L2" | "new";
54
- /** What the layer decided. */
55
- status: "deduped" | "passed" | "stored";
54
+ /** What the layer decided. `skipped` = matched but policy declined to collapse. */
55
+ status: "deduped" | "passed" | "stored" | "skipped";
56
56
  /** Checkpoint the region collapsed onto, or the nearest one scored. */
57
57
  matchedEntry?: string;
58
58
  /** Checkpoint created, when the outcome was a new write. */
@@ -129,6 +129,20 @@ export interface DedupAuditRecorder {
129
129
  matchedEntry: string,
130
130
  similarity: number,
131
131
  ): void;
132
+ /**
133
+ * A tier MATCHED but policy declined the collapse (degenerate-match guard).
134
+ *
135
+ * Distinct from `passed`: the threshold WAS cleared, so this line is how an
136
+ * operator sees that a skeleton checkpoint was prevented from absorbing richer
137
+ * content — the signal for "the store is healing". `similarity` is carried only
138
+ * where the tier scored one (L2 does, L1 does not).
139
+ */
140
+ skipped(
141
+ tier: "L0" | "L1" | "L2",
142
+ matchedEntry: string,
143
+ dedupReason: string,
144
+ similarity?: number,
145
+ ): void;
132
146
  /** Final outcome: nothing collapsed, a new checkpoint was written. */
133
147
  stored(storedEntry: string, dedupReason: string, tokenEstimate: number): void;
134
148
  }
@@ -161,6 +175,15 @@ export function dedupAuditRecorder(
161
175
  matchedEntry,
162
176
  similarity,
163
177
  }),
178
+ skipped: (tier, matchedEntry, dedupReason, similarity) =>
179
+ emitDedupAudit(ctx, {
180
+ ...base,
181
+ tier,
182
+ status: "skipped",
183
+ matchedEntry,
184
+ dedupReason,
185
+ ...(similarity === undefined ? {} : { similarity }),
186
+ }),
164
187
  stored: (storedEntry, dedupReason, tokenEstimate) =>
165
188
  emitDedupAudit(ctx, {
166
189
  ...base,