pi-mega-compact 0.21.8 → 0.21.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dedup/degenerate.js +68 -0
- package/dist/extensions/dashboard-server/routes-dedup-attribution.js +8 -1
- package/dist/extensions/dashboard-server/routes-rag-settings-compaction.js +42 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +8 -8
- package/dist/extensions/mega-config.js +12 -0
- package/dist/extensions/mega-events/context-handler/gateCheck.js +51 -1
- package/dist/extensions/mega-events/context-handler/headroom.js +128 -0
- package/dist/extensions/mega-events/context-handler/liveTrim.js +37 -51
- package/dist/extensions/mega-events/context-handler/pipelineRun.js +14 -1
- package/dist/extensions/mega-events/context-handler.js +26 -3
- package/dist/extensions/mega-pipeline/compact/run.js +14 -3
- package/dist/extensions/mega-runtime/dashboard-snapshot.js +1 -0
- package/dist/extensions/mega-runtime/runtime-instrumentation.js +1 -0
- package/dist/extensions/mega-runtime/runtime-snapshot.js +1 -0
- package/dist/src/config/dedup.js +3 -0
- package/dist/src/dedup/degenerate.js +68 -0
- package/dist/src/extractive-salvage.js +195 -0
- package/dist/src/extractive.js +63 -72
- package/dist/src/vector-cortex/dedup-attr/rollup.js +5 -0
- package/dist/src/vectorStore/add-degenerate.js +25 -0
- package/dist/src/vectorStore/add.js +28 -5
- package/dist/src/vectorStore/dedup-audit.js +8 -0
- package/dist/vector-cortex/dedup-attr/rollup.js +5 -0
- package/dist/vectorStore/dedup-audit.js +8 -0
- package/extensions/dashboard-server/api-contracts/endpoints/types.ts +2 -0
- package/extensions/dashboard-server/routes-dedup-attribution.ts +11 -1
- package/extensions/dashboard-server/routes-rag-settings-compaction.ts +91 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +13 -27
- package/extensions/mega-config-types.ts +25 -0
- package/extensions/mega-config.ts +12 -0
- package/extensions/mega-dashboard.ts +4 -1
- package/extensions/mega-events/context-handler/gateCheck.ts +67 -0
- package/extensions/mega-events/context-handler/headroom.ts +190 -0
- package/extensions/mega-events/context-handler/liveTrim.ts +37 -57
- package/extensions/mega-events/context-handler/pipelineRun.ts +14 -1
- package/extensions/mega-events/context-handler.ts +26 -3
- package/extensions/mega-pipeline/compact/run.ts +14 -4
- package/extensions/mega-runtime/dashboard-snapshot.ts +3 -0
- package/extensions/mega-runtime/runtime-instrumentation.ts +4 -0
- package/extensions/mega-runtime/runtime-snapshot.ts +2 -0
- package/package.json +1 -1
- package/src/config/dedup.ts +15 -0
- package/src/dedup/degenerate.ts +125 -0
- package/src/extractive-salvage.ts +212 -0
- package/src/extractive.ts +70 -75
- package/src/vector-cortex/dedup-attr/rollup.ts +4 -0
- package/src/vectorStore/add-degenerate.ts +64 -0
- package/src/vectorStore/add.ts +29 -5
- package/src/vectorStore/dedup-audit.ts +25 -2
|
@@ -128,6 +128,7 @@ export function buildDashboardSnapshot(ctx) {
|
|
|
128
128
|
ctxFastGate: ctx.diagCtxFastGate,
|
|
129
129
|
liveTrimFires: ctx.diagLiveTrimFires,
|
|
130
130
|
liveTrimReplays: ctx.diagLiveTrimReplays,
|
|
131
|
+
headroomTrips: ctx.diagCtxHeadroomTrip,
|
|
131
132
|
},
|
|
132
133
|
retries: {
|
|
133
134
|
errorRetryCount: ctx.errorRetryCount,
|
|
@@ -23,6 +23,7 @@ export class RuntimeInstrumentation {
|
|
|
23
23
|
diagCtxCutNull = 0; // computeLiveTrimCut returned null (anchor/boundary)
|
|
24
24
|
diagCtxThrown = 0; // live-trim try threw (caught)
|
|
25
25
|
diagCtxOutputErrorTrip = 0; // Phase H: output-error catch tripped a forced compaction
|
|
26
|
+
diagCtxHeadroomTrip = 0; // v0.21.9: output-headroom gate tripped a pre-overflow compaction
|
|
26
27
|
// Context health instrumentation (v0.12): rolling ring buffers for
|
|
27
28
|
// drift detection + cache poison Layer 1 hash baseline.
|
|
28
29
|
recentTurnEmbeddings = [];
|
|
@@ -94,6 +94,7 @@ export function snapshotImpl(self, ctx) {
|
|
|
94
94
|
diagCtxFastGate: self.diagCtxFastGate,
|
|
95
95
|
diagLiveTrimFires: self.diagLiveTrimFires,
|
|
96
96
|
diagLiveTrimReplays: self.diagLiveTrimReplays,
|
|
97
|
+
diagCtxHeadroomTrip: self.diagCtxHeadroomTrip,
|
|
97
98
|
errorRetryCount: self.rt.errorRetryCount,
|
|
98
99
|
consecutiveErrors: self.rt.consecutiveErrors,
|
|
99
100
|
ERROR_RETRY_MAX_CONSECUTIVE: self.config.maxConsecutiveErrors,
|
package/dist/src/config/dedup.js
CHANGED
|
@@ -63,6 +63,9 @@ export function loadDedupConfig() {
|
|
|
63
63
|
L2_COSINE_CODE: envNumOrNull("MEGACOMPACT_L2_THRESHOLD_CODE"),
|
|
64
64
|
L2_COSINE_PROSE: envNumOrNull("MEGACOMPACT_L2_THRESHOLD_PROSE"),
|
|
65
65
|
L1_JACCARD: envNum("MEGACOMPACT_L1_JACCARD", 0.8),
|
|
66
|
+
DEDUP_DEGENERATE_GUARD: envBool("MEGACOMPACT_DEDUP_DEGENERATE_GUARD", true),
|
|
67
|
+
DEDUP_DEGEN_MIN_TOKENS: envNum("MEGACOMPACT_DEDUP_DEGEN_MIN_TOKENS", 48),
|
|
68
|
+
DEDUP_DEGEN_MIN_PCT: envNum("MEGACOMPACT_DEDUP_DEGEN_MIN_PCT", 0.005),
|
|
66
69
|
DEDUP_SIM: envNum("MEGACOMPACT_DEDUP_SIM", 0.9),
|
|
67
70
|
MMR_LAMBDA: envNum("MEGACOMPACT_MMR_LAMBDA", 0.5),
|
|
68
71
|
SEMDEDUP_COSINE: envNum("MEGACOMPACT_SEMDEDUP_COSINE", 0.95),
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The effective token floor for a checkpoint: the larger of the absolute floor
|
|
3
|
+
* and `MIN_PCT × originalTokenEstimate`.
|
|
4
|
+
*
|
|
5
|
+
* A missing / zero / non-finite `originalTokenEstimate` contributes nothing, so
|
|
6
|
+
* the absolute floor applies alone — direct add() callers and pre-v0.4 rows that
|
|
7
|
+
* never recorded the original region size are judged on absolute size only,
|
|
8
|
+
* never accidentally deemed degenerate by a 0-valued percentage term.
|
|
9
|
+
*/
|
|
10
|
+
export function degenerateFloor(subject, tunables) {
|
|
11
|
+
const orig = subject.originalTokenEstimate;
|
|
12
|
+
const relative = typeof orig === "number" && Number.isFinite(orig) && orig > 0
|
|
13
|
+
? orig * tunables.DEDUP_DEGEN_MIN_PCT
|
|
14
|
+
: 0;
|
|
15
|
+
return Math.max(tunables.DEDUP_DEGEN_MIN_TOKENS, relative);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Is this stored checkpoint a degenerate (content-free) summary?
|
|
19
|
+
*
|
|
20
|
+
* Calibration against the incident data:
|
|
21
|
+
* - skeleton: tokenEstimate 34, original ≈19166 → 34 < max(48, 95.8) → TRUE
|
|
22
|
+
* - normal: tokenEstimate 2000, original 70000 → 2000 > max(48, 350) → FALSE
|
|
23
|
+
*
|
|
24
|
+
* The relative term is what makes this scale: a 34-token summary of a 900-token
|
|
25
|
+
* region is a legitimate 26× compression, while the same 34 tokens standing in
|
|
26
|
+
* for 19k is a skeleton.
|
|
27
|
+
*/
|
|
28
|
+
export function isDegenerateCheckpoint(subject, tunables) {
|
|
29
|
+
const tokens = subject.tokenEstimate ?? 0;
|
|
30
|
+
return tokens < degenerateFloor(subject, tunables);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Should an L1/L2 match be DECLINED because it would collapse richer incoming
|
|
34
|
+
* content onto a degenerate stored checkpoint?
|
|
35
|
+
*
|
|
36
|
+
* Returns true only when all four hold:
|
|
37
|
+
* 1. the umbrella flag is ON,
|
|
38
|
+
* 2. the matched (stored) checkpoint is degenerate,
|
|
39
|
+
* 3. the candidate is strictly richer than the match,
|
|
40
|
+
* 4. the candidate's content is not byte-identical to the match's.
|
|
41
|
+
*
|
|
42
|
+
* Condition 3 uses a strict `>`: equal-size skeletons collapsing is harmless and
|
|
43
|
+
* keeps the store from growing one row per compaction while the summarizer is
|
|
44
|
+
* broken. Only a genuine improvement is worth declining a collapse for.
|
|
45
|
+
*
|
|
46
|
+
* Condition 4 is a CORRECTNESS requirement, not a refinement. `context_chunks`
|
|
47
|
+
* carries a partial UNIQUE index on (session_id, content_hash) (schema/core.ts
|
|
48
|
+
* QA #1), so declining a match whose content hash already exists would fall
|
|
49
|
+
* through to an INSERT that throws — inside add(), which sits on the agent loop.
|
|
50
|
+
* It is also the semantically right call: identical bytes are the SAME region,
|
|
51
|
+
* so re-storing them adds no information and heals nothing. Only L0 may own the
|
|
52
|
+
* exact-match case; the guard exists for fuzzy matches on genuinely different
|
|
53
|
+
* text, which is exactly the incident's shape (each compaction produced a
|
|
54
|
+
* *similar but distinct* skeleton).
|
|
55
|
+
*/
|
|
56
|
+
export function shouldSkipDegenerateMatch(matched, candidate, tunables) {
|
|
57
|
+
if (!tunables.DEDUP_DEGENERATE_GUARD)
|
|
58
|
+
return false;
|
|
59
|
+
if (!isDegenerateCheckpoint(matched, tunables))
|
|
60
|
+
return false;
|
|
61
|
+
if ((candidate.tokenEstimate ?? 0) <= (matched.tokenEstimate ?? 0))
|
|
62
|
+
return false;
|
|
63
|
+
// Byte-identical content → not a healing opportunity (and would violate the
|
|
64
|
+
// UNIQUE index). Compared only when both hashes are known.
|
|
65
|
+
const a = candidate.contentHash;
|
|
66
|
+
const b = matched.contentHash;
|
|
67
|
+
return !(a !== undefined && b !== undefined && a === b);
|
|
68
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* extractive-salvage.ts — file-path policy + skeleton salvage for extractive.ts.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from extractive.ts (A1/A2 sprint) to keep that file under the
|
|
5
|
+
* 300-line src/ soft limit. Pure functions only — no I/O, no logging.
|
|
6
|
+
*
|
|
7
|
+
* DESIGN (A1): blocklist, not allowlist. The old `INTERESTING_EXT` allowlist
|
|
8
|
+
* (rs/ts/tsx/js/json/md) silently produced content-free summaries for every
|
|
9
|
+
* other language — a .go/.py/.c project got a 34-token skeleton. An allowlist
|
|
10
|
+
* has to be *right* about ~40 ecosystems to be useful and fails closed (drops
|
|
11
|
+
* real work) when it is wrong; a NOISE blocklist only has to be right about the
|
|
12
|
+
* small, stable set of binary/generated/asset extensions and fails open (an
|
|
13
|
+
* unknown extension is surfaced, which is the safe direction for a summary).
|
|
14
|
+
*/
|
|
15
|
+
// ---- File path policy ------------------------------------------------------
|
|
16
|
+
/**
|
|
17
|
+
* Generic extension capture: any 1–6 char ALPHABETIC extension. Path
|
|
18
|
+
* character class is unchanged from the original FILE_PATH_RE so existing
|
|
19
|
+
* matching behaviour (quotes/backticks/whitespace as delimiters) is preserved.
|
|
20
|
+
* Alphabetic-only is deliberate: every real source/config extension is alpha
|
|
21
|
+
* (c..tsx, tsconfig.json), while an alphanumeric class matches version strings
|
|
22
|
+
* ("GLM-4.7", "v0.21.9", "Node 18.2") as "files" and spams Key files.
|
|
23
|
+
*/
|
|
24
|
+
export const FILE_PATH_RE = /(?:^|\s)([^\s"`']+\.([A-Za-z]{1,6}))\b/g;
|
|
25
|
+
/** Same policy, single-match, for inferCurrentWork (also excludes ':'). */
|
|
26
|
+
export const CURRENT_WORK_PATH_RE = /(?:^|\s)([^\s"`':]+\.([A-Za-z]{1,6}))\b/m;
|
|
27
|
+
/**
|
|
28
|
+
* Binary / generated / asset / vendored extensions that carry no summary value.
|
|
29
|
+
* Deliberately small and stable. Note `md`, `json`, `toml`, `yaml`, `sql`, `css`
|
|
30
|
+
* and `html` are NOT noise — they are hand-edited source in most repos.
|
|
31
|
+
*/
|
|
32
|
+
export const NOISE_EXT = new Set([
|
|
33
|
+
// lockfiles & logs
|
|
34
|
+
"lock", "log", "sum",
|
|
35
|
+
// images & media
|
|
36
|
+
"png", "jpg", "jpeg", "gif", "webp", "svg", "ico", "bmp", "tiff",
|
|
37
|
+
"mp3", "mp4", "mov", "wav", "webm", "avi",
|
|
38
|
+
// fonts
|
|
39
|
+
"woff", "woff2", "ttf", "otf", "eot",
|
|
40
|
+
// archives & binaries
|
|
41
|
+
"zip", "gz", "tgz", "bz2", "xz", "7z", "rar", "tar",
|
|
42
|
+
"exe", "dll", "so", "dylib", "bin", "o", "a", "obj", "class", "pyc", "pyo",
|
|
43
|
+
"wasm", "node", "jar", "war", "deb", "rpm", "dmg", "iso", "img",
|
|
44
|
+
// generated / build artifacts
|
|
45
|
+
"map", "min", "lockb", "snap", "cache", "tmp", "temp", "swp", "bak", "orig",
|
|
46
|
+
// data blobs & databases
|
|
47
|
+
"pdf", "db", "sqlite", "sqlite3", "pack", "idx", "pem", "key", "crt",
|
|
48
|
+
]);
|
|
49
|
+
/** Directory fragments whose files are never "key files" for a summary. */
|
|
50
|
+
const NOISE_DIR_RE = /(?:^|\/)(?:node_modules|\.git|dist|build|coverage|vendor|target|__pycache__|\.venv|venv)(?:\/|$)/;
|
|
51
|
+
/**
|
|
52
|
+
* TLD-shaped extensions: `example.com`, `github.com`, `npm.cmd` are domains and
|
|
53
|
+
* launchers, not code. (QA lens 1 finding, 2026-08-19.) Conservative list only —
|
|
54
|
+
* no real source extension lives here (.go/.rs/.ts stay interesting).
|
|
55
|
+
*/
|
|
56
|
+
const TLD_EXT = new Set([
|
|
57
|
+
"com", "org", "net", "io", "gov", "edu", "biz", "info", "dev", "app",
|
|
58
|
+
"page", "xyz", "site", "online", "cloud", "me", "co", "us", "uk", "de",
|
|
59
|
+
"fr", "jp", "cn", "nl", "se", "eu", "int", "mil", "cmd",
|
|
60
|
+
]);
|
|
61
|
+
/** True when a matched path is worth surfacing in a summary. */
|
|
62
|
+
export function isInterestingPath(filePath, ext) {
|
|
63
|
+
const lowerExt = ext.toLowerCase();
|
|
64
|
+
if (NOISE_EXT.has(lowerExt))
|
|
65
|
+
return false;
|
|
66
|
+
if (TLD_EXT.has(lowerExt))
|
|
67
|
+
return false; // bare domains, not code
|
|
68
|
+
if (/^(?:https?|ftp):\/\//i.test(filePath) || filePath.toLowerCase().startsWith("www."))
|
|
69
|
+
return false;
|
|
70
|
+
if (NOISE_DIR_RE.test(filePath))
|
|
71
|
+
return false;
|
|
72
|
+
// `app.min.js` / `bundle.min.css` style double extensions.
|
|
73
|
+
if (/\.min\.[A-Za-z0-9]{1,6}$/.test(filePath))
|
|
74
|
+
return false;
|
|
75
|
+
if (/\.map$/.test(filePath))
|
|
76
|
+
return false;
|
|
77
|
+
// Prose abbreviations ("e.g", "i.e", "U.S"): a single-char base with no slash
|
|
78
|
+
// and no digit is never a filename. ("a.c" is rare collateral; "q1.py" and
|
|
79
|
+
// "src/a.ts" survive — digit / slash both exempt.)
|
|
80
|
+
const base = filePath.slice(0, filePath.lastIndexOf("."));
|
|
81
|
+
if (!filePath.includes("/") && !/\d/.test(filePath) && base.length <= 1)
|
|
82
|
+
return false;
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
// ---- Path collection -------------------------------------------------------
|
|
86
|
+
const MAX_KEY_FILES = 5;
|
|
87
|
+
const MAX_FILES = 10;
|
|
88
|
+
const FRESHNESS_WINDOW = 10;
|
|
89
|
+
/** All interesting file paths mentioned in a blob of text. */
|
|
90
|
+
export function extractFilePaths(text) {
|
|
91
|
+
const paths = [];
|
|
92
|
+
for (const m of text.matchAll(FILE_PATH_RE)) {
|
|
93
|
+
if (isInterestingPath(m[1], m[2]))
|
|
94
|
+
paths.push(m[1]);
|
|
95
|
+
}
|
|
96
|
+
return paths;
|
|
97
|
+
}
|
|
98
|
+
/** Most-mentioned paths within the recency window (behaviour unchanged). */
|
|
99
|
+
export function collectKeyFiles(messages) {
|
|
100
|
+
const recent = messages.slice(-FRESHNESS_WINDOW);
|
|
101
|
+
const pathFreq = new Map();
|
|
102
|
+
for (const m of recent) {
|
|
103
|
+
for (const p of extractFilePaths(m.text)) {
|
|
104
|
+
pathFreq.set(p, (pathFreq.get(p) ?? 0) + 1);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return [...pathFreq.entries()]
|
|
108
|
+
.sort((a, b) => b[1] - a[1])
|
|
109
|
+
.slice(0, MAX_KEY_FILES)
|
|
110
|
+
.map(([p]) => p);
|
|
111
|
+
}
|
|
112
|
+
/** Paths written/edited by tools (extension-agnostic; behaviour unchanged). */
|
|
113
|
+
export function extractFilesModified(tools) {
|
|
114
|
+
const files = new Set();
|
|
115
|
+
for (const m of tools) {
|
|
116
|
+
if (!m.toolName)
|
|
117
|
+
continue;
|
|
118
|
+
const name = m.toolName.toLowerCase();
|
|
119
|
+
if (name === "write" || name === "edit" || name === "notebookedit") {
|
|
120
|
+
const input = m.input ?? m.text;
|
|
121
|
+
const pathMatch = input.match(/["']?(\/[^\s"']+\.\w+)["']?/);
|
|
122
|
+
if (pathMatch)
|
|
123
|
+
files.add(pathMatch[1]);
|
|
124
|
+
}
|
|
125
|
+
if (name === "bash") {
|
|
126
|
+
const cmd = m.input ?? m.text;
|
|
127
|
+
if (cmd.includes("git add") || cmd.includes("git commit") || cmd.includes("git diff")) {
|
|
128
|
+
for (const p of extractFilePaths(cmd))
|
|
129
|
+
files.add(p);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return [...files].slice(0, MAX_FILES);
|
|
134
|
+
}
|
|
135
|
+
// ---- Placeholder user turns (A2b) ------------------------------------------
|
|
136
|
+
/**
|
|
137
|
+
* Content-free user turns. Extremely common in resumed sessions; taking them
|
|
138
|
+
* verbatim as "User requests" is what produced the three "• resume" bullets.
|
|
139
|
+
*/
|
|
140
|
+
const PLACEHOLDER_RE = /^(?:resume|continue|go on|go ahead|proceed|next|yes|yeah|yep|y|ok|okay|k|sure|thanks|thank you|ty|done|please continue|carry on)\W*$/i;
|
|
141
|
+
export function isPlaceholderRequest(text) {
|
|
142
|
+
return PLACEHOLDER_RE.test(text.trim());
|
|
143
|
+
}
|
|
144
|
+
// ---- Skeleton salvage (A2c) ------------------------------------------------
|
|
145
|
+
const MAX_SALVAGE_LINES = 5;
|
|
146
|
+
const SALVAGE_LINE_LEN = 120;
|
|
147
|
+
/**
|
|
148
|
+
* A "skeleton" summary is the scope line and nothing else — no files, no current
|
|
149
|
+
* work, no decisions, no pending items, and no *substantive* user request. That
|
|
150
|
+
* is ~34 tokens of zero information and is what breaks a resumed session.
|
|
151
|
+
*
|
|
152
|
+
* NOTE: `recentUser` is treated as empty when it holds only placeholders. The
|
|
153
|
+
* incident summary DID have three "• resume" bullets, so a plain
|
|
154
|
+
* `recentUser.length === 0` test would never have fired on the very case this
|
|
155
|
+
* salvage exists for.
|
|
156
|
+
*/
|
|
157
|
+
export function isSkeletonSummary(parts) {
|
|
158
|
+
const hasRealRequest = parts.recentUser.some((r) => !isPlaceholderRequest(r));
|
|
159
|
+
return (!hasRealRequest &&
|
|
160
|
+
parts.keyFiles.length === 0 &&
|
|
161
|
+
!parts.currentWork &&
|
|
162
|
+
parts.decisions.length === 0 &&
|
|
163
|
+
parts.pending.length === 0);
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Last-resort content: the first meaningful line of the most recent assistant
|
|
167
|
+
* and tool messages. Deterministic (pure scan, newest-first, then re-ordered
|
|
168
|
+
* oldest-first for reading). Returns [] when there is genuinely nothing.
|
|
169
|
+
*/
|
|
170
|
+
export function buildSalvageDigest(messages) {
|
|
171
|
+
const out = [];
|
|
172
|
+
const seen = new Set();
|
|
173
|
+
for (let i = messages.length - 1; i >= 0 && out.length < MAX_SALVAGE_LINES; i--) {
|
|
174
|
+
const m = messages[i];
|
|
175
|
+
if (m.role !== "assistant" && m.role !== "tool")
|
|
176
|
+
continue;
|
|
177
|
+
const raw = m.text || m.output || m.input || "";
|
|
178
|
+
const line = raw
|
|
179
|
+
.split("\n")
|
|
180
|
+
.map((l) => l.trim())
|
|
181
|
+
.find((l) => l.length > 0);
|
|
182
|
+
if (!line)
|
|
183
|
+
continue;
|
|
184
|
+
const label = m.role === "tool" ? `${m.toolName ?? "tool"}: ` : "";
|
|
185
|
+
const entry = truncateLine(`${label}${line}`, SALVAGE_LINE_LEN);
|
|
186
|
+
if (seen.has(entry))
|
|
187
|
+
continue;
|
|
188
|
+
seen.add(entry);
|
|
189
|
+
out.push(entry);
|
|
190
|
+
}
|
|
191
|
+
return out.reverse();
|
|
192
|
+
}
|
|
193
|
+
function truncateLine(s, maxLen) {
|
|
194
|
+
return s.length <= maxLen ? s : s.slice(0, maxLen - 1) + "…";
|
|
195
|
+
}
|
package/dist/src/extractive.js
CHANGED
|
@@ -8,12 +8,14 @@
|
|
|
8
8
|
* Deterministic: same messages → same output, every time.
|
|
9
9
|
*/
|
|
10
10
|
import { estimateBlockTokens } from "./tokens.js";
|
|
11
|
+
import { CURRENT_WORK_PATH_RE, isInterestingPath, isPlaceholderRequest, isSkeletonSummary, buildSalvageDigest, collectKeyFiles, extractFilesModified, } from "./extractive-salvage.js";
|
|
11
12
|
// ---- Limits ----------------------------------------------------------------
|
|
12
13
|
const MAX_RECENT_USER = 3;
|
|
13
14
|
const MAX_DECISIONS = 5;
|
|
14
|
-
const MAX_FILES = 10;
|
|
15
15
|
const MAX_PENDING = 5;
|
|
16
16
|
const MAX_TOPIC_LINES = 12;
|
|
17
|
+
/** Cap for the merged keyFiles ∪ filesModified "Key files" line (A2a). */
|
|
18
|
+
const MAX_SUMMARY_FILES = 8;
|
|
17
19
|
// ---- Truncation helper -----------------------------------------------------
|
|
18
20
|
function truncate(s, maxLen) {
|
|
19
21
|
if (s.length <= maxLen)
|
|
@@ -27,8 +29,22 @@ function truncate(s, maxLen) {
|
|
|
27
29
|
* Captures: tools used, recent user requests, current work, key files,
|
|
28
30
|
* pending work. Typically 12 lines / ~500 tokens instead of ~70K.
|
|
29
31
|
*/
|
|
30
|
-
function buildTopicSummary(messages, tools, recentUser, currentWork, keyFiles, pending) {
|
|
32
|
+
function buildTopicSummary(messages, tools, recentUser, currentWork, keyFiles, pending, filesModified, decisions) {
|
|
31
33
|
const lines = [];
|
|
34
|
+
// A2a: files captured from write/edit tool inputs are extracted
|
|
35
|
+
// extension-agnostically but never reached the summary. Fold them in so work
|
|
36
|
+
// outside the recency window (and outside the path regex) is still reported.
|
|
37
|
+
// Drop an absolute path when a kept relative path already names the SAME file.
|
|
38
|
+
// Only MULTI-COMPONENT relative paths fold ("engine/mesh.go" absorbs
|
|
39
|
+
// "/proj/engine/mesh.go"); a bare basename ("mesh.go") is never folded into an
|
|
40
|
+
// absolute path, since "/proj/other/x/mesh.go" may be a genuinely different
|
|
41
|
+
// file (QA lens 1 finding: the naive endsWith dropped different directories).
|
|
42
|
+
const combined = [...keyFiles, ...filesModified];
|
|
43
|
+
const allFiles = [...new Set(combined)].filter((p) => {
|
|
44
|
+
if (!p.startsWith("/"))
|
|
45
|
+
return true;
|
|
46
|
+
return !combined.some((r) => r !== p && !r.startsWith("/") && r.includes("/") && p.endsWith("/" + r));
|
|
47
|
+
}).slice(0, MAX_SUMMARY_FILES);
|
|
32
48
|
// Scope line
|
|
33
49
|
const users = messages.filter((m) => m.role === "user");
|
|
34
50
|
const assistants = messages.filter((m) => m.role === "assistant");
|
|
@@ -45,45 +61,59 @@ function buildTopicSummary(messages, tools, recentUser, currentWork, keyFiles, p
|
|
|
45
61
|
// Current work
|
|
46
62
|
if (currentWork)
|
|
47
63
|
lines.push(`Current work: ${currentWork}`);
|
|
48
|
-
// Key files
|
|
49
|
-
if (
|
|
50
|
-
lines.push(`Key files: ${
|
|
64
|
+
// Key files (keyFiles ∪ filesModified)
|
|
65
|
+
if (allFiles.length)
|
|
66
|
+
lines.push(`Key files: ${allFiles.join(", ")}.`);
|
|
51
67
|
// Pending work
|
|
52
68
|
if (pending.length) {
|
|
53
69
|
lines.push("Pending work:");
|
|
54
70
|
for (const p of pending)
|
|
55
71
|
lines.push(` • ${p}`);
|
|
56
72
|
}
|
|
73
|
+
// A2c: a scope-line-only summary carries zero information and strands a
|
|
74
|
+
// resumed session. Salvage the tail of the conversation instead. The line cap
|
|
75
|
+
// is raised ONLY here: the salvage block is bounded at 5 lines + 1 header, and
|
|
76
|
+
// a skeleton by definition contributed just the 1 scope line, so the worst
|
|
77
|
+
// case is 7 lines — still well under the normal 12-line budget.
|
|
78
|
+
const skeleton = isSkeletonSummary({ recentUser, keyFiles: allFiles, currentWork, decisions, pending });
|
|
79
|
+
if (skeleton) {
|
|
80
|
+
const digest = buildSalvageDigest(messages);
|
|
81
|
+
if (digest.length) {
|
|
82
|
+
lines.push("Recent activity:");
|
|
83
|
+
for (const d of digest)
|
|
84
|
+
lines.push(` • ${d}`);
|
|
85
|
+
}
|
|
86
|
+
return lines.join("\n");
|
|
87
|
+
}
|
|
57
88
|
// Cap total length
|
|
58
89
|
return lines.slice(0, MAX_TOPIC_LINES).join("\n");
|
|
59
90
|
}
|
|
60
|
-
// ----
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
const ext = m[2];
|
|
68
|
-
const basename = filePath.split("/").pop() ?? filePath;
|
|
69
|
-
if (basename === "node_modules" || filePath.includes("node_modules/"))
|
|
70
|
-
continue;
|
|
71
|
-
if (INTERESTING_EXT.has(ext))
|
|
72
|
-
paths.push(filePath);
|
|
73
|
-
}
|
|
74
|
-
return paths;
|
|
75
|
-
}
|
|
76
|
-
// ---- Recent user requests (existing logic, kept) ---------------------------
|
|
91
|
+
// ---- Recent user requests --------------------------------------------------
|
|
92
|
+
/**
|
|
93
|
+
* A2b: skip content-free "resume"/"continue" turns and look further back to
|
|
94
|
+
* fill the quota, so a resumed session surfaces its real requests. Falls back
|
|
95
|
+
* to the placeholders when EVERY user turn is one (an honest "• resume" beats
|
|
96
|
+
* an empty section).
|
|
97
|
+
*/
|
|
77
98
|
function collectRecentUserRequests(messages, limit) {
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
99
|
+
const substantive = [];
|
|
100
|
+
const placeholders = [];
|
|
101
|
+
for (let i = messages.length - 1; i >= 0 && substantive.length < limit; i--) {
|
|
102
|
+
if (messages[i].role !== "user")
|
|
103
|
+
continue;
|
|
104
|
+
let snippet = messages[i].text.split("\n").slice(0, 3).join(" ");
|
|
105
|
+
snippet = snippet.replace(/^.+\nProcessed\$?\s*/i, "").replace(/\n/g, " ");
|
|
106
|
+
const cleaned = truncate(snippet, 200);
|
|
107
|
+
if (!cleaned.trim())
|
|
108
|
+
continue;
|
|
109
|
+
if (isPlaceholderRequest(cleaned)) {
|
|
110
|
+
if (placeholders.length < limit)
|
|
111
|
+
placeholders.push(cleaned);
|
|
112
|
+
continue;
|
|
84
113
|
}
|
|
114
|
+
substantive.push(cleaned);
|
|
85
115
|
}
|
|
86
|
-
return
|
|
116
|
+
return (substantive.length ? substantive : placeholders).reverse();
|
|
87
117
|
}
|
|
88
118
|
// ---- Pending work (existing logic, kept) -----------------------------------
|
|
89
119
|
const PENDING_WORDS = ["todo", "next", "pending", "follow up", "remaining"];
|
|
@@ -106,8 +136,9 @@ function inferCurrentWork(messages) {
|
|
|
106
136
|
const m = messages[i];
|
|
107
137
|
if (m.role !== "assistant")
|
|
108
138
|
continue;
|
|
109
|
-
|
|
110
|
-
|
|
139
|
+
// A1: same language-agnostic policy as extractFilePaths.
|
|
140
|
+
const path = m.text.match(CURRENT_WORK_PATH_RE);
|
|
141
|
+
if (path && isInterestingPath(path[1], path[2])) {
|
|
111
142
|
const line = m.text.split("\n").slice(0, 2).join(" ");
|
|
112
143
|
return truncate(line, 200);
|
|
113
144
|
}
|
|
@@ -144,30 +175,6 @@ function extractDecisions(messages) {
|
|
|
144
175
|
}
|
|
145
176
|
return [...new Set(decisions)];
|
|
146
177
|
}
|
|
147
|
-
// ---- Files modified --------------------------------------------------------
|
|
148
|
-
function extractFilesModified(tools) {
|
|
149
|
-
const files = new Set();
|
|
150
|
-
for (const m of tools) {
|
|
151
|
-
if (!m.toolName)
|
|
152
|
-
continue;
|
|
153
|
-
const name = m.toolName.toLowerCase();
|
|
154
|
-
if (name === "write" || name === "edit" || name === "notebookedit") {
|
|
155
|
-
// Extract file path from input payload
|
|
156
|
-
const input = m.input ?? m.text;
|
|
157
|
-
const pathMatch = input.match(/["']?(\/[^\s"']+\.\w+)["']?/);
|
|
158
|
-
if (pathMatch)
|
|
159
|
-
files.add(pathMatch[1]);
|
|
160
|
-
}
|
|
161
|
-
if (name === "bash") {
|
|
162
|
-
const cmd = m.input ?? m.text;
|
|
163
|
-
if (cmd.includes("git add") || cmd.includes("git commit") || cmd.includes("git diff")) {
|
|
164
|
-
for (const p of extractFilePaths(cmd))
|
|
165
|
-
files.add(p);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
return [...files].slice(0, MAX_FILES);
|
|
170
|
-
}
|
|
171
178
|
// ---- Public API ------------------------------------------------------------
|
|
172
179
|
/**
|
|
173
180
|
* Deterministic extractive summary. Same messages → same output, every time.
|
|
@@ -192,23 +199,7 @@ export function extractiveSummarize(messages) {
|
|
|
192
199
|
const pending = inferPendingWork(safe);
|
|
193
200
|
const keyDecisions = extractDecisions(safe);
|
|
194
201
|
const filesModified = extractFilesModified(toolMsgs);
|
|
195
|
-
const topicSummary = buildTopicSummary(safe, tools, recentUser, currentWork, keyFiles, pending);
|
|
202
|
+
const topicSummary = buildTopicSummary(safe, tools, recentUser, currentWork, keyFiles, pending, filesModified, keyDecisions);
|
|
196
203
|
const tokenEstimate = estimateBlockTokens(topicSummary);
|
|
197
204
|
return { topicSummary, keyDecisions, nextSteps: pending, filesModified, tokenEstimate };
|
|
198
205
|
}
|
|
199
|
-
// ---- Key files (existing logic from compact.ts, moved here) ----------------
|
|
200
|
-
const MAX_KEY_FILES = 5;
|
|
201
|
-
const FRESHNESS_WINDOW = 10;
|
|
202
|
-
function collectKeyFiles(messages) {
|
|
203
|
-
const recent = messages.slice(-FRESHNESS_WINDOW);
|
|
204
|
-
const pathFreq = new Map();
|
|
205
|
-
for (const m of recent) {
|
|
206
|
-
for (const p of extractFilePaths(m.text)) {
|
|
207
|
-
pathFreq.set(p, (pathFreq.get(p) ?? 0) + 1);
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
return [...pathFreq.entries()]
|
|
211
|
-
.sort((a, b) => b[1] - a[1])
|
|
212
|
-
.slice(0, MAX_KEY_FILES)
|
|
213
|
-
.map(([p]) => p);
|
|
214
|
-
}
|
|
@@ -49,6 +49,11 @@ export function computeDedupTierRollup(events, windowMs, now) {
|
|
|
49
49
|
const ts = Date.parse(ev.ts);
|
|
50
50
|
if (Number.isNaN(ts) || ts < windowStart || ts > windowEnd)
|
|
51
51
|
continue;
|
|
52
|
+
// "skipped" (degenerate-match guard declined a collapse) is a decision, but
|
|
53
|
+
// NOT a tier catch — it is attributed to no tier below. Excluding it from the
|
|
54
|
+
// denominator keeps l0Share + l1Share + l2Share summing to 1 over the window.
|
|
55
|
+
if (ev.status === "skipped")
|
|
56
|
+
continue;
|
|
52
57
|
total += 1;
|
|
53
58
|
switch (ev.tier) {
|
|
54
59
|
case "L0":
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { shouldSkipDegenerateMatch } from "../dedup/degenerate.js";
|
|
2
|
+
/**
|
|
3
|
+
* Build the decliner for one add() cascade.
|
|
4
|
+
*
|
|
5
|
+
* Returns a predicate the L1/L2 call sites use as a one-line guard. When it
|
|
6
|
+
* returns true it has ALREADY recorded the declined decision (monitoring event +
|
|
7
|
+
* `skipped` audit line + the live `onTier` detail), so the caller only has to
|
|
8
|
+
* fall through. When the umbrella flag is off it always returns false and emits
|
|
9
|
+
* nothing — byte-identical to the pre-guard cascade.
|
|
10
|
+
*/
|
|
11
|
+
export function degenerateDecliner(args) {
|
|
12
|
+
const { store, input, contentHash, cfg, audit, t0 } = args;
|
|
13
|
+
const candidate = { ...input, contentHash };
|
|
14
|
+
return (tier, matched, similarity) => {
|
|
15
|
+
if (!shouldSkipDegenerateMatch(matched, candidate, cfg))
|
|
16
|
+
return false;
|
|
17
|
+
// Reported as `mark_only`: a tier matched but policy declined to collapse —
|
|
18
|
+
// exactly the existing MARK_ONLY shape, with a distinct reason string so the
|
|
19
|
+
// dashboard can tell a guard decline from an operator-configured MARK_ONLY.
|
|
20
|
+
store.record(tier, "mark_only", "degenerateGuard", Date.now() - t0, similarity, matched.checkpointId);
|
|
21
|
+
audit.skipped(tier, matched.checkpointId, "degenerateGuard", similarity);
|
|
22
|
+
input.onTier?.({ tier, status: "passed", detail: "degenerateGuard" });
|
|
23
|
+
return true;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
@@ -27,6 +27,7 @@ import { lshBands } from "../dedup/l1-lsh.js";
|
|
|
27
27
|
import { openBloom, saveBloom } from "../store/bloom.js";
|
|
28
28
|
import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, addTokensSaved, bumpDedupStats, } from "../store/sqlite.js";
|
|
29
29
|
import { computeRegionHash } from "./hash.js";
|
|
30
|
+
import { degenerateDecliner } from "./add-degenerate.js";
|
|
30
31
|
import { runL0Tier, computeSummaryHash } from "./add-l0.js";
|
|
31
32
|
import { findL1Duplicate } from "./add-l1.js";
|
|
32
33
|
import { dedupAuditRecorder } from "./dedup-audit.js";
|
|
@@ -67,8 +68,22 @@ export function addCheckpoint(store, input) {
|
|
|
67
68
|
// Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
|
|
68
69
|
// and which tier.
|
|
69
70
|
let markOnly = null;
|
|
70
|
-
//
|
|
71
|
+
// Content digest for this candidate (also consumed by the L0 tier below).
|
|
71
72
|
const digest = computeContentDigest(input.regionText);
|
|
73
|
+
// Degenerate-match guard (incident 2026-08-19): declines a fuzzy-tier collapse
|
|
74
|
+
// onto a content-free skeleton when the incoming region is richer, so the
|
|
75
|
+
// skeleton stops absorbing every future compaction. Returns true (having
|
|
76
|
+
// already recorded the decision) ⇒ the caller treats the match as a non-match.
|
|
77
|
+
// Flag-off ⇒ always false ⇒ byte-identical predecessor. See add-degenerate.ts.
|
|
78
|
+
const declineDegenerate = degenerateDecliner({
|
|
79
|
+
store,
|
|
80
|
+
input,
|
|
81
|
+
contentHash: digest.contentHash,
|
|
82
|
+
cfg,
|
|
83
|
+
audit,
|
|
84
|
+
t0,
|
|
85
|
+
});
|
|
86
|
+
// L0 exact-match tier (contentHash / regionHash / summaryHash) — see add-l0.ts.
|
|
72
87
|
const bloom = openBloom(store.stateDir);
|
|
73
88
|
const summaryHash = computeSummaryHash(input.topicSummary);
|
|
74
89
|
const l0 = runL0Tier({
|
|
@@ -94,7 +109,10 @@ export function addCheckpoint(store, input) {
|
|
|
94
109
|
onTier?.({ tier: "L1", status: "scanning" });
|
|
95
110
|
if (cfg.L1_ENABLED) {
|
|
96
111
|
const l1 = findL1Duplicate(store, sessionId, input.regionText, all);
|
|
97
|
-
|
|
112
|
+
// Guard first: a declined match must not collapse and must not be recorded
|
|
113
|
+
// as a MARK_ONLY hit either — it is a non-match for the rest of the cascade.
|
|
114
|
+
const l1Declined = l1 !== undefined && declineDegenerate("L1", l1);
|
|
115
|
+
if (l1 && !l1Declined && !cfg.MARK_ONLY_L1) {
|
|
98
116
|
l1.timestamp = input.timestamp;
|
|
99
117
|
upsertCheckpoint(l1, store.stateDir);
|
|
100
118
|
bumpDedupStats(true, store.stateDir);
|
|
@@ -105,7 +123,7 @@ export function addCheckpoint(store, input) {
|
|
|
105
123
|
onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
|
|
106
124
|
return r;
|
|
107
125
|
}
|
|
108
|
-
if (l1 && cfg.MARK_ONLY_L1)
|
|
126
|
+
if (l1 && !l1Declined && cfg.MARK_ONLY_L1)
|
|
109
127
|
markOnly = "L1";
|
|
110
128
|
}
|
|
111
129
|
onTier?.({ tier: "L1", status: "passed" });
|
|
@@ -131,7 +149,12 @@ export function addCheckpoint(store, input) {
|
|
|
131
149
|
const sim = cosineSimilarity(embedding, cp.embedding);
|
|
132
150
|
return sim > best.sim ? { checkpoint: cp, sim } : best;
|
|
133
151
|
}, { checkpoint: all[0], sim: -1 });
|
|
134
|
-
|
|
152
|
+
// A declined match falls through to the "store a fresh checkpoint" path; the
|
|
153
|
+
// guard already audited the decision, so the near-miss emit below is skipped.
|
|
154
|
+
const l2Declined = !timedOut &&
|
|
155
|
+
nearest.sim >= simThreshold &&
|
|
156
|
+
declineDegenerate("L2", nearest.checkpoint, nearest.sim);
|
|
157
|
+
if (!timedOut && !l2Declined && nearest.sim >= simThreshold) {
|
|
135
158
|
if (!cfg.MARK_ONLY_L2) {
|
|
136
159
|
// Near-identical — update timestamp on existing checkpoint
|
|
137
160
|
nearest.checkpoint.timestamp = input.timestamp;
|
|
@@ -164,7 +187,7 @@ export function addCheckpoint(store, input) {
|
|
|
164
187
|
// Near-miss: how close did we come to collapsing? Only emitted when the
|
|
165
188
|
// scan actually completed and scored a candidate — a timed-out scan has no
|
|
166
189
|
// honest best to report.
|
|
167
|
-
if (!timedOut && nearest.sim >= 0) {
|
|
190
|
+
if (!timedOut && !l2Declined && nearest.sim >= 0) {
|
|
168
191
|
audit.passed("L2", nearest.checkpoint.checkpointId, nearest.sim);
|
|
169
192
|
}
|
|
170
193
|
}
|
|
@@ -69,6 +69,14 @@ export function dedupAuditRecorder(ctx, scope) {
|
|
|
69
69
|
matchedEntry,
|
|
70
70
|
similarity,
|
|
71
71
|
}),
|
|
72
|
+
skipped: (tier, matchedEntry, dedupReason, similarity) => emitDedupAudit(ctx, {
|
|
73
|
+
...base,
|
|
74
|
+
tier,
|
|
75
|
+
status: "skipped",
|
|
76
|
+
matchedEntry,
|
|
77
|
+
dedupReason,
|
|
78
|
+
...(similarity === undefined ? {} : { similarity }),
|
|
79
|
+
}),
|
|
72
80
|
stored: (storedEntry, dedupReason, tokenEstimate) => emitDedupAudit(ctx, {
|
|
73
81
|
...base,
|
|
74
82
|
tier: "new",
|
|
@@ -49,6 +49,11 @@ export function computeDedupTierRollup(events, windowMs, now) {
|
|
|
49
49
|
const ts = Date.parse(ev.ts);
|
|
50
50
|
if (Number.isNaN(ts) || ts < windowStart || ts > windowEnd)
|
|
51
51
|
continue;
|
|
52
|
+
// "skipped" (degenerate-match guard declined a collapse) is a decision, but
|
|
53
|
+
// NOT a tier catch — it is attributed to no tier below. Excluding it from the
|
|
54
|
+
// denominator keeps l0Share + l1Share + l2Share summing to 1 over the window.
|
|
55
|
+
if (ev.status === "skipped")
|
|
56
|
+
continue;
|
|
52
57
|
total += 1;
|
|
53
58
|
switch (ev.tier) {
|
|
54
59
|
case "L0":
|
|
@@ -69,6 +69,14 @@ export function dedupAuditRecorder(ctx, scope) {
|
|
|
69
69
|
matchedEntry,
|
|
70
70
|
similarity,
|
|
71
71
|
}),
|
|
72
|
+
skipped: (tier, matchedEntry, dedupReason, similarity) => emitDedupAudit(ctx, {
|
|
73
|
+
...base,
|
|
74
|
+
tier,
|
|
75
|
+
status: "skipped",
|
|
76
|
+
matchedEntry,
|
|
77
|
+
dedupReason,
|
|
78
|
+
...(similarity === undefined ? {} : { similarity }),
|
|
79
|
+
}),
|
|
72
80
|
stored: (storedEntry, dedupReason, tokenEstimate) => emitDedupAudit(ctx, {
|
|
73
81
|
...base,
|
|
74
82
|
tier: "new",
|