pi-mega-compact 0.21.9 → 0.21.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/dedup/degenerate.js +68 -0
  2. package/dist/extensions/dashboard-server/routes-dedup-attribution.js +8 -1
  3. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +3 -0
  4. package/dist/extensions/mega-events/context-handler/headroom.js +58 -3
  5. package/dist/extensions/mega-events/context-handler/liveTrim.js +6 -2
  6. package/dist/extensions/mega-events/context-handler.js +7 -1
  7. package/dist/extensions/mega-pipeline/compact/run.js +14 -3
  8. package/dist/src/config/dedup.js +3 -0
  9. package/dist/src/dedup/degenerate.js +68 -0
  10. package/dist/src/extractive-salvage.js +195 -0
  11. package/dist/src/extractive.js +63 -72
  12. package/dist/src/vector-cortex/dedup-attr/rollup.js +5 -0
  13. package/dist/src/vectorStore/add-degenerate.js +25 -0
  14. package/dist/src/vectorStore/add.js +28 -5
  15. package/dist/src/vectorStore/dedup-audit.js +8 -0
  16. package/dist/vector-cortex/dedup-attr/rollup.js +5 -0
  17. package/dist/vectorStore/dedup-audit.js +8 -0
  18. package/extensions/dashboard-server/routes-dedup-attribution.ts +11 -1
  19. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +8 -0
  20. package/extensions/mega-events/context-handler/headroom.ts +50 -3
  21. package/extensions/mega-events/context-handler/liveTrim.ts +6 -2
  22. package/extensions/mega-events/context-handler.ts +7 -1
  23. package/extensions/mega-pipeline/compact/run.ts +14 -4
  24. package/package.json +1 -1
  25. package/src/config/dedup.ts +15 -0
  26. package/src/dedup/degenerate.ts +125 -0
  27. package/src/extractive-salvage.ts +212 -0
  28. package/src/extractive.ts +70 -75
  29. package/src/vector-cortex/dedup-attr/rollup.ts +4 -0
  30. package/src/vectorStore/add-degenerate.ts +64 -0
  31. package/src/vectorStore/add.ts +29 -5
  32. package/src/vectorStore/dedup-audit.ts +25 -2
@@ -0,0 +1,68 @@
1
+ /**
2
+ * The effective token floor for a checkpoint: the larger of the absolute floor
3
+ * and `MIN_PCT × originalTokenEstimate`.
4
+ *
5
+ * A missing / zero / non-finite `originalTokenEstimate` contributes nothing, so
6
+ * the absolute floor applies alone — direct add() callers and pre-v0.4 rows that
7
+ * never recorded the original region size are judged on absolute size only,
8
+ * never accidentally deemed degenerate by a 0-valued percentage term.
9
+ */
10
+ export function degenerateFloor(subject, tunables) {
11
+ const orig = subject.originalTokenEstimate;
12
+ const relative = typeof orig === "number" && Number.isFinite(orig) && orig > 0
13
+ ? orig * tunables.DEDUP_DEGEN_MIN_PCT
14
+ : 0;
15
+ return Math.max(tunables.DEDUP_DEGEN_MIN_TOKENS, relative);
16
+ }
17
+ /**
18
+ * Is this stored checkpoint a degenerate (content-free) summary?
19
+ *
20
+ * Calibration against the incident data:
21
+ * - skeleton: tokenEstimate 34, original ≈19166 → 34 < max(48, 95.8) → TRUE
22
+ * - normal: tokenEstimate 2000, original 70000 → 2000 > max(48, 350) → FALSE
23
+ *
24
+ * The relative term is what makes this scale: a 34-token summary of a 900-token
25
+ * region is a legitimate 26× compression, while the same 34 tokens standing in
26
+ * for 19k is a skeleton.
27
+ */
28
+ export function isDegenerateCheckpoint(subject, tunables) {
29
+ const tokens = subject.tokenEstimate ?? 0;
30
+ return tokens < degenerateFloor(subject, tunables);
31
+ }
32
+ /**
33
+ * Should an L1/L2 match be DECLINED because it would collapse richer incoming
34
+ * content onto a degenerate stored checkpoint?
35
+ *
36
+ * Returns true only when all four hold:
37
+ * 1. the umbrella flag is ON,
38
+ * 2. the matched (stored) checkpoint is degenerate,
39
+ * 3. the candidate is strictly richer than the match,
40
+ * 4. the candidate's content is not byte-identical to the match's.
41
+ *
42
+ * Condition 3 uses a strict `>`: equal-size skeletons collapsing is harmless and
43
+ * keeps the store from growing one row per compaction while the summarizer is
44
+ * broken. Only a genuine improvement is worth declining a collapse for.
45
+ *
46
+ * Condition 4 is a CORRECTNESS requirement, not a refinement. `context_chunks`
47
+ * carries a partial UNIQUE index on (session_id, content_hash) (schema/core.ts
48
+ * QA #1), so declining a match whose content hash already exists would fall
49
+ * through to an INSERT that throws — inside add(), which sits on the agent loop.
50
+ * It is also the semantically right call: identical bytes are the SAME region,
51
+ * so re-storing them adds no information and heals nothing. Only L0 may own the
52
+ * exact-match case; the guard exists for fuzzy matches on genuinely different
53
+ * text, which is exactly the incident's shape (each compaction produced a
54
+ * *similar but distinct* skeleton).
55
+ */
56
+ export function shouldSkipDegenerateMatch(matched, candidate, tunables) {
57
+ if (!tunables.DEDUP_DEGENERATE_GUARD)
58
+ return false;
59
+ if (!isDegenerateCheckpoint(matched, tunables))
60
+ return false;
61
+ if ((candidate.tokenEstimate ?? 0) <= (matched.tokenEstimate ?? 0))
62
+ return false;
63
+ // Byte-identical content → not a healing opportunity (and would violate the
64
+ // UNIQUE index). Compared only when both hashes are known.
65
+ const a = candidate.contentHash;
66
+ const b = matched.contentHash;
67
+ return !(a !== undefined && b !== undefined && a === b);
68
+ }
@@ -50,7 +50,14 @@ function parseAuditLine(line) {
50
50
  const status = obj.status;
51
51
  if (tier !== "L0" && tier !== "L1" && tier !== "L2" && tier !== "new")
52
52
  return null;
53
- if (status !== "deduped" && status !== "passed" && status !== "stored")
53
+ // "skipped" = a tier matched but the degenerate-match guard declined to
54
+ // collapse. Accepted so the line is not silently dropped from the tail; the
55
+ // rollup below counts only deduped/passed, so tier catch-share math is
56
+ // unchanged by its presence.
57
+ if (status !== "deduped" &&
58
+ status !== "passed" &&
59
+ status !== "stored" &&
60
+ status !== "skipped")
54
61
  return null;
55
62
  // sessionId is never read by the rollup; a parsed line may omit richer fields.
56
63
  return { type: "dedup_audit", ts: obj.ts, tier, status, sessionId: "" };
@@ -85,6 +85,7 @@ export const SETTINGS = [
85
85
  boolDirect("MEGACOMPACT_MARK_ONLY_L1", "Mark Only L1", "L1 runs but does not collapse", false),
86
86
  boolDirect("MEGACOMPACT_MARK_ONLY_L2", "Mark Only L2", "L2 runs but does not collapse", false),
87
87
  boolDirect("MEGACOMPACT_MINILM", "MiniLM Embedder", "Use MiniLM instead of trigram", false),
88
+ boolDirect("MEGACOMPACT_DEDUP_DEGENERATE_GUARD", "Degenerate Match Guard", "Decline an L1/L2 collapse when the MATCHED stored checkpoint is a content-free skeleton (a ~30-40 token structural summary) and the incoming region is richer. Without this, one degenerate checkpoint absorbs every later compaction forever and the store can never heal. OFF = byte-identical pre-guard cascade. Calibrated by the two Degenerate floors under Dedup Thresholds.", true),
88
89
  boolDirect("MEGACOMPACT_DEDUP_AUDIT", "Dedup Audit Trail", "Append one events.log line per tier decision (which layer collapsed a region, onto what, at what similarity) to tune the thresholds below. Pure instrumentation — dedup behavior is identical either way.", true),
89
90
  ],
90
91
  },
@@ -94,6 +95,8 @@ export const SETTINGS = [
94
95
  num("MEGACOMPACT_L2_THRESHOLD", "L2 Cosine Threshold", "L2 semantic dedup firing point", 0.85, 0, 1),
95
96
  num("MEGACOMPACT_L1_JACCARD", "L1 Jaccard Threshold", "L1 MinHash near-dup threshold", 0.8, 0, 1),
96
97
  num("MEGACOMPACT_DEDUP_SIM", "Dedup Similarity", "Legacy content-similarity fallback", 0.9, 0, 1),
98
+ num("MEGACOMPACT_DEDUP_DEGEN_MIN_TOKENS", "Degenerate Min Tokens", "Absolute token floor below which a stored summary counts as a degenerate skeleton (Degenerate Match Guard)", 48, 0, 10000, "tokens"),
99
+ num("MEGACOMPACT_DEDUP_DEGEN_MIN_PCT", "Degenerate Min Percent", "Relative floor as a fraction of the summary's original region size; a summary under max(min-tokens, pct x original) is degenerate", 0.005, 0, 1),
97
100
  num("MEGACOMPACT_RECALL_MIN_COSINE", "Recall Min Cosine (same-repo)", "3WF-3 same-repo floor the 3-source validator applies to the top winner (cross-repo 0.90 stays separate)", 0.12, 0, 1),
98
101
  num("MEGACOMPACT_MMR_LAMBDA", "MMR Lambda", "Maximal Marginal Relevance diversity", 0.5, 0, 1),
99
102
  num("MEGACOMPACT_SEMDEDUP_COSINE", "SemDeDup Cosine", "Offline SemDeDup pair threshold", 0.95, 0, 1),
@@ -1,5 +1,60 @@
1
- import { estimateBlockTokens, estimateMessageTokens } from "../../../src/tokens.js";
1
+ import { estimateBlockTokens } from "../../../src/tokens.js";
2
2
  import { messageContentText } from "./messageText.js";
3
+ /**
4
+ * Full-surface AgentMessage token estimate for BUDGET arithmetic (tail cap).
5
+ *
6
+ * convertToLlm (pi dist/core/messages.js) ships assistant/toolResult messages
7
+ * VERBATIM — every content block goes over the wire: text, thinking, toolCall
8
+ * (name + full `arguments` JSON), toolResult output, role wrappers. The text
9
+ * extractor (messageContentText) is lossy-on-purpose for analytics, and using
10
+ * it here made a GLM-4.7-style assistant message with ~11.6k bytes of toolCall
11
+ * arguments register as ~77 tokens — a 30k-token tail passed an 11.9k budget,
12
+ * the model overflowed, and pi's one-shot compact-and-retry failed
13
+ * ("Context overflow recovery failed", 2026-08-20 incident).
14
+ *
15
+ * Counts every byte the provider actually receives. Still a heuristic (len/4
16
+ * + 1 per block, like estimateBlockTokens) — just no longer lossy. Never
17
+ * throws: unknown block shapes fall back to their JSON serialization length,
18
+ * and a non-array/string content is counted as its serialization.
19
+ */
20
+ export function estimateAgentMessageBudgetTokens(m) {
21
+ try {
22
+ const c = m.content;
23
+ let bytes = 0;
24
+ if (typeof c === "string") {
25
+ bytes += c.length;
26
+ }
27
+ else if (Array.isArray(c)) {
28
+ for (const b of c) {
29
+ if (b == null || typeof b !== "object")
30
+ continue;
31
+ const o = b;
32
+ if (typeof o.text === "string")
33
+ bytes += o.text.length;
34
+ if (typeof o.thinking === "string")
35
+ bytes += o.thinking.length;
36
+ if (typeof o.name === "string")
37
+ bytes += o.name.length;
38
+ if (o.arguments != null)
39
+ bytes += JSON.stringify(o.arguments).length;
40
+ if (typeof o.output === "string")
41
+ bytes += o.output.length;
42
+ // Per-block envelope overhead (role/type markers), matching the
43
+ // len/4+1 block accounting in estimateBlockTokens.
44
+ bytes += 4;
45
+ }
46
+ }
47
+ else if (c != null) {
48
+ bytes += JSON.stringify(c).length;
49
+ }
50
+ return estimateBlockTokens(" ".repeat(Math.max(0, bytes)));
51
+ }
52
+ catch {
53
+ // non-fatal: fall back to the legacy text-only estimate rather than
54
+ // disable the cap on a pathological message.
55
+ return estimateBlockTokens(messageContentText(m));
56
+ }
57
+ }
3
58
  /**
4
59
  * The model's declared maxTokens is only trusted as the output budget when it
5
60
  * is plausible. models.json carries sentinel junk for some entries (1e9,
@@ -91,7 +146,7 @@ export function applyTailCap(opts) {
91
146
  tailTokens +=
92
147
  msgTokens != null
93
148
  ? Math.max(0, msgTokens[i])
94
- : estimateMessageTokens({ text: messageContentText(recentRaw[i]) });
149
+ : estimateAgentMessageBudgetTokens(recentRaw[i]);
95
150
  if (tailTokens > budget) {
96
151
  // Keep from i+1 onward; never drop below the FINAL message.
97
152
  start = Math.min(i + 1, recentRaw.length - 1);
@@ -119,7 +174,7 @@ export function applyTailCap(opts) {
119
174
  export function recapReplayedTail(opts) {
120
175
  return applyTailCap({
121
176
  recentRaw: opts.recentRaw,
122
- summaryTokens: estimateBlockTokens(messageContentText(opts.summaryAgentMsg)),
177
+ summaryTokens: estimateAgentMessageBudgetTokens(opts.summaryAgentMsg),
123
178
  ctxWindow: opts.ctxWindow,
124
179
  maxOutputTokens: opts.maxOutputTokens,
125
180
  outputReservePct: opts.outputReservePct,
@@ -115,8 +115,12 @@ export function buildLiveTrimView(runtime, config, ctx, opts) {
115
115
  // (rt.lastCheckpointId) instead of ran.result.checkpointId, which is
116
116
  // dedup-volatile: on a re-compact that dedups onto a DIFFERENT existing
117
117
  // checkpoint, result.checkpointId is the matched id (engine.ts:188) while
118
- // lastCheckpointId is only updated on a genuinely new checkpoint
119
- // (compact.ts:100-104). Keying on result.checkpointId would make
118
+ // lastCheckpointId was, pre-C1, only updated on a genuinely new checkpoint.
119
+ // C1 (v0.21.10) now stamps lastCheckpointId on the dedup path too (see
120
+ // compact/run.ts) — it means "the checkpoint backing this epoch" — so this
121
+ // key and the D.2/D.3 comparison agree in both directions and the `??`
122
+ // fallbacks below are now only for the truly-no-checkpoint edge case.
123
+ // Keying on result.checkpointId directly would still make
120
124
  // trimCache.checkpointId != rt.lastCheckpointId forever after that
121
125
  // dedup fire, disabling replay for the rest of the epoch (the
122
126
  // alternating cache-miss that 0.8.6 meant to fix). Prefer the stable
@@ -183,8 +183,14 @@ export function registerContextHandler(pi, runtime, config) {
183
183
  }
184
184
  // Debounce so we don't fire on every context event past threshold.
185
185
  // (Replay already returned above — only fresh compacts reach this point.)
186
+ // C2 (v0.21.10): EXEMPT headroom-triggered fires, matching the thrash-guard
187
+ // exemption above. pi's own overflow recovery (400 → compact → immediate
188
+ // retry) re-fires a context event <2s after our last fire; debouncing it
189
+ // returned the RAW untrimmed view, so input + output reserve still blew the
190
+ // window → 400 → "recovery failed after one compact-and-retry attempt".
191
+ // An overflowed session is unrecoverable; a re-fire is merely wasteful.
186
192
  const now = Date.now();
187
- if (now < runtime.debounceUntil) {
193
+ if (now < runtime.debounceUntil && !gate.headroomExceeded) {
188
194
  runtime.diagCtxDebounce++;
189
195
  return tailResult() ?? undefined;
190
196
  }
@@ -71,10 +71,21 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
71
71
  runtime.pulsing = false;
72
72
  if (result.skipped)
73
73
  return { skipped: true };
74
- if (!result.deduped) {
74
+ // C1 (v0.21.10): lastCheckpointId tracks "the checkpoint backing this epoch",
75
+ // so it is stamped on BOTH paths — a matched-dedup checkpoint backs this epoch
76
+ // just as much as a freshly created one. Previously the dedup path left it
77
+ // undefined, so a runtime session whose every compaction deduped (common after
78
+ // a process restart, when checkpoints persist but `rt` is rebuilt) never set it
79
+ // → liveTrim's trimCache fell back to result.checkpointId (the matched id) →
80
+ // `trimCache.checkpointId === rt.lastCheckpointId` was `"chkpt_001" !== undefined`
81
+ // → the D.2/D.3 replay NEVER matched and the full pipeline re-ran on every
82
+ // context event (liveTrimReplays: 0, "comp lag warn"). A later fire matching a
83
+ // DIFFERENT checkpoint now changes the key once (one cache regeneration), then
84
+ // replays stabilise. `persistedThisSession` keeps its narrower meaning ("we
85
+ // wrote NEW state this session") and stays gated on !deduped.
86
+ if (!result.deduped)
75
87
  runtime.rt.persistedThisSession = true;
76
- runtime.rt.lastCheckpointId = result.checkpointId;
77
- }
88
+ runtime.rt.lastCheckpointId = result.checkpointId;
78
89
  runtime.rt.lastCompactedFrom = result.compactedFrom;
79
90
  runtime.rt.lastCompactedTokens = result.tokenEstimate;
80
91
  runtime.rt.dedupAttempts++;
@@ -63,6 +63,9 @@ export function loadDedupConfig() {
63
63
  L2_COSINE_CODE: envNumOrNull("MEGACOMPACT_L2_THRESHOLD_CODE"),
64
64
  L2_COSINE_PROSE: envNumOrNull("MEGACOMPACT_L2_THRESHOLD_PROSE"),
65
65
  L1_JACCARD: envNum("MEGACOMPACT_L1_JACCARD", 0.8),
66
+ DEDUP_DEGENERATE_GUARD: envBool("MEGACOMPACT_DEDUP_DEGENERATE_GUARD", true),
67
+ DEDUP_DEGEN_MIN_TOKENS: envNum("MEGACOMPACT_DEDUP_DEGEN_MIN_TOKENS", 48),
68
+ DEDUP_DEGEN_MIN_PCT: envNum("MEGACOMPACT_DEDUP_DEGEN_MIN_PCT", 0.005),
66
69
  DEDUP_SIM: envNum("MEGACOMPACT_DEDUP_SIM", 0.9),
67
70
  MMR_LAMBDA: envNum("MEGACOMPACT_MMR_LAMBDA", 0.5),
68
71
  SEMDEDUP_COSINE: envNum("MEGACOMPACT_SEMDEDUP_COSINE", 0.95),
@@ -0,0 +1,68 @@
1
+ /**
2
+ * The effective token floor for a checkpoint: the larger of the absolute floor
3
+ * and `MIN_PCT × originalTokenEstimate`.
4
+ *
5
+ * A missing / zero / non-finite `originalTokenEstimate` contributes nothing, so
6
+ * the absolute floor applies alone — direct add() callers and pre-v0.4 rows that
7
+ * never recorded the original region size are judged on absolute size only,
8
+ * never accidentally deemed degenerate by a 0-valued percentage term.
9
+ */
10
+ export function degenerateFloor(subject, tunables) {
11
+ const orig = subject.originalTokenEstimate;
12
+ const relative = typeof orig === "number" && Number.isFinite(orig) && orig > 0
13
+ ? orig * tunables.DEDUP_DEGEN_MIN_PCT
14
+ : 0;
15
+ return Math.max(tunables.DEDUP_DEGEN_MIN_TOKENS, relative);
16
+ }
17
+ /**
18
+ * Is this stored checkpoint a degenerate (content-free) summary?
19
+ *
20
+ * Calibration against the incident data:
21
+ * - skeleton: tokenEstimate 34, original ≈19166 → 34 < max(48, 95.8) → TRUE
22
+ * - normal: tokenEstimate 2000, original 70000 → 2000 > max(48, 350) → FALSE
23
+ *
24
+ * The relative term is what makes this scale: a 34-token summary of a 900-token
25
+ * region is a legitimate 26× compression, while the same 34 tokens standing in
26
+ * for 19k is a skeleton.
27
+ */
28
+ export function isDegenerateCheckpoint(subject, tunables) {
29
+ const tokens = subject.tokenEstimate ?? 0;
30
+ return tokens < degenerateFloor(subject, tunables);
31
+ }
32
+ /**
33
+ * Should an L1/L2 match be DECLINED because it would collapse richer incoming
34
+ * content onto a degenerate stored checkpoint?
35
+ *
36
+ * Returns true only when all four hold:
37
+ * 1. the umbrella flag is ON,
38
+ * 2. the matched (stored) checkpoint is degenerate,
39
+ * 3. the candidate is strictly richer than the match,
40
+ * 4. the candidate's content is not byte-identical to the match's.
41
+ *
42
+ * Condition 3 uses a strict `>`: equal-size skeletons collapsing is harmless and
43
+ * keeps the store from growing one row per compaction while the summarizer is
44
+ * broken. Only a genuine improvement is worth declining a collapse for.
45
+ *
46
+ * Condition 4 is a CORRECTNESS requirement, not a refinement. `context_chunks`
47
+ * carries a partial UNIQUE index on (session_id, content_hash) (schema/core.ts
48
+ * QA #1), so declining a match whose content hash already exists would fall
49
+ * through to an INSERT that throws — inside add(), which sits on the agent loop.
50
+ * It is also the semantically right call: identical bytes are the SAME region,
51
+ * so re-storing them adds no information and heals nothing. Only L0 may own the
52
+ * exact-match case; the guard exists for fuzzy matches on genuinely different
53
+ * text, which is exactly the incident's shape (each compaction produced a
54
+ * *similar but distinct* skeleton).
55
+ */
56
+ export function shouldSkipDegenerateMatch(matched, candidate, tunables) {
57
+ if (!tunables.DEDUP_DEGENERATE_GUARD)
58
+ return false;
59
+ if (!isDegenerateCheckpoint(matched, tunables))
60
+ return false;
61
+ if ((candidate.tokenEstimate ?? 0) <= (matched.tokenEstimate ?? 0))
62
+ return false;
63
+ // Byte-identical content → not a healing opportunity (and would violate the
64
+ // UNIQUE index). Compared only when both hashes are known.
65
+ const a = candidate.contentHash;
66
+ const b = matched.contentHash;
67
+ return !(a !== undefined && b !== undefined && a === b);
68
+ }
@@ -0,0 +1,195 @@
1
+ /**
2
+ * extractive-salvage.ts — file-path policy + skeleton salvage for extractive.ts.
3
+ *
4
+ * Extracted from extractive.ts (A1/A2 sprint) to keep that file under the
5
+ * 300-line src/ soft limit. Pure functions only — no I/O, no logging.
6
+ *
7
+ * DESIGN (A1): blocklist, not allowlist. The old `INTERESTING_EXT` allowlist
8
+ * (rs/ts/tsx/js/json/md) silently produced content-free summaries for every
9
+ * other language — a .go/.py/.c project got a 34-token skeleton. An allowlist
10
+ * has to be *right* about ~40 ecosystems to be useful and fails closed (drops
11
+ * real work) when it is wrong; a NOISE blocklist only has to be right about the
12
+ * small, stable set of binary/generated/asset extensions and fails open (an
13
+ * unknown extension is surfaced, which is the safe direction for a summary).
14
+ */
15
+ // ---- File path policy ------------------------------------------------------
16
+ /**
17
+ * Generic extension capture: any 1–6 char ALPHABETIC extension. Path
18
+ * character class is unchanged from the original FILE_PATH_RE so existing
19
+ * matching behaviour (quotes/backticks/whitespace as delimiters) is preserved.
20
+ * Alphabetic-only is deliberate: every real source/config extension is alpha
21
+ * (c..tsx, tsconfig.json), while an alphanumeric class matches version strings
22
+ * ("GLM-4.7", "v0.21.9", "Node 18.2") as "files" and spams Key files.
23
+ */
24
+ export const FILE_PATH_RE = /(?:^|\s)([^\s"`']+\.([A-Za-z]{1,6}))\b/g;
25
+ /** Same policy, single-match, for inferCurrentWork (also excludes ':'). */
26
+ export const CURRENT_WORK_PATH_RE = /(?:^|\s)([^\s"`':]+\.([A-Za-z]{1,6}))\b/m;
27
+ /**
28
+ * Binary / generated / asset / vendored extensions that carry no summary value.
29
+ * Deliberately small and stable. Note `md`, `json`, `toml`, `yaml`, `sql`, `css`
30
+ * and `html` are NOT noise — they are hand-edited source in most repos.
31
+ */
32
+ export const NOISE_EXT = new Set([
33
+ // lockfiles & logs
34
+ "lock", "log", "sum",
35
+ // images & media
36
+ "png", "jpg", "jpeg", "gif", "webp", "svg", "ico", "bmp", "tiff",
37
+ "mp3", "mp4", "mov", "wav", "webm", "avi",
38
+ // fonts
39
+ "woff", "woff2", "ttf", "otf", "eot",
40
+ // archives & binaries
41
+ "zip", "gz", "tgz", "bz2", "xz", "7z", "rar", "tar",
42
+ "exe", "dll", "so", "dylib", "bin", "o", "a", "obj", "class", "pyc", "pyo",
43
+ "wasm", "node", "jar", "war", "deb", "rpm", "dmg", "iso", "img",
44
+ // generated / build artifacts
45
+ "map", "min", "lockb", "snap", "cache", "tmp", "temp", "swp", "bak", "orig",
46
+ // data blobs & databases
47
+ "pdf", "db", "sqlite", "sqlite3", "pack", "idx", "pem", "key", "crt",
48
+ ]);
49
+ /** Directory fragments whose files are never "key files" for a summary. */
50
+ const NOISE_DIR_RE = /(?:^|\/)(?:node_modules|\.git|dist|build|coverage|vendor|target|__pycache__|\.venv|venv)(?:\/|$)/;
51
+ /**
52
+ * TLD-shaped extensions: `example.com`, `github.com`, `npm.cmd` are domains and
53
+ * launchers, not code. (QA lens 1 finding, 2026-08-19.) Conservative list only —
54
+ * no real source extension lives here (.go/.rs/.ts stay interesting).
55
+ */
56
+ const TLD_EXT = new Set([
57
+ "com", "org", "net", "io", "gov", "edu", "biz", "info", "dev", "app",
58
+ "page", "xyz", "site", "online", "cloud", "me", "co", "us", "uk", "de",
59
+ "fr", "jp", "cn", "nl", "se", "eu", "int", "mil", "cmd",
60
+ ]);
61
+ /** True when a matched path is worth surfacing in a summary. */
62
+ export function isInterestingPath(filePath, ext) {
63
+ const lowerExt = ext.toLowerCase();
64
+ if (NOISE_EXT.has(lowerExt))
65
+ return false;
66
+ if (TLD_EXT.has(lowerExt))
67
+ return false; // bare domains, not code
68
+ if (/^(?:https?|ftp):\/\//i.test(filePath) || filePath.toLowerCase().startsWith("www."))
69
+ return false;
70
+ if (NOISE_DIR_RE.test(filePath))
71
+ return false;
72
+ // `app.min.js` / `bundle.min.css` style double extensions.
73
+ if (/\.min\.[A-Za-z0-9]{1,6}$/.test(filePath))
74
+ return false;
75
+ if (/\.map$/.test(filePath))
76
+ return false;
77
+ // Prose abbreviations ("e.g", "i.e", "U.S"): a single-char base with no slash
78
+ // and no digit is never a filename. ("a.c" is rare collateral; "q1.py" and
79
+ // "src/a.ts" survive — digit / slash both exempt.)
80
+ const base = filePath.slice(0, filePath.lastIndexOf("."));
81
+ if (!filePath.includes("/") && !/\d/.test(filePath) && base.length <= 1)
82
+ return false;
83
+ return true;
84
+ }
85
+ // ---- Path collection -------------------------------------------------------
86
+ const MAX_KEY_FILES = 5;
87
+ const MAX_FILES = 10;
88
+ const FRESHNESS_WINDOW = 10;
89
+ /** All interesting file paths mentioned in a blob of text. */
90
+ export function extractFilePaths(text) {
91
+ const paths = [];
92
+ for (const m of text.matchAll(FILE_PATH_RE)) {
93
+ if (isInterestingPath(m[1], m[2]))
94
+ paths.push(m[1]);
95
+ }
96
+ return paths;
97
+ }
98
+ /** Most-mentioned paths within the recency window (behaviour unchanged). */
99
+ export function collectKeyFiles(messages) {
100
+ const recent = messages.slice(-FRESHNESS_WINDOW);
101
+ const pathFreq = new Map();
102
+ for (const m of recent) {
103
+ for (const p of extractFilePaths(m.text)) {
104
+ pathFreq.set(p, (pathFreq.get(p) ?? 0) + 1);
105
+ }
106
+ }
107
+ return [...pathFreq.entries()]
108
+ .sort((a, b) => b[1] - a[1])
109
+ .slice(0, MAX_KEY_FILES)
110
+ .map(([p]) => p);
111
+ }
112
+ /** Paths written/edited by tools (extension-agnostic; behaviour unchanged). */
113
+ export function extractFilesModified(tools) {
114
+ const files = new Set();
115
+ for (const m of tools) {
116
+ if (!m.toolName)
117
+ continue;
118
+ const name = m.toolName.toLowerCase();
119
+ if (name === "write" || name === "edit" || name === "notebookedit") {
120
+ const input = m.input ?? m.text;
121
+ const pathMatch = input.match(/["']?(\/[^\s"']+\.\w+)["']?/);
122
+ if (pathMatch)
123
+ files.add(pathMatch[1]);
124
+ }
125
+ if (name === "bash") {
126
+ const cmd = m.input ?? m.text;
127
+ if (cmd.includes("git add") || cmd.includes("git commit") || cmd.includes("git diff")) {
128
+ for (const p of extractFilePaths(cmd))
129
+ files.add(p);
130
+ }
131
+ }
132
+ }
133
+ return [...files].slice(0, MAX_FILES);
134
+ }
135
+ // ---- Placeholder user turns (A2b) ------------------------------------------
136
+ /**
137
+ * Content-free user turns. Extremely common in resumed sessions; taking them
138
+ * verbatim as "User requests" is what produced the three "• resume" bullets.
139
+ */
140
+ const PLACEHOLDER_RE = /^(?:resume|continue|go on|go ahead|proceed|next|yes|yeah|yep|y|ok|okay|k|sure|thanks|thank you|ty|done|please continue|carry on)\W*$/i;
141
+ export function isPlaceholderRequest(text) {
142
+ return PLACEHOLDER_RE.test(text.trim());
143
+ }
144
+ // ---- Skeleton salvage (A2c) ------------------------------------------------
145
+ const MAX_SALVAGE_LINES = 5;
146
+ const SALVAGE_LINE_LEN = 120;
147
+ /**
148
+ * A "skeleton" summary is the scope line and nothing else — no files, no current
149
+ * work, no decisions, no pending items, and no *substantive* user request. That
150
+ * is ~34 tokens of zero information and is what breaks a resumed session.
151
+ *
152
+ * NOTE: `recentUser` is treated as empty when it holds only placeholders. The
153
+ * incident summary DID have three "• resume" bullets, so a plain
154
+ * `recentUser.length === 0` test would never have fired on the very case this
155
+ * salvage exists for.
156
+ */
157
+ export function isSkeletonSummary(parts) {
158
+ const hasRealRequest = parts.recentUser.some((r) => !isPlaceholderRequest(r));
159
+ return (!hasRealRequest &&
160
+ parts.keyFiles.length === 0 &&
161
+ !parts.currentWork &&
162
+ parts.decisions.length === 0 &&
163
+ parts.pending.length === 0);
164
+ }
165
+ /**
166
+ * Last-resort content: the first meaningful line of the most recent assistant
167
+ * and tool messages. Deterministic (pure scan, newest-first, then re-ordered
168
+ * oldest-first for reading). Returns [] when there is genuinely nothing.
169
+ */
170
+ export function buildSalvageDigest(messages) {
171
+ const out = [];
172
+ const seen = new Set();
173
+ for (let i = messages.length - 1; i >= 0 && out.length < MAX_SALVAGE_LINES; i--) {
174
+ const m = messages[i];
175
+ if (m.role !== "assistant" && m.role !== "tool")
176
+ continue;
177
+ const raw = m.text || m.output || m.input || "";
178
+ const line = raw
179
+ .split("\n")
180
+ .map((l) => l.trim())
181
+ .find((l) => l.length > 0);
182
+ if (!line)
183
+ continue;
184
+ const label = m.role === "tool" ? `${m.toolName ?? "tool"}: ` : "";
185
+ const entry = truncateLine(`${label}${line}`, SALVAGE_LINE_LEN);
186
+ if (seen.has(entry))
187
+ continue;
188
+ seen.add(entry);
189
+ out.push(entry);
190
+ }
191
+ return out.reverse();
192
+ }
193
+ function truncateLine(s, maxLen) {
194
+ return s.length <= maxLen ? s : s.slice(0, maxLen - 1) + "…";
195
+ }