pi-mega-compact 0.8.24 → 0.8.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +26 -0
  2. package/dist/extensions/mega-compact-s38.test.js +263 -14
  3. package/dist/extensions/mega-compact.js +15 -0
  4. package/dist/extensions/mega-config.js +3 -0
  5. package/dist/extensions/mega-events/agent-handlers.js +211 -26
  6. package/dist/extensions/mega-events/context-handler.js +45 -7
  7. package/dist/extensions/mega-events/error-classifier.js +125 -18
  8. package/dist/extensions/mega-pipeline/compact.js +24 -13
  9. package/dist/extensions/mega-pipeline/recall.js +31 -2
  10. package/dist/extensions/mega-runtime/dashboard-snapshot.js +4 -0
  11. package/dist/extensions/mega-runtime/runtime-snapshot.js +4 -0
  12. package/dist/extensions/mega-runtime/runtime.js +58 -5
  13. package/dist/src/boundary.js +79 -43
  14. package/dist/src/boundary.test.js +119 -2
  15. package/dist/src/canary.js +10 -0
  16. package/dist/src/config/dedup.js +14 -0
  17. package/dist/src/config.js +3 -1
  18. package/dist/src/dedup/raptor/buildHistory.js +164 -0
  19. package/dist/src/dedup/raptor/buildHistory.test.js +292 -0
  20. package/dist/src/dedup/raptor/index.js +38 -0
  21. package/dist/src/dedup/raptor/multilevel-serve.test.js +229 -0
  22. package/dist/src/dedup/raptor/multilevel.js +17 -5
  23. package/dist/src/dedup/raptor/multilevel.test.js +36 -1
  24. package/dist/src/dedup/raptor/raptor.test.js +43 -0
  25. package/dist/src/dedup/raptor/retrieval.js +14 -2
  26. package/dist/src/dedup/raptor/retrieval.test.js +95 -0
  27. package/dist/src/dedup/raptor/serve-gate.test.js +298 -0
  28. package/dist/src/dedup/raptor/summarizer.js +1 -0
  29. package/dist/src/dedup/raptor/tree.js +16 -2
  30. package/dist/src/engine.js +18 -2
  31. package/dist/src/httpEmbedder.js +96 -6
  32. package/dist/src/httpEmbedder.test.js +277 -0
  33. package/dist/src/mechanical-fix.test.js +65 -0
  34. package/dist/src/raptor-inject-summaries.test.js +162 -0
  35. package/dist/src/recall.js +153 -24
  36. package/dist/src/recall.test.js +179 -4
  37. package/dist/src/store/sqlite/dedup-mirror.js +32 -15
  38. package/dist/src/store/sqlite/maintenance.js +2 -2
  39. package/dist/src/store/sqlite/mechanical-fix.test.js +146 -0
  40. package/dist/src/store/sqlite/memories.js +5 -5
  41. package/dist/src/store/sqlite/meta.js +1 -1
  42. package/dist/src/store/sqlite/raptor.js +56 -17
  43. package/dist/src/store/sqlite/raptor.test.js +106 -0
  44. package/dist/src/store/sqlite/schema.js +90 -1
  45. package/dist/src/store/sqlite/session-state.js +9 -3
  46. package/dist/src/store/sqlite/stats.js +9 -5
  47. package/dist/src/store/sqlite/turns.js +179 -0
  48. package/dist/src/store/sqlite/turns.test.js +183 -0
  49. package/dist/src/store/sqlite/utils.js +15 -4
  50. package/dist/src/store/sqlite.js +1 -0
  51. package/dist/src/store.js +2 -2
  52. package/dist/src/vector-search-cache.test.js +157 -0
  53. package/dist/src/vector-search.js +107 -15
  54. package/dist/src/vectorStore.js +36 -8
  55. package/extensions/mega-compact-s38.test.ts +259 -14
  56. package/extensions/mega-compact.ts +15 -0
  57. package/extensions/mega-config.ts +18 -0
  58. package/extensions/mega-dashboard.ts +10 -1
  59. package/extensions/mega-events/agent-handlers.ts +211 -26
  60. package/extensions/mega-events/context-handler.ts +43 -7
  61. package/extensions/mega-events/error-classifier.ts +125 -17
  62. package/extensions/mega-pipeline/compact.ts +28 -16
  63. package/extensions/mega-pipeline/recall.ts +34 -2
  64. package/extensions/mega-runtime/dashboard-snapshot.ts +8 -0
  65. package/extensions/mega-runtime/helpers.ts +25 -1
  66. package/extensions/mega-runtime/runtime-snapshot.ts +4 -0
  67. package/extensions/mega-runtime/runtime.ts +69 -23
  68. package/package.json +1 -1
  69. package/src/boundary.test.ts +128 -2
  70. package/src/boundary.ts +75 -39
  71. package/src/canary.ts +10 -0
  72. package/src/config/dedup.ts +25 -0
  73. package/src/config.ts +3 -1
  74. package/src/dedup/raptor/buildHistory.test.ts +353 -0
  75. package/src/dedup/raptor/buildHistory.ts +259 -0
  76. package/src/dedup/raptor/index.ts +38 -0
  77. package/src/dedup/raptor/multilevel-serve.test.ts +273 -0
  78. package/src/dedup/raptor/multilevel.test.ts +47 -0
  79. package/src/dedup/raptor/multilevel.ts +18 -8
  80. package/src/dedup/raptor/raptor.test.ts +59 -0
  81. package/src/dedup/raptor/retrieval.test.ts +118 -0
  82. package/src/dedup/raptor/retrieval.ts +14 -2
  83. package/src/dedup/raptor/serve-gate.test.ts +348 -0
  84. package/src/dedup/raptor/summarizer.ts +1 -0
  85. package/src/dedup/raptor/tree.ts +17 -2
  86. package/src/engine.ts +32 -3
  87. package/src/httpEmbedder.test.ts +286 -0
  88. package/src/httpEmbedder.ts +98 -8
  89. package/src/mechanical-fix.test.ts +70 -0
  90. package/src/raptor-inject-summaries.test.ts +228 -0
  91. package/src/recall.test.ts +220 -4
  92. package/src/recall.ts +462 -265
  93. package/src/store/sqlite/dedup-mirror.ts +35 -18
  94. package/src/store/sqlite/maintenance.ts +2 -2
  95. package/src/store/sqlite/mechanical-fix.test.ts +162 -0
  96. package/src/store/sqlite/memories.ts +5 -5
  97. package/src/store/sqlite/meta.ts +1 -1
  98. package/src/store/sqlite/raptor.test.ts +139 -0
  99. package/src/store/sqlite/raptor.ts +135 -81
  100. package/src/store/sqlite/schema.ts +90 -1
  101. package/src/store/sqlite/session-state.ts +9 -3
  102. package/src/store/sqlite/stats.ts +10 -8
  103. package/src/store/sqlite/turns.test.ts +218 -0
  104. package/src/store/sqlite/turns.ts +302 -0
  105. package/src/store/sqlite/utils.ts +14 -4
  106. package/src/store/sqlite.ts +1 -0
  107. package/src/store.ts +9 -2
  108. package/src/vector-search-cache.test.ts +190 -0
  109. package/src/vector-search.ts +273 -156
  110. package/src/vectorStore.ts +443 -382
  111. package/extensions/mega-runtime/reset-runtime.ts +0 -80
@@ -16,6 +16,7 @@ import { consolidateMemories } from "../../src/memory.js";
16
16
  import { C, MARKER_TYPE, } from "../mega-runtime.js";
17
17
  import { resolveRepoRoot, preserveRecentForPressure } from "../mega-config.js";
18
18
  import { runRaptor } from "../../src/dedup/raptor/index.js";
19
+ import { isRaptorTreeFresh } from "../../src/dedup/raptor/buildHistory.js";
19
20
  import { loadDedupConfig } from "../../src/config/dedup.js";
20
21
  import { upsertEmbedding as indexUpsertEmbedding } from "../../src/store/vectorIndex.js";
21
22
  import { runMemoryReview } from "./memory-review.js";
@@ -180,19 +181,29 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
180
181
  embedding: cp.embedding,
181
182
  }));
182
183
  if (leaves.length >= 2) {
183
- // S25: stamp the tree with the newest checkpoint epoch so the
184
- // freshness guard in raptorSearchHits can reject stale trees after a
185
- // later compaction adds newer checkpoints.
186
- const builtAt = all.length > 0 ? Math.max(...all.map((c) => c.timestamp)) : Date.now();
187
- runRaptor(leaves, {
188
- stateDir: runtime.currentStateDir,
189
- sessionId: sid,
190
- budgetMs: dd.RAPTOR_BUDGET_MS,
191
- clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
192
- consistencyThreshold: dd.RAPTOR_CONSISTENCY,
193
- logger: runtime.logger,
194
- builtAt: Number.isFinite(builtAt) ? builtAt : Date.now(),
195
- });
184
+ // S42D: skip the rebuild when the last build is fresh (within
185
+ // RAPTOR_FRESHNESS_HOURS) and the checkpoint count hasn't drifted by
186
+ // more than 20%. avoids re-clustering on every compaction when the
187
+ // tree is still representative. 0 disables (always rebuild).
188
+ if (dd.RAPTOR_FRESHNESS_HOURS > 0 &&
189
+ isRaptorTreeFresh(sid, runtime.currentStateDir, dd.RAPTOR_FRESHNESS_HOURS, all.length)) {
190
+ runtime.logger?.info("raptor_skip_fresh", { sessionId: sid });
191
+ }
192
+ else {
193
+ // S25: stamp the tree with the newest checkpoint epoch so the
194
+ // freshness guard in raptorSearchHits can reject stale trees after a
195
+ // later compaction adds newer checkpoints.
196
+ const builtAt = all.length > 0 ? Math.max(...all.map((c) => c.timestamp)) : Date.now();
197
+ runRaptor(leaves, {
198
+ stateDir: runtime.currentStateDir,
199
+ sessionId: sid,
200
+ budgetMs: dd.RAPTOR_BUDGET_MS,
201
+ clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
202
+ consistencyThreshold: dd.RAPTOR_CONSISTENCY,
203
+ logger: runtime.logger,
204
+ builtAt: Number.isFinite(builtAt) ? builtAt : Date.now(),
205
+ });
206
+ }
196
207
  }
197
208
  }
198
209
  catch {
@@ -8,7 +8,8 @@
8
8
  import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
9
9
  import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "../../src/recall.js";
10
10
  import { normalizeSessionId } from "../../src/store.js";
11
- import { incRecallInjected, incCacheHitTokens } from "../../src/store/sqlite.js";
11
+ import { incRecallInjected, incCacheHitTokens, getIndexDir } from "../../src/store/sqlite.js";
12
+ import { ensureConversationId, recordTurn, recordTurnRecall } from "../../src/store/sqlite/turns.js";
12
13
  import { C, } from "../mega-runtime.js";
13
14
  /**
14
15
  * Unified recall (Layer 5). The ONE path that injects. Returns the recall
@@ -50,6 +51,27 @@ export function doRecall(runtime, config, ctx, query, source) {
50
51
  runtime.rt.cacheHitTokens += sumTokens;
51
52
  incRecallInjected(result.toInject.length, runtime.currentStateDir);
52
53
  incCacheHitTokens(sumTokens, runtime.currentStateDir);
54
+ // S43: record recall provenance — which checkpoints/summaries served this
55
+ // turn, their score + source path. Linked to the turn row written at
56
+ // turn_end via the conversation+turnIndex. Best-effort + non-fatal.
57
+ try {
58
+ const convId = ensureConversationId(sid, runtime.currentStateDir);
59
+ const turnId = recordTurn({
60
+ conversationId: convId,
61
+ sessionId: sid,
62
+ turnIndex: runtime.currentTurn,
63
+ startedAt: Date.now(),
64
+ }, runtime.currentStateDir);
65
+ recordTurnRecall(turnId, result.toInject.map((h) => ({
66
+ checkpointId: h.checkpoint.checkpointId,
67
+ score: h.score,
68
+ source: (h.raptorLevel !== undefined ? "raptor" : h.repoId ? "cross-repo" : "flat"),
69
+ raptorLevel: h.raptorLevel,
70
+ })), runtime.currentStateDir);
71
+ }
72
+ catch {
73
+ /* non-fatal: recall provenance never breaks the recall path */
74
+ }
53
75
  }
54
76
  return result;
55
77
  }
@@ -84,7 +106,14 @@ export async function doRecallAsync(runtime, config, ctx, query, source, opts =
84
106
  sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
85
107
  recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
86
108
  liveWindow, dedupSim: config.crossRepoCosine, crossRepo: true,
87
- globalIndexDir: process.env.MEGACOMPACT_INDEX_DIR,
109
+ // F2: resolve the machine-wide index dir via the shared resolver so the
110
+ // cross-repo injected-set dedup works even when MEGACOMPACT_INDEX_DIR is
111
+ // unset. The env var still wins when set (getIndexDir checks it first);
112
+ // the default (~/.mega-compact-index) is the same DB mega-commands and the
113
+ // dashboard read, so injection counts stay consistent. Without this, a
114
+ // bare `process.env` read returns undefined → cross-repo hits re-inject in
115
+ // every new session (the global injected-set is never consulted).
116
+ globalIndexDir: getIndexDir(),
88
117
  }, runtime.store);
89
118
  runtime.dashboard.event("recall-crossrepo", {
90
119
  source, query: query.slice(0, 120), injected: x.toInject.length,
@@ -113,6 +113,10 @@ export function buildDashboardSnapshot(ctx) {
113
113
  consecutiveErrors: ctx.consecutiveErrors,
114
114
  maxConsecutiveErrors: ctx.ERROR_RETRY_MAX_CONSECUTIVE,
115
115
  errorRetryHardStop: ctx.errorRetryHardStop,
116
+ // R7 (retry redesign): additive session-cap + poisoned-context counters.
117
+ sessionRetryCount: ctx.sessionRetryCount,
118
+ sessionMax: ctx.sessionRetryMax,
119
+ poisonedCount: ctx.poisonedCount,
116
120
  },
117
121
  };
118
122
  }
@@ -90,6 +90,10 @@ export function snapshotImpl(self, ctx) {
90
90
  consecutiveErrors: self.rt.consecutiveErrors,
91
91
  ERROR_RETRY_MAX_CONSECUTIVE: self.config.maxConsecutiveErrors,
92
92
  errorRetryHardStop: self.config.errorRetryHardStop,
93
+ // R7 (retry redesign): session-cap + poisoned-context counters.
94
+ sessionRetryCount: self.rt.errorRetrySessionCount,
95
+ sessionRetryMax: self.config.errorRetrySessionMax,
96
+ poisonedCount: self.rt.poisonedCount,
93
97
  activeAgents: self.activeAgents,
94
98
  currentTurn: self.currentTurn,
95
99
  currentModel: self.currentModel,
@@ -5,7 +5,7 @@
5
5
  * Phase 2d (maximal split): the class body is field declarations, the
6
6
  * constructor, and 1-line delegates only. Every method body lives in its own
7
7
  * module following the context-interface + free-function + thin-delegate
8
- * pattern: pressure-getters.ts / reset-runtime.ts / append-event.ts /
8
+ * pattern: pressure-getters.ts / append-event.ts /
9
9
  * get-state-dir.ts / render-widget.ts / status.ts / engine-view.ts /
10
10
  * runtime-snapshot.ts / runtime-helpers.ts / effects.ts / game-state.ts /
11
11
  * capture-model.ts / bind-repo.ts / perf.ts. state.ts re-exports the class for
@@ -23,7 +23,6 @@ import { captureModelImpl } from "./capture-model.js";
23
23
  import { bindRepoImpl } from "./bind-repo.js";
24
24
  import { snapshotImpl } from "./runtime-snapshot.js";
25
25
  import { pressureImpl, effectiveThresholdImpl, pressureBandImpl, } from "./pressure-getters.js";
26
- import { resetRuntimeImpl } from "./reset-runtime.js";
27
26
  import { appendEventImpl } from "./append-event.js";
28
27
  import { getStateDirImpl } from "./get-state-dir.js";
29
28
  import { renderWidgetImpl } from "./render-widget.js";
@@ -57,6 +56,15 @@ export class MegaRuntime {
57
56
  errorRetryCount: 0,
58
57
  errorRetryUntil: 0,
59
58
  consecutiveErrors: 0,
59
+ // R1-R3 (retry redesign): in-flight dedup, session cap, poisoned-context state.
60
+ lastErrorRetryAt: 0,
61
+ retryNudgePending: false,
62
+ errorRetrySessionCount: 0,
63
+ lastErrorText: undefined,
64
+ errorTextRepeatCount: 0,
65
+ poisonedAdviseSent: false,
66
+ poisonedCompactSignatures: new Set(),
67
+ poisonedCount: 0,
60
68
  };
61
69
  // v0.8.6 cache-stability: the cached live-trim view for the current
62
70
  // compaction epoch. Set after a fresh runCompact + computeLiveTrimCut, and
@@ -256,10 +264,55 @@ export class MegaRuntime {
256
264
  setStatus(ctx, text) {
257
265
  setStatusImpl(this, ctx, text);
258
266
  }
259
- /** Per-session state reset (session_start / session_tree) — thin delegate to
260
- * `resetRuntimeImpl` (reset-runtime.ts). */
267
+ /** Per-session state reset (session_start / session_tree) — inlined by the
268
+ * raptor-promotion merge (R1–R3 retry-redesign fields); reset-runtime.ts was
269
+ * retired by that branch. */
261
270
  resetRuntime(sessionId) {
262
- resetRuntimeImpl(this, sessionId);
271
+ const sid = normalizeSessionId(sessionId);
272
+ if (this.rt.sessionId === sid && this.rt.persistedThisSession)
273
+ return; // same session, keep checkpoint memory
274
+ this.rt = {
275
+ sessionId: sid,
276
+ persistedThisSession: false,
277
+ lastCheckpointId: undefined,
278
+ lastCompactedFrom: 0,
279
+ lastCompactedTokens: 0,
280
+ dedupSkips: 0,
281
+ dedupAttempts: 0,
282
+ tokensSaved: 0,
283
+ lastCompactAt: null,
284
+ lastNativeCompactAt: null,
285
+ compactCount: 0,
286
+ recallInjections: 0,
287
+ cacheHitTokens: 0,
288
+ lengthStopPending: false,
289
+ errorRetryCount: 0,
290
+ errorRetryUntil: 0,
291
+ consecutiveErrors: 0,
292
+ // R1-R3 (retry redesign): in-flight dedup, session cap, poisoned-context state.
293
+ lastErrorRetryAt: 0,
294
+ retryNudgePending: false,
295
+ errorRetrySessionCount: 0,
296
+ lastErrorText: undefined,
297
+ errorTextRepeatCount: 0,
298
+ poisonedAdviseSent: false,
299
+ poisonedCompactSignatures: new Set(),
300
+ poisonedCount: 0,
301
+ };
302
+ this.trimCache = null; // v0.8.6: never replay a stale trim into a new session
303
+ this.statusKey = undefined;
304
+ this.activeAgents = 0;
305
+ this.currentTurn = 0;
306
+ this.lastActivityAt = 0;
307
+ this.tierTrace = undefined;
308
+ this.ticker.length = 0;
309
+ this.pulsing = false;
310
+ this.savedGoal = 50_000;
311
+ this.lastWhy = undefined;
312
+ // S31 audit P2: symmetry with bindRepo — a reset can coincide with a context
313
+ // that re-binds the repo, so drop the memo too. Cheap; the next
314
+ // getCachedGameState() re-queries lazily.
315
+ this.cachedGameState = undefined;
263
316
  }
264
317
  captureModel(ctx) {
265
318
  captureModelImpl(this, ctx);
@@ -5,10 +5,19 @@
5
5
  * 1. ANCHOR FLOOR: never drop the most recent N user messages.
6
6
  * 2. TOOL-PAIR: never split an assistant(toolCall) from its following
7
7
  * tool-result message — an orphaned `tool` role with no preceding
8
- * assistant tool call causes a 400 on the OpenAI-compat path.
8
+ * assistant tool call causes a 400 on the OpenAI-compat path. The pair
9
+ * invariant outranks the anchor floor: on conflict we drop LESS (lower the
10
+ * drop end), never cross a pair.
9
11
  *
10
12
  * The engine reasons over EngineMessage; the pi adapter maps role "tool" +
11
- * toolName to the tool-result shape.
13
+ * toolName to the tool-result shape. EngineMessage carries no tool-call id, so
14
+ * ownership is positional: a tool result's owner is its nearest preceding
15
+ * assistant tool-call (the last assistant message with a `toolName` before it).
16
+ * A preserved tool result is orphaned by a cut when its owner is dropped; the
17
+ * guard rejects any cut that drops an owner while preserving its result, for
18
+ * ARBITRARY interleavings (custom/non-tool messages between call and result,
19
+ * consecutive results sharing one call, a cut landing directly on a call whose
20
+ * results follow).
12
21
  */
13
22
  /** Is this message a tool result (pi `tool` role with a tool name)? */
14
23
  function isToolResult(m) {
@@ -18,16 +27,56 @@ function isToolResult(m) {
18
27
  function hasToolUse(m) {
19
28
  return Boolean(m.toolName) && m.role !== "tool";
20
29
  }
30
+ /**
31
+ * Is the drop boundary at `dropEnd` pair-safe? The preserved run is
32
+ * [dropEnd, messages.length). The cut is pair-safe iff NO preserved tool result
33
+ * is orphaned: for every tool result at index >= dropEnd, its nearest preceding
34
+ * assistant tool-call must EXIST and be PRESERVED (index >= dropEnd). A tool
35
+ * result with no preceding assistant tool-call is already orphaned in the
36
+ * input — we treat that as unsafe too, so the guard never endorses shipping an
37
+ * orphaned result to the provider.
38
+ *
39
+ * O(messages.length) single forward pass; early-exits on the first orphan. The
40
+ * owner of each result is the most recent `hasToolUse` message seen so far
41
+ * (tracked across the whole stream, including dropped messages, because a
42
+ * dropped assistant tool-call is exactly the owner we must reject).
43
+ */
44
+ export function isPairSafe(messages, dropEnd) {
45
+ if (dropEnd <= 0 || dropEnd >= messages.length)
46
+ return true;
47
+ let lastToolCall = -1;
48
+ for (let i = 0; i < messages.length; i++) {
49
+ if (hasToolUse(messages[i]))
50
+ lastToolCall = i;
51
+ if (i >= dropEnd && isToolResult(messages[i])) {
52
+ if (lastToolCall === -1)
53
+ return false; // no preceding call → orphaned
54
+ if (lastToolCall < dropEnd)
55
+ return false; // owner dropped → orphaned
56
+ }
57
+ }
58
+ return true;
59
+ }
21
60
  /**
22
61
  * Compute the safe drop range [dropStart, dropEnd) within `messages`.
23
- * `keepFrom` is the caller's desired first-preserved index. We then:
24
- * 1. Walk it back (lower dropEnd = keep more) so the first preserved message
25
- * is never an orphaned tool result (tool-pair invariant).
26
- * 2. Raise it (lower dropEnd) to the anchor floor so the last N user messages
27
- * are never dropped, when enough user messages exist.
28
62
  *
29
- * dropEnd is the first index KEPT. Returns [dropStart, dropEnd]; empty range
30
- * if nothing should be dropped.
63
+ * Contract:
64
+ * - `keepFrom` is the caller's desired first-preserved index (drop [0, keepFrom)).
65
+ * - `dropEnd` is the first index KEPT; we may LOWER it (keep more) to satisfy the
66
+ * guards, never raise it above keepFrom.
67
+ * - The anchor floor (PREVENT-PI-001) caps dropEnd at the index of the
68
+ * Nth-from-last user message so the last N user messages are never dropped.
69
+ * - The tool-pair invariant (PREVENT-PI-002) rejects any dropEnd that orphans a
70
+ * preserved tool result; on conflict with the anchor floor the pair rule wins
71
+ * (we drop less, never cross a pair).
72
+ * - We return the LARGEST pair-safe dropEnd <= min(keepFrom, anchorStart) so the
73
+ * caller drops as much as is safe. When no pair-safe positive cut exists at
74
+ * or below keepFrom, we return [0, 0] (no-op) — the pair rule outranks
75
+ * dropping. dropStart is always 0 today (we drop a prefix); reserved for
76
+ * future two-sided trimming.
77
+ *
78
+ * Returns [0, 0] (empty range, drop nothing) when keepFrom is out of range or no
79
+ * pair-safe positive cut exists.
31
80
  */
32
81
  export function computeDropRange(messages, keepFrom, anchorUserMessages) {
33
82
  if (keepFrom <= 0 || keepFrom >= messages.length)
@@ -36,49 +85,36 @@ export function computeDropRange(messages, keepFrom, anchorUserMessages) {
36
85
  messages.forEach((m, i) => { if (m.role === "user")
37
86
  userIndexes.push(i); });
38
87
  const anchorActive = anchorUserMessages > 0 && userIndexes.length >= anchorUserMessages;
39
- const anchorStart = anchorActive ? userIndexes[userIndexes.length - anchorUserMessages] : 0;
40
- const floor = anchorActive ? anchorStart : 0;
41
- // Walk back for the tool-pair invariant (keep more when needed).
42
- let k = keepFrom;
43
- while (k > floor) {
44
- const firstPreserved = messages[k];
45
- if (!firstPreserved || !isToolResult(firstPreserved))
46
- break;
47
- const preceding = messages[k - 1];
48
- if (preceding && hasToolUse(preceding)) {
49
- k -= 1; // pair intact across boundary — include the assistant turn
50
- break;
51
- }
52
- k -= 1;
88
+ const anchorStart = anchorActive ? userIndexes[userIndexes.length - anchorUserMessages] : keepFrom;
89
+ // Upper bound on dropEnd: never keep less than the caller asked (dropEnd <= keepFrom)
90
+ // and never drop a must-keep user message (dropEnd <= anchorStart).
91
+ const upperBound = Math.min(keepFrom, anchorActive ? anchorStart : keepFrom);
92
+ // Walk down from the upper bound to find the largest pair-safe cut. dropEnd=0
93
+ // (drop nothing) is always pair-safe; the loop finds the largest positive cut,
94
+ // and falls back to [0, 0] when none exists — the pair rule outranks dropping.
95
+ for (let dropEnd = upperBound; dropEnd > 0; dropEnd--) {
96
+ if (isPairSafe(messages, dropEnd))
97
+ return [0, dropEnd];
53
98
  }
54
- if (k < floor)
55
- k = floor;
56
- // Anchor floor: never drop a must-keep user message. Raise dropEnd so we keep
57
- // from anchorStart onward when the walk didn't already.
58
- if (anchorActive && k > anchorStart)
59
- k = anchorStart;
60
- if (k <= 0)
61
- return [0, 0];
62
- return [0, k];
99
+ return [0, 0];
63
100
  }
64
101
  /**
65
102
  * Validate that the intended split at `keepFrom` (drop [0, keepFrom), keep the
66
- * rest) does not start the preserved run on an orphaned tool result. Checks
67
- * messages[keepFrom] against messages[keepFrom-1] directly independent of the
68
- * walk-back that computeDropRange may apply.
103
+ * rest) does not orphan any preserved tool result. Checks the FULL preserved
104
+ * run, not just the first message, so it holds for arbitrary interleavings
105
+ * (custom messages between call and result, consecutive shared-call results, a
106
+ * cut landing on a call whose results follow). Used on the every-LLM-call
107
+ * live-trim hot path (extensions/mega-trim.ts) and by dropCompactedRange
108
+ * (src/adapt.ts).
69
109
  */
70
110
  export function isBoundarySafe(messages, keepFrom) {
71
- if (keepFrom <= 0 || keepFrom >= messages.length)
72
- return true;
73
- const firstPreserved = messages[keepFrom];
74
- if (!isToolResult(firstPreserved))
75
- return true;
76
- const preceding = messages[keepFrom - 1];
77
- return Boolean(preceding && hasToolUse(preceding));
111
+ return isPairSafe(messages, keepFrom);
78
112
  }
79
113
  /**
80
114
  * Drop everything before the safe keep-index, honoring both guards, returning
81
- * the filtered message list.
115
+ * the filtered message list. Returns the original array reference (unchanged)
116
+ * when the safe range is empty so callers can short-circuit on reference
117
+ * equality.
82
118
  */
83
119
  export function dropBefore(messages, keepFrom, anchorUserMessages) {
84
120
  const [dropStart, dropEnd] = computeDropRange(messages, keepFrom, anchorUserMessages);
@@ -5,6 +5,7 @@ function user(t) { return { role: "user", text: t }; }
5
5
  function assistant(t) { return { role: "assistant", text: t }; }
6
6
  function toolUse(n, i = "{}") { return { role: "assistant", text: "", toolName: n, input: i }; }
7
7
  function toolResult(n, o = "ok") { return { role: "tool", text: "", toolName: n, output: o }; }
8
+ function custom(t) { return { role: "custom", text: t }; }
8
9
  test("walks back so first preserved message is not an orphaned tool result", () => {
9
10
  const messages = [
10
11
  user("Search for files"),
@@ -21,9 +22,14 @@ test("walks back so first preserved message is not an orphaned tool result", ()
21
22
  assert.notEqual(kept[0].role, "tool");
22
23
  assert.equal(kept[0].toolName, "search"); // assistant tool-call preserved
23
24
  });
24
- test("isBoundarySafe: tool result at boundary with preceding tool use is safe", () => {
25
+ test("isBoundarySafe: cut that drops a toolCall but keeps its toolResult is unsafe (PREVENT-PI-002)", () => {
25
26
  const messages = [user("a"), toolUse("search"), toolResult("search")];
26
- assert.equal(isBoundarySafe(messages, 2), true);
27
+ // keepFrom=2 drops the toolUse at index 1 but keeps the toolResult at index 2 →
28
+ // the preserved run starts on an orphaned tool result. This is the shape the old
29
+ // check mis-validated (it only compared messages[keepFrom] to messages[keepFrom-1]).
30
+ assert.equal(isBoundarySafe(messages, 2), false);
31
+ // keepFrom=1 keeps the toolCall together with its toolResult → safe.
32
+ assert.equal(isBoundarySafe(messages, 1), true);
27
33
  });
28
34
  test("isBoundarySafe: orphaned tool result without preceding tool use is unsafe", () => {
29
35
  const messages = [user("a"), toolResult("search", "orphan")];
@@ -51,3 +57,114 @@ test("dropBefore returns original when range is empty", () => {
51
57
  const messages = [user("a"), assistant("b")];
52
58
  assert.equal(dropBefore(messages, 0, 1), messages);
53
59
  });
60
+ // --- PREVENT-PI-002 regression cases for arbitrary interleavings ---
61
+ test("interleaved custom message between toolCall and toolResult: walk-back keeps the call", () => {
62
+ // [user, assistant(tc=read), custom/bashExecution, tool(read-result)] with
63
+ // desired keepFrom=3. The old walk-back saw messages[2] was not a toolUse and
64
+ // broke at k=2, dropping the assistant toolCall at index 1 while KEEPING its
65
+ // tool result at index 3 → orphaned tool result → provider 400.
66
+ const messages = [
67
+ user("Search for files"),
68
+ toolUse("read"),
69
+ custom("bash: ls -la"),
70
+ toolResult("read", "file contents"),
71
+ assistant("Done."),
72
+ ];
73
+ const [start, end] = computeDropRange(messages, 3, 0);
74
+ assert.equal(start, 0);
75
+ assert.equal(end, 1); // keep the assistant toolCall at index 1 with its result
76
+ const kept = messages.slice(end);
77
+ assert.equal(kept[0].toolName, "read"); // assistant tool-call preserved
78
+ assert.ok(kept.some((m) => m.role === "tool" && m.toolName === "read"));
79
+ // The toolCall and its toolResult are both in the kept run.
80
+ const callIdx = kept.findIndex((m) => m.role === "assistant" && m.toolName === "read");
81
+ const resultIdx = kept.findIndex((m) => m.role === "tool" && m.toolName === "read");
82
+ assert.ok(callIdx !== -1 && resultIdx !== -1 && callIdx < resultIdx);
83
+ });
84
+ test("consecutive tool results sharing one call: no-op when the call cannot be kept", () => {
85
+ // [assistant(tc), T1, T2] with keepFrom=1: dropping the call orphans BOTH T1
86
+ // and T2. No pair-safe positive cut exists below keepFrom → no-op compaction
87
+ // (the [start,end) contract preserves a non-zero result only when one exists).
88
+ const messages = [toolUse("multi"), toolResult("multi", "r1"), toolResult("multi", "r2")];
89
+ const [start, end] = computeDropRange(messages, 1, 0);
90
+ assert.equal(start, 0);
91
+ assert.equal(end, 0); // no-op — pair rule outranks dropping
92
+ assert.equal(dropBefore(messages, 1, 0), messages);
93
+ });
94
+ test("consecutive tool results sharing one call: safe cut keeps the call with both results", () => {
95
+ // [user, assistant(tc), T1, T2, user2] — keepFrom=3 would orphan T2; the guard
96
+ // walks back to keep the call (dropEnd=1, only the first user is dropped).
97
+ const messages = [
98
+ user("u1"),
99
+ toolUse("multi"),
100
+ toolResult("multi", "r1"),
101
+ toolResult("multi", "r2"),
102
+ user("u2"),
103
+ ];
104
+ const [start, end] = computeDropRange(messages, 3, 0);
105
+ assert.equal(start, 0);
106
+ assert.equal(end, 1); // keep [assistant(tc), T1, T2, user2]
107
+ const kept = messages.slice(end);
108
+ const callIdx = kept.findIndex((m) => m.role === "assistant" && m.toolName === "multi");
109
+ const r1Idx = kept.findIndex((m) => m.role === "tool" && m.output === "r1");
110
+ const r2Idx = kept.findIndex((m) => m.role === "tool" && m.output === "r2");
111
+ assert.ok(callIdx < r1Idx && r1Idx < r2Idx);
112
+ });
113
+ test("consecutive shared-call results: dropping call + all results together is safe", () => {
114
+ // keepFrom=4 preserves only the trailing user — the call and BOTH results are
115
+ // dropped together, so nothing is orphaned.
116
+ const messages = [
117
+ user("u1"),
118
+ toolUse("multi"),
119
+ toolResult("multi", "r1"),
120
+ toolResult("multi", "r2"),
121
+ user("u2"),
122
+ ];
123
+ const [start, end] = computeDropRange(messages, 4, 0);
124
+ assert.equal(start, 0);
125
+ assert.equal(end, 4);
126
+ assert.deepEqual(messages.slice(end), [user("u2")]);
127
+ });
128
+ test("keepFrom landing on a call whose results follow is safe", () => {
129
+ // [user, assistant(tc1), T1, assistant(tc2), T2, user2] keepFrom=3 → preserved
130
+ // run starts on assistant(tc2) at index 3, whose result T2 follows. Safe.
131
+ const messages = [
132
+ user("u1"),
133
+ toolUse("read"),
134
+ toolResult("read", "r1"),
135
+ toolUse("write"),
136
+ toolResult("write", "r2"),
137
+ user("u2"),
138
+ ];
139
+ const [start, end] = computeDropRange(messages, 3, 0);
140
+ assert.equal(start, 0);
141
+ assert.equal(end, 3);
142
+ const kept = messages.slice(end);
143
+ assert.equal(kept[0].role, "assistant");
144
+ assert.equal(kept[0].toolName, "write");
145
+ assert.ok(kept.some((m) => m.role === "tool" && m.toolName === "write"));
146
+ });
147
+ test("anchor floor + pair-rule conflict: pair rule wins, drop less", () => {
148
+ // [assistant(tc), user, T] with anchor=1: the anchor floor wants dropEnd<=1
149
+ // (keep the user at index 1), but keeping from index 1 orphans T (its owner at
150
+ // index 0 would be dropped). The pair rule outranks the floor — we drop LESS,
151
+ // keeping everything (no-op) rather than cross a pair.
152
+ const messages = [toolUse("read"), user("keep me"), toolResult("read", "r")];
153
+ const out = dropBefore(messages, 2, 1);
154
+ assert.equal(out, messages, "anchor floor would orphan the tool result → no-op");
155
+ });
156
+ test("isBoundarySafe: interleaved custom between call and result is detected unsafe", () => {
157
+ const messages = [user("a"), toolUse("read"), custom("bash"), toolResult("read", "r")];
158
+ // keepFrom=3 drops the call at index 1, keeps the result at index 3 → unsafe.
159
+ assert.equal(isBoundarySafe(messages, 3), false);
160
+ // keepFrom=2 ALSO drops the call at index 1 (drop [0,2) = [user, toolUse]) and
161
+ // keeps the result at index 3 → still orphaned → unsafe.
162
+ assert.equal(isBoundarySafe(messages, 2), false);
163
+ // keepFrom=1 keeps the call (index 1) together with its result at index 3 → safe.
164
+ assert.equal(isBoundarySafe(messages, 1), true);
165
+ });
166
+ test("isBoundarySafe: cut before any tool result is safe", () => {
167
+ const messages = [user("a"), toolUse("read"), toolResult("read", "r"), assistant("done")];
168
+ assert.equal(isBoundarySafe(messages, 0), true); // out of range → safe
169
+ assert.equal(isBoundarySafe(messages, messages.length), true); // out of range → safe
170
+ });
@@ -8,6 +8,16 @@
8
8
  * The controller owns a MUTABLE working copy of the dedup config; callers read
9
9
  * `controller.config` after each step. Tiers disabled via MARK_ONLY degrade
10
10
  * gracefully rather than fully off.
11
+ *
12
+ * Persistence design: disabled state is IN-MEMORY ONLY by design. The dedup
13
+ * config loads from MEGACOMPACT_* env vars (config/dedup.ts) with no durable
14
+ * save mechanism. `setEnabled` DOES mutate `this.config.L1_ENABLED` etc. so
15
+ * callers reading `controller.config` see the disabled state for the current
16
+ * session. On restart, env defaults re-apply and the canary sequences again
17
+ * from L0 — this is intentional: a tier disabled due to a cold cache or
18
+ * transient load gets a fresh evaluation each run rather than being locked
19
+ * out forever. To permanently disable a tier, set the corresponding
20
+ * MEGACOMPACT_*_ENABLED env var to false.
11
21
  */
12
22
  import { loadDedupConfig } from "./config/dedup.js";
13
23
  import { p95 } from "./monitoring.js";
@@ -27,6 +27,14 @@ function envNum(name, def) {
27
27
  const n = Number(v);
28
28
  return Number.isFinite(n) ? n : def;
29
29
  }
30
+ /** S42B: parse a comma-separated numeric env var (e.g. "1.0,0.9,0.8"). */
31
+ function envNumArray(name, def) {
32
+ const v = process.env[name];
33
+ if (v === undefined)
34
+ return def;
35
+ const parts = v.split(",").map((x) => Number(x.trim()));
36
+ return parts.length > 0 && parts.every((n) => Number.isFinite(n)) ? parts : def;
37
+ }
30
38
  /** Read the current dedup config from env (file defaults reproduce Sprint 13). */
31
39
  export function loadDedupConfig() {
32
40
  return {
@@ -53,6 +61,12 @@ export function loadDedupConfig() {
53
61
  RAPTOR_BUDGET_MS: envNum("MEGACOMPACT_RAPTOR_BUDGET_MS", 5000),
54
62
  RAPTOR_CLUSTERS_PER_LEVEL: envNum("MEGACOMPACT_RAPTOR_CLUSTERS", 5),
55
63
  RAPTOR_CONSISTENCY: envNum("MEGACOMPACT_RAPTOR_CONSISTENCY", 0.6),
64
+ RAPTOR_MULTILEVEL_ENABLED: envBool("MEGACOMPACT_RAPTOR_MULTILEVEL", true),
65
+ RAPTOR_LEVEL_WEIGHTS: envNumArray("MEGACOMPACT_RAPTOR_LEVEL_WEIGHTS", [1.0, 0.9, 0.8, 0.7, 0.5]),
66
+ RAPTOR_LEAF_EXPANSION: envBool("MEGACOMPACT_RAPTOR_LEAF_EXPANSION", true),
67
+ RAPTOR_MAX_LEAF_EXPANSION: envNum("MEGACOMPACT_RAPTOR_MAX_LEAF_EXP", 10),
68
+ RAPTOR_FRESHNESS_HOURS: envNum("MEGACOMPACT_RAPTOR_FRESHNESS_HOURS", 4),
69
+ RAPTOR_INJECT_SUMMARIES: envBool("MEGACOMPACT_RAPTOR_INJECT_SUMMARIES", true),
56
70
  FP_RATE_L0: envNum("MEGACOMPACT_FP_RATE_L0", 0.01),
57
71
  FP_RATE_L1L2: envNum("MEGACOMPACT_FP_RATE_L1L2", 0.05),
58
72
  ALERT_WINDOW_MS: envNum("MEGACOMPACT_ALERT_WINDOW_MS", 600_000),
@@ -29,7 +29,9 @@ export function pressureFromPct(pct) {
29
29
  export function preserveRecentForPressure(pressure, preserveRecent, preserveRecentMin) {
30
30
  const p = pressure < 0 ? 0 : pressure > 1 ? 1 : pressure;
31
31
  const v = Math.round(preserveRecent - (preserveRecent - preserveRecentMin) * p);
32
- return Math.max(preserveRecentMin, Math.min(preserveRecent, v));
32
+ // Floor of 1: even with preserveRecentMin=0 at full pressure, never compact
33
+ // ALL messages — the boundary guard (computeDropRange) needs ≥1 to anchor on.
34
+ return Math.max(1, preserveRecentMin, Math.min(preserveRecent, v));
33
35
  }
34
36
  /** Clamp a pressure ratio into [0, 1]. */
35
37
  function clamp01(p) {