pi-mega-compact 0.7.8 → 0.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +11 -12
  2. package/dist/extensions/dashboard-server/helpers.js +37 -0
  3. package/dist/extensions/dashboard-server/html/all-repos-tab.js +26 -0
  4. package/dist/extensions/dashboard-server/html/body-open.js +23 -0
  5. package/dist/extensions/dashboard-server/html/current-repo-tab.js +130 -0
  6. package/dist/extensions/dashboard-server/html/head-open.js +16 -0
  7. package/dist/extensions/dashboard-server/html/high-score-tab.js +25 -0
  8. package/dist/extensions/dashboard-server/html/repo-detail-modal.js +26 -0
  9. package/dist/extensions/dashboard-server/html/script.js +259 -0
  10. package/dist/extensions/dashboard-server/html/styles.js +103 -0
  11. package/dist/extensions/dashboard-server/html/summary-tab.js +19 -0
  12. package/dist/extensions/dashboard-server/html-template.js +41 -0
  13. package/dist/extensions/dashboard-server/html.js +756 -0
  14. package/dist/extensions/dashboard-server/index-reader.js +133 -0
  15. package/dist/extensions/dashboard-server/server.js +370 -0
  16. package/dist/extensions/dashboard-server/snapshot.js +43 -0
  17. package/dist/extensions/dashboard-server/state.js +30 -0
  18. package/dist/extensions/dashboard-server/types.js +5 -0
  19. package/dist/extensions/dashboard-server.js +7 -1315
  20. package/dist/extensions/mega-commands.js +162 -134
  21. package/dist/extensions/mega-compact.test.js +90 -21
  22. package/dist/extensions/mega-conflict-cmds.js +5 -1
  23. package/dist/extensions/mega-dashboard-cmds.js +29 -22
  24. package/dist/extensions/mega-db-cmds.js +11 -2
  25. package/dist/extensions/mega-events/agent-handlers.js +173 -0
  26. package/dist/extensions/mega-events/compact-handlers.js +133 -0
  27. package/dist/extensions/mega-events/context-handler.js +249 -0
  28. package/dist/extensions/mega-events/register.js +21 -0
  29. package/dist/extensions/mega-events/session-handlers.js +142 -0
  30. package/dist/extensions/mega-events.js +15 -699
  31. package/dist/extensions/mega-pipeline/compact.js +324 -0
  32. package/dist/extensions/mega-pipeline/memory-review.js +38 -0
  33. package/dist/extensions/mega-pipeline/recall.js +147 -0
  34. package/dist/extensions/mega-pipeline.js +9 -480
  35. package/dist/extensions/mega-runtime/helpers.js +40 -0
  36. package/dist/extensions/mega-runtime/query.js +29 -0
  37. package/dist/extensions/mega-runtime/state.js +711 -0
  38. package/dist/extensions/mega-runtime/widget.js +197 -0
  39. package/dist/extensions/mega-runtime.js +15 -947
  40. package/dist/src/store/sqlite/checkpoints.js +145 -0
  41. package/dist/src/store/sqlite/connection.js +35 -0
  42. package/dist/src/store/sqlite/dedup-mirror.js +64 -0
  43. package/dist/src/store/sqlite/foundation.js +38 -0
  44. package/dist/src/store/sqlite/global-index.js +224 -0
  45. package/dist/src/store/sqlite/index-store.js +167 -0
  46. package/dist/src/store/sqlite/maintenance.js +235 -0
  47. package/dist/src/store/sqlite/memories.js +164 -0
  48. package/dist/src/store/sqlite/memory.js +54 -0
  49. package/dist/src/store/sqlite/meta.js +82 -0
  50. package/dist/src/store/sqlite/minhash-lsh.js +47 -0
  51. package/dist/src/store/sqlite/model-snapshots.js +47 -0
  52. package/dist/src/store/sqlite/raptor.js +57 -0
  53. package/dist/src/store/sqlite/raw-transcript.js +134 -0
  54. package/dist/src/store/sqlite/schema.js +250 -0
  55. package/dist/src/store/sqlite/session-state.js +28 -0
  56. package/dist/src/store/sqlite/sessions.js +39 -0
  57. package/dist/src/store/sqlite/stats.js +66 -0
  58. package/dist/src/store/sqlite/transaction.js +19 -0
  59. package/dist/src/store/sqlite/utils.js +120 -0
  60. package/dist/src/store/sqlite.js +20 -1607
  61. package/dist/src/vectorStore/add.js +260 -0
  62. package/dist/src/vectorStore/dedup.js +52 -0
  63. package/dist/src/vectorStore/index.js +10 -0
  64. package/dist/src/vectorStore/queries.js +83 -0
  65. package/dist/src/vectorStore/search.js +95 -0
  66. package/dist/src/vectorStore/session.js +19 -0
  67. package/dist/src/vectorStore/store.js +105 -0
  68. package/dist/src/vectorStore/types.js +6 -0
  69. package/dist/src/vectorStore/utils.js +23 -0
  70. package/extensions/dashboard-server/html.ts +758 -0
  71. package/extensions/dashboard-server/index-reader.ts +130 -0
  72. package/extensions/dashboard-server/server.ts +358 -0
  73. package/extensions/dashboard-server/snapshot.ts +44 -0
  74. package/extensions/dashboard-server/state.ts +33 -0
  75. package/extensions/dashboard-server/types.ts +134 -0
  76. package/extensions/dashboard-server.ts +7 -1431
  77. package/extensions/mega-commands.ts +33 -10
  78. package/extensions/mega-compact.test.ts +198 -43
  79. package/extensions/mega-conflict-cmds.ts +6 -2
  80. package/extensions/mega-dashboard-cmds.ts +30 -23
  81. package/extensions/mega-db-cmds.ts +11 -3
  82. package/extensions/mega-events/agent-handlers.ts +214 -0
  83. package/extensions/mega-events/compact-handlers.ts +164 -0
  84. package/extensions/mega-events/context-handler.ts +290 -0
  85. package/extensions/mega-events/register.ts +37 -0
  86. package/extensions/mega-events/session-handlers.ts +165 -0
  87. package/extensions/mega-events.ts +15 -780
  88. package/extensions/mega-pipeline/compact.ts +366 -0
  89. package/extensions/mega-pipeline/memory-review.ts +46 -0
  90. package/extensions/mega-pipeline/recall.ts +165 -0
  91. package/extensions/mega-pipeline.ts +9 -537
  92. package/extensions/mega-runtime/helpers.ts +68 -0
  93. package/extensions/mega-runtime/query.ts +29 -0
  94. package/extensions/mega-runtime/state.ts +797 -0
  95. package/extensions/mega-runtime/widget.ts +258 -0
  96. package/extensions/mega-runtime.ts +15 -1093
  97. package/package.json +4 -3
  98. package/src/store/sqlite/checkpoints.ts +204 -0
  99. package/src/store/sqlite/dedup-mirror.ts +114 -0
  100. package/src/store/sqlite/foundation.ts +63 -0
  101. package/src/store/sqlite/global-index.ts +305 -0
  102. package/src/store/sqlite/maintenance.ts +294 -0
  103. package/src/store/sqlite/memories.ts +217 -0
  104. package/src/store/sqlite/meta.ts +108 -0
  105. package/src/store/sqlite/model-snapshots.ts +83 -0
  106. package/src/store/sqlite/raptor.ts +107 -0
  107. package/src/store/sqlite/raw-transcript.ts +221 -0
  108. package/src/store/sqlite/schema.ts +258 -0
  109. package/src/store/sqlite/session-state.ts +38 -0
  110. package/src/store/sqlite/stats.ts +127 -0
  111. package/src/store/sqlite/utils.ts +125 -0
  112. package/src/store/sqlite.ts +20 -2204
@@ -0,0 +1,324 @@
1
+ /**
2
+ * compact.ts — full compaction pipeline (Trident) + pi no-op prediction.
3
+ *
4
+ * `runCompact` runs the full Trident pipeline (fast-gate aside) and persists a
5
+ * checkpoint. `piCompactWouldNoop` predicts whether pi's `ctx.compact()` would
6
+ * throw a no-op error. Both mutate the shared MegaRuntime (token accounting,
7
+ * ticker, status, events) and are driven by the event + command handlers in
8
+ * mega-events.ts / mega-commands.ts.
9
+ */
10
+ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
11
+ import { compactSession } from "../../src/engine.js";
12
+ import { normalizeSessionId } from "../../src/store.js";
13
+ import { estimateBlockTokens } from "../../src/tokens.js";
14
+ import { touchSession, logDaily, incCompactCount, incCacheHitTokens } from "../../src/store/sqlite.js";
15
+ import { consolidateMemories } from "../../src/memory.js";
16
+ import { C, MARKER_TYPE, } from "../mega-runtime.js";
17
+ import { resolveRepoRoot, preserveRecentForPressure } from "../mega-config.js";
18
+ import { runRaptor } from "../../src/dedup/raptor/index.js";
19
+ import { loadDedupConfig } from "../../src/config/dedup.js";
20
+ import { upsertEmbedding as indexUpsertEmbedding } from "../../src/store/vectorIndex.js";
21
+ import { runMemoryReview } from "./memory-review.js";
22
+ /** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
23
+ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
24
+ runtime.bindRepo(ctx.cwd);
25
+ const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
26
+ runtime.resetRuntime(sid);
27
+ runtime.rt.sessionId = sid;
28
+ const view = runtime.engineView(messages);
29
+ // keepFrom deepens with context pressure (Fix E): under high pressure we
30
+ // compact more of the session, down to the preserveRecentMin floor.
31
+ const preserve = preserveRecentForPressure(opts.compressionPressure ?? 0, config.preserveRecent, config.preserveRecentMin);
32
+ const keepFrom = opts.keepFrom ?? Math.max(0, view.length - preserve);
33
+ // For very small sessions (fewer messages than preserveRecent), allow
34
+ // compacting everything except the last message — the user explicitly
35
+ // requested compaction, so don't refuse it just because the session is short.
36
+ if (keepFrom <= 0) {
37
+ if (view.length <= 1)
38
+ return { skipped: true };
39
+ // Use the fallback: compact everything except the last message
40
+ const fallbackKeepFrom = view.length - 1;
41
+ return doCompact(view, fallbackKeepFrom, opts, sid, config, pi, ctx, runtime);
42
+ }
43
+ return doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime);
44
+ }
45
+ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
46
+ runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
47
+ // S21.2: reset the per-compaction memory-op counter so the post-compact
48
+ // consolidate pass only fires when memory rows actually changed during the
49
+ // compaction window (turn_end → auto-review may have written some).
50
+ runtime.memoriesTouchedThisCompaction = 0;
51
+ const result = compactSession({
52
+ sessionId: sid,
53
+ messages: view,
54
+ keepFrom,
55
+ summary: opts.summary,
56
+ timestamp: Date.now(),
57
+ onTier: runtime.makeTierCallback(ctx),
58
+ compressionPressure: opts.compressionPressure,
59
+ }, runtime.store);
60
+ runtime.pulsing = false;
61
+ if (result.skipped)
62
+ return { skipped: true };
63
+ if (!result.deduped) {
64
+ runtime.rt.persistedThisSession = true;
65
+ runtime.rt.lastCheckpointId = result.checkpointId;
66
+ }
67
+ runtime.rt.lastCompactedFrom = result.compactedFrom;
68
+ runtime.rt.lastCompactedTokens = result.tokenEstimate;
69
+ runtime.rt.dedupAttempts++;
70
+ // Honest "tokens saved" for this session-instance only:
71
+ // new checkpoint → original − stored
72
+ // deduped onto existing → whole original region (nothing new stored)
73
+ // Resets to 0 on session_start (rt is rebuilt) — so a fresh session shows 0
74
+ // while the repo's cumulative saved (SQLite meta) keeps the running total.
75
+ const saved = result.deduped
76
+ ? result.originalTokenEstimate
77
+ : Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
78
+ runtime.rt.tokensSaved += saved;
79
+ runtime.rt.compactCount += 1;
80
+ incCompactCount(runtime.currentStateDir);
81
+ if (result.deduped) {
82
+ runtime.rt.cacheHitTokens += saved;
83
+ incCacheHitTokens(saved, runtime.currentStateDir);
84
+ }
85
+ runtime.rt.lastCompactAt = Date.now();
86
+ if (result.deduped)
87
+ runtime.rt.dedupSkips++;
88
+ // Grow the rolling "saved" goal so the progress bar always has a fresh
89
+ // denominator (we don't want it pinned at 100% once we pass an old target).
90
+ if (runtime.rt.tokensSaved > runtime.savedGoal)
91
+ runtime.savedGoal = Math.ceil((runtime.rt.tokensSaved * 1.25) / 10_000) * 10_000;
92
+ // Live toolbar activity: what file/region just got compacted or deduped.
93
+ // Rendered via the rotating ticker line (see snapshot); the ring buffer is
94
+ // cycled one-per-repaint so the single line scrolls through recent files.
95
+ const files = result.filesModified ?? [];
96
+ const fileLabel = files.length
97
+ ? files.map((f) => f.split("/").pop() ?? f).slice(0, 2).join(", ")
98
+ : result.regionHash.slice(0, 8);
99
+ runtime.lastActivityAt = Date.now();
100
+ // Explain-why line: surfaced while fresh. Pulls the dedup reason (which for
101
+ // L2 includes the cosine sim) so the user sees WHY a region was kept/dropped.
102
+ runtime.lastWhy = result.deduped
103
+ ? `why: deduped@${result.dedupReason ?? "tier"}`
104
+ : `why: compacted → ${result.checkpointId}`;
105
+ // Recall/activity ticker: record this event in the ring buffer.
106
+ const savedK = (saved / 1000).toFixed(1);
107
+ runtime.pushTicker(result.deduped
108
+ ? `${C.green}♻${C.reset} deduped ${fileLabel} · ${savedK}k saved`
109
+ : `${C.cyan}🗜${C.reset} ${result.checkpointId} · +${savedK}k · ${fileLabel}`);
110
+ // The per-tier trace has settled into the final outcome — fold it back into
111
+ // the activity line and stop showing the live trace.
112
+ runtime.tierTrace = undefined;
113
+ // Record session activity + a daily-log entry in the per-repo SQLite store
114
+ // (foundation for resume-sessions / daily-log features). Best-effort — never
115
+ // block a compaction on bookkeeping.
116
+ try {
117
+ const root = resolveRepoRoot(ctx.cwd);
118
+ touchSession(sid, root, runtime.currentStateDir);
119
+ logDaily(sid, "compact", result.checkpointId, saved, runtime.currentStateDir);
120
+ }
121
+ catch {
122
+ /* non-fatal: stats bookkeeping only */
123
+ }
124
+ // S21.2: best-effort consolidation of near-duplicate memories for this repo.
125
+ // Runs after the per-repo stats touch so `consolidateMemories` can use the
126
+ // same stateDir. Non-fatal — a failed consolidate never blocks a compaction.
127
+ // Only runs when new memory ops landed in this pass (otherwise the prior
128
+ // compaction's consolidate already had its shot — re-running would just
129
+ // touch every row again with no merges).
130
+ if (!result.deduped && runtime.memoriesTouchedThisCompaction > 0) {
131
+ try {
132
+ const root = resolveRepoRoot(ctx.cwd);
133
+ void consolidateMemories(runtime.currentStateDir, root).then((n) => {
134
+ if (n > 0)
135
+ runtime.pushTicker(`${C.green}∫${C.reset} consolidated ${n} memory dup${n === 1 ? "" : "s"}`);
136
+ }, () => {
137
+ /* swallow: consolidate failures must never surface to the user */
138
+ });
139
+ }
140
+ catch {
141
+ /* non-fatal */
142
+ }
143
+ }
144
+ // S24 review-on-compact: when pressure is high, the just-compacted region is
145
+ // exactly the context worth remembering, so review it immediately rather than
146
+ // waiting for the next turn-cadence tick. Uses the shared runMemoryReview
147
+ // helper (fire-and-forget; doCompact is sync). Best-effort + non-fatal. Only
148
+ // fires above the `high` band so low-pressure compactions don't pay the cost.
149
+ if (!result.deduped && config.memoryAutoReview && runtime.pressureBand !== "low" && runtime.pressureBand !== "medium") {
150
+ void runMemoryReview(runtime, view, "pressure");
151
+ }
152
+ // Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
153
+ // skip re-vectorizing an already-compacted region (zero token cost).
154
+ pi.appendEntry(MARKER_TYPE, {
155
+ checkpointId: result.checkpointId,
156
+ regionHash: result.regionHash,
157
+ tokenEstimate: result.tokenEstimate,
158
+ deduped: result.deduped,
159
+ });
160
+ // Fix D: refresh the RAPTOR tree for this session so live recall (search) can
161
+ // serve high-level summaries. Best-effort + non-fatal: never block compaction.
162
+ // Budget-guarded (RAPTOR_BUDGET_MS) so it can't hang a large session.
163
+ if (config.raptorEnabled && !result.deduped) {
164
+ try {
165
+ const dd = loadDedupConfig();
166
+ const all = runtime.store.list(sid);
167
+ const leaves = all.map((cp) => ({
168
+ id: cp.checkpointId,
169
+ messages: [],
170
+ sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
171
+ embedding: cp.embedding,
172
+ }));
173
+ if (leaves.length >= 2) {
174
+ // S25: stamp the tree with the newest checkpoint epoch so the
175
+ // freshness guard in raptorSearchHits can reject stale trees after a
176
+ // later compaction adds newer checkpoints.
177
+ const builtAt = all.length > 0 ? Math.max(...all.map((c) => c.timestamp)) : Date.now();
178
+ runRaptor(leaves, {
179
+ stateDir: runtime.currentStateDir,
180
+ sessionId: sid,
181
+ budgetMs: dd.RAPTOR_BUDGET_MS,
182
+ clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
183
+ consistencyThreshold: dd.RAPTOR_CONSISTENCY,
184
+ logger: runtime.logger,
185
+ builtAt: Number.isFinite(builtAt) ? builtAt : Date.now(),
186
+ });
187
+ }
188
+ }
189
+ catch {
190
+ /* non-fatal: tree refresh never blocks a compaction */
191
+ }
192
+ }
193
+ // Slice 2: best-effort mirror of the new checkpoint into the async global
194
+ // PGlite/HNSW vector index. Fires once per compaction (not per-add), so the
195
+ // shared global dir is never hammered by concurrent test workers.
196
+ // Non-fatal: a WASM init failure degrades to the sync scan silently.
197
+ if (!result.deduped) {
198
+ try {
199
+ const all = runtime.store.list(sid);
200
+ const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
201
+ if (latest?.embedding) {
202
+ void indexUpsertEmbedding(runtime.currentStateDir, sid, latest.checkpointId, latest.embedding).catch(() => {
203
+ /* non-fatal: index refresh never blocks a compaction */
204
+ });
205
+ }
206
+ }
207
+ catch {
208
+ /* non-fatal: index refresh never blocks a compaction */
209
+ }
210
+ }
211
+ runtime.setStatus(ctx, runtime.rt.persistedThisSession
212
+ ? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
213
+ : `mega-compact: ready`);
214
+ runtime.logger.info("compact", {
215
+ sessionId: sid,
216
+ checkpointId: result.checkpointId ?? "(deduped)",
217
+ deduped: result.deduped,
218
+ tokenEstimate: saved,
219
+ compactedFrom: result.compactedFrom,
220
+ });
221
+ runtime.dashboard.event("compact", {
222
+ sessionId: sid,
223
+ checkpointId: result.checkpointId ?? "(deduped)",
224
+ deduped: result.deduped,
225
+ tokenEstimate: saved,
226
+ compactedFrom: result.compactedFrom,
227
+ });
228
+ runtime.snapshot(ctx);
229
+ return { skipped: false, result, keepFrom, saved };
230
+ }
231
+ /**
232
+ * Predict whether pi's `ctx.compact()` would throw a no-op error — "Already
233
+ * compacted" or "Nothing to compact (session too small)" — so the auto-trigger
234
+ * can SKIP the call instead of surfacing a hard, user-facing error.
235
+ *
236
+ * Why we can't intercept or suppress it: pi's public `compact()` computes
237
+ * `prepareCompaction()` and throws *before* it emits `session_before_compact`,
238
+ * so our handler there never runs on the no-op path. And `ctx.compact()`'s
239
+ * `onError` callback fires only AFTER pi has already emitted a `compaction_end`
240
+ * event carrying the error message (which the interactive UI renders) — so
241
+ * `onError` cannot mute it either. The only robust fix is to not call
242
+ * `ctx.compact()` when pi would no-op. (pi's own `_runAutoCompaction` path is
243
+ * silent on this same condition; the public path we're forced through is the
244
+ * one that throws.)
245
+ *
246
+ * Skipping is correct, not a compromise: by the time this runs, `runCompact()`
247
+ * has already persisted the recall checkpoint (Path A). The durable on-disk
248
+ * trim is only useful when pi can actually summarize a region; a transcript
249
+ * under pi's `keepRecentTokens` budget is small enough that reloading it on
250
+ * resume isn't a token-growth problem, so the durable trim is unnecessary
251
+ * there anyway.
252
+ *
253
+ * Mirrors pi's `prepareCompaction()` return-undefined conditions (compaction.js):
254
+ * (1) last entry is a compaction → "Already compacted"
255
+ * (2) <2 cut-point messages since the last compaction → nothing to summarize
256
+ * (a cut point = any non-toolResult message — user/assistant/bash/custom/
257
+ * branchSummary/compactionSummary — matching pi's isCutPointMessage)
258
+ * (3) transcript tokens since the last compaction < keepRecentTokens → pi
259
+ * keeps everything → nothing to summarize
260
+ * `keepRecentTokens` isn't readable from the extension API, so (3) uses the pi
261
+ * default (20000) as a conservative floor; raise it via
262
+ * `MEGACOMPACT_DURABLE_TRIM_FLOOR` if you raise pi's `compact.keepRecentTokens`.
263
+ *
264
+ * Best-effort: on any read error returns true (skip) — skipping a durable trim
265
+ * is always safe; calling `ctx.compact()` on a no-op throws to the user.
266
+ */
267
+ export function piCompactWouldNoop(ctx) {
268
+ try {
269
+ const branch = ctx.sessionManager.getBranch();
270
+ if (branch.length === 0)
271
+ return true;
272
+ // (1) already compacted — pi throws "Already compacted"
273
+ if (branch[branch.length - 1].type === "compaction")
274
+ return true;
275
+ // boundaryStart = index just after the most recent compaction entry (or 0)
276
+ let boundaryStart = 0;
277
+ for (let i = branch.length - 1; i >= 0; i--) {
278
+ if (branch[i].type === "compaction") {
279
+ boundaryStart = i + 1;
280
+ break;
281
+ }
282
+ }
283
+ let cutPoints = 0;
284
+ let tokens = 0;
285
+ for (let i = boundaryStart; i < branch.length; i++) {
286
+ const e = branch[i];
287
+ if (e.type === "compaction")
288
+ continue;
289
+ let isCut = false;
290
+ for (const m of sessionEntryToContextMessages(e)) {
291
+ // pi's isCutPointMessage: every role except toolResult
292
+ if (m.role !== "toolResult")
293
+ isCut = true;
294
+ const c = m.content;
295
+ const text = typeof c === "string" ? c
296
+ : Array.isArray(c)
297
+ ? c.map((b) => b?.text ?? "").join(" ")
298
+ : "";
299
+ if (text)
300
+ tokens += estimateBlockTokens(text);
301
+ }
302
+ if (isCut)
303
+ cutPoints++;
304
+ }
305
+ // (2) need >=2 cut points so the kept cut isn't the first message
306
+ if (cutPoints < 2)
307
+ return true;
308
+ // (3) transcript under pi's keepRecentTokens budget → pi keeps everything
309
+ if (tokens < durableTrimFloorTokens())
310
+ return true;
311
+ return false;
312
+ }
313
+ catch {
314
+ return true; // safe: skip the durable trim rather than risk a user-facing throw
315
+ }
316
+ }
317
+ /** pi's default keepRecentTokens (compaction settings). Override with
318
+ * MEGACOMPACT_DURABLE_TRIM_FLOOR if you raise pi's compact.keepRecentTokens. */
319
+ function durableTrimFloorTokens() {
320
+ const raw = process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
321
+ if (raw !== undefined && Number.isFinite(Number(raw)))
322
+ return Number(raw);
323
+ return 20_000;
324
+ }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * memory-review.ts — review live conversation & persist durable memories.
3
+ *
4
+ * `runMemoryReview` is shared by the pressure-scaled turn-end cadence
5
+ * (mega-events.ts) AND review-on-compact (compact.ts) so both paths run the
6
+ * identical review body. Best-effort + non-fatal: a review failure is swallowed
7
+ * and never breaks the caller.
8
+ */
9
+ import { C, } from "../mega-runtime.js";
10
+ /**
11
+ * Review the live conversation and persist durable memories (S20+S24). Shared by
12
+ * the pressure-scaled turn-end cadence (mega-events.ts) AND review-on-compact
13
+ * (below) so both paths run the identical review body. Best-effort + non-fatal:
14
+ * a review failure is swallowed and never breaks the caller. On success, the
15
+ * number of applied ops is returned so callers can feed the consolidation gate.
16
+ *
17
+ * @param view the engine message view to review (caller builds it)
18
+ * @param label a short source tag for the ticker line (e.g. "pressure" / "turn")
19
+ */
20
+ export async function runMemoryReview(runtime, view, label) {
21
+ try {
22
+ const { reviewConversation } = await import("../../src/memory.js");
23
+ const { applyMemoryOps } = await import("../../src/memoryOps.js");
24
+ const ops = reviewConversation(view, []);
25
+ if (ops.length) {
26
+ await applyMemoryOps(ops, runtime.currentStateDir);
27
+ // S21.2: ops landed — the compaction path reads this counter and fires
28
+ // `consolidateMemories` only when > 0.
29
+ runtime.memoriesTouchedThisCompaction += ops.length;
30
+ runtime.pushTicker(`${C.green}🧠${C.reset} reviewed ${ops.length} memory op${ops.length === 1 ? "" : "s"} (${label})`);
31
+ }
32
+ return ops.length;
33
+ }
34
+ catch {
35
+ /* non-fatal — auto-review must never break the turn loop / compaction */
36
+ return 0;
37
+ }
38
+ }
@@ -0,0 +1,147 @@
1
+ /**
2
+ * recall.ts — unified Layer-5 recall pipeline.
3
+ *
4
+ * `doRecall` is the ONE path that injects (sync). `doRecallAsync` augments with
5
+ * optional cross-repo HNSW on resume / /mega-recall --cross-repo. Both mutate
6
+ * the shared MegaRuntime (token accounting, ticker, dashboard events).
7
+ */
8
+ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
9
+ import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "../../src/recall.js";
10
+ import { normalizeSessionId } from "../../src/store.js";
11
+ import { incRecallInjected, incCacheHitTokens } from "../../src/store/sqlite.js";
12
+ import { C, } from "../mega-runtime.js";
13
+ /**
14
+ * Unified recall (Layer 5). The ONE path that injects. Returns the recall
15
+ * result; callers decide whether to stage it for before_agent_start (resume)
16
+ * or report it (command).
17
+ */
18
+ export function doRecall(runtime, config, ctx, query, source) {
19
+ runtime.bindRepo(ctx.cwd);
20
+ const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
21
+ // Live window text for inline dedupe (Fix C): drop recalled checkpoints that
22
+ // are already resident in the session, so recall never re-injects context the
23
+ // model can already see. Best-effort — an empty window just skips dedupe.
24
+ const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
25
+ const result = recallAndInline({
26
+ sessionId: sid,
27
+ query,
28
+ limit: config.autoInlineK,
29
+ source,
30
+ skipInjected: true,
31
+ recallMaxTokens: config.recallMaxTokens,
32
+ windowDedupe: config.windowDedupe,
33
+ liveWindow,
34
+ dedupSim: config.dedupSim,
35
+ }, runtime.store);
36
+ runtime.dashboard.event("recall", { source, query: query.slice(0, 120), injected: result.toInject.length, empty: result.empty });
37
+ if (!result.empty && result.toInject.length > 0) {
38
+ const top = result.toInject[0];
39
+ const scorePct = Math.round((top.score ?? 0) * 100);
40
+ const files = top.checkpoint.filesModified ?? [];
41
+ const label = files.length ? files.map((f) => f.split("/").pop() ?? f).slice(0, 2).join(", ") : top.checkpoint.checkpointId;
42
+ runtime.pushTicker(`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`);
43
+ runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
44
+ }
45
+ if (result.toInject.length > 0) {
46
+ let sumTokens = 0;
47
+ for (const h of result.toInject)
48
+ sumTokens += h.checkpoint.tokenEstimate;
49
+ runtime.rt.recallInjections += result.toInject.length;
50
+ runtime.rt.cacheHitTokens += sumTokens;
51
+ incRecallInjected(result.toInject.length, runtime.currentStateDir);
52
+ incCacheHitTokens(sumTokens, runtime.currentStateDir);
53
+ }
54
+ return result;
55
+ }
56
+ /**
57
+ * S17: async recall with optional cross-repo augmentation. Used on resume
58
+ * (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
59
+ * context handler (that stays sync). Runs the sync same-repo scan first; if it
60
+ * returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
61
+ * HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
62
+ * recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
63
+ * never net-inflate the window. Cross-repo uses a stricter cosine floor
64
+ * (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
65
+ * the same-repo result unchanged.
66
+ */
67
+ export async function doRecallAsync(runtime, config, ctx, query, source, opts = {}) {
68
+ runtime.bindRepo(ctx.cwd);
69
+ const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
70
+ const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
71
+ // Sync same-repo first (fast, never blocks).
72
+ const sameRepo = recallAndInline({
73
+ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
74
+ recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
75
+ liveWindow, dedupSim: config.dedupSim,
76
+ }, runtime.store);
77
+ if (!config.crossRepoEnabled || !opts.crossRepo)
78
+ return sameRepo;
79
+ if (sameRepo.toInject.length >= config.autoInlineK)
80
+ return sameRepo; // same-repo satisfied
81
+ // Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
82
+ try {
83
+ const x = await recallAndInlineAsync({
84
+ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
85
+ recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
86
+ liveWindow, dedupSim: config.crossRepoCosine, crossRepo: true,
87
+ globalIndexDir: process.env.MEGACOMPACT_INDEX_DIR,
88
+ }, runtime.store);
89
+ runtime.dashboard.event("recall-crossrepo", {
90
+ source, query: query.slice(0, 120), injected: x.toInject.length,
91
+ sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
92
+ });
93
+ // Merge, dedup by checkpointId, respect the same token cap by reformatting.
94
+ const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
95
+ const merged = [...sameRepo.toInject];
96
+ for (const h of x.toInject) {
97
+ if (!seen.has(h.checkpoint.checkpointId)) {
98
+ merged.push(h);
99
+ seen.add(h.checkpoint.checkpointId);
100
+ }
101
+ }
102
+ const block = merged.length ? formatRecallBlock(merged) : "";
103
+ if (merged.length > 0) {
104
+ let sumTokens = 0;
105
+ for (const h of merged)
106
+ sumTokens += h.checkpoint.tokenEstimate;
107
+ runtime.rt.recallInjections += merged.length;
108
+ runtime.rt.cacheHitTokens += sumTokens;
109
+ incRecallInjected(merged.length, runtime.currentStateDir);
110
+ incCacheHitTokens(sumTokens, runtime.currentStateDir);
111
+ }
112
+ return {
113
+ toInject: merged,
114
+ report: merged.map((h) => ` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
115
+ block,
116
+ empty: merged.length === 0,
117
+ };
118
+ }
119
+ catch {
120
+ return sameRepo; // cross-repo failure → same-repo only (non-fatal)
121
+ }
122
+ }
123
+ /**
124
+ * Extract the live-window message texts from the session manager (Fix C),
125
+ * for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
126
+ * error so recall falls back to unbounded (still correct, just no dedupe).
127
+ * Mirrors recentUserQuery's use of sessionEntryToContextMessages.
128
+ */
129
+ function extractLiveWindow(ctx) {
130
+ try {
131
+ const entries = ctx.sessionManager.getEntries();
132
+ const texts = [];
133
+ for (const e of entries) {
134
+ for (const m of sessionEntryToContextMessages(e)) {
135
+ const c = m.content;
136
+ if (typeof c === "string")
137
+ texts.push(c);
138
+ else if (Array.isArray(c))
139
+ texts.push(c.map((b) => b.text ?? "").join(" "));
140
+ }
141
+ }
142
+ return texts;
143
+ }
144
+ catch {
145
+ return [];
146
+ }
147
+ }