pi-mega-compact 0.7.8 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +11 -12
  2. package/dist/extensions/dashboard-server/html.js +1023 -0
  3. package/dist/extensions/dashboard-server/html.test.js +41 -0
  4. package/dist/extensions/dashboard-server/index-reader.js +133 -0
  5. package/dist/extensions/dashboard-server/server.js +530 -0
  6. package/dist/extensions/dashboard-server/server.test.js +120 -0
  7. package/dist/extensions/dashboard-server/snapshot.js +43 -0
  8. package/dist/extensions/dashboard-server/state.js +30 -0
  9. package/dist/extensions/dashboard-server/types.js +5 -0
  10. package/dist/extensions/dashboard-server-s32.test.js +181 -0
  11. package/dist/extensions/dashboard-server.js +7 -1315
  12. package/dist/extensions/mega-commands.js +162 -134
  13. package/dist/extensions/mega-compact.js +3 -0
  14. package/dist/extensions/mega-compact.test.js +90 -21
  15. package/dist/extensions/mega-conflict-cmds.js +5 -1
  16. package/dist/extensions/mega-dashboard-cmds.js +29 -22
  17. package/dist/extensions/mega-db-cmds.js +11 -2
  18. package/dist/extensions/mega-events/agent-handlers.js +222 -0
  19. package/dist/extensions/mega-events/compact-handlers.js +162 -0
  20. package/dist/extensions/mega-events/context-handler.js +249 -0
  21. package/dist/extensions/mega-events/register.js +21 -0
  22. package/dist/extensions/mega-events/session-handlers.js +142 -0
  23. package/dist/extensions/mega-events.js +15 -699
  24. package/dist/extensions/mega-game-cmds.js +106 -0
  25. package/dist/extensions/mega-game-cmds.test.js +113 -0
  26. package/dist/extensions/mega-pipeline/compact.js +324 -0
  27. package/dist/extensions/mega-pipeline/memory-review.js +38 -0
  28. package/dist/extensions/mega-pipeline/recall.js +147 -0
  29. package/dist/extensions/mega-pipeline.js +9 -480
  30. package/dist/extensions/mega-runtime/helpers.js +40 -0
  31. package/dist/extensions/mega-runtime/query.js +29 -0
  32. package/dist/extensions/mega-runtime/state.js +877 -0
  33. package/dist/extensions/mega-runtime/state.test.js +171 -0
  34. package/dist/extensions/mega-runtime/widget.js +270 -0
  35. package/dist/extensions/mega-runtime/widget.test.js +160 -0
  36. package/dist/extensions/mega-runtime.js +15 -947
  37. package/dist/src/config/themes.js +84 -0
  38. package/dist/src/config/themes.test.js +94 -0
  39. package/dist/src/game/scoring.js +105 -0
  40. package/dist/src/game/scoring.test.js +98 -0
  41. package/dist/src/store/sqlite/checkpoints.js +145 -0
  42. package/dist/src/store/sqlite/dedup-mirror.js +64 -0
  43. package/dist/src/store/sqlite/foundation.js +38 -0
  44. package/dist/src/store/sqlite/game-achievements.js +111 -0
  45. package/dist/src/store/sqlite/game-achievements.test.js +67 -0
  46. package/dist/src/store/sqlite/game-scores.js +105 -0
  47. package/dist/src/store/sqlite/game-scores.test.js +106 -0
  48. package/dist/src/store/sqlite/game-state.js +54 -0
  49. package/dist/src/store/sqlite/game-state.test.js +76 -0
  50. package/dist/src/store/sqlite/global-index.js +224 -0
  51. package/dist/src/store/sqlite/maintenance.js +235 -0
  52. package/dist/src/store/sqlite/memories.js +164 -0
  53. package/dist/src/store/sqlite/meta.js +82 -0
  54. package/dist/src/store/sqlite/model-snapshots.js +47 -0
  55. package/dist/src/store/sqlite/raptor.js +57 -0
  56. package/dist/src/store/sqlite/raw-transcript.js +134 -0
  57. package/dist/src/store/sqlite/schema.js +294 -0
  58. package/dist/src/store/sqlite/session-state.js +28 -0
  59. package/dist/src/store/sqlite/stats.js +66 -0
  60. package/dist/src/store/sqlite/utils.js +120 -0
  61. package/dist/src/store/sqlite.js +23 -1607
  62. package/extensions/dashboard-server/html.test.ts +50 -0
  63. package/extensions/dashboard-server/html.ts +1026 -0
  64. package/extensions/dashboard-server/index-reader.ts +130 -0
  65. package/extensions/dashboard-server/server.test.ts +131 -0
  66. package/extensions/dashboard-server/server.ts +505 -0
  67. package/extensions/dashboard-server/snapshot.ts +44 -0
  68. package/extensions/dashboard-server/state.ts +33 -0
  69. package/extensions/dashboard-server/types.ts +134 -0
  70. package/extensions/dashboard-server-s32.test.ts +195 -0
  71. package/extensions/dashboard-server.ts +7 -1431
  72. package/extensions/mega-commands.ts +33 -10
  73. package/extensions/mega-compact.test.ts +198 -43
  74. package/extensions/mega-compact.ts +3 -0
  75. package/extensions/mega-conflict-cmds.ts +6 -2
  76. package/extensions/mega-dashboard-cmds.ts +30 -23
  77. package/extensions/mega-db-cmds.ts +11 -3
  78. package/extensions/mega-events/agent-handlers.ts +262 -0
  79. package/extensions/mega-events/compact-handlers.ts +192 -0
  80. package/extensions/mega-events/context-handler.ts +290 -0
  81. package/extensions/mega-events/register.ts +37 -0
  82. package/extensions/mega-events/session-handlers.ts +165 -0
  83. package/extensions/mega-events.ts +15 -780
  84. package/extensions/mega-game-cmds.test.ts +137 -0
  85. package/extensions/mega-game-cmds.ts +122 -0
  86. package/extensions/mega-pipeline/compact.ts +366 -0
  87. package/extensions/mega-pipeline/memory-review.ts +46 -0
  88. package/extensions/mega-pipeline/recall.ts +165 -0
  89. package/extensions/mega-pipeline.ts +9 -537
  90. package/extensions/mega-runtime/helpers.ts +68 -0
  91. package/extensions/mega-runtime/query.ts +29 -0
  92. package/extensions/mega-runtime/state.test.ts +171 -0
  93. package/extensions/mega-runtime/state.ts +967 -0
  94. package/extensions/mega-runtime/widget.test.ts +185 -0
  95. package/extensions/mega-runtime/widget.ts +359 -0
  96. package/extensions/mega-runtime.ts +15 -1093
  97. package/package.json +4 -3
  98. package/src/config/themes.test.ts +116 -0
  99. package/src/config/themes.ts +124 -0
  100. package/src/game/scoring.test.ts +103 -0
  101. package/src/game/scoring.ts +158 -0
  102. package/src/store/sqlite/checkpoints.ts +204 -0
  103. package/src/store/sqlite/dedup-mirror.ts +114 -0
  104. package/src/store/sqlite/foundation.ts +63 -0
  105. package/src/store/sqlite/game-achievements.test.ts +80 -0
  106. package/src/store/sqlite/game-achievements.ts +147 -0
  107. package/src/store/sqlite/game-scores.test.ts +132 -0
  108. package/src/store/sqlite/game-scores.ts +168 -0
  109. package/src/store/sqlite/game-state.test.ts +89 -0
  110. package/src/store/sqlite/game-state.ts +87 -0
  111. package/src/store/sqlite/global-index.ts +305 -0
  112. package/src/store/sqlite/maintenance.ts +294 -0
  113. package/src/store/sqlite/memories.ts +217 -0
  114. package/src/store/sqlite/meta.ts +108 -0
  115. package/src/store/sqlite/model-snapshots.ts +83 -0
  116. package/src/store/sqlite/raptor.ts +107 -0
  117. package/src/store/sqlite/raw-transcript.ts +221 -0
  118. package/src/store/sqlite/schema.ts +305 -0
  119. package/src/store/sqlite/session-state.ts +38 -0
  120. package/src/store/sqlite/stats.ts +127 -0
  121. package/src/store/sqlite/utils.ts +125 -0
  122. package/src/store/sqlite.ts +23 -2204
@@ -1,782 +1,17 @@
1
- /**
2
- * mega-events.ts — the pi lifecycle event handlers.
1
+ /** mega-events.ts — barrel re-exporting all pi lifecycle event handlers.
3
2
  *
4
- * Wires every pi event the extension listens for: model/provider capture,
5
- * session lifecycle + state reset, auto-inline injection, agent/turn tracking,
6
- * and the auto-trigger compaction pipeline. Keeps the shared MegaRuntime in
7
- * sync and delegates the heavy lifting to the pipeline + command modules.
3
+ * Split into focused submodules under extensions/mega-events/:
4
+ * - register.ts: lastRuntime + registerEventHandlers (entry point)
5
+ * - session-handlers.ts: session lifecycle (model_select, session_start,
6
+ * session_tree, before_agent_start, session_shutdown)
7
+ * - agent-handlers.ts: agent/turn tracking (agent_start, agent_end,
8
+ * turn_start, turn_end)
9
+ * - context-handler.ts: live-trim auto-trigger (context event)
10
+ * - compact-handlers.ts: native compaction (session_before_compact,
11
+ * session_compact)
8
12
  */
9
-
10
- import type {
11
- ExtensionAPI,
12
- ExtensionContext,
13
- ContextEvent,
14
- SessionBeforeCompactEvent,
15
- SessionCompactEvent,
16
- } from "@earendil-works/pi-coding-agent";
17
- import type { AgentMessage } from "@earendil-works/pi-agent-core";
18
- import { normalizeSessionId } from "../src/store.js";
19
- import { openStore, appendRawTranscript, writeCheckpointEpoch, autoMaintain, type CheckpointEpoch } from "../src/store/sqlite.js";
20
- import { epochIdFor } from "../src/mirror/epoch.js";
21
- import { autoCompactCheck } from "../src/compact.js";
22
- import { estimateSessionTokens, estimateBlockTokens } from "../src/tokens.js";
23
- import {
24
- type MegaRuntime,
25
- recentUserQuery,
26
- WIDGET_KEY,
27
- } from "./mega-runtime.js";
28
- import {
29
- runCompact,
30
- doRecall,
31
- doRecallAsync,
32
- piCompactWouldNoop,
33
- runMemoryReview,
34
- } from "./mega-pipeline.js";
35
- import { recallMemoriesAndInline } from "../src/recall.js";
36
- import {
37
- driveNativeCompaction,
38
- type NativeCompactionResult,
39
- } from "./mega-compact-driver.js";
40
- import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
41
- import {
42
- pressureFromPct,
43
- pressureRatio,
44
- memoryReviewCadence,
45
- type MegaConfig,
46
- } from "./mega-config.js";
47
- import type { RawTranscriptRow } from "../src/store/sqlite.js";
48
- import { createHash } from "node:crypto";
49
-
50
- /**
51
- * Convert a pi AgentMessage to a RawTranscriptRow for the DB mirror.
52
- * content_bytes is canonical JSON (sorted keys) for deterministic hashing.
53
- * Returns null if the message has no usable content.
54
- */
55
- function toRawTranscriptRow(
56
- msg: AgentMessage,
57
- sessionId: string,
58
- epochId: string,
59
- ): RawTranscriptRow | null {
60
- // Narrow to Message union (has content + timestamp).
61
- const m = msg as { role?: string; content?: unknown; timestamp?: number; toolName?: string };
62
- const content = m.content;
63
- if (content == null || content === "") return null;
64
- // Canonical form: sort object keys for deterministic hashing.
65
- const contentBytes = typeof content === "string"
66
- ? content
67
- : JSON.stringify(content, Object.keys(content as object).sort());
68
- const contentHash = createHash("sha256").update(contentBytes).digest("hex");
69
- return {
70
- contentHash,
71
- sessionId,
72
- seq: 0, // assigned by appendRawTranscript (COALESCE(MAX(seq),0)+1)
73
- role: m.role ?? "unknown",
74
- contentBytes,
75
- toolName: m.toolName ?? null,
76
- messageTimestamp: m.timestamp ?? null,
77
- checkpointEpoch: epochId,
78
- };
79
- }
80
-
81
- /**
82
- * DIAG accessor for the headless test harness: the most recently constructed
83
- * MegaRuntime, so a test that loads the compiled extension via its default
84
- * export can read diag counters (diagLiveTrimFires / diagBeforeCompactFires /
85
- * diagBeforeCompactSupplied / diagAgentEndIdle) after firing synthetic events.
86
- * No-op in production — nothing reads this outside tests.
87
- */
88
- export let lastRuntime: MegaRuntime | undefined;
89
-
90
- /** Register all pi lifecycle event handlers. */
91
- export function registerEventHandlers(
92
- pi: ExtensionAPI,
93
- runtime: MegaRuntime,
94
- config: MegaConfig,
95
- ): void {
96
- lastRuntime = runtime;
97
- // ---- Session lifecycle (state reset points) -------------------------------
98
- // Capture model/provider whenever it changes (drives real cost estimation).
99
- pi.on("model_select", async (_event, ctx) => {
100
- runtime.captureModel(ctx);
101
- runtime.snapshot(ctx);
102
- });
103
-
104
- pi.on("session_start", async (event, ctx) => {
105
- runtime.resetRuntime(ctx.sessionManager.getSessionId());
106
- runtime.captureModel(ctx); // best-effort: ctx.model may be set by session start
107
- runtime.setStatus(
108
- ctx,
109
- config.auto ? "mega-compact: ready" : "mega-compact: manual only",
110
- );
111
- // S21: clear any stale memory block from a prior session.
112
- runtime.pendingMemoryRecallBlock = undefined;
113
- // Auto-inline on resume/fork/continue: stage the most relevant checkpoints
114
- // so the next before_agent_start prepends them to the system prompt.
115
- // Triggered whenever this session already has persisted checkpoints AND a
116
- // usable query — that covers reason "resume"/"fork" (explicit) and
117
- // reason "startup" (e.g. `pi --continue`s an existing session, which still
118
- // emits "startup" but with a populated message window). A brand-new empty
119
- // session has no checkpoints, so it's naturally excluded.
120
- if (config.autoInline) {
121
- const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
122
- const query = recentUserQuery(ctx);
123
- if (query && runtime.store.stats(sid).checkpointCount > 0) {
124
- // S17: use the async variant on resume so cross-repo HNSW recall can
125
- // augment when this repo's store is thin. session_start is an async-safe
126
- // point (unlike the mid-turn context handler, which stays sync).
127
- const r = await doRecallAsync(runtime, config, ctx, query, "resume", {
128
- crossRepo: config.crossRepoEnabled,
129
- });
130
- if (!r.empty) {
131
- runtime.pendingRecallBlock = r.block;
132
- const crossLabel = r.toInject.some((h) => h.repoId)
133
- ? " (cross-repo)"
134
- : "";
135
- runtime.setStatus(
136
- ctx,
137
- `mega-compact: recalled ${r.toInject.length} chkpt${crossLabel}`,
138
- );
139
- runtime.logger.info("auto-inline", {
140
- reason: event.reason,
141
- query,
142
- injected: r.toInject.map((h) => h.checkpoint.checkpointId),
143
- crossRepo: r.toInject.some((h) => h.repoId),
144
- });
145
- }
146
- }
147
- // S21: parallel memory recall. Same async context so we can await without
148
- // breaking the handler contract. Best-effort — never throws.
149
- try {
150
- const mr = await recallMemoriesAndInline({
151
- query,
152
- stateDir: runtime.getStateDir(),
153
- limit: 5,
154
- crossRepo: config.crossRepoEnabled,
155
- crossRepoCosine: config.crossRepoCosine,
156
- });
157
- if (!mr.empty) runtime.pendingMemoryRecallBlock = mr.block;
158
- } catch (err) {
159
- runtime.logger.warn("memory-recall skipped", { err: String(err) });
160
- }
161
- }
162
- // S27 Task 10: best-effort auto-maintenance on session start (prune rows
163
- // older than 30d, checkpoint WAL if >10MB, VACUUM if DB >100MB + >20%
164
- // freelist). Never blocks session start — swallows errors and logs a
165
- // one-line summary for diagnostics.
166
- try {
167
- const m = autoMaintain(runtime.currentStateDir);
168
- if (m && !m.endsWith("nothing to do")) runtime.logger.info("db-auto-maintain", { result: m });
169
- } catch (e) {
170
- runtime.logger.warn("db-auto-maintain-fail", { error: String(e) });
171
- }
172
- runtime.dashboard.event("session_start", {
173
- reason: event.reason,
174
- sessionId: runtime.rt.sessionId,
175
- });
176
- runtime.snapshot(ctx);
177
- });
178
-
179
- pi.on("session_tree", async (_event, ctx) => {
180
- // Branch navigation invalidates region indexes — reset checkpoint memory but
181
- // keep the on-disk store (markers replayed from entries below if needed).
182
- runtime.resetRuntime(ctx.sessionManager.getSessionId());
183
- runtime.setStatus(ctx, "mega-compact: ready (branch)");
184
- if (config.autoInline) {
185
- const query = recentUserQuery(ctx);
186
- if (query) {
187
- const r = doRecall(runtime, config, ctx, query, "resume");
188
- if (!r.empty) {
189
- runtime.pendingRecallBlock = r.block;
190
- runtime.logger.info("auto-inline", {
191
- reason: "session_tree",
192
- query,
193
- injected: r.toInject.map((h) => h.checkpoint.checkpointId),
194
- });
195
- }
196
- // S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
197
- try {
198
- const mr = await recallMemoriesAndInline({
199
- query,
200
- stateDir: runtime.getStateDir(),
201
- limit: 5,
202
- crossRepo: config.crossRepoEnabled,
203
- crossRepoCosine: config.crossRepoCosine,
204
- });
205
- if (!mr.empty) runtime.pendingMemoryRecallBlock = mr.block;
206
- } catch (err) {
207
- runtime.logger.warn("memory-recall skipped", { err: String(err) });
208
- }
209
- }
210
- }
211
- runtime.dashboard.event("session_tree", {
212
- sessionId: runtime.rt.sessionId,
213
- });
214
- runtime.snapshot(ctx);
215
- });
216
-
217
- // ---- Auto-inline injection point: prepend staged recall to systemPrompt ----
218
- pi.on("before_agent_start", async (event, ctx) => {
219
- runtime.captureModel(ctx); // most reliable point ctx.model is populated
220
- const cpBlock = runtime.pendingRecallBlock;
221
- const memBlock = runtime.pendingMemoryRecallBlock;
222
- if (!cpBlock && !memBlock) return;
223
- runtime.pendingRecallBlock = undefined;
224
- runtime.pendingMemoryRecallBlock = undefined;
225
- const composed = [cpBlock, memBlock].filter(Boolean).join("\n\n");
226
- return { systemPrompt: `${event.systemPrompt}\n\n${composed}` };
227
- });
228
-
229
- pi.on("session_shutdown", async (_event, ctx) => {
230
- runtime.setStatus(ctx, undefined);
231
- runtime.activeAgents = 0;
232
- runtime.currentTurn = 0;
233
- ctx.ui.setWidget(WIDGET_KEY, [], { placement: "aboveEditor" });
234
- });
235
-
236
- // ---- Agent tracking for real-time widget + status-line updates ---------
237
- pi.on("agent_start", async (_event, ctx) => {
238
- runtime.activeAgents++;
239
- runtime.dashboard.event("agent_start", {
240
- activeAgents: runtime.activeAgents,
241
- });
242
- // Surface live agent activity on the status line (toolbar), not just the
243
- // above-editor widget — otherwise concurrent agents look frozen.
244
- runtime.setStatus(
245
- ctx,
246
- `mega-compact: ▶ ${runtime.activeAgents} agent${runtime.activeAgents === 1 ? "" : "s"}`,
247
- );
248
- runtime.snapshot(ctx);
249
- });
250
-
251
- pi.on("agent_end", async (_event, ctx) => {
252
- runtime.activeAgents = Math.max(0, runtime.activeAgents - 1);
253
- runtime.dashboard.event("agent_end", {
254
- activeAgents: runtime.activeAgents,
255
- });
256
- if (runtime.activeAgents > 0) {
257
- runtime.setStatus(
258
- ctx,
259
- `mega-compact: ▶ ${runtime.activeAgents} agent${runtime.activeAgents === 1 ? "" : "s"}`,
260
- );
261
- } else {
262
- runtime.setStatus(
263
- ctx,
264
- config.auto ? "mega-compact: ready" : "mega-compact: manual only",
265
- );
266
- }
267
- // S16 continuation fallback: if the turn settled idle right after a live-trim
268
- // compaction AND there is queued work AND we haven't nudged recently, nudge
269
- // once so the agent continues (the live trim should make this rare). Guarded
270
- // to never busy-loop: one nudge per 30s, only when truly idle + queued.
271
- if ((config.auto || config.autoContinueLengthStop) && runtime.activeAgents === 0) {
272
- try {
273
- const idle = ctx.isIdle?.() ?? true;
274
- const queued = ctx.hasPendingMessages?.() ?? false;
275
- const now = Date.now();
276
- // DIAG (team-run relief): surface whether the agent is idle + over
277
- // threshold at agent_end so we can see if a mid-run durable-trim trigger
278
- // *should* have fired but didn't.
279
- const overThreshold =
280
- (runtime.lastCtxTokens ?? 0) >= runtime.effectiveThreshold;
281
- runtime.diagAgentEndIdle++;
282
- runtime.logger.info("agent-end-idle", {
283
- sessionId: runtime.rt.sessionId,
284
- idle,
285
- queued,
286
- overThreshold,
287
- ctxPct: runtime.lastCtxPercent,
288
- ctxTokens: runtime.lastCtxTokens,
289
- thresholdTokens: config.thresholdTokens,
290
- wouldNudge:
291
- idle &&
292
- (queued || overThreshold) &&
293
- now >= runtime.resumeNudgeUntil,
294
- });
295
- // S16+S24: MID-RUN DURABLE TRIM. During a long team run (sub-agents),
296
- // pi's native durable compaction only fires from _checkCompaction at
297
- // PARENT settle (agent-session.js:760/844), so the on-disk transcript +
298
- // context meter balloon to ~150k and never relieve until the very end
299
- // ("compacts but doesn't resume"). agent_end with activeAgents===0 is a
300
- // SAFE, settled point: calling ctx.compact() here does NOT abort an
301
- // in-flight turn (the S16 danger is only mid-turn). ctx.compact() runs
302
- // pi's flow, which fires our session_before_compact handler to supply
303
- // the durable trim (truncates the transcript from firstKeptEntryId).
304
- // Guarded three ways: only when truly idle + over threshold, only when
305
- // pi would actually compact (piCompactWouldNoop skips the user-facing
306
- // no-op throw), and debounced (one durable trim per 2s) to avoid
307
- // thrashing the transcript while sub-agents keep settling.
308
- //
309
- // FIX "compacts but doesn't resume": the manual ctx.compact() path
310
- // STOPS the agent loop (agent-session.js:1345). The old resume-nudge
311
- // was gated on `queued`, so when a sub-agent settled with no
312
- // *immediately* queued message, the trim fired but the nudge did not,
313
- // and the (stopped) session hung. The trim still fires on
314
- // `idle && overThreshold` — we intentionally do NOT add a `!queued`
315
- // guard, because that would suppress mid-run relief exactly during
316
- // team-run waves where queued is usually true and relief is needed
317
- // most. Instead we DECOUPLE the nudge from `queued`: after a durable
318
- // trim we ALWAYS nudge so the agent reliably restarts. Debounced 30s.
319
- let didDurableTrim = false;
320
- if (config.auto && idle && overThreshold && now >= runtime.debounceUntil) {
321
- // COMPACT-DEDUP FIX: skip the manual durable-trim trigger when pi's
322
- // NATIVE auto-compaction just fired (or is in-flight). pi emits
323
- // agent_end BEFORE its own _checkCompaction (per its docstring:
324
- // "Called after agent_end and before prompt submission"), so a
325
- // synchronous `piCompactWouldNoop` branch check misses a native
326
- // compaction that hasn't appended its entry yet — calling
327
- // ctx.compact() then races with pi and throws "Already compacted"
328
- // to the user. The `lastCompactAt` cooldown (updated by the
329
- // session_compact listener for EVERY compaction, native or
330
- // extension-supplied) closes that race window.
331
- const sinceCompact = now - (runtime.rt.lastNativeCompactAt ?? 0);
332
- if (sinceCompact < 10_000) {
333
- runtime.diagAgentEndDurableSkipRecent++;
334
- } else if (!piCompactWouldNoop(ctx)) {
335
- runtime.debounceUntil = now + 2000;
336
- runtime.diagAgentEndDurable++;
337
- runtime.logger.info("agent-end-durable-trigger", {
338
- sessionId: runtime.rt.sessionId,
339
- ctxTokens: runtime.lastCtxTokens,
340
- thresholdTokens: config.thresholdTokens,
341
- queued,
342
- });
343
- ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. Race-guarded by lastCompactAt cooldown above (ctx.compact returns void → throw is surfaced by pi as compaction_end; the cooldown prevents the call entirely).
344
- didDurableTrim = true;
345
- }
346
- }
347
- // Restart the agent after a mid-run durable trim (which stopped it), or
348
- // when it settled idle with queued work. Decoupled from `queued` for the
349
- // durable-trim case — see FIX note above. Debounced 30s; never blocks.
350
- const lengthStop = config.autoContinueLengthStop && runtime.rt.lengthStopPending;
351
- if (
352
- idle &&
353
- now >= runtime.resumeNudgeUntil &&
354
- ((config.auto && (didDurableTrim || queued)) || lengthStop)
355
- ) {
356
- runtime.resumeNudgeUntil = now + 30_000;
357
- if (runtime.rt.lengthStopPending) {
358
- runtime.rt.lengthStopPending = false; // one-shot: never re-fire for same stop
359
- runtime.dashboard.event("length_stop_continue", { turnIndex: runtime.currentTurn });
360
- runtime.logger.info("length_stop_continue", {
361
- sessionId: runtime.rt.sessionId,
362
- didDurableTrim,
363
- queued,
364
- });
365
- }
366
- // S28: when a length-stop (max-output-token truncation) fired WITHOUT a durable trim, do NOT claim a compaction happened
367
- // (nothing was compacted on the low-pressure length path). Branch the message so the nudge matches reality.
368
- const nudgeMsg = lengthStop && !didDurableTrim
369
- ? "[mega-compact] the last response hit the output-token cap; continue from where it stopped."
370
- : "[mega-compact] continue from the compacted context above.";
371
- pi.sendUserMessage(nudgeMsg);
372
- }
373
- } catch {
374
- /* non-fatal: a failed nudge never blocks */
375
- }
376
- }
377
- runtime.snapshot(ctx);
378
- });
379
-
380
- pi.on("turn_start", async (event, ctx) => {
381
- runtime.currentTurn = event.turnIndex;
382
- runtime.rt.lengthStopPending = false; // S28: re-arm defensively each user turn
383
- runtime.dashboard.event("turn_start", { turnIndex: event.turnIndex });
384
- runtime.snapshot(ctx);
385
- });
386
-
387
- pi.on("turn_end", async (event, ctx) => {
388
- runtime.dashboard.event("turn_end", { turnIndex: event.turnIndex });
389
- runtime.snapshot(ctx);
390
-
391
- // S20+S24: auto-review the conversation and persist durable memories. The
392
- // review cadence scales with pressure (memoryReviewCadence): as context
393
- // fills, the conversation is reviewed more often so memories keep pace with
394
- // faster churn. Best-effort + non-fatal: a review failure must never break
395
- // the agent loop. Debounced by the pressure-adjusted interval.
396
- if (config.memoryAutoReview && runtime.currentTurn > 0) {
397
- const cadence = memoryReviewCadence(
398
- runtime.pressureBand,
399
- config.memoryReviewInterval,
400
- );
401
- if (runtime.currentTurn % cadence === 0) {
402
- // S20+S24: review the conversation and persist durable memories. The
403
- // cadence scales with pressure (memoryReviewCadence): as context fills,
404
- // the conversation is reviewed more often so memories keep pace with
405
- // faster churn. Shared runMemoryReview body (also used on compact).
406
- const entries = ctx.sessionManager.getEntries();
407
- const view = runtime.engineView(
408
- entries.flatMap((e: any) => (e.message ? [e.message] : [])),
409
- );
410
- await runMemoryReview(runtime, view, "turn");
411
- }
412
- }
413
-
414
- // S28: detect max-output-token truncation. event.message.stopReason is the
415
- // pi-ai StopReason union; 'length' == generation hit max_tokens OUTPUT cap
416
- // (INPUT-orthogonal to context-window overflow). Arm the agent_end nudge.
417
- if (
418
- config.autoContinueLengthStop &&
419
- event.message.role === "assistant" &&
420
- event.message.stopReason === "length"
421
- ) {
422
- runtime.rt.lengthStopPending = true;
423
- runtime.dashboard.event("length_stop", { turnIndex: event.turnIndex });
424
- }
425
- });
426
-
427
- // ---- Auto-trigger: live trim (compact and continue) + native durable ----
428
- // S16 redesign: we NO LONGER call ctx.compact() from the auto-trigger by
429
- // default. That mapped to pi's MANUAL compaction path, which abort()s the
430
- // in-flight turn (agent-session.js:1345) and stops the agent. Instead:
431
- // - LIVE: return { messages: trimmedView } from the context event. This
432
- // feeds pi's transformContext (sdk.js:226 → agent-loop.js:180) so the
433
- // model sees a compacted window EVERY LLM call, with no abort. The turn
434
- // continues. We persist our recall checkpoint (the durable value) first.
435
- // - DURABLE: pi's NATIVE auto-compaction fires at agent-end
436
- // (agent-session.js:1565), continues (return hasQueuedMessages()), and
437
- // emits session_before_compact — where OUR driveNativeCompaction supplies
438
- // the summary and pi truncates the transcript on disk. No ctx.compact().
439
- // Legacy: MEGACOMPACT_LEGACY_DURABLE_TRIM=true restores the v0.4.28 ctx.compact
440
- // path (kept one release as rollback).
441
- pi.on("context", async (event: ContextEvent, ctx: ExtensionContext) => {
442
- if (!config.auto) return;
443
- const usage = ctx.getContextUsage();
444
- const pct = usage?.percent;
445
- // Always track context for the dashboard, even if we return early below.
446
- runtime.lastCtxTokens = usage?.tokens ?? null;
447
- runtime.lastCtxPercent = pct ?? null;
448
- runtime.lastCtxWindow = usage?.contextWindow ?? 0;
449
- runtime.snapshot(ctx);
450
-
451
- const messages = event.messages;
452
- const view = runtime.engineView(messages);
453
- const currentTokens =
454
- usage?.tokens ??
455
- estimateSessionTokens(view) ??
456
- Math.round(((pct ?? 0) / 100) * (usage?.contextWindow ?? 0));
457
-
458
- // S27 DB-mirror: append ALL incoming messages to raw_transcript.
459
- // Runs BEFORE fast-gate so every message is captured, even if we
460
- // don't compact this turn. Append is idempotent (content_hash PK).
461
- if (config.dbMirror) {
462
- try {
463
- const db = openStore(runtime.currentStateDir);
464
- const epochId = epochIdFor(runtime.rt.sessionId);
465
- for (const msg of messages) {
466
- const raw = toRawTranscriptRow(msg, runtime.rt.sessionId, epochId);
467
- if (raw) appendRawTranscript(db, raw);
468
- }
469
- } catch (e) {
470
- runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
471
- }
472
- }
473
-
474
- // S29 FAST GATE: drive the auto-trigger off the context % (the number the
475
- // menu bar shows), NOT the token count — the model under-reports tokens,
476
- // so a token-only gate misses the overshoot that causes max-output-token
477
- // truncation. The fire point is the tier's percent threshold (tierPct)
478
- // unless overridden by MEGACOMPACT_AUTO_PCT_TRIGGER. `custom` (absolute
479
- // MEGACOMPACT_THRESHOLD_TOKENS, tierPct null) is an explicit opt-out of
480
- // percent scaling — it keeps the token gate. When pct is unavailable
481
- // (window unknown / a model that doesn't report percent) a tiered config
482
- // falls back to the token gate (S27 boot-fallback guarantee) instead of
483
- // skipping compaction — a percent-only gate would regress that.
484
- let gatePassed = false;
485
- if (config.tierPct != null && pct != null) {
486
- const firePct = config.autoPctTrigger ?? config.tierPct;
487
- gatePassed = pct / 100 >= firePct;
488
- } else {
489
- // custom tier OR tiered-but-pct-unavailable → token gate (S27 fallback).
490
- if (currentTokens < runtime.effectiveThreshold) {
491
- runtime.diagCtxFastGate++;
492
- return;
493
- }
494
- const check = autoCompactCheck(currentTokens, runtime.effectiveThreshold); // SERVER-STYLE CONFIRM (local)
495
- if (!check.shouldCompact) {
496
- runtime.diagCtxNoCompact++;
497
- return;
498
- }
499
- gatePassed = true;
500
- }
501
- if (!gatePassed) {
502
- runtime.diagCtxFastGate++;
503
- return;
504
- }
505
-
506
- // Debounce so we don't fire on every context event past threshold.
507
- const now = Date.now();
508
- if (now < runtime.debounceUntil) {
509
- runtime.diagCtxDebounce++;
510
- return;
511
- }
512
- runtime.debounceUntil = now + 2000;
513
-
514
- // Adaptive compression (Fix E): scale compression strength + keepFrom depth
515
- // with how close we are to the model context limit. Null-safe: when the
516
- // token-fallback path ran (pct unavailable) use the token-basis pressure
517
- // (the same basis the runtime `pressure` getter uses for custom/no-window).
518
- const pressure = pct != null ? pressureFromPct(pct) : pressureRatio(currentTokens, runtime.effectiveThreshold);
519
- const ran = runCompact(pi, runtime, config, ctx, messages, {
520
- compressionPressure: pressure,
521
- });
522
- if (ran.skipped) {
523
- runtime.diagCtxRunSkipped++;
524
- return;
525
- }
526
-
527
- // S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
528
- // This makes the cache key stable across identical compactions.
529
- if (config.dbMirror) {
530
- try {
531
- const db = openStore(runtime.currentStateDir);
532
- const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
533
- const epoch: CheckpointEpoch = {
534
- epochId: epochIdFor(cpId),
535
- sessionId: runtime.rt.sessionId,
536
- startedSeq: 0,
537
- committedSeq: ran.result.compactedFrom,
538
- checkpointId: cpId,
539
- cutIndex: ran.result.compactedFrom,
540
- summaryMessageText: ran.result.summary,
541
- createdAt: Date.now(),
542
- };
543
- writeCheckpointEpoch(db, epoch);
544
- // S27 Task 6: Fire-and-forget dedup pipeline.
545
- // Deduplicates raw_transcript rows for the compacted range.
546
- try {
547
- const { dedupTranscript } = await import("../src/mirror/dedup.js");
548
- dedupTranscript(
549
- db,
550
- runtime.rt.sessionId,
551
- 0,
552
- ran.result.compactedFrom,
553
- );
554
- } catch (_dedupErr) {
555
- // Fire-and-forget: dedup failure is non-fatal
556
- }
557
- } catch (e) {
558
- runtime.logger.warn("db-mirror-epoch-fail", { error: String(e) });
559
- }
560
- }
561
-
562
- // LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
563
- // manual compact path aborts the in-flight turn — only used behind the flag.
564
- // Read live from env (in addition to the load-time config) so the flag can be
565
- // toggled per-test without reloading the module; config.legacyDurableTrim is
566
- // the cached default. (Mirrors how piCompactWouldNoop re-reads its floor.)
567
- const legacy =
568
- config.legacyDurableTrim ||
569
- process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" ||
570
- process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
571
- if (legacy) {
572
- // COMPACT-DEDUP FIX: same race guard as the agent_end path. Skip when a
573
- // NATIVE compaction just fired (avoids racing pi and surfacing a spurious
574
- // "Already compacted" / "Nothing to compact" toast). Uses lastNativeCompactAt
575
- // (NOT lastCompactAt, which runCompact also stamps for our own checkpoint).
576
- const sinceCompact = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
577
- if (sinceCompact < 10_000 || piCompactWouldNoop(ctx)) return;
578
- ctx.compact({ customInstructions: undefined }); // race-guarded by lastNativeCompactAt cooldown (ctx.compact returns void → not catchable; the cooldown prevents the call)
579
- return;
580
- }
581
-
582
- // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
583
- // Non-destructive: pi keeps the real transcript; only this LLM call sees the
584
- // trimmed window. We compute the cut on the engine view (pure, tested) then
585
- // slice the ORIGINAL pi AgentMessage[] from that index (lossless alignment,
586
- // mirroring dropCompactedRange) and prepend a user-role summary message.
587
- // A build failure or unsafe cut returns nothing (no trim this call — the
588
- // next context event retries). The anchor floor is read live from env (the
589
- // config value is the cached default) so it can be tuned per-test / per-run
590
- // without reloading the module.
591
- try {
592
- const anchorEnv = process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
593
- const anchorUserMessages =
594
- anchorEnv != null &&
595
- anchorEnv !== "" &&
596
- Number.isFinite(Number(anchorEnv))
597
- ? Number(anchorEnv)
598
- : config.anchorUserMessages;
599
- const cut = computeLiveTrimCut(view, {
600
- compactedFrom: ran.result.compactedFrom,
601
- summary: ran.result.summary,
602
- anchorUserMessages,
603
- });
604
- if (cut === null) {
605
- runtime.diagCtxCutNull++;
606
- runtime.logger.info("live-trim-skip", {
607
- sessionId: runtime.rt.sessionId,
608
- compactedFrom: ran.result.compactedFrom,
609
- viewLen: view.length,
610
- anchorUserMessages,
611
- });
612
- return; // unsafe / below anchor floor — no trim this call
613
- }
614
- const summaryMsg = liveTrimSummaryMessage({
615
- compactedFrom: ran.result.compactedFrom,
616
- summary: ran.result.summary,
617
- anchorUserMessages: config.anchorUserMessages,
618
- });
619
- // Synthesize a user-role AgentMessage carrying the compacted summary.
620
- const summaryAgentMsg = {
621
- role: "user" as const,
622
- content: summaryMsg.text,
623
- timestamp: Date.now(),
624
- } as unknown as AgentMessage;
625
- const recent = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
626
- runtime.snapshot(ctx);
627
- // DIAG (team-run relief): confirm the live trim actually fires + how big
628
- // the window still is. The return is non-durable (per-LLM-call only), so
629
- // this is the signal that the model is being fed a compacted view while
630
- // the on-disk transcript + context meter keep growing.
631
- runtime.diagLiveTrimFires++;
632
- runtime.logger.info("live-trim", {
633
- sessionId: runtime.rt.sessionId,
634
- inputMsgs: messages.length,
635
- outputMsgs: recent.length + 1,
636
- compactedFrom: cut,
637
- ctxPct: pct,
638
- ctxTokens: usage?.tokens ?? null,
639
- });
640
- return { messages: [summaryAgentMsg, ...recent] };
641
- } catch {
642
- runtime.diagCtxThrown++;
643
- return; // non-fatal: no trim this call; the next context event retries
644
- }
645
- });
646
-
647
- // ---- Supply a DURABLE trim to pi's native compaction (Fix B) ----------
648
- // We run the Trident pipeline to produce a compressed summary, then return
649
- // it as a CompactionResult. pi writes the summary into a compactionSummary
650
- // entry AND truncates the on-disk transcript from firstKeptEntryId. This is
651
- // the durable fix for "tokens grow on read": the trim survives resume, so
652
- // there is no full-reload + additive recall inflation.
653
- pi.on(
654
- "session_before_compact",
655
- async (event: SessionBeforeCompactEvent, ctx: ExtensionContext) => {
656
- runtime.resetRuntime(ctx.sessionManager.getSessionId());
657
- // DIAG (team-run relief): this is the ONLY durable-trim entry point. Log
658
- // every fire + whether we supplied a compaction (truncates transcript) or
659
- // fell through to {} (pi runs its own). If this is sparse during a team
660
- // run, the durable trim is firing too late (only at parent settle).
661
- const prep = event.preparation;
662
- runtime.diagBeforeCompactFires++;
663
- runtime.logger.info("before-compact-entry", {
664
- sessionId: runtime.rt.sessionId,
665
- reason: event.reason,
666
- hasPrep: !!prep,
667
- msgsToSummarize: prep?.messagesToSummarize?.length ?? 0,
668
- firstKeptEntryId: prep?.firstKeptEntryId ?? null,
669
- activeAgents: runtime.activeAgents,
670
- });
671
- if (!config.auto) return {}; // let pi run its own native compaction
672
- try {
673
- const result = driveNativeCompaction(event, runtime, config);
674
- if (result && result.compaction.summary?.trim()) {
675
- runtime.diagBeforeCompactSupplied++;
676
- runtime.logger.info("native-compact", {
677
- sessionId: runtime.rt.sessionId,
678
- firstKeptEntryId: result.compaction.firstKeptEntryId,
679
- tokensBefore: result.compaction.tokensBefore,
680
- summaryTokens: result.compaction.estimatedTokensAfter,
681
- });
682
- nudgeResume(pi, runtime);
683
- return { compaction: result.compaction };
684
- }
685
- // FIX "compacts but doesn't resume" + "Nothing to compact" regression:
686
- // when we have nothing to summarize (anchor floor protects everything →
687
- // messagesToSummarize empty) or our Trident/RAPTOR summary came back
688
- // EMPTY, pi's OWN compact() throws "Nothing to compact (session too
689
- // small)" and leaves the session stuck with no resume context. Instead
690
- // of returning {} (which makes pi run its throwing compact()), supply a
691
- // fallback compaction from prep.firstKeptEntryId with a minimal resume
692
- // summary. This ALWAYS injects a compact summary so the session
693
- // resumes, and never surfaces the "Nothing to compact" error to the user.
694
- const fb = fallbackCompaction(event);
695
- if (fb) {
696
- runtime.diagBeforeCompactSupplied++;
697
- runtime.logger.info("native-compact-fallback", {
698
- sessionId: runtime.rt.sessionId,
699
- firstKeptEntryId: fb.compaction.firstKeptEntryId,
700
- tokensBefore: fb.compaction.tokensBefore,
701
- reason: event.reason,
702
- });
703
- nudgeResume(pi, runtime);
704
- return { compaction: fb.compaction };
705
- }
706
- } catch (err) {
707
- runtime.logger.error("native-compact-failed", {
708
- sessionId: runtime.rt.sessionId,
709
- error: String(err instanceof Error ? err.message : err),
710
- });
711
- }
712
- // Absolute last resort: let pi run its own (may throw "Nothing to compact").
713
- return {};
714
- },
715
- );
716
-
717
- // COMPACT-DEDUP FIX: track EVERY compaction (native + extension-supplied)
718
- // so the agent_end durable-trim guard can skip a redundant ctx.compact()
719
- // when pi just compacted. Without this, agent_end fires ctx.compact()
720
- // synchronously AFTER pi's native auto-compaction appended a compaction
721
- // entry but BEFORE our branch read sees it on the next tick — racing
722
- // into a user-facing "Already compacted" throw. `lastCompactAt` is the
723
- // race-closing signal: any compaction (manual/threshold/overflow, ours
724
- // or pi's own) stamps it, and the agent_end guard skips for 10s.
725
- pi.on("session_compact", async (_event: SessionCompactEvent, _ctx: ExtensionContext) => {
726
- runtime.rt.lastNativeCompactAt = Date.now();
727
- runtime.rt.lastCompactAt = Date.now();
728
- runtime.logger.info("session-compacted", {
729
- sessionId: runtime.rt.sessionId,
730
- at: runtime.rt.lastCompactAt,
731
- });
732
- });
733
-
734
- /**
735
- * Build a minimal fallback compaction so pi never runs its throwing compact().
736
- *
737
- * Used when our Trident/RAPTOR summary is empty or there is nothing to
738
- * summarize (the anchor floor protects every message). We still record a
739
- * resume summary + truncate from prep.firstKeptEntryId so the session always
740
- * gets a compact summary and resumes. Returns undefined only if pi handed us
741
- * no preparation cut point at all.
742
- */
743
- function fallbackCompaction(
744
- event: SessionBeforeCompactEvent,
745
- ): NativeCompactionResult | undefined {
746
- const prep = event.preparation;
747
- if (!prep?.firstKeptEntryId) return undefined;
748
- // When messagesToSummarize is empty the anchor floor protects everything,
749
- // so firstKeptEntryId == current first entry and the trim is a no-op — but
750
- // we still record a resume summary so the session has context after compaction.
751
- const tokensBefore = prep.tokensBefore ?? 0;
752
- const summary =
753
- `[mega-compact] context compacted at ${tokensBefore.toLocaleString()} tokens ` +
754
- `(anchor floor active). Continue from the most recent messages above.`;
755
- return {
756
- compaction: {
757
- summary,
758
- firstKeptEntryId: prep.firstKeptEntryId,
759
- tokensBefore,
760
- estimatedTokensAfter: estimateBlockTokens(summary),
761
- },
762
- };
763
- }
764
-
765
- /**
766
- * Debounced resume-nudge: restart the agent loop after a compaction (which
767
- * may have stopped it). Idempotent — one nudge per 30s, never blocks.
768
- */
769
- function nudgeResume(pi: ExtensionAPI, runtime: MegaRuntime): void {
770
- try {
771
- const now = Date.now();
772
- if (now >= runtime.resumeNudgeUntil) {
773
- runtime.resumeNudgeUntil = now + 30_000;
774
- pi.sendUserMessage(
775
- "[mega-compact] continue from the compacted context above.",
776
- );
777
- }
778
- } catch {
779
- /* non-fatal: a failed nudge never blocks */
780
- }
781
- }
782
- }
13
+ export * from "./mega-events/register.js";
14
+ export * from "./mega-events/session-handlers.js";
15
+ export * from "./mega-events/agent-handlers.js";
16
+ export * from "./mega-events/context-handler.js";
17
+ export * from "./mega-events/compact-handlers.js";