pi-mega-compact 0.13.1 → 0.13.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/dist/extensions/dashboard-server/api-contracts/embedder-health.js +11 -0
  2. package/dist/extensions/dashboard-server/api-contracts/endpoints/registry.js +225 -0
  3. package/dist/extensions/dashboard-server/api-contracts/endpoints/types.js +1 -0
  4. package/dist/extensions/dashboard-server/api-contracts/endpoints.js +5 -224
  5. package/dist/extensions/dashboard-server/api-contracts/raptor.js +8 -0
  6. package/dist/extensions/dashboard-server/api-contracts.test/_helpers.js +32 -0
  7. package/dist/extensions/dashboard-server/routes-cache.test/_helpers.js +48 -0
  8. package/dist/extensions/dashboard-server/routes-embedder-health.js +67 -0
  9. package/dist/extensions/dashboard-server/routes-raptor.js +88 -0
  10. package/dist/extensions/dashboard-server/routes-setup.js +1 -1
  11. package/dist/extensions/dashboard-server/routes.js +2 -0
  12. package/dist/extensions/dashboard-server/server.js +5 -1
  13. package/dist/extensions/mega-cache-replay.test/_helpers.js +175 -0
  14. package/dist/extensions/mega-commands/dataCommands.js +151 -0
  15. package/dist/extensions/mega-commands/helpers.js +61 -0
  16. package/dist/extensions/mega-commands/historyCommands.js +97 -0
  17. package/dist/extensions/mega-commands/setupCommand.js +200 -0
  18. package/dist/extensions/mega-commands.js +14 -499
  19. package/dist/extensions/mega-compact-s38.test/_helpers.js +194 -0
  20. package/dist/extensions/mega-compact.test/_helpers.js +225 -0
  21. package/dist/extensions/mega-events/agent-handlers/agentEndHandler.js +211 -0
  22. package/dist/extensions/mega-events/agent-handlers/turnEndHandler/cacheStripe.js +49 -0
  23. package/dist/extensions/mega-events/agent-handlers/turnEndHandler/contextHealth.js +33 -0
  24. package/dist/extensions/mega-events/agent-handlers/turnEndHandler/errorRetry.js +387 -0
  25. package/dist/extensions/mega-events/agent-handlers/turnEndHandler/event.js +7 -0
  26. package/dist/extensions/mega-events/agent-handlers/turnEndHandler/gameScoring.js +55 -0
  27. package/dist/extensions/mega-events/agent-handlers/turnEndHandler/lengthStop.js +12 -0
  28. package/dist/extensions/mega-events/agent-handlers/turnEndHandler/memoryReview.js +22 -0
  29. package/dist/extensions/mega-events/agent-handlers/turnEndHandler/recordTurnRow.js +38 -0
  30. package/dist/extensions/mega-events/agent-handlers/turnEndHandler.js +36 -0
  31. package/dist/extensions/mega-events/agent-handlers.js +4 -799
  32. package/dist/extensions/mega-events/context-handler/afterCompact.js +163 -0
  33. package/dist/extensions/mega-events/context-handler/messageText.js +12 -0
  34. package/dist/extensions/mega-events/context-handler/tailResult.js +46 -0
  35. package/dist/extensions/mega-events/context-handler.js +11 -198
  36. package/dist/extensions/mega-events/separated-prompt.test/_helpers.js +25 -0
  37. package/dist/src/config.js +2 -0
  38. package/dist/src/dedup-engine.test/_helpers.js +55 -0
  39. package/dist/src/e2e.test/_helpers.js +52 -0
  40. package/dist/src/embedder.js +1 -0
  41. package/dist/src/httpEmbedder.js +6 -0
  42. package/dist/src/hyde.js +146 -0
  43. package/dist/src/importance.test/_helpers.js +5 -0
  44. package/dist/src/memoryGraph/sources/checkpoints.js +78 -0
  45. package/dist/src/memoryGraph/sources/edges.js +17 -0
  46. package/dist/src/memoryGraph/sources/flags.js +22 -0
  47. package/dist/src/memoryGraph/sources/helpers.js +14 -0
  48. package/dist/src/memoryGraph/sources/memories.js +88 -0
  49. package/dist/src/memoryGraph/sources/raptor.js +26 -0
  50. package/dist/src/memoryGraph/sources/turns.js +176 -0
  51. package/dist/src/memoryGraph/sources.js +15 -441
  52. package/dist/src/memoryGraph-gates.test/_helpers.js +120 -0
  53. package/dist/src/ratio.bench.test/_helpers.js +198 -0
  54. package/dist/src/recall/async.js +112 -0
  55. package/dist/src/recall/format.js +149 -0
  56. package/dist/src/recall/memory.js +62 -0
  57. package/dist/src/recall/reformulate.js +82 -0
  58. package/dist/src/recall/sync.js +169 -0
  59. package/dist/src/recall/types.js +1 -0
  60. package/dist/src/recall.js +13 -530
  61. package/dist/src/store/sqlite/schema/core.js +232 -0
  62. package/dist/src/store/sqlite/schema/game.js +65 -0
  63. package/dist/src/store/sqlite/schema/plan-v2.js +107 -0
  64. package/dist/src/store/sqlite/schema/turns.js +59 -0
  65. package/dist/src/store/sqlite/schema.js +8 -441
  66. package/dist/src/store/turns/contract-compliance.test/_helpers.js +21 -0
  67. package/dist/src/store/turns/sqlite-store/admin.js +171 -0
  68. package/dist/src/store/turns/sqlite-store/ctx.js +1 -0
  69. package/dist/src/store/turns/sqlite-store/read.js +111 -0
  70. package/dist/src/store/turns/sqlite-store/rows.js +42 -0
  71. package/dist/src/store/turns/sqlite-store/write.js +102 -0
  72. package/dist/src/store/turns/sqlite-store.js +31 -367
  73. package/dist/src/vectorStore/class.js +356 -0
  74. package/dist/src/vectorStore/hash.js +10 -0
  75. package/dist/src/vectorStore/types.js +2 -0
  76. package/dist/src/vectorStore.js +8 -359
  77. package/dist/src/vectorStore.test/_helpers.js +17 -0
  78. package/extensions/dashboard-client/dist/assets/{CacheTab-8HRroK9K.js → CacheTab-DPRHjYZ0.js} +2 -2
  79. package/extensions/dashboard-client/dist/assets/{CacheTab-8HRroK9K.js.map → CacheTab-DPRHjYZ0.js.map} +1 -1
  80. package/extensions/dashboard-client/dist/assets/{EventsTab-ncobZw-I.js → EventsTab-CgwYq-Qf.js} +2 -2
  81. package/extensions/dashboard-client/dist/assets/{EventsTab-ncobZw-I.js.map → EventsTab-CgwYq-Qf.js.map} +1 -1
  82. package/extensions/dashboard-client/dist/assets/{HealthTab-C6BI1EZG.js → HealthTab-BgOCjVzR.js} +2 -2
  83. package/extensions/dashboard-client/dist/assets/{HealthTab-C6BI1EZG.js.map → HealthTab-BgOCjVzR.js.map} +1 -1
  84. package/extensions/dashboard-client/dist/assets/MaintenanceTab-Yw1yFpsy.js +2 -0
  85. package/extensions/dashboard-client/dist/assets/MaintenanceTab-Yw1yFpsy.js.map +1 -0
  86. package/extensions/dashboard-client/dist/assets/MemoryMapTab-yHUX7Yzi.js +2 -0
  87. package/extensions/dashboard-client/dist/assets/MemoryMapTab-yHUX7Yzi.js.map +1 -0
  88. package/extensions/dashboard-client/dist/assets/{MetricsTab-B88XTCMk.js → MetricsTab-CcH9fP8P.js} +2 -2
  89. package/extensions/dashboard-client/dist/assets/{MetricsTab-B88XTCMk.js.map → MetricsTab-CcH9fP8P.js.map} +1 -1
  90. package/extensions/dashboard-client/dist/assets/{OverviewTab-CGZrSia-.js → OverviewTab-BOs8ZNev.js} +2 -2
  91. package/extensions/dashboard-client/dist/assets/{OverviewTab-CGZrSia-.js.map → OverviewTab-BOs8ZNev.js.map} +1 -1
  92. package/extensions/dashboard-client/dist/assets/{ReposTab-BZo5cNS1.js → ReposTab-B-MfSPvg.js} +2 -2
  93. package/extensions/dashboard-client/dist/assets/{ReposTab-BZo5cNS1.js.map → ReposTab-B-MfSPvg.js.map} +1 -1
  94. package/extensions/dashboard-client/dist/assets/{SessionsTab-C2RokJcw.js → SessionsTab-BmeN649Q.js} +2 -2
  95. package/extensions/dashboard-client/dist/assets/{SessionsTab-C2RokJcw.js.map → SessionsTab-BmeN649Q.js.map} +1 -1
  96. package/extensions/dashboard-client/dist/assets/SetupTab-nOknoph5.js +2 -0
  97. package/extensions/dashboard-client/dist/assets/SetupTab-nOknoph5.js.map +1 -0
  98. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CewYB2aA.js → TimeSavedCard-DnPg0L0z.js} +2 -2
  99. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CewYB2aA.js.map → TimeSavedCard-DnPg0L0z.js.map} +1 -1
  100. package/extensions/dashboard-client/dist/assets/{TopicsTab-DpVNIuaL.js → TopicsTab-BODHOg_b.js} +2 -2
  101. package/extensions/dashboard-client/dist/assets/{TopicsTab-DpVNIuaL.js.map → TopicsTab-BODHOg_b.js.map} +1 -1
  102. package/extensions/dashboard-client/dist/assets/{TurnsTab-KeuKKVC9.js → TurnsTab-BK0I7hKr.js} +2 -2
  103. package/extensions/dashboard-client/dist/assets/{TurnsTab-KeuKKVC9.js.map → TurnsTab-BK0I7hKr.js.map} +1 -1
  104. package/extensions/dashboard-client/dist/assets/{index-Cucgohga.js → index-mOlljTFI.js} +9 -9
  105. package/extensions/dashboard-client/dist/assets/index-mOlljTFI.js.map +1 -0
  106. package/extensions/dashboard-client/dist/assets/{useSSE-DJsqdPXC.js → useSSE-BYOtv1-3.js} +2 -2
  107. package/extensions/dashboard-client/dist/assets/{useSSE-DJsqdPXC.js.map → useSSE-BYOtv1-3.js.map} +1 -1
  108. package/extensions/dashboard-client/dist/index.html +1 -1
  109. package/extensions/dashboard-client/src/api/client.ts +16 -0
  110. package/extensions/dashboard-client/src/tabs/MaintenanceTab/ActionsCard.tsx +178 -0
  111. package/extensions/dashboard-client/src/tabs/MaintenanceTab/DbStatsCard.tsx +140 -0
  112. package/extensions/dashboard-client/src/tabs/MaintenanceTab/DebugBundleCard.tsx +109 -0
  113. package/extensions/dashboard-client/src/tabs/MaintenanceTab/HealthMitigationCard.tsx +98 -0
  114. package/extensions/dashboard-client/src/tabs/MaintenanceTab/SchemaHealthCard.tsx +108 -0
  115. package/extensions/dashboard-client/src/tabs/MaintenanceTab.tsx +12 -598
  116. package/extensions/dashboard-client/src/tabs/MemoryMapTab/MemoryMapView.tsx +476 -0
  117. package/extensions/dashboard-client/src/tabs/MemoryMapTab/RaptorTreeView.tsx +163 -0
  118. package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +56 -466
  119. package/extensions/dashboard-client/src/tabs/SetupTab/EmbedderHealthCard.tsx +109 -0
  120. package/extensions/dashboard-client/src/tabs/SetupTab/EmbedderSetup.tsx +468 -0
  121. package/extensions/dashboard-client/src/tabs/SetupTab.tsx +13 -483
  122. package/extensions/dashboard-server/api-contracts/embedder-health.ts +32 -0
  123. package/extensions/dashboard-server/api-contracts/endpoints/registry.ts +360 -0
  124. package/extensions/dashboard-server/api-contracts/endpoints/types.ts +350 -0
  125. package/extensions/dashboard-server/api-contracts/endpoints.ts +27 -648
  126. package/extensions/dashboard-server/api-contracts/index.ts +6 -0
  127. package/extensions/dashboard-server/api-contracts/raptor.ts +49 -0
  128. package/extensions/dashboard-server/api-contracts.test/_helpers.ts +47 -0
  129. package/extensions/dashboard-server/routes-cache.test/_helpers.ts +94 -0
  130. package/extensions/dashboard-server/routes-embedder-health.ts +77 -0
  131. package/extensions/dashboard-server/routes-raptor.ts +128 -0
  132. package/extensions/dashboard-server/routes-setup.ts +1 -1
  133. package/extensions/dashboard-server/routes.ts +2 -0
  134. package/extensions/dashboard-server/server.ts +4 -0
  135. package/extensions/mega-cache-replay.test/_helpers.ts +188 -0
  136. package/extensions/mega-commands/dataCommands.ts +169 -0
  137. package/extensions/mega-commands/helpers.ts +62 -0
  138. package/extensions/mega-commands/historyCommands.ts +117 -0
  139. package/extensions/mega-commands/setupCommand.ts +235 -0
  140. package/extensions/mega-commands.ts +25 -538
  141. package/extensions/mega-compact-s38.test/_helpers.ts +201 -0
  142. package/extensions/mega-compact.test/_helpers.ts +237 -0
  143. package/extensions/mega-events/agent-handlers/agentEndHandler.ts +238 -0
  144. package/extensions/mega-events/agent-handlers/turnEndHandler/cacheStripe.ts +74 -0
  145. package/extensions/mega-events/agent-handlers/turnEndHandler/contextHealth.ts +50 -0
  146. package/extensions/mega-events/agent-handlers/turnEndHandler/errorRetry.ts +421 -0
  147. package/extensions/mega-events/agent-handlers/turnEndHandler/event.ts +16 -0
  148. package/extensions/mega-events/agent-handlers/turnEndHandler/gameScoring.ts +73 -0
  149. package/extensions/mega-events/agent-handlers/turnEndHandler/lengthStop.ts +29 -0
  150. package/extensions/mega-events/agent-handlers/turnEndHandler/memoryReview.ts +42 -0
  151. package/extensions/mega-events/agent-handlers/turnEndHandler/recordTurnRow.ts +61 -0
  152. package/extensions/mega-events/agent-handlers/turnEndHandler.ts +64 -0
  153. package/extensions/mega-events/agent-handlers.ts +15 -861
  154. package/extensions/mega-events/context-handler/afterCompact.ts +213 -0
  155. package/extensions/mega-events/context-handler/messageText.ts +20 -0
  156. package/extensions/mega-events/context-handler/tailResult.ts +63 -0
  157. package/extensions/mega-events/context-handler.ts +17 -235
  158. package/extensions/mega-events/separated-prompt.test/_helpers.ts +35 -0
  159. package/package.json +1 -1
  160. package/src/config.ts +4 -0
  161. package/src/dedup-engine.test/_helpers.ts +79 -0
  162. package/src/e2e.test/_helpers.ts +59 -0
  163. package/src/embedder.ts +4 -0
  164. package/src/httpEmbedder.ts +7 -0
  165. package/src/hyde.ts +162 -0
  166. package/src/importance.test/_helpers.ts +5 -0
  167. package/src/memoryGraph/sources/checkpoints.ts +99 -0
  168. package/src/memoryGraph/sources/edges.ts +23 -0
  169. package/src/memoryGraph/sources/flags.ts +26 -0
  170. package/src/memoryGraph/sources/helpers.ts +13 -0
  171. package/src/memoryGraph/sources/memories.ts +108 -0
  172. package/src/memoryGraph/sources/raptor.ts +37 -0
  173. package/src/memoryGraph/sources/turns.ts +226 -0
  174. package/src/memoryGraph/sources.ts +19 -544
  175. package/src/memoryGraph-gates.test/_helpers.ts +164 -0
  176. package/src/ratio.bench.test/_helpers.ts +268 -0
  177. package/src/recall/async.ts +142 -0
  178. package/src/recall/format.ts +173 -0
  179. package/src/recall/memory.ts +90 -0
  180. package/src/recall/reformulate.ts +111 -0
  181. package/src/recall/sync.ts +196 -0
  182. package/src/recall/types.ts +63 -0
  183. package/src/recall.ts +25 -712
  184. package/src/store/sqlite/schema/core.ts +233 -0
  185. package/src/store/sqlite/schema/game.ts +66 -0
  186. package/src/store/sqlite/schema/plan-v2.ts +108 -0
  187. package/src/store/sqlite/schema/turns.ts +60 -0
  188. package/src/store/sqlite/schema.ts +13 -441
  189. package/src/store/turns/contract-compliance.test/_helpers.ts +38 -0
  190. package/src/store/turns/sqlite-store/admin.ts +249 -0
  191. package/src/store/turns/sqlite-store/ctx.ts +14 -0
  192. package/src/store/turns/sqlite-store/read.ts +174 -0
  193. package/src/store/turns/sqlite-store/rows.ts +55 -0
  194. package/src/store/turns/sqlite-store/write.ts +193 -0
  195. package/src/store/turns/sqlite-store.ts +33 -514
  196. package/src/vectorStore/class.ts +433 -0
  197. package/src/vectorStore/hash.ts +11 -0
  198. package/src/vectorStore/types.ts +58 -0
  199. package/src/vectorStore.test/_helpers.ts +20 -0
  200. package/src/vectorStore.ts +13 -489
  201. package/extensions/dashboard-client/dist/assets/MaintenanceTab-DZi1MnZI.js +0 -2
  202. package/extensions/dashboard-client/dist/assets/MaintenanceTab-DZi1MnZI.js.map +0 -1
  203. package/extensions/dashboard-client/dist/assets/MemoryMapTab-CjUa9zzw.js +0 -2
  204. package/extensions/dashboard-client/dist/assets/MemoryMapTab-CjUa9zzw.js.map +0 -1
  205. package/extensions/dashboard-client/dist/assets/SetupTab-C_BwmebC.js +0 -2
  206. package/extensions/dashboard-client/dist/assets/SetupTab-C_BwmebC.js.map +0 -1
  207. package/extensions/dashboard-client/dist/assets/index-Cucgohga.js.map +0 -1
@@ -0,0 +1,213 @@
1
+ /**
2
+ * context-handler/afterCompact.ts — post-compaction DB persistence + maintenance.
3
+ *
4
+ * Extracted from context-handler.ts (delegate-shell split). Runs after a live
5
+ * compaction: writes the checkpoint_epoch row (deterministic nonce), stamps turn
6
+ * epochs, rebuilds the auto-categorizing wiki, seeds the topic model from
7
+ * raw_transcript, and fire-and-forgets the dedup pipeline. All best-effort +
8
+ * non-fatal — a failure never breaks the agent loop.
9
+ */
10
+ import {
11
+ openStore,
12
+ writeCheckpointEpoch,
13
+ type CheckpointEpoch,
14
+ } from "../../../src/store/sqlite.js";
15
+ import { epochIdFor } from "../../../src/mirror/epoch.js";
16
+ import type { MegaRuntime } from "../../mega-runtime.js";
17
+ import { stampTurnsEpochFor } from "../../mega-turn-store.js";
18
+ import { TurnsConfig } from "../../../src/config/turns.js";
19
+ import { openTurnStore } from "../../../src/store/turns/connection.js";
20
+ import {
21
+ buildTopicModel,
22
+ createTopicStore,
23
+ bumpWikiCompactCounter,
24
+ } from "../../../src/topics/index.js";
25
+ import { TrigramEmbedder } from "../../../src/embedder.js";
26
+ import type { EmbeddedChunk } from "../../../src/topics/types.js";
27
+ import type { MegaConfig } from "../../mega-config.js";
28
+
29
+ /** Shape of the compact result consumed by the epoch/maintenance writes. */
30
+ interface CompactResult {
31
+ checkpointId?: string;
32
+ compactedFrom: number;
33
+ summary: string;
34
+ }
35
+
36
+ /**
37
+ * Persist the checkpoint_epoch, stamp turns, rebuild the auto-wiki, seed the
38
+ * topic model, and dedup — gated on dbMirror. Non-fatal end-to-end.
39
+ */
40
+ export async function persistEpochAndMaintain(
41
+ runtime: MegaRuntime,
42
+ config: MegaConfig,
43
+ ran: { result: CompactResult },
44
+ ): Promise<void> {
45
+ // S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
46
+ // This makes the cache key stable across identical compactions.
47
+ if (config.dbMirror) {
48
+ try {
49
+ const db = openStore(runtime.currentStateDir);
50
+ const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
51
+ const epoch: CheckpointEpoch = {
52
+ epochId: epochIdFor(cpId),
53
+ sessionId: runtime.rt.sessionId,
54
+ startedSeq: 0,
55
+ committedSeq: ran.result.compactedFrom,
56
+ checkpointId: cpId,
57
+ cutIndex: ran.result.compactedFrom,
58
+ summaryMessageText: ran.result.summary,
59
+ createdAt: Date.now(),
60
+ };
61
+ writeCheckpointEpoch(db, epoch);
62
+ // S50B: link this session's turns to the epoch that just compacted
63
+ // them (compression-by-conversation-epoch metrics). Isolated-store
64
+ // only; best-effort + non-fatal.
65
+ try {
66
+ stampTurnsEpochFor(
67
+ config,
68
+ runtime.rt.sessionId,
69
+ epoch.epochId,
70
+ runtime.currentStateDir,
71
+ );
72
+ } catch {
73
+ /* non-fatal: epoch stamping never breaks compaction */
74
+ }
75
+
76
+ // S51B: auto-categorizing wiki rebuild — every Nth compaction, derived
77
+ // from real context_chunks embeddings. Isolated-store only, gated on
78
+ // AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
79
+ // Fire regardless of dbMirror — context_chunks is the isolated store.
80
+ // Uses the already-open db when inside the dbMirror block; opens its own
81
+ // connection otherwise.
82
+ // from real context_chunks embeddings. Isolated-store only, gated on
83
+ // AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
84
+ try {
85
+ if (config.autoWikiEnabled && config.turnsDbEnabled) {
86
+ const every = Math.max(
87
+ 1,
88
+ TurnsConfig.WIKI_REBUILD_EVERY_N_COMPACTS,
89
+ );
90
+ const tdb = openTurnStore(runtime.currentStateDir);
91
+ const n = bumpWikiCompactCounter(tdb);
92
+ if (n % every === 0) {
93
+ const model = buildTopicModel(db, {
94
+ kRange: [TurnsConfig.WIKI_K_MIN, TurnsConfig.WIKI_K_MAX],
95
+ labelTopTerms: TurnsConfig.WIKI_LABEL_TOP_TERMS,
96
+ restarts: 5,
97
+ seed: 0x9e3779b9,
98
+ });
99
+ createTopicStore(runtime.currentStateDir).replaceTopicModel(
100
+ model,
101
+ );
102
+ runtime.logger.info("wiki_rebuild", {
103
+ clusterCount: model.k,
104
+ totalChunks: model.totalChunks,
105
+ method: "kmeans+tfidf",
106
+ criterion: model.criterion,
107
+ silhouetteScore: model.silhouetteScore,
108
+ uncalibrated: false,
109
+ });
110
+ }
111
+ }
112
+ } catch (wikiErr) {
113
+ runtime.logger.warn("wiki_rebuild_failed", {
114
+ error: String(wikiErr),
115
+ });
116
+ }
117
+
118
+ // D1: seed the topic model from raw_transcript when context_chunks is
119
+ // thin (pre-compaction). Gated on WIKI_SEED_FROM_TURNS; non-fatal.
120
+ // Seeds buildTopicModel with on-the-fly trigram embeddings from
121
+ // recent raw_transcript rows for the current session.
122
+ try {
123
+ if (
124
+ config.autoWikiEnabled &&
125
+ config.turnsDbEnabled &&
126
+ TurnsConfig.WIKI_SEED_FROM_TURNS
127
+ ) {
128
+ const floor = 50;
129
+ const countRow = db
130
+ .prepare(
131
+ `SELECT COUNT(*) AS cnt FROM context_chunks WHERE session_id = ?`,
132
+ )
133
+ .get(runtime.rt.sessionId) as { cnt: number } | undefined;
134
+ const chunkCount = countRow?.cnt ?? 0;
135
+ if (chunkCount < floor) {
136
+ const transcriptRows = db
137
+ .prepare(
138
+ `SELECT DISTINCT content_bytes
139
+ FROM raw_transcript
140
+ WHERE session_id = ?
141
+ AND length(content_bytes) > 10
142
+ ORDER BY seq ASC
143
+ LIMIT 200`,
144
+ )
145
+ .all(runtime.rt.sessionId) as Array<{
146
+ content_bytes: string;
147
+ }>;
148
+ if (transcriptRows.length > 0) {
149
+ const embedder = new TrigramEmbedder();
150
+ const seedChunks: EmbeddedChunk[] = [];
151
+ for (let i = 0; i < transcriptRows.length; i++) {
152
+ const text = transcriptRows[i].content_bytes.trim();
153
+ if (text.length === 0) continue;
154
+ const vec = embedder.embed(text);
155
+ seedChunks.push({
156
+ chunkId: `seed_transcript_${i}`,
157
+ sessionId: runtime.rt.sessionId,
158
+ vec,
159
+ text,
160
+ });
161
+ }
162
+ if (seedChunks.length > 0) {
163
+ const model = buildTopicModel(
164
+ db,
165
+ {
166
+ kRange: [
167
+ TurnsConfig.WIKI_K_MIN,
168
+ TurnsConfig.WIKI_K_MAX,
169
+ ],
170
+ labelTopTerms:
171
+ TurnsConfig.WIKI_LABEL_TOP_TERMS,
172
+ restarts: 5,
173
+ seed: 0x9e3779b9,
174
+ },
175
+ seedChunks,
176
+ );
177
+ createTopicStore(
178
+ runtime.currentStateDir,
179
+ ).replaceTopicModel(model);
180
+ runtime.logger.info("wiki_seed", {
181
+ clusterCount: model.k,
182
+ sourceChunks: seedChunks.length,
183
+ totalChunks: model.totalChunks,
184
+ method: "kmeans+tfidf",
185
+ });
186
+ }
187
+ }
188
+ }
189
+ }
190
+ } catch (seedErr) {
191
+ runtime.logger.warn("wiki_seed_failed", {
192
+ error: String(seedErr),
193
+ });
194
+ }
195
+
196
+ // S27 Task 6: Fire-and-forget dedup pipeline.
197
+ // Deduplicates raw_transcript rows for the compacted range.
198
+ try {
199
+ const { dedupTranscript } = await import("../../../src/mirror/dedup.js");
200
+ dedupTranscript(
201
+ db,
202
+ runtime.rt.sessionId,
203
+ 0,
204
+ ran.result.compactedFrom,
205
+ );
206
+ } catch (_dedupErr) {
207
+ // Fire-and-forget: dedup failure is non-fatal
208
+ }
209
+ } catch (e) {
210
+ runtime.logger.warn("db-mirror-epoch-fail", { error: String(e) });
211
+ }
212
+ }
213
+ }
@@ -0,0 +1,20 @@
1
+ /**
2
+ * context-handler/messageText.ts — best-effort text extraction helpers.
3
+ *
4
+ * Extracted from context-handler.ts (delegate-shell split). AgentMessage is a
5
+ * discriminated union; .content exists only on some variants
6
+ * (user/assistant/toolResult/custom). Narrow by role, never assume.
7
+ */
8
+ import type { AgentMessage } from "@earendil-works/pi-agent-core";
9
+
10
+ /** Best-effort text extraction from an AgentMessage for analytics/logging. */
11
+ export function messageContentText(m: AgentMessage): string {
12
+ const c = (m as { content?: unknown }).content;
13
+ if (typeof c === "string") return c;
14
+ if (Array.isArray(c)) {
15
+ return c
16
+ .map((b) => (b && typeof b === "object" && "text" in b ? String((b as { text?: string }).text ?? "") : ""))
17
+ .join(" ");
18
+ }
19
+ return "";
20
+ }
@@ -0,0 +1,63 @@
1
+ /**
2
+ * context-handler/tailResult.ts — buildTailResult(factory).
3
+ *
4
+ * Extracted from context-handler.ts (delegate-shell split). Constructs the
5
+ * closure that injects the staged recall/memory block as a user-role tail
6
+ * message at any context view-return point, plus optional cache-striping /
7
+ * message-separation prompt reshapes and prefix-stability logging.
8
+ */
9
+ import type { AgentMessage } from "@earendil-works/pi-agent-core";
10
+ import type { MegaRuntime } from "../../mega-runtime.js";
11
+ import type { MegaConfig } from "../../mega-config.js";
12
+ import { stagedForTail, withRecallTail } from "../recall-tail.js";
13
+ import { buildSeparatedPrompt, buildCacheOptimizedPrompt } from "../separated-prompt.js";
14
+ import { messageContentText } from "./messageText.js";
15
+
16
+ /**
17
+ * Build the tail injection closure. Returns undefined when nothing is staged
18
+ * (or the flags are OFF) so the caller falls through to its normal return.
19
+ */
20
+ export function buildTailResult(
21
+ runtime: MegaRuntime,
22
+ config: MegaConfig,
23
+ messages: readonly AgentMessage[],
24
+ ): ((msgs?: readonly AgentMessage[]) => { messages: AgentMessage[] } | undefined) {
25
+ // S53: helper to inject the staged recall/memory block as a user-role
26
+ // tail message at any view-return point. Returns undefined when nothing
27
+ // is staged (or the flag is OFF) so the caller falls through to its
28
+ // normal return. The F3 mirror append (above the gates) runs on the
29
+ // REAL transcript before any view is built, so the injected tail never
30
+ // reaches raw_transcript (PREVENT-PI: append-only, view-only).
31
+ const tailResult = (msgs?: readonly AgentMessage[]) => {
32
+ const base = msgs ?? messages;
33
+ if (!stagedForTail(runtime, config) && !config.messageSeparation && !config.cacheStriping) return undefined;
34
+ let result: AgentMessage[] = stagedForTail(runtime, config)
35
+ ? withRecallTail(base, runtime, config)
36
+ : (base as AgentMessage[]);
37
+ if (config.cacheStriping) {
38
+ result = buildCacheOptimizedPrompt(result);
39
+ } else if (config.messageSeparation) {
40
+ result = buildSeparatedPrompt(result);
41
+ }
42
+ // P2.5: log prefix stability (fire-and-forget, non-fatal).
43
+ // tailResult is sync, so use .then().catch() on the dynamic import.
44
+ if (result.length > 1) {
45
+ import("../../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
46
+ const stableScore = computeStabilityScore(
47
+ { content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 },
48
+ result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })),
49
+ );
50
+ runtime.logger.info("prefix_stability", {
51
+ stableScore: Number.isFinite(stableScore) ? stableScore : 0,
52
+ prefixMessages: result.length,
53
+ separation: config.messageSeparation ? "v2" : "off",
54
+ striping: config.cacheStriping ? "v3" : "off",
55
+ });
56
+ }).catch(() => {
57
+ // Non-fatal: stability logging is best-effort.
58
+ });
59
+ }
60
+ return { messages: result };
61
+ };
62
+ return tailResult;
63
+ }
@@ -1,9 +1,13 @@
1
1
  /**
2
2
  * mega-events/context-handler.ts — the context event handler (auto-trigger).
3
3
  *
4
- * Handles the live-trim compaction pipeline: DB-mirror append, fast-gate
5
- * threshold check, pipeline invocation, checkpoint epoch write, dedup, and
6
- * the live-trim message reconstruction that feeds pi's transformContext.
4
+ * Delegate-shell (extensions split): handles the live-trim compaction pipeline
5
+ * DB-mirror append, fast-gate threshold check, pipeline invocation, legacy
6
+ * durable compact, and the live-trim message reconstruction that feeds pi's
7
+ * transformContext. The extracted pieces live in ./context-handler/:
8
+ * - messageText.ts (messageContentText — best-effort text extraction)
9
+ * - tailResult.ts (buildTailResult — recall-tail injection factory)
10
+ * - afterCompact.ts (persistEpochAndMaintain — epoch/wiki/seed/dedup writes)
7
11
  */
8
12
  import type {
9
13
  ExtensionAPI,
@@ -13,47 +17,22 @@ import type {
13
17
  import type { AgentMessage } from "@earendil-works/pi-agent-core";
14
18
  import {
15
19
  openStore,
16
- writeCheckpointEpoch,
17
- type CheckpointEpoch,
18
20
  } from "../../src/store/sqlite.js";
19
- import { epochIdFor } from "../../src/mirror/epoch.js";
20
21
  import { autoCompactCheck } from "../../src/compact.js";
21
22
  import { estimateSessionTokens } from "../../src/tokens.js";
22
23
  import type { MegaRuntime } from "../mega-runtime.js";
23
24
  import { runCompact, piCompactWouldNoop } from "../mega-pipeline.js";
24
- import { stampTurnsEpochFor } from "../mega-turn-store.js";
25
- import { TurnsConfig } from "../../src/config/turns.js";
26
- import { openTurnStore } from "../../src/store/turns/connection.js";
27
- import {
28
- buildTopicModel,
29
- createTopicStore,
30
- bumpWikiCompactCounter,
31
- } from "../../src/topics/index.js";
32
- import { TrigramEmbedder } from "../../src/embedder.js";
33
- import type { EmbeddedChunk } from "../../src/topics/types.js";
34
- import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
35
25
  import {
36
26
  pressureFromPct,
37
27
  pressureRatio,
38
28
  type MegaConfig,
39
29
  } from "../mega-config.js";
40
30
  import { appendMirrorMessages } from "./mirror-append.js";
41
- import { stagedForTail, withRecallTail } from "./recall-tail.js";
42
- import { buildSeparatedPrompt, buildCacheOptimizedPrompt } from "./separated-prompt.js";
43
-
44
- /** Best-effort text extraction from an AgentMessage for analytics/logging.
45
- * AgentMessage is a discriminated union; .content exists only on some
46
- * variants (user/assistant/toolResult/custom). Narrow by role, never assume. */
47
- function messageContentText(m: AgentMessage): string {
48
- const c = (m as { content?: unknown }).content;
49
- if (typeof c === "string") return c;
50
- if (Array.isArray(c)) {
51
- return c
52
- .map((b) => (b && typeof b === "object" && "text" in b ? String((b as { text?: string }).text ?? "") : ""))
53
- .join(" ");
54
- }
55
- return "";
56
- }
31
+ import { epochIdFor } from "../../src/mirror/epoch.js";
32
+ import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
33
+ import { messageContentText } from "./context-handler/messageText.js";
34
+ import { buildTailResult } from "./context-handler/tailResult.js";
35
+ import { persistEpochAndMaintain } from "./context-handler/afterCompact.js";
57
36
 
58
37
  /** Register the context event handler (live-trim auto-trigger). */
59
38
  export function registerContextHandler(
@@ -82,40 +61,8 @@ export function registerContextHandler(
82
61
  // S53: helper to inject the staged recall/memory block as a user-role
83
62
  // tail message at any view-return point. Returns undefined when nothing
84
63
  // is staged (or the flag is OFF) so the caller falls through to its
85
- // normal return. The F3 mirror append (above the gates) runs on the
86
- // REAL transcript before any view is built, so the injected tail never
87
- // reaches raw_transcript (PREVENT-PI: append-only, view-only).
88
- const tailResult = (msgs?: readonly AgentMessage[]) => {
89
- const base = msgs ?? messages;
90
- if (!stagedForTail(runtime, config) && !config.messageSeparation && !config.cacheStriping) return undefined;
91
- let result: AgentMessage[] = stagedForTail(runtime, config)
92
- ? withRecallTail(base, runtime, config)
93
- : (base as AgentMessage[]);
94
- if (config.cacheStriping) {
95
- result = buildCacheOptimizedPrompt(result);
96
- } else if (config.messageSeparation) {
97
- result = buildSeparatedPrompt(result);
98
- }
99
- // P2.5: log prefix stability (fire-and-forget, non-fatal).
100
- // tailResult is sync, so use .then().catch() on the dynamic import.
101
- if (result.length > 1) {
102
- import("../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
103
- const stableScore = computeStabilityScore(
104
- { content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 },
105
- result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })),
106
- );
107
- runtime.logger.info("prefix_stability", {
108
- stableScore: Number.isFinite(stableScore) ? stableScore : 0,
109
- prefixMessages: result.length,
110
- separation: config.messageSeparation ? "v2" : "off",
111
- striping: config.cacheStriping ? "v3" : "off",
112
- });
113
- }).catch(() => {
114
- // Non-fatal: stability logging is best-effort.
115
- });
116
- }
117
- return { messages: result };
118
- };
64
+ // normal return.
65
+ const tailResult = buildTailResult(runtime, config, messages);
119
66
  // Always track context for the dashboard/widget, even when auto is off.
120
67
  // (v0.8 regression: !config.auto gate sat above this, leaving ctx stats
121
68
  // null -> widget '?% / ?/?' when auto disabled. Track first, THEN gate.)
@@ -303,174 +250,9 @@ export function registerContextHandler(
303
250
  return tailResult() ?? undefined;
304
251
  }
305
252
 
306
- // S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
307
- // This makes the cache key stable across identical compactions.
308
- if (config.dbMirror) {
309
- try {
310
- const db = openStore(runtime.currentStateDir);
311
- const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
312
- const epoch: CheckpointEpoch = {
313
- epochId: epochIdFor(cpId),
314
- sessionId: runtime.rt.sessionId,
315
- startedSeq: 0,
316
- committedSeq: ran.result.compactedFrom,
317
- checkpointId: cpId,
318
- cutIndex: ran.result.compactedFrom,
319
- summaryMessageText: ran.result.summary,
320
- createdAt: Date.now(),
321
- };
322
- writeCheckpointEpoch(db, epoch);
323
- // S50B: link this session's turns to the epoch that just compacted
324
- // them (compression-by-conversation-epoch metrics). Isolated-store
325
- // only; best-effort + non-fatal.
326
- try {
327
- stampTurnsEpochFor(
328
- config,
329
- runtime.rt.sessionId,
330
- epoch.epochId,
331
- runtime.currentStateDir,
332
- );
333
- } catch {
334
- /* non-fatal: epoch stamping never breaks compaction */
335
- }
336
-
337
- // S51B: auto-categorizing wiki rebuild — every Nth compaction, derived
338
- // from real context_chunks embeddings. Isolated-store only, gated on
339
- // AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
340
- // Fire regardless of dbMirror — context_chunks is the isolated store.
341
- // Uses the already-open db when inside the dbMirror block; opens its own
342
- // connection otherwise.
343
- // from real context_chunks embeddings. Isolated-store only, gated on
344
- // AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
345
- try {
346
- if (config.autoWikiEnabled && config.turnsDbEnabled) {
347
- const every = Math.max(
348
- 1,
349
- TurnsConfig.WIKI_REBUILD_EVERY_N_COMPACTS,
350
- );
351
- const tdb = openTurnStore(runtime.currentStateDir);
352
- const n = bumpWikiCompactCounter(tdb);
353
- if (n % every === 0) {
354
- const model = buildTopicModel(db, {
355
- kRange: [TurnsConfig.WIKI_K_MIN, TurnsConfig.WIKI_K_MAX],
356
- labelTopTerms: TurnsConfig.WIKI_LABEL_TOP_TERMS,
357
- restarts: 5,
358
- seed: 0x9e3779b9,
359
- });
360
- createTopicStore(runtime.currentStateDir).replaceTopicModel(
361
- model,
362
- );
363
- runtime.logger.info("wiki_rebuild", {
364
- clusterCount: model.k,
365
- totalChunks: model.totalChunks,
366
- method: "kmeans+tfidf",
367
- criterion: model.criterion,
368
- silhouetteScore: model.silhouetteScore,
369
- uncalibrated: false,
370
- });
371
- }
372
- }
373
- } catch (wikiErr) {
374
- runtime.logger.warn("wiki_rebuild_failed", {
375
- error: String(wikiErr),
376
- });
377
- }
378
-
379
- // D1: seed the topic model from raw_transcript when context_chunks is
380
- // thin (pre-compaction). Gated on WIKI_SEED_FROM_TURNS; non-fatal.
381
- // Seeds buildTopicModel with on-the-fly trigram embeddings from
382
- // recent raw_transcript rows for the current session.
383
- try {
384
- if (
385
- config.autoWikiEnabled &&
386
- config.turnsDbEnabled &&
387
- TurnsConfig.WIKI_SEED_FROM_TURNS
388
- ) {
389
- const floor = 50;
390
- const countRow = db
391
- .prepare(
392
- `SELECT COUNT(*) AS cnt FROM context_chunks WHERE session_id = ?`,
393
- )
394
- .get(runtime.rt.sessionId) as { cnt: number } | undefined;
395
- const chunkCount = countRow?.cnt ?? 0;
396
- if (chunkCount < floor) {
397
- const transcriptRows = db
398
- .prepare(
399
- `SELECT DISTINCT content_bytes
400
- FROM raw_transcript
401
- WHERE session_id = ?
402
- AND length(content_bytes) > 10
403
- ORDER BY seq ASC
404
- LIMIT 200`,
405
- )
406
- .all(runtime.rt.sessionId) as Array<{
407
- content_bytes: string;
408
- }>;
409
- if (transcriptRows.length > 0) {
410
- const embedder = new TrigramEmbedder();
411
- const seedChunks: EmbeddedChunk[] = [];
412
- for (let i = 0; i < transcriptRows.length; i++) {
413
- const text = transcriptRows[i].content_bytes.trim();
414
- if (text.length === 0) continue;
415
- const vec = embedder.embed(text);
416
- seedChunks.push({
417
- chunkId: `seed_transcript_${i}`,
418
- sessionId: runtime.rt.sessionId,
419
- vec,
420
- text,
421
- });
422
- }
423
- if (seedChunks.length > 0) {
424
- const model = buildTopicModel(
425
- db,
426
- {
427
- kRange: [
428
- TurnsConfig.WIKI_K_MIN,
429
- TurnsConfig.WIKI_K_MAX,
430
- ],
431
- labelTopTerms:
432
- TurnsConfig.WIKI_LABEL_TOP_TERMS,
433
- restarts: 5,
434
- seed: 0x9e3779b9,
435
- },
436
- seedChunks,
437
- );
438
- createTopicStore(
439
- runtime.currentStateDir,
440
- ).replaceTopicModel(model);
441
- runtime.logger.info("wiki_seed", {
442
- clusterCount: model.k,
443
- sourceChunks: seedChunks.length,
444
- totalChunks: model.totalChunks,
445
- method: "kmeans+tfidf",
446
- });
447
- }
448
- }
449
- }
450
- }
451
- } catch (seedErr) {
452
- runtime.logger.warn("wiki_seed_failed", {
453
- error: String(seedErr),
454
- });
455
- }
456
-
457
- // S27 Task 6: Fire-and-forget dedup pipeline.
458
- // Deduplicates raw_transcript rows for the compacted range.
459
- try {
460
- const { dedupTranscript } = await import("../../src/mirror/dedup.js");
461
- dedupTranscript(
462
- db,
463
- runtime.rt.sessionId,
464
- 0,
465
- ran.result.compactedFrom,
466
- );
467
- } catch (_dedupErr) {
468
- // Fire-and-forget: dedup failure is non-fatal
469
- }
470
- } catch (e) {
471
- runtime.logger.warn("db-mirror-epoch-fail", { error: String(e) });
472
- }
473
- }
253
+ // S27 DB-mirror: write checkpoint_epoch + stamp turn epochs + auto-wiki +
254
+ // topic seed + fire-and-forget dedup. Best-effort + non-fatal.
255
+ await persistEpochAndMaintain(runtime, config, ran);
474
256
 
475
257
  // LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
476
258
  // manual compact path aborts the in-flight turn — only used behind the flag.
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Shared helpers for the separated-prompt.test split files.
3
+ * Extracted from extensions/mega-events/separated-prompt.test.ts.
4
+ */
5
+ import { mkdtempSync } from "node:fs";
6
+ import { join } from "node:path";
7
+ import { tmpdir } from "node:os";
8
+ import { DatabaseSync } from "node:sqlite";
9
+ import { initSchema } from "../../../src/store/sqlite/schema.js";
10
+
11
+ /** Cast output to a plain Record array for role/content access in assertions. */
12
+ export type R = Record<string, unknown>;
13
+
14
+ export function asR(arr: unknown[]): R[] {
15
+ return arr as R[];
16
+ }
17
+
18
+ /** Create a temp DB with schema initialized, returns db + dir. */
19
+ export function createTestDb(): { db: DatabaseSync; dir: string } {
20
+ const dir = mkdtempSync(join(tmpdir(), "separated-prompt-test-"));
21
+ const db = new DatabaseSync(join(dir, "sqlite.db"));
22
+ db.exec("PRAGMA journal_mode=WAL");
23
+ db.exec("PRAGMA foreign_keys=ON");
24
+ initSchema(db);
25
+ return { db, dir };
26
+ }
27
+
28
+ /** Helper: create an AgentMessage-like object. */
29
+ export function msg(
30
+ role: string,
31
+ content: unknown,
32
+ extra?: Record<string, unknown>,
33
+ ): Record<string, unknown> {
34
+ return { role, content, ...extra };
35
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.13.1",
3
+ "version": "0.13.2",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-3-Clause",
package/src/config.ts CHANGED
@@ -128,3 +128,7 @@ export const RAG_RECALL_METRICS = (): boolean =>
128
128
  /** B4: Memory graph traversal (dashboard-oriented). */
129
129
  export const RAG_MEMORY_GRAPH = (): boolean =>
130
130
  ragFlag("MEGACOMPACT_MEMORY_GRAPH");
131
+
132
+ /** B5: HyDE — generate a hypothetical answer doc via LLM (HttpEmbedder only). Default OFF. */
133
+ export const RAG_HYDE_ENABLED = (): boolean =>
134
+ ragFlag("MEGACOMPACT_HYDE_ENABLED");