pi-mega-compact 0.11.7 → 0.11.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/dist/extensions/dashboard-server/gather-debug-keys.js +18 -0
  2. package/dist/extensions/dashboard-server/gather-debug.js +129 -0
  3. package/dist/extensions/dashboard-server/routes-maintenance.js +6 -0
  4. package/dist/extensions/dashboard-server/routes-memory-map.js +8 -15
  5. package/dist/extensions/mega-commands.js +65 -5
  6. package/dist/extensions/mega-config.js +7 -0
  7. package/dist/extensions/mega-events/agent-handlers.js +74 -5
  8. package/dist/extensions/mega-events/context-handler.js +132 -3
  9. package/dist/extensions/mega-runtime/snapshot.js +1 -1
  10. package/dist/extensions/mega-trim.js +10 -0
  11. package/dist/src/config/turns.js +2 -1
  12. package/dist/src/memoryGraph/embedding.js +50 -0
  13. package/dist/src/memoryGraph/gates.js +379 -0
  14. package/dist/src/memoryGraph/sources.js +390 -0
  15. package/dist/src/memoryGraph.js +133 -259
  16. package/dist/src/store/sqlite/embedding-cache.js +71 -0
  17. package/dist/src/store/sqlite.js +1 -0
  18. package/dist/src/topics/cluster.js +10 -2
  19. package/extensions/dashboard-client/dist/assets/{AchievementTiles-DR9cLGDL.js → AchievementTiles-DFitrb4Y.js} +2 -2
  20. package/extensions/dashboard-client/dist/assets/{AchievementTiles-DR9cLGDL.js.map → AchievementTiles-DFitrb4Y.js.map} +1 -1
  21. package/extensions/dashboard-client/dist/assets/{AchievementsTab-svNUgJ1e.js → AchievementsTab-PJrmq8pa.js} +2 -2
  22. package/extensions/dashboard-client/dist/assets/{AchievementsTab-svNUgJ1e.js.map → AchievementsTab-PJrmq8pa.js.map} +1 -1
  23. package/extensions/dashboard-client/dist/assets/CacheTab-C1WE7dEx.js +2 -0
  24. package/extensions/dashboard-client/dist/assets/CacheTab-C1WE7dEx.js.map +1 -0
  25. package/extensions/dashboard-client/dist/assets/{ConfigTab-CmmWheuW.js → ConfigTab-Dx-RpO8C.js} +2 -2
  26. package/extensions/dashboard-client/dist/assets/{ConfigTab-CmmWheuW.js.map → ConfigTab-Dx-RpO8C.js.map} +1 -1
  27. package/extensions/dashboard-client/dist/assets/{EventsTab-CFE7jLsa.js → EventsTab-C63BEWEs.js} +2 -2
  28. package/extensions/dashboard-client/dist/assets/{EventsTab-CFE7jLsa.js.map → EventsTab-C63BEWEs.js.map} +1 -1
  29. package/extensions/dashboard-client/dist/assets/{GameTab-CJAX_th_.js → GameTab-DHpJE6km.js} +2 -2
  30. package/extensions/dashboard-client/dist/assets/{GameTab-CJAX_th_.js.map → GameTab-DHpJE6km.js.map} +1 -1
  31. package/extensions/dashboard-client/dist/assets/MaintenanceTab-BSv0GfBb.js +2 -0
  32. package/extensions/dashboard-client/dist/assets/MaintenanceTab-BSv0GfBb.js.map +1 -0
  33. package/extensions/dashboard-client/dist/assets/MemoryMapTab-Cg1fgEx_.js +2 -0
  34. package/extensions/dashboard-client/dist/assets/MemoryMapTab-Cg1fgEx_.js.map +1 -0
  35. package/extensions/dashboard-client/dist/assets/{MetricsTab-CHQscixy.js → MetricsTab-DCahxNOQ.js} +2 -2
  36. package/extensions/dashboard-client/dist/assets/{MetricsTab-CHQscixy.js.map → MetricsTab-DCahxNOQ.js.map} +1 -1
  37. package/extensions/dashboard-client/dist/assets/{OverviewTab-DBs4DJdd.js → OverviewTab-RYvO1SZL.js} +2 -2
  38. package/extensions/dashboard-client/dist/assets/{OverviewTab-DBs4DJdd.js.map → OverviewTab-RYvO1SZL.js.map} +1 -1
  39. package/extensions/dashboard-client/dist/assets/{ReposTab-DeFz76L5.js → ReposTab-BDGr_UXb.js} +2 -2
  40. package/extensions/dashboard-client/dist/assets/{ReposTab-DeFz76L5.js.map → ReposTab-BDGr_UXb.js.map} +1 -1
  41. package/extensions/dashboard-client/dist/assets/{SessionsTab-BWNsj1OB.js → SessionsTab-ByrTnHaB.js} +2 -2
  42. package/extensions/dashboard-client/dist/assets/{SessionsTab-BWNsj1OB.js.map → SessionsTab-ByrTnHaB.js.map} +1 -1
  43. package/extensions/dashboard-client/dist/assets/{SetupTab-uS6A0xmw.js → SetupTab-CELafKis.js} +2 -2
  44. package/extensions/dashboard-client/dist/assets/{SetupTab-uS6A0xmw.js.map → SetupTab-CELafKis.js.map} +1 -1
  45. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CiqCOQuL.js → TimeSavedCard-BLWtmfr_.js} +2 -2
  46. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CiqCOQuL.js.map → TimeSavedCard-BLWtmfr_.js.map} +1 -1
  47. package/extensions/dashboard-client/dist/assets/TopicsTab-8Fsh_Igx.js +2 -0
  48. package/extensions/dashboard-client/dist/assets/TopicsTab-8Fsh_Igx.js.map +1 -0
  49. package/extensions/dashboard-client/dist/assets/{TurnsTab-CVz4f4-o.js → TurnsTab-D7mgXDe2.js} +2 -2
  50. package/extensions/dashboard-client/dist/assets/{TurnsTab-CVz4f4-o.js.map → TurnsTab-D7mgXDe2.js.map} +1 -1
  51. package/extensions/dashboard-client/dist/assets/{index-DrktNOoT.js → index-h3jgEs6G.js} +10 -10
  52. package/extensions/dashboard-client/dist/assets/{index-DrktNOoT.js.map → index-h3jgEs6G.js.map} +1 -1
  53. package/extensions/dashboard-client/dist/assets/{useSSE-Bf5LOLSi.js → useSSE-C5lmipBF.js} +2 -2
  54. package/extensions/dashboard-client/dist/assets/{useSSE-Bf5LOLSi.js.map → useSSE-C5lmipBF.js.map} +1 -1
  55. package/extensions/dashboard-client/dist/index.html +1 -1
  56. package/extensions/dashboard-client/src/api/client.ts +5 -0
  57. package/extensions/dashboard-client/src/memory-map-layout.ts +161 -0
  58. package/extensions/dashboard-client/src/memory-map-shapes.tsx +126 -0
  59. package/extensions/dashboard-client/src/tabs/CacheTab.tsx +29 -2
  60. package/extensions/dashboard-client/src/tabs/MaintenanceTab.tsx +101 -0
  61. package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +201 -165
  62. package/extensions/dashboard-client/src/tabs/ReposTab.tsx +21 -0
  63. package/extensions/dashboard-client/src/tabs/TopicsTab.tsx +1 -1
  64. package/extensions/dashboard-server/api-contracts/index.ts +3 -1
  65. package/extensions/dashboard-server/api-contracts/maintenance.ts +13 -0
  66. package/extensions/dashboard-server/api-contracts/memory-map.ts +14 -6
  67. package/extensions/dashboard-server/gather-debug-keys.ts +18 -0
  68. package/extensions/dashboard-server/gather-debug.ts +136 -0
  69. package/extensions/dashboard-server/routes-maintenance.ts +7 -0
  70. package/extensions/dashboard-server/routes-memory-map.ts +21 -29
  71. package/extensions/mega-commands.ts +63 -6
  72. package/extensions/mega-config.ts +22 -0
  73. package/extensions/mega-events/agent-handlers.ts +88 -5
  74. package/extensions/mega-events/context-handler.ts +152 -3
  75. package/extensions/mega-runtime/snapshot.ts +1 -1
  76. package/extensions/mega-trim.ts +15 -0
  77. package/package.json +1 -1
  78. package/src/config/turns.ts +5 -2
  79. package/src/memoryGraph/embedding.ts +63 -0
  80. package/src/memoryGraph/gates.ts +451 -0
  81. package/src/memoryGraph/sources.ts +484 -0
  82. package/src/memoryGraph.ts +188 -350
  83. package/src/store/sqlite/embedding-cache.ts +81 -0
  84. package/src/store/sqlite.ts +1 -0
  85. package/src/topics/cluster.ts +11 -1
  86. package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js +0 -2
  87. package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js.map +0 -1
  88. package/extensions/dashboard-client/dist/assets/MaintenanceTab-P4SUsvMf.js +0 -2
  89. package/extensions/dashboard-client/dist/assets/MaintenanceTab-P4SUsvMf.js.map +0 -1
  90. package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js +0 -2
  91. package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js.map +0 -1
  92. package/extensions/dashboard-client/dist/assets/TopicsTab-BWWQxltv.js +0 -2
  93. package/extensions/dashboard-client/dist/assets/TopicsTab-BWWQxltv.js.map +0 -1
@@ -0,0 +1,136 @@
1
+ /**
2
+ * gather-debug.ts — assemble a diagnostic bundle for bug reports.
3
+ *
4
+ * POST /api/maintenance/gather-debug → DebugBundleResponse. Reads local
5
+ * SQLite + FS only (PREVENT-PI-004: zero network). Each section is non-fatal:
6
+ * a read failure records `{ error }` for that section and continues, so the
7
+ * bundle always returns 200 with whatever could be gathered.
8
+ */
9
+ import { readFileSync, existsSync } from "node:fs";
10
+ import { createRequire } from "node:module";
11
+ import type { ServerResponse } from "node:http";
12
+ import type { RouteContext } from "./routes-core.js";
13
+ import type { DebugBundleResponse } from "./api-contracts/maintenance.js";
14
+ import { defaultMetricsPath, defaultEventsPath } from "../../src/monitoring.js";
15
+ import { ownVersion } from "../mega-runtime/helpers.js";
16
+ import { MEGACOMPACT_ENV_KEYS } from "./gather-debug-keys.js";
17
+
18
+ /** Event-name patterns that indicate a critical/compaction class of error. */
19
+ const CRITICAL_PATTERN =
20
+ /critical|overflow|already[.\s-]?compacted|compaction[.\s-]?failed|stream[.\s-]?death|poisoned|prefix_stability/i;
21
+
22
+ const require = createRequire(import.meta.url);
23
+
24
+ /** Lazy store opener (mirrors routes-maintenance.ts's lazyStoreUtils). */
25
+ function openStoreDb(ctx: RouteContext) {
26
+ const mod = require("../../src/store/sqlite/utils.js") as typeof import("../../src/store/sqlite/utils.js");
27
+ return mod.openStore(ctx.stateDir);
28
+ }
29
+
30
+ /** Lazy maintenance loaders (mirrors routes-maintenance.ts's lazyMaintenance). */
31
+ function loadMaintenance() {
32
+ return require("../../src/store/sqlite/maintenance.js") as typeof import("../../src/store/sqlite/maintenance.js");
33
+ }
34
+
35
+ /** Send the debug bundle as JSON. Always 200 (partial on read failure). */
36
+ export function sendGatherDebug(res: ServerResponse, ctx: RouteContext): void {
37
+ const result: DebugBundleResponse = {
38
+ builtAt: Date.now(),
39
+ version: ownVersion(),
40
+ stateDir: ctx.stateDir,
41
+ config: {},
42
+ schemaHealth: { error: "not fetched" },
43
+ storeStats: { error: "not fetched" },
44
+ dashboardSnapshot: { error: "not fetched" },
45
+ recentEvents: [],
46
+ criticalEvents: [],
47
+ };
48
+
49
+ // 1. Config flags (no secrets — MEGACOMPACT_* only)
50
+ for (const key of MEGACOMPACT_ENV_KEYS) {
51
+ result.config[key] = process.env[key] ?? "<default>";
52
+ }
53
+
54
+ // 2. Schema health
55
+ try {
56
+ const maint = loadMaintenance();
57
+ const integrity = maint.integrityCheck(ctx.stateDir);
58
+ const db = openStoreDb(ctx);
59
+ const metaRow = db
60
+ .prepare("SELECT value FROM meta WHERE key = ?")
61
+ .get("schema_version") as { value: string } | undefined;
62
+ const schemaVersion = metaRow ? Number(metaRow.value) : 0;
63
+ const fkRows = db.prepare("PRAGMA foreign_key_check").all() as Array<Record<string, unknown>>;
64
+ result.schemaHealth = {
65
+ type: "schema-health" as const,
66
+ schemaVersion,
67
+ integrity,
68
+ fkCheck: fkRows.map((r) => JSON.stringify(r)),
69
+ columns: [],
70
+ healthy: true,
71
+ };
72
+ } catch (e) {
73
+ result.schemaHealth = { error: e instanceof Error ? e.message : String(e) };
74
+ }
75
+
76
+ // 3. Store stats
77
+ try {
78
+ const maint = loadMaintenance();
79
+ const raw = maint.getDbStats(ctx.stateDir);
80
+ result.storeStats = {
81
+ type: "db-stats" as const,
82
+ tables: Object.entries(raw.tableCounts).map(([table, rowCount]) => ({ table, rowCount })),
83
+ storage: {
84
+ files: { dbBytes: raw.dbBytes, walBytes: raw.walBytes, shmBytes: raw.shmBytes },
85
+ pageSize: raw.pageSize,
86
+ pageCount: raw.pageCount,
87
+ freelistPages: raw.freelistPages,
88
+ },
89
+ };
90
+ } catch (e) {
91
+ result.storeStats = { error: e instanceof Error ? e.message : String(e) };
92
+ }
93
+
94
+ // 4. Dashboard snapshot
95
+ try {
96
+ const path = defaultMetricsPath(ctx.stateDir);
97
+ if (existsSync(path)) {
98
+ const raw = readFileSync(path, "utf-8");
99
+ result.dashboardSnapshot = JSON.parse(raw) as unknown;
100
+ } else {
101
+ result.dashboardSnapshot = { error: "no snapshot" };
102
+ }
103
+ } catch (e) {
104
+ result.dashboardSnapshot = { error: e instanceof Error ? e.message : String(e) };
105
+ }
106
+
107
+ // 5. Recent + critical events (tail ~200 lines of events.log)
108
+ try {
109
+ const path = defaultEventsPath(ctx.stateDir);
110
+ if (existsSync(path)) {
111
+ const lines = readFileSync(path, "utf-8").split("\n").filter(Boolean);
112
+ for (const line of lines.slice(-200)) {
113
+ try {
114
+ const ev = JSON.parse(line) as unknown;
115
+ result.recentEvents.push(ev);
116
+ if (CRITICAL_PATTERN.test(line)) result.criticalEvents.push(ev);
117
+ } catch {
118
+ continue; // skip unparseable line (PREVENT-001)
119
+ }
120
+ }
121
+ }
122
+ } catch (e) {
123
+ result.recentEvents = [{ error: e instanceof Error ? e.message : String(e) }];
124
+ }
125
+
126
+ // Send (local helper to avoid importing sendJson from a non-exporting module)
127
+ try {
128
+ const body = JSON.stringify(result);
129
+ res.writeHead(200, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
130
+ res.end(body);
131
+ } catch {
132
+ // last resort — never leave the response hanging
133
+ res.writeHead(500, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
134
+ res.end('{"error":"failed to serialize debug bundle"}');
135
+ }
136
+ }
@@ -13,6 +13,7 @@
13
13
 
14
14
  import { createRequire } from "node:module";
15
15
  import type { IncomingMessage, ServerResponse } from "node:http";
16
+ import { sendGatherDebug } from "./gather-debug.js";
16
17
 
17
18
  import type { RouteContext } from "./routes-core.js";
18
19
 
@@ -77,6 +78,12 @@ export function handleMaintenance(
77
78
  return true;
78
79
  }
79
80
 
81
+ // ── POST /api/maintenance/gather-debug ──────────────────────────
82
+ if (method === "POST" && url === "/api/maintenance/gather-debug") {
83
+ sendGatherDebug(res, ctx);
84
+ return true;
85
+ }
86
+
80
87
  // ── GET /api/maintenance ──────────────────────────────────────────
81
88
  if (method === "GET" && url === "/api/maintenance") {
82
89
  sendDbStats(res, ctx);
@@ -1,7 +1,8 @@
1
1
  /**
2
- * routes-memory-map.ts — GET /api/memory-map route handler (S46).
2
+ * routes-memory-map.ts — GET /api/memory-map route handler (S46/D3).
3
3
  *
4
4
  * Builds a memory graph from the local SQLite store and returns it as JSON.
5
+ * D3: surfaces nodeType on each node and the GraphValidationReport.
5
6
  * Uses an inline require to the pi-agnostic src/memoryGraph.ts builder.
6
7
  */
7
8
  import { createRequire } from "node:module";
@@ -25,14 +26,7 @@ export function handleMemoryMap(
25
26
 
26
27
  // guardrails-allow PREVENT-PI-004: local loopback-only dashboard server — reads the local SQLite store for graph data.
27
28
  const { buildMemoryGraph } = _req("../../src/memoryGraph.js") as {
28
- buildMemoryGraph: (opts?: {
29
- sessionId?: string;
30
- similarityThreshold?: number;
31
- maxEdgesPerNode?: number;
32
- stateDir?: string;
33
- includeRaptorEdges?: boolean;
34
- includeTemporalEdges?: boolean;
35
- }) => MemoryGraphReturn;
29
+ buildMemoryGraph: (sessionId: string, stateDir: string) => MemoryGraphReturn;
36
30
  };
37
31
 
38
32
  type MemoryGraphReturn = {
@@ -48,6 +42,7 @@ export function handleMemoryMap(
48
42
  topicSummary: string | undefined;
49
43
  decisionCount: number;
50
44
  textSnippet: string;
45
+ nodeType: "checkpoint" | "turn" | "turn-content" | "memory";
51
46
  }>;
52
47
  edges: Array<{
53
48
  source: string;
@@ -56,31 +51,25 @@ export function handleMemoryMap(
56
51
  type: "temporal" | "semantic" | "raptor_parent";
57
52
  }>;
58
53
  metadata: {
59
- sessionCount: number;
60
- totalCheckpoints: number;
54
+ totalNodes: number;
61
55
  totalEdges: number;
62
- semanticEdgeCount: number;
63
- temporalEdgeCount: number;
64
- raptorEdgeCount: number;
65
- similarityThresholdUsed: number;
56
+ avgWeight: number;
57
+ nodeTypeBreakdown: Record<string, number>;
58
+ edgeTypeBreakdown: Record<string, number>;
59
+ };
60
+ validation: {
61
+ gatesRun: number;
62
+ gatesPassed: number;
63
+ dropped: { nodes: number; edges: number };
64
+ warnings: Array<{ gate: string; code: string; count: number }>;
65
+ sources: { checkpoint: number; turn: number; turnContent: number; memory: number };
66
66
  builtAt: number;
67
67
  };
68
68
  };
69
69
 
70
- const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : undefined;
71
- const threshold = typeof parsed.query.threshold === "string" ? Number.parseFloat(parsed.query.threshold) : undefined;
72
- const maxEdges = typeof parsed.query.maxEdgesPerNode === "string" ? Number.parseInt(parsed.query.maxEdgesPerNode, 10) : undefined;
73
- const raptorEdges = parsed.query.includeRaptorEdges === "false" ? false : true;
74
- const temporalEdges = parsed.query.includeTemporalEdges === "false" ? false : true;
70
+ const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : "";
75
71
 
76
- const graph = buildMemoryGraph({
77
- sessionId,
78
- similarityThreshold: threshold,
79
- maxEdgesPerNode: maxEdges,
80
- stateDir: ctx.stateDir,
81
- includeRaptorEdges: raptorEdges,
82
- includeTemporalEdges: temporalEdges,
83
- });
72
+ const graph = buildMemoryGraph(sessionId, ctx.stateDir);
84
73
 
85
74
  const nodes: MemoryMapNode[] = graph.nodes.map((n) => ({
86
75
  id: n.id,
@@ -94,10 +83,13 @@ export function handleMemoryMap(
94
83
  topicSummary: n.topicSummary,
95
84
  decisionCount: n.decisionCount,
96
85
  textSnippet: n.textSnippet,
86
+ // D3: pass through nodeType for UI node-shape encoding
87
+ nodeType: n.nodeType,
97
88
  }));
98
89
 
99
90
  const edges: MemoryMapEdgeEntry[] = graph.edges as MemoryMapEdgeEntry[];
100
- const response: MemoryMapResponse = { nodes, edges, metadata: graph.metadata };
91
+ // D3: surface the validation report (optional in the response type for backward compat)
92
+ const response: MemoryMapResponse = { nodes, edges, metadata: graph.metadata, validation: graph.validation };
101
93
 
102
94
  res.writeHead(200, { "Content-Type": "application/json" });
103
95
  res.end(JSON.stringify(response));
@@ -11,13 +11,13 @@ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
11
11
  import { normalizeSessionId } from "../src/store.js";
12
12
  import { listCheckpoints, latestModelSnapshot, countInjectedGlobal, listRepoRegistry } from "../src/store/sqlite.js";
13
13
  import { decompressSmart } from "../src/store/compression.js";
14
- import { loadMetrics, fpRate, p95, defaultMetricsPath } from "../src/monitoring.js";
14
+ import { loadMetrics, fpRate, p95, defaultMetricsPath, defaultEventsPath } from "../src/monitoring.js";
15
15
  import { type MegaRuntime, C, recentUserQuery } from "./mega-runtime.js";
16
16
  import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
17
17
  import { vectorStats, vectorRepoStats, vectorDataInvariant } from "../src/vectorStore.js";
18
18
  import type { MegaConfig } from "./mega-config.js";
19
19
  import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: local subprocess detection only (no network)
20
- import { mkdirSync, writeFileSync } from "node:fs";
20
+ import { mkdirSync, writeFileSync, existsSync, readFileSync } from "node:fs";
21
21
  import path from "node:path";
22
22
  import os from "node:os";
23
23
  import { createRequire } from "node:module";
@@ -280,6 +280,52 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
280
280
  },
281
281
  });
282
282
 
283
+ /**
284
+ * Try to peek at recent events.log and mega-compact.log for low recall quality signals.
285
+ * Best-effort, non-fatal — returns true if a low-quality signal was found.
286
+ */
287
+ /**
288
+ * Try to peek at recent events.log and mega-compact.log for low recall quality signals.
289
+ * Best-effort, non-fatal — returns true if a low-quality signal was found.
290
+ */
291
+ function checkRecallQuality(stateDir: string): boolean {
292
+ const eventPaths = [
293
+ defaultEventsPath(stateDir),
294
+ path.join(stateDir, "mega-compact.log"),
295
+ ];
296
+ for (const fp of eventPaths) {
297
+ try {
298
+ if (!existsSync(fp)) continue;
299
+ const buf = readFileSync(fp, { encoding: "utf-8" });
300
+ // Only scan last 64 KiB of the file
301
+ const tail = buf.length > 65536 ? buf.slice(buf.length - 65536) : buf;
302
+ const lines = tail.split("\n").filter(Boolean);
303
+ for (const line of lines) {
304
+ try {
305
+ // guardrails-allow PREVENT-001: JSON.parse with null check
306
+ const parsed = JSON.parse(line);
307
+ if (typeof parsed !== "object" || parsed === null) continue;
308
+ // Check for recall_metrics_low_quality event (latest format)
309
+ if (parsed.event === "recall_metrics_low_quality" && parsed.score !== undefined) {
310
+ if (typeof parsed.score === "number" && parsed.score < 0.4) return true;
311
+ }
312
+ // Check for recall_metrics with low relevanceScore
313
+ if (parsed.event === "recall_metrics" && parsed.relevanceScore !== undefined) {
314
+ if (typeof parsed.relevanceScore === "number" && parsed.relevanceScore < 0.4) return true;
315
+ }
316
+ // Check for RecallQualityEvent format from monitoring.ts
317
+ if (parsed.score !== undefined && typeof parsed.score === "number" && parsed.score < 0.4) return true;
318
+ } catch {
319
+ // skip unparseable lines
320
+ }
321
+ }
322
+ } catch {
323
+ // non-fatal: skip if can't read
324
+ }
325
+ }
326
+ return false;
327
+ }
328
+
283
329
  pi.registerCommand("mega-setup", {
284
330
  description:
285
331
  "Local embedding setup wizard — detect available embedders and configure mega-compact.",
@@ -355,6 +401,11 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
355
401
  // require error
356
402
  }
357
403
 
404
+ // --- 2.5 Quality check ---
405
+ // Scan recent events for low recall quality (best-effort, non-fatal)
406
+ const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
407
+ const lowRecallQuality = checkRecallQuality(stateDir);
408
+
358
409
  // --- 3. Build choices ---
359
410
  interface Choice {
360
411
  label: string;
@@ -402,6 +453,11 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
402
453
 
403
454
  // --- 4. Prompt user ---
404
455
  log(`Current embedder: ${currentEmbedder}`);
456
+ if (lowRecallQuality) {
457
+ log(
458
+ "Tap: recent recall quality is low. Consider switching to Ollama or llama.cpp for better results.",
459
+ );
460
+ }
405
461
  let selected = "trigram";
406
462
  if (choices.length > 1) {
407
463
  try {
@@ -435,7 +491,6 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
435
491
  const envContent = lines.join("\n") + "\n";
436
492
 
437
493
  // Write .mega-compact.env to state dir
438
- const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
439
494
  try {
440
495
  mkdirSync(stateDir, { recursive: true });
441
496
  writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
@@ -457,8 +512,6 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
457
512
  `export MEGACOMPACT_EMBEDDING_URL="${url}"`,
458
513
  ];
459
514
  const envContent = lines.join("\n") + "\n";
460
-
461
- const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
462
515
  try {
463
516
  mkdirSync(stateDir, { recursive: true });
464
517
  writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
@@ -477,7 +530,11 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
477
530
  "ONNX Runtime configuration is experimental.\n\nSet MEGACOMPACT_EMBEDDING_URL to your local ONNX server URL, or see docs for manual setup.\nKeeping TrigramEmbedder for now.",
478
531
  );
479
532
  } else {
480
- log("Keeping TrigramEmbedder (default). No configuration needed.");
533
+ const qualityTip =
534
+ lowRecallQuality
535
+ ? " Recent recall quality is low — consider running /mega-setup again to switch to Ollama or llama.cpp for better results."
536
+ : "";
537
+ log(`Keeping TrigramEmbedder (default). No configuration needed.${qualityTip}`);
481
538
  }
482
539
  } catch (e) {
483
540
  ctx.ui.notify(`[mega-compact] /mega-setup failed: ${String(e)}`);
@@ -172,6 +172,9 @@ export interface MegaConfig {
172
172
  * from volatile tool results so the prompt-cache prefix stays stable.
173
173
  * Default OFF (flag-OFF = byte-identical pre-sprint). */
174
174
  messageSeparation: boolean;
175
+ /** P3: Cache-aware striping (PLAN_V2 Phase 3). Inserts stability-ordered
176
+ * cache stripes between summaries and thread. Default OFF. */
177
+ cacheStriping: boolean;
175
178
  debug: boolean;
176
179
  /** Master reconciliation: TUI shutdown widget (MEGACOMPACT_TUI_WIDGET=0 to disable). */
177
180
  tuiWidget: boolean;
@@ -183,6 +186,18 @@ export interface MegaConfig {
183
186
  ragRecallMetrics: boolean;
184
187
  /** S57 B4: Memory graph traversal (dashboard-oriented). */
185
188
  ragMemoryGraph: boolean;
189
+ /** D1: Seed initial wiki topic model from live turns when no context_chunks exist yet. */
190
+ wikiSeedFromTurns: boolean;
191
+ /** D3 Source A: Include structural turn nodes in the memory graph (metadata only, no content). */
192
+ memoryGraphSeedTurns: boolean;
193
+ /** D3 Source B: Include raw_transcript content nodes in the memory graph (requires dbMirror). */
194
+ memoryGraphSeedTurnContent: boolean;
195
+ /** D3 Source C: Include memory review nodes in the memory graph. */
196
+ memoryGraphSeedMemories: boolean;
197
+ /** D3 edges: Stricter cosine floor for cross-type edges (e.g. turn↔checkpoint). */
198
+ memoryGraphCrossTypeThreshold: number;
199
+ /** D3 edges: Cosine floor for within-type semantic edges. */
200
+ memoryGraphWithinTypeThreshold: number;
186
201
  }
187
202
 
188
203
  function envFlag(name: string, fallback: number): number {
@@ -345,11 +360,18 @@ export function loadConfig(): MegaConfig {
345
360
  windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
346
361
  recallTailInject: envBool("MEGACOMPACT_RECALL_TAIL_INJECT", true),
347
362
  messageSeparation: envBool("MEGACOMPACT_MESSAGE_SEPARATION", false),
363
+ cacheStriping: envBool("MEGACOMPACT_CACHE_STRIPING", false),
348
364
  tuiWidget: envBool("MEGACOMPACT_TUI_WIDGET", true),
349
365
  ragQueryReformulation: envBool("MEGACOMPACT_QUERY_REFORMULATION", false),
350
366
  ragTieredRouter: envBool("MEGACOMPACT_TIERED_ROUTER", false),
351
367
  ragRecallMetrics: envBool("MEGACOMPACT_RECALL_METRICS", false),
352
368
  ragMemoryGraph: envBool("MEGACOMPACT_MEMORY_GRAPH", false),
369
+ wikiSeedFromTurns: envBool("MEGACOMPACT_WIKI_SEED_FROM_TURNS", true),
370
+ memoryGraphSeedTurns: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS", true),
371
+ memoryGraphSeedTurnContent: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true),
372
+ memoryGraphSeedMemories: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_MEMORIES", true),
373
+ memoryGraphCrossTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_CROSS_TYPE_THRESHOLD", 0.85),
374
+ memoryGraphWithinTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_WITHIN_TYPE_THRESHOLD", 0.7),
353
375
  // D.1: env-overridable recompact delta (minimum context growth % before
354
376
  // re-compacting instead of replaying the cached live trim). Default 50.
355
377
  recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),
@@ -26,6 +26,15 @@ import {
26
26
  import { safeSendUserMessage } from "./send-safe.js";
27
27
  import { maybeSendProviderOutageAdvisory } from "./outage-advisor.js";
28
28
  import { vectorStats } from "../../src/vectorStore.js";
29
+ import { defaultEmbedder } from "../../src/embedder.js";
30
+ import {
31
+ storeTopicEmbedding,
32
+ loadTopicEmbedding,
33
+ detectTopicShift,
34
+ } from "./separated-prompt.js";
35
+ import {
36
+ refreshStripeAssignments as writeStripeAssignments,
37
+ } from "../../src/cache-stripe-impl.js";
29
38
 
30
39
  /** Register agent/turn tracking event handlers. */
31
40
  export function registerAgentHandlers(
@@ -167,9 +176,20 @@ export function registerAgentHandlers(
167
176
  // context-handler.ts:258-287 so both call sites stay in sync.
168
177
  const cooldownMs = config.raceGuardStrict ? 30_000 : 10_000;
169
178
  const sinceCompact = now - (runtime.rt.lastNativeCompactAt ?? 0);
179
+ // CRITICAL-OVER ESCAPE HATCH: when context is critically over the
180
+ // window (>= 90%), force the durable ctx.compact() even if
181
+ // piCompactWouldNoop says it would no-op ("Already compacted").
182
+ // Without this, the on-disk transcript never truncates and the
183
+ // session stays overflowed on every resume — the "Already
184
+ // compacted" + overflow death-spiral (2026-08-01 incident). The
185
+ // cooldown still applies (we don't spam compact), but the no-op
186
+ // gate is bypassed when critical. pi's ctx.compact() may throw
187
+ // "Already compacted" in the truly-no-op case — wrapped in
188
+ // try/catch so the throw is non-fatal.
189
+ const criticalOver = (runtime.lastCtxPercent ?? 0) >= 90;
170
190
  if (sinceCompact < cooldownMs) {
171
191
  runtime.diagAgentEndDurableSkipRecent++;
172
- } else if (!piCompactWouldNoop(ctx)) {
192
+ } else if (criticalOver || !piCompactWouldNoop(ctx)) {
173
193
  runtime.debounceUntil = now + 2000;
174
194
  runtime.diagAgentEndDurable++;
175
195
  runtime.logger.info("agent-end-durable-trigger", {
@@ -186,6 +206,7 @@ export function registerAgentHandlers(
186
206
  // would re-check before it lands.
187
207
  const stamp = runtime.rt.lastNativeCompactAt;
188
208
  const liveSid = runtime.rt.sessionId;
209
+ const liveCritical = criticalOver;
189
210
  setTimeout(() => {
190
211
  try {
191
212
  if (runtime.rt.sessionId !== liveSid) return; // session reset
@@ -198,16 +219,22 @@ export function registerAgentHandlers(
198
219
  since2 < cooldownMs
199
220
  )
200
221
  return;
201
- if (piCompactWouldNoop(ctx)) return;
222
+ // CRITICAL-OVER ESCAPE HATCH: bypass the no-op gate when
223
+ // critical (captured at handler time) so the durable trim fires.
224
+ if (!liveCritical && piCompactWouldNoop(ctx)) return;
202
225
  ctx.compact({
203
226
  customInstructions: undefined,
204
- }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated.
227
+ }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated. May throw "Already compacted" when the escape hatch forces it — caught below.
205
228
  } catch {
206
- /* non-fatal */
229
+ /* non-fatal: "Already compacted" throws are expected when the critical-over hatch forces a no-op compact */
207
230
  }
208
231
  }, 500);
209
232
  } else {
210
- ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. Race-guarded by lastNativeCompactAt cooldown above (ctx.compact returns void → throw is surfaced by pi as compaction_end; the cooldown prevents the call entirely).
233
+ try {
234
+ ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. May throw "Already compacted" when the critical-over escape hatch forces it — caught.
235
+ } catch {
236
+ /* non-fatal */
237
+ }
211
238
  }
212
239
  didDurableTrim = true;
213
240
  }
@@ -777,5 +804,61 @@ export function registerAgentHandlers(
777
804
  } catch {
778
805
  /* non-fatal: a classifier/retry failure never breaks the agent loop */
779
806
  }
807
+
808
+ // P3.5: topic-shift stripe refresh. When MEGACOMPACT_CACHE_STRIPING is ON,
809
+ // embed the current turn, detect topic shift vs. the previous turn, and
810
+ // refresh cache stripe assignments when shifted (or first turn). Best-effort:
811
+ // failures log + continue, never break the agent loop.
812
+ if (config.cacheStriping) {
813
+ try {
814
+ const stateDir = runtime.currentStateDir;
815
+ const sessionId = runtime.rt.sessionId;
816
+ const currentTurn = event.turnIndex;
817
+ const embedder = defaultEmbedder();
818
+
819
+ // (a) Extract text from the assistant's response for topic embedding.
820
+ let textToEmbed = "";
821
+ const msg = event.message;
822
+ if (msg.role === "assistant" && Array.isArray(msg.content)) {
823
+ for (const part of msg.content) {
824
+ if ("text" in part && typeof part.text === "string") {
825
+ textToEmbed += part.text + " ";
826
+ }
827
+ }
828
+ }
829
+ textToEmbed = textToEmbed.trim() || `turn-${currentTurn}`;
830
+ const embedding = new Float32Array(embedder.embed(textToEmbed));
831
+ storeTopicEmbedding(stateDir, sessionId, currentTurn, embedding);
832
+
833
+ // (b) Load previous turn's embedding and detect topic shift.
834
+ const prevEmb = currentTurn > 0
835
+ ? loadTopicEmbedding(stateDir, sessionId, currentTurn - 1)
836
+ : null;
837
+ const shifted = currentTurn === 0 || detectTopicShift(embedding, prevEmb);
838
+
839
+ // (c) If shifted (or first turn), refresh stripe assignments (WRITE path).
840
+ let chunkCount = 0;
841
+ if (shifted) {
842
+ chunkCount = writeStripeAssignments(
843
+ stateDir,
844
+ undefined,
845
+ embedder,
846
+ (detail) => runtime.logger.info("stripe_refresh_detail", {
847
+ detail,
848
+ turnIndex: currentTurn,
849
+ }),
850
+ );
851
+ }
852
+
853
+ // (d) Log the refresh outcome.
854
+ runtime.logger.info("stripe_refresh", {
855
+ turnIndex: currentTurn,
856
+ shifted,
857
+ chunkCount,
858
+ });
859
+ } catch {
860
+ /* non-fatal: stripe refresh never breaks the agent loop */
861
+ }
862
+ }
780
863
  });
781
864
  }