pi-mega-compact 0.11.7 → 0.11.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/dist/extensions/dashboard-server/gather-debug-keys.js +18 -0
  2. package/dist/extensions/dashboard-server/gather-debug.js +129 -0
  3. package/dist/extensions/dashboard-server/routes-maintenance.js +6 -0
  4. package/dist/extensions/dashboard-server/routes-memory-map.js +8 -15
  5. package/dist/extensions/mega-commands.js +65 -5
  6. package/dist/extensions/mega-config.js +7 -0
  7. package/dist/extensions/mega-events/agent-handlers.js +74 -5
  8. package/dist/extensions/mega-events/context-handler.js +132 -3
  9. package/dist/extensions/mega-runtime/snapshot.js +1 -1
  10. package/dist/extensions/mega-trim.js +10 -0
  11. package/dist/src/config/turns.js +2 -1
  12. package/dist/src/memoryGraph/embedding.js +50 -0
  13. package/dist/src/memoryGraph/gates.js +379 -0
  14. package/dist/src/memoryGraph/sources.js +390 -0
  15. package/dist/src/memoryGraph.js +133 -259
  16. package/dist/src/store/sqlite/embedding-cache.js +71 -0
  17. package/dist/src/store/sqlite.js +1 -0
  18. package/dist/src/topics/cluster.js +10 -2
  19. package/extensions/dashboard-client/dist/assets/{AchievementTiles-DR9cLGDL.js → AchievementTiles-DFitrb4Y.js} +2 -2
  20. package/extensions/dashboard-client/dist/assets/{AchievementTiles-DR9cLGDL.js.map → AchievementTiles-DFitrb4Y.js.map} +1 -1
  21. package/extensions/dashboard-client/dist/assets/{AchievementsTab-svNUgJ1e.js → AchievementsTab-PJrmq8pa.js} +2 -2
  22. package/extensions/dashboard-client/dist/assets/{AchievementsTab-svNUgJ1e.js.map → AchievementsTab-PJrmq8pa.js.map} +1 -1
  23. package/extensions/dashboard-client/dist/assets/CacheTab-C1WE7dEx.js +2 -0
  24. package/extensions/dashboard-client/dist/assets/CacheTab-C1WE7dEx.js.map +1 -0
  25. package/extensions/dashboard-client/dist/assets/{ConfigTab-CmmWheuW.js → ConfigTab-Dx-RpO8C.js} +2 -2
  26. package/extensions/dashboard-client/dist/assets/{ConfigTab-CmmWheuW.js.map → ConfigTab-Dx-RpO8C.js.map} +1 -1
  27. package/extensions/dashboard-client/dist/assets/{EventsTab-CFE7jLsa.js → EventsTab-C63BEWEs.js} +2 -2
  28. package/extensions/dashboard-client/dist/assets/{EventsTab-CFE7jLsa.js.map → EventsTab-C63BEWEs.js.map} +1 -1
  29. package/extensions/dashboard-client/dist/assets/{GameTab-CJAX_th_.js → GameTab-DHpJE6km.js} +2 -2
  30. package/extensions/dashboard-client/dist/assets/{GameTab-CJAX_th_.js.map → GameTab-DHpJE6km.js.map} +1 -1
  31. package/extensions/dashboard-client/dist/assets/MaintenanceTab-BSv0GfBb.js +2 -0
  32. package/extensions/dashboard-client/dist/assets/MaintenanceTab-BSv0GfBb.js.map +1 -0
  33. package/extensions/dashboard-client/dist/assets/MemoryMapTab-Cg1fgEx_.js +2 -0
  34. package/extensions/dashboard-client/dist/assets/MemoryMapTab-Cg1fgEx_.js.map +1 -0
  35. package/extensions/dashboard-client/dist/assets/{MetricsTab-CHQscixy.js → MetricsTab-DCahxNOQ.js} +2 -2
  36. package/extensions/dashboard-client/dist/assets/{MetricsTab-CHQscixy.js.map → MetricsTab-DCahxNOQ.js.map} +1 -1
  37. package/extensions/dashboard-client/dist/assets/{OverviewTab-DBs4DJdd.js → OverviewTab-RYvO1SZL.js} +2 -2
  38. package/extensions/dashboard-client/dist/assets/{OverviewTab-DBs4DJdd.js.map → OverviewTab-RYvO1SZL.js.map} +1 -1
  39. package/extensions/dashboard-client/dist/assets/{ReposTab-DeFz76L5.js → ReposTab-BDGr_UXb.js} +2 -2
  40. package/extensions/dashboard-client/dist/assets/{ReposTab-DeFz76L5.js.map → ReposTab-BDGr_UXb.js.map} +1 -1
  41. package/extensions/dashboard-client/dist/assets/{SessionsTab-BWNsj1OB.js → SessionsTab-ByrTnHaB.js} +2 -2
  42. package/extensions/dashboard-client/dist/assets/{SessionsTab-BWNsj1OB.js.map → SessionsTab-ByrTnHaB.js.map} +1 -1
  43. package/extensions/dashboard-client/dist/assets/{SetupTab-uS6A0xmw.js → SetupTab-CELafKis.js} +2 -2
  44. package/extensions/dashboard-client/dist/assets/{SetupTab-uS6A0xmw.js.map → SetupTab-CELafKis.js.map} +1 -1
  45. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CiqCOQuL.js → TimeSavedCard-BLWtmfr_.js} +2 -2
  46. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CiqCOQuL.js.map → TimeSavedCard-BLWtmfr_.js.map} +1 -1
  47. package/extensions/dashboard-client/dist/assets/TopicsTab-8Fsh_Igx.js +2 -0
  48. package/extensions/dashboard-client/dist/assets/TopicsTab-8Fsh_Igx.js.map +1 -0
  49. package/extensions/dashboard-client/dist/assets/{TurnsTab-CVz4f4-o.js → TurnsTab-D7mgXDe2.js} +2 -2
  50. package/extensions/dashboard-client/dist/assets/{TurnsTab-CVz4f4-o.js.map → TurnsTab-D7mgXDe2.js.map} +1 -1
  51. package/extensions/dashboard-client/dist/assets/{index-DrktNOoT.js → index-h3jgEs6G.js} +10 -10
  52. package/extensions/dashboard-client/dist/assets/{index-DrktNOoT.js.map → index-h3jgEs6G.js.map} +1 -1
  53. package/extensions/dashboard-client/dist/assets/{useSSE-Bf5LOLSi.js → useSSE-C5lmipBF.js} +2 -2
  54. package/extensions/dashboard-client/dist/assets/{useSSE-Bf5LOLSi.js.map → useSSE-C5lmipBF.js.map} +1 -1
  55. package/extensions/dashboard-client/dist/index.html +1 -1
  56. package/extensions/dashboard-client/src/api/client.ts +5 -0
  57. package/extensions/dashboard-client/src/memory-map-layout.ts +161 -0
  58. package/extensions/dashboard-client/src/memory-map-shapes.tsx +126 -0
  59. package/extensions/dashboard-client/src/tabs/CacheTab.tsx +29 -2
  60. package/extensions/dashboard-client/src/tabs/MaintenanceTab.tsx +101 -0
  61. package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +201 -165
  62. package/extensions/dashboard-client/src/tabs/ReposTab.tsx +21 -0
  63. package/extensions/dashboard-client/src/tabs/TopicsTab.tsx +1 -1
  64. package/extensions/dashboard-server/api-contracts/index.ts +3 -1
  65. package/extensions/dashboard-server/api-contracts/maintenance.ts +13 -0
  66. package/extensions/dashboard-server/api-contracts/memory-map.ts +14 -6
  67. package/extensions/dashboard-server/gather-debug-keys.ts +18 -0
  68. package/extensions/dashboard-server/gather-debug.ts +136 -0
  69. package/extensions/dashboard-server/routes-maintenance.ts +7 -0
  70. package/extensions/dashboard-server/routes-memory-map.ts +21 -29
  71. package/extensions/mega-commands.ts +63 -6
  72. package/extensions/mega-config.ts +22 -0
  73. package/extensions/mega-events/agent-handlers.ts +88 -5
  74. package/extensions/mega-events/context-handler.ts +152 -3
  75. package/extensions/mega-runtime/snapshot.ts +1 -1
  76. package/extensions/mega-trim.ts +15 -0
  77. package/package.json +1 -1
  78. package/src/config/turns.ts +5 -2
  79. package/src/memoryGraph/embedding.ts +63 -0
  80. package/src/memoryGraph/gates.ts +451 -0
  81. package/src/memoryGraph/sources.ts +484 -0
  82. package/src/memoryGraph.ts +188 -350
  83. package/src/store/sqlite/embedding-cache.ts +81 -0
  84. package/src/store/sqlite.ts +1 -0
  85. package/src/topics/cluster.ts +11 -1
  86. package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js +0 -2
  87. package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js.map +0 -1
  88. package/extensions/dashboard-client/dist/assets/MaintenanceTab-P4SUsvMf.js +0 -2
  89. package/extensions/dashboard-client/dist/assets/MaintenanceTab-P4SUsvMf.js.map +0 -1
  90. package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js +0 -2
  91. package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js.map +0 -1
  92. package/extensions/dashboard-client/dist/assets/TopicsTab-BWWQxltv.js +0 -2
  93. package/extensions/dashboard-client/dist/assets/TopicsTab-BWWQxltv.js.map +0 -1
@@ -0,0 +1,18 @@
1
+ export const MEGACOMPACT_ENV_KEYS = [
2
+ "MEGACOMPACT_AUTO",
3
+ "MEGACOMPACT_THRESHOLD_TOKENS",
4
+ "MEGACOMPACT_TIER",
5
+ "MEGACOMPACT_DBMIRROR",
6
+ "MEGACOMPACT_MESSAGE_SEPARATION",
7
+ "MEGACOMPACT_CACHE_STRIPING",
8
+ "MEGACOMPACT_LEGACY_DURABLE_TRIM",
9
+ "MEGACOMPACT_RACE_GUARD_STRICT",
10
+ "MEGACOMPACT_RECALL_TAIL_INJECT",
11
+ "MEGACOMPACT_PGLITE_DISABLED",
12
+ "MEGACOMPACT_WIKI_REBUILD_EVERY",
13
+ "MEGACOMPACT_WIKI_SEED_FROM_TURNS",
14
+ "MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS",
15
+ "MEGACOMPACT_MEMORY_GRAPH_SEED_TIER",
16
+ "MEGACOMPACT_MEMORY_GRAPH_SEED_BATCH",
17
+ "MEGACOMPACT_MEMORY_GRAPH_SEED_MAX",
18
+ ];
@@ -0,0 +1,129 @@
1
+ /**
2
+ * gather-debug.ts — assemble a diagnostic bundle for bug reports.
3
+ *
4
+ * POST /api/maintenance/gather-debug → DebugBundleResponse. Reads local
5
+ * SQLite + FS only (PREVENT-PI-004: zero network). Each section is non-fatal:
6
+ * a read failure records `{ error }` for that section and continues, so the
7
+ * bundle always returns 200 with whatever could be gathered.
8
+ */
9
+ import { readFileSync, existsSync } from "node:fs";
10
+ import { createRequire } from "node:module";
11
+ import { defaultMetricsPath, defaultEventsPath } from "../../src/monitoring.js";
12
+ import { ownVersion } from "../mega-runtime/helpers.js";
13
+ import { MEGACOMPACT_ENV_KEYS } from "./gather-debug-keys.js";
14
+ /** Event-name patterns that indicate a critical/compaction class of error. */
15
+ const CRITICAL_PATTERN = /critical|overflow|already[.\s-]?compacted|compaction[.\s-]?failed|stream[.\s-]?death|poisoned|prefix_stability/i;
16
+ const require = createRequire(import.meta.url);
17
+ /** Lazy store opener (mirrors routes-maintenance.ts's lazyStoreUtils). */
18
+ function openStoreDb(ctx) {
19
+ const mod = require("../../src/store/sqlite/utils.js");
20
+ return mod.openStore(ctx.stateDir);
21
+ }
22
+ /** Lazy maintenance loaders (mirrors routes-maintenance.ts's lazyMaintenance). */
23
+ function loadMaintenance() {
24
+ return require("../../src/store/sqlite/maintenance.js");
25
+ }
26
+ /** Send the debug bundle as JSON. Always 200 (partial on read failure). */
27
+ export function sendGatherDebug(res, ctx) {
28
+ const result = {
29
+ builtAt: Date.now(),
30
+ version: ownVersion(),
31
+ stateDir: ctx.stateDir,
32
+ config: {},
33
+ schemaHealth: { error: "not fetched" },
34
+ storeStats: { error: "not fetched" },
35
+ dashboardSnapshot: { error: "not fetched" },
36
+ recentEvents: [],
37
+ criticalEvents: [],
38
+ };
39
+ // 1. Config flags (no secrets — MEGACOMPACT_* only)
40
+ for (const key of MEGACOMPACT_ENV_KEYS) {
41
+ result.config[key] = process.env[key] ?? "<default>";
42
+ }
43
+ // 2. Schema health
44
+ try {
45
+ const maint = loadMaintenance();
46
+ const integrity = maint.integrityCheck(ctx.stateDir);
47
+ const db = openStoreDb(ctx);
48
+ const metaRow = db
49
+ .prepare("SELECT value FROM meta WHERE key = ?")
50
+ .get("schema_version");
51
+ const schemaVersion = metaRow ? Number(metaRow.value) : 0;
52
+ const fkRows = db.prepare("PRAGMA foreign_key_check").all();
53
+ result.schemaHealth = {
54
+ type: "schema-health",
55
+ schemaVersion,
56
+ integrity,
57
+ fkCheck: fkRows.map((r) => JSON.stringify(r)),
58
+ columns: [],
59
+ healthy: true,
60
+ };
61
+ }
62
+ catch (e) {
63
+ result.schemaHealth = { error: e instanceof Error ? e.message : String(e) };
64
+ }
65
+ // 3. Store stats
66
+ try {
67
+ const maint = loadMaintenance();
68
+ const raw = maint.getDbStats(ctx.stateDir);
69
+ result.storeStats = {
70
+ type: "db-stats",
71
+ tables: Object.entries(raw.tableCounts).map(([table, rowCount]) => ({ table, rowCount })),
72
+ storage: {
73
+ files: { dbBytes: raw.dbBytes, walBytes: raw.walBytes, shmBytes: raw.shmBytes },
74
+ pageSize: raw.pageSize,
75
+ pageCount: raw.pageCount,
76
+ freelistPages: raw.freelistPages,
77
+ },
78
+ };
79
+ }
80
+ catch (e) {
81
+ result.storeStats = { error: e instanceof Error ? e.message : String(e) };
82
+ }
83
+ // 4. Dashboard snapshot
84
+ try {
85
+ const path = defaultMetricsPath(ctx.stateDir);
86
+ if (existsSync(path)) {
87
+ const raw = readFileSync(path, "utf-8");
88
+ result.dashboardSnapshot = JSON.parse(raw);
89
+ }
90
+ else {
91
+ result.dashboardSnapshot = { error: "no snapshot" };
92
+ }
93
+ }
94
+ catch (e) {
95
+ result.dashboardSnapshot = { error: e instanceof Error ? e.message : String(e) };
96
+ }
97
+ // 5. Recent + critical events (tail ~200 lines of events.log)
98
+ try {
99
+ const path = defaultEventsPath(ctx.stateDir);
100
+ if (existsSync(path)) {
101
+ const lines = readFileSync(path, "utf-8").split("\n").filter(Boolean);
102
+ for (const line of lines.slice(-200)) {
103
+ try {
104
+ const ev = JSON.parse(line);
105
+ result.recentEvents.push(ev);
106
+ if (CRITICAL_PATTERN.test(line))
107
+ result.criticalEvents.push(ev);
108
+ }
109
+ catch {
110
+ continue; // skip unparseable line (PREVENT-001)
111
+ }
112
+ }
113
+ }
114
+ }
115
+ catch (e) {
116
+ result.recentEvents = [{ error: e instanceof Error ? e.message : String(e) }];
117
+ }
118
+ // Send (local helper to avoid importing sendJson from a non-exporting module)
119
+ try {
120
+ const body = JSON.stringify(result);
121
+ res.writeHead(200, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
122
+ res.end(body);
123
+ }
124
+ catch {
125
+ // last resort — never leave the response hanging
126
+ res.writeHead(500, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
127
+ res.end('{"error":"failed to serialize debug bundle"}');
128
+ }
129
+ }
@@ -11,6 +11,7 @@
11
11
  * SQLite only; zero network calls.
12
12
  */
13
13
  import { createRequire } from "node:module";
14
+ import { sendGatherDebug } from "./gather-debug.js";
14
15
  // ---------------------------------------------------------------------------
15
16
  // Lazy imports (via createRequire)
16
17
  // ---------------------------------------------------------------------------
@@ -48,6 +49,11 @@ export function handleMaintenance(req, res, ctx) {
48
49
  handleMaintenanceAction(req, res, ctx);
49
50
  return true;
50
51
  }
52
+ // ── POST /api/maintenance/gather-debug ──────────────────────────
53
+ if (method === "POST" && url === "/api/maintenance/gather-debug") {
54
+ sendGatherDebug(res, ctx);
55
+ return true;
56
+ }
51
57
  // ── GET /api/maintenance ──────────────────────────────────────────
52
58
  if (method === "GET" && url === "/api/maintenance") {
53
59
  sendDbStats(res, ctx);
@@ -1,7 +1,8 @@
1
1
  /**
2
- * routes-memory-map.ts — GET /api/memory-map route handler (S46).
2
+ * routes-memory-map.ts — GET /api/memory-map route handler (S46/D3).
3
3
  *
4
4
  * Builds a memory graph from the local SQLite store and returns it as JSON.
5
+ * D3: surfaces nodeType on each node and the GraphValidationReport.
5
6
  * Uses an inline require to the pi-agnostic src/memoryGraph.ts builder.
6
7
  */
7
8
  import { createRequire } from "node:module";
@@ -16,19 +17,8 @@ export function handleMemoryMap(req, res, ctx) {
16
17
  return false;
17
18
  // guardrails-allow PREVENT-PI-004: local loopback-only dashboard server — reads the local SQLite store for graph data.
18
19
  const { buildMemoryGraph } = _req("../../src/memoryGraph.js");
19
- const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : undefined;
20
- const threshold = typeof parsed.query.threshold === "string" ? Number.parseFloat(parsed.query.threshold) : undefined;
21
- const maxEdges = typeof parsed.query.maxEdgesPerNode === "string" ? Number.parseInt(parsed.query.maxEdgesPerNode, 10) : undefined;
22
- const raptorEdges = parsed.query.includeRaptorEdges === "false" ? false : true;
23
- const temporalEdges = parsed.query.includeTemporalEdges === "false" ? false : true;
24
- const graph = buildMemoryGraph({
25
- sessionId,
26
- similarityThreshold: threshold,
27
- maxEdgesPerNode: maxEdges,
28
- stateDir: ctx.stateDir,
29
- includeRaptorEdges: raptorEdges,
30
- includeTemporalEdges: temporalEdges,
31
- });
20
+ const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : "";
21
+ const graph = buildMemoryGraph(sessionId, ctx.stateDir);
32
22
  const nodes = graph.nodes.map((n) => ({
33
23
  id: n.id,
34
24
  sessionId: n.sessionId,
@@ -41,9 +31,12 @@ export function handleMemoryMap(req, res, ctx) {
41
31
  topicSummary: n.topicSummary,
42
32
  decisionCount: n.decisionCount,
43
33
  textSnippet: n.textSnippet,
34
+ // D3: pass through nodeType for UI node-shape encoding
35
+ nodeType: n.nodeType,
44
36
  }));
45
37
  const edges = graph.edges;
46
- const response = { nodes, edges, metadata: graph.metadata };
38
+ // D3: surface the validation report (optional in the response type for backward compat)
39
+ const response = { nodes, edges, metadata: graph.metadata, validation: graph.validation };
47
40
  res.writeHead(200, { "Content-Type": "application/json" });
48
41
  res.end(JSON.stringify(response));
49
42
  return true;
@@ -9,12 +9,12 @@ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
9
9
  import { normalizeSessionId } from "../src/store.js";
10
10
  import { listCheckpoints, latestModelSnapshot, countInjectedGlobal, listRepoRegistry } from "../src/store/sqlite.js";
11
11
  import { decompressSmart } from "../src/store/compression.js";
12
- import { loadMetrics, fpRate, p95, defaultMetricsPath } from "../src/monitoring.js";
12
+ import { loadMetrics, fpRate, p95, defaultMetricsPath, defaultEventsPath } from "../src/monitoring.js";
13
13
  import { C, recentUserQuery } from "./mega-runtime.js";
14
14
  import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
15
15
  import { vectorStats, vectorRepoStats, vectorDataInvariant } from "../src/vectorStore.js";
16
16
  import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: local subprocess detection only (no network)
17
- import { mkdirSync, writeFileSync } from "node:fs";
17
+ import { mkdirSync, writeFileSync, existsSync, readFileSync } from "node:fs";
18
18
  import path from "node:path";
19
19
  import os from "node:os";
20
20
  import { createRequire } from "node:module";
@@ -260,6 +260,58 @@ export function registerCommands(pi, runtime, config) {
260
260
  `• data safety — every compressed region is kept verbatim; nothing is permanently deleted. /mega-restore brings any of it back.`);
261
261
  },
262
262
  });
263
+ /**
264
+ * Try to peek at recent events.log and mega-compact.log for low recall quality signals.
265
+ * Best-effort, non-fatal — returns true if a low-quality signal was found.
266
+ */
267
+ /**
268
+ * Try to peek at recent events.log and mega-compact.log for low recall quality signals.
269
+ * Best-effort, non-fatal — returns true if a low-quality signal was found.
270
+ */
271
+ function checkRecallQuality(stateDir) {
272
+ const eventPaths = [
273
+ defaultEventsPath(stateDir),
274
+ path.join(stateDir, "mega-compact.log"),
275
+ ];
276
+ for (const fp of eventPaths) {
277
+ try {
278
+ if (!existsSync(fp))
279
+ continue;
280
+ const buf = readFileSync(fp, { encoding: "utf-8" });
281
+ // Only scan last 64 KiB of the file
282
+ const tail = buf.length > 65536 ? buf.slice(buf.length - 65536) : buf;
283
+ const lines = tail.split("\n").filter(Boolean);
284
+ for (const line of lines) {
285
+ try {
286
+ // guardrails-allow PREVENT-001: JSON.parse with null check
287
+ const parsed = JSON.parse(line);
288
+ if (typeof parsed !== "object" || parsed === null)
289
+ continue;
290
+ // Check for recall_metrics_low_quality event (latest format)
291
+ if (parsed.event === "recall_metrics_low_quality" && parsed.score !== undefined) {
292
+ if (typeof parsed.score === "number" && parsed.score < 0.4)
293
+ return true;
294
+ }
295
+ // Check for recall_metrics with low relevanceScore
296
+ if (parsed.event === "recall_metrics" && parsed.relevanceScore !== undefined) {
297
+ if (typeof parsed.relevanceScore === "number" && parsed.relevanceScore < 0.4)
298
+ return true;
299
+ }
300
+ // Check for RecallQualityEvent format from monitoring.ts
301
+ if (parsed.score !== undefined && typeof parsed.score === "number" && parsed.score < 0.4)
302
+ return true;
303
+ }
304
+ catch {
305
+ // skip unparseable lines
306
+ }
307
+ }
308
+ }
309
+ catch {
310
+ // non-fatal: skip if can't read
311
+ }
312
+ }
313
+ return false;
314
+ }
263
315
  pi.registerCommand("mega-setup", {
264
316
  description: "Local embedding setup wizard — detect available embedders and configure mega-compact.",
265
317
  handler: async (_args, ctx) => {
@@ -335,6 +387,10 @@ export function registerCommands(pi, runtime, config) {
335
387
  catch {
336
388
  // require error
337
389
  }
390
+ // --- 2.5 Quality check ---
391
+ // Scan recent events for low recall quality (best-effort, non-fatal)
392
+ const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
393
+ const lowRecallQuality = checkRecallQuality(stateDir);
338
394
  const choices = [];
339
395
  if (ollamaDetected && ollamaRunning) {
340
396
  const model = ollamaModels.find((m) => m.includes("nomic-embed") ||
@@ -370,6 +426,9 @@ export function registerCommands(pi, runtime, config) {
370
426
  });
371
427
  // --- 4. Prompt user ---
372
428
  log(`Current embedder: ${currentEmbedder}`);
429
+ if (lowRecallQuality) {
430
+ log("Tap: recent recall quality is low. Consider switching to Ollama or llama.cpp for better results.");
431
+ }
373
432
  let selected = "trigram";
374
433
  if (choices.length > 1) {
375
434
  try {
@@ -403,7 +462,6 @@ export function registerCommands(pi, runtime, config) {
403
462
  ];
404
463
  const envContent = lines.join("\n") + "\n";
405
464
  // Write .mega-compact.env to state dir
406
- const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
407
465
  try {
408
466
  mkdirSync(stateDir, { recursive: true });
409
467
  writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
@@ -421,7 +479,6 @@ export function registerCommands(pi, runtime, config) {
421
479
  `export MEGACOMPACT_EMBEDDING_URL="${url}"`,
422
480
  ];
423
481
  const envContent = lines.join("\n") + "\n";
424
- const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
425
482
  try {
426
483
  mkdirSync(stateDir, { recursive: true });
427
484
  writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
@@ -435,7 +492,10 @@ export function registerCommands(pi, runtime, config) {
435
492
  log("ONNX Runtime configuration is experimental.\n\nSet MEGACOMPACT_EMBEDDING_URL to your local ONNX server URL, or see docs for manual setup.\nKeeping TrigramEmbedder for now.");
436
493
  }
437
494
  else {
438
- log("Keeping TrigramEmbedder (default). No configuration needed.");
495
+ const qualityTip = lowRecallQuality
496
+ ? " Recent recall quality is low — consider running /mega-setup again to switch to Ollama or llama.cpp for better results."
497
+ : "";
498
+ log(`Keeping TrigramEmbedder (default). No configuration needed.${qualityTip}`);
439
499
  }
440
500
  }
441
501
  catch (e) {
@@ -168,11 +168,18 @@ export function loadConfig() {
168
168
  windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
169
169
  recallTailInject: envBool("MEGACOMPACT_RECALL_TAIL_INJECT", true),
170
170
  messageSeparation: envBool("MEGACOMPACT_MESSAGE_SEPARATION", false),
171
+ cacheStriping: envBool("MEGACOMPACT_CACHE_STRIPING", false),
171
172
  tuiWidget: envBool("MEGACOMPACT_TUI_WIDGET", true),
172
173
  ragQueryReformulation: envBool("MEGACOMPACT_QUERY_REFORMULATION", false),
173
174
  ragTieredRouter: envBool("MEGACOMPACT_TIERED_ROUTER", false),
174
175
  ragRecallMetrics: envBool("MEGACOMPACT_RECALL_METRICS", false),
175
176
  ragMemoryGraph: envBool("MEGACOMPACT_MEMORY_GRAPH", false),
177
+ wikiSeedFromTurns: envBool("MEGACOMPACT_WIKI_SEED_FROM_TURNS", true),
178
+ memoryGraphSeedTurns: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS", true),
179
+ memoryGraphSeedTurnContent: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true),
180
+ memoryGraphSeedMemories: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_MEMORIES", true),
181
+ memoryGraphCrossTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_CROSS_TYPE_THRESHOLD", 0.85),
182
+ memoryGraphWithinTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_WITHIN_TYPE_THRESHOLD", 0.7),
176
183
  // D.1: env-overridable recompact delta (minimum context growth % before
177
184
  // re-compacting instead of replaying the cached live trim). Default 50.
178
185
  recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),
@@ -9,6 +9,9 @@ import { classifyError, classifyErrorDetailed, errorRetryBackoffMs, extractError
9
9
  import { safeSendUserMessage } from "./send-safe.js";
10
10
  import { maybeSendProviderOutageAdvisory } from "./outage-advisor.js";
11
11
  import { vectorStats } from "../../src/vectorStore.js";
12
+ import { defaultEmbedder } from "../../src/embedder.js";
13
+ import { storeTopicEmbedding, loadTopicEmbedding, detectTopicShift, } from "./separated-prompt.js";
14
+ import { refreshStripeAssignments as writeStripeAssignments, } from "../../src/cache-stripe-impl.js";
12
15
  /** Register agent/turn tracking event handlers. */
13
16
  export function registerAgentHandlers(pi, runtime, config) {
14
17
  // ---- Agent tracking for real-time widget + status-line updates ---------
@@ -130,10 +133,21 @@ export function registerAgentHandlers(pi, runtime, config) {
130
133
  // context-handler.ts:258-287 so both call sites stay in sync.
131
134
  const cooldownMs = config.raceGuardStrict ? 30_000 : 10_000;
132
135
  const sinceCompact = now - (runtime.rt.lastNativeCompactAt ?? 0);
136
+ // CRITICAL-OVER ESCAPE HATCH: when context is critically over the
137
+ // window (>= 90%), force the durable ctx.compact() even if
138
+ // piCompactWouldNoop says it would no-op ("Already compacted").
139
+ // Without this, the on-disk transcript never truncates and the
140
+ // session stays overflowed on every resume — the "Already
141
+ // compacted" + overflow death-spiral (2026-08-01 incident). The
142
+ // cooldown still applies (we don't spam compact), but the no-op
143
+ // gate is bypassed when critical. pi's ctx.compact() may throw
144
+ // "Already compacted" in the truly-no-op case — wrapped in
145
+ // try/catch so the throw is non-fatal.
146
+ const criticalOver = (runtime.lastCtxPercent ?? 0) >= 90;
133
147
  if (sinceCompact < cooldownMs) {
134
148
  runtime.diagAgentEndDurableSkipRecent++;
135
149
  }
136
- else if (!piCompactWouldNoop(ctx)) {
150
+ else if (criticalOver || !piCompactWouldNoop(ctx)) {
137
151
  runtime.debounceUntil = now + 2000;
138
152
  runtime.diagAgentEndDurable++;
139
153
  runtime.logger.info("agent-end-durable-trigger", {
@@ -150,6 +164,7 @@ export function registerAgentHandlers(pi, runtime, config) {
150
164
  // would re-check before it lands.
151
165
  const stamp = runtime.rt.lastNativeCompactAt;
152
166
  const liveSid = runtime.rt.sessionId;
167
+ const liveCritical = criticalOver;
153
168
  setTimeout(() => {
154
169
  try {
155
170
  if (runtime.rt.sessionId !== liveSid)
@@ -161,19 +176,26 @@ export function registerAgentHandlers(pi, runtime, config) {
161
176
  if (runtime.rt.lastNativeCompactAt !== stamp &&
162
177
  since2 < cooldownMs)
163
178
  return;
164
- if (piCompactWouldNoop(ctx))
179
+ // CRITICAL-OVER ESCAPE HATCH: bypass the no-op gate when
180
+ // critical (captured at handler time) so the durable trim fires.
181
+ if (!liveCritical && piCompactWouldNoop(ctx))
165
182
  return;
166
183
  ctx.compact({
167
184
  customInstructions: undefined,
168
- }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated.
185
+ }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated. May throw "Already compacted" when the escape hatch forces it — caught below.
169
186
  }
170
187
  catch {
171
- /* non-fatal */
188
+ /* non-fatal: "Already compacted" throws are expected when the critical-over hatch forces a no-op compact */
172
189
  }
173
190
  }, 500);
174
191
  }
175
192
  else {
176
- ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. Race-guarded by lastNativeCompactAt cooldown above (ctx.compact returns void → throw is surfaced by pi as compaction_end; the cooldown prevents the call entirely).
193
+ try {
194
+ ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. May throw "Already compacted" when the critical-over escape hatch forces it — caught.
195
+ }
196
+ catch {
197
+ /* non-fatal */
198
+ }
177
199
  }
178
200
  didDurableTrim = true;
179
201
  }
@@ -712,5 +734,52 @@ export function registerAgentHandlers(pi, runtime, config) {
712
734
  catch {
713
735
  /* non-fatal: a classifier/retry failure never breaks the agent loop */
714
736
  }
737
+ // P3.5: topic-shift stripe refresh. When MEGACOMPACT_CACHE_STRIPING is ON,
738
+ // embed the current turn, detect topic shift vs. the previous turn, and
739
+ // refresh cache stripe assignments when shifted (or first turn). Best-effort:
740
+ // failures log + continue, never break the agent loop.
741
+ if (config.cacheStriping) {
742
+ try {
743
+ const stateDir = runtime.currentStateDir;
744
+ const sessionId = runtime.rt.sessionId;
745
+ const currentTurn = event.turnIndex;
746
+ const embedder = defaultEmbedder();
747
+ // (a) Extract text from the assistant's response for topic embedding.
748
+ let textToEmbed = "";
749
+ const msg = event.message;
750
+ if (msg.role === "assistant" && Array.isArray(msg.content)) {
751
+ for (const part of msg.content) {
752
+ if ("text" in part && typeof part.text === "string") {
753
+ textToEmbed += part.text + " ";
754
+ }
755
+ }
756
+ }
757
+ textToEmbed = textToEmbed.trim() || `turn-${currentTurn}`;
758
+ const embedding = new Float32Array(embedder.embed(textToEmbed));
759
+ storeTopicEmbedding(stateDir, sessionId, currentTurn, embedding);
760
+ // (b) Load previous turn's embedding and detect topic shift.
761
+ const prevEmb = currentTurn > 0
762
+ ? loadTopicEmbedding(stateDir, sessionId, currentTurn - 1)
763
+ : null;
764
+ const shifted = currentTurn === 0 || detectTopicShift(embedding, prevEmb);
765
+ // (c) If shifted (or first turn), refresh stripe assignments (WRITE path).
766
+ let chunkCount = 0;
767
+ if (shifted) {
768
+ chunkCount = writeStripeAssignments(stateDir, undefined, embedder, (detail) => runtime.logger.info("stripe_refresh_detail", {
769
+ detail,
770
+ turnIndex: currentTurn,
771
+ }));
772
+ }
773
+ // (d) Log the refresh outcome.
774
+ runtime.logger.info("stripe_refresh", {
775
+ turnIndex: currentTurn,
776
+ shifted,
777
+ chunkCount,
778
+ });
779
+ }
780
+ catch {
781
+ /* non-fatal: stripe refresh never breaks the agent loop */
782
+ }
783
+ }
715
784
  });
716
785
  }