pi-mega-compact 0.11.7 → 0.11.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server/gather-debug-keys.js +18 -0
- package/dist/extensions/dashboard-server/gather-debug.js +129 -0
- package/dist/extensions/dashboard-server/routes-maintenance.js +6 -0
- package/dist/extensions/dashboard-server/routes-memory-map.js +8 -15
- package/dist/extensions/mega-commands.js +65 -5
- package/dist/extensions/mega-config.js +7 -0
- package/dist/extensions/mega-events/agent-handlers.js +74 -5
- package/dist/extensions/mega-events/context-handler.js +132 -3
- package/dist/extensions/mega-runtime/snapshot.js +1 -1
- package/dist/extensions/mega-trim.js +10 -0
- package/dist/src/config/turns.js +2 -1
- package/dist/src/memoryGraph/embedding.js +50 -0
- package/dist/src/memoryGraph/gates.js +379 -0
- package/dist/src/memoryGraph/sources.js +390 -0
- package/dist/src/memoryGraph.js +133 -259
- package/dist/src/store/sqlite/embedding-cache.js +71 -0
- package/dist/src/store/sqlite.js +1 -0
- package/dist/src/topics/cluster.js +10 -2
- package/extensions/dashboard-client/dist/assets/{AchievementTiles-DR9cLGDL.js → AchievementTiles-DFitrb4Y.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AchievementTiles-DR9cLGDL.js.map → AchievementTiles-DFitrb4Y.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{AchievementsTab-svNUgJ1e.js → AchievementsTab-PJrmq8pa.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AchievementsTab-svNUgJ1e.js.map → AchievementsTab-PJrmq8pa.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/CacheTab-C1WE7dEx.js +2 -0
- package/extensions/dashboard-client/dist/assets/CacheTab-C1WE7dEx.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{ConfigTab-CmmWheuW.js → ConfigTab-Dx-RpO8C.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ConfigTab-CmmWheuW.js.map → ConfigTab-Dx-RpO8C.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{EventsTab-CFE7jLsa.js → EventsTab-C63BEWEs.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{EventsTab-CFE7jLsa.js.map → EventsTab-C63BEWEs.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{GameTab-CJAX_th_.js → GameTab-DHpJE6km.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{GameTab-CJAX_th_.js.map → GameTab-DHpJE6km.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-BSv0GfBb.js +2 -0
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-BSv0GfBb.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-Cg1fgEx_.js +2 -0
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-Cg1fgEx_.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{MetricsTab-CHQscixy.js → MetricsTab-DCahxNOQ.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MetricsTab-CHQscixy.js.map → MetricsTab-DCahxNOQ.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{OverviewTab-DBs4DJdd.js → OverviewTab-RYvO1SZL.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{OverviewTab-DBs4DJdd.js.map → OverviewTab-RYvO1SZL.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{ReposTab-DeFz76L5.js → ReposTab-BDGr_UXb.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ReposTab-DeFz76L5.js.map → ReposTab-BDGr_UXb.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BWNsj1OB.js → SessionsTab-ByrTnHaB.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BWNsj1OB.js.map → SessionsTab-ByrTnHaB.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SetupTab-uS6A0xmw.js → SetupTab-CELafKis.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SetupTab-uS6A0xmw.js.map → SetupTab-CELafKis.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CiqCOQuL.js → TimeSavedCard-BLWtmfr_.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CiqCOQuL.js.map → TimeSavedCard-BLWtmfr_.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/TopicsTab-8Fsh_Igx.js +2 -0
- package/extensions/dashboard-client/dist/assets/TopicsTab-8Fsh_Igx.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{TurnsTab-CVz4f4-o.js → TurnsTab-D7mgXDe2.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TurnsTab-CVz4f4-o.js.map → TurnsTab-D7mgXDe2.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{index-DrktNOoT.js → index-h3jgEs6G.js} +10 -10
- package/extensions/dashboard-client/dist/assets/{index-DrktNOoT.js.map → index-h3jgEs6G.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{useSSE-Bf5LOLSi.js → useSSE-C5lmipBF.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{useSSE-Bf5LOLSi.js.map → useSSE-C5lmipBF.js.map} +1 -1
- package/extensions/dashboard-client/dist/index.html +1 -1
- package/extensions/dashboard-client/src/api/client.ts +5 -0
- package/extensions/dashboard-client/src/memory-map-layout.ts +161 -0
- package/extensions/dashboard-client/src/memory-map-shapes.tsx +126 -0
- package/extensions/dashboard-client/src/tabs/CacheTab.tsx +29 -2
- package/extensions/dashboard-client/src/tabs/MaintenanceTab.tsx +101 -0
- package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +201 -165
- package/extensions/dashboard-client/src/tabs/ReposTab.tsx +21 -0
- package/extensions/dashboard-client/src/tabs/TopicsTab.tsx +1 -1
- package/extensions/dashboard-server/api-contracts/index.ts +3 -1
- package/extensions/dashboard-server/api-contracts/maintenance.ts +13 -0
- package/extensions/dashboard-server/api-contracts/memory-map.ts +14 -6
- package/extensions/dashboard-server/gather-debug-keys.ts +18 -0
- package/extensions/dashboard-server/gather-debug.ts +136 -0
- package/extensions/dashboard-server/routes-maintenance.ts +7 -0
- package/extensions/dashboard-server/routes-memory-map.ts +21 -29
- package/extensions/mega-commands.ts +63 -6
- package/extensions/mega-config.ts +22 -0
- package/extensions/mega-events/agent-handlers.ts +88 -5
- package/extensions/mega-events/context-handler.ts +152 -3
- package/extensions/mega-runtime/snapshot.ts +1 -1
- package/extensions/mega-trim.ts +15 -0
- package/package.json +1 -1
- package/src/config/turns.ts +5 -2
- package/src/memoryGraph/embedding.ts +63 -0
- package/src/memoryGraph/gates.ts +451 -0
- package/src/memoryGraph/sources.ts +484 -0
- package/src/memoryGraph.ts +188 -350
- package/src/store/sqlite/embedding-cache.ts +81 -0
- package/src/store/sqlite.ts +1 -0
- package/src/topics/cluster.ts +11 -1
- package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js +0 -2
- package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-P4SUsvMf.js +0 -2
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-P4SUsvMf.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js +0 -2
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/TopicsTab-BWWQxltv.js +0 -2
- package/extensions/dashboard-client/dist/assets/TopicsTab-BWWQxltv.js.map +0 -1
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export const MEGACOMPACT_ENV_KEYS = [
|
|
2
|
+
"MEGACOMPACT_AUTO",
|
|
3
|
+
"MEGACOMPACT_THRESHOLD_TOKENS",
|
|
4
|
+
"MEGACOMPACT_TIER",
|
|
5
|
+
"MEGACOMPACT_DBMIRROR",
|
|
6
|
+
"MEGACOMPACT_MESSAGE_SEPARATION",
|
|
7
|
+
"MEGACOMPACT_CACHE_STRIPING",
|
|
8
|
+
"MEGACOMPACT_LEGACY_DURABLE_TRIM",
|
|
9
|
+
"MEGACOMPACT_RACE_GUARD_STRICT",
|
|
10
|
+
"MEGACOMPACT_RECALL_TAIL_INJECT",
|
|
11
|
+
"MEGACOMPACT_PGLITE_DISABLED",
|
|
12
|
+
"MEGACOMPACT_WIKI_REBUILD_EVERY",
|
|
13
|
+
"MEGACOMPACT_WIKI_SEED_FROM_TURNS",
|
|
14
|
+
"MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS",
|
|
15
|
+
"MEGACOMPACT_MEMORY_GRAPH_SEED_TIER",
|
|
16
|
+
"MEGACOMPACT_MEMORY_GRAPH_SEED_BATCH",
|
|
17
|
+
"MEGACOMPACT_MEMORY_GRAPH_SEED_MAX",
|
|
18
|
+
];
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* gather-debug.ts — assemble a diagnostic bundle for bug reports.
|
|
3
|
+
*
|
|
4
|
+
* POST /api/maintenance/gather-debug → DebugBundleResponse. Reads local
|
|
5
|
+
* SQLite + FS only (PREVENT-PI-004: zero network). Each section is non-fatal:
|
|
6
|
+
* a read failure records `{ error }` for that section and continues, so the
|
|
7
|
+
* bundle always returns 200 with whatever could be gathered.
|
|
8
|
+
*/
|
|
9
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
10
|
+
import { createRequire } from "node:module";
|
|
11
|
+
import { defaultMetricsPath, defaultEventsPath } from "../../src/monitoring.js";
|
|
12
|
+
import { ownVersion } from "../mega-runtime/helpers.js";
|
|
13
|
+
import { MEGACOMPACT_ENV_KEYS } from "./gather-debug-keys.js";
|
|
14
|
+
/** Event-name patterns that indicate a critical/compaction class of error. */
|
|
15
|
+
const CRITICAL_PATTERN = /critical|overflow|already[.\s-]?compacted|compaction[.\s-]?failed|stream[.\s-]?death|poisoned|prefix_stability/i;
|
|
16
|
+
const require = createRequire(import.meta.url);
|
|
17
|
+
/** Lazy store opener (mirrors routes-maintenance.ts's lazyStoreUtils). */
|
|
18
|
+
function openStoreDb(ctx) {
|
|
19
|
+
const mod = require("../../src/store/sqlite/utils.js");
|
|
20
|
+
return mod.openStore(ctx.stateDir);
|
|
21
|
+
}
|
|
22
|
+
/** Lazy maintenance loaders (mirrors routes-maintenance.ts's lazyMaintenance). */
|
|
23
|
+
function loadMaintenance() {
|
|
24
|
+
return require("../../src/store/sqlite/maintenance.js");
|
|
25
|
+
}
|
|
26
|
+
/** Send the debug bundle as JSON. Always 200 (partial on read failure). */
|
|
27
|
+
export function sendGatherDebug(res, ctx) {
|
|
28
|
+
const result = {
|
|
29
|
+
builtAt: Date.now(),
|
|
30
|
+
version: ownVersion(),
|
|
31
|
+
stateDir: ctx.stateDir,
|
|
32
|
+
config: {},
|
|
33
|
+
schemaHealth: { error: "not fetched" },
|
|
34
|
+
storeStats: { error: "not fetched" },
|
|
35
|
+
dashboardSnapshot: { error: "not fetched" },
|
|
36
|
+
recentEvents: [],
|
|
37
|
+
criticalEvents: [],
|
|
38
|
+
};
|
|
39
|
+
// 1. Config flags (no secrets — MEGACOMPACT_* only)
|
|
40
|
+
for (const key of MEGACOMPACT_ENV_KEYS) {
|
|
41
|
+
result.config[key] = process.env[key] ?? "<default>";
|
|
42
|
+
}
|
|
43
|
+
// 2. Schema health
|
|
44
|
+
try {
|
|
45
|
+
const maint = loadMaintenance();
|
|
46
|
+
const integrity = maint.integrityCheck(ctx.stateDir);
|
|
47
|
+
const db = openStoreDb(ctx);
|
|
48
|
+
const metaRow = db
|
|
49
|
+
.prepare("SELECT value FROM meta WHERE key = ?")
|
|
50
|
+
.get("schema_version");
|
|
51
|
+
const schemaVersion = metaRow ? Number(metaRow.value) : 0;
|
|
52
|
+
const fkRows = db.prepare("PRAGMA foreign_key_check").all();
|
|
53
|
+
result.schemaHealth = {
|
|
54
|
+
type: "schema-health",
|
|
55
|
+
schemaVersion,
|
|
56
|
+
integrity,
|
|
57
|
+
fkCheck: fkRows.map((r) => JSON.stringify(r)),
|
|
58
|
+
columns: [],
|
|
59
|
+
healthy: true,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
catch (e) {
|
|
63
|
+
result.schemaHealth = { error: e instanceof Error ? e.message : String(e) };
|
|
64
|
+
}
|
|
65
|
+
// 3. Store stats
|
|
66
|
+
try {
|
|
67
|
+
const maint = loadMaintenance();
|
|
68
|
+
const raw = maint.getDbStats(ctx.stateDir);
|
|
69
|
+
result.storeStats = {
|
|
70
|
+
type: "db-stats",
|
|
71
|
+
tables: Object.entries(raw.tableCounts).map(([table, rowCount]) => ({ table, rowCount })),
|
|
72
|
+
storage: {
|
|
73
|
+
files: { dbBytes: raw.dbBytes, walBytes: raw.walBytes, shmBytes: raw.shmBytes },
|
|
74
|
+
pageSize: raw.pageSize,
|
|
75
|
+
pageCount: raw.pageCount,
|
|
76
|
+
freelistPages: raw.freelistPages,
|
|
77
|
+
},
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
catch (e) {
|
|
81
|
+
result.storeStats = { error: e instanceof Error ? e.message : String(e) };
|
|
82
|
+
}
|
|
83
|
+
// 4. Dashboard snapshot
|
|
84
|
+
try {
|
|
85
|
+
const path = defaultMetricsPath(ctx.stateDir);
|
|
86
|
+
if (existsSync(path)) {
|
|
87
|
+
const raw = readFileSync(path, "utf-8");
|
|
88
|
+
result.dashboardSnapshot = JSON.parse(raw);
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
result.dashboardSnapshot = { error: "no snapshot" };
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
catch (e) {
|
|
95
|
+
result.dashboardSnapshot = { error: e instanceof Error ? e.message : String(e) };
|
|
96
|
+
}
|
|
97
|
+
// 5. Recent + critical events (tail ~200 lines of events.log)
|
|
98
|
+
try {
|
|
99
|
+
const path = defaultEventsPath(ctx.stateDir);
|
|
100
|
+
if (existsSync(path)) {
|
|
101
|
+
const lines = readFileSync(path, "utf-8").split("\n").filter(Boolean);
|
|
102
|
+
for (const line of lines.slice(-200)) {
|
|
103
|
+
try {
|
|
104
|
+
const ev = JSON.parse(line);
|
|
105
|
+
result.recentEvents.push(ev);
|
|
106
|
+
if (CRITICAL_PATTERN.test(line))
|
|
107
|
+
result.criticalEvents.push(ev);
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
continue; // skip unparseable line (PREVENT-001)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
catch (e) {
|
|
116
|
+
result.recentEvents = [{ error: e instanceof Error ? e.message : String(e) }];
|
|
117
|
+
}
|
|
118
|
+
// Send (local helper to avoid importing sendJson from a non-exporting module)
|
|
119
|
+
try {
|
|
120
|
+
const body = JSON.stringify(result);
|
|
121
|
+
res.writeHead(200, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
|
|
122
|
+
res.end(body);
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
// last resort — never leave the response hanging
|
|
126
|
+
res.writeHead(500, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
|
|
127
|
+
res.end('{"error":"failed to serialize debug bundle"}');
|
|
128
|
+
}
|
|
129
|
+
}
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* SQLite only; zero network calls.
|
|
12
12
|
*/
|
|
13
13
|
import { createRequire } from "node:module";
|
|
14
|
+
import { sendGatherDebug } from "./gather-debug.js";
|
|
14
15
|
// ---------------------------------------------------------------------------
|
|
15
16
|
// Lazy imports (via createRequire)
|
|
16
17
|
// ---------------------------------------------------------------------------
|
|
@@ -48,6 +49,11 @@ export function handleMaintenance(req, res, ctx) {
|
|
|
48
49
|
handleMaintenanceAction(req, res, ctx);
|
|
49
50
|
return true;
|
|
50
51
|
}
|
|
52
|
+
// ── POST /api/maintenance/gather-debug ──────────────────────────
|
|
53
|
+
if (method === "POST" && url === "/api/maintenance/gather-debug") {
|
|
54
|
+
sendGatherDebug(res, ctx);
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
51
57
|
// ── GET /api/maintenance ──────────────────────────────────────────
|
|
52
58
|
if (method === "GET" && url === "/api/maintenance") {
|
|
53
59
|
sendDbStats(res, ctx);
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* routes-memory-map.ts — GET /api/memory-map route handler (S46).
|
|
2
|
+
* routes-memory-map.ts — GET /api/memory-map route handler (S46/D3).
|
|
3
3
|
*
|
|
4
4
|
* Builds a memory graph from the local SQLite store and returns it as JSON.
|
|
5
|
+
* D3: surfaces nodeType on each node and the GraphValidationReport.
|
|
5
6
|
* Uses an inline require to the pi-agnostic src/memoryGraph.ts builder.
|
|
6
7
|
*/
|
|
7
8
|
import { createRequire } from "node:module";
|
|
@@ -16,19 +17,8 @@ export function handleMemoryMap(req, res, ctx) {
|
|
|
16
17
|
return false;
|
|
17
18
|
// guardrails-allow PREVENT-PI-004: local loopback-only dashboard server — reads the local SQLite store for graph data.
|
|
18
19
|
const { buildMemoryGraph } = _req("../../src/memoryGraph.js");
|
|
19
|
-
const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId :
|
|
20
|
-
const
|
|
21
|
-
const maxEdges = typeof parsed.query.maxEdgesPerNode === "string" ? Number.parseInt(parsed.query.maxEdgesPerNode, 10) : undefined;
|
|
22
|
-
const raptorEdges = parsed.query.includeRaptorEdges === "false" ? false : true;
|
|
23
|
-
const temporalEdges = parsed.query.includeTemporalEdges === "false" ? false : true;
|
|
24
|
-
const graph = buildMemoryGraph({
|
|
25
|
-
sessionId,
|
|
26
|
-
similarityThreshold: threshold,
|
|
27
|
-
maxEdgesPerNode: maxEdges,
|
|
28
|
-
stateDir: ctx.stateDir,
|
|
29
|
-
includeRaptorEdges: raptorEdges,
|
|
30
|
-
includeTemporalEdges: temporalEdges,
|
|
31
|
-
});
|
|
20
|
+
const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : "";
|
|
21
|
+
const graph = buildMemoryGraph(sessionId, ctx.stateDir);
|
|
32
22
|
const nodes = graph.nodes.map((n) => ({
|
|
33
23
|
id: n.id,
|
|
34
24
|
sessionId: n.sessionId,
|
|
@@ -41,9 +31,12 @@ export function handleMemoryMap(req, res, ctx) {
|
|
|
41
31
|
topicSummary: n.topicSummary,
|
|
42
32
|
decisionCount: n.decisionCount,
|
|
43
33
|
textSnippet: n.textSnippet,
|
|
34
|
+
// D3: pass through nodeType for UI node-shape encoding
|
|
35
|
+
nodeType: n.nodeType,
|
|
44
36
|
}));
|
|
45
37
|
const edges = graph.edges;
|
|
46
|
-
|
|
38
|
+
// D3: surface the validation report (optional in the response type for backward compat)
|
|
39
|
+
const response = { nodes, edges, metadata: graph.metadata, validation: graph.validation };
|
|
47
40
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
48
41
|
res.end(JSON.stringify(response));
|
|
49
42
|
return true;
|
|
@@ -9,12 +9,12 @@ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
|
9
9
|
import { normalizeSessionId } from "../src/store.js";
|
|
10
10
|
import { listCheckpoints, latestModelSnapshot, countInjectedGlobal, listRepoRegistry } from "../src/store/sqlite.js";
|
|
11
11
|
import { decompressSmart } from "../src/store/compression.js";
|
|
12
|
-
import { loadMetrics, fpRate, p95, defaultMetricsPath } from "../src/monitoring.js";
|
|
12
|
+
import { loadMetrics, fpRate, p95, defaultMetricsPath, defaultEventsPath } from "../src/monitoring.js";
|
|
13
13
|
import { C, recentUserQuery } from "./mega-runtime.js";
|
|
14
14
|
import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
|
|
15
15
|
import { vectorStats, vectorRepoStats, vectorDataInvariant } from "../src/vectorStore.js";
|
|
16
16
|
import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: local subprocess detection only (no network)
|
|
17
|
-
import { mkdirSync, writeFileSync } from "node:fs";
|
|
17
|
+
import { mkdirSync, writeFileSync, existsSync, readFileSync } from "node:fs";
|
|
18
18
|
import path from "node:path";
|
|
19
19
|
import os from "node:os";
|
|
20
20
|
import { createRequire } from "node:module";
|
|
@@ -260,6 +260,58 @@ export function registerCommands(pi, runtime, config) {
|
|
|
260
260
|
`• data safety — every compressed region is kept verbatim; nothing is permanently deleted. /mega-restore brings any of it back.`);
|
|
261
261
|
},
|
|
262
262
|
});
|
|
263
|
+
/**
|
|
264
|
+
* Try to peek at recent events.log and mega-compact.log for low recall quality signals.
|
|
265
|
+
* Best-effort, non-fatal — returns true if a low-quality signal was found.
|
|
266
|
+
*/
|
|
267
|
+
/**
|
|
268
|
+
* Try to peek at recent events.log and mega-compact.log for low recall quality signals.
|
|
269
|
+
* Best-effort, non-fatal — returns true if a low-quality signal was found.
|
|
270
|
+
*/
|
|
271
|
+
function checkRecallQuality(stateDir) {
|
|
272
|
+
const eventPaths = [
|
|
273
|
+
defaultEventsPath(stateDir),
|
|
274
|
+
path.join(stateDir, "mega-compact.log"),
|
|
275
|
+
];
|
|
276
|
+
for (const fp of eventPaths) {
|
|
277
|
+
try {
|
|
278
|
+
if (!existsSync(fp))
|
|
279
|
+
continue;
|
|
280
|
+
const buf = readFileSync(fp, { encoding: "utf-8" });
|
|
281
|
+
// Only scan last 64 KiB of the file
|
|
282
|
+
const tail = buf.length > 65536 ? buf.slice(buf.length - 65536) : buf;
|
|
283
|
+
const lines = tail.split("\n").filter(Boolean);
|
|
284
|
+
for (const line of lines) {
|
|
285
|
+
try {
|
|
286
|
+
// guardrails-allow PREVENT-001: JSON.parse with null check
|
|
287
|
+
const parsed = JSON.parse(line);
|
|
288
|
+
if (typeof parsed !== "object" || parsed === null)
|
|
289
|
+
continue;
|
|
290
|
+
// Check for recall_metrics_low_quality event (latest format)
|
|
291
|
+
if (parsed.event === "recall_metrics_low_quality" && parsed.score !== undefined) {
|
|
292
|
+
if (typeof parsed.score === "number" && parsed.score < 0.4)
|
|
293
|
+
return true;
|
|
294
|
+
}
|
|
295
|
+
// Check for recall_metrics with low relevanceScore
|
|
296
|
+
if (parsed.event === "recall_metrics" && parsed.relevanceScore !== undefined) {
|
|
297
|
+
if (typeof parsed.relevanceScore === "number" && parsed.relevanceScore < 0.4)
|
|
298
|
+
return true;
|
|
299
|
+
}
|
|
300
|
+
// Check for RecallQualityEvent format from monitoring.ts
|
|
301
|
+
if (parsed.score !== undefined && typeof parsed.score === "number" && parsed.score < 0.4)
|
|
302
|
+
return true;
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
// skip unparseable lines
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
// non-fatal: skip if can't read
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
return false;
|
|
314
|
+
}
|
|
263
315
|
pi.registerCommand("mega-setup", {
|
|
264
316
|
description: "Local embedding setup wizard — detect available embedders and configure mega-compact.",
|
|
265
317
|
handler: async (_args, ctx) => {
|
|
@@ -335,6 +387,10 @@ export function registerCommands(pi, runtime, config) {
|
|
|
335
387
|
catch {
|
|
336
388
|
// require error
|
|
337
389
|
}
|
|
390
|
+
// --- 2.5 Quality check ---
|
|
391
|
+
// Scan recent events for low recall quality (best-effort, non-fatal)
|
|
392
|
+
const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
|
|
393
|
+
const lowRecallQuality = checkRecallQuality(stateDir);
|
|
338
394
|
const choices = [];
|
|
339
395
|
if (ollamaDetected && ollamaRunning) {
|
|
340
396
|
const model = ollamaModels.find((m) => m.includes("nomic-embed") ||
|
|
@@ -370,6 +426,9 @@ export function registerCommands(pi, runtime, config) {
|
|
|
370
426
|
});
|
|
371
427
|
// --- 4. Prompt user ---
|
|
372
428
|
log(`Current embedder: ${currentEmbedder}`);
|
|
429
|
+
if (lowRecallQuality) {
|
|
430
|
+
log("Tap: recent recall quality is low. Consider switching to Ollama or llama.cpp for better results.");
|
|
431
|
+
}
|
|
373
432
|
let selected = "trigram";
|
|
374
433
|
if (choices.length > 1) {
|
|
375
434
|
try {
|
|
@@ -403,7 +462,6 @@ export function registerCommands(pi, runtime, config) {
|
|
|
403
462
|
];
|
|
404
463
|
const envContent = lines.join("\n") + "\n";
|
|
405
464
|
// Write .mega-compact.env to state dir
|
|
406
|
-
const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
|
|
407
465
|
try {
|
|
408
466
|
mkdirSync(stateDir, { recursive: true });
|
|
409
467
|
writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
|
|
@@ -421,7 +479,6 @@ export function registerCommands(pi, runtime, config) {
|
|
|
421
479
|
`export MEGACOMPACT_EMBEDDING_URL="${url}"`,
|
|
422
480
|
];
|
|
423
481
|
const envContent = lines.join("\n") + "\n";
|
|
424
|
-
const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
|
|
425
482
|
try {
|
|
426
483
|
mkdirSync(stateDir, { recursive: true });
|
|
427
484
|
writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
|
|
@@ -435,7 +492,10 @@ export function registerCommands(pi, runtime, config) {
|
|
|
435
492
|
log("ONNX Runtime configuration is experimental.\n\nSet MEGACOMPACT_EMBEDDING_URL to your local ONNX server URL, or see docs for manual setup.\nKeeping TrigramEmbedder for now.");
|
|
436
493
|
}
|
|
437
494
|
else {
|
|
438
|
-
|
|
495
|
+
const qualityTip = lowRecallQuality
|
|
496
|
+
? " Recent recall quality is low — consider running /mega-setup again to switch to Ollama or llama.cpp for better results."
|
|
497
|
+
: "";
|
|
498
|
+
log(`Keeping TrigramEmbedder (default). No configuration needed.${qualityTip}`);
|
|
439
499
|
}
|
|
440
500
|
}
|
|
441
501
|
catch (e) {
|
|
@@ -168,11 +168,18 @@ export function loadConfig() {
|
|
|
168
168
|
windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
|
|
169
169
|
recallTailInject: envBool("MEGACOMPACT_RECALL_TAIL_INJECT", true),
|
|
170
170
|
messageSeparation: envBool("MEGACOMPACT_MESSAGE_SEPARATION", false),
|
|
171
|
+
cacheStriping: envBool("MEGACOMPACT_CACHE_STRIPING", false),
|
|
171
172
|
tuiWidget: envBool("MEGACOMPACT_TUI_WIDGET", true),
|
|
172
173
|
ragQueryReformulation: envBool("MEGACOMPACT_QUERY_REFORMULATION", false),
|
|
173
174
|
ragTieredRouter: envBool("MEGACOMPACT_TIERED_ROUTER", false),
|
|
174
175
|
ragRecallMetrics: envBool("MEGACOMPACT_RECALL_METRICS", false),
|
|
175
176
|
ragMemoryGraph: envBool("MEGACOMPACT_MEMORY_GRAPH", false),
|
|
177
|
+
wikiSeedFromTurns: envBool("MEGACOMPACT_WIKI_SEED_FROM_TURNS", true),
|
|
178
|
+
memoryGraphSeedTurns: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS", true),
|
|
179
|
+
memoryGraphSeedTurnContent: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true),
|
|
180
|
+
memoryGraphSeedMemories: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_MEMORIES", true),
|
|
181
|
+
memoryGraphCrossTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_CROSS_TYPE_THRESHOLD", 0.85),
|
|
182
|
+
memoryGraphWithinTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_WITHIN_TYPE_THRESHOLD", 0.7),
|
|
176
183
|
// D.1: env-overridable recompact delta (minimum context growth % before
|
|
177
184
|
// re-compacting instead of replaying the cached live trim). Default 50.
|
|
178
185
|
recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),
|
|
@@ -9,6 +9,9 @@ import { classifyError, classifyErrorDetailed, errorRetryBackoffMs, extractError
|
|
|
9
9
|
import { safeSendUserMessage } from "./send-safe.js";
|
|
10
10
|
import { maybeSendProviderOutageAdvisory } from "./outage-advisor.js";
|
|
11
11
|
import { vectorStats } from "../../src/vectorStore.js";
|
|
12
|
+
import { defaultEmbedder } from "../../src/embedder.js";
|
|
13
|
+
import { storeTopicEmbedding, loadTopicEmbedding, detectTopicShift, } from "./separated-prompt.js";
|
|
14
|
+
import { refreshStripeAssignments as writeStripeAssignments, } from "../../src/cache-stripe-impl.js";
|
|
12
15
|
/** Register agent/turn tracking event handlers. */
|
|
13
16
|
export function registerAgentHandlers(pi, runtime, config) {
|
|
14
17
|
// ---- Agent tracking for real-time widget + status-line updates ---------
|
|
@@ -130,10 +133,21 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
130
133
|
// context-handler.ts:258-287 so both call sites stay in sync.
|
|
131
134
|
const cooldownMs = config.raceGuardStrict ? 30_000 : 10_000;
|
|
132
135
|
const sinceCompact = now - (runtime.rt.lastNativeCompactAt ?? 0);
|
|
136
|
+
// CRITICAL-OVER ESCAPE HATCH: when context is critically over the
|
|
137
|
+
// window (>= 90%), force the durable ctx.compact() even if
|
|
138
|
+
// piCompactWouldNoop says it would no-op ("Already compacted").
|
|
139
|
+
// Without this, the on-disk transcript never truncates and the
|
|
140
|
+
// session stays overflowed on every resume — the "Already
|
|
141
|
+
// compacted" + overflow death-spiral (2026-08-01 incident). The
|
|
142
|
+
// cooldown still applies (we don't spam compact), but the no-op
|
|
143
|
+
// gate is bypassed when critical. pi's ctx.compact() may throw
|
|
144
|
+
// "Already compacted" in the truly-no-op case — wrapped in
|
|
145
|
+
// try/catch so the throw is non-fatal.
|
|
146
|
+
const criticalOver = (runtime.lastCtxPercent ?? 0) >= 90;
|
|
133
147
|
if (sinceCompact < cooldownMs) {
|
|
134
148
|
runtime.diagAgentEndDurableSkipRecent++;
|
|
135
149
|
}
|
|
136
|
-
else if (!piCompactWouldNoop(ctx)) {
|
|
150
|
+
else if (criticalOver || !piCompactWouldNoop(ctx)) {
|
|
137
151
|
runtime.debounceUntil = now + 2000;
|
|
138
152
|
runtime.diagAgentEndDurable++;
|
|
139
153
|
runtime.logger.info("agent-end-durable-trigger", {
|
|
@@ -150,6 +164,7 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
150
164
|
// would re-check before it lands.
|
|
151
165
|
const stamp = runtime.rt.lastNativeCompactAt;
|
|
152
166
|
const liveSid = runtime.rt.sessionId;
|
|
167
|
+
const liveCritical = criticalOver;
|
|
153
168
|
setTimeout(() => {
|
|
154
169
|
try {
|
|
155
170
|
if (runtime.rt.sessionId !== liveSid)
|
|
@@ -161,19 +176,26 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
161
176
|
if (runtime.rt.lastNativeCompactAt !== stamp &&
|
|
162
177
|
since2 < cooldownMs)
|
|
163
178
|
return;
|
|
164
|
-
|
|
179
|
+
// CRITICAL-OVER ESCAPE HATCH: bypass the no-op gate when
|
|
180
|
+
// critical (captured at handler time) so the durable trim fires.
|
|
181
|
+
if (!liveCritical && piCompactWouldNoop(ctx))
|
|
165
182
|
return;
|
|
166
183
|
ctx.compact({
|
|
167
184
|
customInstructions: undefined,
|
|
168
|
-
}); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated.
|
|
185
|
+
}); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated. May throw "Already compacted" when the escape hatch forces it — caught below.
|
|
169
186
|
}
|
|
170
187
|
catch {
|
|
171
|
-
/* non-fatal */
|
|
188
|
+
/* non-fatal: "Already compacted" throws are expected when the critical-over hatch forces a no-op compact */
|
|
172
189
|
}
|
|
173
190
|
}, 500);
|
|
174
191
|
}
|
|
175
192
|
else {
|
|
176
|
-
|
|
193
|
+
try {
|
|
194
|
+
ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. May throw "Already compacted" when the critical-over escape hatch forces it — caught.
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
/* non-fatal */
|
|
198
|
+
}
|
|
177
199
|
}
|
|
178
200
|
didDurableTrim = true;
|
|
179
201
|
}
|
|
@@ -712,5 +734,52 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
712
734
|
catch {
|
|
713
735
|
/* non-fatal: a classifier/retry failure never breaks the agent loop */
|
|
714
736
|
}
|
|
737
|
+
// P3.5: topic-shift stripe refresh. When MEGACOMPACT_CACHE_STRIPING is ON,
|
|
738
|
+
// embed the current turn, detect topic shift vs. the previous turn, and
|
|
739
|
+
// refresh cache stripe assignments when shifted (or first turn). Best-effort:
|
|
740
|
+
// failures log + continue, never break the agent loop.
|
|
741
|
+
if (config.cacheStriping) {
|
|
742
|
+
try {
|
|
743
|
+
const stateDir = runtime.currentStateDir;
|
|
744
|
+
const sessionId = runtime.rt.sessionId;
|
|
745
|
+
const currentTurn = event.turnIndex;
|
|
746
|
+
const embedder = defaultEmbedder();
|
|
747
|
+
// (a) Extract text from the assistant's response for topic embedding.
|
|
748
|
+
let textToEmbed = "";
|
|
749
|
+
const msg = event.message;
|
|
750
|
+
if (msg.role === "assistant" && Array.isArray(msg.content)) {
|
|
751
|
+
for (const part of msg.content) {
|
|
752
|
+
if ("text" in part && typeof part.text === "string") {
|
|
753
|
+
textToEmbed += part.text + " ";
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
textToEmbed = textToEmbed.trim() || `turn-${currentTurn}`;
|
|
758
|
+
const embedding = new Float32Array(embedder.embed(textToEmbed));
|
|
759
|
+
storeTopicEmbedding(stateDir, sessionId, currentTurn, embedding);
|
|
760
|
+
// (b) Load previous turn's embedding and detect topic shift.
|
|
761
|
+
const prevEmb = currentTurn > 0
|
|
762
|
+
? loadTopicEmbedding(stateDir, sessionId, currentTurn - 1)
|
|
763
|
+
: null;
|
|
764
|
+
const shifted = currentTurn === 0 || detectTopicShift(embedding, prevEmb);
|
|
765
|
+
// (c) If shifted (or first turn), refresh stripe assignments (WRITE path).
|
|
766
|
+
let chunkCount = 0;
|
|
767
|
+
if (shifted) {
|
|
768
|
+
chunkCount = writeStripeAssignments(stateDir, undefined, embedder, (detail) => runtime.logger.info("stripe_refresh_detail", {
|
|
769
|
+
detail,
|
|
770
|
+
turnIndex: currentTurn,
|
|
771
|
+
}));
|
|
772
|
+
}
|
|
773
|
+
// (d) Log the refresh outcome.
|
|
774
|
+
runtime.logger.info("stripe_refresh", {
|
|
775
|
+
turnIndex: currentTurn,
|
|
776
|
+
shifted,
|
|
777
|
+
chunkCount,
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
catch {
|
|
781
|
+
/* non-fatal: stripe refresh never breaks the agent loop */
|
|
782
|
+
}
|
|
783
|
+
}
|
|
715
784
|
});
|
|
716
785
|
}
|