pi-mega-compact 0.11.7 → 0.11.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server/gather-debug-keys.js +18 -0
- package/dist/extensions/dashboard-server/gather-debug.js +129 -0
- package/dist/extensions/dashboard-server/routes-maintenance.js +6 -0
- package/dist/extensions/dashboard-server/routes-memory-map.js +8 -15
- package/dist/extensions/mega-commands.js +65 -5
- package/dist/extensions/mega-config.js +7 -0
- package/dist/extensions/mega-events/agent-handlers.js +74 -5
- package/dist/extensions/mega-events/context-handler.js +132 -3
- package/dist/extensions/mega-runtime/snapshot.js +1 -1
- package/dist/extensions/mega-trim.js +10 -0
- package/dist/src/config/turns.js +2 -1
- package/dist/src/memoryGraph/embedding.js +50 -0
- package/dist/src/memoryGraph/gates.js +379 -0
- package/dist/src/memoryGraph/sources.js +390 -0
- package/dist/src/memoryGraph.js +133 -259
- package/dist/src/store/sqlite/embedding-cache.js +71 -0
- package/dist/src/store/sqlite.js +1 -0
- package/dist/src/topics/cluster.js +10 -2
- package/extensions/dashboard-client/dist/assets/{AchievementTiles-DR9cLGDL.js → AchievementTiles-DFitrb4Y.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AchievementTiles-DR9cLGDL.js.map → AchievementTiles-DFitrb4Y.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{AchievementsTab-svNUgJ1e.js → AchievementsTab-PJrmq8pa.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AchievementsTab-svNUgJ1e.js.map → AchievementsTab-PJrmq8pa.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/CacheTab-C1WE7dEx.js +2 -0
- package/extensions/dashboard-client/dist/assets/CacheTab-C1WE7dEx.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{ConfigTab-CmmWheuW.js → ConfigTab-Dx-RpO8C.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ConfigTab-CmmWheuW.js.map → ConfigTab-Dx-RpO8C.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{EventsTab-CFE7jLsa.js → EventsTab-C63BEWEs.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{EventsTab-CFE7jLsa.js.map → EventsTab-C63BEWEs.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{GameTab-CJAX_th_.js → GameTab-DHpJE6km.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{GameTab-CJAX_th_.js.map → GameTab-DHpJE6km.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-BSv0GfBb.js +2 -0
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-BSv0GfBb.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-Cg1fgEx_.js +2 -0
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-Cg1fgEx_.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{MetricsTab-CHQscixy.js → MetricsTab-DCahxNOQ.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MetricsTab-CHQscixy.js.map → MetricsTab-DCahxNOQ.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{OverviewTab-DBs4DJdd.js → OverviewTab-RYvO1SZL.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{OverviewTab-DBs4DJdd.js.map → OverviewTab-RYvO1SZL.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{ReposTab-DeFz76L5.js → ReposTab-BDGr_UXb.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ReposTab-DeFz76L5.js.map → ReposTab-BDGr_UXb.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BWNsj1OB.js → SessionsTab-ByrTnHaB.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BWNsj1OB.js.map → SessionsTab-ByrTnHaB.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SetupTab-uS6A0xmw.js → SetupTab-CELafKis.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SetupTab-uS6A0xmw.js.map → SetupTab-CELafKis.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CiqCOQuL.js → TimeSavedCard-BLWtmfr_.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-CiqCOQuL.js.map → TimeSavedCard-BLWtmfr_.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/TopicsTab-8Fsh_Igx.js +2 -0
- package/extensions/dashboard-client/dist/assets/TopicsTab-8Fsh_Igx.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{TurnsTab-CVz4f4-o.js → TurnsTab-D7mgXDe2.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TurnsTab-CVz4f4-o.js.map → TurnsTab-D7mgXDe2.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{index-DrktNOoT.js → index-h3jgEs6G.js} +10 -10
- package/extensions/dashboard-client/dist/assets/{index-DrktNOoT.js.map → index-h3jgEs6G.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{useSSE-Bf5LOLSi.js → useSSE-C5lmipBF.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{useSSE-Bf5LOLSi.js.map → useSSE-C5lmipBF.js.map} +1 -1
- package/extensions/dashboard-client/dist/index.html +1 -1
- package/extensions/dashboard-client/src/api/client.ts +5 -0
- package/extensions/dashboard-client/src/memory-map-layout.ts +161 -0
- package/extensions/dashboard-client/src/memory-map-shapes.tsx +126 -0
- package/extensions/dashboard-client/src/tabs/CacheTab.tsx +29 -2
- package/extensions/dashboard-client/src/tabs/MaintenanceTab.tsx +101 -0
- package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +201 -165
- package/extensions/dashboard-client/src/tabs/ReposTab.tsx +21 -0
- package/extensions/dashboard-client/src/tabs/TopicsTab.tsx +1 -1
- package/extensions/dashboard-server/api-contracts/index.ts +3 -1
- package/extensions/dashboard-server/api-contracts/maintenance.ts +13 -0
- package/extensions/dashboard-server/api-contracts/memory-map.ts +14 -6
- package/extensions/dashboard-server/gather-debug-keys.ts +18 -0
- package/extensions/dashboard-server/gather-debug.ts +136 -0
- package/extensions/dashboard-server/routes-maintenance.ts +7 -0
- package/extensions/dashboard-server/routes-memory-map.ts +21 -29
- package/extensions/mega-commands.ts +63 -6
- package/extensions/mega-config.ts +22 -0
- package/extensions/mega-events/agent-handlers.ts +88 -5
- package/extensions/mega-events/context-handler.ts +152 -3
- package/extensions/mega-runtime/snapshot.ts +1 -1
- package/extensions/mega-trim.ts +15 -0
- package/package.json +1 -1
- package/src/config/turns.ts +5 -2
- package/src/memoryGraph/embedding.ts +63 -0
- package/src/memoryGraph/gates.ts +451 -0
- package/src/memoryGraph/sources.ts +484 -0
- package/src/memoryGraph.ts +188 -350
- package/src/store/sqlite/embedding-cache.ts +81 -0
- package/src/store/sqlite.ts +1 -0
- package/src/topics/cluster.ts +11 -1
- package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js +0 -2
- package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-P4SUsvMf.js +0 -2
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-P4SUsvMf.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js +0 -2
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/TopicsTab-BWWQxltv.js +0 -2
- package/extensions/dashboard-client/dist/assets/TopicsTab-BWWQxltv.js.map +0 -1
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* gather-debug.ts — assemble a diagnostic bundle for bug reports.
|
|
3
|
+
*
|
|
4
|
+
* POST /api/maintenance/gather-debug → DebugBundleResponse. Reads local
|
|
5
|
+
* SQLite + FS only (PREVENT-PI-004: zero network). Each section is non-fatal:
|
|
6
|
+
* a read failure records `{ error }` for that section and continues, so the
|
|
7
|
+
* bundle always returns 200 with whatever could be gathered.
|
|
8
|
+
*/
|
|
9
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
10
|
+
import { createRequire } from "node:module";
|
|
11
|
+
import type { ServerResponse } from "node:http";
|
|
12
|
+
import type { RouteContext } from "./routes-core.js";
|
|
13
|
+
import type { DebugBundleResponse } from "./api-contracts/maintenance.js";
|
|
14
|
+
import { defaultMetricsPath, defaultEventsPath } from "../../src/monitoring.js";
|
|
15
|
+
import { ownVersion } from "../mega-runtime/helpers.js";
|
|
16
|
+
import { MEGACOMPACT_ENV_KEYS } from "./gather-debug-keys.js";
|
|
17
|
+
|
|
18
|
+
/** Event-name patterns that indicate a critical/compaction class of error. */
|
|
19
|
+
const CRITICAL_PATTERN =
|
|
20
|
+
/critical|overflow|already[.\s-]?compacted|compaction[.\s-]?failed|stream[.\s-]?death|poisoned|prefix_stability/i;
|
|
21
|
+
|
|
22
|
+
const require = createRequire(import.meta.url);
|
|
23
|
+
|
|
24
|
+
/** Lazy store opener (mirrors routes-maintenance.ts's lazyStoreUtils). */
|
|
25
|
+
function openStoreDb(ctx: RouteContext) {
|
|
26
|
+
const mod = require("../../src/store/sqlite/utils.js") as typeof import("../../src/store/sqlite/utils.js");
|
|
27
|
+
return mod.openStore(ctx.stateDir);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Lazy maintenance loaders (mirrors routes-maintenance.ts's lazyMaintenance). */
|
|
31
|
+
function loadMaintenance() {
|
|
32
|
+
return require("../../src/store/sqlite/maintenance.js") as typeof import("../../src/store/sqlite/maintenance.js");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Send the debug bundle as JSON. Always 200 (partial on read failure). */
|
|
36
|
+
export function sendGatherDebug(res: ServerResponse, ctx: RouteContext): void {
|
|
37
|
+
const result: DebugBundleResponse = {
|
|
38
|
+
builtAt: Date.now(),
|
|
39
|
+
version: ownVersion(),
|
|
40
|
+
stateDir: ctx.stateDir,
|
|
41
|
+
config: {},
|
|
42
|
+
schemaHealth: { error: "not fetched" },
|
|
43
|
+
storeStats: { error: "not fetched" },
|
|
44
|
+
dashboardSnapshot: { error: "not fetched" },
|
|
45
|
+
recentEvents: [],
|
|
46
|
+
criticalEvents: [],
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
// 1. Config flags (no secrets — MEGACOMPACT_* only)
|
|
50
|
+
for (const key of MEGACOMPACT_ENV_KEYS) {
|
|
51
|
+
result.config[key] = process.env[key] ?? "<default>";
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// 2. Schema health
|
|
55
|
+
try {
|
|
56
|
+
const maint = loadMaintenance();
|
|
57
|
+
const integrity = maint.integrityCheck(ctx.stateDir);
|
|
58
|
+
const db = openStoreDb(ctx);
|
|
59
|
+
const metaRow = db
|
|
60
|
+
.prepare("SELECT value FROM meta WHERE key = ?")
|
|
61
|
+
.get("schema_version") as { value: string } | undefined;
|
|
62
|
+
const schemaVersion = metaRow ? Number(metaRow.value) : 0;
|
|
63
|
+
const fkRows = db.prepare("PRAGMA foreign_key_check").all() as Array<Record<string, unknown>>;
|
|
64
|
+
result.schemaHealth = {
|
|
65
|
+
type: "schema-health" as const,
|
|
66
|
+
schemaVersion,
|
|
67
|
+
integrity,
|
|
68
|
+
fkCheck: fkRows.map((r) => JSON.stringify(r)),
|
|
69
|
+
columns: [],
|
|
70
|
+
healthy: true,
|
|
71
|
+
};
|
|
72
|
+
} catch (e) {
|
|
73
|
+
result.schemaHealth = { error: e instanceof Error ? e.message : String(e) };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// 3. Store stats
|
|
77
|
+
try {
|
|
78
|
+
const maint = loadMaintenance();
|
|
79
|
+
const raw = maint.getDbStats(ctx.stateDir);
|
|
80
|
+
result.storeStats = {
|
|
81
|
+
type: "db-stats" as const,
|
|
82
|
+
tables: Object.entries(raw.tableCounts).map(([table, rowCount]) => ({ table, rowCount })),
|
|
83
|
+
storage: {
|
|
84
|
+
files: { dbBytes: raw.dbBytes, walBytes: raw.walBytes, shmBytes: raw.shmBytes },
|
|
85
|
+
pageSize: raw.pageSize,
|
|
86
|
+
pageCount: raw.pageCount,
|
|
87
|
+
freelistPages: raw.freelistPages,
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
} catch (e) {
|
|
91
|
+
result.storeStats = { error: e instanceof Error ? e.message : String(e) };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// 4. Dashboard snapshot
|
|
95
|
+
try {
|
|
96
|
+
const path = defaultMetricsPath(ctx.stateDir);
|
|
97
|
+
if (existsSync(path)) {
|
|
98
|
+
const raw = readFileSync(path, "utf-8");
|
|
99
|
+
result.dashboardSnapshot = JSON.parse(raw) as unknown;
|
|
100
|
+
} else {
|
|
101
|
+
result.dashboardSnapshot = { error: "no snapshot" };
|
|
102
|
+
}
|
|
103
|
+
} catch (e) {
|
|
104
|
+
result.dashboardSnapshot = { error: e instanceof Error ? e.message : String(e) };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// 5. Recent + critical events (tail ~200 lines of events.log)
|
|
108
|
+
try {
|
|
109
|
+
const path = defaultEventsPath(ctx.stateDir);
|
|
110
|
+
if (existsSync(path)) {
|
|
111
|
+
const lines = readFileSync(path, "utf-8").split("\n").filter(Boolean);
|
|
112
|
+
for (const line of lines.slice(-200)) {
|
|
113
|
+
try {
|
|
114
|
+
const ev = JSON.parse(line) as unknown;
|
|
115
|
+
result.recentEvents.push(ev);
|
|
116
|
+
if (CRITICAL_PATTERN.test(line)) result.criticalEvents.push(ev);
|
|
117
|
+
} catch {
|
|
118
|
+
continue; // skip unparseable line (PREVENT-001)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
} catch (e) {
|
|
123
|
+
result.recentEvents = [{ error: e instanceof Error ? e.message : String(e) }];
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Send (local helper to avoid importing sendJson from a non-exporting module)
|
|
127
|
+
try {
|
|
128
|
+
const body = JSON.stringify(result);
|
|
129
|
+
res.writeHead(200, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
|
|
130
|
+
res.end(body);
|
|
131
|
+
} catch {
|
|
132
|
+
// last resort — never leave the response hanging
|
|
133
|
+
res.writeHead(500, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
|
|
134
|
+
res.end('{"error":"failed to serialize debug bundle"}');
|
|
135
|
+
}
|
|
136
|
+
}
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
import { createRequire } from "node:module";
|
|
15
15
|
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
16
|
+
import { sendGatherDebug } from "./gather-debug.js";
|
|
16
17
|
|
|
17
18
|
import type { RouteContext } from "./routes-core.js";
|
|
18
19
|
|
|
@@ -77,6 +78,12 @@ export function handleMaintenance(
|
|
|
77
78
|
return true;
|
|
78
79
|
}
|
|
79
80
|
|
|
81
|
+
// ── POST /api/maintenance/gather-debug ──────────────────────────
|
|
82
|
+
if (method === "POST" && url === "/api/maintenance/gather-debug") {
|
|
83
|
+
sendGatherDebug(res, ctx);
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
|
|
80
87
|
// ── GET /api/maintenance ──────────────────────────────────────────
|
|
81
88
|
if (method === "GET" && url === "/api/maintenance") {
|
|
82
89
|
sendDbStats(res, ctx);
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* routes-memory-map.ts — GET /api/memory-map route handler (S46).
|
|
2
|
+
* routes-memory-map.ts — GET /api/memory-map route handler (S46/D3).
|
|
3
3
|
*
|
|
4
4
|
* Builds a memory graph from the local SQLite store and returns it as JSON.
|
|
5
|
+
* D3: surfaces nodeType on each node and the GraphValidationReport.
|
|
5
6
|
* Uses an inline require to the pi-agnostic src/memoryGraph.ts builder.
|
|
6
7
|
*/
|
|
7
8
|
import { createRequire } from "node:module";
|
|
@@ -25,14 +26,7 @@ export function handleMemoryMap(
|
|
|
25
26
|
|
|
26
27
|
// guardrails-allow PREVENT-PI-004: local loopback-only dashboard server — reads the local SQLite store for graph data.
|
|
27
28
|
const { buildMemoryGraph } = _req("../../src/memoryGraph.js") as {
|
|
28
|
-
buildMemoryGraph: (
|
|
29
|
-
sessionId?: string;
|
|
30
|
-
similarityThreshold?: number;
|
|
31
|
-
maxEdgesPerNode?: number;
|
|
32
|
-
stateDir?: string;
|
|
33
|
-
includeRaptorEdges?: boolean;
|
|
34
|
-
includeTemporalEdges?: boolean;
|
|
35
|
-
}) => MemoryGraphReturn;
|
|
29
|
+
buildMemoryGraph: (sessionId: string, stateDir: string) => MemoryGraphReturn;
|
|
36
30
|
};
|
|
37
31
|
|
|
38
32
|
type MemoryGraphReturn = {
|
|
@@ -48,6 +42,7 @@ export function handleMemoryMap(
|
|
|
48
42
|
topicSummary: string | undefined;
|
|
49
43
|
decisionCount: number;
|
|
50
44
|
textSnippet: string;
|
|
45
|
+
nodeType: "checkpoint" | "turn" | "turn-content" | "memory";
|
|
51
46
|
}>;
|
|
52
47
|
edges: Array<{
|
|
53
48
|
source: string;
|
|
@@ -56,31 +51,25 @@ export function handleMemoryMap(
|
|
|
56
51
|
type: "temporal" | "semantic" | "raptor_parent";
|
|
57
52
|
}>;
|
|
58
53
|
metadata: {
|
|
59
|
-
|
|
60
|
-
totalCheckpoints: number;
|
|
54
|
+
totalNodes: number;
|
|
61
55
|
totalEdges: number;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
56
|
+
avgWeight: number;
|
|
57
|
+
nodeTypeBreakdown: Record<string, number>;
|
|
58
|
+
edgeTypeBreakdown: Record<string, number>;
|
|
59
|
+
};
|
|
60
|
+
validation: {
|
|
61
|
+
gatesRun: number;
|
|
62
|
+
gatesPassed: number;
|
|
63
|
+
dropped: { nodes: number; edges: number };
|
|
64
|
+
warnings: Array<{ gate: string; code: string; count: number }>;
|
|
65
|
+
sources: { checkpoint: number; turn: number; turnContent: number; memory: number };
|
|
66
66
|
builtAt: number;
|
|
67
67
|
};
|
|
68
68
|
};
|
|
69
69
|
|
|
70
|
-
const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId :
|
|
71
|
-
const threshold = typeof parsed.query.threshold === "string" ? Number.parseFloat(parsed.query.threshold) : undefined;
|
|
72
|
-
const maxEdges = typeof parsed.query.maxEdgesPerNode === "string" ? Number.parseInt(parsed.query.maxEdgesPerNode, 10) : undefined;
|
|
73
|
-
const raptorEdges = parsed.query.includeRaptorEdges === "false" ? false : true;
|
|
74
|
-
const temporalEdges = parsed.query.includeTemporalEdges === "false" ? false : true;
|
|
70
|
+
const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : "";
|
|
75
71
|
|
|
76
|
-
const graph = buildMemoryGraph(
|
|
77
|
-
sessionId,
|
|
78
|
-
similarityThreshold: threshold,
|
|
79
|
-
maxEdgesPerNode: maxEdges,
|
|
80
|
-
stateDir: ctx.stateDir,
|
|
81
|
-
includeRaptorEdges: raptorEdges,
|
|
82
|
-
includeTemporalEdges: temporalEdges,
|
|
83
|
-
});
|
|
72
|
+
const graph = buildMemoryGraph(sessionId, ctx.stateDir);
|
|
84
73
|
|
|
85
74
|
const nodes: MemoryMapNode[] = graph.nodes.map((n) => ({
|
|
86
75
|
id: n.id,
|
|
@@ -94,10 +83,13 @@ export function handleMemoryMap(
|
|
|
94
83
|
topicSummary: n.topicSummary,
|
|
95
84
|
decisionCount: n.decisionCount,
|
|
96
85
|
textSnippet: n.textSnippet,
|
|
86
|
+
// D3: pass through nodeType for UI node-shape encoding
|
|
87
|
+
nodeType: n.nodeType,
|
|
97
88
|
}));
|
|
98
89
|
|
|
99
90
|
const edges: MemoryMapEdgeEntry[] = graph.edges as MemoryMapEdgeEntry[];
|
|
100
|
-
|
|
91
|
+
// D3: surface the validation report (optional in the response type for backward compat)
|
|
92
|
+
const response: MemoryMapResponse = { nodes, edges, metadata: graph.metadata, validation: graph.validation };
|
|
101
93
|
|
|
102
94
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
103
95
|
res.end(JSON.stringify(response));
|
|
@@ -11,13 +11,13 @@ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
|
11
11
|
import { normalizeSessionId } from "../src/store.js";
|
|
12
12
|
import { listCheckpoints, latestModelSnapshot, countInjectedGlobal, listRepoRegistry } from "../src/store/sqlite.js";
|
|
13
13
|
import { decompressSmart } from "../src/store/compression.js";
|
|
14
|
-
import { loadMetrics, fpRate, p95, defaultMetricsPath } from "../src/monitoring.js";
|
|
14
|
+
import { loadMetrics, fpRate, p95, defaultMetricsPath, defaultEventsPath } from "../src/monitoring.js";
|
|
15
15
|
import { type MegaRuntime, C, recentUserQuery } from "./mega-runtime.js";
|
|
16
16
|
import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
|
|
17
17
|
import { vectorStats, vectorRepoStats, vectorDataInvariant } from "../src/vectorStore.js";
|
|
18
18
|
import type { MegaConfig } from "./mega-config.js";
|
|
19
19
|
import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: local subprocess detection only (no network)
|
|
20
|
-
import { mkdirSync, writeFileSync } from "node:fs";
|
|
20
|
+
import { mkdirSync, writeFileSync, existsSync, readFileSync } from "node:fs";
|
|
21
21
|
import path from "node:path";
|
|
22
22
|
import os from "node:os";
|
|
23
23
|
import { createRequire } from "node:module";
|
|
@@ -280,6 +280,52 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
280
280
|
},
|
|
281
281
|
});
|
|
282
282
|
|
|
283
|
+
/**
|
|
284
|
+
* Try to peek at recent events.log and mega-compact.log for low recall quality signals.
|
|
285
|
+
* Best-effort, non-fatal — returns true if a low-quality signal was found.
|
|
286
|
+
*/
|
|
287
|
+
/**
|
|
288
|
+
* Try to peek at recent events.log and mega-compact.log for low recall quality signals.
|
|
289
|
+
* Best-effort, non-fatal — returns true if a low-quality signal was found.
|
|
290
|
+
*/
|
|
291
|
+
function checkRecallQuality(stateDir: string): boolean {
|
|
292
|
+
const eventPaths = [
|
|
293
|
+
defaultEventsPath(stateDir),
|
|
294
|
+
path.join(stateDir, "mega-compact.log"),
|
|
295
|
+
];
|
|
296
|
+
for (const fp of eventPaths) {
|
|
297
|
+
try {
|
|
298
|
+
if (!existsSync(fp)) continue;
|
|
299
|
+
const buf = readFileSync(fp, { encoding: "utf-8" });
|
|
300
|
+
// Only scan last 64 KiB of the file
|
|
301
|
+
const tail = buf.length > 65536 ? buf.slice(buf.length - 65536) : buf;
|
|
302
|
+
const lines = tail.split("\n").filter(Boolean);
|
|
303
|
+
for (const line of lines) {
|
|
304
|
+
try {
|
|
305
|
+
// guardrails-allow PREVENT-001: JSON.parse with null check
|
|
306
|
+
const parsed = JSON.parse(line);
|
|
307
|
+
if (typeof parsed !== "object" || parsed === null) continue;
|
|
308
|
+
// Check for recall_metrics_low_quality event (latest format)
|
|
309
|
+
if (parsed.event === "recall_metrics_low_quality" && parsed.score !== undefined) {
|
|
310
|
+
if (typeof parsed.score === "number" && parsed.score < 0.4) return true;
|
|
311
|
+
}
|
|
312
|
+
// Check for recall_metrics with low relevanceScore
|
|
313
|
+
if (parsed.event === "recall_metrics" && parsed.relevanceScore !== undefined) {
|
|
314
|
+
if (typeof parsed.relevanceScore === "number" && parsed.relevanceScore < 0.4) return true;
|
|
315
|
+
}
|
|
316
|
+
// Check for RecallQualityEvent format from monitoring.ts
|
|
317
|
+
if (parsed.score !== undefined && typeof parsed.score === "number" && parsed.score < 0.4) return true;
|
|
318
|
+
} catch {
|
|
319
|
+
// skip unparseable lines
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
} catch {
|
|
323
|
+
// non-fatal: skip if can't read
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
return false;
|
|
327
|
+
}
|
|
328
|
+
|
|
283
329
|
pi.registerCommand("mega-setup", {
|
|
284
330
|
description:
|
|
285
331
|
"Local embedding setup wizard — detect available embedders and configure mega-compact.",
|
|
@@ -355,6 +401,11 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
355
401
|
// require error
|
|
356
402
|
}
|
|
357
403
|
|
|
404
|
+
// --- 2.5 Quality check ---
|
|
405
|
+
// Scan recent events for low recall quality (best-effort, non-fatal)
|
|
406
|
+
const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
|
|
407
|
+
const lowRecallQuality = checkRecallQuality(stateDir);
|
|
408
|
+
|
|
358
409
|
// --- 3. Build choices ---
|
|
359
410
|
interface Choice {
|
|
360
411
|
label: string;
|
|
@@ -402,6 +453,11 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
402
453
|
|
|
403
454
|
// --- 4. Prompt user ---
|
|
404
455
|
log(`Current embedder: ${currentEmbedder}`);
|
|
456
|
+
if (lowRecallQuality) {
|
|
457
|
+
log(
|
|
458
|
+
"Tap: recent recall quality is low. Consider switching to Ollama or llama.cpp for better results.",
|
|
459
|
+
);
|
|
460
|
+
}
|
|
405
461
|
let selected = "trigram";
|
|
406
462
|
if (choices.length > 1) {
|
|
407
463
|
try {
|
|
@@ -435,7 +491,6 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
435
491
|
const envContent = lines.join("\n") + "\n";
|
|
436
492
|
|
|
437
493
|
// Write .mega-compact.env to state dir
|
|
438
|
-
const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
|
|
439
494
|
try {
|
|
440
495
|
mkdirSync(stateDir, { recursive: true });
|
|
441
496
|
writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
|
|
@@ -457,8 +512,6 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
457
512
|
`export MEGACOMPACT_EMBEDDING_URL="${url}"`,
|
|
458
513
|
];
|
|
459
514
|
const envContent = lines.join("\n") + "\n";
|
|
460
|
-
|
|
461
|
-
const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
|
|
462
515
|
try {
|
|
463
516
|
mkdirSync(stateDir, { recursive: true });
|
|
464
517
|
writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
|
|
@@ -477,7 +530,11 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
477
530
|
"ONNX Runtime configuration is experimental.\n\nSet MEGACOMPACT_EMBEDDING_URL to your local ONNX server URL, or see docs for manual setup.\nKeeping TrigramEmbedder for now.",
|
|
478
531
|
);
|
|
479
532
|
} else {
|
|
480
|
-
|
|
533
|
+
const qualityTip =
|
|
534
|
+
lowRecallQuality
|
|
535
|
+
? " Recent recall quality is low — consider running /mega-setup again to switch to Ollama or llama.cpp for better results."
|
|
536
|
+
: "";
|
|
537
|
+
log(`Keeping TrigramEmbedder (default). No configuration needed.${qualityTip}`);
|
|
481
538
|
}
|
|
482
539
|
} catch (e) {
|
|
483
540
|
ctx.ui.notify(`[mega-compact] /mega-setup failed: ${String(e)}`);
|
|
@@ -172,6 +172,9 @@ export interface MegaConfig {
|
|
|
172
172
|
* from volatile tool results so the prompt-cache prefix stays stable.
|
|
173
173
|
* Default OFF (flag-OFF = byte-identical pre-sprint). */
|
|
174
174
|
messageSeparation: boolean;
|
|
175
|
+
/** P3: Cache-aware striping (PLAN_V2 Phase 3). Inserts stability-ordered
|
|
176
|
+
* cache stripes between summaries and thread. Default OFF. */
|
|
177
|
+
cacheStriping: boolean;
|
|
175
178
|
debug: boolean;
|
|
176
179
|
/** Master reconciliation: TUI shutdown widget (MEGACOMPACT_TUI_WIDGET=0 to disable). */
|
|
177
180
|
tuiWidget: boolean;
|
|
@@ -183,6 +186,18 @@ export interface MegaConfig {
|
|
|
183
186
|
ragRecallMetrics: boolean;
|
|
184
187
|
/** S57 B4: Memory graph traversal (dashboard-oriented). */
|
|
185
188
|
ragMemoryGraph: boolean;
|
|
189
|
+
/** D1: Seed initial wiki topic model from live turns when no context_chunks exist yet. */
|
|
190
|
+
wikiSeedFromTurns: boolean;
|
|
191
|
+
/** D3 Source A: Include structural turn nodes in the memory graph (metadata only, no content). */
|
|
192
|
+
memoryGraphSeedTurns: boolean;
|
|
193
|
+
/** D3 Source B: Include raw_transcript content nodes in the memory graph (requires dbMirror). */
|
|
194
|
+
memoryGraphSeedTurnContent: boolean;
|
|
195
|
+
/** D3 Source C: Include memory review nodes in the memory graph. */
|
|
196
|
+
memoryGraphSeedMemories: boolean;
|
|
197
|
+
/** D3 edges: Stricter cosine floor for cross-type edges (e.g. turn↔checkpoint). */
|
|
198
|
+
memoryGraphCrossTypeThreshold: number;
|
|
199
|
+
/** D3 edges: Cosine floor for within-type semantic edges. */
|
|
200
|
+
memoryGraphWithinTypeThreshold: number;
|
|
186
201
|
}
|
|
187
202
|
|
|
188
203
|
function envFlag(name: string, fallback: number): number {
|
|
@@ -345,11 +360,18 @@ export function loadConfig(): MegaConfig {
|
|
|
345
360
|
windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
|
|
346
361
|
recallTailInject: envBool("MEGACOMPACT_RECALL_TAIL_INJECT", true),
|
|
347
362
|
messageSeparation: envBool("MEGACOMPACT_MESSAGE_SEPARATION", false),
|
|
363
|
+
cacheStriping: envBool("MEGACOMPACT_CACHE_STRIPING", false),
|
|
348
364
|
tuiWidget: envBool("MEGACOMPACT_TUI_WIDGET", true),
|
|
349
365
|
ragQueryReformulation: envBool("MEGACOMPACT_QUERY_REFORMULATION", false),
|
|
350
366
|
ragTieredRouter: envBool("MEGACOMPACT_TIERED_ROUTER", false),
|
|
351
367
|
ragRecallMetrics: envBool("MEGACOMPACT_RECALL_METRICS", false),
|
|
352
368
|
ragMemoryGraph: envBool("MEGACOMPACT_MEMORY_GRAPH", false),
|
|
369
|
+
wikiSeedFromTurns: envBool("MEGACOMPACT_WIKI_SEED_FROM_TURNS", true),
|
|
370
|
+
memoryGraphSeedTurns: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS", true),
|
|
371
|
+
memoryGraphSeedTurnContent: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true),
|
|
372
|
+
memoryGraphSeedMemories: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_MEMORIES", true),
|
|
373
|
+
memoryGraphCrossTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_CROSS_TYPE_THRESHOLD", 0.85),
|
|
374
|
+
memoryGraphWithinTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_WITHIN_TYPE_THRESHOLD", 0.7),
|
|
353
375
|
// D.1: env-overridable recompact delta (minimum context growth % before
|
|
354
376
|
// re-compacting instead of replaying the cached live trim). Default 50.
|
|
355
377
|
recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),
|
|
@@ -26,6 +26,15 @@ import {
|
|
|
26
26
|
import { safeSendUserMessage } from "./send-safe.js";
|
|
27
27
|
import { maybeSendProviderOutageAdvisory } from "./outage-advisor.js";
|
|
28
28
|
import { vectorStats } from "../../src/vectorStore.js";
|
|
29
|
+
import { defaultEmbedder } from "../../src/embedder.js";
|
|
30
|
+
import {
|
|
31
|
+
storeTopicEmbedding,
|
|
32
|
+
loadTopicEmbedding,
|
|
33
|
+
detectTopicShift,
|
|
34
|
+
} from "./separated-prompt.js";
|
|
35
|
+
import {
|
|
36
|
+
refreshStripeAssignments as writeStripeAssignments,
|
|
37
|
+
} from "../../src/cache-stripe-impl.js";
|
|
29
38
|
|
|
30
39
|
/** Register agent/turn tracking event handlers. */
|
|
31
40
|
export function registerAgentHandlers(
|
|
@@ -167,9 +176,20 @@ export function registerAgentHandlers(
|
|
|
167
176
|
// context-handler.ts:258-287 so both call sites stay in sync.
|
|
168
177
|
const cooldownMs = config.raceGuardStrict ? 30_000 : 10_000;
|
|
169
178
|
const sinceCompact = now - (runtime.rt.lastNativeCompactAt ?? 0);
|
|
179
|
+
// CRITICAL-OVER ESCAPE HATCH: when context is critically over the
|
|
180
|
+
// window (>= 90%), force the durable ctx.compact() even if
|
|
181
|
+
// piCompactWouldNoop says it would no-op ("Already compacted").
|
|
182
|
+
// Without this, the on-disk transcript never truncates and the
|
|
183
|
+
// session stays overflowed on every resume — the "Already
|
|
184
|
+
// compacted" + overflow death-spiral (2026-08-01 incident). The
|
|
185
|
+
// cooldown still applies (we don't spam compact), but the no-op
|
|
186
|
+
// gate is bypassed when critical. pi's ctx.compact() may throw
|
|
187
|
+
// "Already compacted" in the truly-no-op case — wrapped in
|
|
188
|
+
// try/catch so the throw is non-fatal.
|
|
189
|
+
const criticalOver = (runtime.lastCtxPercent ?? 0) >= 90;
|
|
170
190
|
if (sinceCompact < cooldownMs) {
|
|
171
191
|
runtime.diagAgentEndDurableSkipRecent++;
|
|
172
|
-
} else if (!piCompactWouldNoop(ctx)) {
|
|
192
|
+
} else if (criticalOver || !piCompactWouldNoop(ctx)) {
|
|
173
193
|
runtime.debounceUntil = now + 2000;
|
|
174
194
|
runtime.diagAgentEndDurable++;
|
|
175
195
|
runtime.logger.info("agent-end-durable-trigger", {
|
|
@@ -186,6 +206,7 @@ export function registerAgentHandlers(
|
|
|
186
206
|
// would re-check before it lands.
|
|
187
207
|
const stamp = runtime.rt.lastNativeCompactAt;
|
|
188
208
|
const liveSid = runtime.rt.sessionId;
|
|
209
|
+
const liveCritical = criticalOver;
|
|
189
210
|
setTimeout(() => {
|
|
190
211
|
try {
|
|
191
212
|
if (runtime.rt.sessionId !== liveSid) return; // session reset
|
|
@@ -198,16 +219,22 @@ export function registerAgentHandlers(
|
|
|
198
219
|
since2 < cooldownMs
|
|
199
220
|
)
|
|
200
221
|
return;
|
|
201
|
-
|
|
222
|
+
// CRITICAL-OVER ESCAPE HATCH: bypass the no-op gate when
|
|
223
|
+
// critical (captured at handler time) so the durable trim fires.
|
|
224
|
+
if (!liveCritical && piCompactWouldNoop(ctx)) return;
|
|
202
225
|
ctx.compact({
|
|
203
226
|
customInstructions: undefined,
|
|
204
|
-
}); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated.
|
|
227
|
+
}); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated. May throw "Already compacted" when the escape hatch forces it — caught below.
|
|
205
228
|
} catch {
|
|
206
|
-
/* non-fatal */
|
|
229
|
+
/* non-fatal: "Already compacted" throws are expected when the critical-over hatch forces a no-op compact */
|
|
207
230
|
}
|
|
208
231
|
}, 500);
|
|
209
232
|
} else {
|
|
210
|
-
|
|
233
|
+
try {
|
|
234
|
+
ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. May throw "Already compacted" when the critical-over escape hatch forces it — caught.
|
|
235
|
+
} catch {
|
|
236
|
+
/* non-fatal */
|
|
237
|
+
}
|
|
211
238
|
}
|
|
212
239
|
didDurableTrim = true;
|
|
213
240
|
}
|
|
@@ -777,5 +804,61 @@ export function registerAgentHandlers(
|
|
|
777
804
|
} catch {
|
|
778
805
|
/* non-fatal: a classifier/retry failure never breaks the agent loop */
|
|
779
806
|
}
|
|
807
|
+
|
|
808
|
+
// P3.5: topic-shift stripe refresh. When MEGACOMPACT_CACHE_STRIPING is ON,
|
|
809
|
+
// embed the current turn, detect topic shift vs. the previous turn, and
|
|
810
|
+
// refresh cache stripe assignments when shifted (or first turn). Best-effort:
|
|
811
|
+
// failures log + continue, never break the agent loop.
|
|
812
|
+
if (config.cacheStriping) {
|
|
813
|
+
try {
|
|
814
|
+
const stateDir = runtime.currentStateDir;
|
|
815
|
+
const sessionId = runtime.rt.sessionId;
|
|
816
|
+
const currentTurn = event.turnIndex;
|
|
817
|
+
const embedder = defaultEmbedder();
|
|
818
|
+
|
|
819
|
+
// (a) Extract text from the assistant's response for topic embedding.
|
|
820
|
+
let textToEmbed = "";
|
|
821
|
+
const msg = event.message;
|
|
822
|
+
if (msg.role === "assistant" && Array.isArray(msg.content)) {
|
|
823
|
+
for (const part of msg.content) {
|
|
824
|
+
if ("text" in part && typeof part.text === "string") {
|
|
825
|
+
textToEmbed += part.text + " ";
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
textToEmbed = textToEmbed.trim() || `turn-${currentTurn}`;
|
|
830
|
+
const embedding = new Float32Array(embedder.embed(textToEmbed));
|
|
831
|
+
storeTopicEmbedding(stateDir, sessionId, currentTurn, embedding);
|
|
832
|
+
|
|
833
|
+
// (b) Load previous turn's embedding and detect topic shift.
|
|
834
|
+
const prevEmb = currentTurn > 0
|
|
835
|
+
? loadTopicEmbedding(stateDir, sessionId, currentTurn - 1)
|
|
836
|
+
: null;
|
|
837
|
+
const shifted = currentTurn === 0 || detectTopicShift(embedding, prevEmb);
|
|
838
|
+
|
|
839
|
+
// (c) If shifted (or first turn), refresh stripe assignments (WRITE path).
|
|
840
|
+
let chunkCount = 0;
|
|
841
|
+
if (shifted) {
|
|
842
|
+
chunkCount = writeStripeAssignments(
|
|
843
|
+
stateDir,
|
|
844
|
+
undefined,
|
|
845
|
+
embedder,
|
|
846
|
+
(detail) => runtime.logger.info("stripe_refresh_detail", {
|
|
847
|
+
detail,
|
|
848
|
+
turnIndex: currentTurn,
|
|
849
|
+
}),
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
// (d) Log the refresh outcome.
|
|
854
|
+
runtime.logger.info("stripe_refresh", {
|
|
855
|
+
turnIndex: currentTurn,
|
|
856
|
+
shifted,
|
|
857
|
+
chunkCount,
|
|
858
|
+
});
|
|
859
|
+
} catch {
|
|
860
|
+
/* non-fatal: stripe refresh never breaks the agent loop */
|
|
861
|
+
}
|
|
862
|
+
}
|
|
780
863
|
});
|
|
781
864
|
}
|