pi-mega-compact 0.8.21 → 0.8.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/LICENSE +6 -2
  2. package/README.md +1 -1
  3. package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
  4. package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
  5. package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
  6. package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
  7. package/dist/extensions/dashboard-server/html.js +2 -621
  8. package/dist/extensions/dashboard-server/routes-core.js +62 -0
  9. package/dist/extensions/dashboard-server/routes-game.js +323 -0
  10. package/dist/extensions/dashboard-server/routes-repo.js +170 -0
  11. package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
  12. package/dist/extensions/dashboard-server/routes.js +10 -0
  13. package/dist/extensions/dashboard-server/server.js +26 -623
  14. package/dist/extensions/mega-commands.js +4 -3
  15. package/dist/extensions/mega-events/agent-handlers.js +2 -1
  16. package/dist/extensions/mega-events/compact-handlers.js +26 -0
  17. package/dist/extensions/mega-events/session-handlers.js +2 -1
  18. package/dist/extensions/mega-pipeline/compact.js +3 -2
  19. package/dist/extensions/mega-runtime/state.js +7 -7
  20. package/dist/src/dedup/raptor/multilevel.js +172 -0
  21. package/dist/src/dedup/raptor/multilevel.test.js +203 -0
  22. package/dist/src/dedup/raptor/promote.test.js +5 -5
  23. package/dist/src/dedup/raptor/retrieval.js +1 -1
  24. package/dist/src/dedup/sprint12.test.js +7 -7
  25. package/dist/src/dedup-engine.test.js +29 -29
  26. package/dist/src/e2e.test.js +38 -38
  27. package/dist/src/engine.js +3 -3
  28. package/dist/src/engine.test.js +6 -6
  29. package/dist/src/importance.js +197 -0
  30. package/dist/src/importance.test.js +372 -0
  31. package/dist/src/ratio.bench.test.js +18 -18
  32. package/dist/src/recall.js +6 -5
  33. package/dist/src/recall.test.js +85 -27
  34. package/dist/src/sprint14.test.js +2 -2
  35. package/dist/src/store/migrate.test.js +5 -5
  36. package/dist/src/store/sprint10.test.js +5 -5
  37. package/dist/src/store/sqlite/global-index.js +5 -174
  38. package/dist/src/store/sqlite/global-sessions.js +190 -0
  39. package/dist/src/vector-read.js +168 -0
  40. package/dist/src/vector-search.js +191 -0
  41. package/dist/src/vectorStore.js +10 -297
  42. package/dist/src/vectorStore.test.js +32 -32
  43. package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
  44. package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
  45. package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
  46. package/extensions/dashboard-server/dashboard-client.ts +21 -0
  47. package/extensions/dashboard-server/html.ts +2 -621
  48. package/extensions/dashboard-server/routes-core.ts +113 -0
  49. package/extensions/dashboard-server/routes-game.ts +386 -0
  50. package/extensions/dashboard-server/routes-repo.ts +212 -0
  51. package/extensions/dashboard-server/routes-sessions.ts +195 -0
  52. package/extensions/dashboard-server/routes.ts +13 -0
  53. package/extensions/dashboard-server/server.ts +37 -700
  54. package/extensions/mega-commands.ts +4 -3
  55. package/extensions/mega-events/agent-handlers.ts +2 -1
  56. package/extensions/mega-events/compact-handlers.ts +28 -0
  57. package/extensions/mega-events/session-handlers.ts +2 -1
  58. package/extensions/mega-pipeline/compact.ts +3 -2
  59. package/extensions/mega-runtime/state.ts +7 -7
  60. package/extensions/openclaw-mega-compact.ts +2 -2
  61. package/package.json +2 -2
  62. package/src/dedup/raptor/multilevel.test.ts +278 -0
  63. package/src/dedup/raptor/multilevel.ts +246 -0
  64. package/src/dedup/raptor/promote.test.ts +5 -5
  65. package/src/dedup/raptor/retrieval.ts +1 -1
  66. package/src/dedup/sprint12.test.ts +7 -7
  67. package/src/dedup-engine.test.ts +30 -30
  68. package/src/e2e.test.ts +38 -38
  69. package/src/engine.test.ts +6 -6
  70. package/src/engine.ts +3 -3
  71. package/src/importance.test.ts +538 -0
  72. package/src/importance.ts +312 -0
  73. package/src/ratio.bench.test.ts +18 -18
  74. package/src/recall.test.ts +101 -29
  75. package/src/recall.ts +9 -9
  76. package/src/sprint14.test.ts +2 -2
  77. package/src/store/migrate.test.ts +5 -5
  78. package/src/store/sprint10.test.ts +5 -5
  79. package/src/store/sqlite/global-index.ts +18 -290
  80. package/src/store/sqlite/global-sessions.ts +291 -0
  81. package/src/vector-read.ts +237 -0
  82. package/src/vector-search.ts +231 -0
  83. package/src/vectorStore.test.ts +32 -32
  84. package/src/vectorStore.ts +29 -356
@@ -0,0 +1,159 @@
1
+ /**
2
+ * dashboard-server/routes-sessions.ts — Session and SSE event route handlers.
3
+ */
4
+ import { createRequire } from "node:module";
5
+ import { existsSync, watch } from "node:fs";
6
+ import { readFrom } from "./snapshot.js";
7
+ // ---------------------------------------------------------------------------
8
+ // handleEvents — "/api/events" SSE
9
+ // ---------------------------------------------------------------------------
10
+ export function handleEvents(req, res, ctx) {
11
+ if (req.url !== "/api/events")
12
+ return false;
13
+ const { eventsPath, eventOffsetRef } = ctx;
14
+ res.writeHead(200, {
15
+ "Content-Type": "text/event-stream",
16
+ "Cache-Control": "no-cache",
17
+ Connection: "keep-alive",
18
+ });
19
+ // Drain existing events so the client starts with history
20
+ const { data: existing, offset: initialOffset } = readFrom(eventsPath, 0);
21
+ eventOffsetRef.value = initialOffset;
22
+ const lines = existing.split("\n").filter((l) => l.trim());
23
+ for (const line of lines) {
24
+ res.write(`data: ${line}\n\n`);
25
+ }
26
+ // Tail new events via fs.watch (coalesced with 100ms debounce)
27
+ let watchTimer = null;
28
+ const onWatch = () => {
29
+ if (watchTimer)
30
+ return;
31
+ watchTimer = setTimeout(() => {
32
+ watchTimer = null;
33
+ const { data, offset } = readFrom(eventsPath, eventOffsetRef.value);
34
+ eventOffsetRef.value = offset;
35
+ const newLines = data.split("\n").filter((l) => l.trim());
36
+ for (const line of newLines) {
37
+ res.write(`data: ${line}\n\n`);
38
+ }
39
+ }, 100);
40
+ };
41
+ // Set up file watching: if file exists, watch it directly;
42
+ // otherwise poll for creation every 1s then switch to fs.watch.
43
+ let watcher = null;
44
+ let pollInterval = null;
45
+ function startFileWatch() {
46
+ try {
47
+ watcher = watch(eventsPath, onWatch);
48
+ }
49
+ catch {
50
+ /* give up */
51
+ }
52
+ }
53
+ if (existsSync(eventsPath)) {
54
+ startFileWatch();
55
+ }
56
+ else {
57
+ pollInterval = setInterval(() => {
58
+ if (existsSync(eventsPath)) {
59
+ if (pollInterval) {
60
+ clearInterval(pollInterval);
61
+ pollInterval = null;
62
+ }
63
+ startFileWatch();
64
+ }
65
+ }, 1000);
66
+ }
67
+ req.on("close", () => {
68
+ if (watchTimer)
69
+ clearTimeout(watchTimer);
70
+ if (pollInterval)
71
+ clearInterval(pollInterval);
72
+ watcher?.close();
73
+ });
74
+ return true;
75
+ }
76
+ // ---------------------------------------------------------------------------
77
+ // handleSessions — "/api/sessions" (GET sessions list) and
78
+ // "/api/sessions/timeseries" (GET timeseries).
79
+ // ---------------------------------------------------------------------------
80
+ export function handleSessions(req, res, _ctx) {
81
+ if (!req.url?.startsWith("/api/sessions"))
82
+ return false;
83
+ // /api/sessions/timeseries handled inline below.
84
+ if (req.url.startsWith("/api/sessions/timeseries")) {
85
+ // /api/sessions/timeseries — S39: stacked per-session token timeseries for
86
+ // the recharts memory graph. GET ?minutes=N (clamped [1,1440]) returns
87
+ // {updatedAt, windowMinutes, series[], totals[]} in recharts-ready shape.
88
+ // Non-GET -> 405. PREVENT-PI-004: loopback.
89
+ const tsReq = createRequire(import.meta.url);
90
+ const { readSessionTimeseries, pruneTokenSamples, } = tsReq("../../src/store/sqlite.js");
91
+ if (req.method !== "GET") {
92
+ res.writeHead(405, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
93
+ res.end(JSON.stringify({ error: "method_not_allowed" }));
94
+ return true;
95
+ }
96
+ try {
97
+ const url = new URL(req.url, "http://x"); // guardrails-allow PREVENT-PI-004: localhost dashboard URL base (loopback-only)
98
+ let minutes = Number(url.searchParams.get("minutes") ?? "30");
99
+ if (!Number.isFinite(minutes) || minutes <= 0)
100
+ minutes = 30;
101
+ minutes = Math.min(Math.max(minutes, 1), 1440);
102
+ const sinceTs = Date.now() - minutes * 60_000;
103
+ const pruneMs = Math.max(minutes * 60_000, 1_800_000);
104
+ pruneTokenSamples(pruneMs); // guardrails-allow PREVENT-PI-004: local SQLite read (loopback dashboard)
105
+ const result = readSessionTimeseries(sinceTs); // guardrails-allow PREVENT-PI-004: local SQLite read (loopback dashboard)
106
+ res.writeHead(200, { "Content-Type": "application/json" });
107
+ res.end(JSON.stringify({
108
+ updatedAt: new Date().toISOString(),
109
+ windowMinutes: minutes,
110
+ series: result.series,
111
+ totals: result.totals,
112
+ }));
113
+ }
114
+ catch (e) {
115
+ res.writeHead(500, { "Content-Type": "application/json" });
116
+ res.end(JSON.stringify({ error: "timeseries_unavailable", detail: String(e) }));
117
+ }
118
+ return true;
119
+ }
120
+ // /api/sessions — S39: active pi sessions with latest token usage + heartbeat.
121
+ // GET returns {updatedAt, pruned, sessions[]} after pruning stale heartbeats.
122
+ // The dashboard server is a detached child with no MegaRuntime ref, so it
123
+ // reads session_heartbeats via a require()'d sqlite helper (same pattern as
124
+ // /api/achievements, /api/perf). Non-GET -> 405. PREVENT-PI-004: loopback.
125
+ const sReq = createRequire(import.meta.url);
126
+ const { readActiveSessions, pruneStaleSessions, listRepoRegistry, } = sReq("../../src/store/sqlite.js");
127
+ if (req.method !== "GET") {
128
+ res.writeHead(405, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
129
+ res.end(JSON.stringify({ error: "method_not_allowed" }));
130
+ return true;
131
+ }
132
+ try {
133
+ const pruned = pruneStaleSessions(); // guardrails-allow PREVENT-PI-004: local SQLite read (loopback dashboard)
134
+ const active = readActiveSessions();
135
+ const repos = listRepoRegistry();
136
+ const repoMap = new Map(repos.map((r) => [r.repoRoot, r]));
137
+ const sessions = active.map((s) => ({
138
+ pid: s.pid,
139
+ sessionId: s.sessionId,
140
+ repoRoot: s.repoRoot,
141
+ displayName: s.repoRoot
142
+ ? (s.repoRoot.split(/[\\/]/).filter(Boolean).pop() ?? s.repoRoot)
143
+ : (s.stateDir?.split(/[\\/]/).filter(Boolean).pop() ?? "unknown"),
144
+ model: s.repoRoot ? (repoMap.get(s.repoRoot)?.modelName ?? null) : null,
145
+ tokens: s.tokens,
146
+ percent: s.percent,
147
+ ctxWindow: s.ctxWindow,
148
+ lastSeen: s.lastSeen,
149
+ stateDir: s.stateDir,
150
+ }));
151
+ res.writeHead(200, { "Content-Type": "application/json" });
152
+ res.end(JSON.stringify({ updatedAt: new Date().toISOString(), pruned, sessions }));
153
+ }
154
+ catch (e) {
155
+ res.writeHead(500, { "Content-Type": "application/json" });
156
+ res.end(JSON.stringify({ error: "sessions_unavailable", detail: String(e) }));
157
+ }
158
+ return true;
159
+ }
@@ -0,0 +1,10 @@
1
+ /**
2
+ * dashboard-server/routes.ts — Barrel: re-exports all route handlers + types.
3
+ *
4
+ * server.ts imports only from this barrel; the actual handler implementations
5
+ * live in the split files below.
6
+ */
7
+ export { buildRouteContext } from "./routes-core.js";
8
+ export { handleIndex, handleRepoIndex, handleStatic } from "./routes-repo.js";
9
+ export { handleGameState, handleGameScores, handlePerf, handleAchievements } from "./routes-game.js";
10
+ export { handleEvents, handleSessions } from "./routes-sessions.js";