pi-mega-compact 0.8.21 → 0.8.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/LICENSE +6 -2
  2. package/README.md +1 -1
  3. package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
  4. package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
  5. package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
  6. package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
  7. package/dist/extensions/dashboard-server/html.js +2 -621
  8. package/dist/extensions/dashboard-server/routes-core.js +62 -0
  9. package/dist/extensions/dashboard-server/routes-game.js +323 -0
  10. package/dist/extensions/dashboard-server/routes-repo.js +170 -0
  11. package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
  12. package/dist/extensions/dashboard-server/routes.js +10 -0
  13. package/dist/extensions/dashboard-server/server.js +26 -623
  14. package/dist/extensions/mega-commands.js +4 -3
  15. package/dist/extensions/mega-events/agent-handlers.js +2 -1
  16. package/dist/extensions/mega-events/compact-handlers.js +26 -0
  17. package/dist/extensions/mega-events/session-handlers.js +2 -1
  18. package/dist/extensions/mega-pipeline/compact.js +3 -2
  19. package/dist/extensions/mega-runtime/state.js +7 -7
  20. package/dist/src/dedup/raptor/multilevel.js +172 -0
  21. package/dist/src/dedup/raptor/multilevel.test.js +203 -0
  22. package/dist/src/dedup/raptor/promote.test.js +5 -5
  23. package/dist/src/dedup/raptor/retrieval.js +1 -1
  24. package/dist/src/dedup/sprint12.test.js +7 -7
  25. package/dist/src/dedup-engine.test.js +29 -29
  26. package/dist/src/e2e.test.js +38 -38
  27. package/dist/src/engine.js +3 -3
  28. package/dist/src/engine.test.js +6 -6
  29. package/dist/src/importance.js +197 -0
  30. package/dist/src/importance.test.js +372 -0
  31. package/dist/src/ratio.bench.test.js +18 -18
  32. package/dist/src/recall.js +6 -5
  33. package/dist/src/recall.test.js +85 -27
  34. package/dist/src/sprint14.test.js +2 -2
  35. package/dist/src/store/migrate.test.js +5 -5
  36. package/dist/src/store/sprint10.test.js +5 -5
  37. package/dist/src/store/sqlite/global-index.js +5 -174
  38. package/dist/src/store/sqlite/global-sessions.js +190 -0
  39. package/dist/src/vector-read.js +168 -0
  40. package/dist/src/vector-search.js +191 -0
  41. package/dist/src/vectorStore.js +10 -297
  42. package/dist/src/vectorStore.test.js +32 -32
  43. package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
  44. package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
  45. package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
  46. package/extensions/dashboard-server/dashboard-client.ts +21 -0
  47. package/extensions/dashboard-server/html.ts +2 -621
  48. package/extensions/dashboard-server/routes-core.ts +113 -0
  49. package/extensions/dashboard-server/routes-game.ts +386 -0
  50. package/extensions/dashboard-server/routes-repo.ts +212 -0
  51. package/extensions/dashboard-server/routes-sessions.ts +195 -0
  52. package/extensions/dashboard-server/routes.ts +13 -0
  53. package/extensions/dashboard-server/server.ts +37 -700
  54. package/extensions/mega-commands.ts +4 -3
  55. package/extensions/mega-events/agent-handlers.ts +2 -1
  56. package/extensions/mega-events/compact-handlers.ts +28 -0
  57. package/extensions/mega-events/session-handlers.ts +2 -1
  58. package/extensions/mega-pipeline/compact.ts +3 -2
  59. package/extensions/mega-runtime/state.ts +7 -7
  60. package/extensions/openclaw-mega-compact.ts +2 -2
  61. package/package.json +2 -2
  62. package/src/dedup/raptor/multilevel.test.ts +278 -0
  63. package/src/dedup/raptor/multilevel.ts +246 -0
  64. package/src/dedup/raptor/promote.test.ts +5 -5
  65. package/src/dedup/raptor/retrieval.ts +1 -1
  66. package/src/dedup/sprint12.test.ts +7 -7
  67. package/src/dedup-engine.test.ts +30 -30
  68. package/src/e2e.test.ts +38 -38
  69. package/src/engine.test.ts +6 -6
  70. package/src/engine.ts +3 -3
  71. package/src/importance.test.ts +538 -0
  72. package/src/importance.ts +312 -0
  73. package/src/ratio.bench.test.ts +18 -18
  74. package/src/recall.test.ts +101 -29
  75. package/src/recall.ts +9 -9
  76. package/src/sprint14.test.ts +2 -2
  77. package/src/store/migrate.test.ts +5 -5
  78. package/src/store/sprint10.test.ts +5 -5
  79. package/src/store/sqlite/global-index.ts +18 -290
  80. package/src/store/sqlite/global-sessions.ts +291 -0
  81. package/src/vector-read.ts +237 -0
  82. package/src/vector-search.ts +231 -0
  83. package/src/vectorStore.test.ts +32 -32
  84. package/src/vectorStore.ts +29 -356
@@ -0,0 +1,212 @@
1
+ /**
2
+ * dashboard-client-repos.ts — verbatim JS chunk: multi-repo index rendering.
3
+ *
4
+ * Extracted from html.ts (PR0 split) — ZERO behavior change. Second of three
5
+ * chunks concatenated into the dashboard IIFE (core → repos → game). Shares
6
+ * the closure with core + game: uses sanitize() from core, fmtSec/renderGameScores
7
+ * from core/game.
8
+ */
9
+ /** Multi-repo index rendering, repo detail modal, active-repos table. */
10
+ export function dashboardClientReposJs() {
11
+ return `// --- Multi-repo (index.sqlite via /api/index) ---------------------------
12
+ function fmtBytesTop(b) {
13
+ b = b || 0;
14
+ if (b >= 1048576) return (b / 1048576).toFixed(1) + ' MiB';
15
+ if (b >= 1024) return (b / 1024).toFixed(1) + ' KiB';
16
+ return b + ' B';
17
+ }
18
+ function renderIndex(d) {
19
+ d = d || { updatedAt: null, summary: null, repos: [] };
20
+ var repos = d.repos || [];
21
+ var s = d.summary || { totalRepos: 0, totalCheckpoints: 0, totalTokensSaved: 0, totalCompressedOriginalBytes: 0 };
22
+ document.getElementById('sm-repos').textContent = (s.totalRepos || 0).toLocaleString();
23
+ document.getElementById('sm-checkpoints').textContent = (s.totalCheckpoints || 0).toLocaleString();
24
+ document.getElementById('sm-saved').textContent = (s.totalTokensSaved || 0).toLocaleString();
25
+ document.getElementById('sm-bytes').textContent = fmtBytesTop(s.totalCompressedOriginalBytes);
26
+
27
+ // Shared clickable-row renderer for both the in-current table and the
28
+ // All-repos tab — each row opens the per-repo detail modal.
29
+ function rowsHtml() {
30
+ if (!repos.length) return '<tr><td colspan="6" class="repo-none">No repositories registered yet.</td></tr>';
31
+ return repos.map(function(r) {
32
+ var model = r.modelName
33
+ ? '<span class="repo-model">' + sanitize(r.modelName) + '</span>'
34
+ : '<span class="repo-none">—</span>';
35
+ var when = r.lastCompactedAt ? new Date(r.lastCompactedAt).toLocaleString() : '—';
36
+ return '<tr class="repo-link" data-repo="' + sanitize(r.repoRoot) + '">' +
37
+ '<td title="' + sanitize(r.repoRoot) + '">' + sanitize(r.displayName || r.repoRoot) + '</td>' +
38
+ '<td>' + model + '</td>' +
39
+ '<td class="num">' + (r.checkpointCount || 0).toLocaleString() + '</td>' +
40
+ '<td class="num">' + (r.tokensSaved || 0).toLocaleString() + '</td>' +
41
+ '<td class="num">' + fmtBytesTop(r.compressedOriginalBytes) + '</td>' +
42
+ '<td class="num">' + sanitize(when) + '</td>' +
43
+ '</tr>';
44
+ }).join('');
45
+ }
46
+ document.getElementById('cur-rows').innerHTML = rowsHtml();
47
+ document.getElementById('all-rows').innerHTML = rowsHtml();
48
+ bindRepoRows();
49
+
50
+ var stamp = d.updatedAt ? 'Updated ' + new Date(d.updatedAt).toLocaleTimeString() : '';
51
+ document.getElementById('cur-updated').textContent = stamp;
52
+ document.getElementById('all-updated').textContent = stamp;
53
+ document.getElementById('sm-updated').textContent = stamp;
54
+ renderByModel(repos);
55
+ }
56
+
57
+ // Savings-by-model aggregation for the Summary tab — groups the machine-
58
+ // wide repo registry by (modelName || '(unknown)') so the user can see how
59
+ // much context + cost mega-compact has reclaimed, broken down by which model
60
+ // they were running. $ Saved = Σ(tokensSaved × inputRate) per model. Sorted
61
+ // by tokens saved descending so the biggest-reclaim model wins the top row.
62
+ function renderByModel(repos) {
63
+ var rows = document.getElementById('bm-rows');
64
+ if (!rows) return;
65
+ if (!repos || !repos.length) {
66
+ rows.innerHTML = '<tr><td colspan="14" class="repo-none">No repositories registered yet.</td></tr>';
67
+ return;
68
+ }
69
+ var groups = {};
70
+ for (var i = 0; i < repos.length; i++) {
71
+ var r = repos[i];
72
+ var key = (r.modelName && String(r.modelName).trim()) || '(unknown)';
73
+ if (!groups[key]) groups[key] = {
74
+ model: key, provider: r.providerName || r.provider || '—', repos: 0, checkpoints: 0,
75
+ tokensSaved: 0, tokensIn: 0, tokensOut: 0, sessions: 0, usd: 0, lastAt: 0,
76
+ inRates: [], outRates: [], ctxWindows: [], maxTokens: [], reasoning: null,
77
+ };
78
+ var g = groups[key];
79
+ g.repos++;
80
+ g.checkpoints += (r.checkpointCount || 0);
81
+ g.tokensSaved += (r.tokensSaved || 0);
82
+ g.tokensIn += (r.tokensDropped || 0);
83
+ g.tokensOut += (r.tokensKept || 0);
84
+ g.sessions += (r.sessions || 0);
85
+ if (r.inputRate) { g.usd += (r.tokensSaved || 0) * r.inputRate; g.inRates.push(r.inputRate); }
86
+ if (r.outputRate) g.outRates.push(r.outputRate);
87
+ if (r.contextWindow) g.ctxWindows.push(r.contextWindow);
88
+ if (r.maxTokens) g.maxTokens.push(r.maxTokens);
89
+ if (r.reasoning != null) g.reasoning = r.reasoning;
90
+ if (r.lastCompactedAt && r.lastCompactedAt > g.lastAt) g.lastAt = r.lastCompactedAt;
91
+ }
92
+ var arr = [];
93
+ for (var k in groups) { if (Object.prototype.hasOwnProperty.call(groups, k)) arr.push(groups[k]); }
94
+ arr.sort(function(a, b) { return b.tokensSaved - a.tokensSaved; });
95
+ // Helper: a set of numeric samples collapses to a single value when all
96
+ // repos in the group agree, otherwise shows the range (min–max) so the
97
+ // user can see mixed-config model groups at a glance.
98
+ function collapseNum(samples) {
99
+ if (!samples || !samples.length) return '—';
100
+ var lo = Math.min.apply(null, samples), hi = Math.max.apply(null, samples);
101
+ return lo === hi ? lo.toLocaleString() : lo.toLocaleString() + '–' + hi.toLocaleString();
102
+ }
103
+ function collapseRate(samples) {
104
+ if (!samples || !samples.length) return '—';
105
+ var lo = Math.min.apply(null, samples), hi = Math.max.apply(null, samples);
106
+ var fmt = function(v) { return '$' + v.toFixed(6); };
107
+ return lo === hi ? fmt(lo) : fmt(lo) + '–' + fmt(hi);
108
+ }
109
+ rows.innerHTML = arr.map(function(g) {
110
+ var freed = (g.tokensIn || 0) - (g.tokensOut || 0);
111
+ var usd = g.usd > 0 ? '$' + g.usd.toFixed(4) : '—';
112
+ var when = g.lastAt ? new Date(g.lastAt).toLocaleString() : '—';
113
+ var reas = g.reasoning == null ? '—' : (g.reasoning ? 'yes' : 'no');
114
+ return '<tr>' +
115
+ '<td><span class="repo-model">' + sanitize(g.model) + '</span></td>' +
116
+ '<td>' + sanitize(g.provider) + '</td>' +
117
+ '<td class="num">' + (g.tokensIn || 0).toLocaleString() + '</td>' +
118
+ '<td class="num">' + (g.tokensOut || 0).toLocaleString() + '</td>' +
119
+ '<td class="num">' + freed.toLocaleString() + '</td>' +
120
+ '<td class="num">' + collapseNum(g.ctxWindows) + '</td>' +
121
+ '<td class="num">' + collapseNum(g.maxTokens) + '</td>' +
122
+ '<td class="num">' + reas + '</td>' +
123
+ '<td class="num">' + g.sessions.toLocaleString() + '</td>' +
124
+ '<td class="num">' + g.checkpoints.toLocaleString() + '</td>' +
125
+ '<td class="num">' + collapseRate(g.inRates) + '</td>' +
126
+ '<td class="num">' + collapseRate(g.outRates) + '</td>' +
127
+ '<td class="num">' + sanitize(usd) + '</td>' +
128
+ '<td class="num">' + sanitize(when) + '</td>' +
129
+ '</tr>';
130
+ }).join('');
131
+ }
132
+
133
+ // Per-repo detail modal ---------------------------------------------------
134
+ var detailEl = document.getElementById('repo-detail');
135
+ var indexCache = { repos: [] };
136
+ function openRepoDetail(root) {
137
+ var r = null;
138
+ for (var i = 0; i < indexCache.repos.length; i++) {
139
+ if (indexCache.repos[i].repoRoot === root) { r = indexCache.repos[i]; break; }
140
+ }
141
+ if (!r) return;
142
+ document.getElementById('rd-name').textContent = r.displayName || r.repoRoot;
143
+ document.getElementById('rd-path').textContent = r.repoRoot;
144
+ document.getElementById('rd-model').textContent = r.modelName || '—';
145
+ document.getElementById('rd-provider').textContent = r.providerName || (r.provider || '—');
146
+ document.getElementById('rd-cp').textContent = (r.checkpointCount || 0).toLocaleString();
147
+ document.getElementById('rd-saved').textContent = (r.tokensSaved || 0).toLocaleString();
148
+ document.getElementById('rd-bytes').textContent = fmtBytesTop(r.compressedOriginalBytes);
149
+ document.getElementById('rd-when').textContent = r.lastCompactedAt ? new Date(r.lastCompactedAt).toLocaleString() : '—';
150
+ detailEl.classList.add('open');
151
+ }
152
+ document.getElementById('rd-close').addEventListener('click', function() { detailEl.classList.remove('open'); });
153
+ detailEl.addEventListener('click', function(e) { if (e.target === detailEl) detailEl.classList.remove('open'); });
154
+ function bindRepoRows() {
155
+ var rows = document.querySelectorAll('.repo-link');
156
+ for (var i = 0; i < rows.length; i++) {
157
+ rows[i].addEventListener('click', function() { openRepoDetail(this.getAttribute('data-repo')); });
158
+ }
159
+ }
160
+ function pollIndex() {
161
+ fetch('/api/index').then(function(r) { return r.json(); }).then(function(d) { // guardrails-allow PREVENT-PI-004: browser-side fetch in dashboard HTML template (not Node runtime)
162
+ indexCache = d && d.repos ? d : indexCache;
163
+ renderIndex(d);
164
+ }).catch(function() {});
165
+ }
166
+ pollIndex();
167
+ setInterval(pollIndex, 5000);
168
+
169
+ // --- Active repos (live cache-hit / compaction stats) ---------------------
170
+ function fmtSec(s) {
171
+ s = s || 0;
172
+ if (s >= 3600) return (s / 3600).toFixed(1) + 'h';
173
+ if (s >= 60) return Math.round(s / 60) + 'm';
174
+ if (s >= 1) return s.toFixed(1) + 's';
175
+ return Math.round(s * 1000) + 'ms';
176
+ }
177
+ function renderActiveRepos(d) {
178
+ d = d || { updatedAt: null, servers: [] };
179
+ var servers = d.servers || [];
180
+ var rowsEl = document.getElementById('active-rows');
181
+ if (!rowsEl) return;
182
+ if (!servers.length) {
183
+ rowsEl.innerHTML = '<tr><td colspan="9" class="repo-none">No active repositories.</td></tr>';
184
+ } else {
185
+ rowsEl.innerHTML = servers.map(function(r) {
186
+ var ch = r.cacheHits || { session: 0, total: 0, sessionTokensSaved: 0, totalTokensSaved: 0 };
187
+ var cp = r.compacts || { session: 0, total: 0 };
188
+ var ts = r.timeSaved || { compact: { sessionSec: 0, totalSec: 0 }, cacheHit: { sessionSec: 0, totalSec: 0 } };
189
+ return '<tr>' +
190
+ '<td title="' + sanitize(r.repoRoot) + '">' + sanitize(r.displayName || r.repoRoot) + '</td>' +
191
+ '<td>' + sanitize(r.model || '—') + '</td>' +
192
+ '<td>' + sanitize(r.tier || '—') + '</td>' +
193
+ '<td class="num">' + (r.contextPct != null ? Math.round(r.contextPct * 100) + '%' : '—') + '</td>' +
194
+ '<td>' + sanitize(r.state || '—') + '</td>' +
195
+ '<td class="num">' + (cp.session || 0) + ' / ' + (cp.total || 0) + '</td>' +
196
+ '<td class="num">' + (ch.session || 0) + ' / ' + (ch.total || 0) + '</td>' +
197
+ '<td class="num">' + fmtSec(ts.compact.sessionSec) + ' / ' + fmtSec(ts.compact.totalSec) + '</td>' +
198
+ '<td class="num">' + fmtSec(ts.cacheHit.sessionSec) + ' / ' + fmtSec(ts.cacheHit.totalSec) + '</td>' +
199
+ '</tr>';
200
+ }).join('');
201
+ }
202
+ var upd = document.getElementById('active-updated');
203
+ if (upd) upd.textContent = d.updatedAt ? 'Updated ' + new Date(d.updatedAt).toLocaleTimeString() : '';
204
+ }
205
+ function pollServers() {
206
+ fetch('/api/servers').then(function(r) { return r.json(); }).then(renderActiveRepos).catch(function() {}); // guardrails-allow PREVENT-PI-004: browser-side fetch in dashboard HTML template (not Node runtime)
207
+ }
208
+ pollServers();
209
+ setInterval(pollServers, 5000);
210
+
211
+ `;
212
+ }
@@ -0,0 +1,19 @@
1
+ /**
2
+ * dashboard-client.ts — thin concatenator for the dashboard's embedded client JS.
3
+ *
4
+ * The actual JS lives in three sibling files — dashboard-client-core.ts
5
+ * (snapshot + SSE), dashboard-client-repos.ts (multi-repo index), and
6
+ * dashboard-client-game.ts (game-mode + achievements + perf + tabs). Each
7
+ * exports its JS as a verbatim string; this file wraps them in the shared IIFE.
8
+ *
9
+ * Split from html.ts (PR0) — ZERO behavior change. The JS is one closure, so
10
+ * the chunks must concatenate in order: core → repos → game.
11
+ */
12
+ import { dashboardClientCoreJs } from "./dashboard-client-core.js";
13
+ import { dashboardClientReposJs } from "./dashboard-client-repos.js";
14
+ import { dashboardClientGameJs } from "./dashboard-client-game.js";
15
+ /** The full embedded client JS — an IIFE wrapping the three chunks in order. */
16
+ export function dashboardClientJs() {
17
+ return `(function() {
18
+ ${dashboardClientCoreJs()}${dashboardClientReposJs()}${dashboardClientGameJs()}`;
19
+ }