pi-mega-compact 0.4.4 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server.js +450 -0
- package/dist/extensions/dashboard-server.test.js +111 -0
- package/dist/extensions/error-patterns.js +115 -0
- package/dist/extensions/mega-compact.js +782 -0
- package/dist/extensions/mega-compact.test.js +328 -0
- package/dist/extensions/openclaw-mega-compact.js +291 -0
- package/dist/src/adapt.js +106 -0
- package/dist/src/boundary.js +88 -0
- package/dist/src/boundary.test.js +53 -0
- package/dist/src/canary.js +118 -0
- package/dist/src/compact.js +250 -0
- package/dist/src/compact.test.js +78 -0
- package/dist/src/config/dedup.js +81 -0
- package/dist/src/config.js +12 -0
- package/dist/src/dedup/dedup.test.js +41 -0
- package/dist/src/dedup/digest.js +30 -0
- package/dist/src/dedup/l1-lsh.js +52 -0
- package/dist/src/dedup/l1-minhash.js +91 -0
- package/dist/src/dedup/l1-verify.js +54 -0
- package/dist/src/dedup/l1.test.js +50 -0
- package/dist/src/dedup/mmr.js +45 -0
- package/dist/src/dedup/normalize.js +39 -0
- package/dist/src/dedup/raptor/guardrails.js +83 -0
- package/dist/src/dedup/raptor/index.js +94 -0
- package/dist/src/dedup/raptor/kmeans.js +152 -0
- package/dist/src/dedup/raptor/raptor.test.js +205 -0
- package/dist/src/dedup/raptor/retrieval.js +81 -0
- package/dist/src/dedup/raptor/summarizer.js +85 -0
- package/dist/src/dedup/raptor/tree.js +177 -0
- package/dist/src/dedup/sprint12.test.js +219 -0
- package/dist/src/dedup/topk.js +60 -0
- package/dist/src/dedup-engine.test.js +447 -0
- package/dist/src/e2e.test.js +698 -0
- package/dist/src/embedder.js +102 -0
- package/dist/src/engine.js +137 -0
- package/dist/src/engine.test.js +111 -0
- package/dist/src/extractive.js +209 -0
- package/dist/src/extractive.test.js +130 -0
- package/dist/src/httpEmbedder.js +143 -0
- package/dist/src/log.js +47 -0
- package/dist/src/log.test.js +42 -0
- package/dist/src/minilm.js +92 -0
- package/dist/src/monitoring.js +131 -0
- package/dist/src/ratio.bench.test.js +897 -0
- package/dist/src/recall.integration.test.js +77 -0
- package/dist/src/recall.js +60 -0
- package/dist/src/recall.test.js +50 -0
- package/dist/src/sprint14.test.js +219 -0
- package/dist/src/store/backfill.js +189 -0
- package/dist/src/store/bloom.js +114 -0
- package/dist/src/store/compression.js +177 -0
- package/dist/src/store/compression.test.js +67 -0
- package/dist/src/store/integrity.js +44 -0
- package/dist/src/store/migrate.js +79 -0
- package/dist/src/store/migrate.test.js +139 -0
- package/dist/src/store/sprint10.test.js +186 -0
- package/dist/src/store/sqlite.js +574 -0
- package/dist/src/store.js +115 -0
- package/dist/src/store.test.js +142 -0
- package/dist/src/supersede.js +68 -0
- package/dist/src/supersede.test.js +36 -0
- package/dist/src/tokens.js +31 -0
- package/dist/src/types.js +8 -0
- package/dist/src/types.test.js +9 -0
- package/dist/src/vectorStore.js +465 -0
- package/dist/src/vectorStore.test.js +479 -0
- package/dist/src/wordpiece.js +129 -0
- package/extensions/mega-compact.ts +9 -3
- package/package.json +4 -2
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-server.ts — lightweight local web dashboard for mega-compact.
|
|
3
|
+
*
|
|
4
|
+
* Zero npm dependencies. Uses only Node built-in modules (http, fs, path).
|
|
5
|
+
* Serves a single-page HTML dashboard, a JSON snapshot API, and an SSE
|
|
6
|
+
* endpoint that live-streams new events.log entries.
|
|
7
|
+
*
|
|
8
|
+
* Designed to be spawned as a detached child process from the pi extension
|
|
9
|
+
* and discovered by the /dashboard command via a port.pid file.
|
|
10
|
+
*
|
|
11
|
+
* @module
|
|
12
|
+
*/
|
|
13
|
+
import { createServer } from "node:http";
|
|
14
|
+
import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync } from "node:fs";
|
|
15
|
+
import { join } from "node:path";
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
// Helpers
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
function readSnapshot(snapshotPath) {
|
|
20
|
+
try {
|
|
21
|
+
const raw = readFileSync(snapshotPath, "utf-8");
|
|
22
|
+
return JSON.parse(raw);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return {
|
|
26
|
+
version: 1,
|
|
27
|
+
updatedAt: null,
|
|
28
|
+
tier: "unknown",
|
|
29
|
+
config: { fastGatePct: 80, thresholdTokens: 100_000, anchorUserMessages: 1, preserveRecent: 2, auto: true, autoInlineK: 3 },
|
|
30
|
+
session: { id: null, state: null, persistedThisSession: false, lastCheckpointId: null, lastCompactedFrom: 0 },
|
|
31
|
+
context: { tokens: null, percent: null, contextWindow: 0 },
|
|
32
|
+
trigger: { armed: false, ready: false, currentTokens: null, thresholdTokens: 100_000, fastGatePct: 80 },
|
|
33
|
+
store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
|
|
34
|
+
crew: { activeAgents: 0, currentTurn: 0 },
|
|
35
|
+
repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function readFrom(path, charOffset) {
|
|
40
|
+
try {
|
|
41
|
+
const content = readFileSync(path, "utf-8");
|
|
42
|
+
if (content.length <= charOffset)
|
|
43
|
+
return { data: "", offset: charOffset };
|
|
44
|
+
return { data: content.slice(charOffset), offset: content.length };
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return { data: "", offset: charOffset };
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
// ---------------------------------------------------------------------------
|
|
51
|
+
// HTML template
|
|
52
|
+
// ---------------------------------------------------------------------------
|
|
53
|
+
function dashboardHtml(tierName) {
|
|
54
|
+
return `<!DOCTYPE html>
|
|
55
|
+
<html lang="en">
|
|
56
|
+
<head>
|
|
57
|
+
<meta charset="utf-8">
|
|
58
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
59
|
+
<title>mega-compact dashboard</title>
|
|
60
|
+
<style>
|
|
61
|
+
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
62
|
+
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; background: #0d1117; color: #c9d1d9; padding: 24px; line-height: 1.5; }
|
|
63
|
+
h1 { font-size: 20px; font-weight: 600; margin-bottom: 20px; display: flex; align-items: center; gap: 10px; color: #f0f6fc; }
|
|
64
|
+
h1 .tier { background: #1f6feb; color: #fff; font-size: 11px; font-weight: 700; padding: 2px 8px; border-radius: 10px; text-transform: uppercase; letter-spacing: .5px; }
|
|
65
|
+
.grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin-bottom: 20px; }
|
|
66
|
+
.card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; }
|
|
67
|
+
.card h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .5px; color: #8b949e; margin-bottom: 12px; font-weight: 600; }
|
|
68
|
+
.meter-track { background: #21262d; border-radius: 4px; height: 20px; overflow: hidden; margin: 8px 0; }
|
|
69
|
+
.meter-fill { height: 100%; border-radius: 4px; transition: width .6s ease; min-width: 2px; }
|
|
70
|
+
.meter-green { background: #238636; }
|
|
71
|
+
.meter-yellow { background: #d29922; }
|
|
72
|
+
.meter-red { background: #f85149; }
|
|
73
|
+
.meter-label { font-size: 24px; font-weight: 700; color: #f0f6fc; }
|
|
74
|
+
.meter-sub { font-size: 12px; color: #8b949e; }
|
|
75
|
+
.status-row { display: flex; align-items: center; gap: 8px; margin: 6px 0; font-size: 14px; }
|
|
76
|
+
.status-row .bullet { width: 8px; height: 8px; border-radius: 50%; flex-shrink: 0; }
|
|
77
|
+
.bullet-on { background: #3fb950; box-shadow: 0 0 6px #3fb95088; }
|
|
78
|
+
.bullet-off { background: #484f58; }
|
|
79
|
+
.bullet-na { background: #d29922; }
|
|
80
|
+
.state-text { font-size: 13px; color: #8b949e; margin-top: 8px; font-family: monospace; }
|
|
81
|
+
.stat-grid { display: grid; grid-template-columns: auto 1fr; gap: 4px 12px; font-size: 14px; }
|
|
82
|
+
.stat-grid .label { color: #8b949e; }
|
|
83
|
+
.stat-grid .value { color: #f0f6fc; font-weight: 600; font-family: monospace; }
|
|
84
|
+
.conf-grid { display: grid; grid-template-columns: auto 1fr; gap: 4px 12px; font-size: 14px; }
|
|
85
|
+
.conf-grid .label { color: #8b949e; }
|
|
86
|
+
.conf-grid .value { color: #f0f6fc; font-family: monospace; }
|
|
87
|
+
.events { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; }
|
|
88
|
+
.events h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .5px; color: #8b949e; margin-bottom: 12px; font-weight: 600; }
|
|
89
|
+
.events-wrap { max-height: 240px; overflow-y: auto; font-family: monospace; font-size: 12px; }
|
|
90
|
+
.ev { padding: 3px 0; border-bottom: 1px solid #21262d; display: flex; gap: 8px; align-items: baseline; }
|
|
91
|
+
.ev:last-child { border-bottom: none; }
|
|
92
|
+
.ev-type { font-weight: 700; min-width: 70px; text-align: right; }
|
|
93
|
+
.ev-type-compact { color: #3fb950; }
|
|
94
|
+
.ev-type-recall { color: #a371f7; }
|
|
95
|
+
.ev-time { color: #484f58; font-size: 10px; min-width: 80px; }
|
|
96
|
+
.ev-detail { color: #8b949e; flex: 1; }
|
|
97
|
+
.updated { font-size: 11px; color: #484f58; margin-top: 16px; text-align: right; }
|
|
98
|
+
.empty { color: #484f58; font-style: italic; font-size: 13px; padding: 8px 0; }
|
|
99
|
+
.offline-banner { background: #f8514922; border: 1px solid #f85149; border-radius: 6px; padding: 10px 16px; margin-bottom: 16px; font-size: 13px; color: #f85149; display: none; }
|
|
100
|
+
</style>
|
|
101
|
+
</head>
|
|
102
|
+
<body>
|
|
103
|
+
|
|
104
|
+
<div class="offline-banner" id="offline-banner">Dashboard data unavailable — waiting for a pi session to write snapshot...</div>
|
|
105
|
+
|
|
106
|
+
<h1><span>mega-compact</span><span class="tier">${tierName}</span></h1>
|
|
107
|
+
|
|
108
|
+
<div class="grid">
|
|
109
|
+
<div class="card">
|
|
110
|
+
<h2>Context Window</h2>
|
|
111
|
+
<div class="meter-label" id="ctx-pct">—</div>
|
|
112
|
+
<div class="meter-track"><div class="meter-fill" id="ctx-bar" style="width:0%"></div></div>
|
|
113
|
+
<div class="meter-sub" id="ctx-sub">waiting for data</div>
|
|
114
|
+
</div>
|
|
115
|
+
<div class="card">
|
|
116
|
+
<h2>Trigger Status</h2>
|
|
117
|
+
<div class="status-row"><div class="bullet" id="tr-armed"></div><span>Armed (context ≥ fast gate)</span></div>
|
|
118
|
+
<div class="status-row"><div class="bullet" id="tr-ready"></div><span>Ready (tokens ≥ threshold)</span></div>
|
|
119
|
+
<div class="state-text" id="tr-state">waiting</div>
|
|
120
|
+
</div>
|
|
121
|
+
<div class="card">
|
|
122
|
+
<h2>Vector Store</h2>
|
|
123
|
+
<div class="stat-grid">
|
|
124
|
+
<span class="label">Checkpoints</span><span class="value" id="st-count">0</span>
|
|
125
|
+
<span class="label">Tokens Stored</span><span class="value" id="st-tokens">0</span>
|
|
126
|
+
<span class="label">Original Tokens</span><span class="value" id="st-orig">0</span>
|
|
127
|
+
<span class="label">Tokens Saved</span><span class="value" id="st-saved">0</span>
|
|
128
|
+
<span class="label">Injected</span><span class="value" id="st-injected">0</span>
|
|
129
|
+
<span class="label">Dedup Rate</span><span class="value" id="st-dedup">0%</span>
|
|
130
|
+
<span class="label">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
|
|
131
|
+
<span class="label">Collapsed</span><span class="value" id="st-collapsed">0</span>
|
|
132
|
+
<span class="label">Last ID</span><span class="value" id="st-lastid">—</span>
|
|
133
|
+
</div>
|
|
134
|
+
</div>
|
|
135
|
+
<div class="card">
|
|
136
|
+
<h2>Repo (all sessions)</h2>
|
|
137
|
+
<div class="stat-grid">
|
|
138
|
+
<span class="label">Checkpoints</span><span class="value" id="rp-count">0</span>
|
|
139
|
+
<span class="label">Tokens Stored</span><span class="value" id="rp-tokens">0</span>
|
|
140
|
+
<span class="label">Original Tokens</span><span class="value" id="rp-orig">0</span>
|
|
141
|
+
<span class="label">Tokens Saved</span><span class="value" id="rp-saved">0</span>
|
|
142
|
+
<span class="label">Sessions</span><span class="value" id="rp-sessions">0</span>
|
|
143
|
+
<span class="label">Collapsed</span><span class="value" id="rp-collapsed">0</span>
|
|
144
|
+
<span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
|
|
145
|
+
</div>
|
|
146
|
+
</div>
|
|
147
|
+
<div class="card">
|
|
148
|
+
<h2>Configuration</h2>
|
|
149
|
+
<div class="conf-grid">
|
|
150
|
+
<span class="label">Tier</span><span class="value" id="cf-tier">${tierName}</span>
|
|
151
|
+
<span class="label">Threshold</span><span class="value" id="cf-threshold">—</span>
|
|
152
|
+
<span class="label">Fast Gate</span><span class="value" id="cf-gate">—</span>
|
|
153
|
+
<span class="label">Auto</span><span class="value" id="cf-auto">—</span>
|
|
154
|
+
<span class="label">Anchor</span><span class="value" id="cf-anchor">—</span>
|
|
155
|
+
</div>
|
|
156
|
+
</div>
|
|
157
|
+
<div class="card">
|
|
158
|
+
<h2>Crew / Agents</h2>
|
|
159
|
+
<div class="stat-grid">
|
|
160
|
+
<span class="label">Active Agents</span><span class="value" id="cr-agents">0</span>
|
|
161
|
+
<span class="label">Current Turn</span><span class="value" id="cr-turn">0</span>
|
|
162
|
+
<span class="label">Status</span><span class="value" id="cr-status">idle</span>
|
|
163
|
+
</div>
|
|
164
|
+
</div>
|
|
165
|
+
</div>
|
|
166
|
+
|
|
167
|
+
<div class="events">
|
|
168
|
+
<h2>Event Stream</h2>
|
|
169
|
+
<div class="events-wrap" id="events"><div class="empty">connecting…</div></div>
|
|
170
|
+
</div>
|
|
171
|
+
|
|
172
|
+
<div class="updated" id="updated"></div>
|
|
173
|
+
|
|
174
|
+
<script>
|
|
175
|
+
(function() {
|
|
176
|
+
var evBox = document.getElementById('events');
|
|
177
|
+
var evBuffer = [];
|
|
178
|
+
var MAX_EV = 50;
|
|
179
|
+
var offlineBanner = document.getElementById('offline-banner');
|
|
180
|
+
|
|
181
|
+
function bullet(el, on, na) {
|
|
182
|
+
el.className = 'bullet ' + (na ? 'bullet-na' : on ? 'bullet-on' : 'bullet-off');
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function renderSnapshot(d) {
|
|
186
|
+
if (!d || !d.updatedAt) { offlineBanner.style.display = 'block'; return; }
|
|
187
|
+
offlineBanner.style.display = 'none';
|
|
188
|
+
|
|
189
|
+
var pct = d.context.percent || 0;
|
|
190
|
+
document.getElementById('ctx-pct').textContent = pct + '%';
|
|
191
|
+
var bar = document.getElementById('ctx-bar');
|
|
192
|
+
bar.style.width = Math.max(pct, 1) + '%';
|
|
193
|
+
bar.className = 'meter-fill ' + (pct >= 90 ? 'meter-red' : pct >= 70 ? 'meter-yellow' : 'meter-green');
|
|
194
|
+
var tok = d.context.tokens != null ? d.context.tokens.toLocaleString() : '?';
|
|
195
|
+
var win = d.context.contextWindow ? d.context.contextWindow.toLocaleString() : '?';
|
|
196
|
+
document.getElementById('ctx-sub').textContent = tok + ' / ' + win + ' tokens';
|
|
197
|
+
|
|
198
|
+
bullet(document.getElementById('tr-armed'), d.trigger.armed, false);
|
|
199
|
+
bullet(document.getElementById('tr-ready'), d.trigger.ready, !d.trigger.armed);
|
|
200
|
+
var state = d.trigger.ready ? 'THRESHOLD EXCEEDED — compacting next event' :
|
|
201
|
+
d.trigger.armed ? 'past fast gate — monitoring token count' : 'idle — below fast gate';
|
|
202
|
+
document.getElementById('tr-state').textContent = state;
|
|
203
|
+
|
|
204
|
+
document.getElementById('st-count').textContent = d.store.checkpointCount;
|
|
205
|
+
document.getElementById('st-tokens').textContent = d.store.totalTokenEstimate.toLocaleString();
|
|
206
|
+
document.getElementById('st-orig').textContent = (d.store.originalTokens || 0).toLocaleString();
|
|
207
|
+
document.getElementById('st-saved').textContent = (d.store.tokensSaved || 0).toLocaleString();
|
|
208
|
+
document.getElementById('st-injected').textContent = d.store.injectedCount;
|
|
209
|
+
document.getElementById('st-dedup').textContent = Math.round(d.store.dedupHitRate * 100) + '%';
|
|
210
|
+
var sdr = d.store.storageDedupRate || 0;
|
|
211
|
+
document.getElementById('st-sdedup').textContent = (sdr * 100 >= 10 ? Math.round(sdr * 100) : (sdr * 100).toFixed(1)) + '%';
|
|
212
|
+
document.getElementById('st-collapsed').textContent = d.store.dedupCollapsed || 0;
|
|
213
|
+
document.getElementById('st-lastid').textContent = d.session.lastCheckpointId || '—';
|
|
214
|
+
|
|
215
|
+
// Repo-wide (all sessions in this repo's SQLite store).
|
|
216
|
+
var repo = d.repo || { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupCollapsed: 0, storageDedupRate: 0 };
|
|
217
|
+
document.getElementById('rp-count').textContent = repo.checkpointCount;
|
|
218
|
+
document.getElementById('rp-tokens').textContent = repo.totalTokenEstimate.toLocaleString();
|
|
219
|
+
document.getElementById('rp-orig').textContent = (repo.originalTokens || 0).toLocaleString();
|
|
220
|
+
document.getElementById('rp-saved').textContent = (repo.tokensSaved || 0).toLocaleString();
|
|
221
|
+
document.getElementById('rp-sessions').textContent = repo.sessionCount || 0;
|
|
222
|
+
document.getElementById('rp-collapsed').textContent = repo.dedupCollapsed || 0;
|
|
223
|
+
var rsdr = repo.storageDedupRate || 0;
|
|
224
|
+
document.getElementById('rp-sdedup').textContent = (rsdr * 100 >= 10 ? Math.round(rsdr * 100) : (rsdr * 100).toFixed(1)) + '%';
|
|
225
|
+
|
|
226
|
+
// Crew / agents (live sub-agent activity + turn).
|
|
227
|
+
var crew = d.crew || { activeAgents: 0, currentTurn: 0 };
|
|
228
|
+
document.getElementById('cr-agents').textContent = crew.activeAgents || 0;
|
|
229
|
+
document.getElementById('cr-turn').textContent = crew.currentTurn || 0;
|
|
230
|
+
document.getElementById('cr-status').textContent = (crew.activeAgents > 0)
|
|
231
|
+
? ('▶ ' + crew.activeAgents + ' running') : 'idle';
|
|
232
|
+
|
|
233
|
+
document.getElementById('cf-tier').textContent = d.tier;
|
|
234
|
+
document.getElementById('cf-threshold').textContent = d.config.thresholdTokens.toLocaleString();
|
|
235
|
+
document.getElementById('cf-gate').textContent = d.config.fastGatePct + '%';
|
|
236
|
+
document.getElementById('cf-auto').textContent = d.config.auto ? 'enabled' : 'disabled';
|
|
237
|
+
document.getElementById('cf-anchor').textContent = d.config.anchorUserMessages;
|
|
238
|
+
|
|
239
|
+
document.getElementById('updated').textContent = 'Updated ' + new Date(d.updatedAt).toLocaleTimeString();
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function sanitize(s) {
|
|
243
|
+
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"');
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function renderEvent(ev) {
|
|
247
|
+
evBuffer.unshift(ev);
|
|
248
|
+
if (evBuffer.length > MAX_EV) evBuffer.length = MAX_EV;
|
|
249
|
+
evBox.innerHTML = evBuffer.map(function(e) {
|
|
250
|
+
var t = e.ts ? new Date(e.ts).toLocaleTimeString() : '';
|
|
251
|
+
var detail = '';
|
|
252
|
+
if (e.data) {
|
|
253
|
+
if (e.data.checkpointId) detail = sanitize(e.data.checkpointId);
|
|
254
|
+
else if (e.data.query) detail = sanitize(e.data.query.slice(0, 80));
|
|
255
|
+
if (e.data.tokenEstimate != null) detail += ' ' + e.data.tokenEstimate + ' tok';
|
|
256
|
+
if (e.data.deduped) detail += ' (deduped)';
|
|
257
|
+
if (e.data.injected != null) detail = 'injected: ' + e.data.injected + (e.data.empty ? ' (empty)' : '');
|
|
258
|
+
}
|
|
259
|
+
return '<div class="ev">' +
|
|
260
|
+
'<span class="ev-time">' + t + '</span>' +
|
|
261
|
+
'<span class="ev-type ev-type-' + sanitize(e.type) + '">' + sanitize(e.type) + '</span>' +
|
|
262
|
+
'<span class="ev-detail">' + detail + '</span></div>';
|
|
263
|
+
}).join('');
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// Poll snapshot every 2s
|
|
267
|
+
function pollSnapshot() {
|
|
268
|
+
fetch('/api/snapshot').then(function(r) { return r.json(); }).then(renderSnapshot).catch(function() {});
|
|
269
|
+
}
|
|
270
|
+
pollSnapshot();
|
|
271
|
+
setInterval(pollSnapshot, 2000);
|
|
272
|
+
|
|
273
|
+
// SSE for events
|
|
274
|
+
function connectSSE() {
|
|
275
|
+
var es = new EventSource('/api/events');
|
|
276
|
+
es.onmessage = function(msg) {
|
|
277
|
+
try { renderEvent(JSON.parse(msg.data)); } catch(e) {}
|
|
278
|
+
};
|
|
279
|
+
es.onerror = function() {
|
|
280
|
+
es.close();
|
|
281
|
+
setTimeout(connectSSE, 3000);
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
connectSSE();
|
|
285
|
+
})();
|
|
286
|
+
</script>
|
|
287
|
+
</body>
|
|
288
|
+
</html>`;
|
|
289
|
+
}
|
|
290
|
+
// ---------------------------------------------------------------------------
|
|
291
|
+
// Server
|
|
292
|
+
// ---------------------------------------------------------------------------
|
|
293
|
+
export function launchDashboardServer(stateDir) {
|
|
294
|
+
const portFile = join(stateDir, "port.pid");
|
|
295
|
+
const snapshotPath = join(stateDir, "dashboard.json");
|
|
296
|
+
const eventsPath = join(stateDir, "events.log");
|
|
297
|
+
// ── Existing server? ──────────────────────────────────────────────────────
|
|
298
|
+
if (existsSync(portFile)) {
|
|
299
|
+
try {
|
|
300
|
+
const info = JSON.parse(readFileSync(portFile, "utf-8"));
|
|
301
|
+
if (info && info.port) {
|
|
302
|
+
return Promise.resolve({ port: info.port, url: `http://localhost:${info.port}` });
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
catch {
|
|
306
|
+
// stale file, overwrite
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
// ── New server ────────────────────────────────────────────────────────────
|
|
310
|
+
mkdirSync(stateDir, { recursive: true });
|
|
311
|
+
let eventOffset = 0;
|
|
312
|
+
const server = createServer((req, res) => {
|
|
313
|
+
// CORS for local access
|
|
314
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
315
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, OPTIONS");
|
|
316
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type");
|
|
317
|
+
if (req.method === "OPTIONS") {
|
|
318
|
+
res.writeHead(204);
|
|
319
|
+
res.end();
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
if (req.url === "/" || req.url === "/index.html") {
|
|
323
|
+
const tier = readSnapshot(snapshotPath).tier;
|
|
324
|
+
res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
|
|
325
|
+
res.end(dashboardHtml(tier));
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
if (req.url === "/api/snapshot") {
|
|
329
|
+
const snap = readSnapshot(snapshotPath);
|
|
330
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
331
|
+
res.end(JSON.stringify(snap));
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
if (req.url === "/api/events") {
|
|
335
|
+
res.writeHead(200, {
|
|
336
|
+
"Content-Type": "text/event-stream",
|
|
337
|
+
"Cache-Control": "no-cache",
|
|
338
|
+
"Connection": "keep-alive",
|
|
339
|
+
});
|
|
340
|
+
// Drain existing events so the client starts with history
|
|
341
|
+
const { data: existing, offset: initialOffset } = readFrom(eventsPath, 0);
|
|
342
|
+
eventOffset = initialOffset;
|
|
343
|
+
const lines = existing.split("\n").filter((l) => l.trim());
|
|
344
|
+
for (const line of lines) {
|
|
345
|
+
res.write(`data: ${line}\n\n`);
|
|
346
|
+
}
|
|
347
|
+
// Tail new events via fs.watch (coalesced with 100ms debounce)
|
|
348
|
+
let watchTimer = null;
|
|
349
|
+
const onWatch = () => {
|
|
350
|
+
if (watchTimer)
|
|
351
|
+
return;
|
|
352
|
+
watchTimer = setTimeout(() => {
|
|
353
|
+
watchTimer = null;
|
|
354
|
+
const { data, offset } = readFrom(eventsPath, eventOffset);
|
|
355
|
+
eventOffset = offset;
|
|
356
|
+
const newLines = data.split("\n").filter((l) => l.trim());
|
|
357
|
+
for (const line of newLines) {
|
|
358
|
+
res.write(`data: ${line}\n\n`);
|
|
359
|
+
}
|
|
360
|
+
}, 100);
|
|
361
|
+
};
|
|
362
|
+
// Set up file watching: if file exists, watch it directly;
|
|
363
|
+
// otherwise poll for creation every 1s then switch to fs.watch.
|
|
364
|
+
let watcher = null;
|
|
365
|
+
let pollInterval = null;
|
|
366
|
+
function startFileWatch() {
|
|
367
|
+
try {
|
|
368
|
+
watcher = watch(eventsPath, onWatch);
|
|
369
|
+
}
|
|
370
|
+
catch { /* give up */ }
|
|
371
|
+
}
|
|
372
|
+
if (existsSync(eventsPath)) {
|
|
373
|
+
startFileWatch();
|
|
374
|
+
}
|
|
375
|
+
else {
|
|
376
|
+
pollInterval = setInterval(() => {
|
|
377
|
+
if (existsSync(eventsPath)) {
|
|
378
|
+
if (pollInterval) {
|
|
379
|
+
clearInterval(pollInterval);
|
|
380
|
+
pollInterval = null;
|
|
381
|
+
}
|
|
382
|
+
startFileWatch();
|
|
383
|
+
}
|
|
384
|
+
}, 1000);
|
|
385
|
+
}
|
|
386
|
+
req.on("close", () => {
|
|
387
|
+
if (watchTimer)
|
|
388
|
+
clearTimeout(watchTimer);
|
|
389
|
+
if (pollInterval)
|
|
390
|
+
clearInterval(pollInterval);
|
|
391
|
+
watcher?.close();
|
|
392
|
+
});
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
// Fallback — serve the dashboard
|
|
396
|
+
const tier = readSnapshot(snapshotPath).tier;
|
|
397
|
+
res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
|
|
398
|
+
res.end(dashboardHtml(tier));
|
|
399
|
+
});
|
|
400
|
+
const TARGET_PORT = 9320;
|
|
401
|
+
const PORT_RANGE = 10; // 9320–9329
|
|
402
|
+
return new Promise((resolve, reject) => {
|
|
403
|
+
function tryPort(port) {
|
|
404
|
+
server.once("error", (err) => {
|
|
405
|
+
if (err.code === "EADDRINUSE" && port < TARGET_PORT + PORT_RANGE - 1) {
|
|
406
|
+
tryPort(port + 1);
|
|
407
|
+
}
|
|
408
|
+
else {
|
|
409
|
+
reject(err);
|
|
410
|
+
}
|
|
411
|
+
});
|
|
412
|
+
server.listen(port, "127.0.0.1", () => {
|
|
413
|
+
const url = `http://localhost:${port}`;
|
|
414
|
+
console.log(`[mega-compact] dashboard server running: ${url}`);
|
|
415
|
+
// Write port.pid
|
|
416
|
+
try {
|
|
417
|
+
writeFileSync(portFile, JSON.stringify({ port, pid: process.pid }));
|
|
418
|
+
}
|
|
419
|
+
catch { /* non-fatal */ }
|
|
420
|
+
// Graceful cleanup
|
|
421
|
+
const cleanup = () => {
|
|
422
|
+
try {
|
|
423
|
+
unlinkSync(portFile);
|
|
424
|
+
}
|
|
425
|
+
catch { /* already gone */ }
|
|
426
|
+
server.close();
|
|
427
|
+
process.exit(0);
|
|
428
|
+
};
|
|
429
|
+
process.on("SIGTERM", cleanup);
|
|
430
|
+
process.on("SIGINT", cleanup);
|
|
431
|
+
resolve({ port, url });
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
tryPort(TARGET_PORT);
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
// ---------------------------------------------------------------------------
|
|
438
|
+
// CLI entry point — when run directly as `node dashboard-server.js <stateDir>`
|
|
439
|
+
// ---------------------------------------------------------------------------
|
|
440
|
+
if (process.argv[1] && process.argv[1].includes("dashboard-server")) {
|
|
441
|
+
const stateDir = process.argv[2];
|
|
442
|
+
if (!stateDir) {
|
|
443
|
+
console.error("Usage: node dashboard-server.js <stateDir>");
|
|
444
|
+
process.exit(1);
|
|
445
|
+
}
|
|
446
|
+
launchDashboardServer(stateDir).catch((err) => {
|
|
447
|
+
console.error("[mega-compact] dashboard server failed:", err);
|
|
448
|
+
process.exit(1);
|
|
449
|
+
});
|
|
450
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-server.test.ts — unit tests for the standalone HTTP dashboard server.
|
|
3
|
+
*
|
|
4
|
+
* Tests the core logic (snapshot building, API responses) without requiring
|
|
5
|
+
* a live pi session or model.
|
|
6
|
+
*/
|
|
7
|
+
import { test, describe } from "node:test";
|
|
8
|
+
import assert from "node:assert/strict";
|
|
9
|
+
import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync } from "node:fs";
|
|
10
|
+
import { tmpdir } from "node:os";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
// helpers
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
function tmpDir() {
|
|
16
|
+
return mkdtempSync(join(tmpdir(), "dashboard-test-"));
|
|
17
|
+
}
|
|
18
|
+
function writeSnapshot(dir, data) {
|
|
19
|
+
writeFileSync(join(dir, "snapshot.json"), JSON.stringify(data, null, 2));
|
|
20
|
+
}
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// Tests
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
describe("snapshot.json reading", () => {
|
|
25
|
+
test("returns valid snapshot when file exists", () => {
|
|
26
|
+
const dir = tmpDir();
|
|
27
|
+
const snapshot = {
|
|
28
|
+
updatedAt: new Date().toISOString(),
|
|
29
|
+
tier: "medium",
|
|
30
|
+
version: 1,
|
|
31
|
+
config: { activeTier: "medium" },
|
|
32
|
+
session: { id: "test-123", state: "running", persistedThisSession: true },
|
|
33
|
+
context: { tokens: 50000, percent: 50, contextWindow: 100000 },
|
|
34
|
+
trigger: { armed: true, ready: false, currentTokens: 50000, thresholdTokens: 100000 },
|
|
35
|
+
store: { checkpointCount: 3, totalTokenEstimate: 15000 },
|
|
36
|
+
};
|
|
37
|
+
writeSnapshot(dir, snapshot);
|
|
38
|
+
const content = readFileSync(join(dir, "snapshot.json"), "utf-8");
|
|
39
|
+
const parsed = JSON.parse(content);
|
|
40
|
+
assert.equal(parsed.tier, "medium");
|
|
41
|
+
assert.equal(parsed.session.id, "test-123");
|
|
42
|
+
assert.equal(parsed.context.percent, 50);
|
|
43
|
+
assert.equal(parsed.store.checkpointCount, 3);
|
|
44
|
+
rmSync(dir, { recursive: true });
|
|
45
|
+
});
|
|
46
|
+
test("handles missing snapshot.json gracefully", () => {
|
|
47
|
+
const dir = tmpDir();
|
|
48
|
+
const snapshotPath = join(dir, "snapshot.json");
|
|
49
|
+
assert.equal(existsSync(snapshotPath), false);
|
|
50
|
+
rmSync(dir, { recursive: true });
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
describe("dashboard.json (compact snapshot)", () => {
|
|
54
|
+
test("round-trips correctly", () => {
|
|
55
|
+
const dir = tmpDir();
|
|
56
|
+
const data = {
|
|
57
|
+
updatedAt: "2025-01-01T00:00:00.000Z",
|
|
58
|
+
tier: "mega",
|
|
59
|
+
version: 1,
|
|
60
|
+
config: { activeTier: "mega", thresholdTokens: 10000000 },
|
|
61
|
+
session: { id: "abc", state: "running", persistedThisSession: false },
|
|
62
|
+
context: { tokens: null, percent: null, contextWindow: 200000 },
|
|
63
|
+
trigger: { armed: false, ready: false, currentTokens: null, thresholdTokens: 10000000 },
|
|
64
|
+
store: { checkpointCount: 0, totalTokenEstimate: 0 },
|
|
65
|
+
};
|
|
66
|
+
writeFileSync(join(dir, "dashboard.json"), JSON.stringify(data));
|
|
67
|
+
const content = readFileSync(join(dir, "dashboard.json"), "utf-8");
|
|
68
|
+
const parsed = JSON.parse(content);
|
|
69
|
+
assert.equal(parsed.tier, "mega");
|
|
70
|
+
assert.equal(parsed.config.thresholdTokens, 10000000);
|
|
71
|
+
assert.equal(parsed.context.tokens, null);
|
|
72
|
+
rmSync(dir, { recursive: true });
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
describe("events.log (JSONL)", () => {
|
|
76
|
+
test("parses multiple JSONL lines", () => {
|
|
77
|
+
const dir = tmpDir();
|
|
78
|
+
const events = [
|
|
79
|
+
{ ts: "2025-01-01T00:00:00.000Z", type: "compact_start", trigger: "auto", tier: "medium", sessionId: "s1" },
|
|
80
|
+
{ ts: "2025-01-01T00:00:01.000Z", type: "compact_end", trigger: "auto", durationMs: 1500, mode: "mega", fromTokens: 100000, toTokens: 5000 },
|
|
81
|
+
{ ts: "2025-01-01T00:00:02.000Z", type: "checkpoint_persisted", checkpointId: "chk_1", totalCheckpoints: 1 },
|
|
82
|
+
];
|
|
83
|
+
writeFileSync(join(dir, "events.log"), events.map((e) => JSON.stringify(e)).join("\n") + "\n");
|
|
84
|
+
const lines = readFileSync(join(dir, "events.log"), "utf-8").trim().split("\n");
|
|
85
|
+
assert.equal(lines.length, 3);
|
|
86
|
+
const parsed = lines.map((l) => JSON.parse(l));
|
|
87
|
+
assert.equal(parsed[0].type, "compact_start");
|
|
88
|
+
assert.equal(parsed[1].durationMs, 1500);
|
|
89
|
+
assert.equal(parsed[2].checkpointId, "chk_1");
|
|
90
|
+
rmSync(dir, { recursive: true });
|
|
91
|
+
});
|
|
92
|
+
test("ignores empty lines", () => {
|
|
93
|
+
const dir = tmpDir();
|
|
94
|
+
writeFileSync(join(dir, "events.log"), '{"type":"test"}\n\n\n{"type":"test2"}\n');
|
|
95
|
+
const lines = readFileSync(join(dir, "events.log"), "utf-8").trim().split("\n").filter(Boolean);
|
|
96
|
+
assert.equal(lines.length, 2);
|
|
97
|
+
rmSync(dir, { recursive: true });
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
describe("port.pid file", () => {
|
|
101
|
+
test("round-trips port and pid", () => {
|
|
102
|
+
const dir = tmpDir();
|
|
103
|
+
const info = { port: 3847, pid: 12345 };
|
|
104
|
+
writeFileSync(join(dir, "port.pid"), JSON.stringify(info));
|
|
105
|
+
const content = readFileSync(join(dir, "port.pid"), "utf-8");
|
|
106
|
+
const parsed = JSON.parse(content);
|
|
107
|
+
assert.equal(parsed.port, 3847);
|
|
108
|
+
assert.equal(parsed.pid, 12345);
|
|
109
|
+
rmSync(dir, { recursive: true });
|
|
110
|
+
});
|
|
111
|
+
});
|