pi-mega-compact 0.7.8 → 0.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -12
- package/dist/extensions/dashboard-server/helpers.js +37 -0
- package/dist/extensions/dashboard-server/html/all-repos-tab.js +26 -0
- package/dist/extensions/dashboard-server/html/body-open.js +23 -0
- package/dist/extensions/dashboard-server/html/current-repo-tab.js +130 -0
- package/dist/extensions/dashboard-server/html/head-open.js +16 -0
- package/dist/extensions/dashboard-server/html/high-score-tab.js +25 -0
- package/dist/extensions/dashboard-server/html/repo-detail-modal.js +26 -0
- package/dist/extensions/dashboard-server/html/script.js +259 -0
- package/dist/extensions/dashboard-server/html/styles.js +103 -0
- package/dist/extensions/dashboard-server/html/summary-tab.js +19 -0
- package/dist/extensions/dashboard-server/html-template.js +41 -0
- package/dist/extensions/dashboard-server/html.js +756 -0
- package/dist/extensions/dashboard-server/index-reader.js +133 -0
- package/dist/extensions/dashboard-server/server.js +370 -0
- package/dist/extensions/dashboard-server/snapshot.js +43 -0
- package/dist/extensions/dashboard-server/state.js +30 -0
- package/dist/extensions/dashboard-server/types.js +5 -0
- package/dist/extensions/dashboard-server.js +7 -1315
- package/dist/extensions/mega-commands.js +162 -134
- package/dist/extensions/mega-compact.test.js +90 -21
- package/dist/extensions/mega-conflict-cmds.js +5 -1
- package/dist/extensions/mega-dashboard-cmds.js +29 -22
- package/dist/extensions/mega-db-cmds.js +11 -2
- package/dist/extensions/mega-events/agent-handlers.js +173 -0
- package/dist/extensions/mega-events/compact-handlers.js +133 -0
- package/dist/extensions/mega-events/context-handler.js +249 -0
- package/dist/extensions/mega-events/register.js +21 -0
- package/dist/extensions/mega-events/session-handlers.js +142 -0
- package/dist/extensions/mega-events.js +15 -699
- package/dist/extensions/mega-pipeline/compact.js +324 -0
- package/dist/extensions/mega-pipeline/memory-review.js +38 -0
- package/dist/extensions/mega-pipeline/recall.js +147 -0
- package/dist/extensions/mega-pipeline.js +9 -480
- package/dist/extensions/mega-runtime/helpers.js +40 -0
- package/dist/extensions/mega-runtime/query.js +29 -0
- package/dist/extensions/mega-runtime/state.js +711 -0
- package/dist/extensions/mega-runtime/widget.js +197 -0
- package/dist/extensions/mega-runtime.js +15 -947
- package/dist/src/store/sqlite/checkpoints.js +145 -0
- package/dist/src/store/sqlite/connection.js +35 -0
- package/dist/src/store/sqlite/dedup-mirror.js +64 -0
- package/dist/src/store/sqlite/foundation.js +38 -0
- package/dist/src/store/sqlite/global-index.js +224 -0
- package/dist/src/store/sqlite/index-store.js +167 -0
- package/dist/src/store/sqlite/maintenance.js +235 -0
- package/dist/src/store/sqlite/memories.js +164 -0
- package/dist/src/store/sqlite/memory.js +54 -0
- package/dist/src/store/sqlite/meta.js +82 -0
- package/dist/src/store/sqlite/minhash-lsh.js +47 -0
- package/dist/src/store/sqlite/model-snapshots.js +47 -0
- package/dist/src/store/sqlite/raptor.js +57 -0
- package/dist/src/store/sqlite/raw-transcript.js +134 -0
- package/dist/src/store/sqlite/schema.js +250 -0
- package/dist/src/store/sqlite/session-state.js +28 -0
- package/dist/src/store/sqlite/sessions.js +39 -0
- package/dist/src/store/sqlite/stats.js +66 -0
- package/dist/src/store/sqlite/transaction.js +19 -0
- package/dist/src/store/sqlite/utils.js +120 -0
- package/dist/src/store/sqlite.js +20 -1607
- package/dist/src/vectorStore/add.js +260 -0
- package/dist/src/vectorStore/dedup.js +52 -0
- package/dist/src/vectorStore/index.js +10 -0
- package/dist/src/vectorStore/queries.js +83 -0
- package/dist/src/vectorStore/search.js +95 -0
- package/dist/src/vectorStore/session.js +19 -0
- package/dist/src/vectorStore/store.js +105 -0
- package/dist/src/vectorStore/types.js +6 -0
- package/dist/src/vectorStore/utils.js +23 -0
- package/extensions/dashboard-server/html.ts +758 -0
- package/extensions/dashboard-server/index-reader.ts +130 -0
- package/extensions/dashboard-server/server.ts +358 -0
- package/extensions/dashboard-server/snapshot.ts +44 -0
- package/extensions/dashboard-server/state.ts +33 -0
- package/extensions/dashboard-server/types.ts +134 -0
- package/extensions/dashboard-server.ts +7 -1431
- package/extensions/mega-commands.ts +33 -10
- package/extensions/mega-compact.test.ts +198 -43
- package/extensions/mega-conflict-cmds.ts +6 -2
- package/extensions/mega-dashboard-cmds.ts +30 -23
- package/extensions/mega-db-cmds.ts +11 -3
- package/extensions/mega-events/agent-handlers.ts +214 -0
- package/extensions/mega-events/compact-handlers.ts +164 -0
- package/extensions/mega-events/context-handler.ts +290 -0
- package/extensions/mega-events/register.ts +37 -0
- package/extensions/mega-events/session-handlers.ts +165 -0
- package/extensions/mega-events.ts +15 -780
- package/extensions/mega-pipeline/compact.ts +366 -0
- package/extensions/mega-pipeline/memory-review.ts +46 -0
- package/extensions/mega-pipeline/recall.ts +165 -0
- package/extensions/mega-pipeline.ts +9 -537
- package/extensions/mega-runtime/helpers.ts +68 -0
- package/extensions/mega-runtime/query.ts +29 -0
- package/extensions/mega-runtime/state.ts +797 -0
- package/extensions/mega-runtime/widget.ts +258 -0
- package/extensions/mega-runtime.ts +15 -1093
- package/package.json +4 -3
- package/src/store/sqlite/checkpoints.ts +204 -0
- package/src/store/sqlite/dedup-mirror.ts +114 -0
- package/src/store/sqlite/foundation.ts +63 -0
- package/src/store/sqlite/global-index.ts +305 -0
- package/src/store/sqlite/maintenance.ts +294 -0
- package/src/store/sqlite/memories.ts +217 -0
- package/src/store/sqlite/meta.ts +108 -0
- package/src/store/sqlite/model-snapshots.ts +83 -0
- package/src/store/sqlite/raptor.ts +107 -0
- package/src/store/sqlite/raw-transcript.ts +221 -0
- package/src/store/sqlite/schema.ts +258 -0
- package/src/store/sqlite/session-state.ts +38 -0
- package/src/store/sqlite/stats.ts +127 -0
- package/src/store/sqlite/utils.ts +125 -0
- package/src/store/sqlite.ts +20 -2204
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-server/index-reader.ts — machine-wide repo registry reader.
|
|
3
|
+
*
|
|
4
|
+
* The extension writes a machine-wide repo registry into a single SQLite DB
|
|
5
|
+
* (<indexDir>/index.sqlite) as the concurrency-safe write path; the dashboard
|
|
6
|
+
* reads that table directly (one read-only connection, opened per request so a
|
|
7
|
+
* concurrent writer's WAL never blocks the request). All registry data lives in
|
|
8
|
+
* SQLite (the project's one-store invariant) — there is no JSON mirror. Same
|
|
9
|
+
* index-dir resolution as src/store/sqlite.ts getIndexDir().
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync } from "node:fs";
|
|
12
|
+
import { homedir } from "node:os";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { DatabaseSync } from "node:sqlite";
|
|
15
|
+
export function getIndexDir() {
|
|
16
|
+
const override = process.env.MEGACOMPACT_INDEX_DIR;
|
|
17
|
+
if (override && override.trim() !== "")
|
|
18
|
+
return override;
|
|
19
|
+
try {
|
|
20
|
+
return join(homedir(), ".mega-compact-index");
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return join("/tmp", ".mega-compact-index");
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/** Read the machine-wide repo registry from SQLite (read-only, single shot). */
|
|
27
|
+
export function readIndex() {
|
|
28
|
+
const indexPath = join(getIndexDir(), "index.sqlite");
|
|
29
|
+
if (!existsSync(indexPath))
|
|
30
|
+
return null;
|
|
31
|
+
let db;
|
|
32
|
+
try {
|
|
33
|
+
// Read-only + immutable WAL so a concurrent writer's WAL never blocks us.
|
|
34
|
+
db = new DatabaseSync(indexPath, { readOnly: true });
|
|
35
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
36
|
+
const rows = db
|
|
37
|
+
.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC")
|
|
38
|
+
.all();
|
|
39
|
+
const mapped = rows.map((r) => ({
|
|
40
|
+
repoRoot: String(r.repo_root ?? ""),
|
|
41
|
+
displayName: String(r.display_name ?? ""),
|
|
42
|
+
stateDir: String(r.state_dir ?? ""),
|
|
43
|
+
checkpointCount: Number(r.checkpoint_count ?? 0),
|
|
44
|
+
tokensSaved: Number(r.tokens_saved ?? 0),
|
|
45
|
+
compressedOriginalBytes: Number(r.compressed_original_bytes ?? 0),
|
|
46
|
+
lastCompactedAt: r.last_compacted_at ?? null,
|
|
47
|
+
provider: r.provider ?? null,
|
|
48
|
+
providerName: r.provider_name ?? null,
|
|
49
|
+
modelName: r.model_name ?? null,
|
|
50
|
+
inputRate: r.input_rate ?? null,
|
|
51
|
+
outputRate: r.output_rate ?? null,
|
|
52
|
+
lastSeen: Number(r.last_seen ?? 0),
|
|
53
|
+
// Defaults — enriched below from each repo's own store.
|
|
54
|
+
tokensKept: 0,
|
|
55
|
+
tokensDropped: 0,
|
|
56
|
+
sessions: 0,
|
|
57
|
+
contextWindow: null,
|
|
58
|
+
maxTokens: null,
|
|
59
|
+
reasoning: null,
|
|
60
|
+
}));
|
|
61
|
+
// Enrich each repo with per-store token + model detail read directly via
|
|
62
|
+
// node:sqlite (same zero-dependency invariant as readIndex; no store graph
|
|
63
|
+
// import). Best-effort: a missing/corrupt store degrades to the defaults
|
|
64
|
+
// above so the dashboard never fails to render.
|
|
65
|
+
for (const repo of mapped) {
|
|
66
|
+
try {
|
|
67
|
+
const storePath = join(repo.stateDir, "sqlite.db");
|
|
68
|
+
if (existsSync(storePath)) {
|
|
69
|
+
const sdb = new DatabaseSync(storePath, { readOnly: true });
|
|
70
|
+
try {
|
|
71
|
+
const tok = sdb
|
|
72
|
+
.prepare(`SELECT COALESCE(SUM(token_estimate),0) AS kept,
|
|
73
|
+
COALESCE(SUM(original_token_estimate),0) AS dropped,
|
|
74
|
+
COUNT(DISTINCT session_id) AS sess
|
|
75
|
+
FROM context_chunks WHERE dedup_status != 'removed'`)
|
|
76
|
+
.get();
|
|
77
|
+
repo.tokensKept = Number(tok.kept ?? 0);
|
|
78
|
+
repo.tokensDropped = Number(tok.dropped ?? 0);
|
|
79
|
+
repo.sessions = Number(tok.sess ?? 0);
|
|
80
|
+
const mrow = sdb
|
|
81
|
+
.prepare(`SELECT context_window, max_tokens, reasoning
|
|
82
|
+
FROM model_snapshots ORDER BY captured_at DESC LIMIT 1`)
|
|
83
|
+
.get();
|
|
84
|
+
if (mrow) {
|
|
85
|
+
repo.contextWindow = Number(mrow.context_window ?? 0) || null;
|
|
86
|
+
repo.maxTokens = Number(mrow.max_tokens ?? 0) || null;
|
|
87
|
+
repo.reasoning = Number(mrow.reasoning ?? 0) === 1;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
sdb.close();
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
/* best-effort — keep the defaults */
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
// Defensive display hygiene (belt-and-suspenders — the real fix is that
|
|
100
|
+
// tests now isolate via MEGACOMPACT_INDEX_DIR): drop transient test/temp
|
|
101
|
+
// paths that should never have been real repos, and collapse duplicate
|
|
102
|
+
// display names to the most-recently-seen row (rows are last_seen DESC, so
|
|
103
|
+
// the first occurrence wins). Keeps the All-repos list readable.
|
|
104
|
+
const isTransient = (p) => /^\/tmp\//.test(p) || /^\/private\/tmp\//.test(p) || /^\/var\/folders\//.test(p) ||
|
|
105
|
+
/\/mc-(ext|e2e|resume|recall)-/.test(p);
|
|
106
|
+
const seenName = new Set();
|
|
107
|
+
const repos = [];
|
|
108
|
+
for (const r of mapped) {
|
|
109
|
+
if (isTransient(r.repoRoot))
|
|
110
|
+
continue;
|
|
111
|
+
if (seenName.has(r.displayName))
|
|
112
|
+
continue;
|
|
113
|
+
seenName.add(r.displayName);
|
|
114
|
+
repos.push(r);
|
|
115
|
+
}
|
|
116
|
+
const summary = {
|
|
117
|
+
totalRepos: repos.length,
|
|
118
|
+
totalCheckpoints: repos.reduce((a, r) => a + r.checkpointCount, 0),
|
|
119
|
+
totalTokensSaved: repos.reduce((a, r) => a + r.tokensSaved, 0),
|
|
120
|
+
totalCompressedOriginalBytes: repos.reduce((a, r) => a + r.compressedOriginalBytes, 0),
|
|
121
|
+
};
|
|
122
|
+
return { updatedAt: new Date().toISOString(), summary, repos };
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
try {
|
|
129
|
+
db?.close();
|
|
130
|
+
}
|
|
131
|
+
catch { /* ignore */ }
|
|
132
|
+
}
|
|
133
|
+
}
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-server/server.ts — HTTP server creation + launch + CLI entry point.
|
|
3
|
+
*/
|
|
4
|
+
import { createServer } from "node:http";
|
|
5
|
+
import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync } from "node:fs";
|
|
6
|
+
import { join, dirname } from "node:path";
|
|
7
|
+
import { fileURLToPath } from "node:url";
|
|
8
|
+
import { createRequire } from "node:module";
|
|
9
|
+
import { log, setLogPath, setDashboardServerVersion } from "./state.js";
|
|
10
|
+
import { readIndex, getIndexDir } from "./index-reader.js";
|
|
11
|
+
import { readSnapshot, readFrom } from "./snapshot.js";
|
|
12
|
+
import { dashboardHtml } from "./html.js";
|
|
13
|
+
import { ACTIVE_WINDOW_SEC } from "./types.js";
|
|
14
|
+
export async function launchDashboardServer(stateDir) {
|
|
15
|
+
// Our own package version — exposed at /api/version so the launcher can
|
|
16
|
+
// detect a stale server (started by an older build) and replace it on
|
|
17
|
+
// upgrade instead of reuse it.
|
|
18
|
+
let SERVER_VERSION = "0.0.0";
|
|
19
|
+
try {
|
|
20
|
+
// dashboard-server.js lives at <pkg>/dist/extensions/, so package.json is
|
|
21
|
+
// two levels up. Guard each candidate so a dev-checkout layout still works.
|
|
22
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
23
|
+
const candidates = [join(here, "..", "..", "package.json"), join(here, "..", "package.json")];
|
|
24
|
+
for (const p of candidates) {
|
|
25
|
+
if (!existsSync(p))
|
|
26
|
+
continue;
|
|
27
|
+
const pkg = JSON.parse(readFileSync(p, "utf-8"));
|
|
28
|
+
if (pkg.version) {
|
|
29
|
+
SERVER_VERSION = pkg.version;
|
|
30
|
+
setDashboardServerVersion(pkg.version);
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
catch { /* non-fatal */ }
|
|
36
|
+
// Lazy-loaded via require so the dashboard stays cheap to boot and we don't
|
|
37
|
+
// need a top-level await in the handler.
|
|
38
|
+
const driftReq = createRequire(import.meta.url);
|
|
39
|
+
const detectCrossRepoDrift = (idxDir) => driftReq("../../src/driftDetection.js")
|
|
40
|
+
.detectCrossRepoDrift(idxDir);
|
|
41
|
+
const portFile = join(stateDir, "port.pid");
|
|
42
|
+
const snapshotPath = join(stateDir, "dashboard.json");
|
|
43
|
+
const eventsPath = join(stateDir, "events.log");
|
|
44
|
+
setLogPath(join(stateDir, "dashboard.log"));
|
|
45
|
+
log("launch invoked", { stateDir });
|
|
46
|
+
// ── Existing server? ───────────────────────────────────────────────────────
|
|
47
|
+
// A stale port.pid pointing at a dead/competing process is the classic cause
|
|
48
|
+
// of "dashboard failed to start" — we return a port that is NOT actually
|
|
49
|
+
// serving. Probe for a live server on that port first; only reuse the marker
|
|
50
|
+
// when something real answers /api/version. Otherwise drop it and start fresh.
|
|
51
|
+
if (existsSync(portFile)) {
|
|
52
|
+
try {
|
|
53
|
+
const info = JSON.parse(readFileSync(portFile, "utf-8"));
|
|
54
|
+
if (info && info.port) {
|
|
55
|
+
let live = false;
|
|
56
|
+
try {
|
|
57
|
+
const probe = await fetch(`http://localhost:${info.port}/api/version`, { signal: AbortSignal.timeout(800) }); // guardrails-allow PREVENT-PI-004: optional localhost dashboard server probe (loopback-only)
|
|
58
|
+
live = probe.ok;
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
live = false;
|
|
62
|
+
}
|
|
63
|
+
if (live) {
|
|
64
|
+
log("reusing live server from port.pid", { port: info.port });
|
|
65
|
+
return { port: info.port, url: `http://localhost:${info.port}` }; // guardrails-allow PREVENT-PI-004: localhost dashboard URL (loopback-only)
|
|
66
|
+
}
|
|
67
|
+
log("port.pid present but no live server — treating as stale", { port: info.port });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
log("port.pid unparseable — treating as stale");
|
|
72
|
+
}
|
|
73
|
+
// stale file, remove so the fresh bind does not collide with a lingering
|
|
74
|
+
// process that still holds the port
|
|
75
|
+
try {
|
|
76
|
+
unlinkSync(portFile);
|
|
77
|
+
}
|
|
78
|
+
catch { /* ignore */ }
|
|
79
|
+
}
|
|
80
|
+
// ── New server ────────────────────────────────────────────────────────────
|
|
81
|
+
mkdirSync(stateDir, { recursive: true });
|
|
82
|
+
let eventOffset = 0;
|
|
83
|
+
// Overlay the live current-repo snapshot (snapshot.json, rewritten every
|
|
84
|
+
// context event) onto its registry row so the All-repos / Summary views stay
|
|
85
|
+
// in sync with the live menu bar + Current-repo card in real time. The
|
|
86
|
+
// registry (index.sqlite) is only written on repo-switch (bindRepo), so
|
|
87
|
+
// without this the current repo's row freezes between switches. Read-only —
|
|
88
|
+
// no extra writes to index.sqlite. Matched by stateDir, which equals the
|
|
89
|
+
// value this server was launched with (runtime.currentStateDir).
|
|
90
|
+
function overlayCurrentRepo(idx) {
|
|
91
|
+
if (!idx || !idx.repos.length)
|
|
92
|
+
return;
|
|
93
|
+
let snap = null;
|
|
94
|
+
try {
|
|
95
|
+
snap = readSnapshot(snapshotPath);
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
if (!snap || !snap.repo)
|
|
101
|
+
return;
|
|
102
|
+
const cur = idx.repos.find((r) => r.stateDir === stateDir);
|
|
103
|
+
if (!cur)
|
|
104
|
+
return;
|
|
105
|
+
const prevSaved = cur.tokensSaved;
|
|
106
|
+
const prevCp = cur.checkpointCount;
|
|
107
|
+
const prevBytes = cur.compressedOriginalBytes;
|
|
108
|
+
const comp = snap.compression?.repo;
|
|
109
|
+
const liveSaved = comp ? comp.tokensFreed : (snap.repo.tokensSaved ?? prevSaved);
|
|
110
|
+
const liveCp = snap.repo.checkpointCount ?? prevCp;
|
|
111
|
+
const liveBytes = snap.integrity?.compressedOriginalBytes ?? prevBytes;
|
|
112
|
+
cur.tokensSaved = liveSaved;
|
|
113
|
+
cur.checkpointCount = liveCp;
|
|
114
|
+
cur.compressedOriginalBytes = liveBytes;
|
|
115
|
+
if (idx.summary) {
|
|
116
|
+
idx.summary.totalTokensSaved += liveSaved - prevSaved;
|
|
117
|
+
idx.summary.totalCheckpoints += liveCp - prevCp;
|
|
118
|
+
idx.summary.totalCompressedOriginalBytes += liveBytes - prevBytes;
|
|
119
|
+
}
|
|
120
|
+
idx.updatedAt = snap.updatedAt ?? idx.updatedAt;
|
|
121
|
+
}
|
|
122
|
+
const server = createServer((req, res) => {
|
|
123
|
+
// guardrails-allow PREVENT-PI-004: optional, user-triggered /dashboard localhost server (loopback-only) — CORS open for local browser access
|
|
124
|
+
// CORS for local access
|
|
125
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
126
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, OPTIONS");
|
|
127
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type");
|
|
128
|
+
if (req.method === "OPTIONS") {
|
|
129
|
+
res.writeHead(204);
|
|
130
|
+
res.end();
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
if (req.url === "/" || req.url === "/index.html") {
|
|
134
|
+
const tier = readSnapshot(snapshotPath).tier;
|
|
135
|
+
res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
|
|
136
|
+
res.end(dashboardHtml(tier));
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
if (req.url === "/api/snapshot") {
|
|
140
|
+
const snap = readSnapshot(snapshotPath);
|
|
141
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
142
|
+
res.end(JSON.stringify(snap));
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
// Server version — lets the /dashboard launcher detect a stale server from
|
|
146
|
+
// an older build and replace it on upgrade rather than reuse it.
|
|
147
|
+
if (req.url === "/api/version") {
|
|
148
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
149
|
+
res.end(JSON.stringify({ version: SERVER_VERSION }));
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
// Multi-repo aggregate (Phase 5b): the machine-wide repo registry read
|
|
153
|
+
// directly from SQLite (index.sqlite). Lets one dashboard show every repo's
|
|
154
|
+
// checkpoints, tokens saved, and active model. Read-only.
|
|
155
|
+
if (req.url === "/api/index") {
|
|
156
|
+
const idx = readIndex();
|
|
157
|
+
if (idx)
|
|
158
|
+
overlayCurrentRepo(idx);
|
|
159
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
160
|
+
res.end(JSON.stringify(idx ?? { updatedAt: null, summary: null, repos: [] }));
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
// /api/repos — registry list. Optional `?active=24h` filters to repos
|
|
164
|
+
// seen within the last N hours (default: all). The dashboard uses this to
|
|
165
|
+
// drive its "active vs archived" badge without refetching /api/index.
|
|
166
|
+
if (req.url?.startsWith("/api/repos")) {
|
|
167
|
+
const url = new URL(req.url, "http://x"); // guardrails-allow PREVENT-PI-004: localhost dashboard URL base (loopback-only)
|
|
168
|
+
const activeParam = url.searchParams.get("active");
|
|
169
|
+
const idx = readIndex();
|
|
170
|
+
if (idx)
|
|
171
|
+
overlayCurrentRepo(idx);
|
|
172
|
+
let repos = idx?.repos ?? [];
|
|
173
|
+
if (activeParam) {
|
|
174
|
+
const m = /^(\d+)h$/.exec(activeParam);
|
|
175
|
+
if (m) {
|
|
176
|
+
const cutoffSec = Math.floor(Date.now() / 1000) - Number(m[1]) * 3600;
|
|
177
|
+
repos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
181
|
+
res.end(JSON.stringify({ updatedAt: idx?.updatedAt ?? null, repos, count: repos.length }));
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
// /api/summary — header tiles without the full repo list (keeps payload
|
|
185
|
+
// small for embed scenarios). activeRepos mirrors the /api/repos?active=24h
|
|
186
|
+
// count so the dashboard can render the active badge alongside totals.
|
|
187
|
+
if (req.url?.startsWith("/api/summary")) {
|
|
188
|
+
const idx = readIndex();
|
|
189
|
+
if (idx)
|
|
190
|
+
overlayCurrentRepo(idx);
|
|
191
|
+
const repos = idx?.repos ?? [];
|
|
192
|
+
const cutoffSec = Math.floor(Date.now() / 1000) - 24 * 3600;
|
|
193
|
+
const activeRepos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec).length;
|
|
194
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
195
|
+
res.end(JSON.stringify({
|
|
196
|
+
updatedAt: idx?.updatedAt ?? null,
|
|
197
|
+
summary: idx?.summary ?? null,
|
|
198
|
+
activeRepos,
|
|
199
|
+
totalRepos: repos.length,
|
|
200
|
+
}));
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
// /api/drift — R4: cross-repo drift report over repo_registry. Flags stale
|
|
204
|
+
// repos (>30d idle), compaction lag (active but >24h since last
|
|
205
|
+
// compaction), and recent model churn. Read-only.
|
|
206
|
+
if (req.url?.startsWith("/api/drift")) {
|
|
207
|
+
const report = detectCrossRepoDrift(getIndexDir());
|
|
208
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
209
|
+
res.end(JSON.stringify(report));
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (req.url === "/api/servers") {
|
|
213
|
+
try {
|
|
214
|
+
const idx = readIndex();
|
|
215
|
+
const nowSec = Math.floor(Date.now() / 1000);
|
|
216
|
+
const servers = (idx?.repos ?? []).filter((r) => (r.lastSeen ?? 0) >= nowSec - ACTIVE_WINDOW_SEC).map((r) => {
|
|
217
|
+
const out = { repoRoot: r.repoRoot, displayName: r.displayName, model: r.modelName, provider: r.providerName, lastSeen: r.lastSeen, lastCompactedAt: r.lastCompactedAt };
|
|
218
|
+
try {
|
|
219
|
+
const p = join(r.stateDir, "dashboard.json");
|
|
220
|
+
if (existsSync(p)) {
|
|
221
|
+
const snap = JSON.parse(readFileSync(p, "utf-8"));
|
|
222
|
+
out.tier = snap.tier ?? null;
|
|
223
|
+
out.contextPct = (snap.context && snap.context.percent != null) ? snap.context.percent : null;
|
|
224
|
+
out.state = (snap.session && snap.session.state) || null;
|
|
225
|
+
out.cacheHits = snap.cacheHits ?? null;
|
|
226
|
+
out.compacts = snap.compacts ?? null;
|
|
227
|
+
out.timeSaved = snap.timeSaved ?? null;
|
|
228
|
+
out.updatedAt = snap.updatedAt ?? null;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
catch { /* best-effort */ }
|
|
232
|
+
return out;
|
|
233
|
+
}).sort((a, b) => b.lastSeen - a.lastSeen);
|
|
234
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
235
|
+
res.end(JSON.stringify({ updatedAt: new Date().toISOString(), servers }));
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
239
|
+
res.end(JSON.stringify({ error: "servers_unavailable" }));
|
|
240
|
+
}
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
if (req.url === "/api/events") {
|
|
244
|
+
res.writeHead(200, {
|
|
245
|
+
"Content-Type": "text/event-stream",
|
|
246
|
+
"Cache-Control": "no-cache",
|
|
247
|
+
"Connection": "keep-alive",
|
|
248
|
+
});
|
|
249
|
+
// Drain existing events so the client starts with history
|
|
250
|
+
const { data: existing, offset: initialOffset } = readFrom(eventsPath, 0);
|
|
251
|
+
eventOffset = initialOffset;
|
|
252
|
+
const lines = existing.split("\n").filter((l) => l.trim());
|
|
253
|
+
for (const line of lines) {
|
|
254
|
+
res.write(`data: ${line}\n\n`);
|
|
255
|
+
}
|
|
256
|
+
// Tail new events via fs.watch (coalesced with 100ms debounce)
|
|
257
|
+
let watchTimer = null;
|
|
258
|
+
const onWatch = () => {
|
|
259
|
+
if (watchTimer)
|
|
260
|
+
return;
|
|
261
|
+
watchTimer = setTimeout(() => {
|
|
262
|
+
watchTimer = null;
|
|
263
|
+
const { data, offset } = readFrom(eventsPath, eventOffset);
|
|
264
|
+
eventOffset = offset;
|
|
265
|
+
const newLines = data.split("\n").filter((l) => l.trim());
|
|
266
|
+
for (const line of newLines) {
|
|
267
|
+
res.write(`data: ${line}\n\n`);
|
|
268
|
+
}
|
|
269
|
+
}, 100);
|
|
270
|
+
};
|
|
271
|
+
// Set up file watching: if file exists, watch it directly;
|
|
272
|
+
// otherwise poll for creation every 1s then switch to fs.watch.
|
|
273
|
+
let watcher = null;
|
|
274
|
+
let pollInterval = null;
|
|
275
|
+
function startFileWatch() {
|
|
276
|
+
try {
|
|
277
|
+
watcher = watch(eventsPath, onWatch);
|
|
278
|
+
}
|
|
279
|
+
catch { /* give up */ }
|
|
280
|
+
}
|
|
281
|
+
if (existsSync(eventsPath)) {
|
|
282
|
+
startFileWatch();
|
|
283
|
+
}
|
|
284
|
+
else {
|
|
285
|
+
pollInterval = setInterval(() => {
|
|
286
|
+
if (existsSync(eventsPath)) {
|
|
287
|
+
if (pollInterval) {
|
|
288
|
+
clearInterval(pollInterval);
|
|
289
|
+
pollInterval = null;
|
|
290
|
+
}
|
|
291
|
+
startFileWatch();
|
|
292
|
+
}
|
|
293
|
+
}, 1000);
|
|
294
|
+
}
|
|
295
|
+
req.on("close", () => {
|
|
296
|
+
if (watchTimer)
|
|
297
|
+
clearTimeout(watchTimer);
|
|
298
|
+
if (pollInterval)
|
|
299
|
+
clearInterval(pollInterval);
|
|
300
|
+
watcher?.close();
|
|
301
|
+
});
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
// Fallback — serve the dashboard
|
|
305
|
+
const tier = readSnapshot(snapshotPath).tier;
|
|
306
|
+
res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
|
|
307
|
+
res.end(dashboardHtml(tier));
|
|
308
|
+
});
|
|
309
|
+
// Bind base + range are env-configurable so tests can use a private,
|
|
310
|
+
// non-colliding range (parallel runs / leftover servers from killed runs
|
|
311
|
+
// would otherwise EADDRINUSE on the machine-global 9320 range). Default
|
|
312
|
+
// MEGACOMPACT_DASHBOARD_PORT=9320 (10-port range 9320–9329) preserves the
|
|
313
|
+
// production behavior.
|
|
314
|
+
const TARGET_PORT = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
|
|
315
|
+
const PORT_RANGE = 10; // TARGET_PORT..TARGET_PORT+9
|
|
316
|
+
return new Promise((resolve, reject) => {
|
|
317
|
+
function tryPort(port) {
|
|
318
|
+
server.once("error", (err) => {
|
|
319
|
+
if (err.code === "EADDRINUSE" && port < TARGET_PORT + PORT_RANGE - 1) {
|
|
320
|
+
log("port in use, trying next", { port });
|
|
321
|
+
tryPort(port + 1);
|
|
322
|
+
}
|
|
323
|
+
else {
|
|
324
|
+
log("listen failed", { port, code: err.code, message: err.message });
|
|
325
|
+
reject(err);
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
server.listen(port, "127.0.0.1", () => {
|
|
329
|
+
const url = `http://localhost:${port}`; // guardrails-allow PREVENT-PI-004: localhost dashboard URL (loopback-only)
|
|
330
|
+
log("server running", { url });
|
|
331
|
+
// eslint-disable-next-line no-console
|
|
332
|
+
console.log(`[mega-compact] dashboard server running: ${url}`);
|
|
333
|
+
// Write port.pid
|
|
334
|
+
try {
|
|
335
|
+
writeFileSync(portFile, JSON.stringify({ port, pid: process.pid }));
|
|
336
|
+
}
|
|
337
|
+
catch (e) {
|
|
338
|
+
log("could not write port.pid", { error: String(e) });
|
|
339
|
+
}
|
|
340
|
+
// Graceful cleanup
|
|
341
|
+
const cleanup = () => {
|
|
342
|
+
try {
|
|
343
|
+
unlinkSync(portFile);
|
|
344
|
+
}
|
|
345
|
+
catch { /* already gone */ }
|
|
346
|
+
server.close();
|
|
347
|
+
process.exit(0);
|
|
348
|
+
};
|
|
349
|
+
process.on("SIGTERM", cleanup);
|
|
350
|
+
process.on("SIGINT", cleanup);
|
|
351
|
+
resolve({ port, url });
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
tryPort(TARGET_PORT);
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
// ---------------------------------------------------------------------------
|
|
358
|
+
// CLI entry point — when run directly as `node dashboard-server.js <stateDir>`
|
|
359
|
+
// ---------------------------------------------------------------------------
|
|
360
|
+
if (process.argv[1] && process.argv[1].includes("dashboard-server")) {
|
|
361
|
+
const stateDir = process.argv[2];
|
|
362
|
+
if (!stateDir) {
|
|
363
|
+
console.error("Usage: node dashboard-server.js <stateDir>");
|
|
364
|
+
process.exit(1);
|
|
365
|
+
}
|
|
366
|
+
launchDashboardServer(stateDir).catch((err) => {
|
|
367
|
+
console.error("[mega-compact] dashboard server failed:", err);
|
|
368
|
+
process.exit(1);
|
|
369
|
+
});
|
|
370
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-server/snapshot.ts — snapshot + events.log file readers.
|
|
3
|
+
*/
|
|
4
|
+
import { readFileSync } from "node:fs";
|
|
5
|
+
export function readSnapshot(snapshotPath) {
|
|
6
|
+
try {
|
|
7
|
+
const raw = readFileSync(snapshotPath, "utf-8");
|
|
8
|
+
return JSON.parse(raw);
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return {
|
|
12
|
+
version: 1,
|
|
13
|
+
updatedAt: null,
|
|
14
|
+
tier: "unknown",
|
|
15
|
+
presetTier: "unknown",
|
|
16
|
+
pressure: 0,
|
|
17
|
+
config: { fastGatePct: 80, thresholdTokens: 100_000, tierPct: null, effectiveThresholdPct: null, anchorUserMessages: 1, preserveRecent: 2, auto: true, autoInlineK: 3 },
|
|
18
|
+
session: { id: null, state: null, persistedThisSession: false, lastCheckpointId: null, lastCompactedFrom: 0 },
|
|
19
|
+
context: { tokens: null, percent: null, contextWindow: 0 },
|
|
20
|
+
trigger: { armed: false, ready: false, currentTokens: null, thresholdTokens: 100_000, fastGatePct: 80, tierPct: null, effectiveThresholdPct: null },
|
|
21
|
+
store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
|
|
22
|
+
crew: { activeAgents: 0, currentTurn: 0 },
|
|
23
|
+
repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
|
|
24
|
+
integrity: { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 },
|
|
25
|
+
cacheHits: { session: 0, total: 0, sessionTokensSaved: 0, totalTokensSaved: 0 },
|
|
26
|
+
compacts: { session: 0, total: 0 },
|
|
27
|
+
timeSaved: { compact: { sessionSec: 0, totalSec: 0 }, cacheHit: { sessionSec: 0, totalSec: 0 } },
|
|
28
|
+
compression: { session: { tokensIn: 0, tokensOut: 0, tokensFreed: 0, compressionPct: 0, dedupPct: 0 }, repo: { tokensIn: 0, tokensOut: 0, tokensFreed: 0, compressionPct: 0, dedupPct: 0 } },
|
|
29
|
+
model: undefined,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
export function readFrom(path, charOffset) {
|
|
34
|
+
try {
|
|
35
|
+
const content = readFileSync(path, "utf-8");
|
|
36
|
+
if (content.length <= charOffset)
|
|
37
|
+
return { data: "", offset: charOffset };
|
|
38
|
+
return { data: content.slice(charOffset), offset: content.length };
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return { data: "", offset: charOffset };
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-server/state.ts — local runtime log + version state.
|
|
3
|
+
*
|
|
4
|
+
* The dashboard server is spawned as a DETACHED child. When it is launched with
|
|
5
|
+
* `stdio: "ignore"` (the old default) any crash before the first console.log is
|
|
6
|
+
* invisible — there is no log to "check". We therefore mirror every lifecycle
|
|
7
|
+
* line to a file in the state dir so a failed start is always diagnosable. The
|
|
8
|
+
* launcher also captures stderr, so this doubles as defense-in-depth.
|
|
9
|
+
*/
|
|
10
|
+
import { appendFileSync } from "node:fs";
|
|
11
|
+
let LOG_PATH = null;
|
|
12
|
+
export function setLogPath(path) {
|
|
13
|
+
LOG_PATH = path;
|
|
14
|
+
}
|
|
15
|
+
export function log(...parts) {
|
|
16
|
+
const line = `[mega-compact][dashboard] ${parts.map((p) => (typeof p === "string" ? p : JSON.stringify(p))).join(" ")}`;
|
|
17
|
+
// eslint-disable-next-line no-console
|
|
18
|
+
console.error(line); // stderr — captured by the launcher pipe
|
|
19
|
+
if (LOG_PATH) {
|
|
20
|
+
try {
|
|
21
|
+
appendFileSync(LOG_PATH, new Date().toISOString() + " " + line + "\n");
|
|
22
|
+
}
|
|
23
|
+
catch { /* non-fatal */ }
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/** Package version of this extension, surfaced in the dashboard header. */
|
|
27
|
+
export let dashboardServerVersion = "0.0.0";
|
|
28
|
+
export function setDashboardServerVersion(v) {
|
|
29
|
+
dashboardServerVersion = v;
|
|
30
|
+
}
|