pi-mega-compact 0.8.21 → 0.8.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/LICENSE +6 -2
  2. package/README.md +1 -1
  3. package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
  4. package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
  5. package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
  6. package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
  7. package/dist/extensions/dashboard-server/html.js +2 -621
  8. package/dist/extensions/dashboard-server/routes-core.js +62 -0
  9. package/dist/extensions/dashboard-server/routes-game.js +323 -0
  10. package/dist/extensions/dashboard-server/routes-repo.js +170 -0
  11. package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
  12. package/dist/extensions/dashboard-server/routes.js +10 -0
  13. package/dist/extensions/dashboard-server/server.js +26 -623
  14. package/dist/extensions/mega-commands.js +4 -3
  15. package/dist/extensions/mega-events/agent-handlers.js +2 -1
  16. package/dist/extensions/mega-events/compact-handlers.js +26 -0
  17. package/dist/extensions/mega-events/session-handlers.js +2 -1
  18. package/dist/extensions/mega-pipeline/compact.js +3 -2
  19. package/dist/extensions/mega-runtime/state.js +7 -7
  20. package/dist/src/dedup/raptor/multilevel.js +172 -0
  21. package/dist/src/dedup/raptor/multilevel.test.js +203 -0
  22. package/dist/src/dedup/raptor/promote.test.js +5 -5
  23. package/dist/src/dedup/raptor/retrieval.js +1 -1
  24. package/dist/src/dedup/sprint12.test.js +7 -7
  25. package/dist/src/dedup-engine.test.js +29 -29
  26. package/dist/src/e2e.test.js +38 -38
  27. package/dist/src/engine.js +3 -3
  28. package/dist/src/engine.test.js +6 -6
  29. package/dist/src/importance.js +197 -0
  30. package/dist/src/importance.test.js +372 -0
  31. package/dist/src/ratio.bench.test.js +18 -18
  32. package/dist/src/recall.js +6 -5
  33. package/dist/src/recall.test.js +85 -27
  34. package/dist/src/sprint14.test.js +2 -2
  35. package/dist/src/store/migrate.test.js +5 -5
  36. package/dist/src/store/sprint10.test.js +5 -5
  37. package/dist/src/store/sqlite/global-index.js +5 -174
  38. package/dist/src/store/sqlite/global-sessions.js +190 -0
  39. package/dist/src/vector-read.js +168 -0
  40. package/dist/src/vector-search.js +191 -0
  41. package/dist/src/vectorStore.js +10 -297
  42. package/dist/src/vectorStore.test.js +32 -32
  43. package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
  44. package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
  45. package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
  46. package/extensions/dashboard-server/dashboard-client.ts +21 -0
  47. package/extensions/dashboard-server/html.ts +2 -621
  48. package/extensions/dashboard-server/routes-core.ts +113 -0
  49. package/extensions/dashboard-server/routes-game.ts +386 -0
  50. package/extensions/dashboard-server/routes-repo.ts +212 -0
  51. package/extensions/dashboard-server/routes-sessions.ts +195 -0
  52. package/extensions/dashboard-server/routes.ts +13 -0
  53. package/extensions/dashboard-server/server.ts +37 -700
  54. package/extensions/mega-commands.ts +4 -3
  55. package/extensions/mega-events/agent-handlers.ts +2 -1
  56. package/extensions/mega-events/compact-handlers.ts +28 -0
  57. package/extensions/mega-events/session-handlers.ts +2 -1
  58. package/extensions/mega-pipeline/compact.ts +3 -2
  59. package/extensions/mega-runtime/state.ts +7 -7
  60. package/extensions/openclaw-mega-compact.ts +2 -2
  61. package/package.json +2 -2
  62. package/src/dedup/raptor/multilevel.test.ts +278 -0
  63. package/src/dedup/raptor/multilevel.ts +246 -0
  64. package/src/dedup/raptor/promote.test.ts +5 -5
  65. package/src/dedup/raptor/retrieval.ts +1 -1
  66. package/src/dedup/sprint12.test.ts +7 -7
  67. package/src/dedup-engine.test.ts +30 -30
  68. package/src/e2e.test.ts +38 -38
  69. package/src/engine.test.ts +6 -6
  70. package/src/engine.ts +3 -3
  71. package/src/importance.test.ts +538 -0
  72. package/src/importance.ts +312 -0
  73. package/src/ratio.bench.test.ts +18 -18
  74. package/src/recall.test.ts +101 -29
  75. package/src/recall.ts +9 -9
  76. package/src/sprint14.test.ts +2 -2
  77. package/src/store/migrate.test.ts +5 -5
  78. package/src/store/sprint10.test.ts +5 -5
  79. package/src/store/sqlite/global-index.ts +18 -290
  80. package/src/store/sqlite/global-sessions.ts +291 -0
  81. package/src/vector-read.ts +237 -0
  82. package/src/vector-search.ts +231 -0
  83. package/src/vectorStore.test.ts +32 -32
  84. package/src/vectorStore.ts +29 -356
@@ -14,6 +14,7 @@
14
14
  * extension decides where it lands.
15
15
  */
16
16
  import { recall as searchRecall } from "./engine.js";
17
+ import { vectorWasInjected, vectorMarkInjected, vectorSearchAsync } from "./vectorStore.js";
17
18
  import { estimateBlockTokens } from "./tokens.js";
18
19
  import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
19
20
  /** Wrap a recall block so the model reads it as restored compacted context. */
@@ -98,7 +99,7 @@ export function recallAndInline(opts, store) {
98
99
  const parts = [];
99
100
  let blockTokens = 0;
100
101
  for (const h of hits) {
101
- if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId))
102
+ if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId))
102
103
  continue;
103
104
  // Inline dedupe: skip a hit already resident in the live window (Fix C).
104
105
  if (doWindowDedupe && liveEmbeddings.length > 0) {
@@ -114,7 +115,7 @@ export function recallAndInline(opts, store) {
114
115
  parts.push(part);
115
116
  toInject.push(h);
116
117
  blockTokens += partTokens;
117
- store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
118
+ vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
118
119
  }
119
120
  const block = parts.join("\n");
120
121
  const report = toInject.map((h) => ` • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
@@ -210,7 +211,7 @@ export async function recallAndInlineAsync(opts, store) {
210
211
  const dedupSim = opts.dedupSim ?? 0.9;
211
212
  let hits = [];
212
213
  try {
213
- hits = await store.searchAsync(opts.sessionId, opts.query, limit, {
214
+ hits = await vectorSearchAsync(store, opts.sessionId, opts.query, limit, {
214
215
  crossRepo: opts.crossRepo,
215
216
  repoId: opts.repoId,
216
217
  });
@@ -227,7 +228,7 @@ export async function recallAndInlineAsync(opts, store) {
227
228
  const parts = [];
228
229
  let blockTokens = 0;
229
230
  for (const h of hits) {
230
- if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId))
231
+ if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId))
231
232
  continue;
232
233
  // S18: machine-wide injected-set — a foreign checkpoint already injected
233
234
  // (in any session) is never re-injected. Only applies to cross-repo hits
@@ -254,7 +255,7 @@ export async function recallAndInlineAsync(opts, store) {
254
255
  parts.push(part);
255
256
  toInject.push(h);
256
257
  blockTokens += partTokens;
257
- store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
258
+ vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
258
259
  // S18: record the cross-repo injection machine-wide so it's not re-injected
259
260
  // by a later recall (same or different session).
260
261
  if (opts.globalIndexDir && h.repoId) {
@@ -6,7 +6,9 @@ import { join } from "node:path";
6
6
  import { VectorStore } from "./vectorStore.js";
7
7
  import { compactSession } from "./engine.js";
8
8
  import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "./recall.js";
9
+ import { vectorList } from "./vectorStore.js";
9
10
  import { markInjectedGlobal, wasInjectedGlobal, closeIndexStore } from "./store/sqlite.js";
11
+ import { closeVectorIndex, initVectorIndex, rebuildFromSqlite, } from "./store/vectorIndex.js";
10
12
  const baseTmp = mkdtempSync(join(tmpdir(), "mc-recall-"));
11
13
  let counter = 0;
12
14
  function store() {
@@ -92,53 +94,109 @@ test("Fix C: inline dedupe drops a hit already resident in the live window", ()
92
94
  assert.ok(rDedup.toInject.length <= rNoDedup.toInject.length, "dedupe never adds hits");
93
95
  assert.ok(rDedup.toInject.length < rNoDedup.toInject.length, "inline dedupe dropped a resident hit");
94
96
  });
97
+ // ---- S18 cross-repo global injected-set (real-data, no mocks) ----------------
98
+ //
99
+ // Earlier versions of these tests used `as any` mock stores with canned
100
+ // `searchAsync` returns. That was mock data: it asserted recall's orchestration
101
+ // against a fake search result, not the real embed → HNSW → hydrate → inject
102
+ // path. Per the no-mock-data principle, both tests now seed a REAL foreign
103
+ // VectorStore (real checkpoint + real TrigramEmbedder embedding persisted to
104
+ // SQLite), rebuild the real PGlite index from it via `rebuildFromSqlite`, and
105
+ // run the full `recallAndInlineAsync` cross-repo path against a separate self
106
+ // store. The foreign checkpoint hydrates from the foreign store via its real
107
+ // repoId (== stateDir, per VectorStore's repoId convention).
108
+ async function seedForeignRepo(foreignStateDir) {
109
+ // A real VectorStore at the foreign repo's stateDir. repoId == stateDir.
110
+ const foreign = new VectorStore({ stateDir: foreignStateDir, dedupSim: 0.9 });
111
+ const sess = "sess_foreign";
112
+ // Seed a real checkpoint via the real compactSession pipeline. The summary
113
+ // text determines the embedding; the query below must land near it.
114
+ const summary = "foreign repo authentication jwt token validation";
115
+ const result = compactSession({
116
+ sessionId: sess,
117
+ messages: [msg("user", summary), msg("assistant", "ok", "Edit")],
118
+ keepFrom: 2,
119
+ timestamp: 1,
120
+ }, foreign);
121
+ return { sessionId: sess, checkpointId: result.checkpointId ?? "", summary };
122
+ }
95
123
  test("S18: global injected-set skips a foreign checkpoint already injected machine-wide", async () => {
124
+ if (process.env.MEGACOMPACT_PGLITE_DISABLED === "true") {
125
+ return;
126
+ } // skip when WASM index is off
96
127
  const indexDir = mkdtempSync(join(tmpdir(), "mc-gi-"));
128
+ const foreignStateDir = mkdtempSync(join(tmpdir(), "mc-gi-foreign-"));
129
+ const selfStateDir = mkdtempSync(join(tmpdir(), "mc-gi-self-"));
130
+ process.env.MEGACOMPACT_VECTOR_INDEX_DIR = mkdtempSync(join(tmpdir(), "mc-gi-vidx-"));
97
131
  try {
132
+ await closeVectorIndex(); // fresh singleton per this index dir
133
+ const seed = await seedForeignRepo(foreignStateDir);
134
+ // Populate the real PGlite index from the foreign store's real checkpoints.
135
+ await rebuildFromSqlite(() => [{ repoId: foreignStateDir, stateDir: foreignStateDir }], (sd) => {
136
+ // readCheckpoints: yield (sessionId, checkpointId, embedding) for every
137
+ // real checkpoint in this store. Mirrors the production enumerator.
138
+ const store = new VectorStore({ stateDir: sd, dedupSim: 0.9 });
139
+ return vectorList(store, seed.sessionId).map((cp) => ({
140
+ sessionId: seed.sessionId,
141
+ checkpointId: cp.checkpointId,
142
+ embedding: cp.embedding,
143
+ }));
144
+ });
145
+ const pg = await initVectorIndex();
146
+ assert.ok(pg, "PGlite index should initialize (WASM available)");
98
147
  const sess = "sess_cross";
99
- // A foreign checkpoint already marked injected globally (in this session).
100
- markInjectedGlobal("chkpt_foreign", "/repo/other", sess, indexDir);
101
- assert.equal(wasInjectedGlobal("chkpt_foreign", sess, indexDir), true);
102
- // searchAsync returns the foreign hit; recallAndInlineAsync must skip it
103
- // (globally injected) toInject is empty.
104
- const mockStore = {
105
- searchAsync: async () => [{
106
- checkpoint: { checkpointId: "chkpt_foreign", summary: "foreign work", filesModified: [], dedupStatus: "active" },
107
- score: 0.92,
108
- repoId: "/repo/other",
109
- }],
110
- wasInjected: () => false,
111
- markInjected: () => { },
112
- };
113
- const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, mockStore);
148
+ // Pre-mark the foreign checkpoint as already injected machine-wide.
149
+ markInjectedGlobal(seed.checkpointId, foreignStateDir, sess, indexDir);
150
+ assert.equal(wasInjectedGlobal(seed.checkpointId, sess, indexDir), true);
151
+ // Self store (different repo) — the cross-repo query runs against the index.
152
+ const selfStore = new VectorStore({ stateDir: selfStateDir, dedupSim: 0.9 });
153
+ const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign repo authentication jwt", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, selfStore);
114
154
  assert.equal(r.toInject.length, 0, "globally-injected foreign checkpoint skipped");
115
155
  }
116
156
  finally {
157
+ await closeVectorIndex();
117
158
  closeIndexStore();
159
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
118
160
  rmSync(indexDir, { recursive: true, force: true });
161
+ rmSync(foreignStateDir, { recursive: true, force: true });
162
+ rmSync(selfStateDir, { recursive: true, force: true });
119
163
  }
120
164
  });
121
165
  test("S18: a fresh foreign checkpoint is injected AND recorded globally", async () => {
166
+ if (process.env.MEGACOMPACT_PGLITE_DISABLED === "true") {
167
+ return;
168
+ } // skip when WASM index is off
122
169
  const indexDir = mkdtempSync(join(tmpdir(), "mc-gi2-"));
170
+ const foreignStateDir = mkdtempSync(join(tmpdir(), "mc-gi2-foreign-"));
171
+ const selfStateDir = mkdtempSync(join(tmpdir(), "mc-gi2-self-"));
172
+ process.env.MEGACOMPACT_VECTOR_INDEX_DIR = mkdtempSync(join(tmpdir(), "mc-gi2-vidx-"));
123
173
  try {
174
+ await closeVectorIndex(); // fresh singleton per this index dir
175
+ const seed = await seedForeignRepo(foreignStateDir);
176
+ assert.equal(wasInjectedGlobal(seed.checkpointId, "sess_fresh", indexDir), false);
177
+ await rebuildFromSqlite(() => [{ repoId: foreignStateDir, stateDir: foreignStateDir }], (sd) => {
178
+ const store = new VectorStore({ stateDir: sd, dedupSim: 0.9 });
179
+ return vectorList(store, seed.sessionId).map((cp) => ({
180
+ sessionId: seed.sessionId,
181
+ checkpointId: cp.checkpointId,
182
+ embedding: cp.embedding,
183
+ }));
184
+ });
185
+ const pg = await initVectorIndex();
186
+ assert.ok(pg, "PGlite index should initialize (WASM available)");
124
187
  const sess = "sess_fresh";
125
- assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), false);
126
- const mockStore = {
127
- searchAsync: async () => [{
128
- checkpoint: { checkpointId: "chkpt_new", summary: "brand new foreign work", filesModified: [], dedupStatus: "active" },
129
- score: 0.93,
130
- repoId: "/repo/alpha",
131
- }],
132
- wasInjected: () => false,
133
- markInjected: () => { },
134
- };
135
- const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, mockStore);
188
+ const selfStore = new VectorStore({ stateDir: selfStateDir, dedupSim: 0.9 });
189
+ const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign repo authentication jwt", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, selfStore);
136
190
  assert.equal(r.toInject.length, 1, "fresh foreign checkpoint injected");
137
- assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), true, "recorded machine-wide");
191
+ assert.equal(wasInjectedGlobal(seed.checkpointId, sess, indexDir), true, "recorded machine-wide");
138
192
  }
139
193
  finally {
194
+ await closeVectorIndex();
140
195
  closeIndexStore();
196
+ delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
141
197
  rmSync(indexDir, { recursive: true, force: true });
198
+ rmSync(foreignStateDir, { recursive: true, force: true });
199
+ rmSync(selfStateDir, { recursive: true, force: true });
142
200
  }
143
201
  });
144
202
  test("cleanup", () => {
@@ -7,7 +7,7 @@ import assert from "node:assert/strict";
7
7
  import { mkdtempSync, rmSync, readFileSync, existsSync } from "node:fs";
8
8
  import { tmpdir } from "node:os";
9
9
  import { join } from "node:path";
10
- import { VectorStore } from "./vectorStore.js";
10
+ import { VectorStore, vectorSearch } from "./vectorStore.js";
11
11
  import { defaultEmbedder } from "./embedder.js";
12
12
  import { loadDedupConfig } from "./config/dedup.js";
13
13
  import { loadMetrics, saveMetrics, recordDecision, evaluateAlerts, fpRate, p95, } from "./monitoring.js";
@@ -35,7 +35,7 @@ test("flag matrix: all 16 L0/L1/L2/RAPTOR enable combos are safe", () => {
35
35
  s.add({ sessionId: "s", summary: "x", regionText: `region A for combo ${combos} about the cache`, timestamp: 1 });
36
36
  const r2 = s.add({ sessionId: "s", summary: "x", regionText: `region B for combo ${combos} about the parser`, timestamp: 2 });
37
37
  // search must not throw under any combination
38
- const hits = s.search("s", "cache", 3);
38
+ const hits = vectorSearch(s, "s", "cache", 3);
39
39
  assert.ok(Array.isArray(hits));
40
40
  // With all tiers off, the second add is always "new" (no collapse).
41
41
  if (!l0 && !l1 && !l2)
@@ -15,7 +15,7 @@ import assert from "node:assert/strict";
15
15
  import { mkdtempSync, rmSync, existsSync } from "node:fs";
16
16
  import { tmpdir } from "node:os";
17
17
  import { join } from "node:path";
18
- import { VectorStore } from "../vectorStore.js";
18
+ import { VectorStore, vectorMarkInjected, vectorWasInjected, vectorSearch } from "../vectorStore.js";
19
19
  import { writeGzJson } from "../store.js";
20
20
  import { migrateJsonToSqlite, readLegacyCheckpointFile } from "../store/migrate.js";
21
21
  import { listCheckpoints, closeStore } from "../store/sqlite.js";
@@ -110,7 +110,7 @@ test("cross-process recall: fresh VectorStore over same dir recalls prior checkp
110
110
  closeStore(dir);
111
111
  // Process B: brand-new VectorStore, same stateDir.
112
112
  const b = new VectorStore({ stateDir: dir });
113
- const hits = b.search(sid, "cross process recall proof", 5);
113
+ const hits = vectorSearch(b, sid, "cross process recall proof", 5);
114
114
  assert.equal(hits.length, 1, "checkpoint survives cross-process reopen");
115
115
  assert.equal(hits[0].checkpoint.checkpointId, "chkpt_001");
116
116
  assert.ok(hits[0].score > 0.5, "recall is relevant");
@@ -127,11 +127,11 @@ test("cross-process recall: injected state persists across reopen", () => {
127
127
  regionText: "injected state persists across process reopen test",
128
128
  timestamp: 1,
129
129
  });
130
- a.markInjected(sid, added.checkpoint.checkpointId);
131
- assert.equal(a.wasInjected(sid, added.checkpoint.checkpointId), true);
130
+ vectorMarkInjected(a, sid, added.checkpoint.checkpointId);
131
+ assert.equal(vectorWasInjected(a, sid, added.checkpoint.checkpointId), true);
132
132
  closeStore(dir);
133
133
  const b = new VectorStore({ stateDir: dir });
134
- assert.equal(b.wasInjected(sid, added.checkpoint.checkpointId), true, "injection remembered");
134
+ assert.equal(vectorWasInjected(b, sid, added.checkpoint.checkpointId), true, "injection remembered");
135
135
  closeStore(dir);
136
136
  });
137
137
  test("cleanup", () => {
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
3
3
  import { mkdtempSync, rmSync } from "node:fs";
4
4
  import { tmpdir } from "node:os";
5
5
  import { join } from "node:path";
6
- import { VectorStore } from "../vectorStore.js";
6
+ import { VectorStore, vectorList, vectorRepoStats } from "../vectorStore.js";
7
7
  import { normalize } from "../dedup/normalize.js";
8
8
  import { openBloom, closeBloom } from "./bloom.js";
9
9
  import { backfillContentHashes, isBackfillComplete } from "./backfill.js";
@@ -25,7 +25,7 @@ test("Sprint 10 L0: case/whitespace/ANSI variants dedup to one row", () => {
25
25
  if (!r.deduped)
26
26
  added++;
27
27
  }
28
- assert.equal(s.list("sess_norm").length, 1);
28
+ assert.equal(vectorList(s, "sess_norm").length, 1);
29
29
  });
30
30
  test("normalize case-folds so Foo/foo/FOO are equal", () => {
31
31
  assert.equal(normalize("Foo Bar"), normalize("foo bar"));
@@ -63,7 +63,7 @@ test("duplicate add is idempotent (no partial rows, single checkpoint)", () => {
63
63
  const b = s.add({ sessionId: "sess_atom", summary: "x", regionText: raw, timestamp: 2 });
64
64
  assert.equal(a.deduped, false);
65
65
  assert.equal(b.deduped, true);
66
- assert.equal(s.list("sess_atom").length, 1);
66
+ assert.equal(vectorList(s, "sess_atom").length, 1);
67
67
  });
68
68
  // --- Backfill orchestrator -------------------------------------------------
69
69
  test("backfill populates null content_hash rows and is idempotent", () => {
@@ -170,14 +170,14 @@ test("Sprint 10 migration: pre-0.4.2 db gains original_token_estimate and repoSt
170
170
  originalTokenEstimate: 500,
171
171
  timestamp: 1,
172
172
  });
173
- const repo = vs.repoStats();
173
+ const repo = vectorRepoStats(vs);
174
174
  // No "no such column" crash = migration succeeded; totals reflect the new col.
175
175
  assert.equal(repo.checkpointCount, 1);
176
176
  assert.equal(repo.originalTokens, 500, "originalTokens read from migrated column");
177
177
  // Re-open a second time to prove idempotency (column already exists).
178
178
  closeStore(dir);
179
179
  const vs2 = new VectorStore({ dedupSim: 0.9, stateDir: dir });
180
- assert.equal(vs2.repoStats().originalTokens, 500, "stable across re-open");
180
+ assert.equal(vectorRepoStats(vs2).originalTokens, 500, "stable across re-open");
181
181
  });
182
182
  // --- cleanup ---------------------------------------------------------------
183
183
  test("Sprint 10 cleanup", () => {
@@ -12,9 +12,13 @@
12
12
  * infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
13
13
  */
14
14
  import { DatabaseSync } from "node:sqlite";
15
- import { appendFileSync, existsSync, mkdirSync } from "node:fs";
16
15
  import { homedir, tmpdir } from "node:os";
16
+ import { existsSync, mkdirSync } from "node:fs";
17
17
  import { join } from "node:path";
18
+ // Back-compat re-exports: session heartbeat + timeseries helpers moved to
19
+ // global-sessions.ts (file-size split). Existing imports from this module
20
+ // keep resolving; behavior is unchanged.
21
+ export { recordSessionHeartbeat, appendTokenSample, pruneStaleSessions, pruneTokenSamples, readActiveSessions, readSessionTimeseries, clearSessionHeartbeat, } from "./global-sessions.js";
18
22
  /** Resolve the machine-wide index directory (env-overridable). */
19
23
  export function getIndexDir() {
20
24
  const override = process.env.MEGACOMPACT_INDEX_DIR;
@@ -247,176 +251,3 @@ export function countInjectedGlobal(indexDir = getIndexDir()) {
247
251
  const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get();
248
252
  return row?.n ?? 0;
249
253
  }
250
- /** Stable color palette for per-session series (hash-based, no randomness). */
251
- const SESSION_COLORS = [
252
- "#60a5fa", // blue-400
253
- "#34d399", // emerald-400
254
- "#fbbf24", // amber-400
255
- "#f87171", // red-400
256
- "#a78bfa", // violet-400
257
- "#f472b6", // pink-400
258
- "#22d3ee", // cyan-400
259
- "#a3e635", // lime-400
260
- ];
261
- /** Hash a sessionId to a stable color index. */
262
- function sessionColor(sessionId) {
263
- let h = 0;
264
- for (let i = 0; i < sessionId.length; i++) {
265
- h = (h * 31 + sessionId.charCodeAt(i)) | 0;
266
- }
267
- return SESSION_COLORS[Math.abs(h) % SESSION_COLORS.length];
268
- }
269
- /**
270
- * Record (upsert) a session heartbeat. Called from snapshot() on every material
271
- * change. PRIMARY KEY (pid, session_id) means concurrent pi processes each get
272
- * their own row. Non-fatal on conflict (INSERT ... ON CONFLICT DO UPDATE).
273
- */
274
- export function recordSessionHeartbeat(pid, sessionId, repoRoot, stateDir, ctxWindow, indexDir = getIndexDir()) {
275
- const db = openIndexStore(indexDir);
276
- const now = Date.now();
277
- db.prepare(`INSERT INTO session_heartbeats (pid, session_id, repo_root, state_dir, ctx_window, last_seen)
278
- VALUES (@pid, @session_id, @repo_root, @state_dir, @ctx_window, @last_seen)
279
- ON CONFLICT(pid, session_id) DO UPDATE SET
280
- repo_root = excluded.repo_root,
281
- state_dir = excluded.state_dir,
282
- ctx_window = excluded.ctx_window,
283
- last_seen = excluded.last_seen`).run({
284
- pid,
285
- session_id: sessionId,
286
- repo_root: repoRoot,
287
- state_dir: stateDir,
288
- ctx_window: ctxWindow,
289
- last_seen: now,
290
- });
291
- }
292
- /**
293
- * Append a token sample row + optionally a session_sample line to events.log
294
- * (for SSE real-time push via /api/events). The eventsLogPath is optional —
295
- * callers without an events.log (e.g. tests) can omit it.
296
- */
297
- export function appendTokenSample(sessionId, repoRoot, tokens, percent, ctxWindow, eventsLogPath, indexDir = getIndexDir()) {
298
- const db = openIndexStore(indexDir);
299
- const now = Date.now();
300
- db.prepare(`INSERT INTO token_samples (session_id, repo_root, tokens, percent, ctx_window, ts)
301
- VALUES (@session_id, @repo_root, @tokens, @percent, @ctx_window, @ts)`).run({
302
- session_id: sessionId,
303
- repo_root: repoRoot,
304
- tokens,
305
- percent,
306
- ctx_window: ctxWindow,
307
- ts: now,
308
- });
309
- // Step 5: also append a session_sample JSON line to events.log so the
310
- // existing /api/events SSE tail streams it for free (real-time chart push).
311
- // Mirrors the DashboardEmitter events.log append pattern in
312
- // extensions/mega-dashboard.ts:{ event(type, data) }: a JSON object with
313
- // `ts` (ISO 8601 string), `type`, and the event-specific payload. The ISO
314
- // timestamp matches the shape of every other SSE variant (SseSessionSample
315
- // contract; every SSE variant's `ts` is a string); a numeric ms `ts` would
316
- // violate the contract union's "every SSE variant has a ts field of type
317
- // string" invariant and break DashboardEmitter consumers.
318
- if (eventsLogPath) {
319
- try {
320
- const dir = eventsLogPath.includes("/") ? eventsLogPath.slice(0, eventsLogPath.lastIndexOf("/")) : ".";
321
- if (!existsSync(dir))
322
- mkdirSync(dir, { recursive: true });
323
- appendFileSync(eventsLogPath, JSON.stringify({
324
- ts: new Date(now).toISOString(),
325
- type: "session_sample",
326
- sessionId,
327
- tokens,
328
- percent,
329
- }) + "\n");
330
- }
331
- catch {
332
- /* non-fatal: SSE push is best-effort */
333
- }
334
- }
335
- }
336
- /**
337
- * Prune stale session heartbeats (sessions not seen within maxAgeMs).
338
- * Default 30-min retention. Called by /api/sessions.
339
- */
340
- export function pruneStaleSessions(maxAgeMs = 1_800_000, indexDir = getIndexDir()) {
341
- const db = openIndexStore(indexDir);
342
- const cutoff = Date.now() - maxAgeMs;
343
- const result = db.prepare("DELETE FROM session_heartbeats WHERE last_seen < @cutoff").run({ cutoff });
344
- return Number(result.changes);
345
- }
346
- /**
347
- * Prune old token samples older than maxAgeMs. Default 30-min retention.
348
- * Called by /api/sessions/timeseries.
349
- */
350
- export function pruneTokenSamples(maxAgeMs = 1_800_000, indexDir = getIndexDir()) {
351
- const db = openIndexStore(indexDir);
352
- const cutoff = Date.now() - maxAgeMs;
353
- const result = db.prepare("DELETE FROM token_samples WHERE ts < @cutoff").run({ cutoff });
354
- return Number(result.changes);
355
- }
356
- /**
357
- * Read all active sessions with their latest token sample (if any).
358
- * JOINs session_heartbeats with the latest token_samples per session_id
359
- * via a correlated subquery. Returns rows sorted by last_seen descending.
360
- */
361
- export function readActiveSessions(indexDir = getIndexDir()) {
362
- const db = openIndexStore(indexDir);
363
- const rows = db.prepare(`SELECT h.pid, h.session_id, h.repo_root, h.state_dir, h.ctx_window, h.last_seen,
364
- s.tokens, s.percent
365
- FROM session_heartbeats h
366
- LEFT JOIN token_samples s ON s.id = (
367
- SELECT id FROM token_samples t
368
- WHERE t.session_id = h.session_id
369
- ORDER BY t.ts DESC LIMIT 1
370
- )
371
- ORDER BY h.last_seen DESC`).all();
372
- return rows.map((r) => ({
373
- pid: r.pid,
374
- sessionId: r.session_id,
375
- repoRoot: r.repo_root,
376
- stateDir: r.state_dir,
377
- ctxWindow: r.ctx_window ?? 0,
378
- lastSeen: r.last_seen,
379
- tokens: r.tokens,
380
- percent: r.percent,
381
- }));
382
- }
383
- /**
384
- * Read token samples since sinceMs, returning a recharts-ready stacked shape:
385
- * per-session `SessionSeries` (with stable color) + a `totals` array
386
- * [{ts, tokens}] (sum of all sessions at each timestamp).
387
- */
388
- export function readSessionTimeseries(sinceMs, indexDir = getIndexDir()) {
389
- const db = openIndexStore(indexDir);
390
- const rows = db.prepare(`SELECT session_id, tokens, percent, ts FROM token_samples WHERE ts >= @since ORDER BY ts ASC`).all({ since: sinceMs });
391
- // Group by session_id → series; + accumulate totals per timestamp.
392
- const seriesMap = new Map();
393
- const totalsMap = new Map();
394
- for (const r of rows) {
395
- let pts = seriesMap.get(r.session_id);
396
- if (!pts) {
397
- pts = [];
398
- seriesMap.set(r.session_id, pts);
399
- }
400
- pts.push({ ts: r.ts, tokens: r.tokens, percent: r.percent });
401
- totalsMap.set(r.ts, (totalsMap.get(r.ts) ?? 0) + r.tokens);
402
- }
403
- const series = [];
404
- for (const [sessionId, data] of seriesMap) {
405
- const label = sessionId.length > 12 ? sessionId.slice(0, 12) : sessionId;
406
- series.push({ sessionId, label, color: sessionColor(sessionId), data });
407
- }
408
- // Sort series by first-timestamp for stable legend order.
409
- series.sort((a, b) => (a.data[0]?.ts ?? 0) - (b.data[0]?.ts ?? 0));
410
- const totals = Array.from(totalsMap.entries())
411
- .sort((a, b) => a[0] - b[0])
412
- .map(([ts, tokens]) => ({ ts, tokens }));
413
- return { series, totals };
414
- }
415
- /**
416
- * Clear a session's heartbeat row (e.g. on clean shutdown / session reset).
417
- * Non-fatal: no-op if the row doesn't exist.
418
- */
419
- export function clearSessionHeartbeat(pid, sessionId, indexDir = getIndexDir()) {
420
- const db = openIndexStore(indexDir);
421
- db.prepare("DELETE FROM session_heartbeats WHERE pid = @pid AND session_id = @session_id").run({ pid, session_id: sessionId });
422
- }