pi-mega-compact 0.8.21 → 0.8.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +6 -2
- package/README.md +1 -1
- package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
- package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
- package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
- package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
- package/dist/extensions/dashboard-server/html.js +2 -621
- package/dist/extensions/dashboard-server/routes-core.js +62 -0
- package/dist/extensions/dashboard-server/routes-game.js +323 -0
- package/dist/extensions/dashboard-server/routes-repo.js +170 -0
- package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
- package/dist/extensions/dashboard-server/routes.js +10 -0
- package/dist/extensions/dashboard-server/server.js +26 -623
- package/dist/extensions/mega-commands.js +4 -3
- package/dist/extensions/mega-events/agent-handlers.js +2 -1
- package/dist/extensions/mega-events/compact-handlers.js +26 -0
- package/dist/extensions/mega-events/session-handlers.js +2 -1
- package/dist/extensions/mega-pipeline/compact.js +3 -2
- package/dist/extensions/mega-runtime/state.js +7 -7
- package/dist/src/dedup/raptor/multilevel.js +172 -0
- package/dist/src/dedup/raptor/multilevel.test.js +203 -0
- package/dist/src/dedup/raptor/promote.test.js +5 -5
- package/dist/src/dedup/raptor/retrieval.js +1 -1
- package/dist/src/dedup/sprint12.test.js +7 -7
- package/dist/src/dedup-engine.test.js +29 -29
- package/dist/src/e2e.test.js +38 -38
- package/dist/src/engine.js +3 -3
- package/dist/src/engine.test.js +6 -6
- package/dist/src/importance.js +197 -0
- package/dist/src/importance.test.js +372 -0
- package/dist/src/ratio.bench.test.js +18 -18
- package/dist/src/recall.js +6 -5
- package/dist/src/recall.test.js +85 -27
- package/dist/src/sprint14.test.js +2 -2
- package/dist/src/store/migrate.test.js +5 -5
- package/dist/src/store/sprint10.test.js +5 -5
- package/dist/src/store/sqlite/global-index.js +5 -174
- package/dist/src/store/sqlite/global-sessions.js +190 -0
- package/dist/src/vector-read.js +168 -0
- package/dist/src/vector-search.js +191 -0
- package/dist/src/vectorStore.js +10 -297
- package/dist/src/vectorStore.test.js +32 -32
- package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
- package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
- package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
- package/extensions/dashboard-server/dashboard-client.ts +21 -0
- package/extensions/dashboard-server/html.ts +2 -621
- package/extensions/dashboard-server/routes-core.ts +113 -0
- package/extensions/dashboard-server/routes-game.ts +386 -0
- package/extensions/dashboard-server/routes-repo.ts +212 -0
- package/extensions/dashboard-server/routes-sessions.ts +195 -0
- package/extensions/dashboard-server/routes.ts +13 -0
- package/extensions/dashboard-server/server.ts +37 -700
- package/extensions/mega-commands.ts +4 -3
- package/extensions/mega-events/agent-handlers.ts +2 -1
- package/extensions/mega-events/compact-handlers.ts +28 -0
- package/extensions/mega-events/session-handlers.ts +2 -1
- package/extensions/mega-pipeline/compact.ts +3 -2
- package/extensions/mega-runtime/state.ts +7 -7
- package/extensions/openclaw-mega-compact.ts +2 -2
- package/package.json +2 -2
- package/src/dedup/raptor/multilevel.test.ts +278 -0
- package/src/dedup/raptor/multilevel.ts +246 -0
- package/src/dedup/raptor/promote.test.ts +5 -5
- package/src/dedup/raptor/retrieval.ts +1 -1
- package/src/dedup/sprint12.test.ts +7 -7
- package/src/dedup-engine.test.ts +30 -30
- package/src/e2e.test.ts +38 -38
- package/src/engine.test.ts +6 -6
- package/src/engine.ts +3 -3
- package/src/importance.test.ts +538 -0
- package/src/importance.ts +312 -0
- package/src/ratio.bench.test.ts +18 -18
- package/src/recall.test.ts +101 -29
- package/src/recall.ts +9 -9
- package/src/sprint14.test.ts +2 -2
- package/src/store/migrate.test.ts +5 -5
- package/src/store/sprint10.test.ts +5 -5
- package/src/store/sqlite/global-index.ts +18 -290
- package/src/store/sqlite/global-sessions.ts +291 -0
- package/src/vector-read.ts +237 -0
- package/src/vector-search.ts +231 -0
- package/src/vectorStore.test.ts +32 -32
- package/src/vectorStore.ts +29 -356
package/dist/src/recall.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* extension decides where it lands.
|
|
15
15
|
*/
|
|
16
16
|
import { recall as searchRecall } from "./engine.js";
|
|
17
|
+
import { vectorWasInjected, vectorMarkInjected, vectorSearchAsync } from "./vectorStore.js";
|
|
17
18
|
import { estimateBlockTokens } from "./tokens.js";
|
|
18
19
|
import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
|
|
19
20
|
/** Wrap a recall block so the model reads it as restored compacted context. */
|
|
@@ -98,7 +99,7 @@ export function recallAndInline(opts, store) {
|
|
|
98
99
|
const parts = [];
|
|
99
100
|
let blockTokens = 0;
|
|
100
101
|
for (const h of hits) {
|
|
101
|
-
if (skip && store
|
|
102
|
+
if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId))
|
|
102
103
|
continue;
|
|
103
104
|
// Inline dedupe: skip a hit already resident in the live window (Fix C).
|
|
104
105
|
if (doWindowDedupe && liveEmbeddings.length > 0) {
|
|
@@ -114,7 +115,7 @@ export function recallAndInline(opts, store) {
|
|
|
114
115
|
parts.push(part);
|
|
115
116
|
toInject.push(h);
|
|
116
117
|
blockTokens += partTokens;
|
|
117
|
-
store
|
|
118
|
+
vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
|
|
118
119
|
}
|
|
119
120
|
const block = parts.join("\n");
|
|
120
121
|
const report = toInject.map((h) => ` • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
|
|
@@ -210,7 +211,7 @@ export async function recallAndInlineAsync(opts, store) {
|
|
|
210
211
|
const dedupSim = opts.dedupSim ?? 0.9;
|
|
211
212
|
let hits = [];
|
|
212
213
|
try {
|
|
213
|
-
hits = await store
|
|
214
|
+
hits = await vectorSearchAsync(store, opts.sessionId, opts.query, limit, {
|
|
214
215
|
crossRepo: opts.crossRepo,
|
|
215
216
|
repoId: opts.repoId,
|
|
216
217
|
});
|
|
@@ -227,7 +228,7 @@ export async function recallAndInlineAsync(opts, store) {
|
|
|
227
228
|
const parts = [];
|
|
228
229
|
let blockTokens = 0;
|
|
229
230
|
for (const h of hits) {
|
|
230
|
-
if (skip && store
|
|
231
|
+
if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId))
|
|
231
232
|
continue;
|
|
232
233
|
// S18: machine-wide injected-set — a foreign checkpoint already injected
|
|
233
234
|
// (in any session) is never re-injected. Only applies to cross-repo hits
|
|
@@ -254,7 +255,7 @@ export async function recallAndInlineAsync(opts, store) {
|
|
|
254
255
|
parts.push(part);
|
|
255
256
|
toInject.push(h);
|
|
256
257
|
blockTokens += partTokens;
|
|
257
|
-
store
|
|
258
|
+
vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
|
|
258
259
|
// S18: record the cross-repo injection machine-wide so it's not re-injected
|
|
259
260
|
// by a later recall (same or different session).
|
|
260
261
|
if (opts.globalIndexDir && h.repoId) {
|
package/dist/src/recall.test.js
CHANGED
|
@@ -6,7 +6,9 @@ import { join } from "node:path";
|
|
|
6
6
|
import { VectorStore } from "./vectorStore.js";
|
|
7
7
|
import { compactSession } from "./engine.js";
|
|
8
8
|
import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "./recall.js";
|
|
9
|
+
import { vectorList } from "./vectorStore.js";
|
|
9
10
|
import { markInjectedGlobal, wasInjectedGlobal, closeIndexStore } from "./store/sqlite.js";
|
|
11
|
+
import { closeVectorIndex, initVectorIndex, rebuildFromSqlite, } from "./store/vectorIndex.js";
|
|
10
12
|
const baseTmp = mkdtempSync(join(tmpdir(), "mc-recall-"));
|
|
11
13
|
let counter = 0;
|
|
12
14
|
function store() {
|
|
@@ -92,53 +94,109 @@ test("Fix C: inline dedupe drops a hit already resident in the live window", ()
|
|
|
92
94
|
assert.ok(rDedup.toInject.length <= rNoDedup.toInject.length, "dedupe never adds hits");
|
|
93
95
|
assert.ok(rDedup.toInject.length < rNoDedup.toInject.length, "inline dedupe dropped a resident hit");
|
|
94
96
|
});
|
|
97
|
+
// ---- S18 cross-repo global injected-set (real-data, no mocks) ----------------
|
|
98
|
+
//
|
|
99
|
+
// Earlier versions of these tests used `as any` mock stores with canned
|
|
100
|
+
// `searchAsync` returns. That was mock data: it asserted recall's orchestration
|
|
101
|
+
// against a fake search result, not the real embed → HNSW → hydrate → inject
|
|
102
|
+
// path. Per the no-mock-data principle, both tests now seed a REAL foreign
|
|
103
|
+
// VectorStore (real checkpoint + real TrigramEmbedder embedding persisted to
|
|
104
|
+
// SQLite), rebuild the real PGlite index from it via `rebuildFromSqlite`, and
|
|
105
|
+
// run the full `recallAndInlineAsync` cross-repo path against a separate self
|
|
106
|
+
// store. The foreign checkpoint hydrates from the foreign store via its real
|
|
107
|
+
// repoId (== stateDir, per VectorStore's repoId convention).
|
|
108
|
+
async function seedForeignRepo(foreignStateDir) {
|
|
109
|
+
// A real VectorStore at the foreign repo's stateDir. repoId == stateDir.
|
|
110
|
+
const foreign = new VectorStore({ stateDir: foreignStateDir, dedupSim: 0.9 });
|
|
111
|
+
const sess = "sess_foreign";
|
|
112
|
+
// Seed a real checkpoint via the real compactSession pipeline. The summary
|
|
113
|
+
// text determines the embedding; the query below must land near it.
|
|
114
|
+
const summary = "foreign repo authentication jwt token validation";
|
|
115
|
+
const result = compactSession({
|
|
116
|
+
sessionId: sess,
|
|
117
|
+
messages: [msg("user", summary), msg("assistant", "ok", "Edit")],
|
|
118
|
+
keepFrom: 2,
|
|
119
|
+
timestamp: 1,
|
|
120
|
+
}, foreign);
|
|
121
|
+
return { sessionId: sess, checkpointId: result.checkpointId ?? "", summary };
|
|
122
|
+
}
|
|
95
123
|
test("S18: global injected-set skips a foreign checkpoint already injected machine-wide", async () => {
|
|
124
|
+
if (process.env.MEGACOMPACT_PGLITE_DISABLED === "true") {
|
|
125
|
+
return;
|
|
126
|
+
} // skip when WASM index is off
|
|
96
127
|
const indexDir = mkdtempSync(join(tmpdir(), "mc-gi-"));
|
|
128
|
+
const foreignStateDir = mkdtempSync(join(tmpdir(), "mc-gi-foreign-"));
|
|
129
|
+
const selfStateDir = mkdtempSync(join(tmpdir(), "mc-gi-self-"));
|
|
130
|
+
process.env.MEGACOMPACT_VECTOR_INDEX_DIR = mkdtempSync(join(tmpdir(), "mc-gi-vidx-"));
|
|
97
131
|
try {
|
|
132
|
+
await closeVectorIndex(); // fresh singleton per this index dir
|
|
133
|
+
const seed = await seedForeignRepo(foreignStateDir);
|
|
134
|
+
// Populate the real PGlite index from the foreign store's real checkpoints.
|
|
135
|
+
await rebuildFromSqlite(() => [{ repoId: foreignStateDir, stateDir: foreignStateDir }], (sd) => {
|
|
136
|
+
// readCheckpoints: yield (sessionId, checkpointId, embedding) for every
|
|
137
|
+
// real checkpoint in this store. Mirrors the production enumerator.
|
|
138
|
+
const store = new VectorStore({ stateDir: sd, dedupSim: 0.9 });
|
|
139
|
+
return vectorList(store, seed.sessionId).map((cp) => ({
|
|
140
|
+
sessionId: seed.sessionId,
|
|
141
|
+
checkpointId: cp.checkpointId,
|
|
142
|
+
embedding: cp.embedding,
|
|
143
|
+
}));
|
|
144
|
+
});
|
|
145
|
+
const pg = await initVectorIndex();
|
|
146
|
+
assert.ok(pg, "PGlite index should initialize (WASM available)");
|
|
98
147
|
const sess = "sess_cross";
|
|
99
|
-
//
|
|
100
|
-
markInjectedGlobal(
|
|
101
|
-
assert.equal(wasInjectedGlobal(
|
|
102
|
-
//
|
|
103
|
-
|
|
104
|
-
const
|
|
105
|
-
searchAsync: async () => [{
|
|
106
|
-
checkpoint: { checkpointId: "chkpt_foreign", summary: "foreign work", filesModified: [], dedupStatus: "active" },
|
|
107
|
-
score: 0.92,
|
|
108
|
-
repoId: "/repo/other",
|
|
109
|
-
}],
|
|
110
|
-
wasInjected: () => false,
|
|
111
|
-
markInjected: () => { },
|
|
112
|
-
};
|
|
113
|
-
const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, mockStore);
|
|
148
|
+
// Pre-mark the foreign checkpoint as already injected machine-wide.
|
|
149
|
+
markInjectedGlobal(seed.checkpointId, foreignStateDir, sess, indexDir);
|
|
150
|
+
assert.equal(wasInjectedGlobal(seed.checkpointId, sess, indexDir), true);
|
|
151
|
+
// Self store (different repo) — the cross-repo query runs against the index.
|
|
152
|
+
const selfStore = new VectorStore({ stateDir: selfStateDir, dedupSim: 0.9 });
|
|
153
|
+
const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign repo authentication jwt", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, selfStore);
|
|
114
154
|
assert.equal(r.toInject.length, 0, "globally-injected foreign checkpoint skipped");
|
|
115
155
|
}
|
|
116
156
|
finally {
|
|
157
|
+
await closeVectorIndex();
|
|
117
158
|
closeIndexStore();
|
|
159
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
118
160
|
rmSync(indexDir, { recursive: true, force: true });
|
|
161
|
+
rmSync(foreignStateDir, { recursive: true, force: true });
|
|
162
|
+
rmSync(selfStateDir, { recursive: true, force: true });
|
|
119
163
|
}
|
|
120
164
|
});
|
|
121
165
|
test("S18: a fresh foreign checkpoint is injected AND recorded globally", async () => {
|
|
166
|
+
if (process.env.MEGACOMPACT_PGLITE_DISABLED === "true") {
|
|
167
|
+
return;
|
|
168
|
+
} // skip when WASM index is off
|
|
122
169
|
const indexDir = mkdtempSync(join(tmpdir(), "mc-gi2-"));
|
|
170
|
+
const foreignStateDir = mkdtempSync(join(tmpdir(), "mc-gi2-foreign-"));
|
|
171
|
+
const selfStateDir = mkdtempSync(join(tmpdir(), "mc-gi2-self-"));
|
|
172
|
+
process.env.MEGACOMPACT_VECTOR_INDEX_DIR = mkdtempSync(join(tmpdir(), "mc-gi2-vidx-"));
|
|
123
173
|
try {
|
|
174
|
+
await closeVectorIndex(); // fresh singleton per this index dir
|
|
175
|
+
const seed = await seedForeignRepo(foreignStateDir);
|
|
176
|
+
assert.equal(wasInjectedGlobal(seed.checkpointId, "sess_fresh", indexDir), false);
|
|
177
|
+
await rebuildFromSqlite(() => [{ repoId: foreignStateDir, stateDir: foreignStateDir }], (sd) => {
|
|
178
|
+
const store = new VectorStore({ stateDir: sd, dedupSim: 0.9 });
|
|
179
|
+
return vectorList(store, seed.sessionId).map((cp) => ({
|
|
180
|
+
sessionId: seed.sessionId,
|
|
181
|
+
checkpointId: cp.checkpointId,
|
|
182
|
+
embedding: cp.embedding,
|
|
183
|
+
}));
|
|
184
|
+
});
|
|
185
|
+
const pg = await initVectorIndex();
|
|
186
|
+
assert.ok(pg, "PGlite index should initialize (WASM available)");
|
|
124
187
|
const sess = "sess_fresh";
|
|
125
|
-
|
|
126
|
-
const
|
|
127
|
-
searchAsync: async () => [{
|
|
128
|
-
checkpoint: { checkpointId: "chkpt_new", summary: "brand new foreign work", filesModified: [], dedupStatus: "active" },
|
|
129
|
-
score: 0.93,
|
|
130
|
-
repoId: "/repo/alpha",
|
|
131
|
-
}],
|
|
132
|
-
wasInjected: () => false,
|
|
133
|
-
markInjected: () => { },
|
|
134
|
-
};
|
|
135
|
-
const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, mockStore);
|
|
188
|
+
const selfStore = new VectorStore({ stateDir: selfStateDir, dedupSim: 0.9 });
|
|
189
|
+
const r = await recallAndInlineAsync({ sessionId: sess, query: "foreign repo authentication jwt", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir }, selfStore);
|
|
136
190
|
assert.equal(r.toInject.length, 1, "fresh foreign checkpoint injected");
|
|
137
|
-
assert.equal(wasInjectedGlobal(
|
|
191
|
+
assert.equal(wasInjectedGlobal(seed.checkpointId, sess, indexDir), true, "recorded machine-wide");
|
|
138
192
|
}
|
|
139
193
|
finally {
|
|
194
|
+
await closeVectorIndex();
|
|
140
195
|
closeIndexStore();
|
|
196
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
141
197
|
rmSync(indexDir, { recursive: true, force: true });
|
|
198
|
+
rmSync(foreignStateDir, { recursive: true, force: true });
|
|
199
|
+
rmSync(selfStateDir, { recursive: true, force: true });
|
|
142
200
|
}
|
|
143
201
|
});
|
|
144
202
|
test("cleanup", () => {
|
|
@@ -7,7 +7,7 @@ import assert from "node:assert/strict";
|
|
|
7
7
|
import { mkdtempSync, rmSync, readFileSync, existsSync } from "node:fs";
|
|
8
8
|
import { tmpdir } from "node:os";
|
|
9
9
|
import { join } from "node:path";
|
|
10
|
-
import { VectorStore } from "./vectorStore.js";
|
|
10
|
+
import { VectorStore, vectorSearch } from "./vectorStore.js";
|
|
11
11
|
import { defaultEmbedder } from "./embedder.js";
|
|
12
12
|
import { loadDedupConfig } from "./config/dedup.js";
|
|
13
13
|
import { loadMetrics, saveMetrics, recordDecision, evaluateAlerts, fpRate, p95, } from "./monitoring.js";
|
|
@@ -35,7 +35,7 @@ test("flag matrix: all 16 L0/L1/L2/RAPTOR enable combos are safe", () => {
|
|
|
35
35
|
s.add({ sessionId: "s", summary: "x", regionText: `region A for combo ${combos} about the cache`, timestamp: 1 });
|
|
36
36
|
const r2 = s.add({ sessionId: "s", summary: "x", regionText: `region B for combo ${combos} about the parser`, timestamp: 2 });
|
|
37
37
|
// search must not throw under any combination
|
|
38
|
-
const hits = s
|
|
38
|
+
const hits = vectorSearch(s, "s", "cache", 3);
|
|
39
39
|
assert.ok(Array.isArray(hits));
|
|
40
40
|
// With all tiers off, the second add is always "new" (no collapse).
|
|
41
41
|
if (!l0 && !l1 && !l2)
|
|
@@ -15,7 +15,7 @@ import assert from "node:assert/strict";
|
|
|
15
15
|
import { mkdtempSync, rmSync, existsSync } from "node:fs";
|
|
16
16
|
import { tmpdir } from "node:os";
|
|
17
17
|
import { join } from "node:path";
|
|
18
|
-
import { VectorStore } from "../vectorStore.js";
|
|
18
|
+
import { VectorStore, vectorMarkInjected, vectorWasInjected, vectorSearch } from "../vectorStore.js";
|
|
19
19
|
import { writeGzJson } from "../store.js";
|
|
20
20
|
import { migrateJsonToSqlite, readLegacyCheckpointFile } from "../store/migrate.js";
|
|
21
21
|
import { listCheckpoints, closeStore } from "../store/sqlite.js";
|
|
@@ -110,7 +110,7 @@ test("cross-process recall: fresh VectorStore over same dir recalls prior checkp
|
|
|
110
110
|
closeStore(dir);
|
|
111
111
|
// Process B: brand-new VectorStore, same stateDir.
|
|
112
112
|
const b = new VectorStore({ stateDir: dir });
|
|
113
|
-
const hits = b
|
|
113
|
+
const hits = vectorSearch(b, sid, "cross process recall proof", 5);
|
|
114
114
|
assert.equal(hits.length, 1, "checkpoint survives cross-process reopen");
|
|
115
115
|
assert.equal(hits[0].checkpoint.checkpointId, "chkpt_001");
|
|
116
116
|
assert.ok(hits[0].score > 0.5, "recall is relevant");
|
|
@@ -127,11 +127,11 @@ test("cross-process recall: injected state persists across reopen", () => {
|
|
|
127
127
|
regionText: "injected state persists across process reopen test",
|
|
128
128
|
timestamp: 1,
|
|
129
129
|
});
|
|
130
|
-
a
|
|
131
|
-
assert.equal(a
|
|
130
|
+
vectorMarkInjected(a, sid, added.checkpoint.checkpointId);
|
|
131
|
+
assert.equal(vectorWasInjected(a, sid, added.checkpoint.checkpointId), true);
|
|
132
132
|
closeStore(dir);
|
|
133
133
|
const b = new VectorStore({ stateDir: dir });
|
|
134
|
-
assert.equal(b
|
|
134
|
+
assert.equal(vectorWasInjected(b, sid, added.checkpoint.checkpointId), true, "injection remembered");
|
|
135
135
|
closeStore(dir);
|
|
136
136
|
});
|
|
137
137
|
test("cleanup", () => {
|
|
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
|
|
|
3
3
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
|
-
import { VectorStore } from "../vectorStore.js";
|
|
6
|
+
import { VectorStore, vectorList, vectorRepoStats } from "../vectorStore.js";
|
|
7
7
|
import { normalize } from "../dedup/normalize.js";
|
|
8
8
|
import { openBloom, closeBloom } from "./bloom.js";
|
|
9
9
|
import { backfillContentHashes, isBackfillComplete } from "./backfill.js";
|
|
@@ -25,7 +25,7 @@ test("Sprint 10 L0: case/whitespace/ANSI variants dedup to one row", () => {
|
|
|
25
25
|
if (!r.deduped)
|
|
26
26
|
added++;
|
|
27
27
|
}
|
|
28
|
-
assert.equal(s
|
|
28
|
+
assert.equal(vectorList(s, "sess_norm").length, 1);
|
|
29
29
|
});
|
|
30
30
|
test("normalize case-folds so Foo/foo/FOO are equal", () => {
|
|
31
31
|
assert.equal(normalize("Foo Bar"), normalize("foo bar"));
|
|
@@ -63,7 +63,7 @@ test("duplicate add is idempotent (no partial rows, single checkpoint)", () => {
|
|
|
63
63
|
const b = s.add({ sessionId: "sess_atom", summary: "x", regionText: raw, timestamp: 2 });
|
|
64
64
|
assert.equal(a.deduped, false);
|
|
65
65
|
assert.equal(b.deduped, true);
|
|
66
|
-
assert.equal(s
|
|
66
|
+
assert.equal(vectorList(s, "sess_atom").length, 1);
|
|
67
67
|
});
|
|
68
68
|
// --- Backfill orchestrator -------------------------------------------------
|
|
69
69
|
test("backfill populates null content_hash rows and is idempotent", () => {
|
|
@@ -170,14 +170,14 @@ test("Sprint 10 migration: pre-0.4.2 db gains original_token_estimate and repoSt
|
|
|
170
170
|
originalTokenEstimate: 500,
|
|
171
171
|
timestamp: 1,
|
|
172
172
|
});
|
|
173
|
-
const repo = vs
|
|
173
|
+
const repo = vectorRepoStats(vs);
|
|
174
174
|
// No "no such column" crash = migration succeeded; totals reflect the new col.
|
|
175
175
|
assert.equal(repo.checkpointCount, 1);
|
|
176
176
|
assert.equal(repo.originalTokens, 500, "originalTokens read from migrated column");
|
|
177
177
|
// Re-open a second time to prove idempotency (column already exists).
|
|
178
178
|
closeStore(dir);
|
|
179
179
|
const vs2 = new VectorStore({ dedupSim: 0.9, stateDir: dir });
|
|
180
|
-
assert.equal(vs2
|
|
180
|
+
assert.equal(vectorRepoStats(vs2).originalTokens, 500, "stable across re-open");
|
|
181
181
|
});
|
|
182
182
|
// --- cleanup ---------------------------------------------------------------
|
|
183
183
|
test("Sprint 10 cleanup", () => {
|
|
@@ -12,9 +12,13 @@
|
|
|
12
12
|
* infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
|
|
13
13
|
*/
|
|
14
14
|
import { DatabaseSync } from "node:sqlite";
|
|
15
|
-
import { appendFileSync, existsSync, mkdirSync } from "node:fs";
|
|
16
15
|
import { homedir, tmpdir } from "node:os";
|
|
16
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
17
17
|
import { join } from "node:path";
|
|
18
|
+
// Back-compat re-exports: session heartbeat + timeseries helpers moved to
|
|
19
|
+
// global-sessions.ts (file-size split). Existing imports from this module
|
|
20
|
+
// keep resolving; behavior is unchanged.
|
|
21
|
+
export { recordSessionHeartbeat, appendTokenSample, pruneStaleSessions, pruneTokenSamples, readActiveSessions, readSessionTimeseries, clearSessionHeartbeat, } from "./global-sessions.js";
|
|
18
22
|
/** Resolve the machine-wide index directory (env-overridable). */
|
|
19
23
|
export function getIndexDir() {
|
|
20
24
|
const override = process.env.MEGACOMPACT_INDEX_DIR;
|
|
@@ -247,176 +251,3 @@ export function countInjectedGlobal(indexDir = getIndexDir()) {
|
|
|
247
251
|
const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get();
|
|
248
252
|
return row?.n ?? 0;
|
|
249
253
|
}
|
|
250
|
-
/** Stable color palette for per-session series (hash-based, no randomness). */
|
|
251
|
-
const SESSION_COLORS = [
|
|
252
|
-
"#60a5fa", // blue-400
|
|
253
|
-
"#34d399", // emerald-400
|
|
254
|
-
"#fbbf24", // amber-400
|
|
255
|
-
"#f87171", // red-400
|
|
256
|
-
"#a78bfa", // violet-400
|
|
257
|
-
"#f472b6", // pink-400
|
|
258
|
-
"#22d3ee", // cyan-400
|
|
259
|
-
"#a3e635", // lime-400
|
|
260
|
-
];
|
|
261
|
-
/** Hash a sessionId to a stable color index. */
|
|
262
|
-
function sessionColor(sessionId) {
|
|
263
|
-
let h = 0;
|
|
264
|
-
for (let i = 0; i < sessionId.length; i++) {
|
|
265
|
-
h = (h * 31 + sessionId.charCodeAt(i)) | 0;
|
|
266
|
-
}
|
|
267
|
-
return SESSION_COLORS[Math.abs(h) % SESSION_COLORS.length];
|
|
268
|
-
}
|
|
269
|
-
/**
|
|
270
|
-
* Record (upsert) a session heartbeat. Called from snapshot() on every material
|
|
271
|
-
* change. PRIMARY KEY (pid, session_id) means concurrent pi processes each get
|
|
272
|
-
* their own row. Non-fatal on conflict (INSERT ... ON CONFLICT DO UPDATE).
|
|
273
|
-
*/
|
|
274
|
-
export function recordSessionHeartbeat(pid, sessionId, repoRoot, stateDir, ctxWindow, indexDir = getIndexDir()) {
|
|
275
|
-
const db = openIndexStore(indexDir);
|
|
276
|
-
const now = Date.now();
|
|
277
|
-
db.prepare(`INSERT INTO session_heartbeats (pid, session_id, repo_root, state_dir, ctx_window, last_seen)
|
|
278
|
-
VALUES (@pid, @session_id, @repo_root, @state_dir, @ctx_window, @last_seen)
|
|
279
|
-
ON CONFLICT(pid, session_id) DO UPDATE SET
|
|
280
|
-
repo_root = excluded.repo_root,
|
|
281
|
-
state_dir = excluded.state_dir,
|
|
282
|
-
ctx_window = excluded.ctx_window,
|
|
283
|
-
last_seen = excluded.last_seen`).run({
|
|
284
|
-
pid,
|
|
285
|
-
session_id: sessionId,
|
|
286
|
-
repo_root: repoRoot,
|
|
287
|
-
state_dir: stateDir,
|
|
288
|
-
ctx_window: ctxWindow,
|
|
289
|
-
last_seen: now,
|
|
290
|
-
});
|
|
291
|
-
}
|
|
292
|
-
/**
|
|
293
|
-
* Append a token sample row + optionally a session_sample line to events.log
|
|
294
|
-
* (for SSE real-time push via /api/events). The eventsLogPath is optional —
|
|
295
|
-
* callers without an events.log (e.g. tests) can omit it.
|
|
296
|
-
*/
|
|
297
|
-
export function appendTokenSample(sessionId, repoRoot, tokens, percent, ctxWindow, eventsLogPath, indexDir = getIndexDir()) {
|
|
298
|
-
const db = openIndexStore(indexDir);
|
|
299
|
-
const now = Date.now();
|
|
300
|
-
db.prepare(`INSERT INTO token_samples (session_id, repo_root, tokens, percent, ctx_window, ts)
|
|
301
|
-
VALUES (@session_id, @repo_root, @tokens, @percent, @ctx_window, @ts)`).run({
|
|
302
|
-
session_id: sessionId,
|
|
303
|
-
repo_root: repoRoot,
|
|
304
|
-
tokens,
|
|
305
|
-
percent,
|
|
306
|
-
ctx_window: ctxWindow,
|
|
307
|
-
ts: now,
|
|
308
|
-
});
|
|
309
|
-
// Step 5: also append a session_sample JSON line to events.log so the
|
|
310
|
-
// existing /api/events SSE tail streams it for free (real-time chart push).
|
|
311
|
-
// Mirrors the DashboardEmitter events.log append pattern in
|
|
312
|
-
// extensions/mega-dashboard.ts:{ event(type, data) }: a JSON object with
|
|
313
|
-
// `ts` (ISO 8601 string), `type`, and the event-specific payload. The ISO
|
|
314
|
-
// timestamp matches the shape of every other SSE variant (SseSessionSample
|
|
315
|
-
// contract; every SSE variant's `ts` is a string); a numeric ms `ts` would
|
|
316
|
-
// violate the contract union's "every SSE variant has a ts field of type
|
|
317
|
-
// string" invariant and break DashboardEmitter consumers.
|
|
318
|
-
if (eventsLogPath) {
|
|
319
|
-
try {
|
|
320
|
-
const dir = eventsLogPath.includes("/") ? eventsLogPath.slice(0, eventsLogPath.lastIndexOf("/")) : ".";
|
|
321
|
-
if (!existsSync(dir))
|
|
322
|
-
mkdirSync(dir, { recursive: true });
|
|
323
|
-
appendFileSync(eventsLogPath, JSON.stringify({
|
|
324
|
-
ts: new Date(now).toISOString(),
|
|
325
|
-
type: "session_sample",
|
|
326
|
-
sessionId,
|
|
327
|
-
tokens,
|
|
328
|
-
percent,
|
|
329
|
-
}) + "\n");
|
|
330
|
-
}
|
|
331
|
-
catch {
|
|
332
|
-
/* non-fatal: SSE push is best-effort */
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
/**
|
|
337
|
-
* Prune stale session heartbeats (sessions not seen within maxAgeMs).
|
|
338
|
-
* Default 30-min retention. Called by /api/sessions.
|
|
339
|
-
*/
|
|
340
|
-
export function pruneStaleSessions(maxAgeMs = 1_800_000, indexDir = getIndexDir()) {
|
|
341
|
-
const db = openIndexStore(indexDir);
|
|
342
|
-
const cutoff = Date.now() - maxAgeMs;
|
|
343
|
-
const result = db.prepare("DELETE FROM session_heartbeats WHERE last_seen < @cutoff").run({ cutoff });
|
|
344
|
-
return Number(result.changes);
|
|
345
|
-
}
|
|
346
|
-
/**
|
|
347
|
-
* Prune old token samples older than maxAgeMs. Default 30-min retention.
|
|
348
|
-
* Called by /api/sessions/timeseries.
|
|
349
|
-
*/
|
|
350
|
-
export function pruneTokenSamples(maxAgeMs = 1_800_000, indexDir = getIndexDir()) {
|
|
351
|
-
const db = openIndexStore(indexDir);
|
|
352
|
-
const cutoff = Date.now() - maxAgeMs;
|
|
353
|
-
const result = db.prepare("DELETE FROM token_samples WHERE ts < @cutoff").run({ cutoff });
|
|
354
|
-
return Number(result.changes);
|
|
355
|
-
}
|
|
356
|
-
/**
|
|
357
|
-
* Read all active sessions with their latest token sample (if any).
|
|
358
|
-
* JOINs session_heartbeats with the latest token_samples per session_id
|
|
359
|
-
* via a correlated subquery. Returns rows sorted by last_seen descending.
|
|
360
|
-
*/
|
|
361
|
-
export function readActiveSessions(indexDir = getIndexDir()) {
|
|
362
|
-
const db = openIndexStore(indexDir);
|
|
363
|
-
const rows = db.prepare(`SELECT h.pid, h.session_id, h.repo_root, h.state_dir, h.ctx_window, h.last_seen,
|
|
364
|
-
s.tokens, s.percent
|
|
365
|
-
FROM session_heartbeats h
|
|
366
|
-
LEFT JOIN token_samples s ON s.id = (
|
|
367
|
-
SELECT id FROM token_samples t
|
|
368
|
-
WHERE t.session_id = h.session_id
|
|
369
|
-
ORDER BY t.ts DESC LIMIT 1
|
|
370
|
-
)
|
|
371
|
-
ORDER BY h.last_seen DESC`).all();
|
|
372
|
-
return rows.map((r) => ({
|
|
373
|
-
pid: r.pid,
|
|
374
|
-
sessionId: r.session_id,
|
|
375
|
-
repoRoot: r.repo_root,
|
|
376
|
-
stateDir: r.state_dir,
|
|
377
|
-
ctxWindow: r.ctx_window ?? 0,
|
|
378
|
-
lastSeen: r.last_seen,
|
|
379
|
-
tokens: r.tokens,
|
|
380
|
-
percent: r.percent,
|
|
381
|
-
}));
|
|
382
|
-
}
|
|
383
|
-
/**
|
|
384
|
-
* Read token samples since sinceMs, returning a recharts-ready stacked shape:
|
|
385
|
-
* per-session `SessionSeries` (with stable color) + a `totals` array
|
|
386
|
-
* [{ts, tokens}] (sum of all sessions at each timestamp).
|
|
387
|
-
*/
|
|
388
|
-
export function readSessionTimeseries(sinceMs, indexDir = getIndexDir()) {
|
|
389
|
-
const db = openIndexStore(indexDir);
|
|
390
|
-
const rows = db.prepare(`SELECT session_id, tokens, percent, ts FROM token_samples WHERE ts >= @since ORDER BY ts ASC`).all({ since: sinceMs });
|
|
391
|
-
// Group by session_id → series; + accumulate totals per timestamp.
|
|
392
|
-
const seriesMap = new Map();
|
|
393
|
-
const totalsMap = new Map();
|
|
394
|
-
for (const r of rows) {
|
|
395
|
-
let pts = seriesMap.get(r.session_id);
|
|
396
|
-
if (!pts) {
|
|
397
|
-
pts = [];
|
|
398
|
-
seriesMap.set(r.session_id, pts);
|
|
399
|
-
}
|
|
400
|
-
pts.push({ ts: r.ts, tokens: r.tokens, percent: r.percent });
|
|
401
|
-
totalsMap.set(r.ts, (totalsMap.get(r.ts) ?? 0) + r.tokens);
|
|
402
|
-
}
|
|
403
|
-
const series = [];
|
|
404
|
-
for (const [sessionId, data] of seriesMap) {
|
|
405
|
-
const label = sessionId.length > 12 ? sessionId.slice(0, 12) : sessionId;
|
|
406
|
-
series.push({ sessionId, label, color: sessionColor(sessionId), data });
|
|
407
|
-
}
|
|
408
|
-
// Sort series by first-timestamp for stable legend order.
|
|
409
|
-
series.sort((a, b) => (a.data[0]?.ts ?? 0) - (b.data[0]?.ts ?? 0));
|
|
410
|
-
const totals = Array.from(totalsMap.entries())
|
|
411
|
-
.sort((a, b) => a[0] - b[0])
|
|
412
|
-
.map(([ts, tokens]) => ({ ts, tokens }));
|
|
413
|
-
return { series, totals };
|
|
414
|
-
}
|
|
415
|
-
/**
|
|
416
|
-
* Clear a session's heartbeat row (e.g. on clean shutdown / session reset).
|
|
417
|
-
* Non-fatal: no-op if the row doesn't exist.
|
|
418
|
-
*/
|
|
419
|
-
export function clearSessionHeartbeat(pid, sessionId, indexDir = getIndexDir()) {
|
|
420
|
-
const db = openIndexStore(indexDir);
|
|
421
|
-
db.prepare("DELETE FROM session_heartbeats WHERE pid = @pid AND session_id = @session_id").run({ pid, session_id: sessionId });
|
|
422
|
-
}
|