pi-mega-compact 0.8.21 → 0.8.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +6 -2
- package/README.md +1 -1
- package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
- package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
- package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
- package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
- package/dist/extensions/dashboard-server/html.js +2 -621
- package/dist/extensions/dashboard-server/routes-core.js +62 -0
- package/dist/extensions/dashboard-server/routes-game.js +323 -0
- package/dist/extensions/dashboard-server/routes-repo.js +170 -0
- package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
- package/dist/extensions/dashboard-server/routes.js +10 -0
- package/dist/extensions/dashboard-server/server.js +26 -623
- package/dist/extensions/mega-commands.js +4 -3
- package/dist/extensions/mega-events/agent-handlers.js +2 -1
- package/dist/extensions/mega-events/compact-handlers.js +26 -0
- package/dist/extensions/mega-events/session-handlers.js +2 -1
- package/dist/extensions/mega-pipeline/compact.js +3 -2
- package/dist/extensions/mega-runtime/state.js +7 -7
- package/dist/src/dedup/raptor/multilevel.js +172 -0
- package/dist/src/dedup/raptor/multilevel.test.js +203 -0
- package/dist/src/dedup/raptor/promote.test.js +5 -5
- package/dist/src/dedup/raptor/retrieval.js +1 -1
- package/dist/src/dedup/sprint12.test.js +7 -7
- package/dist/src/dedup-engine.test.js +29 -29
- package/dist/src/e2e.test.js +38 -38
- package/dist/src/engine.js +3 -3
- package/dist/src/engine.test.js +6 -6
- package/dist/src/importance.js +197 -0
- package/dist/src/importance.test.js +372 -0
- package/dist/src/ratio.bench.test.js +18 -18
- package/dist/src/recall.js +6 -5
- package/dist/src/recall.test.js +85 -27
- package/dist/src/sprint14.test.js +2 -2
- package/dist/src/store/migrate.test.js +5 -5
- package/dist/src/store/sprint10.test.js +5 -5
- package/dist/src/store/sqlite/global-index.js +5 -174
- package/dist/src/store/sqlite/global-sessions.js +190 -0
- package/dist/src/vector-read.js +168 -0
- package/dist/src/vector-search.js +191 -0
- package/dist/src/vectorStore.js +10 -297
- package/dist/src/vectorStore.test.js +32 -32
- package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
- package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
- package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
- package/extensions/dashboard-server/dashboard-client.ts +21 -0
- package/extensions/dashboard-server/html.ts +2 -621
- package/extensions/dashboard-server/routes-core.ts +113 -0
- package/extensions/dashboard-server/routes-game.ts +386 -0
- package/extensions/dashboard-server/routes-repo.ts +212 -0
- package/extensions/dashboard-server/routes-sessions.ts +195 -0
- package/extensions/dashboard-server/routes.ts +13 -0
- package/extensions/dashboard-server/server.ts +37 -700
- package/extensions/mega-commands.ts +4 -3
- package/extensions/mega-events/agent-handlers.ts +2 -1
- package/extensions/mega-events/compact-handlers.ts +28 -0
- package/extensions/mega-events/session-handlers.ts +2 -1
- package/extensions/mega-pipeline/compact.ts +3 -2
- package/extensions/mega-runtime/state.ts +7 -7
- package/extensions/openclaw-mega-compact.ts +2 -2
- package/package.json +2 -2
- package/src/dedup/raptor/multilevel.test.ts +278 -0
- package/src/dedup/raptor/multilevel.ts +246 -0
- package/src/dedup/raptor/promote.test.ts +5 -5
- package/src/dedup/raptor/retrieval.ts +1 -1
- package/src/dedup/sprint12.test.ts +7 -7
- package/src/dedup-engine.test.ts +30 -30
- package/src/e2e.test.ts +38 -38
- package/src/engine.test.ts +6 -6
- package/src/engine.ts +3 -3
- package/src/importance.test.ts +538 -0
- package/src/importance.ts +312 -0
- package/src/ratio.bench.test.ts +18 -18
- package/src/recall.test.ts +101 -29
- package/src/recall.ts +9 -9
- package/src/sprint14.test.ts +2 -2
- package/src/store/migrate.test.ts +5 -5
- package/src/store/sprint10.test.ts +5 -5
- package/src/store/sqlite/global-index.ts +18 -290
- package/src/store/sqlite/global-sessions.ts +291 -0
- package/src/vector-read.ts +237 -0
- package/src/vector-search.ts +231 -0
- package/src/vectorStore.test.ts +32 -32
- package/src/vectorStore.ts +29 -356
package/dist/src/vectorStore.js
CHANGED
|
@@ -16,13 +16,8 @@ import { computeContentDigest } from "./dedup/digest.js";
|
|
|
16
16
|
import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "./dedup/l1-minhash.js";
|
|
17
17
|
import { lshBands } from "./dedup/l1-lsh.js";
|
|
18
18
|
import { isNearDuplicate } from "./dedup/l1-verify.js";
|
|
19
|
-
import { mmrRerank } from "./dedup/mmr.js";
|
|
20
|
-
import { topK } from "./dedup/topk.js";
|
|
21
19
|
import { openBloom, saveBloom } from "./store/bloom.js";
|
|
22
|
-
import { listCheckpoints, nextCheckpointId, upsertCheckpoint,
|
|
23
|
-
import { initVectorIndex, searchAsync as vectorIndexSearch, } from "./store/vectorIndex.js";
|
|
24
|
-
import { rehydrateRaptorTree, isShadowMode } from "./dedup/raptor/index.js";
|
|
25
|
-
import { stagedExpansion } from "./dedup/raptor/retrieval.js";
|
|
20
|
+
import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, addTokensSaved, bumpDedupStats, } from "./store/sqlite.js";
|
|
26
21
|
import { migrateJsonToSqlite } from "./store/migrate.js";
|
|
27
22
|
/** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
|
|
28
23
|
export const L2_ENABLED = true;
|
|
@@ -33,6 +28,11 @@ export function computeRegionHash(regionText) {
|
|
|
33
28
|
return createHash("sha256").update(normalized).digest("hex").slice(0, 16);
|
|
34
29
|
}
|
|
35
30
|
export class VectorStore {
|
|
31
|
+
// These fields are `readonly` (set once in the constructor) but NOT private:
|
|
32
|
+
// the read/search/dedup helpers split into vector-read.ts, vector-search.ts,
|
|
33
|
+
// and vector-dedup.ts access them directly. Marking them private would force
|
|
34
|
+
// ugly `as unknown as` casts in those modules; keeping them package-public
|
|
35
|
+
// makes VectorStore a thin barrel whose helpers live in sibling files.
|
|
36
36
|
embedder;
|
|
37
37
|
stateDir;
|
|
38
38
|
l2Threshold;
|
|
@@ -327,295 +327,8 @@ export class VectorStore {
|
|
|
327
327
|
}
|
|
328
328
|
return undefined;
|
|
329
329
|
}
|
|
330
|
-
/**
|
|
331
|
-
* Semantic search within a session's checkpoints. Returns top-K by cosine
|
|
332
|
-
* similarity, diversified via MMR (QA #10) so a cluster of near-identical
|
|
333
|
-
* hits yields at most a few distinct-relevance results.
|
|
334
|
-
*
|
|
335
|
-
* Heap-based top-K (QA #4, O(N log k)) replaces the old full sort; MMR then
|
|
336
|
-
* reranks the candidate window for diversity.
|
|
337
|
-
*/
|
|
338
|
-
search(sessionId, query, k = 3) {
|
|
339
|
-
const sid = normalizeSessionId(sessionId);
|
|
340
|
-
const checkpoints = listCheckpoints(sid, this.stateDir).filter((cp) => cp.dedupStatus !== "removed");
|
|
341
|
-
if (checkpoints.length === 0)
|
|
342
|
-
return [];
|
|
343
|
-
const qv = this.embedder.embed(query);
|
|
344
|
-
const scored = checkpoints.map((cp) => ({
|
|
345
|
-
checkpoint: cp,
|
|
346
|
-
score: cosineSimilarity(qv, cp.embedding),
|
|
347
|
-
}));
|
|
348
|
-
// Heap top-K over a widened window (2k) so MMR has diverse candidates.
|
|
349
|
-
const window = topK(scored.map((h) => ({ item: h, score: h.score })), Math.max(k * 2, k)).map((s) => s.item);
|
|
350
|
-
// MMR (QA #10) is part of the L2 semantic tier: skip it when L2 is disabled
|
|
351
|
-
// (Sprint 14 flag), returning the plain relevance-ranked window instead.
|
|
352
|
-
if (!this.cfg.L2_ENABLED)
|
|
353
|
-
return window.slice(0, k);
|
|
354
|
-
// Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
|
|
355
|
-
// merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
|
|
356
|
-
// RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
|
|
357
|
-
// O(n) flat leaves, tightening the block at read time.
|
|
358
|
-
if (this.cfg.RAPTOR_ENABLED) {
|
|
359
|
-
const raptorHits = this.raptorSearchHits(sid, query, k);
|
|
360
|
-
if (raptorHits.length > 0) {
|
|
361
|
-
const merged = [...window];
|
|
362
|
-
for (const rh of raptorHits) {
|
|
363
|
-
if (!merged.some((m) => m.checkpoint.checkpointId === rh.checkpoint.checkpointId)) {
|
|
364
|
-
merged.push(rh);
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
const mmrItems = merged.map((h) => ({
|
|
368
|
-
item: h,
|
|
369
|
-
vector: h.checkpoint.embedding,
|
|
370
|
-
relevance: h.score,
|
|
371
|
-
}));
|
|
372
|
-
return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
|
|
373
|
-
}
|
|
374
|
-
}
|
|
375
|
-
const mmrItems = window.map((h) => ({
|
|
376
|
-
item: h,
|
|
377
|
-
vector: h.checkpoint.embedding,
|
|
378
|
-
relevance: h.score,
|
|
379
|
-
}));
|
|
380
|
-
const ranked = mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
|
|
381
|
-
return ranked;
|
|
382
|
-
}
|
|
383
|
-
/**
|
|
384
|
-
* Slice 2: async cross-repo (or single-repo) recall via the PGlite/HNSW index.
|
|
385
|
-
*
|
|
386
|
-
* This is the ONLY async recall surface and is a BONUS path — the synchronous
|
|
387
|
-
* `search()` above remains the default. `opts.repoId` scopes to one repo; omit
|
|
388
|
-
* it for cross-repo nearest-neighbor recall (the headline capability the sync
|
|
389
|
-
* per-session scan cannot provide).
|
|
390
|
-
*
|
|
391
|
-
* Best-effort: if the index is disabled/empty/failing, we fall back to the
|
|
392
|
-
* synchronous per-session `search()` for THIS repo so callers always get a
|
|
393
|
-
* sensible result. Hydrates each hit's StoredCheckpoint from the authoritative
|
|
394
|
-
* node:sqlite store (the hit's repoId doubles as that repo's stateDir), then
|
|
395
|
-
* MMR-dedupes the merged set.
|
|
396
|
-
*/
|
|
397
|
-
async searchAsync(sessionId, query, k = 3, opts = {}) {
|
|
398
|
-
const sid = normalizeSessionId(sessionId);
|
|
399
|
-
const qv = this.embedder.embed(query);
|
|
400
|
-
// repoId filter: explicit opts.repoId wins; else this repo unless crossRepo.
|
|
401
|
-
const repoId = opts.repoId ?? (opts.crossRepo ? undefined : this.repoId);
|
|
402
|
-
let indexHits = [];
|
|
403
|
-
try {
|
|
404
|
-
await initVectorIndex();
|
|
405
|
-
indexHits = await vectorIndexSearch(qv, { k: Math.max(k * 2, k), repoId });
|
|
406
|
-
}
|
|
407
|
-
catch {
|
|
408
|
-
indexHits = [];
|
|
409
|
-
}
|
|
410
|
-
if (indexHits.length === 0) {
|
|
411
|
-
// Index empty/unavailable → synchronous per-session fallback (this repo).
|
|
412
|
-
return this.search(sid, query, k);
|
|
413
|
-
}
|
|
414
|
-
// Hydrate each index hit from the authoritative node:sqlite store. repoId is
|
|
415
|
-
// that repo's stateDir, so cross-repo hits resolve against their own store.
|
|
416
|
-
// Tag cross-repo hits with their source repoId so the recall block can label
|
|
417
|
-
// them ("from repo <name>"); same-repo hits stay unlabeled.
|
|
418
|
-
const selfRepo = this.repoId;
|
|
419
|
-
const hydrated = [];
|
|
420
|
-
for (const h of indexHits) {
|
|
421
|
-
const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
|
|
422
|
-
if (cp && cp.dedupStatus !== "removed") {
|
|
423
|
-
const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
|
|
424
|
-
hydrated.push({ checkpoint: cp, score: h.score, repoId: crossRepo ? h.repoId : undefined });
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
if (hydrated.length === 0)
|
|
428
|
-
return this.search(sid, query, k);
|
|
429
|
-
// MMR-dedupe the merged candidate set for diversity (mirrors sync search).
|
|
430
|
-
const mmrItems = hydrated.map((h) => ({
|
|
431
|
-
item: h,
|
|
432
|
-
vector: h.checkpoint.embedding,
|
|
433
|
-
relevance: h.score,
|
|
434
|
-
}));
|
|
435
|
-
return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
|
|
436
|
-
}
|
|
437
|
-
/**
|
|
438
|
-
* Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
|
|
439
|
-
* return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
|
|
440
|
-
* exists (small sessions — flat search remains the path). Best-effort/non-fatal.
|
|
441
|
-
*/
|
|
442
|
-
raptorSearchHits(sid, query, k) {
|
|
443
|
-
const t0 = Date.now();
|
|
444
|
-
try {
|
|
445
|
-
// S25 gate (a): honor the shadow contract at SERVE time. The tree is still
|
|
446
|
-
// built + persisted (logging-only) but NOT merged into recall while
|
|
447
|
-
// RAPTOR_SHADOW_MODE is anything other than "false".
|
|
448
|
-
if (isShadowMode())
|
|
449
|
-
return [];
|
|
450
|
-
const tree = rehydrateRaptorTree(sid, this.stateDir);
|
|
451
|
-
if (!tree || !tree.rootId)
|
|
452
|
-
return [];
|
|
453
|
-
// S25 gate (b): freshness + fallback guards. Skip a tree built before the
|
|
454
|
-
// newest checkpoint (stale → may reference trimmed/deduped leaves) or one
|
|
455
|
-
// whose root is a budget-exhausted extractive fallback (level 99).
|
|
456
|
-
if (tree.timedOut)
|
|
457
|
-
return [];
|
|
458
|
-
const maxTs = maxCheckpointTimestamp(sid, this.stateDir);
|
|
459
|
-
if (tree.builtAt && tree.builtAt < maxTs)
|
|
460
|
-
return [];
|
|
461
|
-
const leafIds = stagedExpansion(query, tree, {
|
|
462
|
-
embedder: this.embedder,
|
|
463
|
-
k,
|
|
464
|
-
topM: this.cfg.RAPTOR_CLUSTERS_PER_LEVEL,
|
|
465
|
-
mmrLambda: this.cfg.MMR_LAMBDA,
|
|
466
|
-
});
|
|
467
|
-
if (leafIds.length === 0)
|
|
468
|
-
return [];
|
|
469
|
-
const all = listCheckpoints(sid, this.stateDir).filter((cp) => cp.dedupStatus !== "removed");
|
|
470
|
-
const qv = this.embedder.embed(query);
|
|
471
|
-
const hits = [];
|
|
472
|
-
for (const id of leafIds) {
|
|
473
|
-
const cp = all.find((c) => c.checkpointId === id);
|
|
474
|
-
if (cp)
|
|
475
|
-
hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
|
|
476
|
-
}
|
|
477
|
-
// S25 monitoring: emit a raptor_serve decision so canary.ts can track
|
|
478
|
-
// p95 latency + the tier's live traffic (non-fatal, best-effort).
|
|
479
|
-
this.record("RAPTOR", hits.length > 0 ? "new" : "mark_only", `leaves=${leafIds.length}`, Date.now() - t0);
|
|
480
|
-
return hits;
|
|
481
|
-
}
|
|
482
|
-
catch {
|
|
483
|
-
return [];
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
/**
|
|
487
|
-
* SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
|
|
488
|
-
* lower-quality row of any pair scoring cosine > `threshold` as
|
|
489
|
-
* `dedup_status='removed'` (kept, not deleted — retrieval excludes it). Keeps
|
|
490
|
-
* the row with the higher `tokenEstimate` (more context preserved). Runs as a
|
|
491
|
-
* single scan; idempotent (re-running skips already-removed rows).
|
|
492
|
-
*
|
|
493
|
-
* Returns the number of rows marked removed.
|
|
494
|
-
*/
|
|
495
|
-
semDedup(sessionId, threshold = this.cfg.SEMDEDUP_COSINE) {
|
|
496
|
-
const sid = normalizeSessionId(sessionId);
|
|
497
|
-
const cps = listCheckpoints(sid, this.stateDir).filter((c) => c.dedupStatus !== "removed");
|
|
498
|
-
let removed = 0;
|
|
499
|
-
for (let i = 0; i < cps.length; i++) {
|
|
500
|
-
for (let j = i + 1; j < cps.length; j++) {
|
|
501
|
-
const a = cps[i];
|
|
502
|
-
const b = cps[j];
|
|
503
|
-
if (a.dedupStatus === "removed" || b.dedupStatus === "removed")
|
|
504
|
-
continue;
|
|
505
|
-
if (cosineSimilarity(a.embedding, b.embedding) > threshold) {
|
|
506
|
-
// Keep the higher-tokenEstimate row; remove the other.
|
|
507
|
-
const keep = a.tokenEstimate >= b.tokenEstimate ? a : b;
|
|
508
|
-
const drop = keep === a ? b : a;
|
|
509
|
-
setDedupStatus(drop.checkpointId, sid, "removed", this.stateDir);
|
|
510
|
-
drop.dedupStatus = "removed";
|
|
511
|
-
removed++;
|
|
512
|
-
}
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
|
-
return removed;
|
|
516
|
-
}
|
|
517
|
-
/**
|
|
518
|
-
* Dedup sentinel check: has this region already been stored/represented?
|
|
519
|
-
* Consulted by both the persist path and the recall/inline path.
|
|
520
|
-
*/
|
|
521
|
-
dedupe(sessionId, regionHashOrText, isText = false) {
|
|
522
|
-
const sid = normalizeSessionId(sessionId);
|
|
523
|
-
const hash = isText
|
|
524
|
-
? computeRegionHash(regionHashOrText)
|
|
525
|
-
: regionHashOrText;
|
|
526
|
-
const state = loadSessionState(sid, this.stateDir);
|
|
527
|
-
if (state.storedRegionHashes.includes(hash))
|
|
528
|
-
return true;
|
|
529
|
-
return listCheckpoints(sid, this.stateDir).some((c) => c.regionHash === hash);
|
|
530
|
-
}
|
|
531
|
-
/** Mark a checkpoint as injected into the window (recall dedup). */
|
|
532
|
-
markInjected(sessionId, checkpointId) {
|
|
533
|
-
const sid = normalizeSessionId(sessionId);
|
|
534
|
-
const state = loadSessionState(sid, this.stateDir);
|
|
535
|
-
if (!state.injectedCheckpointIds.includes(checkpointId)) {
|
|
536
|
-
state.injectedCheckpointIds.push(checkpointId);
|
|
537
|
-
saveSessionState(sid, state, this.stateDir);
|
|
538
|
-
}
|
|
539
|
-
}
|
|
540
|
-
/** True if this checkpoint was already injected this session. */
|
|
541
|
-
wasInjected(sessionId, checkpointId) {
|
|
542
|
-
const state = loadSessionState(normalizeSessionId(sessionId), this.stateDir);
|
|
543
|
-
return state.injectedCheckpointIds.includes(checkpointId);
|
|
544
|
-
}
|
|
545
|
-
/** Convenience for a raw vector cosine (exposed for tests). */
|
|
546
|
-
similarity(a, b) {
|
|
547
|
-
return cosineSimilarity(a, b);
|
|
548
|
-
}
|
|
549
|
-
/** All checkpoints for a session (sorted by checkpointId). */
|
|
550
|
-
list(sessionId) {
|
|
551
|
-
return listCheckpoints(normalizeSessionId(sessionId), this.stateDir);
|
|
552
|
-
}
|
|
553
|
-
/**
|
|
554
|
-
* Return the n most similar checkpoints to the current (most recent) checkpoint
|
|
555
|
-
* by cosine similarity. Returns fewer than n if the session has fewer checkpoints.
|
|
556
|
-
* The current checkpoint itself is excluded from results.
|
|
557
|
-
*/
|
|
558
|
-
topSimilar(sessionId, n) {
|
|
559
|
-
const sid = normalizeSessionId(sessionId);
|
|
560
|
-
const checkpoints = listCheckpoints(sid, this.stateDir);
|
|
561
|
-
if (checkpoints.length <= 1)
|
|
562
|
-
return [];
|
|
563
|
-
// Find the most recent checkpoint (by checkpointId, which is sequential)
|
|
564
|
-
const ordered = [...checkpoints].sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
|
|
565
|
-
const current = ordered[ordered.length - 1];
|
|
566
|
-
// Score all other checkpoints by similarity to current
|
|
567
|
-
const scored = ordered
|
|
568
|
-
.filter((cp) => cp.checkpointId !== current.checkpointId)
|
|
569
|
-
.map((cp) => ({
|
|
570
|
-
checkpoint: cp,
|
|
571
|
-
score: cosineSimilarity(current.embedding, cp.embedding),
|
|
572
|
-
}))
|
|
573
|
-
.sort((a, b) => b.score - a.score);
|
|
574
|
-
return scored.slice(0, n);
|
|
575
|
-
}
|
|
576
|
-
/**
|
|
577
|
-
* Store statistics for status reporting / logging. Returns counts + the last
|
|
578
|
-
* (highest-numbered) checkpoint, or nulls when the session is empty.
|
|
579
|
-
*/
|
|
580
|
-
stats(sessionId) {
|
|
581
|
-
const sid = normalizeSessionId(sessionId);
|
|
582
|
-
const cps = listCheckpoints(sid, this.stateDir);
|
|
583
|
-
const state = loadSessionState(sid, this.stateDir);
|
|
584
|
-
const ordered = [...cps].sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
|
|
585
|
-
const last = ordered[ordered.length - 1];
|
|
586
|
-
const injected = state.injectedCheckpointIds.length;
|
|
587
|
-
const ds = getDedupStats(this.stateDir);
|
|
588
|
-
const sessionTok = cps.reduce((s, c) => s + (c.tokenEstimate ?? 0), 0);
|
|
589
|
-
const sessionOrig = cps.reduce((s, c) => s + (c.originalTokenEstimate ?? 0), 0);
|
|
590
|
-
// Per-session "tokens saved" = Σ(original − stored) over this session's
|
|
591
|
-
// stored checkpoints. Deduped adds (whole region discarded, nothing stored)
|
|
592
|
-
// are counted in the repo-wide meta counter via repoStats(); the per-session
|
|
593
|
-
// DB sum here covers the rows that exist.
|
|
594
|
-
const sessionSaved = cps.reduce((s, c) => s + Math.max(0, (c.originalTokenEstimate ?? 0) - (c.tokenEstimate ?? 0)), 0);
|
|
595
|
-
return {
|
|
596
|
-
checkpointCount: cps.length,
|
|
597
|
-
totalTokenEstimate: sessionTok,
|
|
598
|
-
lastCheckpointId: last?.checkpointId,
|
|
599
|
-
lastSummary: last?.summary,
|
|
600
|
-
injectedCount: injected,
|
|
601
|
-
dedupHitRate: cps.length === 0 ? 0 : injected / cps.length,
|
|
602
|
-
storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
|
|
603
|
-
tokensSaved: sessionSaved,
|
|
604
|
-
originalTokens: sessionOrig,
|
|
605
|
-
dedupAttempts: ds.attempts,
|
|
606
|
-
dedupCollapsed: ds.deduped,
|
|
607
|
-
};
|
|
608
|
-
}
|
|
609
|
-
/**
|
|
610
|
-
* Repo-wide stats — aggregates every session in this store (one per repo).
|
|
611
|
-
* Cumulative, resumable, cross-device. Surfaces the dashboard's "Repo …"
|
|
612
|
-
* figures; distinct from {@link stats} (per-session).
|
|
613
|
-
*/
|
|
614
|
-
repoStats() {
|
|
615
|
-
return repoStatsFromStore(this.stateDir);
|
|
616
|
-
}
|
|
617
|
-
/** Data-safety invariant (Phase 0): regions retained vs bytes permanently deleted. */
|
|
618
|
-
dataInvariant() {
|
|
619
|
-
return dataInvariantStats(this.stateDir);
|
|
620
|
-
}
|
|
621
330
|
}
|
|
331
|
+
// Re-exports (back-compat): existing call sites keep importing from "./vectorStore.js"
|
|
332
|
+
export { vectorSemDedup, vectorDedupe, vectorMarkInjected, vectorWasInjected, vectorSimilarity, vectorList, vectorTopSimilar, vectorStats, vectorRepoStats, vectorDataInvariant, } from "./vector-read.js";
|
|
333
|
+
// Re-exports: vectorSearch / vectorSearchAsync (moved to vector-search.ts)
|
|
334
|
+
export { vectorSearch, vectorSearchAsync } from "./vector-search.js";
|
|
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
|
|
|
3
3
|
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
|
-
import { VectorStore, computeRegionHash } from "./vectorStore.js";
|
|
6
|
+
import { VectorStore, computeRegionHash, vectorStats, vectorDedupe, vectorWasInjected, vectorMarkInjected, vectorList, vectorTopSimilar, vectorRepoStats, vectorSearch } from "./vectorStore.js";
|
|
7
7
|
import { TrigramEmbedder, cosineSimilarity, l2Normalize, defaultEmbedder, } from "./embedder.js";
|
|
8
8
|
import { normalizeSessionId, decompressSmart } from "./store.js";
|
|
9
9
|
const baseTmp = mkdtempSync(join(tmpdir(), "mc-test-"));
|
|
@@ -45,7 +45,7 @@ test("add then search returns the planted checkpoint top-1", () => {
|
|
|
45
45
|
tokenEstimate: 1200,
|
|
46
46
|
timestamp: 1000,
|
|
47
47
|
});
|
|
48
|
-
const hits = s
|
|
48
|
+
const hits = vectorSearch(s, "sess_abc", "src/compact.ts truncate helper", 3);
|
|
49
49
|
assert.equal(hits.length, 1);
|
|
50
50
|
assert.ok(hits[0].score > 0.5);
|
|
51
51
|
assert.equal(hits[0].checkpoint.summary.includes("src/compact.ts"), true);
|
|
@@ -68,7 +68,7 @@ test("dedup by regionHash: identical region is not double-stored", () => {
|
|
|
68
68
|
assert.equal(r1.deduped, false);
|
|
69
69
|
assert.equal(r2.deduped, true);
|
|
70
70
|
assert.equal(r1.checkpoint.checkpointId, r2.checkpoint.checkpointId);
|
|
71
|
-
assert.equal(s
|
|
71
|
+
assert.equal(vectorSearch(s, "sess_dup", "anything", 10).length, 1);
|
|
72
72
|
});
|
|
73
73
|
test("dedup cascade: summaryHash catches same-topic incremental compactions", () => {
|
|
74
74
|
const s = store();
|
|
@@ -100,7 +100,7 @@ test("dedup cascade: summaryHash dedup still stores only one checkpoint", () =>
|
|
|
100
100
|
const ts = "some topic summary that is identical";
|
|
101
101
|
s.add({ sessionId: "sess_sh2", summary, topicSummary: ts, regionText: "region a", timestamp: 1 });
|
|
102
102
|
s.add({ sessionId: "sess_sh2", summary, topicSummary: ts, regionText: "region b", timestamp: 2 });
|
|
103
|
-
assert.equal(s
|
|
103
|
+
assert.equal(vectorStats(s, "sess_sh2").checkpointCount, 1);
|
|
104
104
|
});
|
|
105
105
|
test("dedupe() sentinel returns true for a stored region", () => {
|
|
106
106
|
const s = store();
|
|
@@ -112,8 +112,8 @@ test("dedupe() sentinel returns true for a stored region", () => {
|
|
|
112
112
|
regionText: region,
|
|
113
113
|
timestamp: 1,
|
|
114
114
|
});
|
|
115
|
-
assert.equal(s
|
|
116
|
-
assert.equal(s
|
|
115
|
+
assert.equal(vectorDedupe(s, "sess_sent", hash), true);
|
|
116
|
+
assert.equal(vectorDedupe(s, "sess_sent", "deadbeef"), false);
|
|
117
117
|
});
|
|
118
118
|
test("near-duplicate collapse keeps only the top of a near-identical pair", () => {
|
|
119
119
|
const s = store();
|
|
@@ -129,7 +129,7 @@ test("near-duplicate collapse keeps only the top of a near-identical pair", () =
|
|
|
129
129
|
regionText: "user investigated src/compact.ts and added a truncate helper for the summaries",
|
|
130
130
|
timestamp: 2,
|
|
131
131
|
});
|
|
132
|
-
const hits = s
|
|
132
|
+
const hits = vectorSearch(s, "sess_nd", "user investigated src/compact.ts and added a truncate helper for summaries", 5);
|
|
133
133
|
assert.equal(hits.length, 1);
|
|
134
134
|
});
|
|
135
135
|
test("markInjected / wasInjected track injection", () => {
|
|
@@ -140,9 +140,9 @@ test("markInjected / wasInjected track injection", () => {
|
|
|
140
140
|
regionText: "inject region",
|
|
141
141
|
timestamp: 1,
|
|
142
142
|
});
|
|
143
|
-
assert.equal(s
|
|
144
|
-
s
|
|
145
|
-
assert.equal(s
|
|
143
|
+
assert.equal(vectorWasInjected(s, "sess_inj", r.checkpoint.checkpointId), false);
|
|
144
|
+
vectorMarkInjected(s, "sess_inj", r.checkpoint.checkpointId);
|
|
145
|
+
assert.equal(vectorWasInjected(s, "sess_inj", r.checkpoint.checkpointId), true);
|
|
146
146
|
});
|
|
147
147
|
test("normalizeSessionId handles null, prefixed, and uuid forms", () => {
|
|
148
148
|
assert.match(normalizeSessionId("sess_abc"), /^sess_/);
|
|
@@ -177,7 +177,7 @@ test("checkpoints survive a fresh store instance (on-disk)", () => {
|
|
|
177
177
|
timestamp: 1,
|
|
178
178
|
});
|
|
179
179
|
const s2 = new VectorStore({ dedupSim: 0.9, stateDir: dir }); // new instance, same disk state
|
|
180
|
-
const hits = s2
|
|
180
|
+
const hits = vectorSearch(s2, "sess_persist", "persist region text", 3);
|
|
181
181
|
assert.equal(hits.length, 1);
|
|
182
182
|
assert.equal(hits[0].checkpoint.summary, "persisted");
|
|
183
183
|
});
|
|
@@ -187,7 +187,7 @@ test("corrupt checkpoint file falls back to empty (no throw)", () => {
|
|
|
187
187
|
const file = join(dir, "sess_corrupt.checkpoints.json.gz");
|
|
188
188
|
writeFileSync(file, Buffer.from("not a gzip"));
|
|
189
189
|
const s = new VectorStore({ dedupSim: 0.9, stateDir: dir });
|
|
190
|
-
assert.equal(s
|
|
190
|
+
assert.equal(vectorSearch(s, "sess_corrupt", "q", 3).length, 0);
|
|
191
191
|
});
|
|
192
192
|
test("stats reports counts, last checkpoint, and dedup rate", () => {
|
|
193
193
|
const s = store();
|
|
@@ -205,14 +205,14 @@ test("stats reports counts, last checkpoint, and dedup rate", () => {
|
|
|
205
205
|
tokenEstimate: 700,
|
|
206
206
|
timestamp: 2,
|
|
207
207
|
});
|
|
208
|
-
const st1 = s
|
|
208
|
+
const st1 = vectorStats(s, "sess_stats");
|
|
209
209
|
assert.equal(st1.checkpointCount, 2);
|
|
210
210
|
assert.equal(st1.lastCheckpointId, "chkpt_002");
|
|
211
211
|
assert.equal(st1.totalTokenEstimate, 1200);
|
|
212
212
|
assert.equal(st1.injectedCount, 0);
|
|
213
213
|
assert.equal(st1.dedupHitRate, 0);
|
|
214
|
-
s
|
|
215
|
-
const st2 = s
|
|
214
|
+
vectorMarkInjected(s, "sess_stats", "chkpt_001");
|
|
215
|
+
const st2 = vectorStats(s, "sess_stats");
|
|
216
216
|
assert.equal(st2.injectedCount, 1);
|
|
217
217
|
assert.ok(Math.abs(st2.dedupHitRate - 0.5) < 1e-9);
|
|
218
218
|
});
|
|
@@ -223,7 +223,7 @@ test("tokensSaved = original − stored per session; deduped add saves the whole
|
|
|
223
223
|
// cp2: orig 3000, stored 700 → saved 2300
|
|
224
224
|
s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
|
|
225
225
|
s.add({ sessionId: "sess_saved", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
|
|
226
|
-
const st = s
|
|
226
|
+
const st = vectorStats(s, "sess_saved");
|
|
227
227
|
assert.equal(st.totalTokenEstimate, 1200, "Σ stored summaries");
|
|
228
228
|
assert.equal(st.originalTokens, 5000, "Σ original region tokens");
|
|
229
229
|
assert.equal(st.tokensSaved, 3800, "per-session saved = Σ(original − stored) = 1500 + 2300");
|
|
@@ -234,14 +234,14 @@ test("tokensSaved = original − stored per session; deduped add saves the whole
|
|
|
234
234
|
// dedupCollapsed bumps, and no new checkpoint row is created.
|
|
235
235
|
const deduped = s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
|
|
236
236
|
assert.ok(deduped.deduped, "identical region should dedup");
|
|
237
|
-
const st3 = s
|
|
237
|
+
const st3 = vectorStats(s, "sess_saved");
|
|
238
238
|
// Per-session DB sum only covers stored rows (deduped adds create no row), so
|
|
239
239
|
// the per-session figure is unchanged; the deduped save lands in the repo meta.
|
|
240
240
|
assert.equal(st3.tokensSaved, 3800, "per-session DB sum unchanged by deduped add");
|
|
241
241
|
assert.equal(st3.dedupCollapsed, 1, "deduped collapse counted");
|
|
242
242
|
assert.equal(st3.dedupAttempts, 3);
|
|
243
243
|
// Repo cumulative counter DID capture the deduped region's full original size.
|
|
244
|
-
assert.equal(s
|
|
244
|
+
assert.equal(vectorRepoStats(s).tokensSaved, 3800 + 2000, "repo saved includes deduped original");
|
|
245
245
|
});
|
|
246
246
|
test("repoStats aggregates every session + counts deduped original tokens", () => {
|
|
247
247
|
const dir = join(baseTmp, `repo-${counter++}`);
|
|
@@ -249,7 +249,7 @@ test("repoStats aggregates every session + counts deduped original tokens", () =
|
|
|
249
249
|
const b = new VectorStore({ dedupSim: 0.9, stateDir: dir }); // same disk store, diff instance
|
|
250
250
|
a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
|
|
251
251
|
b.add({ sessionId: "sess_b", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
|
|
252
|
-
const repo = a
|
|
252
|
+
const repo = vectorRepoStats(a);
|
|
253
253
|
assert.equal(repo.checkpointCount, 2, "checkpoints across both sessions");
|
|
254
254
|
assert.equal(repo.sessionCount, 2, "two distinct sessions");
|
|
255
255
|
assert.equal(repo.totalTokenEstimate, 1200, "Σ stored");
|
|
@@ -259,7 +259,7 @@ test("repoStats aggregates every session + counts deduped original tokens", () =
|
|
|
259
259
|
// A deduped add into sess_a: whole original region saved, no new row.
|
|
260
260
|
const deduped = a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
|
|
261
261
|
assert.ok(deduped.deduped);
|
|
262
|
-
const repo2 = a
|
|
262
|
+
const repo2 = vectorRepoStats(a);
|
|
263
263
|
assert.equal(repo2.tokensSaved, 3800 + 2000, "deduped collapse adds full original region to repo saved");
|
|
264
264
|
assert.equal(repo2.dedupCollapsed, 1);
|
|
265
265
|
assert.equal(repo2.checkpointCount, 2, "still two stored checkpoints");
|
|
@@ -289,7 +289,7 @@ test("whitespace-variant region is deduplicated", () => {
|
|
|
289
289
|
timestamp: 2,
|
|
290
290
|
});
|
|
291
291
|
assert.equal(r2.deduped, true, "whitespace-variant should be deduplicated");
|
|
292
|
-
assert.equal(s
|
|
292
|
+
assert.equal(vectorStats(s, "sess_ws").checkpointCount, 1, "only one checkpoint stored");
|
|
293
293
|
});
|
|
294
294
|
test("topSimilar returns n most similar checkpoints to the current (most recent)", () => {
|
|
295
295
|
const s = store();
|
|
@@ -313,7 +313,7 @@ test("topSimilar returns n most similar checkpoints to the current (most recent)
|
|
|
313
313
|
regionText: "fix the buffer overflow in src/compact.ts by adding a bounds check before truncate",
|
|
314
314
|
timestamp: 3,
|
|
315
315
|
});
|
|
316
|
-
const hits = s
|
|
316
|
+
const hits = vectorTopSimilar(s, "sess_top", 10);
|
|
317
317
|
assert.equal(hits.length, 2); // two other checkpoints
|
|
318
318
|
// The compact.ts checkpoint should rank above the guitar checkpoint
|
|
319
319
|
assert.equal(hits[0].checkpoint.summary, "compact");
|
|
@@ -334,20 +334,20 @@ test("topSimilar excludes the current checkpoint itself", () => {
|
|
|
334
334
|
regionText: "alpha region text two",
|
|
335
335
|
timestamp: 2,
|
|
336
336
|
});
|
|
337
|
-
const hits = s
|
|
337
|
+
const hits = vectorTopSimilar(s, "sess_self", 5);
|
|
338
338
|
assert.equal(hits.length, 1);
|
|
339
339
|
assert.notEqual(hits[0].checkpoint.checkpointId, "chkpt_002"); // not the current
|
|
340
340
|
});
|
|
341
341
|
test("topSimilar returns empty for sessions with 0 or 1 checkpoints", () => {
|
|
342
342
|
const s = store();
|
|
343
|
-
assert.deepEqual(s
|
|
343
|
+
assert.deepEqual(vectorTopSimilar(s, "sess_none", 5), []);
|
|
344
344
|
s.add({
|
|
345
345
|
sessionId: "sess_one",
|
|
346
346
|
summary: "solo",
|
|
347
347
|
regionText: "only checkpoint",
|
|
348
348
|
timestamp: 1,
|
|
349
349
|
});
|
|
350
|
-
assert.deepEqual(s
|
|
350
|
+
assert.deepEqual(vectorTopSimilar(s, "sess_one", 5), []);
|
|
351
351
|
});
|
|
352
352
|
test("topSimilar respects the n limit", () => {
|
|
353
353
|
const s = store();
|
|
@@ -366,12 +366,12 @@ test("topSimilar respects the n limit", () => {
|
|
|
366
366
|
timestamp: i + 1,
|
|
367
367
|
});
|
|
368
368
|
}
|
|
369
|
-
const hits = s
|
|
369
|
+
const hits = vectorTopSimilar(s, "sess_limit", 2);
|
|
370
370
|
assert.equal(hits.length, 2);
|
|
371
371
|
});
|
|
372
372
|
test("stats on empty session returns zeros and nulls", () => {
|
|
373
373
|
const s = store();
|
|
374
|
-
const st = s
|
|
374
|
+
const st = vectorStats(s, "sess_empty");
|
|
375
375
|
assert.equal(st.checkpointCount, 0);
|
|
376
376
|
assert.equal(st.lastCheckpointId, undefined);
|
|
377
377
|
assert.equal(st.totalTokenEstimate, 0);
|
|
@@ -395,7 +395,7 @@ test("L0 content-hash dedup: identical content under different regionText collap
|
|
|
395
395
|
assert.equal(r1.deduped, false);
|
|
396
396
|
assert.equal(r2.deduped, true);
|
|
397
397
|
assert.equal(r2.reason, "contentHash");
|
|
398
|
-
assert.equal(s
|
|
398
|
+
assert.equal(vectorList(s, "sess_l0").length, 1);
|
|
399
399
|
});
|
|
400
400
|
test("L0 content-hash dedup stores both hash fields and bumps timestamp on hit", () => {
|
|
401
401
|
const s = store();
|
|
@@ -409,7 +409,7 @@ test("L0 content-hash dedup stores both hash fields and bumps timestamp on hit",
|
|
|
409
409
|
});
|
|
410
410
|
assert.equal(r2.deduped, true);
|
|
411
411
|
assert.equal(r2.reason, "contentHash");
|
|
412
|
-
const cp = s
|
|
412
|
+
const cp = vectorList(s, "sess_l0ts")[0];
|
|
413
413
|
assert.equal(cp.contentHash?.length, 64);
|
|
414
414
|
assert.equal(cp.contentHash2?.length, 64);
|
|
415
415
|
assert.equal(cp.contentHashVersion, 1);
|
|
@@ -419,7 +419,7 @@ test("compressed_original roundtrips through versioned compression", () => {
|
|
|
419
419
|
const s = store();
|
|
420
420
|
const raw = "raw region text preserved for audit and replay";
|
|
421
421
|
s.add({ sessionId: "sess_co", summary: "x", regionText: raw, timestamp: 1 });
|
|
422
|
-
const cp = s
|
|
422
|
+
const cp = vectorList(s, "sess_co")[0];
|
|
423
423
|
assert.ok(cp.compressedOriginal instanceof Buffer);
|
|
424
424
|
const restored = decompressSmart(cp.compressedOriginal).toString("utf-8");
|
|
425
425
|
assert.equal(restored, raw);
|
|
@@ -465,14 +465,14 @@ test("L1 catches a one-word-diff near-duplicate that L0 misses", () => {
|
|
|
465
465
|
assert.equal(r1.deduped, false);
|
|
466
466
|
assert.equal(r2.deduped, true);
|
|
467
467
|
assert.equal(r2.reason, "l1MinHash");
|
|
468
|
-
assert.equal(s
|
|
468
|
+
assert.equal(vectorList(s, "sess_l1").length, 1);
|
|
469
469
|
});
|
|
470
470
|
test("L1 does NOT falsely dedup genuinely different content", () => {
|
|
471
471
|
const s = store();
|
|
472
472
|
s.add({ sessionId: "sess_l1b", summary: "a", regionText: "the database migration added three indexes", timestamp: 1 });
|
|
473
473
|
const r2 = s.add({ sessionId: "sess_l1b", summary: "b", regionText: "the frontend added a dark mode toggle", timestamp: 2 });
|
|
474
474
|
assert.equal(r2.deduped, false);
|
|
475
|
-
assert.equal(s
|
|
475
|
+
assert.equal(vectorList(s, "sess_l1b").length, 2);
|
|
476
476
|
});
|
|
477
477
|
test("cleanup", () => {
|
|
478
478
|
rmSync(baseTmp, { recursive: true, force: true });
|