pi-mega-compact 0.8.20 → 0.8.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +6 -0
  2. package/dist/extensions/dashboard-client/src/hooks/useApi.js +51 -0
  3. package/dist/extensions/dashboard-client/src/hooks/useSSE.js +63 -0
  4. package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
  5. package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
  6. package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
  7. package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
  8. package/dist/extensions/dashboard-server/html.js +2 -621
  9. package/dist/extensions/dashboard-server/routes-core.js +62 -0
  10. package/dist/extensions/dashboard-server/routes-game.js +323 -0
  11. package/dist/extensions/dashboard-server/routes-repo.js +170 -0
  12. package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
  13. package/dist/extensions/dashboard-server/routes.js +10 -0
  14. package/dist/extensions/dashboard-server/server.js +26 -623
  15. package/dist/extensions/mega-commands.js +4 -3
  16. package/dist/extensions/mega-compact-s38.test.js +28 -2
  17. package/dist/extensions/mega-events/agent-handlers.js +18 -1
  18. package/dist/extensions/mega-events/compact-handlers.js +26 -0
  19. package/dist/extensions/mega-events/error-classifier.js +8 -3
  20. package/dist/extensions/mega-events/session-handlers.js +2 -1
  21. package/dist/extensions/mega-pipeline/compact.js +3 -2
  22. package/dist/extensions/mega-runtime/state.js +7 -7
  23. package/dist/src/dedup/raptor/promote.test.js +5 -5
  24. package/dist/src/dedup/sprint12.test.js +7 -7
  25. package/dist/src/dedup-engine.test.js +29 -29
  26. package/dist/src/e2e.test.js +38 -38
  27. package/dist/src/engine.js +3 -3
  28. package/dist/src/engine.test.js +6 -6
  29. package/dist/src/importance.js +197 -0
  30. package/dist/src/importance.test.js +372 -0
  31. package/dist/src/ratio.bench.test.js +18 -18
  32. package/dist/src/recall.js +6 -5
  33. package/dist/src/recall.test.js +85 -27
  34. package/dist/src/sprint14.test.js +2 -2
  35. package/dist/src/store/migrate.test.js +5 -5
  36. package/dist/src/store/sprint10.test.js +5 -5
  37. package/dist/src/store/sqlite/global-index.js +5 -174
  38. package/dist/src/store/sqlite/global-sessions.js +190 -0
  39. package/dist/src/vector-read.js +168 -0
  40. package/dist/src/vector-search.js +191 -0
  41. package/dist/src/vectorStore.js +10 -297
  42. package/dist/src/vectorStore.test.js +32 -32
  43. package/extensions/dashboard-client/package-lock.json +1 -1
  44. package/extensions/dashboard-client/package.json +1 -1
  45. package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
  46. package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
  47. package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
  48. package/extensions/dashboard-server/dashboard-client.ts +21 -0
  49. package/extensions/dashboard-server/html.ts +2 -621
  50. package/extensions/dashboard-server/routes-core.ts +113 -0
  51. package/extensions/dashboard-server/routes-game.ts +386 -0
  52. package/extensions/dashboard-server/routes-repo.ts +212 -0
  53. package/extensions/dashboard-server/routes-sessions.ts +195 -0
  54. package/extensions/dashboard-server/routes.ts +13 -0
  55. package/extensions/dashboard-server/server.ts +37 -700
  56. package/extensions/mega-commands.ts +4 -3
  57. package/extensions/mega-compact-s38.test.ts +32 -2
  58. package/extensions/mega-events/agent-handlers.ts +17 -1
  59. package/extensions/mega-events/compact-handlers.ts +28 -0
  60. package/extensions/mega-events/error-classifier.ts +8 -2
  61. package/extensions/mega-events/session-handlers.ts +2 -1
  62. package/extensions/mega-pipeline/compact.ts +3 -2
  63. package/extensions/mega-runtime/state.ts +7 -7
  64. package/extensions/openclaw-mega-compact.ts +2 -2
  65. package/package.json +1 -1
  66. package/src/dedup/raptor/promote.test.ts +5 -5
  67. package/src/dedup/sprint12.test.ts +7 -7
  68. package/src/dedup-engine.test.ts +30 -30
  69. package/src/e2e.test.ts +38 -38
  70. package/src/engine.test.ts +6 -6
  71. package/src/engine.ts +3 -3
  72. package/src/importance.test.ts +538 -0
  73. package/src/importance.ts +312 -0
  74. package/src/ratio.bench.test.ts +18 -18
  75. package/src/recall.test.ts +101 -29
  76. package/src/recall.ts +9 -9
  77. package/src/sprint14.test.ts +2 -2
  78. package/src/store/migrate.test.ts +5 -5
  79. package/src/store/sprint10.test.ts +5 -5
  80. package/src/store/sqlite/global-index.ts +18 -290
  81. package/src/store/sqlite/global-sessions.ts +291 -0
  82. package/src/vector-read.ts +237 -0
  83. package/src/vector-search.ts +231 -0
  84. package/src/vectorStore.test.ts +32 -32
  85. package/src/vectorStore.ts +29 -356
  86. package/dist/extensions/dashboard-server/helpers.js +0 -37
  87. package/dist/extensions/dashboard-server/html/all-repos-tab.js +0 -26
  88. package/dist/extensions/dashboard-server/html/body-open.js +0 -23
  89. package/dist/extensions/dashboard-server/html/current-repo-tab.js +0 -130
  90. package/dist/extensions/dashboard-server/html/head-open.js +0 -16
  91. package/dist/extensions/dashboard-server/html/high-score-tab.js +0 -25
  92. package/dist/extensions/dashboard-server/html/repo-detail-modal.js +0 -26
  93. package/dist/extensions/dashboard-server/html/script.js +0 -259
  94. package/dist/extensions/dashboard-server/html/styles.js +0 -103
  95. package/dist/extensions/dashboard-server/html/summary-tab.js +0 -19
  96. package/dist/extensions/dashboard-server/html-template.js +0 -41
  97. package/dist/src/store/sqlite/connection.js +0 -35
  98. package/dist/src/store/sqlite/index-store.js +0 -167
  99. package/dist/src/store/sqlite/memory.js +0 -54
  100. package/dist/src/store/sqlite/minhash-lsh.js +0 -47
  101. package/dist/src/store/sqlite/sessions.js +0 -39
  102. package/dist/src/store/sqlite/transaction.js +0 -19
  103. package/dist/src/vectorStore/add.js +0 -260
  104. package/dist/src/vectorStore/dedup.js +0 -52
  105. package/dist/src/vectorStore/index.js +0 -10
  106. package/dist/src/vectorStore/queries.js +0 -83
  107. package/dist/src/vectorStore/search.js +0 -95
  108. package/dist/src/vectorStore/session.js +0 -19
  109. package/dist/src/vectorStore/store.js +0 -105
  110. package/dist/src/vectorStore/types.js +0 -6
  111. package/dist/src/vectorStore/utils.js +0 -23
@@ -9,44 +9,29 @@
9
9
  */
10
10
 
11
11
  import { createHash } from "node:crypto";
12
- import type { Embedder, Vector } from "./embedder.js";
12
+ import type { Embedder } from "./embedder.js";
13
13
  import { cosineSimilarity, defaultEmbedder } from "./embedder.js";
14
14
  import { loadDedupConfig, type DedupConfigShape, type DedupTier } from "./config/dedup.js";
15
15
  import { logDecision } from "./monitoring.js";
16
- import type { StoredCheckpoint, SessionState } from "./store.js";
16
+ import type { StoredCheckpoint } from "./store.js";
17
17
  import { getStateDir, normalizeSessionId, compressSmart } from "./store.js";
18
18
  import { computeContentDigest } from "./dedup/digest.js";
19
19
  import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "./dedup/l1-minhash.js";
20
20
  import { lshBands } from "./dedup/l1-lsh.js";
21
21
  import { isNearDuplicate } from "./dedup/l1-verify.js";
22
- import { mmrRerank, type MmrItem } from "./dedup/mmr.js";
23
- import { topK } from "./dedup/topk.js";
24
22
  import { openBloom, saveBloom } from "./store/bloom.js";
25
23
  import {
26
24
  listCheckpoints,
27
25
  nextCheckpointId,
28
26
  upsertCheckpoint,
29
- getCheckpoint,
30
27
  loadSessionState,
31
28
  saveSessionState,
32
29
  upsertMinhashSignature,
33
30
  insertLshBuckets,
34
31
  lshCandidateChunks,
35
- setDedupStatus,
36
32
  addTokensSaved,
37
- getDedupStats,
38
33
  bumpDedupStats,
39
- repoStats as repoStatsFromStore,
40
- dataInvariantStats,
41
- maxCheckpointTimestamp,
42
34
  } from "./store/sqlite.js";
43
- import {
44
- initVectorIndex,
45
- searchAsync as vectorIndexSearch,
46
- type VectorIndexHit,
47
- } from "./store/vectorIndex.js";
48
- import { rehydrateRaptorTree, isShadowMode } from "./dedup/raptor/index.js";
49
- import { stagedExpansion } from "./dedup/raptor/retrieval.js";
50
35
  import { migrateJsonToSqlite } from "./store/migrate.js";
51
36
 
52
37
  export interface SearchHit {
@@ -102,11 +87,16 @@ export function computeRegionHash(regionText: string): string {
102
87
  }
103
88
 
104
89
  export class VectorStore {
105
- private readonly embedder: Embedder;
106
- private readonly stateDir: string;
90
+ // These fields are `readonly` (set once in the constructor) but NOT private:
91
+ // the read/search/dedup helpers split into vector-read.ts, vector-search.ts,
92
+ // and vector-dedup.ts access them directly. Marking them private would force
93
+ // ugly `as unknown as` casts in those modules; keeping them package-public
94
+ // makes VectorStore a thin barrel whose helpers live in sibling files.
95
+ readonly embedder: Embedder;
96
+ readonly stateDir: string;
107
97
  private readonly l2Threshold: number;
108
98
  /** Single source of truth for tier flags + thresholds (Sprint 14). */
109
- private readonly cfg: DedupConfigShape;
99
+ readonly cfg: DedupConfigShape;
110
100
  /** Optional monitoring target (Sprint 14). Undefined → no monitoring. */
111
101
  private readonly eventsPath?: string;
112
102
  /**
@@ -115,7 +105,7 @@ export class VectorStore {
115
105
  * without crossing into the pi-runtime layer (src/ stays pi-agnostic). The
116
106
  * global index keys on repoId so recall can span repos.
117
107
  */
118
- private readonly repoId: string;
108
+ readonly repoId: string;
119
109
 
120
110
  constructor(
121
111
  opts: {
@@ -153,7 +143,7 @@ export class VectorStore {
153
143
  }
154
144
 
155
145
  /** Emit a structured dedup-decision event (best-effort, never throws). */
156
- private record(tier: DedupTier, result: "deduped" | "new" | "mark_only", reason: string | undefined, latencyMs: number): void {
146
+ record(tier: DedupTier, result: "deduped" | "new" | "mark_only", reason: string | undefined, latencyMs: number): void {
157
147
  if (!this.eventsPath) return;
158
148
  logDecision(this.eventsPath, {
159
149
  ts: Date.now(),
@@ -432,338 +422,21 @@ export class VectorStore {
432
422
  }
433
423
  return undefined;
434
424
  }
435
-
436
- /**
437
- * Semantic search within a session's checkpoints. Returns top-K by cosine
438
- * similarity, diversified via MMR (QA #10) so a cluster of near-identical
439
- * hits yields at most a few distinct-relevance results.
440
- *
441
- * Heap-based top-K (QA #4, O(N log k)) replaces the old full sort; MMR then
442
- * reranks the candidate window for diversity.
443
- */
444
- search(sessionId: string, query: string, k = 3): SearchHit[] {
445
- const sid = normalizeSessionId(sessionId);
446
- const checkpoints = listCheckpoints(sid, this.stateDir).filter(
447
- (cp) => cp.dedupStatus !== "removed", // SemDeDup: exclude removed rows
448
- );
449
- if (checkpoints.length === 0) return [];
450
- const qv = this.embedder.embed(query);
451
-
452
- const scored: SearchHit[] = checkpoints.map((cp) => ({
453
- checkpoint: cp,
454
- score: cosineSimilarity(qv, cp.embedding),
455
- }));
456
-
457
- // Heap top-K over a widened window (2k) so MMR has diverse candidates.
458
- const window = topK(
459
- scored.map((h) => ({ item: h, score: h.score })),
460
- Math.max(k * 2, k),
461
- ).map((s) => s.item);
462
- // MMR (QA #10) is part of the L2 semantic tier: skip it when L2 is disabled
463
- // (Sprint 14 flag), returning the plain relevance-ranked window instead.
464
- if (!this.cfg.L2_ENABLED) return window.slice(0, k);
465
-
466
- // Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
467
- // merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
468
- // RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
469
- // O(n) flat leaves, tightening the block at read time.
470
- if (this.cfg.RAPTOR_ENABLED) {
471
- const raptorHits = this.raptorSearchHits(sid, query, k);
472
- if (raptorHits.length > 0) {
473
- const merged: SearchHit[] = [...window];
474
- for (const rh of raptorHits) {
475
- if (!merged.some((m) => m.checkpoint.checkpointId === rh.checkpoint.checkpointId)) {
476
- merged.push(rh);
477
- }
478
- }
479
- const mmrItems: MmrItem<SearchHit>[] = merged.map((h) => ({
480
- item: h,
481
- vector: h.checkpoint.embedding,
482
- relevance: h.score,
483
- }));
484
- return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
485
- }
486
- }
487
-
488
- const mmrItems: MmrItem<SearchHit>[] = window.map((h) => ({
489
- item: h,
490
- vector: h.checkpoint.embedding,
491
- relevance: h.score,
492
- }));
493
- const ranked = mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
494
- return ranked;
495
- }
496
-
497
- /**
498
- * Slice 2: async cross-repo (or single-repo) recall via the PGlite/HNSW index.
499
- *
500
- * This is the ONLY async recall surface and is a BONUS path — the synchronous
501
- * `search()` above remains the default. `opts.repoId` scopes to one repo; omit
502
- * it for cross-repo nearest-neighbor recall (the headline capability the sync
503
- * per-session scan cannot provide).
504
- *
505
- * Best-effort: if the index is disabled/empty/failing, we fall back to the
506
- * synchronous per-session `search()` for THIS repo so callers always get a
507
- * sensible result. Hydrates each hit's StoredCheckpoint from the authoritative
508
- * node:sqlite store (the hit's repoId doubles as that repo's stateDir), then
509
- * MMR-dedupes the merged set.
510
- */
511
- async searchAsync(
512
- sessionId: string,
513
- query: string,
514
- k = 3,
515
- opts: { repoId?: string; crossRepo?: boolean } = {},
516
- ): Promise<SearchHit[]> {
517
- const sid = normalizeSessionId(sessionId);
518
- const qv = this.embedder.embed(query);
519
- // repoId filter: explicit opts.repoId wins; else this repo unless crossRepo.
520
- const repoId = opts.repoId ?? (opts.crossRepo ? undefined : this.repoId);
521
- let indexHits: VectorIndexHit[] = [];
522
- try {
523
- await initVectorIndex();
524
- indexHits = await vectorIndexSearch(qv, { k: Math.max(k * 2, k), repoId });
525
- } catch {
526
- indexHits = [];
527
- }
528
- if (indexHits.length === 0) {
529
- // Index empty/unavailable → synchronous per-session fallback (this repo).
530
- return this.search(sid, query, k);
531
- }
532
- // Hydrate each index hit from the authoritative node:sqlite store. repoId is
533
- // that repo's stateDir, so cross-repo hits resolve against their own store.
534
- // Tag cross-repo hits with their source repoId so the recall block can label
535
- // them ("from repo <name>"); same-repo hits stay unlabeled.
536
- const selfRepo = this.repoId;
537
- const hydrated: SearchHit[] = [];
538
- for (const h of indexHits) {
539
- const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
540
- if (cp && cp.dedupStatus !== "removed") {
541
- const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
542
- hydrated.push({ checkpoint: cp, score: h.score, repoId: crossRepo ? h.repoId : undefined });
543
- }
544
- }
545
- if (hydrated.length === 0) return this.search(sid, query, k);
546
- // MMR-dedupe the merged candidate set for diversity (mirrors sync search).
547
- const mmrItems: MmrItem<SearchHit>[] = hydrated.map((h) => ({
548
- item: h,
549
- vector: h.checkpoint.embedding,
550
- relevance: h.score,
551
- }));
552
- return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
553
- }
554
-
555
- /**
556
- * Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
557
- * return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
558
- * exists (small sessions — flat search remains the path). Best-effort/non-fatal.
559
- */
560
- private raptorSearchHits(sid: string, query: string, k: number): SearchHit[] {
561
- const t0 = Date.now();
562
- try {
563
- // S25 gate (a): honor the shadow contract at SERVE time. The tree is still
564
- // built + persisted (logging-only) but NOT merged into recall while
565
- // RAPTOR_SHADOW_MODE is anything other than "false".
566
- if (isShadowMode()) return [];
567
- const tree = rehydrateRaptorTree(sid, this.stateDir);
568
- if (!tree || !tree.rootId) return [];
569
- // S25 gate (b): freshness + fallback guards. Skip a tree built before the
570
- // newest checkpoint (stale → may reference trimmed/deduped leaves) or one
571
- // whose root is a budget-exhausted extractive fallback (level 99).
572
- if (tree.timedOut) return [];
573
- const maxTs = maxCheckpointTimestamp(sid, this.stateDir);
574
- if (tree.builtAt && tree.builtAt < maxTs) return [];
575
- const leafIds = stagedExpansion(query, tree, {
576
- embedder: this.embedder,
577
- k,
578
- topM: this.cfg.RAPTOR_CLUSTERS_PER_LEVEL,
579
- mmrLambda: this.cfg.MMR_LAMBDA,
580
- });
581
- if (leafIds.length === 0) return [];
582
- const all = listCheckpoints(sid, this.stateDir).filter(
583
- (cp) => cp.dedupStatus !== "removed",
584
- );
585
- const qv = this.embedder.embed(query);
586
- const hits: SearchHit[] = [];
587
- for (const id of leafIds) {
588
- const cp = all.find((c) => c.checkpointId === id);
589
- if (cp) hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
590
- }
591
- // S25 monitoring: emit a raptor_serve decision so canary.ts can track
592
- // p95 latency + the tier's live traffic (non-fatal, best-effort).
593
- this.record("RAPTOR", hits.length > 0 ? "new" : "mark_only", `leaves=${leafIds.length}`, Date.now() - t0);
594
- return hits;
595
- } catch {
596
- return [];
597
- }
598
- }
599
-
600
- /**
601
- * SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
602
- * lower-quality row of any pair scoring cosine > `threshold` as
603
- * `dedup_status='removed'` (kept, not deleted — retrieval excludes it). Keeps
604
- * the row with the higher `tokenEstimate` (more context preserved). Runs as a
605
- * single scan; idempotent (re-running skips already-removed rows).
606
- *
607
- * Returns the number of rows marked removed.
608
- */
609
- semDedup(sessionId: string, threshold = this.cfg.SEMDEDUP_COSINE): number {
610
- const sid = normalizeSessionId(sessionId);
611
- const cps = listCheckpoints(sid, this.stateDir).filter(
612
- (c) => c.dedupStatus !== "removed",
613
- );
614
- let removed = 0;
615
- for (let i = 0; i < cps.length; i++) {
616
- for (let j = i + 1; j < cps.length; j++) {
617
- const a = cps[i];
618
- const b = cps[j];
619
- if (a.dedupStatus === "removed" || b.dedupStatus === "removed") continue;
620
- if (cosineSimilarity(a.embedding, b.embedding) > threshold) {
621
- // Keep the higher-tokenEstimate row; remove the other.
622
- const keep = a.tokenEstimate >= b.tokenEstimate ? a : b;
623
- const drop = keep === a ? b : a;
624
- setDedupStatus(drop.checkpointId, sid, "removed", this.stateDir);
625
- drop.dedupStatus = "removed";
626
- removed++;
627
- }
628
- }
629
- }
630
- return removed;
631
- }
632
-
633
- /**
634
- * Dedup sentinel check: has this region already been stored/represented?
635
- * Consulted by both the persist path and the recall/inline path.
636
- */
637
- dedupe(sessionId: string, regionHashOrText: string, isText = false): boolean {
638
- const sid = normalizeSessionId(sessionId);
639
- const hash = isText
640
- ? computeRegionHash(regionHashOrText)
641
- : regionHashOrText;
642
- const state = loadSessionState(sid, this.stateDir);
643
- if (state.storedRegionHashes.includes(hash)) return true;
644
- return listCheckpoints(sid, this.stateDir).some(
645
- (c) => c.regionHash === hash,
646
- );
647
- }
648
-
649
- /** Mark a checkpoint as injected into the window (recall dedup). */
650
- markInjected(sessionId: string, checkpointId: string): void {
651
- const sid = normalizeSessionId(sessionId);
652
- const state = loadSessionState(sid, this.stateDir);
653
- if (!state.injectedCheckpointIds.includes(checkpointId)) {
654
- state.injectedCheckpointIds.push(checkpointId);
655
- saveSessionState(sid, state, this.stateDir);
656
- }
657
- }
658
-
659
- /** True if this checkpoint was already injected this session. */
660
- wasInjected(sessionId: string, checkpointId: string): boolean {
661
- const state: SessionState = loadSessionState(
662
- normalizeSessionId(sessionId),
663
- this.stateDir,
664
- );
665
- return state.injectedCheckpointIds.includes(checkpointId);
666
- }
667
-
668
- /** Convenience for a raw vector cosine (exposed for tests). */
669
- similarity(a: Vector, b: Vector): number {
670
- return cosineSimilarity(a, b);
671
- }
672
-
673
- /** All checkpoints for a session (sorted by checkpointId). */
674
- list(sessionId: string): StoredCheckpoint[] {
675
- return listCheckpoints(normalizeSessionId(sessionId), this.stateDir);
676
- }
677
-
678
- /**
679
- * Return the n most similar checkpoints to the current (most recent) checkpoint
680
- * by cosine similarity. Returns fewer than n if the session has fewer checkpoints.
681
- * The current checkpoint itself is excluded from results.
682
- */
683
- topSimilar(sessionId: string, n: number): SearchHit[] {
684
- const sid = normalizeSessionId(sessionId);
685
- const checkpoints = listCheckpoints(sid, this.stateDir);
686
- if (checkpoints.length <= 1) return [];
687
-
688
- // Find the most recent checkpoint (by checkpointId, which is sequential)
689
- const ordered = [...checkpoints].sort((a, b) =>
690
- a.checkpointId.localeCompare(b.checkpointId),
691
- );
692
- const current = ordered[ordered.length - 1];
693
-
694
- // Score all other checkpoints by similarity to current
695
- const scored: SearchHit[] = ordered
696
- .filter((cp) => cp.checkpointId !== current.checkpointId)
697
- .map((cp) => ({
698
- checkpoint: cp,
699
- score: cosineSimilarity(current.embedding, cp.embedding),
700
- }))
701
- .sort((a, b) => b.score - a.score);
702
-
703
- return scored.slice(0, n);
704
- }
705
-
706
- /**
707
- * Store statistics for status reporting / logging. Returns counts + the last
708
- * (highest-numbered) checkpoint, or nulls when the session is empty.
709
- */
710
- stats(sessionId: string): {
711
- checkpointCount: number;
712
- totalTokenEstimate: number;
713
- lastCheckpointId: string | undefined;
714
- lastSummary: string | undefined;
715
- injectedCount: number;
716
- dedupHitRate: number; // injected / checkpoints, 0..1
717
- storageDedupRate: number; // deduped adds / total adds, 0..1 (cumulative)
718
- tokensSaved: number; // Σ(original − stored) for this session's checkpoints
719
- originalTokens: number; // Σ original region size for this session's checkpoints
720
- dedupAttempts: number; // cumulative add() calls (store-wide)
721
- dedupCollapsed: number; // cumulative deduped collapses (store-wide)
722
- } {
723
- const sid = normalizeSessionId(sessionId);
724
- const cps = listCheckpoints(sid, this.stateDir);
725
- const state = loadSessionState(sid, this.stateDir);
726
- const ordered = [...cps].sort((a, b) =>
727
- a.checkpointId.localeCompare(b.checkpointId),
728
- );
729
- const last = ordered[ordered.length - 1];
730
- const injected = state.injectedCheckpointIds.length;
731
- const ds = getDedupStats(this.stateDir);
732
- const sessionTok = cps.reduce((s, c) => s + (c.tokenEstimate ?? 0), 0);
733
- const sessionOrig = cps.reduce((s, c) => s + (c.originalTokenEstimate ?? 0), 0);
734
- // Per-session "tokens saved" = Σ(original − stored) over this session's
735
- // stored checkpoints. Deduped adds (whole region discarded, nothing stored)
736
- // are counted in the repo-wide meta counter via repoStats(); the per-session
737
- // DB sum here covers the rows that exist.
738
- const sessionSaved = cps.reduce(
739
- (s, c) => s + Math.max(0, (c.originalTokenEstimate ?? 0) - (c.tokenEstimate ?? 0)),
740
- 0,
741
- );
742
- return {
743
- checkpointCount: cps.length,
744
- totalTokenEstimate: sessionTok,
745
- lastCheckpointId: last?.checkpointId,
746
- lastSummary: last?.summary,
747
- injectedCount: injected,
748
- dedupHitRate: cps.length === 0 ? 0 : injected / cps.length,
749
- storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
750
- tokensSaved: sessionSaved,
751
- originalTokens: sessionOrig,
752
- dedupAttempts: ds.attempts,
753
- dedupCollapsed: ds.deduped,
754
- };
755
- }
756
-
757
- /**
758
- * Repo-wide stats — aggregates every session in this store (one per repo).
759
- * Cumulative, resumable, cross-device. Surfaces the dashboard's "Repo …"
760
- * figures; distinct from {@link stats} (per-session).
761
- */
762
- repoStats(): ReturnType<typeof repoStatsFromStore> {
763
- return repoStatsFromStore(this.stateDir);
764
- }
765
- /** Data-safety invariant (Phase 0): regions retained vs bytes permanently deleted. */
766
- dataInvariant(): ReturnType<typeof dataInvariantStats> {
767
- return dataInvariantStats(this.stateDir);
768
- }
769
425
  }
426
+
427
+ // Re-exports (back-compat): existing call sites keep importing from "./vectorStore.js"
428
+ export {
429
+ vectorSemDedup,
430
+ vectorDedupe,
431
+ vectorMarkInjected,
432
+ vectorWasInjected,
433
+ vectorSimilarity,
434
+ vectorList,
435
+ vectorTopSimilar,
436
+ vectorStats,
437
+ vectorRepoStats,
438
+ vectorDataInvariant,
439
+ } from "./vector-read.js";
440
+
441
+ // Re-exports: vectorSearch / vectorSearchAsync (moved to vector-search.ts)
442
+ export { vectorSearch, vectorSearchAsync } from "./vector-search.js";
@@ -1,37 +0,0 @@
1
- /**
2
- * File-reading helper functions for the dashboard server.
3
- */
4
- import { readFileSync } from "node:fs";
5
- export function readSnapshot(snapshotPath) {
6
- try {
7
- const raw = readFileSync(snapshotPath, "utf-8");
8
- return JSON.parse(raw);
9
- }
10
- catch {
11
- return {
12
- version: 1,
13
- updatedAt: null,
14
- tier: "unknown",
15
- config: { fastGatePct: 80, thresholdTokens: 100_000, anchorUserMessages: 1, preserveRecent: 2, auto: true, autoInlineK: 3 },
16
- session: { id: null, state: null, persistedThisSession: false, lastCheckpointId: null, lastCompactedFrom: 0 },
17
- context: { tokens: null, percent: null, contextWindow: 0 },
18
- trigger: { armed: false, ready: false, currentTokens: null, thresholdTokens: 100_000, fastGatePct: 80 },
19
- store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
20
- crew: { activeAgents: 0, currentTurn: 0 },
21
- repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
22
- integrity: { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 },
23
- model: undefined,
24
- };
25
- }
26
- }
27
- export function readFrom(path, charOffset) {
28
- try {
29
- const content = readFileSync(path, "utf-8");
30
- if (content.length <= charOffset)
31
- return { data: "", offset: charOffset };
32
- return { data: content.slice(charOffset), offset: content.length };
33
- }
34
- catch {
35
- return { data: "", offset: charOffset };
36
- }
37
- }
@@ -1,26 +0,0 @@
1
- /**
2
- * "All repos" tab panel — machine-wide registry table (index.sqlite).
3
- *
4
- * Rows are populated by the index poller in `script.ts` (shared with the
5
- * in-panel table on the Current-repo tab via the same render call).
6
- */
7
- export function allReposTab() {
8
- return `<!-- All repos (machine-wide registry from index.sqlite) -->
9
- <div class="tab-panel" id="panel-all">
10
- <table class="repos">
11
- <thead>
12
- <tr>
13
- <th>Repo</th><th>Model</th>
14
- <th style="text-align:right">Checkpoints</th>
15
- <th style="text-align:right">Tokens Saved</th>
16
- <th style="text-align:right">Retained</th>
17
- <th style="text-align:right">Last Compacted</th>
18
- </tr>
19
- </thead>
20
- <tbody id="all-rows"><tr><td colspan="6" class="repo-none">loading…</td></tr></tbody>
21
- </table>
22
- <div class="updated" id="all-updated"></div>
23
- </div>
24
-
25
- `;
26
- }
@@ -1,23 +0,0 @@
1
- /**
2
- * Body opening: </head><body>, offline banner, page heading, and the tab nav.
3
- *
4
- * `tierName` is interpolated into the heading tier badge.
5
- * Includes the new "High Score" future-tab button (next project).
6
- */
7
- export function bodyOpen(tierName) {
8
- return `</head>
9
- <body>
10
-
11
- <div class="offline-banner" id="offline-banner">Dashboard data unavailable — waiting for a pi session to write snapshot...</div>
12
-
13
- <h1><span>mega-compact</span><span class="tier">${tierName}</span><span class="model-pill" id="hdr-model">—</span></h1>
14
-
15
- <nav class="tabs">
16
- <button class="tab active" data-tab="current">Current repo</button>
17
- <button class="tab" data-tab="all">All repos</button>
18
- <button class="tab" data-tab="summary">Summary</button>
19
- <button class="tab future" data-tab="highscore">High Score<span class="soon">soon</span></button>
20
- </nav>
21
-
22
- `;
23
- }
@@ -1,130 +0,0 @@
1
- /**
2
- * "Current repo" tab panel — the per-repo dashboard view.
3
- *
4
- * Contains: context-window meter, trigger status, vector-store stats,
5
- * repo-wide stats, data-safety card, configuration, model & cost savings,
6
- * crew/agents, a legend, the live event stream, and the in-panel all-repos
7
- * table (shared rows with the All-repos tab via the index poller).
8
- *
9
- * `tierName` is interpolated into the Configuration card.
10
- */
11
- export function currentRepoTab(tierName) {
12
- return `<!-- Current repo (existing single-repo view) -->
13
- <div class="tab-panel" id="panel-current">
14
- <div class="grid">
15
- <div class="card">
16
- <h2>Context Window</h2>
17
- <div class="meter-label" id="ctx-pct">—</div>
18
- <div class="meter-track"><div class="meter-fill" id="ctx-bar" style="width:0%"></div></div>
19
- <div class="meter-sub" id="ctx-sub">waiting for data</div>
20
- </div>
21
- <div class="card">
22
- <h2>Trigger Status</h2>
23
- <div class="status-row"><div class="bullet" id="tr-armed"></div><span>Armed (context ≥ fast gate)</span></div>
24
- <div class="status-row"><div class="bullet" id="tr-ready"></div><span>Ready (tokens ≥ threshold)</span></div>
25
- <div class="state-text" id="tr-state">waiting</div>
26
- </div>
27
- <div class="card">
28
- <h2>Vector Store</h2>
29
- <div class="stat-grid">
30
- <span class="label" title="A saved summary of a chunk of your conversation that was compacted to free up space.">Checkpoints</span><span class="value" id="st-count">0</span>
31
- <span class="label" title="How much conversation we are currently holding as compact summaries (the 'memory' this extension keeps). Smaller is better.">Tokens Stored</span><span class="value" id="st-tokens">0</span>
32
- <span class="label" title="Total size of the original conversation text before it was compacted.">Original Tokens</span><span class="value" id="st-orig">0</span>
33
- <span class="label" title="How much conversation space we have freed up for you (original size minus the compact summary we kept).">Tokens Saved</span><span class="value" id="st-saved">0</span>
34
- <span class="label" title="How many times old context was automatically brought back into the conversation because it was relevant to what you were doing.">Injected</span><span class="value" id="st-injected">0</span>
35
- <span class="label" title="Of the times we recalled old context, how often it was actually on-topic.">Recall Relevance</span><span class="value" id="st-dedup">0%</span>
36
- <span class="label" title="How often new content matched something we already had, so we skipped storing a duplicate copy. Higher = less wasted space.">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
37
- <span class="label" title="How many duplicate chunks we collapsed into one instead of storing separately.">Collapsed</span><span class="value" id="st-collapsed">0</span>
38
- <span class="label" title="The ID of the most recent saved checkpoint.">Last ID</span><span class="value" id="st-lastid">—</span>
39
- </div>
40
- </div>
41
- <div class="card">
42
- <h2>Repo (all sessions)</h2>
43
- <div class="stat-grid">
44
- <span class="label">Checkpoints</span><span class="value" id="rp-count">0</span>
45
- <span class="label">Tokens Stored</span><span class="value" id="rp-tokens">0</span>
46
- <span class="label">Original Tokens</span><span class="value" id="rp-orig">0</span>
47
- <span class="label">Tokens Saved</span><span class="value" id="rp-saved">0</span>
48
- <span class="label">Sessions</span><span class="value" id="rp-sessions">0</span>
49
- <span class="label">Collapsed</span><span class="value" id="rp-collapsed">0</span>
50
- <span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
51
- </div>
52
- </div>
53
- <div class="card safe">
54
- <h2>🛡 Data Safety</h2>
55
- <div class="stat-grid">
56
- <span class="label">Regions Retained</span><span class="value" id="ig-retained">0</span>
57
- <span class="label">Compressed-Original</span><span class="value" id="ig-bytes">0 B</span>
58
- <span class="label">Dedup Duplicates</span><span class="value" id="ig-dupes">0</span>
59
- <span class="label">Permanently Deleted</span><span class="value ok" id="ig-deleted">0 B</span>
60
- </div>
61
- <p class="safe-note">Every compacted region is kept verbatim (compressed). "Drop" = removed from the live window only. We never delete your data.</p>
62
- </div>
63
- <div class="card">
64
- <h2>Configuration</h2>
65
- <div class="conf-grid">
66
- <span class="label">Tier</span><span class="value" id="cf-tier">${tierName}</span>
67
- <span class="label">Threshold</span><span class="value" id="cf-threshold">—</span>
68
- <span class="label">Fast Gate</span><span class="value" id="cf-gate">—</span>
69
- <span class="label">Auto</span><span class="value" id="cf-auto">—</span>
70
- <span class="label">Anchor</span><span class="value" id="cf-anchor">—</span>
71
- </div>
72
- </div>
73
- <div class="card cost">
74
- <h2>💰 Model &amp; Cost Savings</h2>
75
- <div class="cost-usd" id="cost-usd">≈ $0.00 saved</div>
76
- <div class="cost-sub" id="cost-windows">0 context-windows extended</div>
77
- <div class="stat-grid" style="margin-top:12px">
78
- <span class="label" title="The model pi is currently using — its pricing drives the cost figure.">Model</span><span class="value" id="md-name">—</span>
79
- <span class="label" title="The provider serving the model.">Provider</span><span class="value" id="md-provider">—</span>
80
- <span class="label" title="USD per input token, from the model's pricing.">Input Rate</span><span class="value" id="md-input">—</span>
81
- <span class="label" title="USD per output token, from the model's pricing.">Output Rate</span><span class="value" id="md-output">—</span>
82
- </div>
83
- </div>
84
- <div class="card">
85
- <h2>Crew / Agents</h2>
86
- <div class="stat-grid">
87
- <span class="label">Active Agents</span><span class="value" id="cr-agents">0</span>
88
- <span class="label">Current Turn</span><span class="value" id="cr-turn">0</span>
89
- <span class="label">Status</span><span class="value" id="cr-status">idle</span>
90
- </div>
91
- </div>
92
- <div class="card legend">
93
- <h2>What these numbers mean</h2>
94
- <ul class="legend-list">
95
- <li><b>Tokens saved</b> — conversation space this extension has freed up for you (it compacted old text into short summaries).</li>
96
- <li><b>Tokens stored</b> — how much "memory" (compact summaries) the extension is currently holding for this repo.</li>
97
- <li><b>Injected</b> — times old context was automatically pasted back in because it was relevant to your current task.</li>
98
- <li><b>Recall relevance</b> — of those, how often the recalled context was actually on-topic.</li>
99
- <li><b>Storage dedup</b> — how often new content matched something already saved, so a duplicate copy was skipped (saves space).</li>
100
- <li><b>Data safety</b> — every compacted region is kept verbatim (compressed). Nothing is permanently deleted; you can restore any of it.</li>
101
- </ul>
102
- <p class="legend-note">Hover any label above for a quick explanation.</p>
103
- </div>
104
- </div>
105
-
106
- <div class="events">
107
- <h2>Event Stream</h2>
108
- <div class="events-wrap" id="events"><div class="empty">connecting…</div></div>
109
- </div>
110
-
111
- <h2 style="margin-top:24px;font-size:13px;color:#8b949e;text-transform:uppercase;letter-spacing:.5px">All Repositories</h2>
112
- <table class="repos">
113
- <thead>
114
- <tr>
115
- <th>Repo</th><th>Model</th>
116
- <th style="text-align:right">Checkpoints</th>
117
- <th style="text-align:right">Tokens Saved</th>
118
- <th style="text-align:right">Retained</th>
119
- <th style="text-align:right">Last Compacted</th>
120
- </tr>
121
- </thead>
122
- <tbody id="cur-rows"><tr><td colspan="6" class="repo-none">loading…</td></tr></tbody>
123
- </table>
124
- <div class="updated" id="cur-updated"></div>
125
-
126
- <div class="updated" id="updated"></div>
127
- </div><!-- /panel-current -->
128
-
129
- `;
130
- }