pi-mega-compact 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -115,48 +115,64 @@ OpenAI-style contract and the `MEGACOMPACT_EMBEDDING_KEY` / `MEGACOMPACT_EMBEDDI
115
115
  compile). No network call and no API key are needed at runtime.
116
116
  - A pi coding agent install that loads extensions from `~/.pi/agent/extensions/`.
117
117
 
118
- ### From a git checkout
118
+ ### Install from npm (recommended)
119
119
 
120
120
  ```bash
121
- git clone https://github.com/TheArchitectit/pi-mega-compact.git \
122
- ~/.pi/agent/extensions/pi-mega-compact
123
- cd ~/.pi/agent/extensions/pi-mega-compact
124
- npm install
125
- npm run build
121
+ npm install pi-mega-compact
126
122
  ```
127
123
 
124
+ This places the package in `node_modules` and exposes the extension entry at
125
+ `node_modules/pi-mega-compact/extensions/mega-compact.ts`. Then point pi at it
126
+ (see "Register with pi" below).
127
+
128
128
  ### Register with pi
129
129
 
130
- Either copy/link the extension into pi's extensions dir (the clone above already
131
- does), **or** add it to your pi config's `pi.extensions` list:
130
+ Add the extension to your pi config's `pi.extensions` list, pointing at the
131
+ installed entry (npm path or a symlink into pi's extensions dir — either works):
132
132
 
133
133
  ```jsonc
134
134
  {
135
135
  "pi": {
136
- "extensions": ["~/.pi/agent/extensions/pi-mega-compact/extensions/mega-compact.ts"]
136
+ "extensions": ["pi-mega-compact/extensions/mega-compact.ts"]
137
137
  }
138
138
  }
139
139
  ```
140
140
 
141
- Or use the bundled helper (needs `jq`):
141
+ Or symlink the installed package into pi's extensions dir (the simplest path if
142
+ you run pi from the same machine):
142
143
 
143
144
  ```bash
144
- ./install.sh # copy into ~/.pi/agent/extensions/pi-mega-compact
145
- ./install.sh -s # symlink instead of copy (dev mode)
145
+ ln -s "$(npm root)/pi-mega-compact" ~/.pi/agent/extensions/pi-mega-compact
146
146
  ```
147
147
 
148
+ > **From a git checkout (development).** To hack on the extension, clone instead
149
+ > and build locally:
150
+ > ```bash
151
+ > git clone https://github.com/TheArchitectit/pi-mega-compact.git \
152
+ > ~/.pi/agent/extensions/pi-mega-compact
153
+ > cd ~/.pi/agent/extensions/pi-mega-compact
154
+ > npm install && npm run build
155
+ > ```
156
+ > The bundled `./install.sh` helper (`copy`) / `./install.sh -s` (`symlink`) does
157
+ > the same and also registers the path in pi's config (needs `jq`).
158
+
148
159
  ### Verify
149
160
 
150
161
  ```bash
151
- cd ~/.pi/agent/extensions/pi-mega-compact
152
- npm test # all unit/integration tests pass (192 as of v0.2.0)
162
+ npm test # all unit/integration tests pass (278 as of v0.4.0)
153
163
  npm run lint # tsc --noEmit + guardrails scan clean
154
164
  ```
155
165
 
156
166
  ### Uninstall
157
167
 
158
168
  ```bash
159
- rm -rf ~/.pi/agent/extensions/pi-mega-compact
169
+ npm uninstall pi-mega-compact
170
+ ```
171
+
172
+ If you symlinked it into pi's extensions dir, also remove that link:
173
+
174
+ ```bash
175
+ rm -f ~/.pi/agent/extensions/pi-mega-compact
160
176
  ```
161
177
 
162
178
  Then remove the path from pi's `pi.extensions` array.
@@ -53,8 +53,26 @@ interface Snapshot {
53
53
  store: {
54
54
  checkpointCount: number;
55
55
  totalTokenEstimate: number;
56
+ originalTokens: number;
57
+ tokensSaved: number;
56
58
  injectedCount: number;
57
59
  dedupHitRate: number;
60
+ storageDedupRate: number;
61
+ dedupCollapsed: number;
62
+ };
63
+ crew: {
64
+ activeAgents: number;
65
+ currentTurn: number;
66
+ };
67
+ repo: {
68
+ checkpointCount: number;
69
+ totalTokenEstimate: number;
70
+ originalTokens: number;
71
+ tokensSaved: number;
72
+ sessionCount: number;
73
+ dedupAttempts: number;
74
+ dedupCollapsed: number;
75
+ storageDedupRate: number;
58
76
  };
59
77
  }
60
78
 
@@ -75,7 +93,9 @@ function readSnapshot(snapshotPath: string) {
75
93
  session: { id: null, state: null, persistedThisSession: false, lastCheckpointId: null, lastCompactedFrom: 0 },
76
94
  context: { tokens: null, percent: null, contextWindow: 0 },
77
95
  trigger: { armed: false, ready: false, currentTokens: null, thresholdTokens: 100_000, fastGatePct: 80 },
78
- store: { checkpointCount: 0, totalTokenEstimate: 0, injectedCount: 0, dedupHitRate: 0 },
96
+ store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
97
+ crew: { activeAgents: 0, currentTurn: 0 },
98
+ repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
79
99
  } as Snapshot;
80
100
  }
81
101
  }
@@ -166,12 +186,28 @@ function dashboardHtml(tierName: string): string {
166
186
  <h2>Vector Store</h2>
167
187
  <div class="stat-grid">
168
188
  <span class="label">Checkpoints</span><span class="value" id="st-count">0</span>
169
- <span class="label">Total Tokens</span><span class="value" id="st-tokens">0</span>
189
+ <span class="label">Tokens Stored</span><span class="value" id="st-tokens">0</span>
190
+ <span class="label">Original Tokens</span><span class="value" id="st-orig">0</span>
191
+ <span class="label">Tokens Saved</span><span class="value" id="st-saved">0</span>
170
192
  <span class="label">Injected</span><span class="value" id="st-injected">0</span>
171
193
  <span class="label">Dedup Rate</span><span class="value" id="st-dedup">0%</span>
194
+ <span class="label">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
195
+ <span class="label">Collapsed</span><span class="value" id="st-collapsed">0</span>
172
196
  <span class="label">Last ID</span><span class="value" id="st-lastid">—</span>
173
197
  </div>
174
198
  </div>
199
+ <div class="card">
200
+ <h2>Repo (all sessions)</h2>
201
+ <div class="stat-grid">
202
+ <span class="label">Checkpoints</span><span class="value" id="rp-count">0</span>
203
+ <span class="label">Tokens Stored</span><span class="value" id="rp-tokens">0</span>
204
+ <span class="label">Original Tokens</span><span class="value" id="rp-orig">0</span>
205
+ <span class="label">Tokens Saved</span><span class="value" id="rp-saved">0</span>
206
+ <span class="label">Sessions</span><span class="value" id="rp-sessions">0</span>
207
+ <span class="label">Collapsed</span><span class="value" id="rp-collapsed">0</span>
208
+ <span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
209
+ </div>
210
+ </div>
175
211
  <div class="card">
176
212
  <h2>Configuration</h2>
177
213
  <div class="conf-grid">
@@ -182,6 +218,14 @@ function dashboardHtml(tierName: string): string {
182
218
  <span class="label">Anchor</span><span class="value" id="cf-anchor">—</span>
183
219
  </div>
184
220
  </div>
221
+ <div class="card">
222
+ <h2>Crew / Agents</h2>
223
+ <div class="stat-grid">
224
+ <span class="label">Active Agents</span><span class="value" id="cr-agents">0</span>
225
+ <span class="label">Current Turn</span><span class="value" id="cr-turn">0</span>
226
+ <span class="label">Status</span><span class="value" id="cr-status">idle</span>
227
+ </div>
228
+ </div>
185
229
  </div>
186
230
 
187
231
  <div class="events">
@@ -223,10 +267,33 @@ function dashboardHtml(tierName: string): string {
223
267
 
224
268
  document.getElementById('st-count').textContent = d.store.checkpointCount;
225
269
  document.getElementById('st-tokens').textContent = d.store.totalTokenEstimate.toLocaleString();
270
+ document.getElementById('st-orig').textContent = (d.store.originalTokens || 0).toLocaleString();
271
+ document.getElementById('st-saved').textContent = (d.store.tokensSaved || 0).toLocaleString();
226
272
  document.getElementById('st-injected').textContent = d.store.injectedCount;
227
273
  document.getElementById('st-dedup').textContent = Math.round(d.store.dedupHitRate * 100) + '%';
274
+ var sdr = d.store.storageDedupRate || 0;
275
+ document.getElementById('st-sdedup').textContent = (sdr * 100 >= 10 ? Math.round(sdr * 100) : (sdr * 100).toFixed(1)) + '%';
276
+ document.getElementById('st-collapsed').textContent = d.store.dedupCollapsed || 0;
228
277
  document.getElementById('st-lastid').textContent = d.session.lastCheckpointId || '—';
229
278
 
279
+ // Repo-wide (all sessions in this repo's SQLite store).
280
+ var repo = d.repo || { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupCollapsed: 0, storageDedupRate: 0 };
281
+ document.getElementById('rp-count').textContent = repo.checkpointCount;
282
+ document.getElementById('rp-tokens').textContent = repo.totalTokenEstimate.toLocaleString();
283
+ document.getElementById('rp-orig').textContent = (repo.originalTokens || 0).toLocaleString();
284
+ document.getElementById('rp-saved').textContent = (repo.tokensSaved || 0).toLocaleString();
285
+ document.getElementById('rp-sessions').textContent = repo.sessionCount || 0;
286
+ document.getElementById('rp-collapsed').textContent = repo.dedupCollapsed || 0;
287
+ var rsdr = repo.storageDedupRate || 0;
288
+ document.getElementById('rp-sdedup').textContent = (rsdr * 100 >= 10 ? Math.round(rsdr * 100) : (rsdr * 100).toFixed(1)) + '%';
289
+
290
+ // Crew / agents (live sub-agent activity + turn).
291
+ var crew = d.crew || { activeAgents: 0, currentTurn: 0 };
292
+ document.getElementById('cr-agents').textContent = crew.activeAgents || 0;
293
+ document.getElementById('cr-turn').textContent = crew.currentTurn || 0;
294
+ document.getElementById('cr-status').textContent = (crew.activeAgents > 0)
295
+ ? ('▶ ' + crew.activeAgents + ' running') : 'idle';
296
+
230
297
  document.getElementById('cf-tier').textContent = d.tier;
231
298
  document.getElementById('cf-threshold').textContent = d.config.thresholdTokens.toLocaleString();
232
299
  document.getElementById('cf-gate').textContent = d.config.fastGatePct + '%';
@@ -322,9 +322,24 @@ test("state snapshot writes dashboard.json after compaction", async () => {
322
322
  const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
323
323
  // Fire auto-trigger compaction (context event above 80% threshold)
324
324
  await h.fire("context", { type: "context", messages: h.session }, ctx);
325
- const { existsSync: ex } = await import("node:fs");
325
+ const { existsSync: ex, readFileSync: rf } = await import("node:fs");
326
326
  const { join: j } = await import("node:path");
327
- assert.ok(ex(j(h.stateDir, "dashboard.json")), "dashboard.json written after compaction");
327
+ const snapPath = j(h.stateDir, "dashboard.json");
328
+ assert.ok(ex(snapPath), "dashboard.json written after compaction");
329
+ const snap = JSON.parse(rf(snapPath, "utf-8"));
330
+ // Item B: the honest token model is wired — the original dropped region was
331
+ // captured (originalTokens > 0), and the saved amount never exceeds the
332
+ // original (saved = max(0, original − stored) ≤ original). For this tiny
333
+ // harness session the summary can be ≥ the region, so saved may be 0; the
334
+ // positive "saved > 0" case with a large region is covered by the
335
+ // vectorStore unit tests.
336
+ assert.ok(snap.store.originalTokens > 0, "snapshot.store.originalTokens captured after compaction");
337
+ assert.ok(
338
+ snap.store.originalTokens >= snap.store.tokensSaved,
339
+ "model invariant: original region >= tokens saved",
340
+ );
341
+ // Item A: crew (live agent) block is present in the dashboard snapshot.
342
+ assert.ok(snap.crew && typeof snap.crew.activeAgents === "number", "snapshot.crew.activeAgents present");
328
343
  });
329
344
 
330
345
  test("events.log receives compaction events", async () => {
@@ -37,6 +37,7 @@ import { recallAndInline } from "../src/recall.js";
37
37
  import { autoCompactCheck } from "../src/compact.js";
38
38
  import { estimateSessionTokens } from "../src/tokens.js";
39
39
  import { normalizeSessionId } from "../src/store.js";
40
+ import { touchSession, logDaily } from "../src/store/sqlite.js";
40
41
  import { Logger } from "../src/log.js";
41
42
  import type { EngineMessage } from "../src/types.js";
42
43
  import { writeFileSync, appendFileSync, readFileSync } from "node:fs";
@@ -57,6 +58,7 @@ interface SessionRuntime {
57
58
  lastCompactedTokens: number;
58
59
  dedupSkips: number; // compactions skipped because regionHash already stored
59
60
  dedupAttempts: number; // total compaction attempts (for hit-rate denominator)
61
+ tokensSaved: number; // this session-instance only: reset on session_start
60
62
  }
61
63
 
62
64
  function envFlag(name: string, fallback: number): number {
@@ -190,12 +192,28 @@ interface DashboardSnapshot {
190
192
  store: {
191
193
  checkpointCount: number;
192
194
  totalTokenEstimate: number;
195
+ originalTokens: number; // Σ original dropped-region tokens (this session)
196
+ tokensSaved: number; // Σ(original − stored) for this session
193
197
  injectedCount: number;
194
198
  dedupHitRate: number;
195
199
  storageDedupRate: number;
196
200
  dedupAttempts: number;
197
201
  dedupCollapsed: number;
198
202
  };
203
+ crew: {
204
+ activeAgents: number;
205
+ currentTurn: number;
206
+ };
207
+ repo: {
208
+ checkpointCount: number; // across all sessions in this repo's store
209
+ totalTokenEstimate: number; // repo-wide stored checkpoint tokens
210
+ originalTokens: number; // repo-wide Σ original dropped-region tokens
211
+ tokensSaved: number; // repo-wide cumulative (original − stored) + deduped orig
212
+ sessionCount: number; // distinct sessions with checkpoints
213
+ dedupAttempts: number; // cumulative add() calls (store-wide)
214
+ dedupCollapsed: number; // cumulative deduped collapses (store-wide)
215
+ storageDedupRate: number; // deduped / attempts, 0..1
216
+ };
199
217
  }
200
218
 
201
219
  class Dashboard {
@@ -261,6 +279,7 @@ export default function (pi: ExtensionAPI) {
261
279
  function snapshot(ctx?: ExtensionContext): void {
262
280
  if (ctx) bindRepo(ctx.cwd);
263
281
  const st = store.stats(rt.sessionId);
282
+ const repo = store.repoStats();
264
283
  const armed = lastCtxPercent != null && lastCtxPercent >= config.fastGatePct;
265
284
  const ready = armed && (lastCtxTokens ?? 0) >= config.thresholdTokens;
266
285
  dashboard.snapshot({
@@ -287,7 +306,18 @@ export default function (pi: ExtensionAPI) {
287
306
  },
288
307
  context: { tokens: lastCtxTokens, percent: lastCtxPercent, contextWindow: lastCtxWindow },
289
308
  trigger: { armed, ready, currentTokens: lastCtxTokens, thresholdTokens: config.thresholdTokens, fastGatePct: config.fastGatePct },
290
- store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
309
+ crew: { activeAgents, currentTurn },
310
+ store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, originalTokens: st.originalTokens, tokensSaved: rt.tokensSaved, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
311
+ repo: {
312
+ checkpointCount: repo.checkpointCount,
313
+ totalTokenEstimate: repo.totalTokenEstimate,
314
+ originalTokens: repo.originalTokens,
315
+ tokensSaved: repo.tokensSaved,
316
+ sessionCount: repo.sessionCount,
317
+ dedupAttempts: repo.dedupAttempts,
318
+ dedupCollapsed: repo.dedupCollapsed,
319
+ storageDedupRate: repo.storageDedupRate,
320
+ },
291
321
  });
292
322
 
293
323
  // Live stats widget above the editor
@@ -303,7 +333,12 @@ export default function (pi: ExtensionAPI) {
303
333
  const dedupStr = storageRate * 100 >= 10
304
334
  ? `${Math.round(storageRate * 100)}%`
305
335
  : `${(storageRate * 100).toFixed(1)}%`;
306
- const savedStr = st.totalTokenEstimate > 0 ? `${Math.round(st.totalTokenEstimate / 1000)}k` : "0";
336
+ // saved = cumulative original stored tokens (this session). Show real
337
+ // token counts; use "k" only at/above 1000 so small-but-real savings are
338
+ // visible (previously Math.round(x/1000) rounded everything <1000 to 0).
339
+ const savedStr = rt.tokensSaved >= 1000
340
+ ? `${(rt.tokensSaved / 1000).toFixed(1)}k`
341
+ : `${rt.tokensSaved}`;
307
342
  const agentStr = activeAgents > 0 ? ` │ 🤖 ${activeAgents} agent${activeAgents === 1 ? "" : "s"}` : "";
308
343
  const turnStr = currentTurn > 0 ? ` │ turn ${currentTurn}` : "";
309
344
  ctx.ui.setWidget(
@@ -326,6 +361,7 @@ export default function (pi: ExtensionAPI) {
326
361
  lastCompactedTokens: 0,
327
362
  dedupSkips: 0,
328
363
  dedupAttempts: 0,
364
+ tokensSaved: 0,
329
365
  };
330
366
  let debounceUntil = 0;
331
367
  // Agent tracking for real-time widget updates
@@ -352,6 +388,7 @@ export default function (pi: ExtensionAPI) {
352
388
  lastCompactedTokens: 0,
353
389
  dedupSkips: 0,
354
390
  dedupAttempts: 0,
391
+ tokensSaved: 0,
355
392
  };
356
393
  statusKey = undefined;
357
394
  activeAgents = 0;
@@ -392,8 +429,28 @@ export default function (pi: ExtensionAPI) {
392
429
  rt.lastCompactedFrom = result.compactedFrom;
393
430
  rt.lastCompactedTokens = result.tokenEstimate;
394
431
  rt.dedupAttempts++;
432
+ // Honest "tokens saved" for this session-instance only:
433
+ // new checkpoint → original − stored
434
+ // deduped onto existing → whole original region (nothing new stored)
435
+ // Resets to 0 on session_start (rt is rebuilt) — so a fresh session shows 0
436
+ // while the repo's cumulative saved (SQLite meta) keeps the running total.
437
+ const saved = result.deduped
438
+ ? result.originalTokenEstimate
439
+ : Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
440
+ rt.tokensSaved += saved;
395
441
  if (result.deduped) rt.dedupSkips++;
396
442
 
443
+ // Record session activity + a daily-log entry in the per-repo SQLite store
444
+ // (foundation for resume-sessions / daily-log features). Best-effort — never
445
+ // block a compaction on bookkeeping.
446
+ try {
447
+ const repo = resolveRepoRoot(ctx.cwd);
448
+ touchSession(sid, repo, currentStateDir);
449
+ logDaily(sid, "compact", result.checkpointId, saved, currentStateDir);
450
+ } catch {
451
+ /* non-fatal: stats bookkeeping only */
452
+ }
453
+
397
454
  // Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
398
455
  // skip re-vectorizing an already-compacted region (zero token cost).
399
456
  pi.appendEntry(MARKER_TYPE, {
@@ -403,7 +460,6 @@ export default function (pi: ExtensionAPI) {
403
460
  deduped: result.deduped,
404
461
  });
405
462
 
406
- const saved = result.tokenEstimate;
407
463
  setStatus(
408
464
  ctx,
409
465
  rt.persistedThisSession
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.4.0",
3
+ "version": "0.4.2",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-2-Clause",
package/src/engine.ts CHANGED
@@ -15,7 +15,7 @@
15
15
  import { findSuperseded, supersede } from "./supersede.js";
16
16
  import { summarizeMessages, mergeCompactSummaries, formatCompactSummary } from "./compact.js";
17
17
  import { extractiveSummarize } from "./extractive.js";
18
- import { estimateSessionTokens } from "./tokens.js";
18
+ import { estimateSessionTokens, estimateBlockTokens } from "./tokens.js";
19
19
  import { computeRegionHash, VectorStore, type SearchHit } from "./vectorStore.js";
20
20
  import type { EngineMessage } from "./types.js";
21
21
 
@@ -51,6 +51,11 @@ export interface CompactResult {
51
51
  summary: string;
52
52
  regionHash: string;
53
53
  tokenEstimate: number;
54
+ /** Token count of the original dropped region (before compaction). The honest
55
+ * "tokens saved" base = originalTokenEstimate − tokenEstimate (stored), or the
56
+ * full originalTokenEstimate when the region deduped onto an existing
57
+ * checkpoint (nothing new stored). */
58
+ originalTokenEstimate: number;
54
59
  /** Index in `messages` where the compacted slice begins (for the caller to
55
60
  * build a drop range). */
56
61
  compactedFrom: number;
@@ -87,6 +92,7 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
87
92
  summary: "",
88
93
  regionHash: "",
89
94
  tokenEstimate: 0,
95
+ originalTokenEstimate: 0,
90
96
  compactedFrom,
91
97
  };
92
98
  }
@@ -105,7 +111,6 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
105
111
  let keyDecisions: string[];
106
112
  let nextSteps: string[];
107
113
  let filesModified: string[];
108
- let tokenEstimate: number;
109
114
 
110
115
  if (useExtractive && !input.summary) {
111
116
  const ext = extractiveSummarize(keep);
@@ -114,7 +119,6 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
114
119
  keyDecisions = input.keyDecisions ?? ext.keyDecisions;
115
120
  nextSteps = input.nextSteps ?? ext.nextSteps;
116
121
  filesModified = input.filesModified ?? ext.filesModified;
117
- tokenEstimate = input.tokenEstimate ?? ext.tokenEstimate;
118
122
  } else {
119
123
  const collapsed = input.summary ?? summarizeMessages(keep);
120
124
  summary = formatCompactSummary(collapsed);
@@ -122,9 +126,18 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
122
126
  keyDecisions = input.keyDecisions ?? [];
123
127
  nextSteps = input.nextSteps ?? [];
124
128
  filesModified = input.filesModified ?? [];
125
- tokenEstimate = input.tokenEstimate ?? estimateSessionTokens(compactable);
126
129
  }
127
130
 
131
+ // Honest "tokens saved" accounting:
132
+ // - originalTokenEstimate = the dropped region's token count (what context
133
+ // held before compaction) = the compacted slice's tokens.
134
+ // - storedTokens = the persisted summary's token count, computed from the
135
+ // actual summary string so it's honest for BOTH the extractive and legacy
136
+ // COLLAPSE paths (the legacy path's fallback estimateSessionTokens is the
137
+ // *original* size, not the stored size).
138
+ const originalTokenEstimate = estimateSessionTokens(compactable);
139
+ const storedTokens = estimateBlockTokens(summary);
140
+
128
141
  // Region text = the compacted slice, used for dedup + embedding.
129
142
  const regionText = input.regionText ?? keep.map((m) => m.text).join("\n");
130
143
  const regionHash = computeRegionHash(regionText);
@@ -137,7 +150,8 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
137
150
  nextSteps,
138
151
  filesModified,
139
152
  regionText,
140
- tokenEstimate,
153
+ tokenEstimate: storedTokens,
154
+ originalTokenEstimate,
141
155
  timestamp: input.timestamp ?? 0,
142
156
  });
143
157
 
@@ -148,7 +162,8 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
148
162
  checkpointId: add.checkpoint.checkpointId,
149
163
  summary,
150
164
  regionHash,
151
- tokenEstimate,
165
+ tokenEstimate: storedTokens,
166
+ originalTokenEstimate,
152
167
  compactedFrom,
153
168
  };
154
169
  }
@@ -80,6 +80,7 @@ function initSchema(db: Database.Database): void {
80
80
  files_modified TEXT, -- JSON array
81
81
  embedding_blob BLOB, -- float32 vector
82
82
  token_estimate INTEGER,
83
+ original_token_estimate INTEGER, -- dropped region size (tokens saved = orig − stored)
83
84
  timestamp INTEGER,
84
85
  dedup_status TEXT DEFAULT 'active',
85
86
  compressed_original BLOB -- optional DR copy
@@ -145,6 +146,49 @@ function initSchema(db: Database.Database): void {
145
146
  );
146
147
  CREATE INDEX IF NOT EXISTS idx_raptor_session ON raptor_nodes(session_id);
147
148
  CREATE INDEX IF NOT EXISTS idx_raptor_parent ON raptor_nodes(parent_id);
149
+
150
+ -- Foundation for future features (resume sessions, daily log, lessons
151
+ -- learned). Scaffolded now so all store data lives in SQLite from day one;
152
+ -- population is minimal (touchSession / logDaily on compact) and the full
153
+ -- UI/recall for these lands in later sprints.
154
+
155
+ -- Per-session registry (resume + per-repo session history).
156
+ CREATE TABLE IF NOT EXISTS sessions (
157
+ session_id TEXT PRIMARY KEY,
158
+ repo TEXT,
159
+ started_at INTEGER,
160
+ ended_at INTEGER,
161
+ last_compacted_at INTEGER,
162
+ status TEXT DEFAULT 'active'
163
+ );
164
+
165
+ -- Append-only daily activity log (the "daily log" feature seed).
166
+ CREATE TABLE IF NOT EXISTS daily_log (
167
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
168
+ day TEXT NOT NULL, -- YYYY-MM-DD
169
+ session_id TEXT,
170
+ event TEXT, -- e.g. 'compact'
171
+ detail TEXT,
172
+ tokens_saved INTEGER DEFAULT 0,
173
+ ts INTEGER
174
+ );
175
+ CREATE INDEX IF NOT EXISTS idx_daily_log_day ON daily_log(day);
176
+
177
+ -- Lessons learned (future recall/browse feature seed).
178
+ CREATE TABLE IF NOT EXISTS lessons (
179
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
180
+ session_id TEXT,
181
+ repo TEXT,
182
+ lesson TEXT,
183
+ ts INTEGER
184
+ );
185
+
186
+ -- FTS5 trigram virtual table (Sprint 9+ pg_trgm-equivalent verification).
187
+ CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_trgm USING fts5(
188
+ id UNINDEXED,
189
+ normalized_text,
190
+ tokenize='trigram'
191
+ );
148
192
  `);
149
193
  const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get() as
150
194
  | { value: string }
@@ -152,15 +196,139 @@ function initSchema(db: Database.Database): void {
152
196
  if (!v) {
153
197
  db.prepare("INSERT INTO meta(key, value) VALUES(?, ?)").run("schema_version", String(SCHEMA_VERSION));
154
198
  }
199
+ }
155
200
 
156
- // FTS5 trigram virtual table (Sprint 9+ pg_trgm-equivalent verification).
157
- db.exec(`
158
- CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_trgm USING fts5(
159
- id UNINDEXED,
160
- normalized_text,
161
- tokenize='trigram'
162
- );
163
- `);
201
+ /** Read a string-valued meta key (or undefined). Used for cumulative counters. */
202
+ export function getMeta(key: string, stateDir: string = getStateDir()): string | undefined {
203
+ const db = openStore(stateDir);
204
+ const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key) as
205
+ | { value: string }
206
+ | undefined;
207
+ return row?.value;
208
+ }
209
+
210
+ /**
211
+ * Cumulative "tokens saved" — the sum of stored checkpoint token estimates across
212
+ * all compactions in this store (one per repo). Persisted in the SQLite `meta`
213
+ * table so it survives session restarts and travels with the repo's state dir,
214
+ * mirroring how `storageDedupRate` is cumulative. Incremented in VectorStore.add()
215
+ * when a new (non-deduped) checkpoint is persisted.
216
+ */
217
+ export function getTokensSaved(stateDir: string = getStateDir()): number {
218
+ const raw = getMeta("tokens_saved", stateDir);
219
+ const n = raw == null ? 0 : Number(raw);
220
+ return Number.isFinite(n) ? n : 0;
221
+ }
222
+
223
+ /** Add `delta` (>=0) to the cumulative tokens-saved counter. */
224
+ export function addTokensSaved(delta: number, stateDir: string = getStateDir()): void {
225
+ if (!(delta > 0)) return;
226
+ const db = openStore(stateDir);
227
+ db.prepare(
228
+ `INSERT INTO meta(key, value) VALUES('tokens_saved', ?)
229
+ ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
230
+ ).run(String(delta), delta);
231
+ }
232
+
233
+ /** Cumulative store-wide dedup accounting (Sprint 9+). Persisted in the SQLite
234
+ * `meta` table so it survives session restarts and travels with the repo's
235
+ * state dir — mirroring `tokens_saved`. Replaces the legacy JSON
236
+ * `dedup-stats.json` file (all stats now live in the SQLite store). */
237
+ export interface DedupStats {
238
+ /** Total add() calls (new checkpoints + deduped collapses). */
239
+ attempts: number;
240
+ /** add() calls that collapsed onto an existing checkpoint. */
241
+ deduped: number;
242
+ }
243
+
244
+ /** Read a store-wide integer counter from the meta table (0 if absent). */
245
+ export function getMetaNumber(key: string, stateDir: string = getStateDir()): number {
246
+ const raw = getMeta(key, stateDir);
247
+ const n = raw == null ? 0 : Number(raw);
248
+ return Number.isFinite(n) ? n : 0;
249
+ }
250
+
251
+ /** Atomically add `delta` to an integer meta counter. */
252
+ function incMeta(key: string, delta: number, stateDir: string = getStateDir()): void {
253
+ if (!(delta > 0)) return;
254
+ const db = openStore(stateDir);
255
+ db.prepare(
256
+ `INSERT INTO meta(key, value) VALUES(?, ?)
257
+ ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
258
+ ).run(key, String(delta), delta);
259
+ }
260
+
261
+ /** Read the cumulative store-wide dedup counters. */
262
+ export function getDedupStats(stateDir: string = getStateDir()): DedupStats {
263
+ return {
264
+ attempts: getMetaNumber("dedup_attempts", stateDir),
265
+ deduped: getMetaNumber("deduped", stateDir),
266
+ };
267
+ }
268
+
269
+ /** Increment the store-wide dedup counters for one add() call. */
270
+ export function bumpDedupStats(deduped: boolean, stateDir: string = getStateDir()): void {
271
+ incMeta("dedup_attempts", 1, stateDir);
272
+ if (deduped) incMeta("deduped", 1, stateDir);
273
+ }
274
+
275
+ // --- Future-feature foundation (resume sessions / daily log / lessons) -------
276
+ // Scaffolded tables + minimal helpers so all store data lives in SQLite from
277
+ // day one. Full UI/recall for these lands in later sprints.
278
+
279
+ /** Upsert a `sessions` row (resume + per-repo session history). */
280
+ export function touchSession(
281
+ sessionId: string,
282
+ repo: string | undefined,
283
+ stateDir: string = getStateDir(),
284
+ ): void {
285
+ const db = openStore(stateDir);
286
+ const sid = normalizeSessionId(sessionId);
287
+ const existing = db
288
+ .prepare("SELECT started_at FROM sessions WHERE session_id = ?")
289
+ .get(sid) as { started_at: number | null } | undefined;
290
+ const now = Math.floor(Date.now() / 1000);
291
+ if (!existing) {
292
+ db.prepare(
293
+ `INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
294
+ VALUES(?, ?, ?, ?, 'active')`,
295
+ ).run(sid, repo ?? null, now, now);
296
+ } else {
297
+ db.prepare(
298
+ "UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?",
299
+ ).run(now, repo ?? null, sid);
300
+ }
301
+ }
302
+
303
+ /** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
304
+ export function logDaily(
305
+ sessionId: string,
306
+ event: string,
307
+ detail: string | undefined,
308
+ tokensSaved: number,
309
+ stateDir: string = getStateDir(),
310
+ ): void {
311
+ const db = openStore(stateDir);
312
+ const day = new Date().toISOString().slice(0, 10);
313
+ const now = Math.floor(Date.now() / 1000);
314
+ db.prepare(
315
+ `INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
316
+ VALUES(?, ?, ?, ?, ?, ?)`,
317
+ ).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
318
+ }
319
+
320
+ /** Append a `lessons` entry (future lessons-learned browse/recall). */
321
+ export function addLesson(
322
+ sessionId: string,
323
+ repo: string | undefined,
324
+ lesson: string,
325
+ stateDir: string = getStateDir(),
326
+ ): void {
327
+ const db = openStore(stateDir);
328
+ const now = Math.floor(Date.now() / 1000);
329
+ db.prepare(
330
+ `INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`,
331
+ ).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
164
332
  }
165
333
 
166
334
  /** Map a DB row to the public StoredCheckpoint shape. */
@@ -175,6 +343,7 @@ function rowToCheckpoint(row: any): StoredCheckpoint {
175
343
  nextSteps: row.next_steps ? JSON.parse(row.next_steps) : [],
176
344
  filesModified: row.files_modified ? JSON.parse(row.files_modified) : [],
177
345
  tokenEstimate: row.token_estimate ?? 0,
346
+ originalTokenEstimate: row.original_token_estimate ?? undefined,
178
347
  regionHash: row.region_hash ?? "",
179
348
  contentHash: row.content_hash ?? undefined,
180
349
  contentHash2: row.content_hash2 ?? undefined,
@@ -197,11 +366,11 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
197
366
  (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
198
367
  normalized_text, summary, topic_summary, summary_hash,
199
368
  key_decisions, next_steps, files_modified, embedding_blob,
200
- token_estimate, timestamp, dedup_status, compressed_original)
369
+ token_estimate, original_token_estimate, timestamp, dedup_status, compressed_original)
201
370
  VALUES (@id, @sid, @region_hash, @content_hash, @content_hash2, @content_hash_version,
202
371
  @normalized_text, @summary, @topic_summary, @summary_hash,
203
372
  @key_decisions, @next_steps, @files_modified, @embedding_blob,
204
- @token_estimate, @timestamp, @dedup_status, @compressed_original)
373
+ @token_estimate, @original_token_estimate, @timestamp, @dedup_status, @compressed_original)
205
374
  ON CONFLICT(session_id, id) DO UPDATE SET
206
375
  summary=excluded.summary,
207
376
  topic_summary=excluded.topic_summary,
@@ -211,6 +380,7 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
211
380
  files_modified=excluded.files_modified,
212
381
  embedding_blob=excluded.embedding_blob,
213
382
  token_estimate=excluded.token_estimate,
383
+ original_token_estimate=excluded.original_token_estimate,
214
384
  timestamp=excluded.timestamp,
215
385
  dedup_status=excluded.dedup_status,
216
386
  compressed_original=excluded.compressed_original`,
@@ -230,6 +400,7 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
230
400
  files_modified: jsonText(cp.filesModified),
231
401
  embedding_blob: encodeEmbedding(cp.embedding ?? []),
232
402
  token_estimate: cp.tokenEstimate ?? 0,
403
+ original_token_estimate: cp.originalTokenEstimate ?? null,
233
404
  timestamp: cp.timestamp ?? 0,
234
405
  dedup_status: "active",
235
406
  compressed_original: cp.compressedOriginal ?? null,
@@ -423,6 +594,52 @@ export function storeStats(sessionId: string, stateDir: string = getStateDir()):
423
594
  };
424
595
  }
425
596
 
597
+ /** Repo-wide stats — aggregates every session in this store (one per repo).
598
+ * Backed by the SQLite `meta` cumulative counters (`tokens_saved`,
599
+ * `dedup_attempts`, `deduped`) plus a SUM over all `context_chunks`. This is the
600
+ * cumulative, resumable, cross-device view the dashboard surfaces as "Repo …". */
601
+ export interface RepoStats {
602
+ /** Total checkpoints across all sessions (excludes SemDeDup-removed rows). */
603
+ checkpointCount: number;
604
+ /** Sum of all stored checkpoint token estimates (repo-wide). */
605
+ totalTokenEstimate: number;
606
+ /** Total active sessions with at least one checkpoint. */
607
+ sessionCount: number;
608
+ /** Cumulative stored-summary tokens saved (Σ stored summaries). */
609
+ tokensSaved: number;
610
+ /** Sum of original dropped-region token estimates (repo-wide). */
611
+ originalTokens: number;
612
+ /** Cumulative dedup add() attempts (store-wide). */
613
+ dedupAttempts: number;
614
+ /** Cumulative deduped collapses (store-wide). */
615
+ dedupCollapsed: number;
616
+ /** Storage dedup rate (deduped / attempts), 0..1. */
617
+ storageDedupRate: number;
618
+ }
619
+
620
+ export function repoStats(stateDir: string = getStateDir()): RepoStats {
621
+ const db = openStore(stateDir);
622
+ const row = db
623
+ .prepare(
624
+ `SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
625
+ COALESCE(SUM(original_token_estimate),0) AS orig,
626
+ COUNT(DISTINCT session_id) AS sessions
627
+ FROM context_chunks WHERE dedup_status != 'removed'`,
628
+ )
629
+ .get() as { c: number; tok: number; orig: number; sessions: number };
630
+ const ds = getDedupStats(stateDir);
631
+ return {
632
+ checkpointCount: row.c,
633
+ totalTokenEstimate: row.tok,
634
+ originalTokens: row.orig,
635
+ sessionCount: row.sessions,
636
+ tokensSaved: getMetaNumber("tokens_saved", stateDir),
637
+ dedupAttempts: ds.attempts,
638
+ dedupCollapsed: ds.deduped,
639
+ storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
640
+ };
641
+ }
642
+
426
643
  /** Close and evict a cached connection (test teardown only). */
427
644
  export function closeStore(stateDir: string): void {
428
645
  const db = cache.get(stateDir);
package/src/store.ts CHANGED
@@ -58,6 +58,9 @@ export interface StoredCheckpoint {
58
58
  nextSteps: string[];
59
59
  filesModified: string[];
60
60
  tokenEstimate: number;
61
+ /** Token count of the ORIGINAL dropped region (before compaction). Drives the
62
+ * honest "tokens saved" = originalTokenEstimate − tokenEstimate. */
63
+ originalTokenEstimate?: number;
61
64
  regionHash: string;
62
65
  /** Primary content-addressable hash (full 64-hex SHA-256 of normalized text). */
63
66
  contentHash?: string;
@@ -168,25 +171,8 @@ export function saveSessionState(sessionId: string, state: SessionState, stateDi
168
171
  }
169
172
 
170
173
  /**
171
- * Cumulative store-wide dedup accounting. Lives outside the per-session
172
- * SessionState (which resets on every session instance) so the dedup rate
173
- * stays stable across session restarts. Persisted as plain JSON in the
174
- * state dir.
174
+ * Cumulative store-wide dedup accounting now lives in the SQLite `meta` table
175
+ * (see store/sqlite.ts: getDedupStats / bumpDedupStats). All store stats are
176
+ * SQLite-backed so they survive session restarts and travel with the repo's
177
+ * state dir. The legacy JSON `dedup-stats.json` path was removed.
175
178
  */
176
- export interface DedupStats {
177
- /** Total add() calls (new checkpoints + deduped collapses). */
178
- attempts: number;
179
- /** add() calls that collapsed onto an existing checkpoint. */
180
- deduped: number;
181
- }
182
-
183
- const DEDUP_STATS_FILE = "dedup-stats.json";
184
-
185
- export function loadDedupStats(stateDir: string = getStateDir()): DedupStats {
186
- const file = join(stateDir, DEDUP_STATS_FILE);
187
- return readGzJson<DedupStats>(file, { attempts: 0, deduped: 0 });
188
- }
189
-
190
- export function saveDedupStats(stats: DedupStats, stateDir: string = getStateDir()): void {
191
- writeGzJson(join(stateDir, DEDUP_STATS_FILE), stats);
192
- }
@@ -250,6 +250,59 @@ test("stats reports counts, last checkpoint, and dedup rate", () => {
250
250
  assert.ok(Math.abs(st2.dedupHitRate - 0.5) < 1e-9);
251
251
  });
252
252
 
253
+ test("tokensSaved = original − stored per session; deduped add saves the whole region", () => {
254
+ const s = store();
255
+ // Two genuinely new checkpoints. saved = original − stored.
256
+ // cp1: orig 2000, stored 500 → saved 1500
257
+ // cp2: orig 3000, stored 700 → saved 2300
258
+ s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
259
+ s.add({ sessionId: "sess_saved", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
260
+ const st = s.stats("sess_saved");
261
+ assert.equal(st.totalTokenEstimate, 1200, "Σ stored summaries");
262
+ assert.equal(st.originalTokens, 5000, "Σ original region tokens");
263
+ assert.equal(st.tokensSaved, 3800, "per-session saved = Σ(original − stored) = 1500 + 2300");
264
+ assert.equal(st.dedupCollapsed, 0);
265
+ assert.equal(st.dedupAttempts, 2);
266
+
267
+ // A third add that dedups onto an existing region: whole original region (2000)
268
+ // is discarded (nothing new stored) → repo saved grows by the full original,
269
+ // dedupCollapsed bumps, and no new checkpoint row is created.
270
+ const deduped = s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
271
+ assert.ok(deduped.deduped, "identical region should dedup");
272
+ const st3 = s.stats("sess_saved");
273
+ // Per-session DB sum only covers stored rows (deduped adds create no row), so
274
+ // the per-session figure is unchanged; the deduped save lands in the repo meta.
275
+ assert.equal(st3.tokensSaved, 3800, "per-session DB sum unchanged by deduped add");
276
+ assert.equal(st3.dedupCollapsed, 1, "deduped collapse counted");
277
+ assert.equal(st3.dedupAttempts, 3);
278
+ // Repo cumulative counter DID capture the deduped region's full original size.
279
+ assert.equal(s.repoStats().tokensSaved, 3800 + 2000, "repo saved includes deduped original");
280
+ });
281
+
282
+ test("repoStats aggregates every session + counts deduped original tokens", () => {
283
+ const dir = join(baseTmp, `repo-${counter++}`);
284
+ const a = new VectorStore({ dedupSim: 0.9, stateDir: dir });
285
+ const b = new VectorStore({ dedupSim: 0.9, stateDir: dir }); // same disk store, diff instance
286
+ a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
287
+ b.add({ sessionId: "sess_b", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
288
+
289
+ const repo = a.repoStats();
290
+ assert.equal(repo.checkpointCount, 2, "checkpoints across both sessions");
291
+ assert.equal(repo.sessionCount, 2, "two distinct sessions");
292
+ assert.equal(repo.totalTokenEstimate, 1200, "Σ stored");
293
+ assert.equal(repo.originalTokens, 5000, "Σ original");
294
+ assert.equal(repo.tokensSaved, 3800, "repo saved = Σ(original − stored) = 1500 + 2300");
295
+ assert.equal(repo.dedupCollapsed, 0);
296
+
297
+ // A deduped add into sess_a: whole original region saved, no new row.
298
+ const deduped = a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
299
+ assert.ok(deduped.deduped);
300
+ const repo2 = a.repoStats();
301
+ assert.equal(repo2.tokensSaved, 3800 + 2000, "deduped collapse adds full original region to repo saved");
302
+ assert.equal(repo2.dedupCollapsed, 1);
303
+ assert.equal(repo2.checkpointCount, 2, "still two stored checkpoints");
304
+ });
305
+
253
306
  test("computeRegionHash normalizes whitespace before hashing", () => {
254
307
  const h1 = computeRegionHash("foo bar");
255
308
  const h2 = computeRegionHash("foo bar");
@@ -14,7 +14,7 @@ import { cosineSimilarity, defaultEmbedder } from "./embedder.js";
14
14
  import { loadDedupConfig, type DedupConfigShape, type DedupTier } from "./config/dedup.js";
15
15
  import { logDecision } from "./monitoring.js";
16
16
  import type { StoredCheckpoint, SessionState } from "./store.js";
17
- import { getStateDir, normalizeSessionId, compressSmart, loadDedupStats, saveDedupStats } from "./store.js";
17
+ import { getStateDir, normalizeSessionId, compressSmart } from "./store.js";
18
18
  import { computeContentDigest } from "./dedup/digest.js";
19
19
  import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "./dedup/l1-minhash.js";
20
20
  import { lshBands } from "./dedup/l1-lsh.js";
@@ -32,6 +32,10 @@ import {
32
32
  insertLshBuckets,
33
33
  lshCandidateChunks,
34
34
  setDedupStatus,
35
+ addTokensSaved,
36
+ getDedupStats,
37
+ bumpDedupStats,
38
+ repoStats as repoStatsFromStore,
35
39
  } from "./store/sqlite.js";
36
40
  import { migrateJsonToSqlite } from "./store/migrate.js";
37
41
 
@@ -49,6 +53,11 @@ export interface AddInput {
49
53
  nextSteps?: string[];
50
54
  filesModified?: string[];
51
55
  tokenEstimate?: number;
56
+ /** Token count of the ORIGINAL dropped region (before compaction). Drives the
57
+ * honest "tokens saved" = originalTokenEstimate − tokenEstimate (stored), or
58
+ * the full originalTokenEstimate when the region dedups (nothing new stored).
59
+ * Optional for back-compat with direct add() callers; defaults to stored. */
60
+ originalTokenEstimate?: number;
52
61
  /** Raw text of the compacted region — used to derive the regionHash + vector. */
53
62
  regionText: string;
54
63
  timestamp: number;
@@ -136,10 +145,11 @@ export class VectorStore {
136
145
  const sessionId = normalizeSessionId(input.sessionId);
137
146
  const regionHash = computeRegionHash(input.regionText);
138
147
  const all = listCheckpoints(sessionId, this.stateDir);
148
+ // Honest "tokens saved" base for this region. For a deduped add the whole
149
+ // original region is discarded (nothing new stored); for a new checkpoint
150
+ // we persist (orig − stored). Falls back to stored when orig is unknown.
151
+ const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
139
152
  const cfg = this.cfg;
140
- // Cumulative store-wide dedup accounting (survives session resets).
141
- const ds = loadDedupStats(this.stateDir);
142
- ds.attempts++;
143
153
  // Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
144
154
  // and which tier.
145
155
  let markOnly: DedupTier | null = null;
@@ -165,8 +175,9 @@ export class VectorStore {
165
175
  } else {
166
176
  contentMatch.timestamp = input.timestamp;
167
177
  upsertCheckpoint(contentMatch, this.stateDir);
168
- ds.deduped++;
169
- saveDedupStats(ds, this.stateDir);
178
+ bumpDedupStats(true, this.stateDir);
179
+ // Deduped: whole original region discarded, nothing new stored.
180
+ addTokensSaved(origTokens, this.stateDir);
170
181
  const r = { checkpoint: contentMatch, deduped: true, reason: "contentHash" };
171
182
  this.record("L0", "deduped", "contentHash", Date.now() - t0);
172
183
  return r;
@@ -181,8 +192,9 @@ export class VectorStore {
181
192
  if (cfg.MARK_ONLY_L0) {
182
193
  markOnly = "L0"; // fall through
183
194
  } else {
184
- ds.deduped++;
185
- saveDedupStats(ds, this.stateDir);
195
+ bumpDedupStats(true, this.stateDir);
196
+ // Deduped: whole original region discarded, nothing new stored.
197
+ addTokensSaved(origTokens, this.stateDir);
186
198
  const r = { checkpoint: regionMatch, deduped: true, reason: "regionHash" };
187
199
  this.record("L0", "deduped", "regionHash", Date.now() - t0);
188
200
  return r;
@@ -203,8 +215,9 @@ export class VectorStore {
203
215
  } else {
204
216
  summaryMatch.timestamp = input.timestamp;
205
217
  upsertCheckpoint(summaryMatch, this.stateDir);
206
- ds.deduped++;
207
- saveDedupStats(ds, this.stateDir);
218
+ bumpDedupStats(true, this.stateDir);
219
+ // Deduped: whole original region discarded, nothing new stored.
220
+ addTokensSaved(origTokens, this.stateDir);
208
221
  const r = { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" };
209
222
  this.record("L0", "deduped", "summaryHash", Date.now() - t0);
210
223
  return r;
@@ -221,8 +234,7 @@ export class VectorStore {
221
234
  if (l1 && !cfg.MARK_ONLY_L1) {
222
235
  l1.timestamp = input.timestamp;
223
236
  upsertCheckpoint(l1, this.stateDir);
224
- ds.deduped++;
225
- saveDedupStats(ds, this.stateDir);
237
+ bumpDedupStats(true, this.stateDir);
226
238
  const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
227
239
  this.record("L1", "deduped", "l1MinHash", Date.now() - t0);
228
240
  return r;
@@ -257,8 +269,9 @@ export class VectorStore {
257
269
  // Near-identical — update timestamp on existing checkpoint
258
270
  nearest.checkpoint.timestamp = input.timestamp;
259
271
  upsertCheckpoint(nearest.checkpoint, this.stateDir);
260
- ds.deduped++;
261
- saveDedupStats(ds, this.stateDir);
272
+ bumpDedupStats(true, this.stateDir);
273
+ // Deduped: whole original region discarded, nothing new stored.
274
+ addTokensSaved(origTokens, this.stateDir);
262
275
  const r = { checkpoint: nearest.checkpoint, deduped: true, reason: "contentSimilarity" };
263
276
  this.record("L2", "deduped", "contentSimilarity", Date.now() - t0);
264
277
  return r;
@@ -279,6 +292,7 @@ export class VectorStore {
279
292
  nextSteps: input.nextSteps ?? [],
280
293
  filesModified: input.filesModified ?? [],
281
294
  tokenEstimate: input.tokenEstimate ?? 0,
295
+ originalTokenEstimate: input.originalTokenEstimate,
282
296
  regionHash,
283
297
  contentHash: digest.contentHash,
284
298
  contentHash2: digest.contentHash2,
@@ -291,6 +305,12 @@ export class VectorStore {
291
305
  // Persistence is SQLite (store/sqlite.ts). upsertCheckpoint keeps the
292
306
  // idempotent-by-id semantics the old JSON append implied.
293
307
  upsertCheckpoint(checkpoint, this.stateDir);
308
+ // Cumulative "tokens saved" counter (per-repo SQLite meta). For a NEW
309
+ // checkpoint the saved amount is (original − stored); for a deduped add the
310
+ // whole original region is discarded (handled in the deduped return paths
311
+ // below). Survives sessions and travels with the repo.
312
+ const stored = input.tokenEstimate ?? 0;
313
+ addTokensSaved(Math.max(0, origTokens - stored), this.stateDir);
294
314
  // L1: persist this checkpoint's MinHash signature + LSH buckets so future
295
315
  // near-duplicate inserts can find it. Deterministic given the seed.
296
316
  const sig = minhashSignature(input.regionText);
@@ -320,7 +340,8 @@ export class VectorStore {
320
340
  } else {
321
341
  this.record("L0", "new", undefined, Date.now() - t0);
322
342
  }
323
- saveDedupStats(ds, this.stateDir);
343
+ // Cumulative store-wide dedup accounting (attempt, not collapsed).
344
+ bumpDedupStats(false, this.stateDir);
324
345
  return { checkpoint, deduped: false };
325
346
  }
326
347
 
@@ -517,6 +538,8 @@ export class VectorStore {
517
538
  injectedCount: number;
518
539
  dedupHitRate: number; // injected / checkpoints, 0..1
519
540
  storageDedupRate: number; // deduped adds / total adds, 0..1 (cumulative)
541
+ tokensSaved: number; // Σ(original − stored) for this session's checkpoints
542
+ originalTokens: number; // Σ original region size for this session's checkpoints
520
543
  dedupAttempts: number; // cumulative add() calls (store-wide)
521
544
  dedupCollapsed: number; // cumulative deduped collapses (store-wide)
522
545
  } {
@@ -528,17 +551,38 @@ export class VectorStore {
528
551
  );
529
552
  const last = ordered[ordered.length - 1];
530
553
  const injected = state.injectedCheckpointIds.length;
531
- const ds = loadDedupStats(this.stateDir);
554
+ const ds = getDedupStats(this.stateDir);
555
+ const sessionTok = cps.reduce((s, c) => s + (c.tokenEstimate ?? 0), 0);
556
+ const sessionOrig = cps.reduce((s, c) => s + (c.originalTokenEstimate ?? 0), 0);
557
+ // Per-session "tokens saved" = Σ(original − stored) over this session's
558
+ // stored checkpoints. Deduped adds (whole region discarded, nothing stored)
559
+ // are counted in the repo-wide meta counter via repoStats(); the per-session
560
+ // DB sum here covers the rows that exist.
561
+ const sessionSaved = cps.reduce(
562
+ (s, c) => s + Math.max(0, (c.originalTokenEstimate ?? 0) - (c.tokenEstimate ?? 0)),
563
+ 0,
564
+ );
532
565
  return {
533
566
  checkpointCount: cps.length,
534
- totalTokenEstimate: cps.reduce((s, c) => s + (c.tokenEstimate ?? 0), 0),
567
+ totalTokenEstimate: sessionTok,
535
568
  lastCheckpointId: last?.checkpointId,
536
569
  lastSummary: last?.summary,
537
570
  injectedCount: injected,
538
571
  dedupHitRate: cps.length === 0 ? 0 : injected / cps.length,
539
572
  storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
573
+ tokensSaved: sessionSaved,
574
+ originalTokens: sessionOrig,
540
575
  dedupAttempts: ds.attempts,
541
576
  dedupCollapsed: ds.deduped,
542
577
  };
543
578
  }
579
+
580
+ /**
581
+ * Repo-wide stats — aggregates every session in this store (one per repo).
582
+ * Cumulative, resumable, cross-device. Surfaces the dashboard's "Repo …"
583
+ * figures; distinct from {@link stats} (per-session).
584
+ */
585
+ repoStats(): ReturnType<typeof repoStatsFromStore> {
586
+ return repoStatsFromStore(this.stateDir);
587
+ }
544
588
  }