pi-mega-compact 0.4.1 → 0.4.3

package/extensions/dashboard-server.ts

@@ -53,6 +53,7 @@ interface Snapshot {
   store: {
     checkpointCount: number;
     totalTokenEstimate: number;
+    originalTokens: number;
     tokensSaved: number;
     injectedCount: number;
     dedupHitRate: number;
@@ -66,6 +67,7 @@ interface Snapshot {
   repo: {
     checkpointCount: number;
     totalTokenEstimate: number;
+    originalTokens: number;
     tokensSaved: number;
     sessionCount: number;
     dedupAttempts: number;
@@ -91,9 +93,9 @@ function readSnapshot(snapshotPath: string) {
       session: { id: null, state: null, persistedThisSession: false, lastCheckpointId: null, lastCompactedFrom: 0 },
       context: { tokens: null, percent: null, contextWindow: 0 },
       trigger: { armed: false, ready: false, currentTokens: null, thresholdTokens: 100_000, fastGatePct: 80 },
-      store: { checkpointCount: 0, totalTokenEstimate: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
+      store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
       crew: { activeAgents: 0, currentTurn: 0 },
-      repo: { checkpointCount: 0, totalTokenEstimate: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
+      repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
     } as Snapshot;
   }
 }
@@ -185,6 +187,7 @@ function dashboardHtml(tierName: string): string {
     <div class="stat-grid">
       <span class="label">Checkpoints</span><span class="value" id="st-count">0</span>
       <span class="label">Tokens Stored</span><span class="value" id="st-tokens">0</span>
+      <span class="label">Original Tokens</span><span class="value" id="st-orig">0</span>
       <span class="label">Tokens Saved</span><span class="value" id="st-saved">0</span>
       <span class="label">Injected</span><span class="value" id="st-injected">0</span>
       <span class="label">Dedup Rate</span><span class="value" id="st-dedup">0%</span>
@@ -198,6 +201,7 @@ function dashboardHtml(tierName: string): string {
     <div class="stat-grid">
       <span class="label">Checkpoints</span><span class="value" id="rp-count">0</span>
       <span class="label">Tokens Stored</span><span class="value" id="rp-tokens">0</span>
+      <span class="label">Original Tokens</span><span class="value" id="rp-orig">0</span>
       <span class="label">Tokens Saved</span><span class="value" id="rp-saved">0</span>
       <span class="label">Sessions</span><span class="value" id="rp-sessions">0</span>
       <span class="label">Collapsed</span><span class="value" id="rp-collapsed">0</span>
@@ -263,6 +267,7 @@ function dashboardHtml(tierName: string): string {
 
     document.getElementById('st-count').textContent = d.store.checkpointCount;
     document.getElementById('st-tokens').textContent = d.store.totalTokenEstimate.toLocaleString();
+    document.getElementById('st-orig').textContent = (d.store.originalTokens || 0).toLocaleString();
     document.getElementById('st-saved').textContent = (d.store.tokensSaved || 0).toLocaleString();
     document.getElementById('st-injected').textContent = d.store.injectedCount;
     document.getElementById('st-dedup').textContent = Math.round(d.store.dedupHitRate * 100) + '%';
@@ -272,9 +277,10 @@ function dashboardHtml(tierName: string): string {
     document.getElementById('st-lastid').textContent = d.session.lastCheckpointId || '—';
 
     // Repo-wide (all sessions in this repo's SQLite store).
-    var repo = d.repo || { checkpointCount: 0, totalTokenEstimate: 0, tokensSaved: 0, sessionCount: 0, dedupCollapsed: 0, storageDedupRate: 0 };
+    var repo = d.repo || { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupCollapsed: 0, storageDedupRate: 0 };
     document.getElementById('rp-count').textContent = repo.checkpointCount;
     document.getElementById('rp-tokens').textContent = repo.totalTokenEstimate.toLocaleString();
+    document.getElementById('rp-orig').textContent = (repo.originalTokens || 0).toLocaleString();
     document.getElementById('rp-saved').textContent = (repo.tokensSaved || 0).toLocaleString();
     document.getElementById('rp-sessions').textContent = repo.sessionCount || 0;
     document.getElementById('rp-collapsed').textContent = repo.dedupCollapsed || 0;

package/extensions/mega-compact.test.ts

@@ -327,8 +327,17 @@ test("state snapshot writes dashboard.json after compaction", async () => {
   const snapPath = j(h.stateDir, "dashboard.json");
   assert.ok(ex(snapPath), "dashboard.json written after compaction");
   const snap = JSON.parse(rf(snapPath, "utf-8"));
-  // Item B: tokensSaved is populated (original − stored) after a real compaction.
-  assert.ok(snap.store.tokensSaved > 0, "snapshot.store.tokensSaved > 0 after compaction");
+  // Item B: the honest token model is wired — the original dropped region was
+  // captured (originalTokens > 0), and the saved amount never exceeds the
+  // original (saved = max(0, original − stored) ≤ original). For this tiny
+  // harness session the summary can be ≥ the region, so saved may be 0; the
+  // positive "saved > 0" case with a large region is covered by the
+  // vectorStore unit tests.
+  assert.ok(snap.store.originalTokens > 0, "snapshot.store.originalTokens captured after compaction");
+  assert.ok(
+    snap.store.originalTokens >= snap.store.tokensSaved,
+    "model invariant: original region >= tokens saved",
+  );
   // Item A: crew (live agent) block is present in the dashboard snapshot.
   assert.ok(snap.crew && typeof snap.crew.activeAgents === "number", "snapshot.crew.activeAgents present");
 });

package/extensions/mega-compact.ts

@@ -37,6 +37,7 @@ import { recallAndInline } from "../src/recall.js";
 import { autoCompactCheck } from "../src/compact.js";
 import { estimateSessionTokens } from "../src/tokens.js";
 import { normalizeSessionId } from "../src/store.js";
+import { touchSession, logDaily } from "../src/store/sqlite.js";
 import { Logger } from "../src/log.js";
 import type { EngineMessage } from "../src/types.js";
 import { writeFileSync, appendFileSync, readFileSync } from "node:fs";
@@ -191,7 +192,8 @@ interface DashboardSnapshot {
   store: {
     checkpointCount: number;
     totalTokenEstimate: number;
-    tokensSaved: number;
+    originalTokens: number;      // Σ original dropped-region tokens (this session)
+    tokensSaved: number;         // Σ(original − stored) for this session
     injectedCount: number;
     dedupHitRate: number;
     storageDedupRate: number;
@@ -205,7 +207,8 @@ interface DashboardSnapshot {
   repo: {
     checkpointCount: number;     // across all sessions in this repo's store
     totalTokenEstimate: number;  // repo-wide stored checkpoint tokens
-    tokensSaved: number;         // repo-wide cumulative stored-summary tokens
+    originalTokens: number;      // repo-wide Σ original dropped-region tokens
+    tokensSaved: number;         // repo-wide cumulative (original − stored) + deduped orig
     sessionCount: number;        // distinct sessions with checkpoints
     dedupAttempts: number;       // cumulative add() calls (store-wide)
     dedupCollapsed: number;      // cumulative deduped collapses (store-wide)
@@ -304,10 +307,11 @@ export default function (pi: ExtensionAPI) {
       context: { tokens: lastCtxTokens, percent: lastCtxPercent, contextWindow: lastCtxWindow },
       trigger: { armed, ready, currentTokens: lastCtxTokens, thresholdTokens: config.thresholdTokens, fastGatePct: config.fastGatePct },
       crew: { activeAgents, currentTurn },
-      store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, tokensSaved: rt.tokensSaved, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
+      store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, originalTokens: st.originalTokens, tokensSaved: rt.tokensSaved, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
       repo: {
         checkpointCount: repo.checkpointCount,
         totalTokenEstimate: repo.totalTokenEstimate,
+        originalTokens: repo.originalTokens,
         tokensSaved: repo.tokensSaved,
         sessionCount: repo.sessionCount,
         dedupAttempts: repo.dedupAttempts,
@@ -425,13 +429,28 @@ export default function (pi: ExtensionAPI) {
     rt.lastCompactedFrom = result.compactedFrom;
     rt.lastCompactedTokens = result.tokenEstimate;
     rt.dedupAttempts++;
-    // Per-session "tokens saved" = this session-instance only: the stored-summary
-    // tokens persisted on each NEW (non-deduped) compaction. It resets to 0 on
-    // session_start (rt is rebuilt) — so a fresh session shows 0 while the repo's
-    // cumulative saved (SQLite meta) keeps the historical running total.
-    if (!result.deduped) rt.tokensSaved += result.tokenEstimate;
+    // Honest "tokens saved" for this session-instance only:
+    //   new checkpoint      → original − stored
+    //   deduped onto existing → whole original region (nothing new stored)
+    // Resets to 0 on session_start (rt is rebuilt) — so a fresh session shows 0
+    // while the repo's cumulative saved (SQLite meta) keeps the running total.
+    const saved = result.deduped
+      ? result.originalTokenEstimate
+      : Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
+    rt.tokensSaved += saved;
     if (result.deduped) rt.dedupSkips++;
 
+    // Record session activity + a daily-log entry in the per-repo SQLite store
+    // (foundation for resume-sessions / daily-log features). Best-effort — never
+    // block a compaction on bookkeeping.
+    try {
+      const repo = resolveRepoRoot(ctx.cwd);
+      touchSession(sid, repo, currentStateDir);
+      logDaily(sid, "compact", result.checkpointId, saved, currentStateDir);
+    } catch {
+      /* non-fatal: stats bookkeeping only */
+    }
+
     // Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
     // skip re-vectorizing an already-compacted region (zero token cost).
     pi.appendEntry(MARKER_TYPE, {
@@ -441,7 +460,6 @@ export default function (pi: ExtensionAPI) {
       deduped: result.deduped,
     });
 
-    const saved = result.tokenEstimate;
     setStatus(
       ctx,
       rt.persistedThisSession

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.4.1",
+  "version": "0.4.3",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-2-Clause",

package/src/engine.ts

@@ -15,7 +15,7 @@
 import { findSuperseded, supersede } from "./supersede.js";
 import { summarizeMessages, mergeCompactSummaries, formatCompactSummary } from "./compact.js";
 import { extractiveSummarize } from "./extractive.js";
-import { estimateSessionTokens } from "./tokens.js";
+import { estimateSessionTokens, estimateBlockTokens } from "./tokens.js";
 import { computeRegionHash, VectorStore, type SearchHit } from "./vectorStore.js";
 import type { EngineMessage } from "./types.js";
 
@@ -51,6 +51,11 @@ export interface CompactResult {
   summary: string;
   regionHash: string;
   tokenEstimate: number;
+  /** Token count of the original dropped region (before compaction). The honest
+   *  "tokens saved" base = originalTokenEstimate − tokenEstimate (stored), or the
+   *  full originalTokenEstimate when the region deduped onto an existing
+   *  checkpoint (nothing new stored). */
+  originalTokenEstimate: number;
   /** Index in `messages` where the compacted slice begins (for the caller to
    *  build a drop range). */
   compactedFrom: number;
@@ -87,6 +92,7 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
       summary: "",
       regionHash: "",
       tokenEstimate: 0,
+      originalTokenEstimate: 0,
       compactedFrom,
     };
   }
@@ -105,7 +111,6 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
   let keyDecisions: string[];
   let nextSteps: string[];
   let filesModified: string[];
-  let tokenEstimate: number;
 
   if (useExtractive && !input.summary) {
     const ext = extractiveSummarize(keep);
@@ -114,7 +119,6 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
     keyDecisions = input.keyDecisions ?? ext.keyDecisions;
     nextSteps = input.nextSteps ?? ext.nextSteps;
     filesModified = input.filesModified ?? ext.filesModified;
-    tokenEstimate = input.tokenEstimate ?? ext.tokenEstimate;
   } else {
     const collapsed = input.summary ?? summarizeMessages(keep);
     summary = formatCompactSummary(collapsed);
@@ -122,9 +126,18 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
     keyDecisions = input.keyDecisions ?? [];
     nextSteps = input.nextSteps ?? [];
     filesModified = input.filesModified ?? [];
-    tokenEstimate = input.tokenEstimate ?? estimateSessionTokens(compactable);
   }
 
+  // Honest "tokens saved" accounting:
+  //  - originalTokenEstimate = the dropped region's token count (what context
+  //    held before compaction) = the compacted slice's tokens.
+  //  - storedTokens = the persisted summary's token count, computed from the
+  //    actual summary string so it's honest for BOTH the extractive and legacy
+  //    COLLAPSE paths (the legacy path's fallback estimateSessionTokens is the
+  //    *original* size, not the stored size).
+  const originalTokenEstimate = estimateSessionTokens(compactable);
+  const storedTokens = estimateBlockTokens(summary);
+
   // Region text = the compacted slice, used for dedup + embedding.
   const regionText = input.regionText ?? keep.map((m) => m.text).join("\n");
   const regionHash = computeRegionHash(regionText);
@@ -137,7 +150,8 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
     nextSteps,
     filesModified,
     regionText,
-    tokenEstimate,
+    tokenEstimate: storedTokens,
+    originalTokenEstimate,
     timestamp: input.timestamp ?? 0,
   });
 
@@ -148,7 +162,8 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
     checkpointId: add.checkpoint.checkpointId,
     summary,
     regionHash,
-    tokenEstimate,
+    tokenEstimate: storedTokens,
+    originalTokenEstimate,
     compactedFrom,
   };
 }

package/src/store/sprint10.test.ts

@@ -174,6 +174,38 @@ test("checkAllIntegrity covers every session", () => {
   assert.ok(reports.every((r: { ok: boolean }) => r.ok));
 });
 
+// --- schema migration (v0.4.2: original_token_estimate) -------------------
+
+test("Sprint 10 migration: pre-0.4.2 db gains original_token_estimate and repoStats works", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  // Simulate a v0.4.1-era store: openStore() builds the full current schema,
+  // then drop the original_token_estimate column that 0.4.2 added. CREATE TABLE
+  // IF NOT EXISTS is a no-op on an existing table, which is exactly why the
+  // shipped v0.4.2 crashed at runtime on old repos ("no such column").
+  const db = openStore(dir);
+  db.exec("ALTER TABLE context_chunks DROP COLUMN original_token_estimate;");
+  closeStore(dir);
+
+  // Re-open through the real code path — must ALTER the column in, not crash.
+  const vs = new VectorStore({ dedupSim: 0.9, stateDir: dir });
+  vs.add({
+    sessionId: "sess_mig",
+    summary: "legacy region",
+    regionText: "some region text to compact",
+    tokenEstimate: 100,
+    originalTokenEstimate: 500,
+    timestamp: 1,
+  });
+  const repo = vs.repoStats();
+  // No "no such column" crash = migration succeeded; totals reflect the new col.
+  assert.equal(repo.checkpointCount, 1);
+  assert.equal(repo.originalTokens, 500, "originalTokens read from migrated column");
+  // Re-open a second time to prove idempotency (column already exists).
+  closeStore(dir);
+  const vs2 = new VectorStore({ dedupSim: 0.9, stateDir: dir });
+  assert.equal(vs2.repoStats().originalTokens, 500, "stable across re-open");
+});
+
 // --- cleanup ---------------------------------------------------------------
 
 test("Sprint 10 cleanup", () => {

package/src/store/sqlite.ts

@@ -80,6 +80,7 @@ function initSchema(db: Database.Database): void {
       files_modified     TEXT,           -- JSON array
       embedding_blob     BLOB,           -- float32 vector
       token_estimate     INTEGER,
+      original_token_estimate INTEGER,    -- dropped region size (tokens saved = orig − stored)
       timestamp          INTEGER,
       dedup_status       TEXT DEFAULT 'active',
       compressed_original BLOB           -- optional DR copy
@@ -146,6 +147,42 @@ function initSchema(db: Database.Database): void {
     CREATE INDEX IF NOT EXISTS idx_raptor_session ON raptor_nodes(session_id);
     CREATE INDEX IF NOT EXISTS idx_raptor_parent ON raptor_nodes(parent_id);
 
+    -- Foundation for future features (resume sessions, daily log, lessons
+    -- learned). Scaffolded now so all store data lives in SQLite from day one;
+    -- population is minimal (touchSession / logDaily on compact) and the full
+    -- UI/recall for these lands in later sprints.
+
+    -- Per-session registry (resume + per-repo session history).
+    CREATE TABLE IF NOT EXISTS sessions (
+      session_id    TEXT PRIMARY KEY,
+      repo          TEXT,
+      started_at    INTEGER,
+      ended_at      INTEGER,
+      last_compacted_at INTEGER,
+      status        TEXT DEFAULT 'active'
+    );
+
+    -- Append-only daily activity log (the "daily log" feature seed).
+    CREATE TABLE IF NOT EXISTS daily_log (
+      id            INTEGER PRIMARY KEY AUTOINCREMENT,
+      day           TEXT NOT NULL,     -- YYYY-MM-DD
+      session_id    TEXT,
+      event         TEXT,             -- e.g. 'compact'
+      detail        TEXT,
+      tokens_saved  INTEGER DEFAULT 0,
+      ts            INTEGER
+    );
+    CREATE INDEX IF NOT EXISTS idx_daily_log_day ON daily_log(day);
+
+    -- Lessons learned (future recall/browse feature seed).
+    CREATE TABLE IF NOT EXISTS lessons (
+      id            INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id    TEXT,
+      repo          TEXT,
+      lesson        TEXT,
+      ts            INTEGER
+    );
+
     -- FTS5 trigram virtual table (Sprint 9+ pg_trgm-equivalent verification).
     CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_trgm USING fts5(
       id UNINDEXED,
@@ -153,6 +190,12 @@ function initSchema(db: Database.Database): void {
       tokenize='trigram'
     );
   `);
+  // Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
+  // pre-existing table, so new columns added to context_chunks after a store was
+  // first created (e.g. original_token_estimate in v0.4.2) must be ALTERed in for
+  // databases created by an older version — otherwise repoStats()/upsert crash
+  // with "no such column" and the extension fails to load. Additive only.
+  ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
   const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get() as
     | { value: string }
     | undefined;
@@ -161,6 +204,19 @@ function initSchema(db: Database.Database): void {
   }
 }
 
+/**
+ * Add `column` (with `decl`, e.g. "INTEGER") to `table` if it does not already
+ * exist. Idempotent: checks PRAGMA table_info first, so it is safe to run on
+ * every open. Table/column/decl are code-controlled constants (never user
+ * input), so the unavoidable identifier interpolation here does not violate
+ * PREVENT-002 (no external data reaches this SQL).
+ */
+function ensureColumn(db: Database.Database, table: string, column: string, decl: string): void {
+  const cols = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
+  if (cols.some((c) => c.name === column)) return;
+  db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${decl}`);
+}
+
 /** Read a string-valued meta key (or undefined). Used for cumulative counters. */
 export function getMeta(key: string, stateDir: string = getStateDir()): string | undefined {
   const db = openStore(stateDir);
@@ -235,6 +291,65 @@ export function bumpDedupStats(deduped: boolean, stateDir: string = getStateDir(
   if (deduped) incMeta("deduped", 1, stateDir);
 }
 
+// --- Future-feature foundation (resume sessions / daily log / lessons) -------
+// Scaffolded tables + minimal helpers so all store data lives in SQLite from
+// day one. Full UI/recall for these lands in later sprints.
+
+/** Upsert a `sessions` row (resume + per-repo session history). */
+export function touchSession(
+  sessionId: string,
+  repo: string | undefined,
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  const sid = normalizeSessionId(sessionId);
+  const existing = db
+    .prepare("SELECT started_at FROM sessions WHERE session_id = ?")
+    .get(sid) as { started_at: number | null } | undefined;
+  const now = Math.floor(Date.now() / 1000);
+  if (!existing) {
+    db.prepare(
+      `INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
+       VALUES(?, ?, ?, ?, 'active')`,
+    ).run(sid, repo ?? null, now, now);
+  } else {
+    db.prepare(
+      "UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?",
+    ).run(now, repo ?? null, sid);
+  }
+}
+
+/** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
+export function logDaily(
+  sessionId: string,
+  event: string,
+  detail: string | undefined,
+  tokensSaved: number,
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  const day = new Date().toISOString().slice(0, 10);
+  const now = Math.floor(Date.now() / 1000);
+  db.prepare(
+    `INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
+     VALUES(?, ?, ?, ?, ?, ?)`,
+  ).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
+}
+
+/** Append a `lessons` entry (future lessons-learned browse/recall). */
+export function addLesson(
+  sessionId: string,
+  repo: string | undefined,
+  lesson: string,
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  const now = Math.floor(Date.now() / 1000);
+  db.prepare(
+    `INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`,
+  ).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
+}
+
 /** Map a DB row to the public StoredCheckpoint shape. */
 function rowToCheckpoint(row: any): StoredCheckpoint {
   return {
@@ -247,6 +362,7 @@ function rowToCheckpoint(row: any): StoredCheckpoint {
     nextSteps: row.next_steps ? JSON.parse(row.next_steps) : [],
     filesModified: row.files_modified ? JSON.parse(row.files_modified) : [],
     tokenEstimate: row.token_estimate ?? 0,
+    originalTokenEstimate: row.original_token_estimate ?? undefined,
     regionHash: row.region_hash ?? "",
     contentHash: row.content_hash ?? undefined,
     contentHash2: row.content_hash2 ?? undefined,
@@ -269,11 +385,11 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
         (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
          normalized_text, summary, topic_summary, summary_hash,
          key_decisions, next_steps, files_modified, embedding_blob,
-         token_estimate, timestamp, dedup_status, compressed_original)
+         token_estimate, original_token_estimate, timestamp, dedup_status, compressed_original)
        VALUES (@id, @sid, @region_hash, @content_hash, @content_hash2, @content_hash_version,
                @normalized_text, @summary, @topic_summary, @summary_hash,
                @key_decisions, @next_steps, @files_modified, @embedding_blob,
-               @token_estimate, @timestamp, @dedup_status, @compressed_original)
+               @token_estimate, @original_token_estimate, @timestamp, @dedup_status, @compressed_original)
        ON CONFLICT(session_id, id) DO UPDATE SET
          summary=excluded.summary,
          topic_summary=excluded.topic_summary,
@@ -283,6 +399,7 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
          files_modified=excluded.files_modified,
          embedding_blob=excluded.embedding_blob,
          token_estimate=excluded.token_estimate,
+         original_token_estimate=excluded.original_token_estimate,
          timestamp=excluded.timestamp,
          dedup_status=excluded.dedup_status,
          compressed_original=excluded.compressed_original`,
@@ -302,6 +419,7 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
       files_modified: jsonText(cp.filesModified),
       embedding_blob: encodeEmbedding(cp.embedding ?? []),
       token_estimate: cp.tokenEstimate ?? 0,
+      original_token_estimate: cp.originalTokenEstimate ?? null,
       timestamp: cp.timestamp ?? 0,
       dedup_status: "active",
       compressed_original: cp.compressedOriginal ?? null,
@@ -508,6 +626,8 @@ export interface RepoStats {
   sessionCount: number;
   /** Cumulative stored-summary tokens saved (Σ stored summaries). */
   tokensSaved: number;
+  /** Sum of original dropped-region token estimates (repo-wide). */
+  originalTokens: number;
   /** Cumulative dedup add() attempts (store-wide). */
   dedupAttempts: number;
   /** Cumulative deduped collapses (store-wide). */
@@ -521,14 +641,16 @@ export function repoStats(stateDir: string = getStateDir()): RepoStats {
   const row = db
     .prepare(
       `SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
+              COALESCE(SUM(original_token_estimate),0) AS orig,
               COUNT(DISTINCT session_id) AS sessions
        FROM context_chunks WHERE dedup_status != 'removed'`,
     )
-    .get() as { c: number; tok: number; sessions: number };
+    .get() as { c: number; tok: number; orig: number; sessions: number };
   const ds = getDedupStats(stateDir);
   return {
     checkpointCount: row.c,
     totalTokenEstimate: row.tok,
+    originalTokens: row.orig,
     sessionCount: row.sessions,
     tokensSaved: getMetaNumber("tokens_saved", stateDir),
     dedupAttempts: ds.attempts,

package/src/store.ts

@@ -58,6 +58,9 @@ export interface StoredCheckpoint {
   nextSteps: string[];
   filesModified: string[];
   tokenEstimate: number;
+  /** Token count of the ORIGINAL dropped region (before compaction). Drives the
+   *  honest "tokens saved" = originalTokenEstimate − tokenEstimate. */
+  originalTokenEstimate?: number;
   regionHash: string;
   /** Primary content-addressable hash (full 64-hex SHA-256 of normalized text). */
   contentHash?: string;

package/src/vectorStore.test.ts

@@ -250,47 +250,55 @@ test("stats reports counts, last checkpoint, and dedup rate", () => {
   assert.ok(Math.abs(st2.dedupHitRate - 0.5) < 1e-9);
 });
 
-test("tokensSaved accumulates on new checkpoints; deduped add bumps collapsed not saved", () => {
+test("tokensSaved = original − stored per session; deduped add saves the whole region", () => {
   const s = store();
-  // Two genuinely new checkpoints (stored-sum definition of tokensSaved).
-  s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, timestamp: 1 });
-  s.add({ sessionId: "sess_saved", summary: "beta", regionText: "region beta text", tokenEstimate: 700, timestamp: 2 });
+  // Two genuinely new checkpoints. saved = original − stored.
+  //   cp1: orig 2000, stored 500 → saved 1500
+  //   cp2: orig 3000, stored 700 → saved 2300
+  s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
+  s.add({ sessionId: "sess_saved", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
   const st = s.stats("sess_saved");
-  assert.equal(st.tokensSaved, 1200, "per-session tokensSaved = Σ stored summary tokens");
+  assert.equal(st.totalTokenEstimate, 1200, "Σ stored summaries");
+  assert.equal(st.originalTokens, 5000, "Σ original region tokens");
+  assert.equal(st.tokensSaved, 3800, "per-session saved = Σ(original − stored) = 1500 + 2300");
   assert.equal(st.dedupCollapsed, 0);
   assert.equal(st.dedupAttempts, 2);
 
-  // A third add that dedups onto an existing region (same summaryHash path):
-  // collapses onto an existing checkpoint, so it must NOT grow tokensSaved, but
-  // must bump dedupCollapsed + dedupAttempts.
-  const deduped = s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, timestamp: 3 });
+  // A third add that dedups onto an existing region: whole original region (2000)
+  // is discarded (nothing new stored) → repo saved grows by the full original,
+  // dedupCollapsed bumps, and no new checkpoint row is created.
+  const deduped = s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
   assert.ok(deduped.deduped, "identical region should dedup");
   const st3 = s.stats("sess_saved");
-  assert.equal(st3.tokensSaved, 1200, "deduped add does not add to tokensSaved (stored-sum)");
-  assert.equal(st3.dedupCollapsed, 1, "deduped collapse counted separately");
+  // Per-session DB sum only covers stored rows (deduped adds create no row), so
+  // the per-session figure is unchanged; the deduped save lands in the repo meta.
+  assert.equal(st3.tokensSaved, 3800, "per-session DB sum unchanged by deduped add");
+  assert.equal(st3.dedupCollapsed, 1, "deduped collapse counted");
   assert.equal(st3.dedupAttempts, 3);
+  // Repo cumulative counter DID capture the deduped region's full original size.
+  assert.equal(s.repoStats().tokensSaved, 3800 + 2000, "repo saved includes deduped original");
 });
 
-test("repoStats aggregates every session in the SQLite store", () => {
+test("repoStats aggregates every session + counts deduped original tokens", () => {
   const dir = join(baseTmp, `repo-${counter++}`);
   const a = new VectorStore({ dedupSim: 0.9, stateDir: dir });
   const b = new VectorStore({ dedupSim: 0.9, stateDir: dir }); // same disk store, diff instance
-  a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, timestamp: 1 });
-  b.add({ sessionId: "sess_b", summary: "beta", regionText: "region beta text", tokenEstimate: 700, timestamp: 2 });
+  a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
+  b.add({ sessionId: "sess_b", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
 
   const repo = a.repoStats();
   assert.equal(repo.checkpointCount, 2, "checkpoints across both sessions");
   assert.equal(repo.sessionCount, 2, "two distinct sessions");
-  assert.equal(repo.totalTokenEstimate, 1200);
-  assert.equal(repo.tokensSaved, 1200, "repo-wide cumulative stored-summary tokens");
+  assert.equal(repo.totalTokenEstimate, 1200, "Σ stored");
+  assert.equal(repo.originalTokens, 5000, "Σ original");
+  assert.equal(repo.tokensSaved, 3800, "repo saved = Σ(original − stored) = 1500 + 2300");
   assert.equal(repo.dedupCollapsed, 0);
 
-  // A deduped add into sess_a collapses onto its existing checkpoint: repo
-  // tokensSaved stays put, dedupCollapsed climbs.
-  const deduped = a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, timestamp: 3 });
+  // A deduped add into sess_a: whole original region saved, no new row.
+  const deduped = a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
   assert.ok(deduped.deduped);
   const repo2 = a.repoStats();
-  assert.equal(repo2.tokensSaved, 1200, "deduped collapse does not change repo tokensSaved");
+  assert.equal(repo2.tokensSaved, 3800 + 2000, "deduped collapse adds full original region to repo saved");
   assert.equal(repo2.dedupCollapsed, 1);
   assert.equal(repo2.checkpointCount, 2, "still two stored checkpoints");
 });

package/src/vectorStore.ts

@@ -53,6 +53,11 @@ export interface AddInput {
   nextSteps?: string[];
   filesModified?: string[];
   tokenEstimate?: number;
+  /** Token count of the ORIGINAL dropped region (before compaction). Drives the
+   *  honest "tokens saved" = originalTokenEstimate − tokenEstimate (stored), or
+   *  the full originalTokenEstimate when the region dedups (nothing new stored).
+   *  Optional for back-compat with direct add() callers; defaults to stored. */
+  originalTokenEstimate?: number;
   /** Raw text of the compacted region — used to derive the regionHash + vector. */
   regionText: string;
   timestamp: number;
@@ -140,6 +145,10 @@ export class VectorStore {
     const sessionId = normalizeSessionId(input.sessionId);
     const regionHash = computeRegionHash(input.regionText);
     const all = listCheckpoints(sessionId, this.stateDir);
+    // Honest "tokens saved" base for this region. For a deduped add the whole
+    // original region is discarded (nothing new stored); for a new checkpoint
+    // we persist (orig − stored). Falls back to stored when orig is unknown.
+    const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
     const cfg = this.cfg;
     // Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
     // and which tier.
@@ -167,6 +176,8 @@ export class VectorStore {
           contentMatch.timestamp = input.timestamp;
           upsertCheckpoint(contentMatch, this.stateDir);
           bumpDedupStats(true, this.stateDir);
+          // Deduped: whole original region discarded, nothing new stored.
+          addTokensSaved(origTokens, this.stateDir);
           const r = { checkpoint: contentMatch, deduped: true, reason: "contentHash" };
           this.record("L0", "deduped", "contentHash", Date.now() - t0);
           return r;
@@ -182,6 +193,8 @@ export class VectorStore {
           markOnly = "L0"; // fall through
         } else {
           bumpDedupStats(true, this.stateDir);
+          // Deduped: whole original region discarded, nothing new stored.
+          addTokensSaved(origTokens, this.stateDir);
           const r = { checkpoint: regionMatch, deduped: true, reason: "regionHash" };
           this.record("L0", "deduped", "regionHash", Date.now() - t0);
           return r;
@@ -203,6 +216,8 @@ export class VectorStore {
           summaryMatch.timestamp = input.timestamp;
           upsertCheckpoint(summaryMatch, this.stateDir);
           bumpDedupStats(true, this.stateDir);
+          // Deduped: whole original region discarded, nothing new stored.
+          addTokensSaved(origTokens, this.stateDir);
           const r = { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" };
           this.record("L0", "deduped", "summaryHash", Date.now() - t0);
           return r;
@@ -255,6 +270,8 @@ export class VectorStore {
           nearest.checkpoint.timestamp = input.timestamp;
           upsertCheckpoint(nearest.checkpoint, this.stateDir);
           bumpDedupStats(true, this.stateDir);
+          // Deduped: whole original region discarded, nothing new stored.
+          addTokensSaved(origTokens, this.stateDir);
           const r = { checkpoint: nearest.checkpoint, deduped: true, reason: "contentSimilarity" };
           this.record("L2", "deduped", "contentSimilarity", Date.now() - t0);
           return r;
@@ -275,6 +292,7 @@ export class VectorStore {
       nextSteps: input.nextSteps ?? [],
       filesModified: input.filesModified ?? [],
       tokenEstimate: input.tokenEstimate ?? 0,
+      originalTokenEstimate: input.originalTokenEstimate,
       regionHash,
       contentHash: digest.contentHash,
       contentHash2: digest.contentHash2,
@@ -287,9 +305,12 @@ export class VectorStore {
     // Persistence is SQLite (store/sqlite.ts). upsertCheckpoint keeps the
     // idempotent-by-id semantics the old JSON append implied.
     upsertCheckpoint(checkpoint, this.stateDir);
-    // Cumulative "tokens saved" counter (per-repo SQLite meta). Bumped for every
-    // new checkpoint persisted, so it survives sessions and travels with the repo.
-    addTokensSaved(input.tokenEstimate ?? 0, this.stateDir);
+    // Cumulative "tokens saved" counter (per-repo SQLite meta). For a NEW
+    // checkpoint the saved amount is (original − stored); for a deduped add the
+    // whole original region is discarded (handled in the deduped return paths
+    // below). Survives sessions and travels with the repo.
+    const stored = input.tokenEstimate ?? 0;
+    addTokensSaved(Math.max(0, origTokens - stored), this.stateDir);
     // L1: persist this checkpoint's MinHash signature + LSH buckets so future
     // near-duplicate inserts can find it. Deterministic given the seed.
     const sig = minhashSignature(input.regionText);
@@ -517,7 +538,8 @@ export class VectorStore {
     injectedCount: number;
     dedupHitRate: number; // injected / checkpoints, 0..1
     storageDedupRate: number; // deduped adds / total adds, 0..1 (cumulative)
-    tokensSaved: number; // cumulative stored checkpoint tokens (per-repo SQLite)
+    tokensSaved: number; // Σ(original − stored) for this session's checkpoints
+    originalTokens: number; // Σ original region size for this session's checkpoints
     dedupAttempts: number; // cumulative add() calls (store-wide)
     dedupCollapsed: number; // cumulative deduped collapses (store-wide)
   } {
@@ -530,10 +552,16 @@ export class VectorStore {
     const last = ordered[ordered.length - 1];
     const injected = state.injectedCheckpointIds.length;
     const ds = getDedupStats(this.stateDir);
-    // Per-session "tokens saved" = this session's Σ stored summary token
-    // estimates (stored-sum definition). Equal to totalTokenEstimate by
-    // construction; repo-wide cumulative saved lives in repoStats().
     const sessionTok = cps.reduce((s, c) => s + (c.tokenEstimate ?? 0), 0);
+    const sessionOrig = cps.reduce((s, c) => s + (c.originalTokenEstimate ?? 0), 0);
+    // Per-session "tokens saved" = Σ(original − stored) over this session's
+    // stored checkpoints. Deduped adds (whole region discarded, nothing stored)
+    // are counted in the repo-wide meta counter via repoStats(); the per-session
+    // DB sum here covers the rows that exist.
+    const sessionSaved = cps.reduce(
+      (s, c) => s + Math.max(0, (c.originalTokenEstimate ?? 0) - (c.tokenEstimate ?? 0)),
+      0,
+    );
     return {
       checkpointCount: cps.length,
       totalTokenEstimate: sessionTok,
@@ -542,7 +570,8 @@ export class VectorStore {
       injectedCount: injected,
       dedupHitRate: cps.length === 0 ? 0 : injected / cps.length,
       storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
-      tokensSaved: sessionTok,
+      tokensSaved: sessionSaved,
+      originalTokens: sessionOrig,
       dedupAttempts: ds.attempts,
       dedupCollapsed: ds.deduped,
     };