pi-mega-compact 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -15
- package/extensions/dashboard-server.ts +69 -2
- package/extensions/mega-compact.test.ts +17 -2
- package/extensions/mega-compact.ts +59 -3
- package/package.json +1 -1
- package/src/engine.ts +21 -6
- package/src/store/sqlite.ts +227 -10
- package/src/store.ts +7 -21
- package/src/vectorStore.test.ts +53 -0
- package/src/vectorStore.ts +61 -17
package/README.md
CHANGED
|
@@ -115,48 +115,64 @@ OpenAI-style contract and the `MEGACOMPACT_EMBEDDING_KEY` / `MEGACOMPACT_EMBEDDI
|
|
|
115
115
|
compile). No network call and no API key are needed at runtime.
|
|
116
116
|
- A pi coding agent install that loads extensions from `~/.pi/agent/extensions/`.
|
|
117
117
|
|
|
118
|
-
###
|
|
118
|
+
### Install from npm (recommended)
|
|
119
119
|
|
|
120
120
|
```bash
|
|
121
|
-
|
|
122
|
-
~/.pi/agent/extensions/pi-mega-compact
|
|
123
|
-
cd ~/.pi/agent/extensions/pi-mega-compact
|
|
124
|
-
npm install
|
|
125
|
-
npm run build
|
|
121
|
+
npm install pi-mega-compact
|
|
126
122
|
```
|
|
127
123
|
|
|
124
|
+
This places the package in `node_modules` and exposes the extension entry at
|
|
125
|
+
`node_modules/pi-mega-compact/extensions/mega-compact.ts`. Then point pi at it
|
|
126
|
+
(see "Register with pi" below).
|
|
127
|
+
|
|
128
128
|
### Register with pi
|
|
129
129
|
|
|
130
|
-
|
|
131
|
-
|
|
130
|
+
Add the extension to your pi config's `pi.extensions` list, pointing at the
|
|
131
|
+
installed entry (npm path or a symlink into pi's extensions dir — either works):
|
|
132
132
|
|
|
133
133
|
```jsonc
|
|
134
134
|
{
|
|
135
135
|
"pi": {
|
|
136
|
-
"extensions": ["
|
|
136
|
+
"extensions": ["pi-mega-compact/extensions/mega-compact.ts"]
|
|
137
137
|
}
|
|
138
138
|
}
|
|
139
139
|
```
|
|
140
140
|
|
|
141
|
-
Or
|
|
141
|
+
Or symlink the installed package into pi's extensions dir (the simplest path if
|
|
142
|
+
you run pi from the same machine):
|
|
142
143
|
|
|
143
144
|
```bash
|
|
144
|
-
|
|
145
|
-
./install.sh -s # symlink instead of copy (dev mode)
|
|
145
|
+
ln -s "$(npm root)/pi-mega-compact" ~/.pi/agent/extensions/pi-mega-compact
|
|
146
146
|
```
|
|
147
147
|
|
|
148
|
+
> **From a git checkout (development).** To hack on the extension, clone instead
|
|
149
|
+
> and build locally:
|
|
150
|
+
> ```bash
|
|
151
|
+
> git clone https://github.com/TheArchitectit/pi-mega-compact.git \
|
|
152
|
+
> ~/.pi/agent/extensions/pi-mega-compact
|
|
153
|
+
> cd ~/.pi/agent/extensions/pi-mega-compact
|
|
154
|
+
> npm install && npm run build
|
|
155
|
+
> ```
|
|
156
|
+
> The bundled `./install.sh` helper (`copy`) / `./install.sh -s` (`symlink`) does
|
|
157
|
+
> the same and also registers the path in pi's config (needs `jq`).
|
|
158
|
+
|
|
148
159
|
### Verify
|
|
149
160
|
|
|
150
161
|
```bash
|
|
151
|
-
|
|
152
|
-
npm test # all unit/integration tests pass (192 as of v0.2.0)
|
|
162
|
+
npm test # all unit/integration tests pass (278 as of v0.4.0)
|
|
153
163
|
npm run lint # tsc --noEmit + guardrails scan clean
|
|
154
164
|
```
|
|
155
165
|
|
|
156
166
|
### Uninstall
|
|
157
167
|
|
|
158
168
|
```bash
|
|
159
|
-
|
|
169
|
+
npm uninstall pi-mega-compact
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
If you symlinked it into pi's extensions dir, also remove that link:
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
rm -f ~/.pi/agent/extensions/pi-mega-compact
|
|
160
176
|
```
|
|
161
177
|
|
|
162
178
|
Then remove the path from pi's `pi.extensions` array.
|
|
@@ -53,8 +53,26 @@ interface Snapshot {
|
|
|
53
53
|
store: {
|
|
54
54
|
checkpointCount: number;
|
|
55
55
|
totalTokenEstimate: number;
|
|
56
|
+
originalTokens: number;
|
|
57
|
+
tokensSaved: number;
|
|
56
58
|
injectedCount: number;
|
|
57
59
|
dedupHitRate: number;
|
|
60
|
+
storageDedupRate: number;
|
|
61
|
+
dedupCollapsed: number;
|
|
62
|
+
};
|
|
63
|
+
crew: {
|
|
64
|
+
activeAgents: number;
|
|
65
|
+
currentTurn: number;
|
|
66
|
+
};
|
|
67
|
+
repo: {
|
|
68
|
+
checkpointCount: number;
|
|
69
|
+
totalTokenEstimate: number;
|
|
70
|
+
originalTokens: number;
|
|
71
|
+
tokensSaved: number;
|
|
72
|
+
sessionCount: number;
|
|
73
|
+
dedupAttempts: number;
|
|
74
|
+
dedupCollapsed: number;
|
|
75
|
+
storageDedupRate: number;
|
|
58
76
|
};
|
|
59
77
|
}
|
|
60
78
|
|
|
@@ -75,7 +93,9 @@ function readSnapshot(snapshotPath: string) {
|
|
|
75
93
|
session: { id: null, state: null, persistedThisSession: false, lastCheckpointId: null, lastCompactedFrom: 0 },
|
|
76
94
|
context: { tokens: null, percent: null, contextWindow: 0 },
|
|
77
95
|
trigger: { armed: false, ready: false, currentTokens: null, thresholdTokens: 100_000, fastGatePct: 80 },
|
|
78
|
-
store: { checkpointCount: 0, totalTokenEstimate: 0, injectedCount: 0, dedupHitRate: 0 },
|
|
96
|
+
store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
|
|
97
|
+
crew: { activeAgents: 0, currentTurn: 0 },
|
|
98
|
+
repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
|
|
79
99
|
} as Snapshot;
|
|
80
100
|
}
|
|
81
101
|
}
|
|
@@ -166,12 +186,28 @@ function dashboardHtml(tierName: string): string {
|
|
|
166
186
|
<h2>Vector Store</h2>
|
|
167
187
|
<div class="stat-grid">
|
|
168
188
|
<span class="label">Checkpoints</span><span class="value" id="st-count">0</span>
|
|
169
|
-
<span class="label">
|
|
189
|
+
<span class="label">Tokens Stored</span><span class="value" id="st-tokens">0</span>
|
|
190
|
+
<span class="label">Original Tokens</span><span class="value" id="st-orig">0</span>
|
|
191
|
+
<span class="label">Tokens Saved</span><span class="value" id="st-saved">0</span>
|
|
170
192
|
<span class="label">Injected</span><span class="value" id="st-injected">0</span>
|
|
171
193
|
<span class="label">Dedup Rate</span><span class="value" id="st-dedup">0%</span>
|
|
194
|
+
<span class="label">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
|
|
195
|
+
<span class="label">Collapsed</span><span class="value" id="st-collapsed">0</span>
|
|
172
196
|
<span class="label">Last ID</span><span class="value" id="st-lastid">—</span>
|
|
173
197
|
</div>
|
|
174
198
|
</div>
|
|
199
|
+
<div class="card">
|
|
200
|
+
<h2>Repo (all sessions)</h2>
|
|
201
|
+
<div class="stat-grid">
|
|
202
|
+
<span class="label">Checkpoints</span><span class="value" id="rp-count">0</span>
|
|
203
|
+
<span class="label">Tokens Stored</span><span class="value" id="rp-tokens">0</span>
|
|
204
|
+
<span class="label">Original Tokens</span><span class="value" id="rp-orig">0</span>
|
|
205
|
+
<span class="label">Tokens Saved</span><span class="value" id="rp-saved">0</span>
|
|
206
|
+
<span class="label">Sessions</span><span class="value" id="rp-sessions">0</span>
|
|
207
|
+
<span class="label">Collapsed</span><span class="value" id="rp-collapsed">0</span>
|
|
208
|
+
<span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
|
|
209
|
+
</div>
|
|
210
|
+
</div>
|
|
175
211
|
<div class="card">
|
|
176
212
|
<h2>Configuration</h2>
|
|
177
213
|
<div class="conf-grid">
|
|
@@ -182,6 +218,14 @@ function dashboardHtml(tierName: string): string {
|
|
|
182
218
|
<span class="label">Anchor</span><span class="value" id="cf-anchor">—</span>
|
|
183
219
|
</div>
|
|
184
220
|
</div>
|
|
221
|
+
<div class="card">
|
|
222
|
+
<h2>Crew / Agents</h2>
|
|
223
|
+
<div class="stat-grid">
|
|
224
|
+
<span class="label">Active Agents</span><span class="value" id="cr-agents">0</span>
|
|
225
|
+
<span class="label">Current Turn</span><span class="value" id="cr-turn">0</span>
|
|
226
|
+
<span class="label">Status</span><span class="value" id="cr-status">idle</span>
|
|
227
|
+
</div>
|
|
228
|
+
</div>
|
|
185
229
|
</div>
|
|
186
230
|
|
|
187
231
|
<div class="events">
|
|
@@ -223,10 +267,33 @@ function dashboardHtml(tierName: string): string {
|
|
|
223
267
|
|
|
224
268
|
document.getElementById('st-count').textContent = d.store.checkpointCount;
|
|
225
269
|
document.getElementById('st-tokens').textContent = d.store.totalTokenEstimate.toLocaleString();
|
|
270
|
+
document.getElementById('st-orig').textContent = (d.store.originalTokens || 0).toLocaleString();
|
|
271
|
+
document.getElementById('st-saved').textContent = (d.store.tokensSaved || 0).toLocaleString();
|
|
226
272
|
document.getElementById('st-injected').textContent = d.store.injectedCount;
|
|
227
273
|
document.getElementById('st-dedup').textContent = Math.round(d.store.dedupHitRate * 100) + '%';
|
|
274
|
+
var sdr = d.store.storageDedupRate || 0;
|
|
275
|
+
document.getElementById('st-sdedup').textContent = (sdr * 100 >= 10 ? Math.round(sdr * 100) : (sdr * 100).toFixed(1)) + '%';
|
|
276
|
+
document.getElementById('st-collapsed').textContent = d.store.dedupCollapsed || 0;
|
|
228
277
|
document.getElementById('st-lastid').textContent = d.session.lastCheckpointId || '—';
|
|
229
278
|
|
|
279
|
+
// Repo-wide (all sessions in this repo's SQLite store).
|
|
280
|
+
var repo = d.repo || { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupCollapsed: 0, storageDedupRate: 0 };
|
|
281
|
+
document.getElementById('rp-count').textContent = repo.checkpointCount;
|
|
282
|
+
document.getElementById('rp-tokens').textContent = repo.totalTokenEstimate.toLocaleString();
|
|
283
|
+
document.getElementById('rp-orig').textContent = (repo.originalTokens || 0).toLocaleString();
|
|
284
|
+
document.getElementById('rp-saved').textContent = (repo.tokensSaved || 0).toLocaleString();
|
|
285
|
+
document.getElementById('rp-sessions').textContent = repo.sessionCount || 0;
|
|
286
|
+
document.getElementById('rp-collapsed').textContent = repo.dedupCollapsed || 0;
|
|
287
|
+
var rsdr = repo.storageDedupRate || 0;
|
|
288
|
+
document.getElementById('rp-sdedup').textContent = (rsdr * 100 >= 10 ? Math.round(rsdr * 100) : (rsdr * 100).toFixed(1)) + '%';
|
|
289
|
+
|
|
290
|
+
// Crew / agents (live sub-agent activity + turn).
|
|
291
|
+
var crew = d.crew || { activeAgents: 0, currentTurn: 0 };
|
|
292
|
+
document.getElementById('cr-agents').textContent = crew.activeAgents || 0;
|
|
293
|
+
document.getElementById('cr-turn').textContent = crew.currentTurn || 0;
|
|
294
|
+
document.getElementById('cr-status').textContent = (crew.activeAgents > 0)
|
|
295
|
+
? ('▶ ' + crew.activeAgents + ' running') : 'idle';
|
|
296
|
+
|
|
230
297
|
document.getElementById('cf-tier').textContent = d.tier;
|
|
231
298
|
document.getElementById('cf-threshold').textContent = d.config.thresholdTokens.toLocaleString();
|
|
232
299
|
document.getElementById('cf-gate').textContent = d.config.fastGatePct + '%';
|
|
@@ -322,9 +322,24 @@ test("state snapshot writes dashboard.json after compaction", async () => {
|
|
|
322
322
|
const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
|
|
323
323
|
// Fire auto-trigger compaction (context event above 80% threshold)
|
|
324
324
|
await h.fire("context", { type: "context", messages: h.session }, ctx);
|
|
325
|
-
const { existsSync: ex } = await import("node:fs");
|
|
325
|
+
const { existsSync: ex, readFileSync: rf } = await import("node:fs");
|
|
326
326
|
const { join: j } = await import("node:path");
|
|
327
|
-
|
|
327
|
+
const snapPath = j(h.stateDir, "dashboard.json");
|
|
328
|
+
assert.ok(ex(snapPath), "dashboard.json written after compaction");
|
|
329
|
+
const snap = JSON.parse(rf(snapPath, "utf-8"));
|
|
330
|
+
// Item B: the honest token model is wired — the original dropped region was
|
|
331
|
+
// captured (originalTokens > 0), and the saved amount never exceeds the
|
|
332
|
+
// original (saved = max(0, original − stored) ≤ original). For this tiny
|
|
333
|
+
// harness session the summary can be ≥ the region, so saved may be 0; the
|
|
334
|
+
// positive "saved > 0" case with a large region is covered by the
|
|
335
|
+
// vectorStore unit tests.
|
|
336
|
+
assert.ok(snap.store.originalTokens > 0, "snapshot.store.originalTokens captured after compaction");
|
|
337
|
+
assert.ok(
|
|
338
|
+
snap.store.originalTokens >= snap.store.tokensSaved,
|
|
339
|
+
"model invariant: original region >= tokens saved",
|
|
340
|
+
);
|
|
341
|
+
// Item A: crew (live agent) block is present in the dashboard snapshot.
|
|
342
|
+
assert.ok(snap.crew && typeof snap.crew.activeAgents === "number", "snapshot.crew.activeAgents present");
|
|
328
343
|
});
|
|
329
344
|
|
|
330
345
|
test("events.log receives compaction events", async () => {
|
|
@@ -37,6 +37,7 @@ import { recallAndInline } from "../src/recall.js";
|
|
|
37
37
|
import { autoCompactCheck } from "../src/compact.js";
|
|
38
38
|
import { estimateSessionTokens } from "../src/tokens.js";
|
|
39
39
|
import { normalizeSessionId } from "../src/store.js";
|
|
40
|
+
import { touchSession, logDaily } from "../src/store/sqlite.js";
|
|
40
41
|
import { Logger } from "../src/log.js";
|
|
41
42
|
import type { EngineMessage } from "../src/types.js";
|
|
42
43
|
import { writeFileSync, appendFileSync, readFileSync } from "node:fs";
|
|
@@ -57,6 +58,7 @@ interface SessionRuntime {
|
|
|
57
58
|
lastCompactedTokens: number;
|
|
58
59
|
dedupSkips: number; // compactions skipped because regionHash already stored
|
|
59
60
|
dedupAttempts: number; // total compaction attempts (for hit-rate denominator)
|
|
61
|
+
tokensSaved: number; // this session-instance only: reset on session_start
|
|
60
62
|
}
|
|
61
63
|
|
|
62
64
|
function envFlag(name: string, fallback: number): number {
|
|
@@ -190,12 +192,28 @@ interface DashboardSnapshot {
|
|
|
190
192
|
store: {
|
|
191
193
|
checkpointCount: number;
|
|
192
194
|
totalTokenEstimate: number;
|
|
195
|
+
originalTokens: number; // Σ original dropped-region tokens (this session)
|
|
196
|
+
tokensSaved: number; // Σ(original − stored) for this session
|
|
193
197
|
injectedCount: number;
|
|
194
198
|
dedupHitRate: number;
|
|
195
199
|
storageDedupRate: number;
|
|
196
200
|
dedupAttempts: number;
|
|
197
201
|
dedupCollapsed: number;
|
|
198
202
|
};
|
|
203
|
+
crew: {
|
|
204
|
+
activeAgents: number;
|
|
205
|
+
currentTurn: number;
|
|
206
|
+
};
|
|
207
|
+
repo: {
|
|
208
|
+
checkpointCount: number; // across all sessions in this repo's store
|
|
209
|
+
totalTokenEstimate: number; // repo-wide stored checkpoint tokens
|
|
210
|
+
originalTokens: number; // repo-wide Σ original dropped-region tokens
|
|
211
|
+
tokensSaved: number; // repo-wide cumulative (original − stored) + deduped orig
|
|
212
|
+
sessionCount: number; // distinct sessions with checkpoints
|
|
213
|
+
dedupAttempts: number; // cumulative add() calls (store-wide)
|
|
214
|
+
dedupCollapsed: number; // cumulative deduped collapses (store-wide)
|
|
215
|
+
storageDedupRate: number; // deduped / attempts, 0..1
|
|
216
|
+
};
|
|
199
217
|
}
|
|
200
218
|
|
|
201
219
|
class Dashboard {
|
|
@@ -261,6 +279,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
261
279
|
function snapshot(ctx?: ExtensionContext): void {
|
|
262
280
|
if (ctx) bindRepo(ctx.cwd);
|
|
263
281
|
const st = store.stats(rt.sessionId);
|
|
282
|
+
const repo = store.repoStats();
|
|
264
283
|
const armed = lastCtxPercent != null && lastCtxPercent >= config.fastGatePct;
|
|
265
284
|
const ready = armed && (lastCtxTokens ?? 0) >= config.thresholdTokens;
|
|
266
285
|
dashboard.snapshot({
|
|
@@ -287,7 +306,18 @@ export default function (pi: ExtensionAPI) {
|
|
|
287
306
|
},
|
|
288
307
|
context: { tokens: lastCtxTokens, percent: lastCtxPercent, contextWindow: lastCtxWindow },
|
|
289
308
|
trigger: { armed, ready, currentTokens: lastCtxTokens, thresholdTokens: config.thresholdTokens, fastGatePct: config.fastGatePct },
|
|
290
|
-
|
|
309
|
+
crew: { activeAgents, currentTurn },
|
|
310
|
+
store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, originalTokens: st.originalTokens, tokensSaved: rt.tokensSaved, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
|
|
311
|
+
repo: {
|
|
312
|
+
checkpointCount: repo.checkpointCount,
|
|
313
|
+
totalTokenEstimate: repo.totalTokenEstimate,
|
|
314
|
+
originalTokens: repo.originalTokens,
|
|
315
|
+
tokensSaved: repo.tokensSaved,
|
|
316
|
+
sessionCount: repo.sessionCount,
|
|
317
|
+
dedupAttempts: repo.dedupAttempts,
|
|
318
|
+
dedupCollapsed: repo.dedupCollapsed,
|
|
319
|
+
storageDedupRate: repo.storageDedupRate,
|
|
320
|
+
},
|
|
291
321
|
});
|
|
292
322
|
|
|
293
323
|
// Live stats widget above the editor
|
|
@@ -303,7 +333,12 @@ export default function (pi: ExtensionAPI) {
|
|
|
303
333
|
const dedupStr = storageRate * 100 >= 10
|
|
304
334
|
? `${Math.round(storageRate * 100)}%`
|
|
305
335
|
: `${(storageRate * 100).toFixed(1)}%`;
|
|
306
|
-
|
|
336
|
+
// saved = cumulative original − stored tokens (this session). Show real
|
|
337
|
+
// token counts; use "k" only at/above 1000 so small-but-real savings are
|
|
338
|
+
// visible (previously Math.round(x/1000) rounded everything <1000 to 0).
|
|
339
|
+
const savedStr = rt.tokensSaved >= 1000
|
|
340
|
+
? `${(rt.tokensSaved / 1000).toFixed(1)}k`
|
|
341
|
+
: `${rt.tokensSaved}`;
|
|
307
342
|
const agentStr = activeAgents > 0 ? ` │ 🤖 ${activeAgents} agent${activeAgents === 1 ? "" : "s"}` : "";
|
|
308
343
|
const turnStr = currentTurn > 0 ? ` │ turn ${currentTurn}` : "";
|
|
309
344
|
ctx.ui.setWidget(
|
|
@@ -326,6 +361,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
326
361
|
lastCompactedTokens: 0,
|
|
327
362
|
dedupSkips: 0,
|
|
328
363
|
dedupAttempts: 0,
|
|
364
|
+
tokensSaved: 0,
|
|
329
365
|
};
|
|
330
366
|
let debounceUntil = 0;
|
|
331
367
|
// Agent tracking for real-time widget updates
|
|
@@ -352,6 +388,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
352
388
|
lastCompactedTokens: 0,
|
|
353
389
|
dedupSkips: 0,
|
|
354
390
|
dedupAttempts: 0,
|
|
391
|
+
tokensSaved: 0,
|
|
355
392
|
};
|
|
356
393
|
statusKey = undefined;
|
|
357
394
|
activeAgents = 0;
|
|
@@ -392,8 +429,28 @@ export default function (pi: ExtensionAPI) {
|
|
|
392
429
|
rt.lastCompactedFrom = result.compactedFrom;
|
|
393
430
|
rt.lastCompactedTokens = result.tokenEstimate;
|
|
394
431
|
rt.dedupAttempts++;
|
|
432
|
+
// Honest "tokens saved" for this session-instance only:
|
|
433
|
+
// new checkpoint → original − stored
|
|
434
|
+
// deduped onto existing → whole original region (nothing new stored)
|
|
435
|
+
// Resets to 0 on session_start (rt is rebuilt) — so a fresh session shows 0
|
|
436
|
+
// while the repo's cumulative saved (SQLite meta) keeps the running total.
|
|
437
|
+
const saved = result.deduped
|
|
438
|
+
? result.originalTokenEstimate
|
|
439
|
+
: Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
|
|
440
|
+
rt.tokensSaved += saved;
|
|
395
441
|
if (result.deduped) rt.dedupSkips++;
|
|
396
442
|
|
|
443
|
+
// Record session activity + a daily-log entry in the per-repo SQLite store
|
|
444
|
+
// (foundation for resume-sessions / daily-log features). Best-effort — never
|
|
445
|
+
// block a compaction on bookkeeping.
|
|
446
|
+
try {
|
|
447
|
+
const repo = resolveRepoRoot(ctx.cwd);
|
|
448
|
+
touchSession(sid, repo, currentStateDir);
|
|
449
|
+
logDaily(sid, "compact", result.checkpointId, saved, currentStateDir);
|
|
450
|
+
} catch {
|
|
451
|
+
/* non-fatal: stats bookkeeping only */
|
|
452
|
+
}
|
|
453
|
+
|
|
397
454
|
// Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
|
|
398
455
|
// skip re-vectorizing an already-compacted region (zero token cost).
|
|
399
456
|
pi.appendEntry(MARKER_TYPE, {
|
|
@@ -403,7 +460,6 @@ export default function (pi: ExtensionAPI) {
|
|
|
403
460
|
deduped: result.deduped,
|
|
404
461
|
});
|
|
405
462
|
|
|
406
|
-
const saved = result.tokenEstimate;
|
|
407
463
|
setStatus(
|
|
408
464
|
ctx,
|
|
409
465
|
rt.persistedThisSession
|
package/package.json
CHANGED
package/src/engine.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import { findSuperseded, supersede } from "./supersede.js";
|
|
16
16
|
import { summarizeMessages, mergeCompactSummaries, formatCompactSummary } from "./compact.js";
|
|
17
17
|
import { extractiveSummarize } from "./extractive.js";
|
|
18
|
-
import { estimateSessionTokens } from "./tokens.js";
|
|
18
|
+
import { estimateSessionTokens, estimateBlockTokens } from "./tokens.js";
|
|
19
19
|
import { computeRegionHash, VectorStore, type SearchHit } from "./vectorStore.js";
|
|
20
20
|
import type { EngineMessage } from "./types.js";
|
|
21
21
|
|
|
@@ -51,6 +51,11 @@ export interface CompactResult {
|
|
|
51
51
|
summary: string;
|
|
52
52
|
regionHash: string;
|
|
53
53
|
tokenEstimate: number;
|
|
54
|
+
/** Token count of the original dropped region (before compaction). The honest
|
|
55
|
+
* "tokens saved" base = originalTokenEstimate − tokenEstimate (stored), or the
|
|
56
|
+
* full originalTokenEstimate when the region deduped onto an existing
|
|
57
|
+
* checkpoint (nothing new stored). */
|
|
58
|
+
originalTokenEstimate: number;
|
|
54
59
|
/** Index in `messages` where the compacted slice begins (for the caller to
|
|
55
60
|
* build a drop range). */
|
|
56
61
|
compactedFrom: number;
|
|
@@ -87,6 +92,7 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
|
|
|
87
92
|
summary: "",
|
|
88
93
|
regionHash: "",
|
|
89
94
|
tokenEstimate: 0,
|
|
95
|
+
originalTokenEstimate: 0,
|
|
90
96
|
compactedFrom,
|
|
91
97
|
};
|
|
92
98
|
}
|
|
@@ -105,7 +111,6 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
|
|
|
105
111
|
let keyDecisions: string[];
|
|
106
112
|
let nextSteps: string[];
|
|
107
113
|
let filesModified: string[];
|
|
108
|
-
let tokenEstimate: number;
|
|
109
114
|
|
|
110
115
|
if (useExtractive && !input.summary) {
|
|
111
116
|
const ext = extractiveSummarize(keep);
|
|
@@ -114,7 +119,6 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
|
|
|
114
119
|
keyDecisions = input.keyDecisions ?? ext.keyDecisions;
|
|
115
120
|
nextSteps = input.nextSteps ?? ext.nextSteps;
|
|
116
121
|
filesModified = input.filesModified ?? ext.filesModified;
|
|
117
|
-
tokenEstimate = input.tokenEstimate ?? ext.tokenEstimate;
|
|
118
122
|
} else {
|
|
119
123
|
const collapsed = input.summary ?? summarizeMessages(keep);
|
|
120
124
|
summary = formatCompactSummary(collapsed);
|
|
@@ -122,9 +126,18 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
|
|
|
122
126
|
keyDecisions = input.keyDecisions ?? [];
|
|
123
127
|
nextSteps = input.nextSteps ?? [];
|
|
124
128
|
filesModified = input.filesModified ?? [];
|
|
125
|
-
tokenEstimate = input.tokenEstimate ?? estimateSessionTokens(compactable);
|
|
126
129
|
}
|
|
127
130
|
|
|
131
|
+
// Honest "tokens saved" accounting:
|
|
132
|
+
// - originalTokenEstimate = the dropped region's token count (what context
|
|
133
|
+
// held before compaction) = the compacted slice's tokens.
|
|
134
|
+
// - storedTokens = the persisted summary's token count, computed from the
|
|
135
|
+
// actual summary string so it's honest for BOTH the extractive and legacy
|
|
136
|
+
// COLLAPSE paths (the legacy path's fallback estimateSessionTokens is the
|
|
137
|
+
// *original* size, not the stored size).
|
|
138
|
+
const originalTokenEstimate = estimateSessionTokens(compactable);
|
|
139
|
+
const storedTokens = estimateBlockTokens(summary);
|
|
140
|
+
|
|
128
141
|
// Region text = the compacted slice, used for dedup + embedding.
|
|
129
142
|
const regionText = input.regionText ?? keep.map((m) => m.text).join("\n");
|
|
130
143
|
const regionHash = computeRegionHash(regionText);
|
|
@@ -137,7 +150,8 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
|
|
|
137
150
|
nextSteps,
|
|
138
151
|
filesModified,
|
|
139
152
|
regionText,
|
|
140
|
-
tokenEstimate,
|
|
153
|
+
tokenEstimate: storedTokens,
|
|
154
|
+
originalTokenEstimate,
|
|
141
155
|
timestamp: input.timestamp ?? 0,
|
|
142
156
|
});
|
|
143
157
|
|
|
@@ -148,7 +162,8 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
|
|
|
148
162
|
checkpointId: add.checkpoint.checkpointId,
|
|
149
163
|
summary,
|
|
150
164
|
regionHash,
|
|
151
|
-
tokenEstimate,
|
|
165
|
+
tokenEstimate: storedTokens,
|
|
166
|
+
originalTokenEstimate,
|
|
152
167
|
compactedFrom,
|
|
153
168
|
};
|
|
154
169
|
}
|
package/src/store/sqlite.ts
CHANGED
|
@@ -80,6 +80,7 @@ function initSchema(db: Database.Database): void {
|
|
|
80
80
|
files_modified TEXT, -- JSON array
|
|
81
81
|
embedding_blob BLOB, -- float32 vector
|
|
82
82
|
token_estimate INTEGER,
|
|
83
|
+
original_token_estimate INTEGER, -- dropped region size (tokens saved = orig − stored)
|
|
83
84
|
timestamp INTEGER,
|
|
84
85
|
dedup_status TEXT DEFAULT 'active',
|
|
85
86
|
compressed_original BLOB -- optional DR copy
|
|
@@ -145,6 +146,49 @@ function initSchema(db: Database.Database): void {
|
|
|
145
146
|
);
|
|
146
147
|
CREATE INDEX IF NOT EXISTS idx_raptor_session ON raptor_nodes(session_id);
|
|
147
148
|
CREATE INDEX IF NOT EXISTS idx_raptor_parent ON raptor_nodes(parent_id);
|
|
149
|
+
|
|
150
|
+
-- Foundation for future features (resume sessions, daily log, lessons
|
|
151
|
+
-- learned). Scaffolded now so all store data lives in SQLite from day one;
|
|
152
|
+
-- population is minimal (touchSession / logDaily on compact) and the full
|
|
153
|
+
-- UI/recall for these lands in later sprints.
|
|
154
|
+
|
|
155
|
+
-- Per-session registry (resume + per-repo session history).
|
|
156
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
157
|
+
session_id TEXT PRIMARY KEY,
|
|
158
|
+
repo TEXT,
|
|
159
|
+
started_at INTEGER,
|
|
160
|
+
ended_at INTEGER,
|
|
161
|
+
last_compacted_at INTEGER,
|
|
162
|
+
status TEXT DEFAULT 'active'
|
|
163
|
+
);
|
|
164
|
+
|
|
165
|
+
-- Append-only daily activity log (the "daily log" feature seed).
|
|
166
|
+
CREATE TABLE IF NOT EXISTS daily_log (
|
|
167
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
168
|
+
day TEXT NOT NULL, -- YYYY-MM-DD
|
|
169
|
+
session_id TEXT,
|
|
170
|
+
event TEXT, -- e.g. 'compact'
|
|
171
|
+
detail TEXT,
|
|
172
|
+
tokens_saved INTEGER DEFAULT 0,
|
|
173
|
+
ts INTEGER
|
|
174
|
+
);
|
|
175
|
+
CREATE INDEX IF NOT EXISTS idx_daily_log_day ON daily_log(day);
|
|
176
|
+
|
|
177
|
+
-- Lessons learned (future recall/browse feature seed).
|
|
178
|
+
CREATE TABLE IF NOT EXISTS lessons (
|
|
179
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
180
|
+
session_id TEXT,
|
|
181
|
+
repo TEXT,
|
|
182
|
+
lesson TEXT,
|
|
183
|
+
ts INTEGER
|
|
184
|
+
);
|
|
185
|
+
|
|
186
|
+
-- FTS5 trigram virtual table (Sprint 9+ pg_trgm-equivalent verification).
|
|
187
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_trgm USING fts5(
|
|
188
|
+
id UNINDEXED,
|
|
189
|
+
normalized_text,
|
|
190
|
+
tokenize='trigram'
|
|
191
|
+
);
|
|
148
192
|
`);
|
|
149
193
|
const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get() as
|
|
150
194
|
| { value: string }
|
|
@@ -152,15 +196,139 @@ function initSchema(db: Database.Database): void {
|
|
|
152
196
|
if (!v) {
|
|
153
197
|
db.prepare("INSERT INTO meta(key, value) VALUES(?, ?)").run("schema_version", String(SCHEMA_VERSION));
|
|
154
198
|
}
|
|
199
|
+
}
|
|
155
200
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
201
|
+
/** Read a string-valued meta key (or undefined). Used for cumulative counters. */
|
|
202
|
+
export function getMeta(key: string, stateDir: string = getStateDir()): string | undefined {
|
|
203
|
+
const db = openStore(stateDir);
|
|
204
|
+
const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key) as
|
|
205
|
+
| { value: string }
|
|
206
|
+
| undefined;
|
|
207
|
+
return row?.value;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Cumulative "tokens saved" — the sum of stored checkpoint token estimates across
|
|
212
|
+
* all compactions in this store (one per repo). Persisted in the SQLite `meta`
|
|
213
|
+
* table so it survives session restarts and travels with the repo's state dir,
|
|
214
|
+
* mirroring how `storageDedupRate` is cumulative. Incremented in VectorStore.add()
|
|
215
|
+
* when a new (non-deduped) checkpoint is persisted.
|
|
216
|
+
*/
|
|
217
|
+
export function getTokensSaved(stateDir: string = getStateDir()): number {
|
|
218
|
+
const raw = getMeta("tokens_saved", stateDir);
|
|
219
|
+
const n = raw == null ? 0 : Number(raw);
|
|
220
|
+
return Number.isFinite(n) ? n : 0;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** Add `delta` (>=0) to the cumulative tokens-saved counter. */
|
|
224
|
+
export function addTokensSaved(delta: number, stateDir: string = getStateDir()): void {
|
|
225
|
+
if (!(delta > 0)) return;
|
|
226
|
+
const db = openStore(stateDir);
|
|
227
|
+
db.prepare(
|
|
228
|
+
`INSERT INTO meta(key, value) VALUES('tokens_saved', ?)
|
|
229
|
+
ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
|
|
230
|
+
).run(String(delta), delta);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Cumulative store-wide dedup accounting (Sprint 9+). Persisted in the SQLite
|
|
234
|
+
* `meta` table so it survives session restarts and travels with the repo's
|
|
235
|
+
* state dir — mirroring `tokens_saved`. Replaces the legacy JSON
|
|
236
|
+
* `dedup-stats.json` file (all stats now live in the SQLite store). */
|
|
237
|
+
export interface DedupStats {
|
|
238
|
+
/** Total add() calls (new checkpoints + deduped collapses). */
|
|
239
|
+
attempts: number;
|
|
240
|
+
/** add() calls that collapsed onto an existing checkpoint. */
|
|
241
|
+
deduped: number;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Read a store-wide integer counter from the meta table (0 if absent). */
|
|
245
|
+
export function getMetaNumber(key: string, stateDir: string = getStateDir()): number {
|
|
246
|
+
const raw = getMeta(key, stateDir);
|
|
247
|
+
const n = raw == null ? 0 : Number(raw);
|
|
248
|
+
return Number.isFinite(n) ? n : 0;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** Atomically add `delta` to an integer meta counter. */
|
|
252
|
+
function incMeta(key: string, delta: number, stateDir: string = getStateDir()): void {
|
|
253
|
+
if (!(delta > 0)) return;
|
|
254
|
+
const db = openStore(stateDir);
|
|
255
|
+
db.prepare(
|
|
256
|
+
`INSERT INTO meta(key, value) VALUES(?, ?)
|
|
257
|
+
ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
|
|
258
|
+
).run(key, String(delta), delta);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** Read the cumulative store-wide dedup counters. */
|
|
262
|
+
export function getDedupStats(stateDir: string = getStateDir()): DedupStats {
|
|
263
|
+
return {
|
|
264
|
+
attempts: getMetaNumber("dedup_attempts", stateDir),
|
|
265
|
+
deduped: getMetaNumber("deduped", stateDir),
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** Increment the store-wide dedup counters for one add() call. */
|
|
270
|
+
export function bumpDedupStats(deduped: boolean, stateDir: string = getStateDir()): void {
|
|
271
|
+
incMeta("dedup_attempts", 1, stateDir);
|
|
272
|
+
if (deduped) incMeta("deduped", 1, stateDir);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// --- Future-feature foundation (resume sessions / daily log / lessons) -------
|
|
276
|
+
// Scaffolded tables + minimal helpers so all store data lives in SQLite from
|
|
277
|
+
// day one. Full UI/recall for these lands in later sprints.
|
|
278
|
+
|
|
279
|
+
/** Upsert a `sessions` row (resume + per-repo session history). */
|
|
280
|
+
export function touchSession(
|
|
281
|
+
sessionId: string,
|
|
282
|
+
repo: string | undefined,
|
|
283
|
+
stateDir: string = getStateDir(),
|
|
284
|
+
): void {
|
|
285
|
+
const db = openStore(stateDir);
|
|
286
|
+
const sid = normalizeSessionId(sessionId);
|
|
287
|
+
const existing = db
|
|
288
|
+
.prepare("SELECT started_at FROM sessions WHERE session_id = ?")
|
|
289
|
+
.get(sid) as { started_at: number | null } | undefined;
|
|
290
|
+
const now = Math.floor(Date.now() / 1000);
|
|
291
|
+
if (!existing) {
|
|
292
|
+
db.prepare(
|
|
293
|
+
`INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
|
|
294
|
+
VALUES(?, ?, ?, ?, 'active')`,
|
|
295
|
+
).run(sid, repo ?? null, now, now);
|
|
296
|
+
} else {
|
|
297
|
+
db.prepare(
|
|
298
|
+
"UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?",
|
|
299
|
+
).run(now, repo ?? null, sid);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
|
|
304
|
+
export function logDaily(
|
|
305
|
+
sessionId: string,
|
|
306
|
+
event: string,
|
|
307
|
+
detail: string | undefined,
|
|
308
|
+
tokensSaved: number,
|
|
309
|
+
stateDir: string = getStateDir(),
|
|
310
|
+
): void {
|
|
311
|
+
const db = openStore(stateDir);
|
|
312
|
+
const day = new Date().toISOString().slice(0, 10);
|
|
313
|
+
const now = Math.floor(Date.now() / 1000);
|
|
314
|
+
db.prepare(
|
|
315
|
+
`INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
|
|
316
|
+
VALUES(?, ?, ?, ?, ?, ?)`,
|
|
317
|
+
).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** Append a `lessons` entry (future lessons-learned browse/recall). */
|
|
321
|
+
export function addLesson(
|
|
322
|
+
sessionId: string,
|
|
323
|
+
repo: string | undefined,
|
|
324
|
+
lesson: string,
|
|
325
|
+
stateDir: string = getStateDir(),
|
|
326
|
+
): void {
|
|
327
|
+
const db = openStore(stateDir);
|
|
328
|
+
const now = Math.floor(Date.now() / 1000);
|
|
329
|
+
db.prepare(
|
|
330
|
+
`INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`,
|
|
331
|
+
).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
|
|
164
332
|
}
|
|
165
333
|
|
|
166
334
|
/** Map a DB row to the public StoredCheckpoint shape. */
|
|
@@ -175,6 +343,7 @@ function rowToCheckpoint(row: any): StoredCheckpoint {
|
|
|
175
343
|
nextSteps: row.next_steps ? JSON.parse(row.next_steps) : [],
|
|
176
344
|
filesModified: row.files_modified ? JSON.parse(row.files_modified) : [],
|
|
177
345
|
tokenEstimate: row.token_estimate ?? 0,
|
|
346
|
+
originalTokenEstimate: row.original_token_estimate ?? undefined,
|
|
178
347
|
regionHash: row.region_hash ?? "",
|
|
179
348
|
contentHash: row.content_hash ?? undefined,
|
|
180
349
|
contentHash2: row.content_hash2 ?? undefined,
|
|
@@ -197,11 +366,11 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
|
|
|
197
366
|
(id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
|
|
198
367
|
normalized_text, summary, topic_summary, summary_hash,
|
|
199
368
|
key_decisions, next_steps, files_modified, embedding_blob,
|
|
200
|
-
token_estimate, timestamp, dedup_status, compressed_original)
|
|
369
|
+
token_estimate, original_token_estimate, timestamp, dedup_status, compressed_original)
|
|
201
370
|
VALUES (@id, @sid, @region_hash, @content_hash, @content_hash2, @content_hash_version,
|
|
202
371
|
@normalized_text, @summary, @topic_summary, @summary_hash,
|
|
203
372
|
@key_decisions, @next_steps, @files_modified, @embedding_blob,
|
|
204
|
-
@token_estimate, @timestamp, @dedup_status, @compressed_original)
|
|
373
|
+
@token_estimate, @original_token_estimate, @timestamp, @dedup_status, @compressed_original)
|
|
205
374
|
ON CONFLICT(session_id, id) DO UPDATE SET
|
|
206
375
|
summary=excluded.summary,
|
|
207
376
|
topic_summary=excluded.topic_summary,
|
|
@@ -211,6 +380,7 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
|
|
|
211
380
|
files_modified=excluded.files_modified,
|
|
212
381
|
embedding_blob=excluded.embedding_blob,
|
|
213
382
|
token_estimate=excluded.token_estimate,
|
|
383
|
+
original_token_estimate=excluded.original_token_estimate,
|
|
214
384
|
timestamp=excluded.timestamp,
|
|
215
385
|
dedup_status=excluded.dedup_status,
|
|
216
386
|
compressed_original=excluded.compressed_original`,
|
|
@@ -230,6 +400,7 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
|
|
|
230
400
|
files_modified: jsonText(cp.filesModified),
|
|
231
401
|
embedding_blob: encodeEmbedding(cp.embedding ?? []),
|
|
232
402
|
token_estimate: cp.tokenEstimate ?? 0,
|
|
403
|
+
original_token_estimate: cp.originalTokenEstimate ?? null,
|
|
233
404
|
timestamp: cp.timestamp ?? 0,
|
|
234
405
|
dedup_status: "active",
|
|
235
406
|
compressed_original: cp.compressedOriginal ?? null,
|
|
@@ -423,6 +594,52 @@ export function storeStats(sessionId: string, stateDir: string = getStateDir()):
|
|
|
423
594
|
};
|
|
424
595
|
}
|
|
425
596
|
|
|
597
|
+
/** Repo-wide stats — aggregates every session in this store (one per repo).
|
|
598
|
+
* Backed by the SQLite `meta` cumulative counters (`tokens_saved`,
|
|
599
|
+
* `dedup_attempts`, `deduped`) plus a SUM over all `context_chunks`. This is the
|
|
600
|
+
* cumulative, resumable, cross-device view the dashboard surfaces as "Repo …". */
|
|
601
|
+
export interface RepoStats {
|
|
602
|
+
/** Total checkpoints across all sessions (excludes SemDeDup-removed rows). */
|
|
603
|
+
checkpointCount: number;
|
|
604
|
+
/** Sum of all stored checkpoint token estimates (repo-wide). */
|
|
605
|
+
totalTokenEstimate: number;
|
|
606
|
+
/** Total active sessions with at least one checkpoint. */
|
|
607
|
+
sessionCount: number;
|
|
608
|
+
/** Cumulative stored-summary tokens saved (Σ stored summaries). */
|
|
609
|
+
tokensSaved: number;
|
|
610
|
+
/** Sum of original dropped-region token estimates (repo-wide). */
|
|
611
|
+
originalTokens: number;
|
|
612
|
+
/** Cumulative dedup add() attempts (store-wide). */
|
|
613
|
+
dedupAttempts: number;
|
|
614
|
+
/** Cumulative deduped collapses (store-wide). */
|
|
615
|
+
dedupCollapsed: number;
|
|
616
|
+
/** Storage dedup rate (deduped / attempts), 0..1. */
|
|
617
|
+
storageDedupRate: number;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
export function repoStats(stateDir: string = getStateDir()): RepoStats {
|
|
621
|
+
const db = openStore(stateDir);
|
|
622
|
+
const row = db
|
|
623
|
+
.prepare(
|
|
624
|
+
`SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
|
|
625
|
+
COALESCE(SUM(original_token_estimate),0) AS orig,
|
|
626
|
+
COUNT(DISTINCT session_id) AS sessions
|
|
627
|
+
FROM context_chunks WHERE dedup_status != 'removed'`,
|
|
628
|
+
)
|
|
629
|
+
.get() as { c: number; tok: number; orig: number; sessions: number };
|
|
630
|
+
const ds = getDedupStats(stateDir);
|
|
631
|
+
return {
|
|
632
|
+
checkpointCount: row.c,
|
|
633
|
+
totalTokenEstimate: row.tok,
|
|
634
|
+
originalTokens: row.orig,
|
|
635
|
+
sessionCount: row.sessions,
|
|
636
|
+
tokensSaved: getMetaNumber("tokens_saved", stateDir),
|
|
637
|
+
dedupAttempts: ds.attempts,
|
|
638
|
+
dedupCollapsed: ds.deduped,
|
|
639
|
+
storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
|
|
426
643
|
/** Close and evict a cached connection (test teardown only). */
|
|
427
644
|
export function closeStore(stateDir: string): void {
|
|
428
645
|
const db = cache.get(stateDir);
|
package/src/store.ts
CHANGED
|
@@ -58,6 +58,9 @@ export interface StoredCheckpoint {
|
|
|
58
58
|
nextSteps: string[];
|
|
59
59
|
filesModified: string[];
|
|
60
60
|
tokenEstimate: number;
|
|
61
|
+
/** Token count of the ORIGINAL dropped region (before compaction). Drives the
|
|
62
|
+
* honest "tokens saved" = originalTokenEstimate − tokenEstimate. */
|
|
63
|
+
originalTokenEstimate?: number;
|
|
61
64
|
regionHash: string;
|
|
62
65
|
/** Primary content-addressable hash (full 64-hex SHA-256 of normalized text). */
|
|
63
66
|
contentHash?: string;
|
|
@@ -168,25 +171,8 @@ export function saveSessionState(sessionId: string, state: SessionState, stateDi
|
|
|
168
171
|
}
|
|
169
172
|
|
|
170
173
|
/**
|
|
171
|
-
* Cumulative store-wide dedup accounting
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
* state dir.
|
|
174
|
+
* Cumulative store-wide dedup accounting now lives in the SQLite `meta` table
|
|
175
|
+
* (see store/sqlite.ts: getDedupStats / bumpDedupStats). All store stats are
|
|
176
|
+
* SQLite-backed so they survive session restarts and travel with the repo's
|
|
177
|
+
* state dir. The legacy JSON `dedup-stats.json` path was removed.
|
|
175
178
|
*/
|
|
176
|
-
export interface DedupStats {
|
|
177
|
-
/** Total add() calls (new checkpoints + deduped collapses). */
|
|
178
|
-
attempts: number;
|
|
179
|
-
/** add() calls that collapsed onto an existing checkpoint. */
|
|
180
|
-
deduped: number;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
const DEDUP_STATS_FILE = "dedup-stats.json";
|
|
184
|
-
|
|
185
|
-
export function loadDedupStats(stateDir: string = getStateDir()): DedupStats {
|
|
186
|
-
const file = join(stateDir, DEDUP_STATS_FILE);
|
|
187
|
-
return readGzJson<DedupStats>(file, { attempts: 0, deduped: 0 });
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
export function saveDedupStats(stats: DedupStats, stateDir: string = getStateDir()): void {
|
|
191
|
-
writeGzJson(join(stateDir, DEDUP_STATS_FILE), stats);
|
|
192
|
-
}
|
package/src/vectorStore.test.ts
CHANGED
|
@@ -250,6 +250,59 @@ test("stats reports counts, last checkpoint, and dedup rate", () => {
|
|
|
250
250
|
assert.ok(Math.abs(st2.dedupHitRate - 0.5) < 1e-9);
|
|
251
251
|
});
|
|
252
252
|
|
|
253
|
+
test("tokensSaved = original − stored per session; deduped add saves the whole region", () => {
|
|
254
|
+
const s = store();
|
|
255
|
+
// Two genuinely new checkpoints. saved = original − stored.
|
|
256
|
+
// cp1: orig 2000, stored 500 → saved 1500
|
|
257
|
+
// cp2: orig 3000, stored 700 → saved 2300
|
|
258
|
+
s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
|
|
259
|
+
s.add({ sessionId: "sess_saved", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
|
|
260
|
+
const st = s.stats("sess_saved");
|
|
261
|
+
assert.equal(st.totalTokenEstimate, 1200, "Σ stored summaries");
|
|
262
|
+
assert.equal(st.originalTokens, 5000, "Σ original region tokens");
|
|
263
|
+
assert.equal(st.tokensSaved, 3800, "per-session saved = Σ(original − stored) = 1500 + 2300");
|
|
264
|
+
assert.equal(st.dedupCollapsed, 0);
|
|
265
|
+
assert.equal(st.dedupAttempts, 2);
|
|
266
|
+
|
|
267
|
+
// A third add that dedups onto an existing region: whole original region (2000)
|
|
268
|
+
// is discarded (nothing new stored) → repo saved grows by the full original,
|
|
269
|
+
// dedupCollapsed bumps, and no new checkpoint row is created.
|
|
270
|
+
const deduped = s.add({ sessionId: "sess_saved", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
|
|
271
|
+
assert.ok(deduped.deduped, "identical region should dedup");
|
|
272
|
+
const st3 = s.stats("sess_saved");
|
|
273
|
+
// Per-session DB sum only covers stored rows (deduped adds create no row), so
|
|
274
|
+
// the per-session figure is unchanged; the deduped save lands in the repo meta.
|
|
275
|
+
assert.equal(st3.tokensSaved, 3800, "per-session DB sum unchanged by deduped add");
|
|
276
|
+
assert.equal(st3.dedupCollapsed, 1, "deduped collapse counted");
|
|
277
|
+
assert.equal(st3.dedupAttempts, 3);
|
|
278
|
+
// Repo cumulative counter DID capture the deduped region's full original size.
|
|
279
|
+
assert.equal(s.repoStats().tokensSaved, 3800 + 2000, "repo saved includes deduped original");
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
test("repoStats aggregates every session + counts deduped original tokens", () => {
|
|
283
|
+
const dir = join(baseTmp, `repo-${counter++}`);
|
|
284
|
+
const a = new VectorStore({ dedupSim: 0.9, stateDir: dir });
|
|
285
|
+
const b = new VectorStore({ dedupSim: 0.9, stateDir: dir }); // same disk store, diff instance
|
|
286
|
+
a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 1 });
|
|
287
|
+
b.add({ sessionId: "sess_b", summary: "beta", regionText: "region beta text", tokenEstimate: 700, originalTokenEstimate: 3000, timestamp: 2 });
|
|
288
|
+
|
|
289
|
+
const repo = a.repoStats();
|
|
290
|
+
assert.equal(repo.checkpointCount, 2, "checkpoints across both sessions");
|
|
291
|
+
assert.equal(repo.sessionCount, 2, "two distinct sessions");
|
|
292
|
+
assert.equal(repo.totalTokenEstimate, 1200, "Σ stored");
|
|
293
|
+
assert.equal(repo.originalTokens, 5000, "Σ original");
|
|
294
|
+
assert.equal(repo.tokensSaved, 3800, "repo saved = Σ(original − stored) = 1500 + 2300");
|
|
295
|
+
assert.equal(repo.dedupCollapsed, 0);
|
|
296
|
+
|
|
297
|
+
// A deduped add into sess_a: whole original region saved, no new row.
|
|
298
|
+
const deduped = a.add({ sessionId: "sess_a", summary: "alpha", regionText: "region alpha text", tokenEstimate: 500, originalTokenEstimate: 2000, timestamp: 3 });
|
|
299
|
+
assert.ok(deduped.deduped);
|
|
300
|
+
const repo2 = a.repoStats();
|
|
301
|
+
assert.equal(repo2.tokensSaved, 3800 + 2000, "deduped collapse adds full original region to repo saved");
|
|
302
|
+
assert.equal(repo2.dedupCollapsed, 1);
|
|
303
|
+
assert.equal(repo2.checkpointCount, 2, "still two stored checkpoints");
|
|
304
|
+
});
|
|
305
|
+
|
|
253
306
|
test("computeRegionHash normalizes whitespace before hashing", () => {
|
|
254
307
|
const h1 = computeRegionHash("foo bar");
|
|
255
308
|
const h2 = computeRegionHash("foo bar");
|
package/src/vectorStore.ts
CHANGED
|
@@ -14,7 +14,7 @@ import { cosineSimilarity, defaultEmbedder } from "./embedder.js";
|
|
|
14
14
|
import { loadDedupConfig, type DedupConfigShape, type DedupTier } from "./config/dedup.js";
|
|
15
15
|
import { logDecision } from "./monitoring.js";
|
|
16
16
|
import type { StoredCheckpoint, SessionState } from "./store.js";
|
|
17
|
-
import { getStateDir, normalizeSessionId, compressSmart
|
|
17
|
+
import { getStateDir, normalizeSessionId, compressSmart } from "./store.js";
|
|
18
18
|
import { computeContentDigest } from "./dedup/digest.js";
|
|
19
19
|
import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "./dedup/l1-minhash.js";
|
|
20
20
|
import { lshBands } from "./dedup/l1-lsh.js";
|
|
@@ -32,6 +32,10 @@ import {
|
|
|
32
32
|
insertLshBuckets,
|
|
33
33
|
lshCandidateChunks,
|
|
34
34
|
setDedupStatus,
|
|
35
|
+
addTokensSaved,
|
|
36
|
+
getDedupStats,
|
|
37
|
+
bumpDedupStats,
|
|
38
|
+
repoStats as repoStatsFromStore,
|
|
35
39
|
} from "./store/sqlite.js";
|
|
36
40
|
import { migrateJsonToSqlite } from "./store/migrate.js";
|
|
37
41
|
|
|
@@ -49,6 +53,11 @@ export interface AddInput {
|
|
|
49
53
|
nextSteps?: string[];
|
|
50
54
|
filesModified?: string[];
|
|
51
55
|
tokenEstimate?: number;
|
|
56
|
+
/** Token count of the ORIGINAL dropped region (before compaction). Drives the
|
|
57
|
+
* honest "tokens saved" = originalTokenEstimate − tokenEstimate (stored), or
|
|
58
|
+
* the full originalTokenEstimate when the region dedups (nothing new stored).
|
|
59
|
+
* Optional for back-compat with direct add() callers; defaults to stored. */
|
|
60
|
+
originalTokenEstimate?: number;
|
|
52
61
|
/** Raw text of the compacted region — used to derive the regionHash + vector. */
|
|
53
62
|
regionText: string;
|
|
54
63
|
timestamp: number;
|
|
@@ -136,10 +145,11 @@ export class VectorStore {
|
|
|
136
145
|
const sessionId = normalizeSessionId(input.sessionId);
|
|
137
146
|
const regionHash = computeRegionHash(input.regionText);
|
|
138
147
|
const all = listCheckpoints(sessionId, this.stateDir);
|
|
148
|
+
// Honest "tokens saved" base for this region. For a deduped add the whole
|
|
149
|
+
// original region is discarded (nothing new stored); for a new checkpoint
|
|
150
|
+
// we persist (orig − stored). Falls back to stored when orig is unknown.
|
|
151
|
+
const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
|
|
139
152
|
const cfg = this.cfg;
|
|
140
|
-
// Cumulative store-wide dedup accounting (survives session resets).
|
|
141
|
-
const ds = loadDedupStats(this.stateDir);
|
|
142
|
-
ds.attempts++;
|
|
143
153
|
// Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
|
|
144
154
|
// and which tier.
|
|
145
155
|
let markOnly: DedupTier | null = null;
|
|
@@ -165,8 +175,9 @@ export class VectorStore {
|
|
|
165
175
|
} else {
|
|
166
176
|
contentMatch.timestamp = input.timestamp;
|
|
167
177
|
upsertCheckpoint(contentMatch, this.stateDir);
|
|
168
|
-
|
|
169
|
-
|
|
178
|
+
bumpDedupStats(true, this.stateDir);
|
|
179
|
+
// Deduped: whole original region discarded, nothing new stored.
|
|
180
|
+
addTokensSaved(origTokens, this.stateDir);
|
|
170
181
|
const r = { checkpoint: contentMatch, deduped: true, reason: "contentHash" };
|
|
171
182
|
this.record("L0", "deduped", "contentHash", Date.now() - t0);
|
|
172
183
|
return r;
|
|
@@ -181,8 +192,9 @@ export class VectorStore {
|
|
|
181
192
|
if (cfg.MARK_ONLY_L0) {
|
|
182
193
|
markOnly = "L0"; // fall through
|
|
183
194
|
} else {
|
|
184
|
-
|
|
185
|
-
|
|
195
|
+
bumpDedupStats(true, this.stateDir);
|
|
196
|
+
// Deduped: whole original region discarded, nothing new stored.
|
|
197
|
+
addTokensSaved(origTokens, this.stateDir);
|
|
186
198
|
const r = { checkpoint: regionMatch, deduped: true, reason: "regionHash" };
|
|
187
199
|
this.record("L0", "deduped", "regionHash", Date.now() - t0);
|
|
188
200
|
return r;
|
|
@@ -203,8 +215,9 @@ export class VectorStore {
|
|
|
203
215
|
} else {
|
|
204
216
|
summaryMatch.timestamp = input.timestamp;
|
|
205
217
|
upsertCheckpoint(summaryMatch, this.stateDir);
|
|
206
|
-
|
|
207
|
-
|
|
218
|
+
bumpDedupStats(true, this.stateDir);
|
|
219
|
+
// Deduped: whole original region discarded, nothing new stored.
|
|
220
|
+
addTokensSaved(origTokens, this.stateDir);
|
|
208
221
|
const r = { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" };
|
|
209
222
|
this.record("L0", "deduped", "summaryHash", Date.now() - t0);
|
|
210
223
|
return r;
|
|
@@ -221,8 +234,7 @@ export class VectorStore {
|
|
|
221
234
|
if (l1 && !cfg.MARK_ONLY_L1) {
|
|
222
235
|
l1.timestamp = input.timestamp;
|
|
223
236
|
upsertCheckpoint(l1, this.stateDir);
|
|
224
|
-
|
|
225
|
-
saveDedupStats(ds, this.stateDir);
|
|
237
|
+
bumpDedupStats(true, this.stateDir);
|
|
226
238
|
const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
|
|
227
239
|
this.record("L1", "deduped", "l1MinHash", Date.now() - t0);
|
|
228
240
|
return r;
|
|
@@ -257,8 +269,9 @@ export class VectorStore {
|
|
|
257
269
|
// Near-identical — update timestamp on existing checkpoint
|
|
258
270
|
nearest.checkpoint.timestamp = input.timestamp;
|
|
259
271
|
upsertCheckpoint(nearest.checkpoint, this.stateDir);
|
|
260
|
-
|
|
261
|
-
|
|
272
|
+
bumpDedupStats(true, this.stateDir);
|
|
273
|
+
// Deduped: whole original region discarded, nothing new stored.
|
|
274
|
+
addTokensSaved(origTokens, this.stateDir);
|
|
262
275
|
const r = { checkpoint: nearest.checkpoint, deduped: true, reason: "contentSimilarity" };
|
|
263
276
|
this.record("L2", "deduped", "contentSimilarity", Date.now() - t0);
|
|
264
277
|
return r;
|
|
@@ -279,6 +292,7 @@ export class VectorStore {
|
|
|
279
292
|
nextSteps: input.nextSteps ?? [],
|
|
280
293
|
filesModified: input.filesModified ?? [],
|
|
281
294
|
tokenEstimate: input.tokenEstimate ?? 0,
|
|
295
|
+
originalTokenEstimate: input.originalTokenEstimate,
|
|
282
296
|
regionHash,
|
|
283
297
|
contentHash: digest.contentHash,
|
|
284
298
|
contentHash2: digest.contentHash2,
|
|
@@ -291,6 +305,12 @@ export class VectorStore {
|
|
|
291
305
|
// Persistence is SQLite (store/sqlite.ts). upsertCheckpoint keeps the
|
|
292
306
|
// idempotent-by-id semantics the old JSON append implied.
|
|
293
307
|
upsertCheckpoint(checkpoint, this.stateDir);
|
|
308
|
+
// Cumulative "tokens saved" counter (per-repo SQLite meta). For a NEW
|
|
309
|
+
// checkpoint the saved amount is (original − stored); for a deduped add the
|
|
310
|
+
// whole original region is discarded (handled in the deduped return paths
|
|
311
|
+
// below). Survives sessions and travels with the repo.
|
|
312
|
+
const stored = input.tokenEstimate ?? 0;
|
|
313
|
+
addTokensSaved(Math.max(0, origTokens - stored), this.stateDir);
|
|
294
314
|
// L1: persist this checkpoint's MinHash signature + LSH buckets so future
|
|
295
315
|
// near-duplicate inserts can find it. Deterministic given the seed.
|
|
296
316
|
const sig = minhashSignature(input.regionText);
|
|
@@ -320,7 +340,8 @@ export class VectorStore {
|
|
|
320
340
|
} else {
|
|
321
341
|
this.record("L0", "new", undefined, Date.now() - t0);
|
|
322
342
|
}
|
|
323
|
-
|
|
343
|
+
// Cumulative store-wide dedup accounting (attempt, not collapsed).
|
|
344
|
+
bumpDedupStats(false, this.stateDir);
|
|
324
345
|
return { checkpoint, deduped: false };
|
|
325
346
|
}
|
|
326
347
|
|
|
@@ -517,6 +538,8 @@ export class VectorStore {
|
|
|
517
538
|
injectedCount: number;
|
|
518
539
|
dedupHitRate: number; // injected / checkpoints, 0..1
|
|
519
540
|
storageDedupRate: number; // deduped adds / total adds, 0..1 (cumulative)
|
|
541
|
+
tokensSaved: number; // Σ(original − stored) for this session's checkpoints
|
|
542
|
+
originalTokens: number; // Σ original region size for this session's checkpoints
|
|
520
543
|
dedupAttempts: number; // cumulative add() calls (store-wide)
|
|
521
544
|
dedupCollapsed: number; // cumulative deduped collapses (store-wide)
|
|
522
545
|
} {
|
|
@@ -528,17 +551,38 @@ export class VectorStore {
|
|
|
528
551
|
);
|
|
529
552
|
const last = ordered[ordered.length - 1];
|
|
530
553
|
const injected = state.injectedCheckpointIds.length;
|
|
531
|
-
const ds =
|
|
554
|
+
const ds = getDedupStats(this.stateDir);
|
|
555
|
+
const sessionTok = cps.reduce((s, c) => s + (c.tokenEstimate ?? 0), 0);
|
|
556
|
+
const sessionOrig = cps.reduce((s, c) => s + (c.originalTokenEstimate ?? 0), 0);
|
|
557
|
+
// Per-session "tokens saved" = Σ(original − stored) over this session's
|
|
558
|
+
// stored checkpoints. Deduped adds (whole region discarded, nothing stored)
|
|
559
|
+
// are counted in the repo-wide meta counter via repoStats(); the per-session
|
|
560
|
+
// DB sum here covers the rows that exist.
|
|
561
|
+
const sessionSaved = cps.reduce(
|
|
562
|
+
(s, c) => s + Math.max(0, (c.originalTokenEstimate ?? 0) - (c.tokenEstimate ?? 0)),
|
|
563
|
+
0,
|
|
564
|
+
);
|
|
532
565
|
return {
|
|
533
566
|
checkpointCount: cps.length,
|
|
534
|
-
totalTokenEstimate:
|
|
567
|
+
totalTokenEstimate: sessionTok,
|
|
535
568
|
lastCheckpointId: last?.checkpointId,
|
|
536
569
|
lastSummary: last?.summary,
|
|
537
570
|
injectedCount: injected,
|
|
538
571
|
dedupHitRate: cps.length === 0 ? 0 : injected / cps.length,
|
|
539
572
|
storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
|
|
573
|
+
tokensSaved: sessionSaved,
|
|
574
|
+
originalTokens: sessionOrig,
|
|
540
575
|
dedupAttempts: ds.attempts,
|
|
541
576
|
dedupCollapsed: ds.deduped,
|
|
542
577
|
};
|
|
543
578
|
}
|
|
579
|
+
|
|
580
|
+
/**
|
|
581
|
+
* Repo-wide stats — aggregates every session in this store (one per repo).
|
|
582
|
+
* Cumulative, resumable, cross-device. Surfaces the dashboard's "Repo …"
|
|
583
|
+
* figures; distinct from {@link stats} (per-session).
|
|
584
|
+
*/
|
|
585
|
+
repoStats(): ReturnType<typeof repoStatsFromStore> {
|
|
586
|
+
return repoStatsFromStore(this.stateDir);
|
|
587
|
+
}
|
|
544
588
|
}
|