pi-mega-compact 0.7.8 → 0.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +11 -12
  2. package/dist/extensions/dashboard-server/helpers.js +37 -0
  3. package/dist/extensions/dashboard-server/html/all-repos-tab.js +26 -0
  4. package/dist/extensions/dashboard-server/html/body-open.js +23 -0
  5. package/dist/extensions/dashboard-server/html/current-repo-tab.js +130 -0
  6. package/dist/extensions/dashboard-server/html/head-open.js +16 -0
  7. package/dist/extensions/dashboard-server/html/high-score-tab.js +25 -0
  8. package/dist/extensions/dashboard-server/html/repo-detail-modal.js +26 -0
  9. package/dist/extensions/dashboard-server/html/script.js +259 -0
  10. package/dist/extensions/dashboard-server/html/styles.js +103 -0
  11. package/dist/extensions/dashboard-server/html/summary-tab.js +19 -0
  12. package/dist/extensions/dashboard-server/html-template.js +41 -0
  13. package/dist/extensions/dashboard-server/html.js +756 -0
  14. package/dist/extensions/dashboard-server/index-reader.js +133 -0
  15. package/dist/extensions/dashboard-server/server.js +370 -0
  16. package/dist/extensions/dashboard-server/snapshot.js +43 -0
  17. package/dist/extensions/dashboard-server/state.js +30 -0
  18. package/dist/extensions/dashboard-server/types.js +5 -0
  19. package/dist/extensions/dashboard-server.js +7 -1315
  20. package/dist/extensions/mega-commands.js +162 -134
  21. package/dist/extensions/mega-compact.test.js +90 -21
  22. package/dist/extensions/mega-conflict-cmds.js +5 -1
  23. package/dist/extensions/mega-dashboard-cmds.js +29 -22
  24. package/dist/extensions/mega-db-cmds.js +11 -2
  25. package/dist/extensions/mega-events/agent-handlers.js +173 -0
  26. package/dist/extensions/mega-events/compact-handlers.js +133 -0
  27. package/dist/extensions/mega-events/context-handler.js +249 -0
  28. package/dist/extensions/mega-events/register.js +21 -0
  29. package/dist/extensions/mega-events/session-handlers.js +142 -0
  30. package/dist/extensions/mega-events.js +15 -699
  31. package/dist/extensions/mega-pipeline/compact.js +324 -0
  32. package/dist/extensions/mega-pipeline/memory-review.js +38 -0
  33. package/dist/extensions/mega-pipeline/recall.js +147 -0
  34. package/dist/extensions/mega-pipeline.js +9 -480
  35. package/dist/extensions/mega-runtime/helpers.js +40 -0
  36. package/dist/extensions/mega-runtime/query.js +29 -0
  37. package/dist/extensions/mega-runtime/state.js +711 -0
  38. package/dist/extensions/mega-runtime/widget.js +197 -0
  39. package/dist/extensions/mega-runtime.js +15 -947
  40. package/dist/src/store/sqlite/checkpoints.js +145 -0
  41. package/dist/src/store/sqlite/connection.js +35 -0
  42. package/dist/src/store/sqlite/dedup-mirror.js +64 -0
  43. package/dist/src/store/sqlite/foundation.js +38 -0
  44. package/dist/src/store/sqlite/global-index.js +224 -0
  45. package/dist/src/store/sqlite/index-store.js +167 -0
  46. package/dist/src/store/sqlite/maintenance.js +235 -0
  47. package/dist/src/store/sqlite/memories.js +164 -0
  48. package/dist/src/store/sqlite/memory.js +54 -0
  49. package/dist/src/store/sqlite/meta.js +82 -0
  50. package/dist/src/store/sqlite/minhash-lsh.js +47 -0
  51. package/dist/src/store/sqlite/model-snapshots.js +47 -0
  52. package/dist/src/store/sqlite/raptor.js +57 -0
  53. package/dist/src/store/sqlite/raw-transcript.js +134 -0
  54. package/dist/src/store/sqlite/schema.js +250 -0
  55. package/dist/src/store/sqlite/session-state.js +28 -0
  56. package/dist/src/store/sqlite/sessions.js +39 -0
  57. package/dist/src/store/sqlite/stats.js +66 -0
  58. package/dist/src/store/sqlite/transaction.js +19 -0
  59. package/dist/src/store/sqlite/utils.js +120 -0
  60. package/dist/src/store/sqlite.js +20 -1607
  61. package/dist/src/vectorStore/add.js +260 -0
  62. package/dist/src/vectorStore/dedup.js +52 -0
  63. package/dist/src/vectorStore/index.js +10 -0
  64. package/dist/src/vectorStore/queries.js +83 -0
  65. package/dist/src/vectorStore/search.js +95 -0
  66. package/dist/src/vectorStore/session.js +19 -0
  67. package/dist/src/vectorStore/store.js +105 -0
  68. package/dist/src/vectorStore/types.js +6 -0
  69. package/dist/src/vectorStore/utils.js +23 -0
  70. package/extensions/dashboard-server/html.ts +758 -0
  71. package/extensions/dashboard-server/index-reader.ts +130 -0
  72. package/extensions/dashboard-server/server.ts +358 -0
  73. package/extensions/dashboard-server/snapshot.ts +44 -0
  74. package/extensions/dashboard-server/state.ts +33 -0
  75. package/extensions/dashboard-server/types.ts +134 -0
  76. package/extensions/dashboard-server.ts +7 -1431
  77. package/extensions/mega-commands.ts +33 -10
  78. package/extensions/mega-compact.test.ts +198 -43
  79. package/extensions/mega-conflict-cmds.ts +6 -2
  80. package/extensions/mega-dashboard-cmds.ts +30 -23
  81. package/extensions/mega-db-cmds.ts +11 -3
  82. package/extensions/mega-events/agent-handlers.ts +214 -0
  83. package/extensions/mega-events/compact-handlers.ts +164 -0
  84. package/extensions/mega-events/context-handler.ts +290 -0
  85. package/extensions/mega-events/register.ts +37 -0
  86. package/extensions/mega-events/session-handlers.ts +165 -0
  87. package/extensions/mega-events.ts +15 -780
  88. package/extensions/mega-pipeline/compact.ts +366 -0
  89. package/extensions/mega-pipeline/memory-review.ts +46 -0
  90. package/extensions/mega-pipeline/recall.ts +165 -0
  91. package/extensions/mega-pipeline.ts +9 -537
  92. package/extensions/mega-runtime/helpers.ts +68 -0
  93. package/extensions/mega-runtime/query.ts +29 -0
  94. package/extensions/mega-runtime/state.ts +797 -0
  95. package/extensions/mega-runtime/widget.ts +258 -0
  96. package/extensions/mega-runtime.ts +15 -1093
  97. package/package.json +4 -3
  98. package/src/store/sqlite/checkpoints.ts +204 -0
  99. package/src/store/sqlite/dedup-mirror.ts +114 -0
  100. package/src/store/sqlite/foundation.ts +63 -0
  101. package/src/store/sqlite/global-index.ts +305 -0
  102. package/src/store/sqlite/maintenance.ts +294 -0
  103. package/src/store/sqlite/memories.ts +217 -0
  104. package/src/store/sqlite/meta.ts +108 -0
  105. package/src/store/sqlite/model-snapshots.ts +83 -0
  106. package/src/store/sqlite/raptor.ts +107 -0
  107. package/src/store/sqlite/raw-transcript.ts +221 -0
  108. package/src/store/sqlite/schema.ts +258 -0
  109. package/src/store/sqlite/session-state.ts +38 -0
  110. package/src/store/sqlite/stats.ts +127 -0
  111. package/src/store/sqlite/utils.ts +125 -0
  112. package/src/store/sqlite.ts +20 -2204
@@ -1,2206 +1,22 @@
1
1
  /**
2
- * sqlite.ts — Sprint 8 storage backbone (the "one store").
2
+ * sqlite.ts — barrel re-export of the SQLite store submodules.
3
3
  *
4
- * Replaces the per-session gzipped-JSON checkpoint files with a single local
5
- * SQLite database (node:sqlite the Node built-in, in-process, FS-backed,
6
- * ZERO network calls honors PREVENT-PI-004). No native build and no install
7
- * scripts, so it survives pi's npm blocked-install-scripts gate (better-sqlite3's
8
- * native binary could not be built under pi, which crashed every `pi update
9
- * --extensions`). node:sqlite is synchronous, so every VectorStore signature
10
- * stays sync. PGlite + pgvector (async) is layered on in vectorIndex.ts for
11
- * real HNSW indexing (Slice 2 of the dual-backend plan).
12
- *
13
- * FTS5 `trigram` tokenizer is created for the Sprint 9+ dedup tiers (MinHash/LSH
14
- * / pg_trgm-equivalent verification). The default cosine path stays a linear
15
- * scan over `embedding_blob` (checkpoint counts are small, no ANN index needed).
16
- *
17
- * All queries are parameterized (PREVENT-002) — never string-concatenated.
18
- */
19
-
20
- import { DatabaseSync } from "node:sqlite";
21
- import { existsSync, mkdirSync, statSync } from "node:fs";
22
- import { homedir, tmpdir } from "node:os";
23
- import { join } from "node:path";
24
- import { getStateDir } from "../store.js";
25
- import type { StoredCheckpoint, SessionState } from "../store.js";
26
- import { normalizeSessionId } from "../store.js";
27
-
28
- const SCHEMA_VERSION = 2;
29
-
30
- /** Encode a float vector as a little-endian Float32 BLOB for cosine scanning. */
31
- function encodeEmbedding(v: number[]): Buffer {
32
- const buf = Buffer.allocUnsafe(v.length * 4);
33
- for (let i = 0; i < v.length; i++) buf.writeFloatLE(v[i] ?? 0, i * 4);
34
- return buf;
35
- }
36
- /** Decode a Float32 BLOB back to a number[]. node:sqlite returns BLOBs as
37
- * Uint8Array, so decode via DataView (Buffer is a Uint8Array subclass — both
38
- * work). */
39
- function decodeEmbedding(buf: Uint8Array | null | undefined): number[] {
40
- if (!buf || buf.length === 0) return [];
41
- const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
42
- const n = buf.length / 4;
43
- const out = new Array<number>(n);
44
- for (let i = 0; i < n; i++) out[i] = dv.getFloat32(i * 4, true);
45
- return out;
46
- }
47
-
48
- function jsonText(v: unknown): string {
49
- return JSON.stringify(v ?? []);
50
- }
51
-
52
- // In-process cache so the same stateDir reuses one connection (and so a fresh
53
- // VectorStore over the same dir shares the open DB). Cross-process durability
54
- // comes from reopening the same file path — proven by the integration test.
55
- const cache = new Map<string, DatabaseSync>();
56
-
57
- /** Open (or reuse) the SQLite store for a state dir. */
58
- export function openStore(stateDir: string = getStateDir()): DatabaseSync {
59
- const existing = cache.get(stateDir);
60
- if (existing) {
61
- // A closed handle in the cache (e.g. a test calling db.close() directly
62
- // instead of closeStore) would surface as "database is not open" on the
63
- // next reuse. Detect and evict so callers never see a dead handle.
64
- try {
65
- existing.prepare("SELECT 1");
66
- return existing;
67
- } catch {
68
- cache.delete(stateDir);
69
- }
70
- }
71
-
72
- if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true });
73
- const db = new DatabaseSync(join(stateDir, "sqlite.db"));
74
- db.exec("PRAGMA journal_mode = WAL");
75
- db.exec("PRAGMA foreign_keys = ON");
76
- initSchema(db);
77
- cache.set(stateDir, db);
78
- return db;
79
- }
80
-
81
- // ---------------------------------------------------------------------------
82
- // Global machine-wide index (Phase 5b): a single SQLite DB, separate from every
83
- // per-repo store, that aggregates one row per repo this machine has run on. The
84
- // multi-repo dashboard (Summary / All-repos tabs) reads it so ONE dashboard can
85
- // show every repo's checkpoints, tokens saved, and active model — instead of a
86
- // per-repo dashboard that only ever sees the repo it was launched from.
87
- //
88
- // Written by every pi process on repo-switch (bindRepo) + model capture; read by
89
- // the dashboard server. Concurrency across 10+ pi processes is handled by WAL +
90
- // infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
91
- // ---------------------------------------------------------------------------
92
-
93
- /** Resolve the machine-wide index directory (env-overridable). */
94
- export function getIndexDir(): string {
95
- const override = process.env.MEGACOMPACT_INDEX_DIR;
96
- if (override && override.trim() !== "") return override;
97
- // homedir() can throw in exotic sandboxes; fall back to tmpdir.
98
- try {
99
- return join(homedir(), ".mega-compact-index");
100
- } catch {
101
- return join(tmpdir(), ".mega-compact-index");
102
- }
103
- }
104
-
105
- let indexCache: DatabaseSync | undefined;
106
- let indexCacheDir: string | undefined;
107
-
108
- /** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
109
- export function openIndexStore(indexDir: string = getIndexDir()): DatabaseSync {
110
- if (indexCache && indexCacheDir === indexDir) return indexCache;
111
- if (!existsSync(indexDir)) mkdirSync(indexDir, { recursive: true });
112
- const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
113
- iddb.exec("PRAGMA journal_mode = WAL");
114
- iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
115
- iddb.exec(`
116
- CREATE TABLE IF NOT EXISTS repo_registry (
117
- repo_root TEXT PRIMARY KEY,
118
- display_name TEXT,
119
- state_dir TEXT NOT NULL,
120
- first_seen INTEGER,
121
- last_seen INTEGER,
122
- last_compacted_at INTEGER,
123
- checkpoint_count INTEGER DEFAULT 0,
124
- tokens_saved INTEGER DEFAULT 0,
125
- compressed_original_bytes INTEGER DEFAULT 0,
126
- provider TEXT,
127
- provider_name TEXT,
128
- model_name TEXT,
129
- input_rate REAL,
130
- output_rate REAL,
131
- model_captured_at INTEGER
132
- );
133
- CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
134
- -- S18: machine-wide injected-set. A foreign checkpoint injected in repo A is
135
- -- recorded here so repo B's recall never re-injects it. Keyed by checkpoint
136
- -- + session (a checkpoint may be injected once per session); repo_id is the
137
- -- source repo (the foreign repo's stateDir) for tracking/source labels.
138
- -- PRAMETERIZED queries (PREVENT-002); local node:sqlite (PREVENT-PI-004).
139
- CREATE TABLE IF NOT EXISTS injected_global (
140
- checkpoint_id TEXT NOT NULL,
141
- repo_id TEXT NOT NULL,
142
- session_id TEXT NOT NULL,
143
- injected_at INTEGER NOT NULL,
144
- PRIMARY KEY (checkpoint_id, session_id)
145
- );
146
- CREATE INDEX IF NOT EXISTS idx_injected_global_cid ON injected_global(checkpoint_id);
147
- `);
148
- indexCache = iddb;
149
- indexCacheDir = indexDir;
150
- return iddb;
151
- }
152
-
153
- /** One row of the global repo registry (multi-repo dashboard source). */
154
- export interface RepoRegistryRow {
155
- repoRoot: string;
156
- displayName: string;
157
- stateDir: string;
158
- firstSeen: number;
159
- lastSeen: number;
160
- lastCompactedAt: number | null;
161
- checkpointCount: number;
162
- tokensSaved: number;
163
- compressedOriginalBytes: number;
164
- provider: string | null;
165
- providerName: string | null;
166
- modelName: string | null;
167
- inputRate: number | null;
168
- outputRate: number | null;
169
- modelCapturedAt: number | null;
170
- }
171
-
172
- /**
173
- * Upsert a repo's aggregate stats into the global index. Called on repo-switch
174
- * (infrequent). Preserves first_seen + the model columns on update (model is
175
- * written separately by recordRepoModel so we never clobber it here with nulls).
176
- */
177
- export function upsertRepoRegistry(
178
- row: {
179
- repoRoot: string;
180
- displayName: string;
181
- stateDir: string;
182
- checkpointCount: number;
183
- tokensSaved: number;
184
- compressedOriginalBytes: number;
185
- lastCompactedAt?: number | null;
186
- // The fields below are optional passthroughs so test fixtures and the
187
- // /api/repos active-window filter can seed them directly. They're also
188
- // written by other paths (recordRepoModel, registry refresh) — passing
189
- // them here is harmless because the ON CONFLICT clause keeps first_seen
190
- // and the model columns from being clobbered.
191
- firstSeen?: number;
192
- lastSeen?: number;
193
- provider?: string | null;
194
- providerName?: string | null;
195
- modelName?: string | null;
196
- inputRate?: number | null;
197
- outputRate?: number | null;
198
- modelCapturedAt?: number | null;
199
- },
200
- indexDir: string = getIndexDir(),
201
- ): void {
202
- const db = openIndexStore(indexDir);
203
- const now = Date.now();
204
- db.prepare(
205
- `INSERT INTO repo_registry
206
- (repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
207
- checkpoint_count, tokens_saved, compressed_original_bytes,
208
- provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
209
- VALUES (@repo_root, @display_name, @state_dir, @first_seen, @last_seen, @last_compacted_at,
210
- @checkpoint_count, @tokens_saved, @compressed_original_bytes,
211
- @provider, @provider_name, @model_name, @input_rate, @output_rate, @model_captured_at)
212
- ON CONFLICT(repo_root) DO UPDATE SET
213
- display_name = excluded.display_name,
214
- state_dir = excluded.state_dir,
215
- last_seen = COALESCE(excluded.last_seen, @now),
216
- last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
217
- checkpoint_count = excluded.checkpoint_count,
218
- tokens_saved = excluded.tokens_saved,
219
- compressed_original_bytes = excluded.compressed_original_bytes,
220
- provider = COALESCE(excluded.provider, repo_registry.provider),
221
- provider_name = COALESCE(excluded.provider_name, repo_registry.provider_name),
222
- model_name = COALESCE(excluded.model_name, repo_registry.model_name),
223
- input_rate = COALESCE(excluded.input_rate, repo_registry.input_rate),
224
- output_rate = COALESCE(excluded.output_rate, repo_registry.output_rate),
225
- model_captured_at = COALESCE(excluded.model_captured_at, repo_registry.model_captured_at)`,
226
- ).run({
227
- repo_root: row.repoRoot,
228
- display_name: row.displayName,
229
- state_dir: row.stateDir,
230
- now,
231
- first_seen: row.firstSeen ?? null,
232
- last_seen: row.lastSeen ?? null,
233
- last_compacted_at: row.lastCompactedAt ?? null,
234
- checkpoint_count: row.checkpointCount,
235
- tokens_saved: row.tokensSaved,
236
- compressed_original_bytes: row.compressedOriginalBytes,
237
- provider: row.provider ?? null,
238
- provider_name: row.providerName ?? null,
239
- model_name: row.modelName ?? null,
240
- input_rate: row.inputRate ?? null,
241
- output_rate: row.outputRate ?? null,
242
- model_captured_at: row.modelCapturedAt ?? null,
243
- });
244
- }
245
-
246
- /**
247
- * Record the active model/provider for a repo in the global index (denormalized
248
- * so the All-repos table shows model without opening each repo's DB). Upserts a
249
- * bare registry row if the repo isn't registered yet.
250
- */
251
- export function recordRepoModel(
252
- repoRoot: string,
253
- model: {
254
- provider: string;
255
- providerName: string | null;
256
- modelName: string | null;
257
- inputRate: number;
258
- outputRate: number;
259
- stateDir: string;
260
- displayName: string;
261
- },
262
- indexDir: string = getIndexDir(),
263
- ): void {
264
- const db = openIndexStore(indexDir);
265
- const now = Date.now();
266
- db.prepare(
267
- `INSERT INTO repo_registry
268
- (repo_root, display_name, state_dir, first_seen, last_seen,
269
- provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
270
- VALUES (@repo_root, @display_name, @state_dir, @now, @now,
271
- @provider, @provider_name, @model_name, @input_rate, @output_rate, @now)
272
- ON CONFLICT(repo_root) DO UPDATE SET
273
- last_seen = excluded.last_seen,
274
- provider = excluded.provider,
275
- provider_name = excluded.provider_name,
276
- model_name = excluded.model_name,
277
- input_rate = excluded.input_rate,
278
- output_rate = excluded.output_rate,
279
- model_captured_at = excluded.model_captured_at`,
280
- ).run({
281
- repo_root: repoRoot,
282
- display_name: model.displayName,
283
- state_dir: model.stateDir,
284
- now,
285
- provider: model.provider,
286
- provider_name: model.providerName,
287
- model_name: model.modelName,
288
- input_rate: model.inputRate,
289
- output_rate: model.outputRate,
290
- });
291
- }
292
-
293
- function mapRegistryRow(row: any): RepoRegistryRow {
294
- return {
295
- repoRoot: row.repo_root,
296
- displayName: row.display_name ?? "",
297
- stateDir: row.state_dir,
298
- firstSeen: row.first_seen ?? 0,
299
- lastSeen: row.last_seen ?? 0,
300
- lastCompactedAt: row.last_compacted_at ?? null,
301
- checkpointCount: row.checkpoint_count ?? 0,
302
- tokensSaved: row.tokens_saved ?? 0,
303
- compressedOriginalBytes: row.compressed_original_bytes ?? 0,
304
- provider: row.provider ?? null,
305
- providerName: row.provider_name ?? null,
306
- modelName: row.model_name ?? null,
307
- inputRate: row.input_rate ?? null,
308
- outputRate: row.output_rate ?? null,
309
- modelCapturedAt: row.model_captured_at ?? null,
310
- };
311
- }
312
-
313
- /** All registered repos, most-recently-seen first. */
314
- export function listRepoRegistry(indexDir: string = getIndexDir()): RepoRegistryRow[] {
315
- const db = openIndexStore(indexDir);
316
- const rows = db.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC").all() as any[];
317
- return rows.map(mapRegistryRow);
318
- }
319
-
320
- /** A single repo's registry row, or undefined. */
321
- export function getRepoRegistry(repoRoot: string, indexDir: string = getIndexDir()): RepoRegistryRow | undefined {
322
- const db = openIndexStore(indexDir);
323
- const row = db.prepare("SELECT * FROM repo_registry WHERE repo_root = ?").get(repoRoot) as any;
324
- return row ? mapRegistryRow(row) : undefined;
325
- }
326
-
327
- /** Close the cached index connection (test teardown only). */
328
- export function closeIndexStore(): void {
329
- if (indexCache) {
330
- indexCache.close();
331
- indexCache = undefined;
332
- indexCacheDir = undefined;
333
- }
334
- }
335
-
336
- // ---------------------------------------------------------------------------
337
- // S18: machine-wide injected-set (cross-repo dedup markers)
338
- //
339
- // A foreign checkpoint injected in repo A is recorded here so repo B's recall
340
- // never re-injects it (a stronger, machine-wide version of the per-session
341
- // injected-set in the local store). Keyed by (checkpoint_id, session_id); the
342
- // session_id here is the RECEIVING session, so the same foreign checkpoint can
343
- // be injected into different sessions but never twice into the same one.
344
- // PRAMETERIZED queries (PREVENT-002); local node:sqlite + WAL (PREVENT-PI-004),
345
- // multi-process safe.
346
- // ---------------------------------------------------------------------------
347
-
348
- /** Record that a (foreign) checkpoint was injected into `sessionId`. Idempotent. */
349
- export function markInjectedGlobal(
350
- checkpointId: string,
351
- repoId: string,
352
- sessionId: string,
353
- indexDir: string = getIndexDir(),
354
- ): void {
355
- const db = openIndexStore(indexDir);
356
- db.prepare(
357
- "INSERT OR IGNORE INTO injected_global (checkpoint_id, repo_id, session_id, injected_at) VALUES ($cid, $rid, $sid, $ts)",
358
- ).run({ $cid: checkpointId, $rid: repoId, $sid: sessionId, $ts: Date.now() });
359
- }
360
-
361
- /** True when a checkpoint was already injected into `sessionId` (machine-wide). */
362
- export function wasInjectedGlobal(
363
- checkpointId: string,
364
- sessionId: string,
365
- indexDir: string = getIndexDir(),
366
- ): boolean {
367
- const db = openIndexStore(indexDir);
368
- const row = db.prepare(
369
- "SELECT 1 FROM injected_global WHERE checkpoint_id = $cid AND session_id = $sid LIMIT 1",
370
- ).get({ $cid: checkpointId, $sid: sessionId }) as { "1": number } | undefined;
371
- return row !== undefined;
372
- }
373
-
374
- /** Count of cross-repo injections recorded (for /mega-status stats). */
375
- export function countInjectedGlobal(indexDir: string = getIndexDir()): number {
376
- const db = openIndexStore(indexDir);
377
- const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get() as { n: number } | undefined;
378
- return row?.n ?? 0;
379
- }
380
-
381
- function initSchema(db: DatabaseSync): void {
382
- db.exec(`
383
- CREATE TABLE IF NOT EXISTS context_chunks (
384
- id TEXT NOT NULL,
385
- session_id TEXT NOT NULL,
386
- region_hash TEXT,
387
- content_hash TEXT,
388
- content_hash2 TEXT,
389
- content_hash_version INTEGER,
390
- normalized_text TEXT,
391
- summary TEXT,
392
- topic_summary TEXT,
393
- summary_hash TEXT,
394
- key_decisions TEXT, -- JSON array
395
- next_steps TEXT, -- JSON array
396
- files_modified TEXT, -- JSON array
397
- embedding_blob BLOB, -- float32 vector
398
- token_estimate INTEGER,
399
- original_token_estimate INTEGER, -- dropped region size (tokens saved = orig − stored)
400
- timestamp INTEGER,
401
- dedup_status TEXT DEFAULT 'active',
402
- compressed_original BLOB -- optional DR copy
403
- );
404
- -- Primary key is (session_id, id): checkpoint ids are unique per session
405
- -- (chkpt_001 per session), not globally, so a bare id PK would collide
406
- -- across sessions on the nextCheckpointId sequence.
407
- CREATE UNIQUE INDEX IF NOT EXISTS idx_chunks_pk
408
- ON context_chunks(session_id, id);
409
- CREATE INDEX IF NOT EXISTS idx_chunks_session ON context_chunks(session_id);
410
- CREATE INDEX IF NOT EXISTS idx_chunks_region ON context_chunks(region_hash);
411
- CREATE INDEX IF NOT EXISTS idx_chunks_content ON context_chunks(content_hash);
412
- -- Partial UNIQUE (QA #1): null content_hash rows never violate the constraint;
413
- -- ON CONFLICT DO NOTHING makes backfill + L0 inserts safe.
414
- CREATE UNIQUE INDEX IF NOT EXISTS idx_content_hash
415
- ON context_chunks(session_id, content_hash) WHERE content_hash IS NOT NULL;
416
-
417
- -- Sprint 11: MinHash signature + LSH bucket tables for L1 near-dup dedup.
418
- CREATE TABLE IF NOT EXISTS minhash_signatures (
419
- chunk_id TEXT NOT NULL,
420
- session_id TEXT NOT NULL,
421
- signature_version INTEGER NOT NULL,
422
- signatures TEXT NOT NULL, -- JSON array of 256 uint32
423
- PRIMARY KEY (chunk_id, signature_version)
424
- );
425
- CREATE INDEX IF NOT EXISTS idx_minhash_session ON minhash_signatures(session_id);
426
-
427
- CREATE TABLE IF NOT EXISTS dedup_lsh_buckets (
428
- bucket_key TEXT NOT NULL,
429
- chunk_id TEXT NOT NULL,
430
- session_id TEXT NOT NULL,
431
- signature_version INTEGER NOT NULL,
432
- PRIMARY KEY (bucket_key, chunk_id)
433
- );
434
- CREATE INDEX IF NOT EXISTS idx_lsh_bucket ON dedup_lsh_buckets(bucket_key, session_id);
435
-
436
- CREATE TABLE IF NOT EXISTS session_state (
437
- session_id TEXT PRIMARY KEY,
438
- injected_checkpoint_ids TEXT, -- JSON array
439
- stored_region_hashes TEXT -- JSON array
440
- );
441
-
442
- CREATE TABLE IF NOT EXISTS meta (
443
- key TEXT PRIMARY KEY,
444
- value TEXT
445
- );
446
-
447
- -- Sprint 13 (RAPTOR): hierarchical summary tree nodes. children are a JSON
448
- -- array of child node ids (or raw leaf ids at the bottom); embedding_blob
449
- -- is the node centroid. Additive; retrieval ignores this table until
450
- -- Sprint 14 promotes RAPTOR out of shadow mode.
451
- CREATE TABLE IF NOT EXISTS raptor_nodes (
452
- id TEXT NOT NULL,
453
- session_id TEXT NOT NULL,
454
- level INTEGER NOT NULL,
455
- parent_id TEXT,
456
- children TEXT, -- JSON array of child ids
457
- summary TEXT,
458
- embedding_blob BLOB, -- float32 centroid
459
- quality_marker TEXT DEFAULT 'low',
460
- token_estimate INTEGER,
461
- built_at INTEGER, -- S25: epoch ms when the tree was built (freshness guard)
462
- PRIMARY KEY (session_id, id)
463
- );
464
- CREATE INDEX IF NOT EXISTS idx_raptor_session ON raptor_nodes(session_id);
465
- CREATE INDEX IF NOT EXISTS idx_raptor_parent ON raptor_nodes(parent_id);
466
-
467
- -- Foundation for future features (resume sessions, daily log, lessons
468
- -- learned). Scaffolded now so all store data lives in SQLite from day one;
469
- -- population is minimal (touchSession / logDaily on compact) and the full
470
- -- UI/recall for these lands in later sprints.
471
-
472
- -- Per-session registry (resume + per-repo session history).
473
- CREATE TABLE IF NOT EXISTS sessions (
474
- session_id TEXT PRIMARY KEY,
475
- repo TEXT,
476
- started_at INTEGER,
477
- ended_at INTEGER,
478
- last_compacted_at INTEGER,
479
- status TEXT DEFAULT 'active'
480
- );
481
-
482
- -- Append-only daily activity log (the "daily log" feature seed).
483
- CREATE TABLE IF NOT EXISTS daily_log (
484
- id INTEGER PRIMARY KEY AUTOINCREMENT,
485
- day TEXT NOT NULL, -- YYYY-MM-DD
486
- session_id TEXT,
487
- event TEXT, -- e.g. 'compact'
488
- detail TEXT,
489
- tokens_saved INTEGER DEFAULT 0,
490
- ts INTEGER
491
- );
492
- CREATE INDEX IF NOT EXISTS idx_daily_log_day ON daily_log(day);
493
-
494
- -- Active model/provider for cost estimation + the future multi-repo
495
- -- dashboard (Phase 5b). One row per (repo, model change); latest wins.
496
- CREATE TABLE IF NOT EXISTS model_snapshots (
497
- id INTEGER PRIMARY KEY AUTOINCREMENT,
498
- repo_root TEXT NOT NULL,
499
- provider TEXT NOT NULL,
500
- provider_name TEXT,
501
- model_id TEXT NOT NULL,
502
- model_name TEXT,
503
- input_rate REAL, -- USD per input token (Model.cost.input)
504
- output_rate REAL, -- USD per output token (Model.cost.output)
505
- context_window INTEGER,
506
- max_tokens INTEGER,
507
- reasoning INTEGER DEFAULT 0,
508
- captured_at INTEGER
509
- );
510
- CREATE INDEX IF NOT EXISTS idx_model_repo ON model_snapshots(repo_root);
511
-
512
- -- Lessons learned (future recall/browse feature seed).
513
- CREATE TABLE IF NOT EXISTS lessons (
514
- id INTEGER PRIMARY KEY AUTOINCREMENT,
515
- session_id TEXT,
516
- repo TEXT,
517
- lesson TEXT,
518
- ts INTEGER
519
- );
520
-
521
- -- Durable "save to memory" store (taken over from memory extensions).
522
- -- One row per saved memory; scoped by repo so memory travels with the
523
- -- clone. All params are parameterized (PREVENT-002).
524
- CREATE TABLE IF NOT EXISTS memories (
525
- id INTEGER PRIMARY KEY AUTOINCREMENT,
526
- repo TEXT,
527
- kind TEXT DEFAULT 'note', -- note | fact | decision | preference
528
- content TEXT NOT NULL,
529
- tags TEXT, -- JSON array of strings
530
- created_at INTEGER,
531
- last_recalled_at INTEGER,
532
- -- S20 memory-RAG extension (auto-review add/replace/remove ops).
533
- category TEXT, -- typed bucket, e.g. decision | fact | preference
534
- target TEXT, -- optional subject/scope this memory targets
535
- last_referenced INTEGER, -- last time memory was referenced by recall (epoch s)
536
- source_turn INTEGER -- conversation turn that produced this memory
537
- );
538
- CREATE INDEX IF NOT EXISTS idx_memories_repo ON memories(repo);
539
-
540
- -- FTS5 trigram virtual table (Sprint 9+ pg_trgm-equivalent verification).
541
- CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_trgm USING fts5(
542
- id UNINDEXED,
543
- normalized_text,
544
- tokenize='trigram'
545
- );
546
-
547
- -- S27: durable raw-transcript mirror (MEGACOMPACT_DB_MIRROR). Appended
548
- -- RAW message bytes per session so a compacted window can be rehydrated
549
- -- from the local store instead of the pi runtime transcript (which is
550
- -- trimmed). PK is (content_hash, session_id) — NOT content_hash alone —
551
- -- so identical content in different sessions never collides. Additive:
552
- -- CREATE TABLE IF NOT EXISTS leaves existing DBs untouched on open until
553
- -- the S27 mirror flag is flipped on. All queries parameterized (PREVENT-002).
554
- CREATE TABLE IF NOT EXISTS raw_transcript (
555
- content_hash TEXT NOT NULL,
556
- session_id TEXT NOT NULL,
557
- seq INTEGER NOT NULL,
558
- role TEXT NOT NULL,
559
- content_bytes TEXT NOT NULL,
560
- tool_name TEXT,
561
- message_timestamp INTEGER, -- ORIGINAL msg ts at append, NOT served
562
- checkpoint_epoch TEXT NOT NULL,
563
- PRIMARY KEY (content_hash, session_id)
564
- );
565
- CREATE INDEX IF NOT EXISTS idx_rt_session_seq ON raw_transcript(session_id, seq);
566
- CREATE INDEX IF NOT EXISTS idx_rt_epoch ON raw_transcript(checkpoint_epoch);
567
-
568
- -- S27: checkpoint-epoch registry. One row per compaction epoch; the
569
- -- summary_message_text is the verbatim system message that replaced the
570
- -- trimmed prefix. Informational bookkeeping (the raw_transcript rows are
571
- -- authoritative); refresh-safe via ON CONFLICT(epoch_id) DO UPDATE.
572
- CREATE TABLE IF NOT EXISTS checkpoint_epochs (
573
- epoch_id TEXT PRIMARY KEY,
574
- session_id TEXT NOT NULL,
575
- started_seq INTEGER NOT NULL,
576
- committed_seq INTEGER NOT NULL,
577
- summary_message_text TEXT NOT NULL,
578
- cut_index INTEGER NOT NULL,
579
- checkpoint_id TEXT NOT NULL,
580
- created_at INTEGER NOT NULL
581
- );
582
- CREATE INDEX IF NOT EXISTS idx_epoch_session ON checkpoint_epochs(session_id, created_at DESC);
583
-
584
- -- S27 Task 6: dedup_mirror for space-efficient deduplicated storage.
585
- -- Each unique content_hash stores its bytes ONCE; raw_transcript rows
586
- -- reference this table via content_ref instead of storing duplicate content_bytes inline.
587
- CREATE TABLE IF NOT EXISTS dedup_mirror (
588
- content_hash TEXT PRIMARY KEY,
589
- content_bytes TEXT NOT NULL,
590
- ref_count INTEGER NOT NULL DEFAULT 1,
591
- first_seen_seq INTEGER NOT NULL,
592
- created_at INTEGER NOT NULL
593
- );
594
- `);
595
- // Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
596
- // pre-existing table, so new columns added to context_chunks after a store was
597
- // first created (e.g. original_token_estimate in v0.4.2) must be ALTERed in for
598
- // databases created by an older version — otherwise repoStats()/upsert crash
599
- // with "no such column" and the extension fails to load. Additive only.
600
- ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
601
- // S27 Task 6: content_ref column in raw_transcript for dedup_mirror references.
602
- ensureColumn(db, "raw_transcript", "content_ref", "TEXT");
603
- // S20 memory-RAG extension: additive columns for auto-review ops. Idempotent —
604
- // only alters DBs created by an older version that lack these columns.
605
- ensureColumn(db, "memories", "category", "TEXT");
606
- ensureColumn(db, "memories", "target", "TEXT");
607
- ensureColumn(db, "memories", "last_referenced", "INTEGER");
608
- ensureColumn(db, "memories", "source_turn", "INTEGER");
609
- // S25: RAPTOR freshness-guard timestamp. Additive; old DBs have NULL → 0 →
610
- // treated as stale → flat fallback (safe).
611
- ensureColumn(db, "raptor_nodes", "built_at", "INTEGER");
612
- const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get() as
613
- | { value: string }
614
- | undefined;
615
- if (!v) {
616
- db.prepare("INSERT INTO meta(key, value) VALUES(?, ?)").run("schema_version", String(SCHEMA_VERSION));
617
- }
618
- }
619
-
620
- /**
621
- * Add `column` (with `decl`, e.g. "INTEGER") to `table` if it does not already
622
- * exist. Idempotent: checks PRAGMA table_info first, so it is safe to run on
623
- * every open. Table/column/decl are code-controlled constants (never user
624
- * input), so the unavoidable identifier interpolation here does not violate
625
- * PREVENT-002 (no external data reaches this SQL).
626
- */
627
- function ensureColumn(db: DatabaseSync, table: string, column: string, decl: string): void {
628
- const cols = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
629
- if (cols.some((c) => c.name === column)) return;
630
- db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${decl}`);
631
- }
632
-
633
- /** Read a string-valued meta key (or undefined). Used for cumulative counters. */
634
- export function getMeta(key: string, stateDir: string = getStateDir()): string | undefined {
635
- const db = openStore(stateDir);
636
- const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key) as
637
- | { value: string }
638
- | undefined;
639
- return row?.value;
640
- }
641
-
642
- /**
643
- * Cumulative "tokens saved" — the sum of stored checkpoint token estimates across
644
- * all compactions in this store (one per repo). Persisted in the SQLite `meta`
645
- * table so it survives session restarts and travels with the repo's state dir,
646
- * mirroring how `storageDedupRate` is cumulative. Incremented in VectorStore.add()
647
- * when a new (non-deduped) checkpoint is persisted.
648
- */
649
- export function getTokensSaved(stateDir: string = getStateDir()): number {
650
- const raw = getMeta("tokens_saved", stateDir);
651
- const n = raw == null ? 0 : Number(raw);
652
- return Number.isFinite(n) ? n : 0;
653
- }
654
-
655
- /** Add `delta` (>=0) to the cumulative tokens-saved counter. */
656
- export function addTokensSaved(delta: number, stateDir: string = getStateDir()): void {
657
- if (!(delta > 0)) return;
658
- const db = openStore(stateDir);
659
- db.prepare(
660
- `INSERT INTO meta(key, value) VALUES('tokens_saved', ?)
661
- ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
662
- ).run(String(delta), delta);
663
- }
664
-
665
- /** Cumulative store-wide dedup accounting (Sprint 9+). Persisted in the SQLite
666
- * `meta` table so it survives session restarts and travels with the repo's
667
- * state dir — mirroring `tokens_saved`. Replaces the legacy JSON
668
- * `dedup-stats.json` file (all stats now live in the SQLite store). */
669
- export interface DedupStats {
670
- /** Total add() calls (new checkpoints + deduped collapses). */
671
- attempts: number;
672
- /** add() calls that collapsed onto an existing checkpoint. */
673
- deduped: number;
674
- }
675
-
676
- /** Read a store-wide integer counter from the meta table (0 if absent). */
677
- export function getMetaNumber(key: string, stateDir: string = getStateDir()): number {
678
- const raw = getMeta(key, stateDir);
679
- const n = raw == null ? 0 : Number(raw);
680
- return Number.isFinite(n) ? n : 0;
681
- }
682
-
683
- /** Atomically add `delta` to an integer meta counter. */
684
- function incMeta(key: string, delta: number, stateDir: string = getStateDir()): void {
685
- if (!(delta > 0)) return;
686
- const db = openStore(stateDir);
687
- db.prepare(
688
- `INSERT INTO meta(key, value) VALUES(?, ?)
689
- ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
690
- ).run(key, String(delta), delta);
691
- }
692
-
693
- /** Read the cumulative store-wide dedup counters. */
694
- export function getDedupStats(stateDir: string = getStateDir()): DedupStats {
695
- return {
696
- attempts: getMetaNumber("dedup_attempts", stateDir),
697
- deduped: getMetaNumber("deduped", stateDir),
698
- };
699
- }
700
-
701
- /** Increment the store-wide dedup counters for one add() call. */
702
- export function bumpDedupStats(deduped: boolean, stateDir: string = getStateDir()): void {
703
- incMeta("dedup_attempts", 1, stateDir);
704
- if (deduped) incMeta("deduped", 1, stateDir);
705
- }
706
-
707
- // --- Live dashboard counters (schemaless meta key/value — NO migration) -----
708
- // These reuse the private `incMeta` atomically-incrementing integer counter so
709
- // all cumulative tallies live in the same `meta` table as tokens_saved etc.
710
-
711
- export function incCompactCount(stateDir: string = getStateDir()): void {
712
- incMeta("compact_count", 1, stateDir);
713
- }
714
-
715
- export function getCompactCount(stateDir: string = getStateDir()): number {
716
- return getMetaNumber("compact_count", stateDir);
717
- }
718
-
719
- export function incRecallInjected(n: number, stateDir: string = getStateDir()): void {
720
- if (n > 0) incMeta("recall_injected", n, stateDir);
721
- }
722
-
723
- export function getRecallInjected(stateDir: string = getStateDir()): number {
724
- return getMetaNumber("recall_injected", stateDir);
725
- }
726
-
727
- export function incCacheHitTokens(delta: number, stateDir: string = getStateDir()): void {
728
- if (delta > 0) incMeta("cache_hit_tokens_saved", delta, stateDir);
729
- }
730
-
731
- export function getCacheHitTokensSaved(stateDir: string = getStateDir()): number {
732
- return getMetaNumber("cache_hit_tokens_saved", stateDir);
733
- }
734
-
735
- // --- Future-feature foundation (resume sessions / daily log / lessons) -------
736
- // Scaffolded tables + minimal helpers so all store data lives in SQLite from
737
- // day one. Full UI/recall for these lands in later sprints.
738
-
739
- /** Upsert a `sessions` row (resume + per-repo session history). */
740
- export function touchSession(
741
- sessionId: string,
742
- repo: string | undefined,
743
- stateDir: string = getStateDir(),
744
- ): void {
745
- const db = openStore(stateDir);
746
- const sid = normalizeSessionId(sessionId);
747
- const existing = db
748
- .prepare("SELECT started_at FROM sessions WHERE session_id = ?")
749
- .get(sid) as { started_at: number | null } | undefined;
750
- const now = Math.floor(Date.now() / 1000);
751
- if (!existing) {
752
- db.prepare(
753
- `INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
754
- VALUES(?, ?, ?, ?, 'active')`,
755
- ).run(sid, repo ?? null, now, now);
756
- } else {
757
- db.prepare(
758
- "UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?",
759
- ).run(now, repo ?? null, sid);
760
- }
761
- }
762
-
763
- /** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
764
- export function logDaily(
765
- sessionId: string,
766
- event: string,
767
- detail: string | undefined,
768
- tokensSaved: number,
769
- stateDir: string = getStateDir(),
770
- ): void {
771
- const db = openStore(stateDir);
772
- const day = new Date().toISOString().slice(0, 10);
773
- const now = Math.floor(Date.now() / 1000);
774
- db.prepare(
775
- `INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
776
- VALUES(?, ?, ?, ?, ?, ?)`,
777
- ).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
778
- }
779
-
780
- /** Append a `lessons` entry (future lessons-learned browse/recall). */
781
- export function addLesson(
782
- sessionId: string,
783
- repo: string | undefined,
784
- lesson: string,
785
- stateDir: string = getStateDir(),
786
- ): void {
787
- const db = openStore(stateDir);
788
- const now = Math.floor(Date.now() / 1000);
789
- db.prepare(
790
- `INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`,
791
- ).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
792
- }
793
-
794
- // --- Durable memory (save-to-memory takeover) ---------------------------------
795
- // One SQLite store for user-saved memories, scoped by repo. Mirrors the
796
- // lessons/sessions pattern: all state lives in SQLite from day one.
797
-
798
- // S24 storage hardening: keep each memory row bounded so the durable store can
799
- // never blow a downstream consumer's per-entry buffer (e.g. pi's native
800
- // file-backed memory caps a single entry at ~5k chars). We truncate content at
801
- // MEMORY_MAX_CHARS and evict the least-recently-referenced rows past
802
- // MEMORY_MAX_ROWS per repo via LRU. Both are SQLite-only (PREVENT-PI-004): no
803
- // file-backed memory is written anywhere. Defaults are overridable via env
804
- // (MEGACOMPACT_MEMORY_MAX_CHARS / MEGACOMPACT_MEMORY_MAX_ROWS).
805
- export const MEMORY_MAX_CHARS = 4000;
806
- export const MEMORY_MAX_ROWS = 500;
807
-
808
- /** Read an env override as a positive int, falling back to `fallback`. */
809
- function envInt(name: string, fallback: number): number {
810
- const v = process.env[name];
811
- if (v == null || v === "") return fallback;
812
- const n = Number(v);
813
- return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
814
- }
815
-
816
- /** Effective per-entry char cap (env-overridable, default MEMORY_MAX_CHARS). */
817
- export function memoryMaxChars(): number {
818
- return envInt("MEGACOMPACT_MEMORY_MAX_CHARS", MEMORY_MAX_CHARS);
819
- }
820
-
821
- /** Effective per-repo row cap (env-overridable, default MEMORY_MAX_ROWS). */
822
- export function memoryMaxRows(): number {
823
- return envInt("MEGACOMPACT_MEMORY_MAX_ROWS", MEMORY_MAX_ROWS);
824
- }
825
-
826
- /** Truncate memory content to the per-entry cap, preserving a trailing marker. */
827
- function capMemoryContent(content: string): string {
828
- const cap = memoryMaxChars();
829
- if (content.length <= cap) return content;
830
- return content.slice(0, cap) + "…[truncated]";
831
- }
832
-
833
- /**
834
- * Evict the least-recently-referenced rows for a repo past MEMORY_MAX_ROWS.
835
- * LRU key = COALESCE(last_referenced, last_recalled_at, created_at) so a memory
836
- * that is recalled/referenced survives over a stale one. Best-effort: any error
837
- * is swallowed by the caller. Repo-scoped so one noisy repo can't evict another.
838
- */
839
- function evictMemoryLru(repo: string | null, stateDir: string): void {
840
- const db = openStore(stateDir);
841
- const maxRows = memoryMaxRows();
842
- // SQLite `= NULL` is never true, so the null-repo scope (memories are
843
- // stateDir-scoped when repo is null — the applyMemoryOps path) needs `IS NULL`.
844
- const where = repo == null ? "repo IS NULL" : "repo = ?";
845
- const countRow = repo == null
846
- ? db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get()
847
- : db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get(repo);
848
- const count = (countRow as { n: number }).n;
849
- const over = count - maxRows;
850
- if (over <= 0) return;
851
- // Delete the `over` least-recently-used rows. ORDER BY the LRU key ASC, id ASC
852
- // (id ASC breaks ties deterministically — oldest created first). The `where`
853
- // clause is a code-controlled constant (never user input) → PREVENT-002 OK.
854
- const sql =
855
- `DELETE FROM memories WHERE ${where} AND id IN (
856
- SELECT id FROM memories WHERE ${where}
857
- ORDER BY COALESCE(last_referenced, last_recalled_at, created_at) ASC, id ASC
858
- LIMIT ?
859
- )`;
860
- if (repo == null) db.prepare(sql).run(over);
861
- else db.prepare(sql).run(repo, repo, over);
862
- }
863
-
864
- export interface MemoryRecord {
865
- id: number;
866
- repo: string | null;
867
- kind: string;
868
- content: string;
869
- tags: string[];
870
- createdAt: number;
871
- lastRecalledAt: number | null;
872
- category: string | null;
873
- target: string | null;
874
- lastReferenced: number | null;
875
- sourceTurn: number | null;
876
- }
877
-
878
- /** Save a memory to the current repo's store. Returns the new row id.
879
- * S24 hardening: content is truncated to MEMORY_MAX_CHARS and, once the per-repo
880
- * row count exceeds MEMORY_MAX_ROWS, the least-recently-used rows are evicted
881
- * (LRU) so the store stays bounded. */
882
- export function addMemory(
883
- memory: { kind?: string; content: string; tags?: string[]; category?: string; target?: string; sourceTurn?: number },
884
- repo: string | null,
885
- stateDir: string = getStateDir(),
886
- ): number {
887
- const db = openStore(stateDir);
888
- const now = Math.floor(Date.now() / 1000);
889
- const res = db
890
- .prepare(
891
- `INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at, category, target, source_turn)
892
- VALUES(?, ?, ?, ?, ?, NULL, ?, ?, ?)`,
893
- )
894
- .run(
895
- repo ?? null,
896
- memory.kind ?? "note",
897
- capMemoryContent(memory.content),
898
- JSON.stringify(memory.tags ?? []),
899
- now,
900
- memory.category ?? null,
901
- memory.target ?? null,
902
- memory.sourceTurn ?? null,
903
- );
904
- try {
905
- evictMemoryLru(repo, stateDir);
906
- } catch {
907
- /* non-fatal: eviction must never fail an add */
908
- }
909
- return Number(res.lastInsertRowid);
910
- }
911
-
912
- /** List recent memories for a repo (or all repos when repo is null). */
913
- export function listMemories(repo: string | null, limit = 50, stateDir: string = getStateDir()): MemoryRecord[] {
914
- const db = openStore(stateDir);
915
- const rows = repo
916
- ? db.prepare("SELECT * FROM memories WHERE repo = ? ORDER BY created_at DESC LIMIT ?").all(repo, limit)
917
- : db.prepare("SELECT * FROM memories ORDER BY created_at DESC LIMIT ?").all(limit);
918
- return (rows as any[]).map(mapMemoryRow);
919
- }
920
-
921
- /** Substring search across content + tags. */
922
- export function searchMemories(query: string, repo: string | null = null, limit = 50, stateDir: string = getStateDir()): MemoryRecord[] {
923
- const db = openStore(stateDir);
924
- const like = `%${query}%`;
925
- const rows = repo
926
- ? db.prepare("SELECT * FROM memories WHERE repo = ? AND (content LIKE ? OR tags LIKE ?) ORDER BY created_at DESC LIMIT ?").all(repo, like, like, limit)
927
- : db.prepare("SELECT * FROM memories WHERE content LIKE ? OR tags LIKE ? ORDER BY created_at DESC LIMIT ?").all(like, like, limit);
928
- return (rows as any[]).map(mapMemoryRow);
929
- }
930
-
931
- /** Mark a memory as recalled (updates last_recalled_at). Returns true if found. */
932
- export function recallMemory(id: number, stateDir: string = getStateDir()): boolean {
933
- const db = openStore(stateDir);
934
- const now = Math.floor(Date.now() / 1000);
935
- const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
936
- return res.changes > 0;
937
- }
938
-
939
- /** Mark a memory as referenced (updates last_referenced). Returns true if found. */
940
- export function referenceMemory(id: number, stateDir: string = getStateDir()): boolean {
941
- const db = openStore(stateDir);
942
- const now = Math.floor(Date.now() / 1000);
943
- const res = db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(now, id);
944
- return res.changes > 0;
945
- }
946
-
947
- /** Replace a memory's mutable fields by id. Returns true if a row was updated. */
948
- export function replaceMemory(
949
- id: number,
950
- patch: { kind?: string; content?: string; tags?: string[]; category?: string; target?: string; sourceTurn?: number },
951
- stateDir: string = getStateDir(),
952
- ): boolean {
953
- const db = openStore(stateDir);
954
- const res = db
955
- .prepare(
956
- `UPDATE memories
957
- SET kind = COALESCE(?, kind),
958
- content = COALESCE(?, content),
959
- tags = COALESCE(?, tags),
960
- category = COALESCE(?, category),
961
- target = COALESCE(?, target),
962
- source_turn = COALESCE(?, source_turn)
963
- WHERE id = ?`,
964
- )
965
- .run(
966
- patch.kind ?? null,
967
- patch.content != null ? capMemoryContent(patch.content) : null,
968
- patch.tags ? JSON.stringify(patch.tags) : null,
969
- "category" in patch ? (patch.category ?? null) : null,
970
- "target" in patch ? (patch.target ?? null) : null,
971
- "sourceTurn" in patch ? (patch.sourceTurn ?? null) : null,
972
- id,
973
- );
974
- return res.changes > 0;
975
- }
976
-
977
- /** Remove a memory by id. Returns true if a row was deleted. */
978
- export function removeMemory(id: number, stateDir: string = getStateDir()): boolean {
979
- const db = openStore(stateDir);
980
- const res = db.prepare("DELETE FROM memories WHERE id = ?").run(id);
981
- return res.changes > 0;
982
- }
983
-
984
- /** Look up a single memory by id (or undefined). */
985
- export function getMemory(id: number, stateDir: string = getStateDir()): MemoryRecord | undefined {
986
- const db = openStore(stateDir);
987
- const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
988
- return row ? mapMemoryRow(row) : undefined;
989
- }
990
-
991
- function mapMemoryRow(row: any): MemoryRecord {
992
- return {
993
- id: row.id,
994
- repo: row.repo ?? null,
995
- kind: row.kind ?? "note",
996
- content: row.content ?? "",
997
- tags: row.tags ? JSON.parse(row.tags) : [],
998
- createdAt: row.created_at ?? 0,
999
- lastRecalledAt: row.last_recalled_at ?? null,
1000
- category: row.category ?? null,
1001
- target: row.target ?? null,
1002
- lastReferenced: row.last_referenced ?? null,
1003
- sourceTurn: row.source_turn ?? null,
1004
- };
1005
- }
1006
-
1007
- /**
1008
- * Run `fn` atomically. Uses SAVEPOINT so it nests safely under an outer
1009
- * transaction (unlike `BEGIN`, which SQLite rejects when one is already open).
1010
- * Mirrors better-sqlite3's `db.transaction(fn)` semantics — callers that wrap a
1011
- * batch in withTx (e.g. backfill) can still call helpers that also use withTx.
1012
- */
1013
- export function withTx(db: DatabaseSync, fn: () => void): void {
1014
- db.exec("SAVEPOINT mc_tx");
1015
- try {
1016
- fn();
1017
- db.exec("RELEASE mc_tx");
1018
- } catch (e) {
1019
- db.exec("ROLLBACK TO mc_tx");
1020
- db.exec("RELEASE mc_tx");
1021
- throw e;
1022
- }
1023
- }
1024
-
1025
- /** Map a DB row to the public StoredCheckpoint shape. */
1026
- function rowToCheckpoint(row: any): StoredCheckpoint {
1027
- return {
1028
- checkpointId: row.id,
1029
- sessionId: row.session_id,
1030
- summary: row.summary ?? "",
1031
- topicSummary: row.topic_summary ?? undefined,
1032
- summaryHash: row.summary_hash ?? undefined,
1033
- keyDecisions: row.key_decisions ? JSON.parse(row.key_decisions) : [],
1034
- nextSteps: row.next_steps ? JSON.parse(row.next_steps) : [],
1035
- filesModified: row.files_modified ? JSON.parse(row.files_modified) : [],
1036
- tokenEstimate: row.token_estimate ?? 0,
1037
- originalTokenEstimate: row.original_token_estimate ?? undefined,
1038
- regionHash: row.region_hash ?? "",
1039
- contentHash: row.content_hash ?? undefined,
1040
- contentHash2: row.content_hash2 ?? undefined,
1041
- contentHashVersion: row.content_hash_version ?? undefined,
1042
- normalizedText: row.normalized_text ?? undefined,
1043
- // node:sqlite returns BLOBs as Uint8Array; normalize to Buffer so callers
1044
- // (e.g. decompressSmart → Buffer.toString) behave as under better-sqlite3.
1045
- compressedOriginal: row.compressed_original ? Buffer.from(row.compressed_original) : undefined,
1046
- embedding: decodeEmbedding(row.embedding_blob),
1047
- timestamp: Number(row.timestamp ?? 0),
1048
- dedupStatus: row.dedup_status ?? undefined,
1049
- };
1050
- }
1051
-
1052
- /** Insert or replace a checkpoint (idempotent by id). */
1053
- export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getStateDir()): void {
1054
- const db = openStore(stateDir);
1055
- const sid = normalizeSessionId(cp.sessionId);
1056
- withTx(db, () => {
1057
- db.prepare(
1058
- `INSERT INTO context_chunks
1059
- (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
1060
- normalized_text, summary, topic_summary, summary_hash,
1061
- key_decisions, next_steps, files_modified, embedding_blob,
1062
- token_estimate, original_token_estimate, timestamp, dedup_status, compressed_original)
1063
- VALUES (@id, @sid, @region_hash, @content_hash, @content_hash2, @content_hash_version,
1064
- @normalized_text, @summary, @topic_summary, @summary_hash,
1065
- @key_decisions, @next_steps, @files_modified, @embedding_blob,
1066
- @token_estimate, @original_token_estimate, @timestamp, @dedup_status, @compressed_original)
1067
- ON CONFLICT(session_id, id) DO UPDATE SET
1068
- summary=excluded.summary,
1069
- topic_summary=excluded.topic_summary,
1070
- summary_hash=excluded.summary_hash,
1071
- key_decisions=excluded.key_decisions,
1072
- next_steps=excluded.next_steps,
1073
- files_modified=excluded.files_modified,
1074
- embedding_blob=excluded.embedding_blob,
1075
- token_estimate=excluded.token_estimate,
1076
- original_token_estimate=excluded.original_token_estimate,
1077
- timestamp=excluded.timestamp,
1078
- dedup_status=excluded.dedup_status,
1079
- compressed_original=excluded.compressed_original`,
1080
- ).run({
1081
- "@id": cp.checkpointId,
1082
- "@sid": sid,
1083
- "@region_hash": cp.regionHash ?? null,
1084
- "@content_hash": cp.contentHash ?? null,
1085
- "@content_hash2": cp.contentHash2 ?? null,
1086
- "@content_hash_version": cp.contentHashVersion ?? null,
1087
- "@normalized_text": cp.normalizedText ?? null,
1088
- "@summary": cp.summary ?? "",
1089
- "@topic_summary": cp.topicSummary ?? null,
1090
- "@summary_hash": cp.summaryHash ?? null,
1091
- "@key_decisions": jsonText(cp.keyDecisions),
1092
- "@next_steps": jsonText(cp.nextSteps),
1093
- "@files_modified": jsonText(cp.filesModified),
1094
- "@embedding_blob": encodeEmbedding(cp.embedding ?? []),
1095
- "@token_estimate": cp.tokenEstimate ?? 0,
1096
- "@original_token_estimate": cp.originalTokenEstimate ?? null,
1097
- "@timestamp": cp.timestamp ?? 0,
1098
- "@dedup_status": "active",
1099
- "@compressed_original": cp.compressedOriginal ?? null,
1100
- });
1101
-
1102
- // FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
1103
- // Store normalized_text (the L1 verify key); fall back to summary for rows
1104
- // that predate normalized_text population.
1105
- db.prepare("DELETE FROM context_chunks_trgm WHERE id = ?").run(cp.checkpointId);
1106
- db.prepare(
1107
- "INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)",
1108
- ).run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
1109
- });
1110
- }
1111
-
1112
- // --- Sprint 11: MinHash signatures + LSH buckets --------------------------
1113
-
1114
- /** Persist a checkpoint's MinHash signature (idempotent by chunk_id + version). */
1115
- export function upsertMinhashSignature(
1116
- chunkId: string,
1117
- sessionId: string,
1118
- signatureVersion: number,
1119
- signatures: number[],
1120
- stateDir: string = getStateDir(),
1121
- ): void {
1122
- const db = openStore(stateDir);
1123
- const sid = normalizeSessionId(sessionId);
1124
- db.prepare(
1125
- `INSERT INTO minhash_signatures(chunk_id, session_id, signature_version, signatures)
1126
- VALUES(?, ?, ?, ?)
1127
- ON CONFLICT(chunk_id, signature_version) DO UPDATE SET
1128
- session_id=excluded.session_id, signatures=excluded.signatures`,
1129
- ).run(chunkId, sid, signatureVersion, JSON.stringify(signatures));
1130
- }
1131
-
1132
- /** Persist LSH bucket memberships for a chunk (one row per bucket key). */
1133
- export function insertLshBuckets(
1134
- chunkId: string,
1135
- sessionId: string,
1136
- signatureVersion: number,
1137
- bucketKeys: string[],
1138
- stateDir: string = getStateDir(),
1139
- ): void {
1140
- const db = openStore(stateDir);
1141
- const sid = normalizeSessionId(sessionId);
1142
- const del = db.prepare("DELETE FROM dedup_lsh_buckets WHERE chunk_id = ?");
1143
- const ins = db.prepare(
1144
- "INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)",
1145
- );
1146
- withTx(db, () => {
1147
- del.run(chunkId);
1148
- for (const key of bucketKeys) ins.run(key, chunkId, sid, signatureVersion);
1149
- });
1150
- }
1151
-
1152
- /**
1153
- * Candidate chunk_ids sharing any LSH bucket with `bucketKeys`, scoped to the
1154
- * session, capped at `limit`. Single query (no N loops) — QA #15 amplification
1155
- * guard. Returns DISTINCT chunk_ids excluding `excludeChunkId` (the new row).
1156
- */
1157
- export function lshCandidateChunks(
1158
- bucketKeys: string[],
1159
- sessionId: string,
1160
- excludeChunkId: string,
1161
- stateDir: string = getStateDir(),
1162
- limit = 100,
1163
- ): string[] {
1164
- if (bucketKeys.length === 0) return [];
1165
- const db = openStore(stateDir);
1166
- const sid = normalizeSessionId(sessionId);
1167
- const placeholders = bucketKeys.map(() => "?").join(",");
1168
- const rows = db
1169
- .prepare(
1170
- `SELECT DISTINCT chunk_id FROM dedup_lsh_buckets
1171
- WHERE bucket_key IN (${placeholders}) AND session_id = ? AND chunk_id != ?
1172
- LIMIT ?`,
1173
- )
1174
- .all(...bucketKeys, sid, excludeChunkId, limit) as { chunk_id: string }[];
1175
- return rows.map((r) => r.chunk_id);
1176
- }
1177
-
1178
- /** All checkpoints for a session, sorted by id. */
1179
- export function listCheckpoints(sessionId: string, stateDir: string = getStateDir()): StoredCheckpoint[] {
1180
- const db = openStore(stateDir);
1181
- const sid = normalizeSessionId(sessionId);
1182
- const rows = db
1183
- .prepare("SELECT * FROM context_chunks WHERE session_id = ? ORDER BY id ASC")
1184
- .all(sid) as any[];
1185
- return rows.map(rowToCheckpoint);
1186
- }
1187
-
1188
- /** S25: the newest checkpoint timestamp for a session, or 0 when none. Used by
1189
- * the RAPTOR freshness guard to reject a tree older than the live checkpoints. */
1190
- export function maxCheckpointTimestamp(sessionId: string, stateDir: string = getStateDir()): number {
1191
- const db = openStore(stateDir);
1192
- const row = db
1193
- .prepare("SELECT MAX(timestamp) AS mx FROM context_chunks WHERE session_id = ?")
1194
- .get(normalizeSessionId(sessionId)) as { mx: number | null } | undefined;
1195
- return Number(row?.mx ?? 0);
1196
- }
1197
-
1198
- /** Next sequential checkpoint id (chkpt_001 …) for a session. */
1199
- export function nextCheckpointId(sessionId: string, stateDir: string = getStateDir()): string {
1200
- const db = openStore(stateDir);
1201
- const sid = normalizeSessionId(sessionId);
1202
- const row = db
1203
- .prepare("SELECT MAX(CAST(SUBSTR(id, 7) AS INTEGER)) AS n FROM context_chunks WHERE session_id = ?")
1204
- .get(sid) as { n: number | null };
1205
- const next = (row.n ?? 0) + 1;
1206
- return `chkpt_${String(next).padStart(3, "0")}`;
1207
- }
1208
-
1209
- /** True if a checkpoint id already exists for a session. */
1210
- export function hasCheckpoint(sessionId: string, checkpointId: string, stateDir: string = getStateDir()): boolean {
1211
- const db = openStore(stateDir);
1212
- const row = db
1213
- .prepare("SELECT 1 FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
1214
- .get(normalizeSessionId(sessionId), checkpointId);
1215
- return row !== undefined;
1216
- }
1217
-
1218
- /** Fetch a single checkpoint by (session, id), or undefined if absent. */
1219
- export function getCheckpoint(
1220
- sessionId: string,
1221
- checkpointId: string,
1222
- stateDir: string = getStateDir(),
1223
- ): StoredCheckpoint | undefined {
1224
- const db = openStore(stateDir);
1225
- const row = db
1226
- .prepare("SELECT * FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
1227
- .get(normalizeSessionId(sessionId), checkpointId) as any;
1228
- return row ? rowToCheckpoint(row) : undefined;
1229
- }
1230
-
1231
- /** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
1232
- export function setDedupStatus(
1233
- checkpointId: string,
1234
- sessionId: string,
1235
- status: string,
1236
- stateDir: string = getStateDir(),
1237
- ): void {
1238
- const db = openStore(stateDir);
1239
- db.prepare(
1240
- "UPDATE context_chunks SET dedup_status = ? WHERE id = ? AND session_id = ?",
1241
- ).run(status, checkpointId, normalizeSessionId(sessionId));
1242
- }
1243
-
1244
- // --- Session state (injection tracking) ------------------------------------
1245
-
1246
- function loadSessionStateRow(sid: string, db: DatabaseSync): SessionState {
1247
- const row = db.prepare("SELECT * FROM session_state WHERE session_id = ?").get(sid) as any;
1248
- if (!row) {
1249
- return { injectedCheckpointIds: [], storedRegionHashes: [] };
1250
- }
1251
- return {
1252
- injectedCheckpointIds: row.injected_checkpoint_ids ? JSON.parse(row.injected_checkpoint_ids) : [],
1253
- storedRegionHashes: row.stored_region_hashes ? JSON.parse(row.stored_region_hashes) : [],
1254
- };
1255
- }
1256
-
1257
- export function loadSessionState(sessionId: string, stateDir: string = getStateDir()): SessionState {
1258
- return loadSessionStateRow(normalizeSessionId(sessionId), openStore(stateDir));
1259
- }
1260
-
1261
- export function saveSessionState(sessionId: string, state: SessionState, stateDir: string = getStateDir()): void {
1262
- const db = openStore(stateDir);
1263
- const sid = normalizeSessionId(sessionId);
1264
- db.prepare(
1265
- `INSERT INTO session_state(session_id, injected_checkpoint_ids, stored_region_hashes)
1266
- VALUES(@sid, @inj, @reg)
1267
- ON CONFLICT(session_id) DO UPDATE SET
1268
- injected_checkpoint_ids=excluded.injected_checkpoint_ids,
1269
- stored_region_hashes=excluded.stored_region_hashes`,
1270
- ).run({
1271
- sid,
1272
- inj: jsonText(state.injectedCheckpointIds),
1273
- reg: jsonText(state.storedRegionHashes),
1274
- });
1275
- }
1276
-
1277
- // --- Stats -----------------------------------------------------------------
1278
-
1279
- export interface StoreStats {
1280
- checkpointCount: number;
1281
- totalTokenEstimate: number;
1282
- lastCheckpointId: string | undefined;
1283
- lastSummary: string | undefined;
1284
- }
1285
-
1286
- export function storeStats(sessionId: string, stateDir: string = getStateDir()): StoreStats {
1287
- const db = openStore(stateDir);
1288
- const sid = normalizeSessionId(sessionId);
1289
- const row = db
1290
- .prepare(
1291
- `SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
1292
- MAX(id) AS lastId
1293
- FROM context_chunks WHERE session_id = ?`,
1294
- )
1295
- .get(sid) as { c: number; tok: number; lastId: string | null };
1296
- let lastSummary: string | undefined;
1297
- if (row.lastId) {
1298
- const s = db.prepare("SELECT summary FROM context_chunks WHERE id = ?").get(row.lastId) as
1299
- | { summary: string }
1300
- | undefined;
1301
- lastSummary = s?.summary;
1302
- }
1303
- return {
1304
- checkpointCount: row.c,
1305
- totalTokenEstimate: row.tok,
1306
- lastCheckpointId: row.lastId ?? undefined,
1307
- lastSummary,
1308
- };
1309
- }
1310
-
1311
- /** Repo-wide stats — aggregates every session in this store (one per repo).
1312
- * Backed by the SQLite `meta` cumulative counters (`tokens_saved`,
1313
- * `dedup_attempts`, `deduped`) plus a SUM over all `context_chunks`. This is the
1314
- * cumulative, resumable, cross-device view the dashboard surfaces as "Repo …". */
1315
- export interface RepoStats {
1316
- /** Total checkpoints across all sessions (excludes SemDeDup-removed rows). */
1317
- checkpointCount: number;
1318
- /** Sum of all stored checkpoint token estimates (repo-wide). */
1319
- totalTokenEstimate: number;
1320
- /** Total active sessions with at least one checkpoint. */
1321
- sessionCount: number;
1322
- /** Cumulative stored-summary tokens saved (Σ stored summaries). */
1323
- tokensSaved: number;
1324
- /** Sum of original dropped-region token estimates (repo-wide). */
1325
- originalTokens: number;
1326
- /** Cumulative dedup add() attempts (store-wide). */
1327
- dedupAttempts: number;
1328
- /** Cumulative deduped collapses (store-wide). */
1329
- dedupCollapsed: number;
1330
- /** Storage dedup rate (deduped / attempts), 0..1. */
1331
- storageDedupRate: number;
1332
- }
1333
-
1334
- /**
1335
- * Data-safety invariant metrics (Phase 0 — trust foundation). Proves that every
1336
- * compacted region is still recoverable: we retain a compressed_original blob for
1337
- * each checkpoint and permanently delete nothing. "removed" rows are SemDeDup
1338
- * duplicates whose ORIGINAL is still retained on the surviving checkpoint — they
1339
- * are not data loss, so they are reported separately, not as deletions.
1340
- */
1341
- export interface DataInvariantStats {
1342
- /** Checkpoints with a recoverable compressed_original blob. */
1343
- regionsRetained: number;
1344
- /** Total bytes of compressed_original retained (recoverable verbatim). */
1345
- compressedOriginalBytes: number;
1346
- /** Checkpoints missing a compressed_original blob (pre-blob or direct add). */
1347
- regionsWithoutBlob: number;
1348
- /** Bytes permanently deleted by the extension. ALWAYS 0 — the invariant. */
1349
- bytesPermanentlyDeleted: number;
1350
- /** Duplicate rows collapsed by dedup (original retained on the survivor). */
1351
- duplicatesCollapsed: number;
1352
- }
1353
-
1354
- export function dataInvariantStats(stateDir: string = getStateDir()): DataInvariantStats {
1355
- const db = openStore(stateDir);
1356
- const row = db
1357
- .prepare(
1358
- `SELECT
1359
- COUNT(compressed_original) AS withBlob,
1360
- COALESCE(SUM(LENGTH(compressed_original)),0) AS blobBytes,
1361
- SUM(CASE WHEN compressed_original IS NULL THEN 1 ELSE 0 END) AS noBlob
1362
- FROM context_chunks WHERE dedup_status != 'removed'`,
1363
- )
1364
- .get() as { withBlob: number; blobBytes: number; noBlob: number };
1365
- const removed = db
1366
- .prepare(`SELECT COUNT(*) AS c FROM context_chunks WHERE dedup_status = 'removed'`)
1367
- .get() as { c: number };
1368
- return {
1369
- regionsRetained: row.withBlob,
1370
- compressedOriginalBytes: row.blobBytes,
1371
- regionsWithoutBlob: row.noBlob ?? 0,
1372
- bytesPermanentlyDeleted: 0,
1373
- duplicatesCollapsed: removed.c,
1374
- };
1375
- }
1376
-
1377
- export function repoStats(stateDir: string = getStateDir()): RepoStats {
1378
- const db = openStore(stateDir);
1379
- const row = db
1380
- .prepare(
1381
- `SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
1382
- COALESCE(SUM(original_token_estimate),0) AS orig,
1383
- COUNT(DISTINCT session_id) AS sessions
1384
- FROM context_chunks WHERE dedup_status != 'removed'`,
1385
- )
1386
- .get() as { c: number; tok: number; orig: number; sessions: number };
1387
- const ds = getDedupStats(stateDir);
1388
- return {
1389
- checkpointCount: row.c,
1390
- totalTokenEstimate: row.tok,
1391
- originalTokens: row.orig,
1392
- sessionCount: row.sessions,
1393
- tokensSaved: getMetaNumber("tokens_saved", stateDir),
1394
- dedupAttempts: ds.attempts,
1395
- dedupCollapsed: ds.deduped,
1396
- storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
1397
- };
1398
- }
1399
-
1400
- /** A captured model/provider snapshot (for cost estimation + dashboard). */
1401
- export interface ModelSnapshot {
1402
- provider: string;
1403
- providerName: string | null;
1404
- modelId: string;
1405
- modelName: string | null;
1406
- inputRate: number; // USD per input token
1407
- outputRate: number; // USD per output token
1408
- contextWindow: number;
1409
- maxTokens: number;
1410
- reasoning: boolean;
1411
- capturedAt: number;
1412
- }
1413
-
1414
- /** Persist the active model/provider for a repo (latest row wins per repo). */
1415
- export function recordModelSnapshot(
1416
- repoRoot: string,
1417
- snap: Omit<ModelSnapshot, "capturedAt">,
1418
- stateDir: string = getStateDir(),
1419
- ): void {
1420
- const db = openStore(stateDir);
1421
- db.prepare(
1422
- `INSERT INTO model_snapshots
1423
- (repo_root, provider, provider_name, model_id, model_name, input_rate,
1424
- output_rate, context_window, max_tokens, reasoning, captured_at)
1425
- VALUES (@repo_root, @provider, @provider_name, @model_id, @model_name,
1426
- @input_rate, @output_rate, @context_window, @max_tokens, @reasoning, @captured_at)`,
1427
- ).run({
1428
- repo_root: repoRoot,
1429
- provider: snap.provider,
1430
- provider_name: snap.providerName,
1431
- model_id: snap.modelId,
1432
- model_name: snap.modelName,
1433
- input_rate: snap.inputRate,
1434
- output_rate: snap.outputRate,
1435
- context_window: snap.contextWindow,
1436
- max_tokens: snap.maxTokens,
1437
- reasoning: snap.reasoning ? 1 : 0,
1438
- captured_at: Date.now(),
1439
- });
1440
- }
1441
-
1442
- /** Most recent model/provider snapshot for a repo, or undefined. */
1443
- export function latestModelSnapshot(stateDir: string = getStateDir()): ModelSnapshot | undefined {
1444
- const db = openStore(stateDir);
1445
- const row = db
1446
- .prepare(
1447
- `SELECT * FROM model_snapshots ORDER BY captured_at DESC LIMIT 1`,
1448
- )
1449
- .get() as
1450
- | {
1451
- provider: string;
1452
- provider_name: string | null;
1453
- model_id: string;
1454
- model_name: string | null;
1455
- input_rate: number;
1456
- output_rate: number;
1457
- context_window: number;
1458
- max_tokens: number;
1459
- reasoning: number;
1460
- captured_at: number;
1461
- }
1462
- | undefined;
1463
- if (!row) return undefined;
1464
- return {
1465
- provider: row.provider,
1466
- providerName: row.provider_name,
1467
- modelId: row.model_id,
1468
- modelName: row.model_name,
1469
- inputRate: row.input_rate,
1470
- outputRate: row.output_rate,
1471
- contextWindow: row.context_window,
1472
- maxTokens: row.max_tokens,
1473
- reasoning: row.reasoning === 1,
1474
- capturedAt: row.captured_at,
1475
- };
1476
- }
1477
-
1478
- /** Close and evict a cached connection (test teardown only). */
1479
- export function closeStore(stateDir: string): void {
1480
- const db = cache.get(stateDir);
1481
- if (db) {
1482
- db.close();
1483
- cache.delete(stateDir);
1484
- }
1485
- }
1486
-
1487
- // ---- Sprint 13: RAPTOR node persistence ----------------------------------
1488
-
1489
- export interface StoredRaptorNode {
1490
- id: string;
1491
- sessionId: string;
1492
- level: number;
1493
- parentId: string | null;
1494
- children: string[];
1495
- summary: string;
1496
- embedding: number[];
1497
- qualityMarker: string;
1498
- tokenEstimate: number;
1499
- /** S25: epoch ms when the tree containing this node was built. */
1500
- builtAt: number;
1501
- }
1502
-
1503
- /** Persist a single RAPTOR node (upsert by (session_id, id)). */
1504
- export function upsertRaptorNode(node: StoredRaptorNode, stateDir: string = getStateDir()): void {
1505
- const db = openStore(stateDir);
1506
- db.prepare(
1507
- `INSERT INTO raptor_nodes(id, session_id, level, parent_id, children, summary, embedding_blob, quality_marker, token_estimate, built_at)
1508
- VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1509
- ON CONFLICT(session_id, id) DO UPDATE SET
1510
- level=excluded.level, parent_id=excluded.parent_id, children=excluded.children,
1511
- summary=excluded.summary, embedding_blob=excluded.embedding_blob,
1512
- quality_marker=excluded.quality_marker, token_estimate=excluded.token_estimate,
1513
- built_at=excluded.built_at`,
1514
- ).run(
1515
- node.id,
1516
- node.sessionId,
1517
- node.level,
1518
- node.parentId,
1519
- jsonText(node.children),
1520
- node.summary,
1521
- encodeEmbedding(node.embedding),
1522
- node.qualityMarker,
1523
- node.tokenEstimate,
1524
- node.builtAt,
1525
- );
1526
- }
1527
-
1528
- /** Persist an entire built RAPTOR tree for a session (shadow or live). */
1529
- export function saveRaptorTree(
1530
- sessionId: string,
1531
- tree: {
1532
- nodes: Map<string, {
1533
- id: string;
1534
- level: number;
1535
- parentId: string | null;
1536
- children: string[];
1537
- summary: string;
1538
- embedding: number[];
1539
- qualityMarker: string;
1540
- tokenEstimate: number;
1541
- }>
1542
- },
1543
- builtAt: number,
1544
- stateDir: string = getStateDir(),
1545
- ): void {
1546
- for (const node of tree.nodes.values()) {
1547
- upsertRaptorNode(
1548
- {
1549
- id: node.id,
1550
- sessionId,
1551
- level: node.level,
1552
- parentId: node.parentId,
1553
- children: node.children,
1554
- summary: node.summary,
1555
- embedding: node.embedding,
1556
- qualityMarker: node.qualityMarker,
1557
- tokenEstimate: node.tokenEstimate,
1558
- builtAt,
1559
- },
1560
- stateDir,
1561
- );
1562
- }
1563
- }
1564
-
1565
- /** Load all RAPTOR nodes for a session. */
1566
- export function listRaptorNodes(sessionId: string, stateDir: string = getStateDir()): StoredRaptorNode[] {
1567
- const db = openStore(stateDir);
1568
- const rows = db
1569
- .prepare("SELECT * FROM raptor_nodes WHERE session_id = ? ORDER BY level ASC, id ASC")
1570
- .all(normalizeSessionId(sessionId)) as any[];
1571
- return rows.map((row) => ({
1572
- id: row.id,
1573
- sessionId: row.session_id,
1574
- level: row.level,
1575
- parentId: row.parent_id ?? null,
1576
- children: row.children ? JSON.parse(row.children) : [],
1577
- summary: row.summary ?? "",
1578
- embedding: decodeEmbedding(row.embedding_blob),
1579
- qualityMarker: row.quality_marker ?? "low",
1580
- tokenEstimate: row.token_estimate ?? 0,
1581
- builtAt: Number(row.built_at ?? 0),
1582
- }));
1583
- }
1584
-
1585
- /** Delete all RAPTOR nodes for a session (rollback/cleanup). */
1586
- export function clearRaptorNodes(sessionId: string, stateDir: string = getStateDir()): void {
1587
- const db = openStore(stateDir);
1588
- db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(normalizeSessionId(sessionId));
1589
- }
1590
-
1591
- // --- S27: durable raw-transcript mirror + checkpoint-epoch registry ---------
1592
- // ADDITIVE + flag-default-OFF (MEGACOMPACT_DB_MIRROR). No behavior change
1593
- // until the flag is flipped on; the tables above are created IF NOT EXISTS on
1594
- // open so existing stores are untouched. These helpers are storage primitives
1595
- // only — the runtime hook (Task 5) wires them into the compaction flow. All
1596
- // queries use @-parameterized placeholders (PREVENT-002); no `any` types
1597
- // (PREVENT-011).
1598
-
1599
- /** One appended raw-message row in the durable mirror. */
1600
- export interface RawTranscriptRow {
1601
- contentHash: string;
1602
- sessionId: string;
1603
- seq: number;
1604
- role: string;
1605
- contentBytes: string;
1606
- toolName: string | null;
1607
- /** ORIGINAL message timestamp captured at append time (NOT a served ts). */
1608
- messageTimestamp: number | null;
1609
- checkpointEpoch: string;
1610
- }
1611
-
1612
- /** One checkpoint-epoch bookkeeping row (informational registry). */
1613
- export interface CheckpointEpoch {
1614
- epochId: string;
1615
- sessionId: string;
1616
- startedSeq: number;
1617
- committedSeq: number;
1618
- summaryMessageText: string;
1619
- cutIndex: number;
1620
- checkpointId: string;
1621
- createdAt: number;
1622
- }
1623
-
1624
- /** DB row shape for raw_transcript (snake_case column names). */
1625
- interface RawTranscriptDBRow {
1626
- content_hash: string;
1627
- session_id: string;
1628
- seq: number;
1629
- role: string;
1630
- content_bytes: string;
1631
- tool_name: string | null;
1632
- message_timestamp: number | null;
1633
- checkpoint_epoch: string;
1634
- }
1635
-
1636
- /** DB row shape for checkpoint_epochs (snake_case column names). */
1637
- interface CheckpointEpochDBRow {
1638
- epoch_id: string;
1639
- session_id: string;
1640
- started_seq: number;
1641
- committed_seq: number;
1642
- summary_message_text: string;
1643
- cut_index: number;
1644
- checkpoint_id: string;
1645
- created_at: number;
1646
- }
1647
-
1648
- function rowToRawTranscript(row: RawTranscriptDBRow): RawTranscriptRow {
1649
- return {
1650
- contentHash: row.content_hash,
1651
- sessionId: row.session_id,
1652
- seq: Number(row.seq),
1653
- role: row.role,
1654
- contentBytes: row.content_bytes,
1655
- toolName: row.tool_name ?? null,
1656
- messageTimestamp:
1657
- row.message_timestamp == null ? null : Number(row.message_timestamp),
1658
- checkpointEpoch: row.checkpoint_epoch,
1659
- };
1660
- }
1661
-
1662
- function rowToCheckpointEpoch(row: CheckpointEpochDBRow): CheckpointEpoch {
1663
- return {
1664
- epochId: row.epoch_id,
1665
- sessionId: row.session_id,
1666
- startedSeq: Number(row.started_seq),
1667
- committedSeq: Number(row.committed_seq),
1668
- summaryMessageText: row.summary_message_text,
1669
- cutIndex: Number(row.cut_index),
1670
- checkpointId: row.checkpoint_id,
1671
- createdAt: Number(row.created_at),
1672
- };
1673
- }
1674
-
1675
- /**
1676
- * Append one raw-message row to the durable mirror. Idempotent by
1677
- * (content_hash, session_id) via INSERT OR IGNORE — re-appending the same
1678
- * content for the same session is a no-op. seq is assigned server-side as
1679
- * COALESCE(MAX(seq),0)+1 within the session, so callers never need to compute
1680
- * it. Pass an open store handle (openStore) — matches the other DatabaseSync
1681
- * helpers. Parameterized (PREVENT-002).
1682
- */
1683
- export function appendRawTranscript(db: DatabaseSync, row: RawTranscriptRow): void {
1684
- withTx(db, () => {
1685
- db.prepare(
1686
- `INSERT OR IGNORE INTO raw_transcript
1687
- (content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch)
1688
- VALUES (
1689
- @content_hash, @session_id,
1690
- COALESCE((SELECT MAX(seq) FROM raw_transcript WHERE session_id = @session_id), 0) + 1,
1691
- @role, @content_bytes, @tool_name, @message_timestamp, @checkpoint_epoch
1692
- )`,
1693
- ).run({
1694
- "@content_hash": row.contentHash,
1695
- "@session_id": row.sessionId,
1696
- "@role": row.role,
1697
- "@content_bytes": row.contentBytes,
1698
- "@tool_name": row.toolName,
1699
- "@message_timestamp": row.messageTimestamp,
1700
- "@checkpoint_epoch": row.checkpointEpoch,
1701
- });
1702
- });
1703
- }
1704
-
1705
- /**
1706
- * List raw-transcript rows for a session in [fromSeq, toSeq], ordered by seq
1707
- * ascending. Returns camel-cased RawTranscriptRow[]. Parameterized.
1708
- */
1709
- export function listRawTranscriptRange(
1710
- db: DatabaseSync,
1711
- sessionId: string,
1712
- fromSeq: number,
1713
- toSeq: number,
1714
- ): RawTranscriptRow[] {
1715
- const rows = db
1716
- .prepare(
1717
- `SELECT content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch
1718
- FROM raw_transcript
1719
- WHERE session_id = @session_id AND seq >= @from_seq AND seq <= @to_seq
1720
- ORDER BY seq ASC`,
1721
- )
1722
- .all({
1723
- "@session_id": sessionId,
1724
- "@from_seq": fromSeq,
1725
- "@to_seq": toSeq,
1726
- }) as unknown as RawTranscriptDBRow[];
1727
- return rows.map(rowToRawTranscript);
1728
- }
1729
-
1730
- /**
1731
- * Insert (or refresh) a checkpoint-epoch row. ON CONFLICT(epoch_id) DO UPDATE
1732
- * so re-running the same compaction epoch is idempotent / refresh-safe.
1733
- * Parameterized (PREVENT-002).
1734
- */
1735
- export function writeCheckpointEpoch(db: DatabaseSync, epoch: CheckpointEpoch): void {
1736
- withTx(db, () => {
1737
- db.prepare(
1738
- `INSERT INTO checkpoint_epochs
1739
- (epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at)
1740
- VALUES (@epoch_id, @session_id, @started_seq, @committed_seq, @summary_message_text, @cut_index, @checkpoint_id, @created_at)
1741
- ON CONFLICT(epoch_id) DO UPDATE SET
1742
- session_id = excluded.session_id,
1743
- started_seq = excluded.started_seq,
1744
- committed_seq = excluded.committed_seq,
1745
- summary_message_text = excluded.summary_message_text,
1746
- cut_index = excluded.cut_index,
1747
- checkpoint_id = excluded.checkpoint_id,
1748
- created_at = excluded.created_at`,
1749
- ).run({
1750
- "@epoch_id": epoch.epochId,
1751
- "@session_id": epoch.sessionId,
1752
- "@started_seq": epoch.startedSeq,
1753
- "@committed_seq": epoch.committedSeq,
1754
- "@summary_message_text": epoch.summaryMessageText,
1755
- "@cut_index": epoch.cutIndex,
1756
- "@checkpoint_id": epoch.checkpointId,
1757
- "@created_at": epoch.createdAt,
1758
- });
1759
- });
1760
- }
1761
-
1762
- /** Read one checkpoint-epoch row by id (or null if absent). Parameterized. */
1763
- export function readCheckpointEpoch(db: DatabaseSync, epochId: string): CheckpointEpoch | null {
1764
- const row = db
1765
- .prepare(
1766
- `SELECT epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at
1767
- FROM checkpoint_epochs WHERE epoch_id = @epoch_id`,
1768
- )
1769
- .get({ "@epoch_id": epochId }) as unknown as CheckpointEpochDBRow | undefined;
1770
- return row ? rowToCheckpointEpoch(row) : null;
1771
- }
1772
-
1773
- /**
1774
- * Latest checkpoint-epoch row for a session (highest created_at), or null if
1775
- * none. Parameterized (PREVENT-002).
1776
- */
1777
- export function getActiveEpochForSession(db: DatabaseSync, sessionId: string): CheckpointEpoch | null {
1778
- const row = db
1779
- .prepare(
1780
- `SELECT epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at
1781
- FROM checkpoint_epochs
1782
- WHERE session_id = @session_id
1783
- ORDER BY created_at DESC
1784
- LIMIT 1`,
1785
- )
1786
- .get({ "@session_id": sessionId }) as unknown as CheckpointEpochDBRow | undefined;
1787
- return row ? rowToCheckpointEpoch(row) : null;
1788
- }
1789
-
1790
- /** List all checkpoint epochs (diagnostic / test helper). */
1791
- export function listCheckpointEpochs(db: DatabaseSync): CheckpointEpoch[] {
1792
- const rows = db
1793
- .prepare(
1794
- `SELECT epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at
1795
- FROM checkpoint_epochs
1796
- ORDER BY created_at DESC`,
1797
- )
1798
- .all() as unknown as CheckpointEpochDBRow[];
1799
- return rows.map(rowToCheckpointEpoch);
1800
- }
1801
-
1802
- /** Count raw transcript rows (diagnostic / test helper). */
1803
- export function countRawTranscript(db: DatabaseSync): number {
1804
- const row = db.prepare(`SELECT COUNT(*) AS cnt FROM raw_transcript`).get() as { cnt: number };
1805
- return row.cnt;
1806
- }
1807
-
1808
- // ────────────────────────────────────────────────────────────────────────
1809
- // S27 Task 6: dedup_mirror functions
1810
- // ────────────────────────────────────────────────────────────────────────
1811
-
1812
- /**
1813
- * Dedup mirror row (DB representation).
1814
- */
1815
- export interface DedupMirrorRowDB {
1816
- content_hash: string;
1817
- content_bytes: string;
1818
- ref_count: number;
1819
- first_seen_seq: number;
1820
- created_at: number;
1821
- }
1822
-
1823
- /**
1824
- * Upsert a row into dedup_mirror. If the hash already exists, increment ref_count.
1825
- * Returns true if this was a NEW unique content (first insert), false if it was a duplicate.
1826
- */
1827
- export function upsertDedupMirror(
1828
- db: DatabaseSync,
1829
- contentHash: string,
1830
- contentBytes: string,
1831
- seq: number,
1832
- ): boolean {
1833
- const now = Date.now();
1834
- const existing = db
1835
- .prepare(`SELECT content_hash FROM dedup_mirror WHERE content_hash = @hash`)
1836
- .get({ "@hash": contentHash }) as { content_hash: string } | undefined;
1837
- if (existing) {
1838
- db.prepare(`UPDATE dedup_mirror SET ref_count = ref_count + 1 WHERE content_hash = @hash`).run({
1839
- "@hash": contentHash,
1840
- });
1841
- return false;
1842
- }
1843
- db.prepare(
1844
- `INSERT INTO dedup_mirror (content_hash, content_bytes, ref_count, first_seen_seq, created_at)
1845
- VALUES (@hash, @bytes, 1, @seq, @now)`,
1846
- ).run({
1847
- "@hash": contentHash,
1848
- "@bytes": contentBytes,
1849
- "@seq": seq,
1850
- "@now": now,
1851
- });
1852
- return true;
1853
- }
1854
-
1855
- /**
1856
- * Get dedup ratio for a session: total bytes vs unique bytes.
1857
- */
1858
- export function getDedupRatio(
1859
- db: DatabaseSync,
1860
- sessionId: string,
1861
- ): { totalBytes: number; uniqueBytes: number; ratio: number } {
1862
- const totalRow = db
1863
- .prepare(
1864
- `SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS total
1865
- FROM raw_transcript
1866
- WHERE session_id = @session_id`,
1867
- )
1868
- .get({ "@session_id": sessionId }) as { total: number };
1869
- const uniqueRow = db
1870
- .prepare(
1871
- `SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS unique_bytes
1872
- FROM dedup_mirror`,
1873
- )
1874
- .get() as { unique_bytes: number };
1875
- const totalBytes = totalRow.total;
1876
- const uniqueBytes = uniqueRow.unique_bytes;
1877
- const ratio = uniqueBytes > 0 ? totalBytes / uniqueBytes : 1;
1878
- return { totalBytes, uniqueBytes, ratio };
1879
- }
1880
-
1881
- /**
1882
- * Get dedup mirror stats (diagnostic / test helper).
1883
- */
1884
- export function getDedupMirrorStats(db: DatabaseSync): {
1885
- rowCount: number;
1886
- totalBytes: number;
1887
- avgRefCount: number;
1888
- } {
1889
- const row = db
1890
- .prepare(
1891
- `SELECT COUNT(*) AS cnt,
1892
- COALESCE(SUM(LENGTH(content_bytes)), 0) AS total_bytes,
1893
- COALESCE(AVG(ref_count), 0) AS avg_ref
1894
- FROM dedup_mirror`,
1895
- )
1896
- .get() as { cnt: number; total_bytes: number; avg_ref: number };
1897
- return { rowCount: row.cnt, totalBytes: row.total_bytes, avgRefCount: row.avg_ref };
1898
- }
1899
-
1900
- /**
1901
- * Update raw_transcript.content_ref to point to dedup_mirror.
1902
- */
1903
- export function updateRawTranscriptRef(
1904
- db: DatabaseSync,
1905
- sessionId: string,
1906
- seq: number,
1907
- contentHash: string,
1908
- ): void {
1909
- db.prepare(
1910
- `UPDATE raw_transcript SET content_ref = @ref WHERE session_id = @sid AND seq = @seq`,
1911
- ).run({
1912
- "@ref": contentHash,
1913
- "@sid": sessionId,
1914
- "@seq": seq,
1915
- });
1916
- }
1917
-
1918
- // ---------------------------------------------------------------------------
1919
- // S27 Task 10 — DB maintenance / housekeeping primitives.
1920
- // All pi-agnostic, all parameterized (PREVENT-002), all local (PREVENT-PI-004).
1921
- // Exposed via the /mega-db-* slash commands in extensions/mega-db-cmds.ts.
1922
- // ---------------------------------------------------------------------------
1923
-
1924
- /** Per-table row counts + DB file sizes for the /mega-db-stats command. */
1925
- export interface DbStats {
1926
- /** Row count per table (keys are table names that exist in this DB). */
1927
- tableCounts: Record<string, number>;
1928
- /** Bytes used by the main DB file on disk. */
1929
- dbBytes: number;
1930
- /** Bytes used by the -wal sidecar file (0 if absent). */
1931
- walBytes: number;
1932
- /** Bytes used by the -shm sidecar file (0 if absent). */
1933
- shmBytes: number;
1934
- /** SQLite page size in bytes. */
1935
- pageSize: number;
1936
- /** Total pages (freelist + in-use). */
1937
- pageCount: number;
1938
- /** Freelist pages (reusable by VACUUM). */
1939
- freelistPages: number;
1940
- /** WAL frame count from PRAGMA wal_info (best-effort; 0 if unsupported). */
1941
- walFrames: number;
1942
- }
1943
-
1944
- const DB_TABLE_NAMES = [
1945
- "context_chunks",
1946
- "session_state",
1947
- "raw_transcript",
1948
- "checkpoint_epochs",
1949
- "dedup_mirror",
1950
- "memories",
1951
- "dedup_stats",
1952
- "daily_log",
1953
- ] as const;
1954
-
1955
- function fileSizeIfExists(path: string): number {
1956
- try {
1957
- const st = statSync(path);
1958
- return st.size;
1959
- } catch {
1960
- return 0;
1961
- }
1962
- }
1963
-
1964
- /**
1965
- * Gather DB stats for /mega-db-stats: per-table row counts, disk footprint
1966
- * (main + WAL + SHM), page count, freelist, WAL frame count.
1967
- *
1968
- * Read-only: no PRAGMA writes, no VACUUM. Safe to call any time.
1969
- */
1970
- export function getDbStats(stateDir: string = getStateDir()): DbStats {
1971
- const db = openStore(stateDir);
1972
- const tableCounts: Record<string, number> = {};
1973
- for (const t of DB_TABLE_NAMES) {
1974
- try {
1975
- const row = db.prepare(`SELECT COUNT(*) AS c FROM ${t}`).get() as { c: number } | undefined;
1976
- if (row) tableCounts[t] = row.c;
1977
- } catch {
1978
- // Table doesn't exist on this DB (e.g. raw_transcript on a pre-S27 store).
1979
- // Skip silently — /mega-db-stats lists only tables that exist.
1980
- }
1981
- }
1982
- const pageStat = db.prepare("PRAGMA page_count").get() as { page_count?: number } | undefined;
1983
- const freelistStat = db.prepare("PRAGMA freelist_count").get() as { freelist_count?: number } | undefined;
1984
- const pageSizeStat = db.prepare("PRAGMA page_size").get() as { page_size?: number } | undefined;
1985
- let walFrames = 0;
1986
- try {
1987
- const walInfo = db.prepare("PRAGMA wal_info").get() as { frames?: number } | undefined;
1988
- walFrames = walInfo?.frames ?? 0;
1989
- } catch {
1990
- // node:sqlite may not expose wal_info on all versions; not fatal.
1991
- }
1992
- const dbPath = join(stateDir, "sqlite.db");
1993
- return {
1994
- tableCounts,
1995
- dbBytes: fileSizeIfExists(dbPath),
1996
- walBytes: fileSizeIfExists(`${dbPath}-wal`),
1997
- shmBytes: fileSizeIfExists(`${dbPath}-shm`),
1998
- pageSize: pageSizeStat?.page_size ?? 0,
1999
- pageCount: pageStat?.page_count ?? 0,
2000
- freelistPages: freelistStat?.freelist_count ?? 0,
2001
- walFrames,
2002
- };
2003
- }
2004
-
2005
- /** Result of a prune / VACUUM / checkpoint operation (reclaimed bytes). */
2006
- export interface MaintenanceResult {
2007
- /** Rows deleted (prune) or pages reclaimed (VACUUM / checkpoint). */
2008
- affected: number;
2009
- /** Bytes reclaimed on disk (best-effort: post-op size minus pre-op size). */
2010
- reclaimedBytes: number;
2011
- /** Human-readable summary line for the command output. */
2012
- summary: string;
2013
- }
2014
-
2015
- /**
2016
- * Prune raw_transcript + checkpoint_epochs rows older than `daysOld`.
2017
- * Uses `message_timestamp` (raw_transcript) and `created_at` (epochs), both
2018
- * epoch-ms. Returns the total deleted rows + reclaimed disk bytes.
2019
- *
2020
- * PREVENT-002: parameterized. PREVENT-PI-004: local SQLite only.
2021
- */
2022
- export function pruneOldRows(stateDir: string = getStateDir(), daysOld = 30): MaintenanceResult {
2023
- const db = openStore(stateDir);
2024
- const cutoff = Date.now() - daysOld * 86_400_000;
2025
- const beforeBytes = fileSizeIfExists(join(stateDir, "sqlite.db"));
2026
- // raw_transcript: message_timestamp may be NULL (pre-S27 rows); those use
2027
- // the row's insertion order implicitly via seq, so we prune NULL-ts rows
2028
- // only when the whole session is older than the cutoff (join via session_id
2029
- // to checkpoint_epochs.created_at). Simpler: prune NULL-ts rows older than
2030
- // cutoff by falling back to the MIN(created_at) of their epoch.
2031
- // Delete raw_transcript rows whose message_timestamp is older than cutoff,
2032
- // OR whose message_timestamp is NULL and the session's latest epoch is older.
2033
- const delRt = db.prepare(
2034
- `DELETE FROM raw_transcript
2035
- WHERE message_timestamp IS NOT NULL AND message_timestamp < ?
2036
- OR (message_timestamp IS NULL
2037
- AND session_id IN (
2038
- SELECT session_id FROM checkpoint_epochs
2039
- GROUP BY session_id HAVING MAX(created_at) < ?
2040
- ))`,
2041
- ).run(cutoff, cutoff) as { changes?: number } | undefined;
2042
- const rtDeleted = delRt?.changes ?? 0;
2043
- // checkpoint_epochs: created_at is NOT NULL.
2044
- const delEp = db.prepare(`DELETE FROM checkpoint_epochs WHERE created_at < ?`).run(cutoff) as {
2045
- changes?: number;
2046
- } | undefined;
2047
- const epDeleted = delEp?.changes ?? 0;
2048
- // dedup_mirror: cascade-delete orphan rows whose ref_count has dropped to 0
2049
- // after the raw_transcript deletes. Safe even if FK is off (raw_transcript has
2050
- // no FK to dedup_mirror; ref_count is maintained by the dedup pipeline).
2051
- const delDedup = db.prepare(`DELETE FROM dedup_mirror WHERE ref_count <= 0`).run() as {
2052
- changes?: number;
2053
- } | undefined;
2054
- const dedupDeleted = delDedup?.changes ?? 0;
2055
- const afterBytes = fileSizeIfExists(join(stateDir, "sqlite.db"));
2056
- const total = rtDeleted + epDeleted + dedupDeleted;
2057
- return {
2058
- affected: total,
2059
- reclaimedBytes: Math.max(0, beforeBytes - afterBytes),
2060
- summary: `pruned ${rtDeleted} raw_transcript + ${epDeleted} epochs + ${dedupDeleted} dedup_mirror rows older than ${daysOld}d`,
2061
- };
2062
- }
2063
-
2064
- /**
2065
- * Force a WAL checkpoint (TRUNCATE mode) so the -wal sidecar is reclaimed.
2066
- * Returns the WAL bytes reclaimed (pre-wal size minus post-wal size).
2067
- */
2068
- export function checkpointWal(stateDir: string = getStateDir()): MaintenanceResult {
2069
- const db = openStore(stateDir);
2070
- const dbPath = join(stateDir, "sqlite.db");
2071
- const beforeWal = fileSizeIfExists(`${dbPath}-wal`);
2072
- // PRAGMA wal_checkpoint(TRUNCATE) blocks until all frames are folded into the
2073
- // main db and the WAL file is truncated to 0 bytes.
2074
- const res = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get() as {
2075
- busy?: number;
2076
- log?: number;
2077
- checkpointed?: number;
2078
- } | undefined;
2079
- const afterWal = fileSizeIfExists(`${dbPath}-wal`);
2080
- const reclaimed = Math.max(0, beforeWal - afterWal);
2081
- return {
2082
- affected: res?.checkpointed ?? 0,
2083
- reclaimedBytes: reclaimed,
2084
- summary: `wal_checkpoint(TRUNCATE): ${res?.checkpointed ?? 0} frames folded, WAL ${beforeWal}→${afterWal} bytes${res?.busy ? " (busy: " + res.busy + ")" : ""}`,
2085
- };
2086
- }
2087
-
2088
- /**
2089
- * VACUUM the main DB file (rebuilds pages, reclaims freelist space).
2090
- * Heavy: briefly doubles disk usage. Run only when freelist is large or the
2091
- * user explicitly invokes /mega-db-vacuum.
2092
- */
2093
- export function vacuumDb(stateDir: string = getStateDir()): MaintenanceResult {
2094
- const db = openStore(stateDir);
2095
- const dbPath = join(stateDir, "sqlite.db");
2096
- const beforeBytes = fileSizeIfExists(dbPath);
2097
- db.exec("VACUUM"); // VACUUM cannot be parameterized; it rewrites the whole DB.
2098
- const afterBytes = fileSizeIfExists(dbPath);
2099
- const reclaimed = Math.max(0, beforeBytes - afterBytes);
2100
- return {
2101
- affected: 0,
2102
- reclaimedBytes: reclaimed,
2103
- summary: `VACUUM: db ${beforeBytes}→${afterBytes} bytes (reclaimed ${reclaimed})`,
2104
- };
2105
- }
2106
-
2107
- /**
2108
- * Run `PRAGMA integrity_check` and return the result lines.
2109
- * Returns ["ok"] when the DB is healthy; otherwise returns the error lines.
2110
- */
2111
- export function integrityCheck(stateDir: string = getStateDir()): string[] {
2112
- const db = openStore(stateDir);
2113
- const rows = db.prepare("PRAGMA integrity_check").all() as Array<{ integrity_check: string }> | undefined;
2114
- return (rows ?? []).map((r) => r.integrity_check);
2115
- }
2116
-
2117
- /** Reconcile drift in dedup_mirror.ref_count vs actual raw_transcript refs. */
2118
- export interface DedupReconcileResult {
2119
- /** Rows whose ref_count was corrected. */
2120
- fixedRefCount: number;
2121
- /** Orphan dedup_mirror rows (content_hash with 0 raw_transcript refs) deleted. */
2122
- orphansDeleted: number;
2123
- /** raw_transcript rows whose content_ref was NULL but now set (backfill). */
2124
- refsBackfilled: number;
2125
- }
2126
-
2127
- /**
2128
- * Reconcile dedup_mirror vs raw_transcript after pruning or crashes:
2129
- * 1. Recompute ref_count = COUNT(raw_transcript rows pointing at this hash).
2130
- * 2. Delete orphan dedup_mirror rows whose recomputed ref_count is 0.
2131
- * 3. Backfill raw_transcript.content_ref for rows still storing inline bytes.
2132
- *
2133
- * Idempotent. Read-modify-write within a single transaction (withTx).
2134
- */
2135
- export function reconcileDedupMirror(stateDir: string = getStateDir()): DedupReconcileResult {
2136
- const db = openStore(stateDir);
2137
- const result: DedupReconcileResult = { fixedRefCount: 0, orphansDeleted: 0, refsBackfilled: 0 };
2138
- withTx(db, () => {
2139
- // 1. Recompute ref_count for every dedup_mirror row from the actual
2140
- // raw_transcript references.
2141
- const recompute = db.prepare(
2142
- `UPDATE dedup_mirror AS dm
2143
- SET ref_count = COALESCE((
2144
- SELECT COUNT(*) FROM raw_transcript rt WHERE rt.content_ref = dm.content_hash
2145
- ), 0)
2146
- WHERE dm.ref_count != COALESCE((
2147
- SELECT COUNT(*) FROM raw_transcript rt WHERE rt.content_ref = dm.content_hash
2148
- ), 0)`,
2149
- ).run() as { changes?: number } | undefined;
2150
- result.fixedRefCount = recompute?.changes ?? 0;
2151
- // 2. Delete orphan dedup_mirror rows (no raw_transcript refs).
2152
- const delOrphans = db.prepare(
2153
- `DELETE FROM dedup_mirror
2154
- WHERE content_hash NOT IN (SELECT DISTINCT content_ref FROM raw_transcript WHERE content_ref IS NOT NULL)`,
2155
- ).run() as { changes?: number } | undefined;
2156
- result.orphansDeleted = delOrphans?.changes ?? 0;
2157
- // 3. Backfill content_ref for rows still storing inline content_bytes (no
2158
- // ref yet). Only safe when a matching dedup_mirror row exists; otherwise
2159
- // we'd need to insert one, which is the dedup pipeline's job, not the
2160
- // reconciler's.
2161
- const backfill = db.prepare(
2162
- `UPDATE raw_transcript AS rt
2163
- SET content_ref = (
2164
- SELECT dm.content_hash FROM dedup_mirror dm WHERE dm.content_bytes = rt.content_bytes
2165
- )
2166
- WHERE rt.content_ref IS NULL
2167
- AND EXISTS (SELECT 1 FROM dedup_mirror dm WHERE dm.content_bytes = rt.content_bytes)`,
2168
- ).run() as { changes?: number } | undefined;
2169
- result.refsBackfilled = backfill?.changes ?? 0;
2170
- });
2171
- return result;
2172
- }
2173
-
2174
- /**
2175
- * One-shot auto-maintenance pass for the session_start hook: prune old rows,
2176
- * checkpoint the WAL if it's grown large, and (only if the DB is huge) VACUUM.
2177
- * Best-effort: swallows errors so a session never fails to start over a
2178
- * housekeeping hiccup. Returns a short summary for the diagnostic log.
2179
- */
2180
- export function autoMaintain(stateDir: string = getStateDir()): string {
2181
- try {
2182
- const stats = getDbStats(stateDir);
2183
- const parts: string[] = [];
2184
- // Prune rows older than 30d (default retention).
2185
- const prune = pruneOldRows(stateDir, 30);
2186
- if (prune.affected > 0) parts.push(`pruned ${prune.affected}`);
2187
- // Checkpoint the WAL if it's over 10 MB (avoid pathological WAL growth).
2188
- if (stats.walBytes > 10 * 1024 * 1024) {
2189
- const ck = checkpointWal(stateDir);
2190
- if (ck.reclaimedBytes > 0) parts.push(`wal -${ck.reclaimedBytes}B`);
2191
- }
2192
- // VACUUM only if the DB is over 100 MB AND freelist is >20% of pages.
2193
- if (
2194
- stats.dbBytes > 100 * 1024 * 1024 &&
2195
- stats.pageCount > 0 &&
2196
- stats.freelistPages / stats.pageCount > 0.2
2197
- ) {
2198
- const v = vacuumDb(stateDir);
2199
- if (v.reclaimedBytes > 0) parts.push(`vacuum -${v.reclaimedBytes}B`);
2200
- }
2201
- return parts.length ? `auto-maintain: ${parts.join(", ")}` : "auto-maintain: nothing to do";
2202
- } catch (err) {
2203
- // Never block session start over housekeeping.
2204
- return `auto-maintain: skipped (${(err as Error).message})`;
2205
- }
2206
- }
4
+ * The implementation has been split into focused submodules under
5
+ * `src/store/sqlite/`; this barrel preserves every existing export so ALL
6
+ * consumer import paths (`from "../store/sqlite.js"` etc.) keep working with
7
+ * zero changes. Do not add new code here — add it to the relevant submodule.
8
+ */
9
+ export * from "./sqlite/utils.js";
10
+ export * from "./sqlite/schema.js";
11
+ export * from "./sqlite/meta.js";
12
+ export * from "./sqlite/global-index.js";
13
+ export * from "./sqlite/foundation.js";
14
+ export * from "./sqlite/memories.js";
15
+ export * from "./sqlite/checkpoints.js";
16
+ export * from "./sqlite/session-state.js";
17
+ export * from "./sqlite/stats.js";
18
+ export * from "./sqlite/model-snapshots.js";
19
+ export * from "./sqlite/raptor.js";
20
+ export * from "./sqlite/raw-transcript.js";
21
+ export * from "./sqlite/dedup-mirror.js";
22
+ export * from "./sqlite/maintenance.js";