pi-mega-compact 0.7.7 → 0.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +11 -12
  2. package/dist/extensions/dashboard-server/helpers.js +37 -0
  3. package/dist/extensions/dashboard-server/html/all-repos-tab.js +26 -0
  4. package/dist/extensions/dashboard-server/html/body-open.js +23 -0
  5. package/dist/extensions/dashboard-server/html/current-repo-tab.js +130 -0
  6. package/dist/extensions/dashboard-server/html/head-open.js +16 -0
  7. package/dist/extensions/dashboard-server/html/high-score-tab.js +25 -0
  8. package/dist/extensions/dashboard-server/html/repo-detail-modal.js +26 -0
  9. package/dist/extensions/dashboard-server/html/script.js +259 -0
  10. package/dist/extensions/dashboard-server/html/styles.js +103 -0
  11. package/dist/extensions/dashboard-server/html/summary-tab.js +19 -0
  12. package/dist/extensions/dashboard-server/html-template.js +41 -0
  13. package/dist/extensions/dashboard-server/html.js +756 -0
  14. package/dist/extensions/dashboard-server/index-reader.js +133 -0
  15. package/dist/extensions/dashboard-server/server.js +370 -0
  16. package/dist/extensions/dashboard-server/snapshot.js +43 -0
  17. package/dist/extensions/dashboard-server/state.js +30 -0
  18. package/dist/extensions/dashboard-server/types.js +5 -0
  19. package/dist/extensions/dashboard-server.js +7 -1315
  20. package/dist/extensions/mega-commands.js +162 -134
  21. package/dist/extensions/mega-compact.test.js +292 -24
  22. package/dist/extensions/mega-config.js +10 -0
  23. package/dist/extensions/mega-conflict-cmds.js +5 -1
  24. package/dist/extensions/mega-dashboard-cmds.js +29 -22
  25. package/dist/extensions/mega-db-cmds.js +11 -2
  26. package/dist/extensions/mega-events/agent-handlers.js +173 -0
  27. package/dist/extensions/mega-events/compact-handlers.js +133 -0
  28. package/dist/extensions/mega-events/context-handler.js +249 -0
  29. package/dist/extensions/mega-events/register.js +21 -0
  30. package/dist/extensions/mega-events/session-handlers.js +142 -0
  31. package/dist/extensions/mega-events.js +15 -652
  32. package/dist/extensions/mega-pipeline/compact.js +324 -0
  33. package/dist/extensions/mega-pipeline/memory-review.js +38 -0
  34. package/dist/extensions/mega-pipeline/recall.js +147 -0
  35. package/dist/extensions/mega-pipeline.js +9 -480
  36. package/dist/extensions/mega-runtime/helpers.js +40 -0
  37. package/dist/extensions/mega-runtime/query.js +29 -0
  38. package/dist/extensions/mega-runtime/state.js +711 -0
  39. package/dist/extensions/mega-runtime/widget.js +197 -0
  40. package/dist/extensions/mega-runtime.js +15 -932
  41. package/dist/src/store/sqlite/checkpoints.js +145 -0
  42. package/dist/src/store/sqlite/connection.js +35 -0
  43. package/dist/src/store/sqlite/dedup-mirror.js +64 -0
  44. package/dist/src/store/sqlite/foundation.js +38 -0
  45. package/dist/src/store/sqlite/global-index.js +224 -0
  46. package/dist/src/store/sqlite/index-store.js +167 -0
  47. package/dist/src/store/sqlite/maintenance.js +235 -0
  48. package/dist/src/store/sqlite/memories.js +164 -0
  49. package/dist/src/store/sqlite/memory.js +54 -0
  50. package/dist/src/store/sqlite/meta.js +82 -0
  51. package/dist/src/store/sqlite/minhash-lsh.js +47 -0
  52. package/dist/src/store/sqlite/model-snapshots.js +47 -0
  53. package/dist/src/store/sqlite/raptor.js +57 -0
  54. package/dist/src/store/sqlite/raw-transcript.js +134 -0
  55. package/dist/src/store/sqlite/schema.js +250 -0
  56. package/dist/src/store/sqlite/session-state.js +28 -0
  57. package/dist/src/store/sqlite/sessions.js +39 -0
  58. package/dist/src/store/sqlite/stats.js +66 -0
  59. package/dist/src/store/sqlite/transaction.js +19 -0
  60. package/dist/src/store/sqlite/utils.js +120 -0
  61. package/dist/src/store/sqlite.js +20 -1607
  62. package/dist/src/vectorStore/add.js +260 -0
  63. package/dist/src/vectorStore/dedup.js +52 -0
  64. package/dist/src/vectorStore/index.js +10 -0
  65. package/dist/src/vectorStore/queries.js +83 -0
  66. package/dist/src/vectorStore/search.js +95 -0
  67. package/dist/src/vectorStore/session.js +19 -0
  68. package/dist/src/vectorStore/store.js +105 -0
  69. package/dist/src/vectorStore/types.js +6 -0
  70. package/dist/src/vectorStore/utils.js +23 -0
  71. package/extensions/dashboard-server/html.ts +758 -0
  72. package/extensions/dashboard-server/index-reader.ts +130 -0
  73. package/extensions/dashboard-server/server.ts +358 -0
  74. package/extensions/dashboard-server/snapshot.ts +44 -0
  75. package/extensions/dashboard-server/state.ts +33 -0
  76. package/extensions/dashboard-server/types.ts +134 -0
  77. package/extensions/dashboard-server.ts +7 -1431
  78. package/extensions/mega-commands.ts +33 -10
  79. package/extensions/mega-compact.test.ts +453 -37
  80. package/extensions/mega-config.ts +22 -0
  81. package/extensions/mega-conflict-cmds.ts +6 -2
  82. package/extensions/mega-dashboard-cmds.ts +30 -23
  83. package/extensions/mega-db-cmds.ts +11 -3
  84. package/extensions/mega-events/agent-handlers.ts +214 -0
  85. package/extensions/mega-events/compact-handlers.ts +164 -0
  86. package/extensions/mega-events/context-handler.ts +290 -0
  87. package/extensions/mega-events/register.ts +37 -0
  88. package/extensions/mega-events/session-handlers.ts +165 -0
  89. package/extensions/mega-events.ts +15 -732
  90. package/extensions/mega-pipeline/compact.ts +366 -0
  91. package/extensions/mega-pipeline/memory-review.ts +46 -0
  92. package/extensions/mega-pipeline/recall.ts +165 -0
  93. package/extensions/mega-pipeline.ts +9 -537
  94. package/extensions/mega-runtime/helpers.ts +68 -0
  95. package/extensions/mega-runtime/query.ts +29 -0
  96. package/extensions/mega-runtime/state.ts +797 -0
  97. package/extensions/mega-runtime/widget.ts +258 -0
  98. package/extensions/mega-runtime.ts +15 -1076
  99. package/package.json +4 -3
  100. package/src/store/sqlite/checkpoints.ts +204 -0
  101. package/src/store/sqlite/dedup-mirror.ts +114 -0
  102. package/src/store/sqlite/foundation.ts +63 -0
  103. package/src/store/sqlite/global-index.ts +305 -0
  104. package/src/store/sqlite/maintenance.ts +294 -0
  105. package/src/store/sqlite/memories.ts +217 -0
  106. package/src/store/sqlite/meta.ts +108 -0
  107. package/src/store/sqlite/model-snapshots.ts +83 -0
  108. package/src/store/sqlite/raptor.ts +107 -0
  109. package/src/store/sqlite/raw-transcript.ts +221 -0
  110. package/src/store/sqlite/schema.ts +258 -0
  111. package/src/store/sqlite/session-state.ts +38 -0
  112. package/src/store/sqlite/stats.ts +127 -0
  113. package/src/store/sqlite/utils.ts +125 -0
  114. package/src/store/sqlite.ts +20 -2204
@@ -1,1609 +1,22 @@
1
1
  /**
2
- * sqlite.ts — Sprint 8 storage backbone (the "one store").
2
+ * sqlite.ts — barrel re-export of the SQLite store submodules.
3
3
  *
4
- * Replaces the per-session gzipped-JSON checkpoint files with a single local
5
- * SQLite database (node:sqlite the Node built-in, in-process, FS-backed,
6
- * ZERO network calls honors PREVENT-PI-004). No native build and no install
7
- * scripts, so it survives pi's npm blocked-install-scripts gate (better-sqlite3's
8
- * native binary could not be built under pi, which crashed every `pi update
9
- * --extensions`). node:sqlite is synchronous, so every VectorStore signature
10
- * stays sync. PGlite + pgvector (async) is layered on in vectorIndex.ts for
11
- * real HNSW indexing (Slice 2 of the dual-backend plan).
12
- *
13
- * FTS5 `trigram` tokenizer is created for the Sprint 9+ dedup tiers (MinHash/LSH
14
- * / pg_trgm-equivalent verification). The default cosine path stays a linear
15
- * scan over `embedding_blob` (checkpoint counts are small, no ANN index needed).
16
- *
17
- * All queries are parameterized (PREVENT-002) — never string-concatenated.
18
- */
19
- import { DatabaseSync } from "node:sqlite";
20
- import { existsSync, mkdirSync, statSync } from "node:fs";
21
- import { homedir, tmpdir } from "node:os";
22
- import { join } from "node:path";
23
- import { getStateDir } from "../store.js";
24
- import { normalizeSessionId } from "../store.js";
25
- const SCHEMA_VERSION = 2;
26
- /** Encode a float vector as a little-endian Float32 BLOB for cosine scanning. */
27
- function encodeEmbedding(v) {
28
- const buf = Buffer.allocUnsafe(v.length * 4);
29
- for (let i = 0; i < v.length; i++)
30
- buf.writeFloatLE(v[i] ?? 0, i * 4);
31
- return buf;
32
- }
33
- /** Decode a Float32 BLOB back to a number[]. node:sqlite returns BLOBs as
34
- * Uint8Array, so decode via DataView (Buffer is a Uint8Array subclass — both
35
- * work). */
36
- function decodeEmbedding(buf) {
37
- if (!buf || buf.length === 0)
38
- return [];
39
- const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
40
- const n = buf.length / 4;
41
- const out = new Array(n);
42
- for (let i = 0; i < n; i++)
43
- out[i] = dv.getFloat32(i * 4, true);
44
- return out;
45
- }
46
- function jsonText(v) {
47
- return JSON.stringify(v ?? []);
48
- }
49
- // In-process cache so the same stateDir reuses one connection (and so a fresh
50
- // VectorStore over the same dir shares the open DB). Cross-process durability
51
- // comes from reopening the same file path — proven by the integration test.
52
- const cache = new Map();
53
- /** Open (or reuse) the SQLite store for a state dir. */
54
- export function openStore(stateDir = getStateDir()) {
55
- const existing = cache.get(stateDir);
56
- if (existing) {
57
- // A closed handle in the cache (e.g. a test calling db.close() directly
58
- // instead of closeStore) would surface as "database is not open" on the
59
- // next reuse. Detect and evict so callers never see a dead handle.
60
- try {
61
- existing.prepare("SELECT 1");
62
- return existing;
63
- }
64
- catch {
65
- cache.delete(stateDir);
66
- }
67
- }
68
- if (!existsSync(stateDir))
69
- mkdirSync(stateDir, { recursive: true });
70
- const db = new DatabaseSync(join(stateDir, "sqlite.db"));
71
- db.exec("PRAGMA journal_mode = WAL");
72
- db.exec("PRAGMA foreign_keys = ON");
73
- initSchema(db);
74
- cache.set(stateDir, db);
75
- return db;
76
- }
77
- // ---------------------------------------------------------------------------
78
- // Global machine-wide index (Phase 5b): a single SQLite DB, separate from every
79
- // per-repo store, that aggregates one row per repo this machine has run on. The
80
- // multi-repo dashboard (Summary / All-repos tabs) reads it so ONE dashboard can
81
- // show every repo's checkpoints, tokens saved, and active model — instead of a
82
- // per-repo dashboard that only ever sees the repo it was launched from.
83
- //
84
- // Written by every pi process on repo-switch (bindRepo) + model capture; read by
85
- // the dashboard server. Concurrency across 10+ pi processes is handled by WAL +
86
- // infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
87
- // ---------------------------------------------------------------------------
88
- /** Resolve the machine-wide index directory (env-overridable). */
89
- export function getIndexDir() {
90
- const override = process.env.MEGACOMPACT_INDEX_DIR;
91
- if (override && override.trim() !== "")
92
- return override;
93
- // homedir() can throw in exotic sandboxes; fall back to tmpdir.
94
- try {
95
- return join(homedir(), ".mega-compact-index");
96
- }
97
- catch {
98
- return join(tmpdir(), ".mega-compact-index");
99
- }
100
- }
101
- let indexCache;
102
- let indexCacheDir;
103
- /** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
104
- export function openIndexStore(indexDir = getIndexDir()) {
105
- if (indexCache && indexCacheDir === indexDir)
106
- return indexCache;
107
- if (!existsSync(indexDir))
108
- mkdirSync(indexDir, { recursive: true });
109
- const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
110
- iddb.exec("PRAGMA journal_mode = WAL");
111
- iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
112
- iddb.exec(`
113
- CREATE TABLE IF NOT EXISTS repo_registry (
114
- repo_root TEXT PRIMARY KEY,
115
- display_name TEXT,
116
- state_dir TEXT NOT NULL,
117
- first_seen INTEGER,
118
- last_seen INTEGER,
119
- last_compacted_at INTEGER,
120
- checkpoint_count INTEGER DEFAULT 0,
121
- tokens_saved INTEGER DEFAULT 0,
122
- compressed_original_bytes INTEGER DEFAULT 0,
123
- provider TEXT,
124
- provider_name TEXT,
125
- model_name TEXT,
126
- input_rate REAL,
127
- output_rate REAL,
128
- model_captured_at INTEGER
129
- );
130
- CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
131
- -- S18: machine-wide injected-set. A foreign checkpoint injected in repo A is
132
- -- recorded here so repo B's recall never re-injects it. Keyed by checkpoint
133
- -- + session (a checkpoint may be injected once per session); repo_id is the
134
- -- source repo (the foreign repo's stateDir) for tracking/source labels.
135
- -- PRAMETERIZED queries (PREVENT-002); local node:sqlite (PREVENT-PI-004).
136
- CREATE TABLE IF NOT EXISTS injected_global (
137
- checkpoint_id TEXT NOT NULL,
138
- repo_id TEXT NOT NULL,
139
- session_id TEXT NOT NULL,
140
- injected_at INTEGER NOT NULL,
141
- PRIMARY KEY (checkpoint_id, session_id)
142
- );
143
- CREATE INDEX IF NOT EXISTS idx_injected_global_cid ON injected_global(checkpoint_id);
144
- `);
145
- indexCache = iddb;
146
- indexCacheDir = indexDir;
147
- return iddb;
148
- }
149
- /**
150
- * Upsert a repo's aggregate stats into the global index. Called on repo-switch
151
- * (infrequent). Preserves first_seen + the model columns on update (model is
152
- * written separately by recordRepoModel so we never clobber it here with nulls).
153
- */
154
- export function upsertRepoRegistry(row, indexDir = getIndexDir()) {
155
- const db = openIndexStore(indexDir);
156
- const now = Date.now();
157
- db.prepare(`INSERT INTO repo_registry
158
- (repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
159
- checkpoint_count, tokens_saved, compressed_original_bytes,
160
- provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
161
- VALUES (@repo_root, @display_name, @state_dir, @first_seen, @last_seen, @last_compacted_at,
162
- @checkpoint_count, @tokens_saved, @compressed_original_bytes,
163
- @provider, @provider_name, @model_name, @input_rate, @output_rate, @model_captured_at)
164
- ON CONFLICT(repo_root) DO UPDATE SET
165
- display_name = excluded.display_name,
166
- state_dir = excluded.state_dir,
167
- last_seen = COALESCE(excluded.last_seen, @now),
168
- last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
169
- checkpoint_count = excluded.checkpoint_count,
170
- tokens_saved = excluded.tokens_saved,
171
- compressed_original_bytes = excluded.compressed_original_bytes,
172
- provider = COALESCE(excluded.provider, repo_registry.provider),
173
- provider_name = COALESCE(excluded.provider_name, repo_registry.provider_name),
174
- model_name = COALESCE(excluded.model_name, repo_registry.model_name),
175
- input_rate = COALESCE(excluded.input_rate, repo_registry.input_rate),
176
- output_rate = COALESCE(excluded.output_rate, repo_registry.output_rate),
177
- model_captured_at = COALESCE(excluded.model_captured_at, repo_registry.model_captured_at)`).run({
178
- repo_root: row.repoRoot,
179
- display_name: row.displayName,
180
- state_dir: row.stateDir,
181
- now,
182
- first_seen: row.firstSeen ?? null,
183
- last_seen: row.lastSeen ?? null,
184
- last_compacted_at: row.lastCompactedAt ?? null,
185
- checkpoint_count: row.checkpointCount,
186
- tokens_saved: row.tokensSaved,
187
- compressed_original_bytes: row.compressedOriginalBytes,
188
- provider: row.provider ?? null,
189
- provider_name: row.providerName ?? null,
190
- model_name: row.modelName ?? null,
191
- input_rate: row.inputRate ?? null,
192
- output_rate: row.outputRate ?? null,
193
- model_captured_at: row.modelCapturedAt ?? null,
194
- });
195
- }
196
- /**
197
- * Record the active model/provider for a repo in the global index (denormalized
198
- * so the All-repos table shows model without opening each repo's DB). Upserts a
199
- * bare registry row if the repo isn't registered yet.
200
- */
201
- export function recordRepoModel(repoRoot, model, indexDir = getIndexDir()) {
202
- const db = openIndexStore(indexDir);
203
- const now = Date.now();
204
- db.prepare(`INSERT INTO repo_registry
205
- (repo_root, display_name, state_dir, first_seen, last_seen,
206
- provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
207
- VALUES (@repo_root, @display_name, @state_dir, @now, @now,
208
- @provider, @provider_name, @model_name, @input_rate, @output_rate, @now)
209
- ON CONFLICT(repo_root) DO UPDATE SET
210
- last_seen = excluded.last_seen,
211
- provider = excluded.provider,
212
- provider_name = excluded.provider_name,
213
- model_name = excluded.model_name,
214
- input_rate = excluded.input_rate,
215
- output_rate = excluded.output_rate,
216
- model_captured_at = excluded.model_captured_at`).run({
217
- repo_root: repoRoot,
218
- display_name: model.displayName,
219
- state_dir: model.stateDir,
220
- now,
221
- provider: model.provider,
222
- provider_name: model.providerName,
223
- model_name: model.modelName,
224
- input_rate: model.inputRate,
225
- output_rate: model.outputRate,
226
- });
227
- }
228
- function mapRegistryRow(row) {
229
- return {
230
- repoRoot: row.repo_root,
231
- displayName: row.display_name ?? "",
232
- stateDir: row.state_dir,
233
- firstSeen: row.first_seen ?? 0,
234
- lastSeen: row.last_seen ?? 0,
235
- lastCompactedAt: row.last_compacted_at ?? null,
236
- checkpointCount: row.checkpoint_count ?? 0,
237
- tokensSaved: row.tokens_saved ?? 0,
238
- compressedOriginalBytes: row.compressed_original_bytes ?? 0,
239
- provider: row.provider ?? null,
240
- providerName: row.provider_name ?? null,
241
- modelName: row.model_name ?? null,
242
- inputRate: row.input_rate ?? null,
243
- outputRate: row.output_rate ?? null,
244
- modelCapturedAt: row.model_captured_at ?? null,
245
- };
246
- }
247
- /** All registered repos, most-recently-seen first. */
248
- export function listRepoRegistry(indexDir = getIndexDir()) {
249
- const db = openIndexStore(indexDir);
250
- const rows = db.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC").all();
251
- return rows.map(mapRegistryRow);
252
- }
253
- /** A single repo's registry row, or undefined. */
254
- export function getRepoRegistry(repoRoot, indexDir = getIndexDir()) {
255
- const db = openIndexStore(indexDir);
256
- const row = db.prepare("SELECT * FROM repo_registry WHERE repo_root = ?").get(repoRoot);
257
- return row ? mapRegistryRow(row) : undefined;
258
- }
259
- /** Close the cached index connection (test teardown only). */
260
- export function closeIndexStore() {
261
- if (indexCache) {
262
- indexCache.close();
263
- indexCache = undefined;
264
- indexCacheDir = undefined;
265
- }
266
- }
267
- // ---------------------------------------------------------------------------
268
- // S18: machine-wide injected-set (cross-repo dedup markers)
269
- //
270
- // A foreign checkpoint injected in repo A is recorded here so repo B's recall
271
- // never re-injects it (a stronger, machine-wide version of the per-session
272
- // injected-set in the local store). Keyed by (checkpoint_id, session_id); the
273
- // session_id here is the RECEIVING session, so the same foreign checkpoint can
274
- // be injected into different sessions but never twice into the same one.
275
- // PRAMETERIZED queries (PREVENT-002); local node:sqlite + WAL (PREVENT-PI-004),
276
- // multi-process safe.
277
- // ---------------------------------------------------------------------------
278
- /** Record that a (foreign) checkpoint was injected into `sessionId`. Idempotent. */
279
- export function markInjectedGlobal(checkpointId, repoId, sessionId, indexDir = getIndexDir()) {
280
- const db = openIndexStore(indexDir);
281
- db.prepare("INSERT OR IGNORE INTO injected_global (checkpoint_id, repo_id, session_id, injected_at) VALUES ($cid, $rid, $sid, $ts)").run({ $cid: checkpointId, $rid: repoId, $sid: sessionId, $ts: Date.now() });
282
- }
283
- /** True when a checkpoint was already injected into `sessionId` (machine-wide). */
284
- export function wasInjectedGlobal(checkpointId, sessionId, indexDir = getIndexDir()) {
285
- const db = openIndexStore(indexDir);
286
- const row = db.prepare("SELECT 1 FROM injected_global WHERE checkpoint_id = $cid AND session_id = $sid LIMIT 1").get({ $cid: checkpointId, $sid: sessionId });
287
- return row !== undefined;
288
- }
289
- /** Count of cross-repo injections recorded (for /mega-status stats). */
290
- export function countInjectedGlobal(indexDir = getIndexDir()) {
291
- const db = openIndexStore(indexDir);
292
- const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get();
293
- return row?.n ?? 0;
294
- }
295
- function initSchema(db) {
296
- db.exec(`
297
- CREATE TABLE IF NOT EXISTS context_chunks (
298
- id TEXT NOT NULL,
299
- session_id TEXT NOT NULL,
300
- region_hash TEXT,
301
- content_hash TEXT,
302
- content_hash2 TEXT,
303
- content_hash_version INTEGER,
304
- normalized_text TEXT,
305
- summary TEXT,
306
- topic_summary TEXT,
307
- summary_hash TEXT,
308
- key_decisions TEXT, -- JSON array
309
- next_steps TEXT, -- JSON array
310
- files_modified TEXT, -- JSON array
311
- embedding_blob BLOB, -- float32 vector
312
- token_estimate INTEGER,
313
- original_token_estimate INTEGER, -- dropped region size (tokens saved = orig − stored)
314
- timestamp INTEGER,
315
- dedup_status TEXT DEFAULT 'active',
316
- compressed_original BLOB -- optional DR copy
317
- );
318
- -- Primary key is (session_id, id): checkpoint ids are unique per session
319
- -- (chkpt_001 per session), not globally, so a bare id PK would collide
320
- -- across sessions on the nextCheckpointId sequence.
321
- CREATE UNIQUE INDEX IF NOT EXISTS idx_chunks_pk
322
- ON context_chunks(session_id, id);
323
- CREATE INDEX IF NOT EXISTS idx_chunks_session ON context_chunks(session_id);
324
- CREATE INDEX IF NOT EXISTS idx_chunks_region ON context_chunks(region_hash);
325
- CREATE INDEX IF NOT EXISTS idx_chunks_content ON context_chunks(content_hash);
326
- -- Partial UNIQUE (QA #1): null content_hash rows never violate the constraint;
327
- -- ON CONFLICT DO NOTHING makes backfill + L0 inserts safe.
328
- CREATE UNIQUE INDEX IF NOT EXISTS idx_content_hash
329
- ON context_chunks(session_id, content_hash) WHERE content_hash IS NOT NULL;
330
-
331
- -- Sprint 11: MinHash signature + LSH bucket tables for L1 near-dup dedup.
332
- CREATE TABLE IF NOT EXISTS minhash_signatures (
333
- chunk_id TEXT NOT NULL,
334
- session_id TEXT NOT NULL,
335
- signature_version INTEGER NOT NULL,
336
- signatures TEXT NOT NULL, -- JSON array of 256 uint32
337
- PRIMARY KEY (chunk_id, signature_version)
338
- );
339
- CREATE INDEX IF NOT EXISTS idx_minhash_session ON minhash_signatures(session_id);
340
-
341
- CREATE TABLE IF NOT EXISTS dedup_lsh_buckets (
342
- bucket_key TEXT NOT NULL,
343
- chunk_id TEXT NOT NULL,
344
- session_id TEXT NOT NULL,
345
- signature_version INTEGER NOT NULL,
346
- PRIMARY KEY (bucket_key, chunk_id)
347
- );
348
- CREATE INDEX IF NOT EXISTS idx_lsh_bucket ON dedup_lsh_buckets(bucket_key, session_id);
349
-
350
- CREATE TABLE IF NOT EXISTS session_state (
351
- session_id TEXT PRIMARY KEY,
352
- injected_checkpoint_ids TEXT, -- JSON array
353
- stored_region_hashes TEXT -- JSON array
354
- );
355
-
356
- CREATE TABLE IF NOT EXISTS meta (
357
- key TEXT PRIMARY KEY,
358
- value TEXT
359
- );
360
-
361
- -- Sprint 13 (RAPTOR): hierarchical summary tree nodes. children are a JSON
362
- -- array of child node ids (or raw leaf ids at the bottom); embedding_blob
363
- -- is the node centroid. Additive; retrieval ignores this table until
364
- -- Sprint 14 promotes RAPTOR out of shadow mode.
365
- CREATE TABLE IF NOT EXISTS raptor_nodes (
366
- id TEXT NOT NULL,
367
- session_id TEXT NOT NULL,
368
- level INTEGER NOT NULL,
369
- parent_id TEXT,
370
- children TEXT, -- JSON array of child ids
371
- summary TEXT,
372
- embedding_blob BLOB, -- float32 centroid
373
- quality_marker TEXT DEFAULT 'low',
374
- token_estimate INTEGER,
375
- built_at INTEGER, -- S25: epoch ms when the tree was built (freshness guard)
376
- PRIMARY KEY (session_id, id)
377
- );
378
- CREATE INDEX IF NOT EXISTS idx_raptor_session ON raptor_nodes(session_id);
379
- CREATE INDEX IF NOT EXISTS idx_raptor_parent ON raptor_nodes(parent_id);
380
-
381
- -- Foundation for future features (resume sessions, daily log, lessons
382
- -- learned). Scaffolded now so all store data lives in SQLite from day one;
383
- -- population is minimal (touchSession / logDaily on compact) and the full
384
- -- UI/recall for these lands in later sprints.
385
-
386
- -- Per-session registry (resume + per-repo session history).
387
- CREATE TABLE IF NOT EXISTS sessions (
388
- session_id TEXT PRIMARY KEY,
389
- repo TEXT,
390
- started_at INTEGER,
391
- ended_at INTEGER,
392
- last_compacted_at INTEGER,
393
- status TEXT DEFAULT 'active'
394
- );
395
-
396
- -- Append-only daily activity log (the "daily log" feature seed).
397
- CREATE TABLE IF NOT EXISTS daily_log (
398
- id INTEGER PRIMARY KEY AUTOINCREMENT,
399
- day TEXT NOT NULL, -- YYYY-MM-DD
400
- session_id TEXT,
401
- event TEXT, -- e.g. 'compact'
402
- detail TEXT,
403
- tokens_saved INTEGER DEFAULT 0,
404
- ts INTEGER
405
- );
406
- CREATE INDEX IF NOT EXISTS idx_daily_log_day ON daily_log(day);
407
-
408
- -- Active model/provider for cost estimation + the future multi-repo
409
- -- dashboard (Phase 5b). One row per (repo, model change); latest wins.
410
- CREATE TABLE IF NOT EXISTS model_snapshots (
411
- id INTEGER PRIMARY KEY AUTOINCREMENT,
412
- repo_root TEXT NOT NULL,
413
- provider TEXT NOT NULL,
414
- provider_name TEXT,
415
- model_id TEXT NOT NULL,
416
- model_name TEXT,
417
- input_rate REAL, -- USD per input token (Model.cost.input)
418
- output_rate REAL, -- USD per output token (Model.cost.output)
419
- context_window INTEGER,
420
- max_tokens INTEGER,
421
- reasoning INTEGER DEFAULT 0,
422
- captured_at INTEGER
423
- );
424
- CREATE INDEX IF NOT EXISTS idx_model_repo ON model_snapshots(repo_root);
425
-
426
- -- Lessons learned (future recall/browse feature seed).
427
- CREATE TABLE IF NOT EXISTS lessons (
428
- id INTEGER PRIMARY KEY AUTOINCREMENT,
429
- session_id TEXT,
430
- repo TEXT,
431
- lesson TEXT,
432
- ts INTEGER
433
- );
434
-
435
- -- Durable "save to memory" store (taken over from memory extensions).
436
- -- One row per saved memory; scoped by repo so memory travels with the
437
- -- clone. All params are parameterized (PREVENT-002).
438
- CREATE TABLE IF NOT EXISTS memories (
439
- id INTEGER PRIMARY KEY AUTOINCREMENT,
440
- repo TEXT,
441
- kind TEXT DEFAULT 'note', -- note | fact | decision | preference
442
- content TEXT NOT NULL,
443
- tags TEXT, -- JSON array of strings
444
- created_at INTEGER,
445
- last_recalled_at INTEGER,
446
- -- S20 memory-RAG extension (auto-review add/replace/remove ops).
447
- category TEXT, -- typed bucket, e.g. decision | fact | preference
448
- target TEXT, -- optional subject/scope this memory targets
449
- last_referenced INTEGER, -- last time memory was referenced by recall (epoch s)
450
- source_turn INTEGER -- conversation turn that produced this memory
451
- );
452
- CREATE INDEX IF NOT EXISTS idx_memories_repo ON memories(repo);
453
-
454
- -- FTS5 trigram virtual table (Sprint 9+ pg_trgm-equivalent verification).
455
- CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_trgm USING fts5(
456
- id UNINDEXED,
457
- normalized_text,
458
- tokenize='trigram'
459
- );
460
-
461
- -- S27: durable raw-transcript mirror (MEGACOMPACT_DB_MIRROR). Appended
462
- -- RAW message bytes per session so a compacted window can be rehydrated
463
- -- from the local store instead of the pi runtime transcript (which is
464
- -- trimmed). PK is (content_hash, session_id) — NOT content_hash alone —
465
- -- so identical content in different sessions never collides. Additive:
466
- -- CREATE TABLE IF NOT EXISTS leaves existing DBs untouched on open until
467
- -- the S27 mirror flag is flipped on. All queries parameterized (PREVENT-002).
468
- CREATE TABLE IF NOT EXISTS raw_transcript (
469
- content_hash TEXT NOT NULL,
470
- session_id TEXT NOT NULL,
471
- seq INTEGER NOT NULL,
472
- role TEXT NOT NULL,
473
- content_bytes TEXT NOT NULL,
474
- tool_name TEXT,
475
- message_timestamp INTEGER, -- ORIGINAL msg ts at append, NOT served
476
- checkpoint_epoch TEXT NOT NULL,
477
- PRIMARY KEY (content_hash, session_id)
478
- );
479
- CREATE INDEX IF NOT EXISTS idx_rt_session_seq ON raw_transcript(session_id, seq);
480
- CREATE INDEX IF NOT EXISTS idx_rt_epoch ON raw_transcript(checkpoint_epoch);
481
-
482
- -- S27: checkpoint-epoch registry. One row per compaction epoch; the
483
- -- summary_message_text is the verbatim system message that replaced the
484
- -- trimmed prefix. Informational bookkeeping (the raw_transcript rows are
485
- -- authoritative); refresh-safe via ON CONFLICT(epoch_id) DO UPDATE.
486
- CREATE TABLE IF NOT EXISTS checkpoint_epochs (
487
- epoch_id TEXT PRIMARY KEY,
488
- session_id TEXT NOT NULL,
489
- started_seq INTEGER NOT NULL,
490
- committed_seq INTEGER NOT NULL,
491
- summary_message_text TEXT NOT NULL,
492
- cut_index INTEGER NOT NULL,
493
- checkpoint_id TEXT NOT NULL,
494
- created_at INTEGER NOT NULL
495
- );
496
- CREATE INDEX IF NOT EXISTS idx_epoch_session ON checkpoint_epochs(session_id, created_at DESC);
497
-
498
- -- S27 Task 6: dedup_mirror for space-efficient deduplicated storage.
499
- -- Each unique content_hash stores its bytes ONCE; raw_transcript rows
500
- -- reference this table via content_ref instead of storing duplicate content_bytes inline.
501
- CREATE TABLE IF NOT EXISTS dedup_mirror (
502
- content_hash TEXT PRIMARY KEY,
503
- content_bytes TEXT NOT NULL,
504
- ref_count INTEGER NOT NULL DEFAULT 1,
505
- first_seen_seq INTEGER NOT NULL,
506
- created_at INTEGER NOT NULL
507
- );
508
- `);
509
- // Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
510
- // pre-existing table, so new columns added to context_chunks after a store was
511
- // first created (e.g. original_token_estimate in v0.4.2) must be ALTERed in for
512
- // databases created by an older version — otherwise repoStats()/upsert crash
513
- // with "no such column" and the extension fails to load. Additive only.
514
- ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
515
- // S27 Task 6: content_ref column in raw_transcript for dedup_mirror references.
516
- ensureColumn(db, "raw_transcript", "content_ref", "TEXT");
517
- // S20 memory-RAG extension: additive columns for auto-review ops. Idempotent —
518
- // only alters DBs created by an older version that lack these columns.
519
- ensureColumn(db, "memories", "category", "TEXT");
520
- ensureColumn(db, "memories", "target", "TEXT");
521
- ensureColumn(db, "memories", "last_referenced", "INTEGER");
522
- ensureColumn(db, "memories", "source_turn", "INTEGER");
523
- // S25: RAPTOR freshness-guard timestamp. Additive; old DBs have NULL → 0 →
524
- // treated as stale → flat fallback (safe).
525
- ensureColumn(db, "raptor_nodes", "built_at", "INTEGER");
526
- const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get();
527
- if (!v) {
528
- db.prepare("INSERT INTO meta(key, value) VALUES(?, ?)").run("schema_version", String(SCHEMA_VERSION));
529
- }
530
- }
531
- /**
532
- * Add `column` (with `decl`, e.g. "INTEGER") to `table` if it does not already
533
- * exist. Idempotent: checks PRAGMA table_info first, so it is safe to run on
534
- * every open. Table/column/decl are code-controlled constants (never user
535
- * input), so the unavoidable identifier interpolation here does not violate
536
- * PREVENT-002 (no external data reaches this SQL).
537
- */
538
- function ensureColumn(db, table, column, decl) {
539
- const cols = db.prepare(`PRAGMA table_info(${table})`).all();
540
- if (cols.some((c) => c.name === column))
541
- return;
542
- db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${decl}`);
543
- }
544
- /** Read a string-valued meta key (or undefined). Used for cumulative counters. */
545
- export function getMeta(key, stateDir = getStateDir()) {
546
- const db = openStore(stateDir);
547
- const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key);
548
- return row?.value;
549
- }
550
- /**
551
- * Cumulative "tokens saved" — the sum of stored checkpoint token estimates across
552
- * all compactions in this store (one per repo). Persisted in the SQLite `meta`
553
- * table so it survives session restarts and travels with the repo's state dir,
554
- * mirroring how `storageDedupRate` is cumulative. Incremented in VectorStore.add()
555
- * when a new (non-deduped) checkpoint is persisted.
556
- */
557
- export function getTokensSaved(stateDir = getStateDir()) {
558
- const raw = getMeta("tokens_saved", stateDir);
559
- const n = raw == null ? 0 : Number(raw);
560
- return Number.isFinite(n) ? n : 0;
561
- }
562
- /** Add `delta` (>=0) to the cumulative tokens-saved counter. */
563
- export function addTokensSaved(delta, stateDir = getStateDir()) {
564
- if (!(delta > 0))
565
- return;
566
- const db = openStore(stateDir);
567
- db.prepare(`INSERT INTO meta(key, value) VALUES('tokens_saved', ?)
568
- ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`).run(String(delta), delta);
569
- }
570
- /** Read a store-wide integer counter from the meta table (0 if absent). */
571
- export function getMetaNumber(key, stateDir = getStateDir()) {
572
- const raw = getMeta(key, stateDir);
573
- const n = raw == null ? 0 : Number(raw);
574
- return Number.isFinite(n) ? n : 0;
575
- }
576
- /** Atomically add `delta` to an integer meta counter. */
577
- function incMeta(key, delta, stateDir = getStateDir()) {
578
- if (!(delta > 0))
579
- return;
580
- const db = openStore(stateDir);
581
- db.prepare(`INSERT INTO meta(key, value) VALUES(?, ?)
582
- ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`).run(key, String(delta), delta);
583
- }
584
- /** Read the cumulative store-wide dedup counters. */
585
- export function getDedupStats(stateDir = getStateDir()) {
586
- return {
587
- attempts: getMetaNumber("dedup_attempts", stateDir),
588
- deduped: getMetaNumber("deduped", stateDir),
589
- };
590
- }
591
- /** Increment the store-wide dedup counters for one add() call. */
592
- export function bumpDedupStats(deduped, stateDir = getStateDir()) {
593
- incMeta("dedup_attempts", 1, stateDir);
594
- if (deduped)
595
- incMeta("deduped", 1, stateDir);
596
- }
597
- // --- Live dashboard counters (schemaless meta key/value — NO migration) -----
598
- // These reuse the private `incMeta` atomically-incrementing integer counter so
599
- // all cumulative tallies live in the same `meta` table as tokens_saved etc.
600
- export function incCompactCount(stateDir = getStateDir()) {
601
- incMeta("compact_count", 1, stateDir);
602
- }
603
- export function getCompactCount(stateDir = getStateDir()) {
604
- return getMetaNumber("compact_count", stateDir);
605
- }
606
- export function incRecallInjected(n, stateDir = getStateDir()) {
607
- if (n > 0)
608
- incMeta("recall_injected", n, stateDir);
609
- }
610
- export function getRecallInjected(stateDir = getStateDir()) {
611
- return getMetaNumber("recall_injected", stateDir);
612
- }
613
- export function incCacheHitTokens(delta, stateDir = getStateDir()) {
614
- if (delta > 0)
615
- incMeta("cache_hit_tokens_saved", delta, stateDir);
616
- }
617
- export function getCacheHitTokensSaved(stateDir = getStateDir()) {
618
- return getMetaNumber("cache_hit_tokens_saved", stateDir);
619
- }
620
- // --- Future-feature foundation (resume sessions / daily log / lessons) -------
621
- // Scaffolded tables + minimal helpers so all store data lives in SQLite from
622
- // day one. Full UI/recall for these lands in later sprints.
623
- /** Upsert a `sessions` row (resume + per-repo session history). */
624
- export function touchSession(sessionId, repo, stateDir = getStateDir()) {
625
- const db = openStore(stateDir);
626
- const sid = normalizeSessionId(sessionId);
627
- const existing = db
628
- .prepare("SELECT started_at FROM sessions WHERE session_id = ?")
629
- .get(sid);
630
- const now = Math.floor(Date.now() / 1000);
631
- if (!existing) {
632
- db.prepare(`INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
633
- VALUES(?, ?, ?, ?, 'active')`).run(sid, repo ?? null, now, now);
634
- }
635
- else {
636
- db.prepare("UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?").run(now, repo ?? null, sid);
637
- }
638
- }
639
- /** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
640
- export function logDaily(sessionId, event, detail, tokensSaved, stateDir = getStateDir()) {
641
- const db = openStore(stateDir);
642
- const day = new Date().toISOString().slice(0, 10);
643
- const now = Math.floor(Date.now() / 1000);
644
- db.prepare(`INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
645
- VALUES(?, ?, ?, ?, ?, ?)`).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
646
- }
647
- /** Append a `lessons` entry (future lessons-learned browse/recall). */
648
- export function addLesson(sessionId, repo, lesson, stateDir = getStateDir()) {
649
- const db = openStore(stateDir);
650
- const now = Math.floor(Date.now() / 1000);
651
- db.prepare(`INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
652
- }
653
- // --- Durable memory (save-to-memory takeover) ---------------------------------
654
- // One SQLite store for user-saved memories, scoped by repo. Mirrors the
655
- // lessons/sessions pattern: all state lives in SQLite from day one.
656
- // S24 storage hardening: keep each memory row bounded so the durable store can
657
- // never blow a downstream consumer's per-entry buffer (e.g. pi's native
658
- // file-backed memory caps a single entry at ~5k chars). We truncate content at
659
- // MEMORY_MAX_CHARS and evict the least-recently-referenced rows past
660
- // MEMORY_MAX_ROWS per repo via LRU. Both are SQLite-only (PREVENT-PI-004): no
661
- // file-backed memory is written anywhere. Defaults are overridable via env
662
- // (MEGACOMPACT_MEMORY_MAX_CHARS / MEGACOMPACT_MEMORY_MAX_ROWS).
663
- export const MEMORY_MAX_CHARS = 4000;
664
- export const MEMORY_MAX_ROWS = 500;
665
- /** Read an env override as a positive int, falling back to `fallback`. */
666
- function envInt(name, fallback) {
667
- const v = process.env[name];
668
- if (v == null || v === "")
669
- return fallback;
670
- const n = Number(v);
671
- return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
672
- }
673
- /** Effective per-entry char cap (env-overridable, default MEMORY_MAX_CHARS). */
674
- export function memoryMaxChars() {
675
- return envInt("MEGACOMPACT_MEMORY_MAX_CHARS", MEMORY_MAX_CHARS);
676
- }
677
- /** Effective per-repo row cap (env-overridable, default MEMORY_MAX_ROWS). */
678
- export function memoryMaxRows() {
679
- return envInt("MEGACOMPACT_MEMORY_MAX_ROWS", MEMORY_MAX_ROWS);
680
- }
681
- /** Truncate memory content to the per-entry cap, preserving a trailing marker. */
682
- function capMemoryContent(content) {
683
- const cap = memoryMaxChars();
684
- if (content.length <= cap)
685
- return content;
686
- return content.slice(0, cap) + "…[truncated]";
687
- }
688
- /**
689
- * Evict the least-recently-referenced rows for a repo past MEMORY_MAX_ROWS.
690
- * LRU key = COALESCE(last_referenced, last_recalled_at, created_at) so a memory
691
- * that is recalled/referenced survives over a stale one. Best-effort: any error
692
- * is swallowed by the caller. Repo-scoped so one noisy repo can't evict another.
693
- */
694
- function evictMemoryLru(repo, stateDir) {
695
- const db = openStore(stateDir);
696
- const maxRows = memoryMaxRows();
697
- // SQLite `= NULL` is never true, so the null-repo scope (memories are
698
- // stateDir-scoped when repo is null — the applyMemoryOps path) needs `IS NULL`.
699
- const where = repo == null ? "repo IS NULL" : "repo = ?";
700
- const countRow = repo == null
701
- ? db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get()
702
- : db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get(repo);
703
- const count = countRow.n;
704
- const over = count - maxRows;
705
- if (over <= 0)
706
- return;
707
- // Delete the `over` least-recently-used rows. ORDER BY the LRU key ASC, id ASC
708
- // (id ASC breaks ties deterministically — oldest created first). The `where`
709
- // clause is a code-controlled constant (never user input) → PREVENT-002 OK.
710
- const sql = `DELETE FROM memories WHERE ${where} AND id IN (
711
- SELECT id FROM memories WHERE ${where}
712
- ORDER BY COALESCE(last_referenced, last_recalled_at, created_at) ASC, id ASC
713
- LIMIT ?
714
- )`;
715
- if (repo == null)
716
- db.prepare(sql).run(over);
717
- else
718
- db.prepare(sql).run(repo, repo, over);
719
- }
720
- /** Save a memory to the current repo's store. Returns the new row id.
721
- * S24 hardening: content is truncated to MEMORY_MAX_CHARS and, once the per-repo
722
- * row count exceeds MEMORY_MAX_ROWS, the least-recently-used rows are evicted
723
- * (LRU) so the store stays bounded. */
724
- export function addMemory(memory, repo, stateDir = getStateDir()) {
725
- const db = openStore(stateDir);
726
- const now = Math.floor(Date.now() / 1000);
727
- const res = db
728
- .prepare(`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at, category, target, source_turn)
729
- VALUES(?, ?, ?, ?, ?, NULL, ?, ?, ?)`)
730
- .run(repo ?? null, memory.kind ?? "note", capMemoryContent(memory.content), JSON.stringify(memory.tags ?? []), now, memory.category ?? null, memory.target ?? null, memory.sourceTurn ?? null);
731
- try {
732
- evictMemoryLru(repo, stateDir);
733
- }
734
- catch {
735
- /* non-fatal: eviction must never fail an add */
736
- }
737
- return Number(res.lastInsertRowid);
738
- }
739
- /** List recent memories for a repo (or all repos when repo is null). */
740
- export function listMemories(repo, limit = 50, stateDir = getStateDir()) {
741
- const db = openStore(stateDir);
742
- const rows = repo
743
- ? db.prepare("SELECT * FROM memories WHERE repo = ? ORDER BY created_at DESC LIMIT ?").all(repo, limit)
744
- : db.prepare("SELECT * FROM memories ORDER BY created_at DESC LIMIT ?").all(limit);
745
- return rows.map(mapMemoryRow);
746
- }
747
- /** Substring search across content + tags. */
748
- export function searchMemories(query, repo = null, limit = 50, stateDir = getStateDir()) {
749
- const db = openStore(stateDir);
750
- const like = `%${query}%`;
751
- const rows = repo
752
- ? db.prepare("SELECT * FROM memories WHERE repo = ? AND (content LIKE ? OR tags LIKE ?) ORDER BY created_at DESC LIMIT ?").all(repo, like, like, limit)
753
- : db.prepare("SELECT * FROM memories WHERE content LIKE ? OR tags LIKE ? ORDER BY created_at DESC LIMIT ?").all(like, like, limit);
754
- return rows.map(mapMemoryRow);
755
- }
756
- /** Mark a memory as recalled (updates last_recalled_at). Returns true if found. */
757
- export function recallMemory(id, stateDir = getStateDir()) {
758
- const db = openStore(stateDir);
759
- const now = Math.floor(Date.now() / 1000);
760
- const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
761
- return res.changes > 0;
762
- }
763
- /** Mark a memory as referenced (updates last_referenced). Returns true if found. */
764
- export function referenceMemory(id, stateDir = getStateDir()) {
765
- const db = openStore(stateDir);
766
- const now = Math.floor(Date.now() / 1000);
767
- const res = db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(now, id);
768
- return res.changes > 0;
769
- }
770
- /** Replace a memory's mutable fields by id. Returns true if a row was updated. */
771
- export function replaceMemory(id, patch, stateDir = getStateDir()) {
772
- const db = openStore(stateDir);
773
- const res = db
774
- .prepare(`UPDATE memories
775
- SET kind = COALESCE(?, kind),
776
- content = COALESCE(?, content),
777
- tags = COALESCE(?, tags),
778
- category = COALESCE(?, category),
779
- target = COALESCE(?, target),
780
- source_turn = COALESCE(?, source_turn)
781
- WHERE id = ?`)
782
- .run(patch.kind ?? null, patch.content != null ? capMemoryContent(patch.content) : null, patch.tags ? JSON.stringify(patch.tags) : null, "category" in patch ? (patch.category ?? null) : null, "target" in patch ? (patch.target ?? null) : null, "sourceTurn" in patch ? (patch.sourceTurn ?? null) : null, id);
783
- return res.changes > 0;
784
- }
785
- /** Remove a memory by id. Returns true if a row was deleted. */
786
- export function removeMemory(id, stateDir = getStateDir()) {
787
- const db = openStore(stateDir);
788
- const res = db.prepare("DELETE FROM memories WHERE id = ?").run(id);
789
- return res.changes > 0;
790
- }
791
- /** Look up a single memory by id (or undefined). */
792
- export function getMemory(id, stateDir = getStateDir()) {
793
- const db = openStore(stateDir);
794
- const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
795
- return row ? mapMemoryRow(row) : undefined;
796
- }
797
- function mapMemoryRow(row) {
798
- return {
799
- id: row.id,
800
- repo: row.repo ?? null,
801
- kind: row.kind ?? "note",
802
- content: row.content ?? "",
803
- tags: row.tags ? JSON.parse(row.tags) : [],
804
- createdAt: row.created_at ?? 0,
805
- lastRecalledAt: row.last_recalled_at ?? null,
806
- category: row.category ?? null,
807
- target: row.target ?? null,
808
- lastReferenced: row.last_referenced ?? null,
809
- sourceTurn: row.source_turn ?? null,
810
- };
811
- }
812
- /**
813
- * Run `fn` atomically. Uses SAVEPOINT so it nests safely under an outer
814
- * transaction (unlike `BEGIN`, which SQLite rejects when one is already open).
815
- * Mirrors better-sqlite3's `db.transaction(fn)` semantics — callers that wrap a
816
- * batch in withTx (e.g. backfill) can still call helpers that also use withTx.
817
- */
818
- export function withTx(db, fn) {
819
- db.exec("SAVEPOINT mc_tx");
820
- try {
821
- fn();
822
- db.exec("RELEASE mc_tx");
823
- }
824
- catch (e) {
825
- db.exec("ROLLBACK TO mc_tx");
826
- db.exec("RELEASE mc_tx");
827
- throw e;
828
- }
829
- }
830
- /** Map a DB row to the public StoredCheckpoint shape. */
831
- function rowToCheckpoint(row) {
832
- return {
833
- checkpointId: row.id,
834
- sessionId: row.session_id,
835
- summary: row.summary ?? "",
836
- topicSummary: row.topic_summary ?? undefined,
837
- summaryHash: row.summary_hash ?? undefined,
838
- keyDecisions: row.key_decisions ? JSON.parse(row.key_decisions) : [],
839
- nextSteps: row.next_steps ? JSON.parse(row.next_steps) : [],
840
- filesModified: row.files_modified ? JSON.parse(row.files_modified) : [],
841
- tokenEstimate: row.token_estimate ?? 0,
842
- originalTokenEstimate: row.original_token_estimate ?? undefined,
843
- regionHash: row.region_hash ?? "",
844
- contentHash: row.content_hash ?? undefined,
845
- contentHash2: row.content_hash2 ?? undefined,
846
- contentHashVersion: row.content_hash_version ?? undefined,
847
- normalizedText: row.normalized_text ?? undefined,
848
- // node:sqlite returns BLOBs as Uint8Array; normalize to Buffer so callers
849
- // (e.g. decompressSmart → Buffer.toString) behave as under better-sqlite3.
850
- compressedOriginal: row.compressed_original ? Buffer.from(row.compressed_original) : undefined,
851
- embedding: decodeEmbedding(row.embedding_blob),
852
- timestamp: Number(row.timestamp ?? 0),
853
- dedupStatus: row.dedup_status ?? undefined,
854
- };
855
- }
856
- /** Insert or replace a checkpoint (idempotent by id). */
857
- export function upsertCheckpoint(cp, stateDir = getStateDir()) {
858
- const db = openStore(stateDir);
859
- const sid = normalizeSessionId(cp.sessionId);
860
- withTx(db, () => {
861
- db.prepare(`INSERT INTO context_chunks
862
- (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
863
- normalized_text, summary, topic_summary, summary_hash,
864
- key_decisions, next_steps, files_modified, embedding_blob,
865
- token_estimate, original_token_estimate, timestamp, dedup_status, compressed_original)
866
- VALUES (@id, @sid, @region_hash, @content_hash, @content_hash2, @content_hash_version,
867
- @normalized_text, @summary, @topic_summary, @summary_hash,
868
- @key_decisions, @next_steps, @files_modified, @embedding_blob,
869
- @token_estimate, @original_token_estimate, @timestamp, @dedup_status, @compressed_original)
870
- ON CONFLICT(session_id, id) DO UPDATE SET
871
- summary=excluded.summary,
872
- topic_summary=excluded.topic_summary,
873
- summary_hash=excluded.summary_hash,
874
- key_decisions=excluded.key_decisions,
875
- next_steps=excluded.next_steps,
876
- files_modified=excluded.files_modified,
877
- embedding_blob=excluded.embedding_blob,
878
- token_estimate=excluded.token_estimate,
879
- original_token_estimate=excluded.original_token_estimate,
880
- timestamp=excluded.timestamp,
881
- dedup_status=excluded.dedup_status,
882
- compressed_original=excluded.compressed_original`).run({
883
- "@id": cp.checkpointId,
884
- "@sid": sid,
885
- "@region_hash": cp.regionHash ?? null,
886
- "@content_hash": cp.contentHash ?? null,
887
- "@content_hash2": cp.contentHash2 ?? null,
888
- "@content_hash_version": cp.contentHashVersion ?? null,
889
- "@normalized_text": cp.normalizedText ?? null,
890
- "@summary": cp.summary ?? "",
891
- "@topic_summary": cp.topicSummary ?? null,
892
- "@summary_hash": cp.summaryHash ?? null,
893
- "@key_decisions": jsonText(cp.keyDecisions),
894
- "@next_steps": jsonText(cp.nextSteps),
895
- "@files_modified": jsonText(cp.filesModified),
896
- "@embedding_blob": encodeEmbedding(cp.embedding ?? []),
897
- "@token_estimate": cp.tokenEstimate ?? 0,
898
- "@original_token_estimate": cp.originalTokenEstimate ?? null,
899
- "@timestamp": cp.timestamp ?? 0,
900
- "@dedup_status": "active",
901
- "@compressed_original": cp.compressedOriginal ?? null,
902
- });
903
- // FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
904
- // Store normalized_text (the L1 verify key); fall back to summary for rows
905
- // that predate normalized_text population.
906
- db.prepare("DELETE FROM context_chunks_trgm WHERE id = ?").run(cp.checkpointId);
907
- db.prepare("INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)").run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
908
- });
909
- }
910
- // --- Sprint 11: MinHash signatures + LSH buckets --------------------------
911
- /** Persist a checkpoint's MinHash signature (idempotent by chunk_id + version). */
912
- export function upsertMinhashSignature(chunkId, sessionId, signatureVersion, signatures, stateDir = getStateDir()) {
913
- const db = openStore(stateDir);
914
- const sid = normalizeSessionId(sessionId);
915
- db.prepare(`INSERT INTO minhash_signatures(chunk_id, session_id, signature_version, signatures)
916
- VALUES(?, ?, ?, ?)
917
- ON CONFLICT(chunk_id, signature_version) DO UPDATE SET
918
- session_id=excluded.session_id, signatures=excluded.signatures`).run(chunkId, sid, signatureVersion, JSON.stringify(signatures));
919
- }
920
- /** Persist LSH bucket memberships for a chunk (one row per bucket key). */
921
- export function insertLshBuckets(chunkId, sessionId, signatureVersion, bucketKeys, stateDir = getStateDir()) {
922
- const db = openStore(stateDir);
923
- const sid = normalizeSessionId(sessionId);
924
- const del = db.prepare("DELETE FROM dedup_lsh_buckets WHERE chunk_id = ?");
925
- const ins = db.prepare("INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)");
926
- withTx(db, () => {
927
- del.run(chunkId);
928
- for (const key of bucketKeys)
929
- ins.run(key, chunkId, sid, signatureVersion);
930
- });
931
- }
932
- /**
933
- * Candidate chunk_ids sharing any LSH bucket with `bucketKeys`, scoped to the
934
- * session, capped at `limit`. Single query (no N loops) — QA #15 amplification
935
- * guard. Returns DISTINCT chunk_ids excluding `excludeChunkId` (the new row).
936
- */
937
- export function lshCandidateChunks(bucketKeys, sessionId, excludeChunkId, stateDir = getStateDir(), limit = 100) {
938
- if (bucketKeys.length === 0)
939
- return [];
940
- const db = openStore(stateDir);
941
- const sid = normalizeSessionId(sessionId);
942
- const placeholders = bucketKeys.map(() => "?").join(",");
943
- const rows = db
944
- .prepare(`SELECT DISTINCT chunk_id FROM dedup_lsh_buckets
945
- WHERE bucket_key IN (${placeholders}) AND session_id = ? AND chunk_id != ?
946
- LIMIT ?`)
947
- .all(...bucketKeys, sid, excludeChunkId, limit);
948
- return rows.map((r) => r.chunk_id);
949
- }
950
- /** All checkpoints for a session, sorted by id. */
951
- export function listCheckpoints(sessionId, stateDir = getStateDir()) {
952
- const db = openStore(stateDir);
953
- const sid = normalizeSessionId(sessionId);
954
- const rows = db
955
- .prepare("SELECT * FROM context_chunks WHERE session_id = ? ORDER BY id ASC")
956
- .all(sid);
957
- return rows.map(rowToCheckpoint);
958
- }
959
- /** S25: the newest checkpoint timestamp for a session, or 0 when none. Used by
960
- * the RAPTOR freshness guard to reject a tree older than the live checkpoints. */
961
- export function maxCheckpointTimestamp(sessionId, stateDir = getStateDir()) {
962
- const db = openStore(stateDir);
963
- const row = db
964
- .prepare("SELECT MAX(timestamp) AS mx FROM context_chunks WHERE session_id = ?")
965
- .get(normalizeSessionId(sessionId));
966
- return Number(row?.mx ?? 0);
967
- }
968
- /** Next sequential checkpoint id (chkpt_001 …) for a session. */
969
- export function nextCheckpointId(sessionId, stateDir = getStateDir()) {
970
- const db = openStore(stateDir);
971
- const sid = normalizeSessionId(sessionId);
972
- const row = db
973
- .prepare("SELECT MAX(CAST(SUBSTR(id, 7) AS INTEGER)) AS n FROM context_chunks WHERE session_id = ?")
974
- .get(sid);
975
- const next = (row.n ?? 0) + 1;
976
- return `chkpt_${String(next).padStart(3, "0")}`;
977
- }
978
- /** True if a checkpoint id already exists for a session. */
979
- export function hasCheckpoint(sessionId, checkpointId, stateDir = getStateDir()) {
980
- const db = openStore(stateDir);
981
- const row = db
982
- .prepare("SELECT 1 FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
983
- .get(normalizeSessionId(sessionId), checkpointId);
984
- return row !== undefined;
985
- }
986
- /** Fetch a single checkpoint by (session, id), or undefined if absent. */
987
- export function getCheckpoint(sessionId, checkpointId, stateDir = getStateDir()) {
988
- const db = openStore(stateDir);
989
- const row = db
990
- .prepare("SELECT * FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
991
- .get(normalizeSessionId(sessionId), checkpointId);
992
- return row ? rowToCheckpoint(row) : undefined;
993
- }
994
- /** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
995
- export function setDedupStatus(checkpointId, sessionId, status, stateDir = getStateDir()) {
996
- const db = openStore(stateDir);
997
- db.prepare("UPDATE context_chunks SET dedup_status = ? WHERE id = ? AND session_id = ?").run(status, checkpointId, normalizeSessionId(sessionId));
998
- }
999
- // --- Session state (injection tracking) ------------------------------------
1000
- function loadSessionStateRow(sid, db) {
1001
- const row = db.prepare("SELECT * FROM session_state WHERE session_id = ?").get(sid);
1002
- if (!row) {
1003
- return { injectedCheckpointIds: [], storedRegionHashes: [] };
1004
- }
1005
- return {
1006
- injectedCheckpointIds: row.injected_checkpoint_ids ? JSON.parse(row.injected_checkpoint_ids) : [],
1007
- storedRegionHashes: row.stored_region_hashes ? JSON.parse(row.stored_region_hashes) : [],
1008
- };
1009
- }
1010
- export function loadSessionState(sessionId, stateDir = getStateDir()) {
1011
- return loadSessionStateRow(normalizeSessionId(sessionId), openStore(stateDir));
1012
- }
1013
- export function saveSessionState(sessionId, state, stateDir = getStateDir()) {
1014
- const db = openStore(stateDir);
1015
- const sid = normalizeSessionId(sessionId);
1016
- db.prepare(`INSERT INTO session_state(session_id, injected_checkpoint_ids, stored_region_hashes)
1017
- VALUES(@sid, @inj, @reg)
1018
- ON CONFLICT(session_id) DO UPDATE SET
1019
- injected_checkpoint_ids=excluded.injected_checkpoint_ids,
1020
- stored_region_hashes=excluded.stored_region_hashes`).run({
1021
- sid,
1022
- inj: jsonText(state.injectedCheckpointIds),
1023
- reg: jsonText(state.storedRegionHashes),
1024
- });
1025
- }
1026
- export function storeStats(sessionId, stateDir = getStateDir()) {
1027
- const db = openStore(stateDir);
1028
- const sid = normalizeSessionId(sessionId);
1029
- const row = db
1030
- .prepare(`SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
1031
- MAX(id) AS lastId
1032
- FROM context_chunks WHERE session_id = ?`)
1033
- .get(sid);
1034
- let lastSummary;
1035
- if (row.lastId) {
1036
- const s = db.prepare("SELECT summary FROM context_chunks WHERE id = ?").get(row.lastId);
1037
- lastSummary = s?.summary;
1038
- }
1039
- return {
1040
- checkpointCount: row.c,
1041
- totalTokenEstimate: row.tok,
1042
- lastCheckpointId: row.lastId ?? undefined,
1043
- lastSummary,
1044
- };
1045
- }
1046
- export function dataInvariantStats(stateDir = getStateDir()) {
1047
- const db = openStore(stateDir);
1048
- const row = db
1049
- .prepare(`SELECT
1050
- COUNT(compressed_original) AS withBlob,
1051
- COALESCE(SUM(LENGTH(compressed_original)),0) AS blobBytes,
1052
- SUM(CASE WHEN compressed_original IS NULL THEN 1 ELSE 0 END) AS noBlob
1053
- FROM context_chunks WHERE dedup_status != 'removed'`)
1054
- .get();
1055
- const removed = db
1056
- .prepare(`SELECT COUNT(*) AS c FROM context_chunks WHERE dedup_status = 'removed'`)
1057
- .get();
1058
- return {
1059
- regionsRetained: row.withBlob,
1060
- compressedOriginalBytes: row.blobBytes,
1061
- regionsWithoutBlob: row.noBlob ?? 0,
1062
- bytesPermanentlyDeleted: 0,
1063
- duplicatesCollapsed: removed.c,
1064
- };
1065
- }
1066
- export function repoStats(stateDir = getStateDir()) {
1067
- const db = openStore(stateDir);
1068
- const row = db
1069
- .prepare(`SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
1070
- COALESCE(SUM(original_token_estimate),0) AS orig,
1071
- COUNT(DISTINCT session_id) AS sessions
1072
- FROM context_chunks WHERE dedup_status != 'removed'`)
1073
- .get();
1074
- const ds = getDedupStats(stateDir);
1075
- return {
1076
- checkpointCount: row.c,
1077
- totalTokenEstimate: row.tok,
1078
- originalTokens: row.orig,
1079
- sessionCount: row.sessions,
1080
- tokensSaved: getMetaNumber("tokens_saved", stateDir),
1081
- dedupAttempts: ds.attempts,
1082
- dedupCollapsed: ds.deduped,
1083
- storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
1084
- };
1085
- }
1086
- /** Persist the active model/provider for a repo (latest row wins per repo). */
1087
- export function recordModelSnapshot(repoRoot, snap, stateDir = getStateDir()) {
1088
- const db = openStore(stateDir);
1089
- db.prepare(`INSERT INTO model_snapshots
1090
- (repo_root, provider, provider_name, model_id, model_name, input_rate,
1091
- output_rate, context_window, max_tokens, reasoning, captured_at)
1092
- VALUES (@repo_root, @provider, @provider_name, @model_id, @model_name,
1093
- @input_rate, @output_rate, @context_window, @max_tokens, @reasoning, @captured_at)`).run({
1094
- repo_root: repoRoot,
1095
- provider: snap.provider,
1096
- provider_name: snap.providerName,
1097
- model_id: snap.modelId,
1098
- model_name: snap.modelName,
1099
- input_rate: snap.inputRate,
1100
- output_rate: snap.outputRate,
1101
- context_window: snap.contextWindow,
1102
- max_tokens: snap.maxTokens,
1103
- reasoning: snap.reasoning ? 1 : 0,
1104
- captured_at: Date.now(),
1105
- });
1106
- }
1107
- /** Most recent model/provider snapshot for a repo, or undefined. */
1108
- export function latestModelSnapshot(stateDir = getStateDir()) {
1109
- const db = openStore(stateDir);
1110
- const row = db
1111
- .prepare(`SELECT * FROM model_snapshots ORDER BY captured_at DESC LIMIT 1`)
1112
- .get();
1113
- if (!row)
1114
- return undefined;
1115
- return {
1116
- provider: row.provider,
1117
- providerName: row.provider_name,
1118
- modelId: row.model_id,
1119
- modelName: row.model_name,
1120
- inputRate: row.input_rate,
1121
- outputRate: row.output_rate,
1122
- contextWindow: row.context_window,
1123
- maxTokens: row.max_tokens,
1124
- reasoning: row.reasoning === 1,
1125
- capturedAt: row.captured_at,
1126
- };
1127
- }
1128
- /** Close and evict a cached connection (test teardown only). */
1129
- export function closeStore(stateDir) {
1130
- const db = cache.get(stateDir);
1131
- if (db) {
1132
- db.close();
1133
- cache.delete(stateDir);
1134
- }
1135
- }
1136
- /** Persist a single RAPTOR node (upsert by (session_id, id)). */
1137
- export function upsertRaptorNode(node, stateDir = getStateDir()) {
1138
- const db = openStore(stateDir);
1139
- db.prepare(`INSERT INTO raptor_nodes(id, session_id, level, parent_id, children, summary, embedding_blob, quality_marker, token_estimate, built_at)
1140
- VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1141
- ON CONFLICT(session_id, id) DO UPDATE SET
1142
- level=excluded.level, parent_id=excluded.parent_id, children=excluded.children,
1143
- summary=excluded.summary, embedding_blob=excluded.embedding_blob,
1144
- quality_marker=excluded.quality_marker, token_estimate=excluded.token_estimate,
1145
- built_at=excluded.built_at`).run(node.id, node.sessionId, node.level, node.parentId, jsonText(node.children), node.summary, encodeEmbedding(node.embedding), node.qualityMarker, node.tokenEstimate, node.builtAt);
1146
- }
1147
- /** Persist an entire built RAPTOR tree for a session (shadow or live). */
1148
- export function saveRaptorTree(sessionId, tree, builtAt, stateDir = getStateDir()) {
1149
- for (const node of tree.nodes.values()) {
1150
- upsertRaptorNode({
1151
- id: node.id,
1152
- sessionId,
1153
- level: node.level,
1154
- parentId: node.parentId,
1155
- children: node.children,
1156
- summary: node.summary,
1157
- embedding: node.embedding,
1158
- qualityMarker: node.qualityMarker,
1159
- tokenEstimate: node.tokenEstimate,
1160
- builtAt,
1161
- }, stateDir);
1162
- }
1163
- }
1164
- /** Load all RAPTOR nodes for a session. */
1165
- export function listRaptorNodes(sessionId, stateDir = getStateDir()) {
1166
- const db = openStore(stateDir);
1167
- const rows = db
1168
- .prepare("SELECT * FROM raptor_nodes WHERE session_id = ? ORDER BY level ASC, id ASC")
1169
- .all(normalizeSessionId(sessionId));
1170
- return rows.map((row) => ({
1171
- id: row.id,
1172
- sessionId: row.session_id,
1173
- level: row.level,
1174
- parentId: row.parent_id ?? null,
1175
- children: row.children ? JSON.parse(row.children) : [],
1176
- summary: row.summary ?? "",
1177
- embedding: decodeEmbedding(row.embedding_blob),
1178
- qualityMarker: row.quality_marker ?? "low",
1179
- tokenEstimate: row.token_estimate ?? 0,
1180
- builtAt: Number(row.built_at ?? 0),
1181
- }));
1182
- }
1183
- /** Delete all RAPTOR nodes for a session (rollback/cleanup). */
1184
- export function clearRaptorNodes(sessionId, stateDir = getStateDir()) {
1185
- const db = openStore(stateDir);
1186
- db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(normalizeSessionId(sessionId));
1187
- }
1188
- function rowToRawTranscript(row) {
1189
- return {
1190
- contentHash: row.content_hash,
1191
- sessionId: row.session_id,
1192
- seq: Number(row.seq),
1193
- role: row.role,
1194
- contentBytes: row.content_bytes,
1195
- toolName: row.tool_name ?? null,
1196
- messageTimestamp: row.message_timestamp == null ? null : Number(row.message_timestamp),
1197
- checkpointEpoch: row.checkpoint_epoch,
1198
- };
1199
- }
1200
- function rowToCheckpointEpoch(row) {
1201
- return {
1202
- epochId: row.epoch_id,
1203
- sessionId: row.session_id,
1204
- startedSeq: Number(row.started_seq),
1205
- committedSeq: Number(row.committed_seq),
1206
- summaryMessageText: row.summary_message_text,
1207
- cutIndex: Number(row.cut_index),
1208
- checkpointId: row.checkpoint_id,
1209
- createdAt: Number(row.created_at),
1210
- };
1211
- }
1212
- /**
1213
- * Append one raw-message row to the durable mirror. Idempotent by
1214
- * (content_hash, session_id) via INSERT OR IGNORE — re-appending the same
1215
- * content for the same session is a no-op. seq is assigned server-side as
1216
- * COALESCE(MAX(seq),0)+1 within the session, so callers never need to compute
1217
- * it. Pass an open store handle (openStore) — matches the other DatabaseSync
1218
- * helpers. Parameterized (PREVENT-002).
1219
- */
1220
- export function appendRawTranscript(db, row) {
1221
- withTx(db, () => {
1222
- db.prepare(`INSERT OR IGNORE INTO raw_transcript
1223
- (content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch)
1224
- VALUES (
1225
- @content_hash, @session_id,
1226
- COALESCE((SELECT MAX(seq) FROM raw_transcript WHERE session_id = @session_id), 0) + 1,
1227
- @role, @content_bytes, @tool_name, @message_timestamp, @checkpoint_epoch
1228
- )`).run({
1229
- "@content_hash": row.contentHash,
1230
- "@session_id": row.sessionId,
1231
- "@role": row.role,
1232
- "@content_bytes": row.contentBytes,
1233
- "@tool_name": row.toolName,
1234
- "@message_timestamp": row.messageTimestamp,
1235
- "@checkpoint_epoch": row.checkpointEpoch,
1236
- });
1237
- });
1238
- }
1239
- /**
1240
- * List raw-transcript rows for a session in [fromSeq, toSeq], ordered by seq
1241
- * ascending. Returns camel-cased RawTranscriptRow[]. Parameterized.
1242
- */
1243
- export function listRawTranscriptRange(db, sessionId, fromSeq, toSeq) {
1244
- const rows = db
1245
- .prepare(`SELECT content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch
1246
- FROM raw_transcript
1247
- WHERE session_id = @session_id AND seq >= @from_seq AND seq <= @to_seq
1248
- ORDER BY seq ASC`)
1249
- .all({
1250
- "@session_id": sessionId,
1251
- "@from_seq": fromSeq,
1252
- "@to_seq": toSeq,
1253
- });
1254
- return rows.map(rowToRawTranscript);
1255
- }
1256
- /**
1257
- * Insert (or refresh) a checkpoint-epoch row. ON CONFLICT(epoch_id) DO UPDATE
1258
- * so re-running the same compaction epoch is idempotent / refresh-safe.
1259
- * Parameterized (PREVENT-002).
1260
- */
1261
- export function writeCheckpointEpoch(db, epoch) {
1262
- withTx(db, () => {
1263
- db.prepare(`INSERT INTO checkpoint_epochs
1264
- (epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at)
1265
- VALUES (@epoch_id, @session_id, @started_seq, @committed_seq, @summary_message_text, @cut_index, @checkpoint_id, @created_at)
1266
- ON CONFLICT(epoch_id) DO UPDATE SET
1267
- session_id = excluded.session_id,
1268
- started_seq = excluded.started_seq,
1269
- committed_seq = excluded.committed_seq,
1270
- summary_message_text = excluded.summary_message_text,
1271
- cut_index = excluded.cut_index,
1272
- checkpoint_id = excluded.checkpoint_id,
1273
- created_at = excluded.created_at`).run({
1274
- "@epoch_id": epoch.epochId,
1275
- "@session_id": epoch.sessionId,
1276
- "@started_seq": epoch.startedSeq,
1277
- "@committed_seq": epoch.committedSeq,
1278
- "@summary_message_text": epoch.summaryMessageText,
1279
- "@cut_index": epoch.cutIndex,
1280
- "@checkpoint_id": epoch.checkpointId,
1281
- "@created_at": epoch.createdAt,
1282
- });
1283
- });
1284
- }
1285
- /** Read one checkpoint-epoch row by id (or null if absent). Parameterized. */
1286
- export function readCheckpointEpoch(db, epochId) {
1287
- const row = db
1288
- .prepare(`SELECT epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at
1289
- FROM checkpoint_epochs WHERE epoch_id = @epoch_id`)
1290
- .get({ "@epoch_id": epochId });
1291
- return row ? rowToCheckpointEpoch(row) : null;
1292
- }
1293
- /**
1294
- * Latest checkpoint-epoch row for a session (highest created_at), or null if
1295
- * none. Parameterized (PREVENT-002).
1296
- */
1297
- export function getActiveEpochForSession(db, sessionId) {
1298
- const row = db
1299
- .prepare(`SELECT epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at
1300
- FROM checkpoint_epochs
1301
- WHERE session_id = @session_id
1302
- ORDER BY created_at DESC
1303
- LIMIT 1`)
1304
- .get({ "@session_id": sessionId });
1305
- return row ? rowToCheckpointEpoch(row) : null;
1306
- }
1307
- /** List all checkpoint epochs (diagnostic / test helper). */
1308
- export function listCheckpointEpochs(db) {
1309
- const rows = db
1310
- .prepare(`SELECT epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at
1311
- FROM checkpoint_epochs
1312
- ORDER BY created_at DESC`)
1313
- .all();
1314
- return rows.map(rowToCheckpointEpoch);
1315
- }
1316
- /** Count raw transcript rows (diagnostic / test helper). */
1317
- export function countRawTranscript(db) {
1318
- const row = db.prepare(`SELECT COUNT(*) AS cnt FROM raw_transcript`).get();
1319
- return row.cnt;
1320
- }
1321
- /**
1322
- * Upsert a row into dedup_mirror. If the hash already exists, increment ref_count.
1323
- * Returns true if this was a NEW unique content (first insert), false if it was a duplicate.
1324
- */
1325
- export function upsertDedupMirror(db, contentHash, contentBytes, seq) {
1326
- const now = Date.now();
1327
- const existing = db
1328
- .prepare(`SELECT content_hash FROM dedup_mirror WHERE content_hash = @hash`)
1329
- .get({ "@hash": contentHash });
1330
- if (existing) {
1331
- db.prepare(`UPDATE dedup_mirror SET ref_count = ref_count + 1 WHERE content_hash = @hash`).run({
1332
- "@hash": contentHash,
1333
- });
1334
- return false;
1335
- }
1336
- db.prepare(`INSERT INTO dedup_mirror (content_hash, content_bytes, ref_count, first_seen_seq, created_at)
1337
- VALUES (@hash, @bytes, 1, @seq, @now)`).run({
1338
- "@hash": contentHash,
1339
- "@bytes": contentBytes,
1340
- "@seq": seq,
1341
- "@now": now,
1342
- });
1343
- return true;
1344
- }
1345
- /**
1346
- * Get dedup ratio for a session: total bytes vs unique bytes.
1347
- */
1348
- export function getDedupRatio(db, sessionId) {
1349
- const totalRow = db
1350
- .prepare(`SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS total
1351
- FROM raw_transcript
1352
- WHERE session_id = @session_id`)
1353
- .get({ "@session_id": sessionId });
1354
- const uniqueRow = db
1355
- .prepare(`SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS unique_bytes
1356
- FROM dedup_mirror`)
1357
- .get();
1358
- const totalBytes = totalRow.total;
1359
- const uniqueBytes = uniqueRow.unique_bytes;
1360
- const ratio = uniqueBytes > 0 ? totalBytes / uniqueBytes : 1;
1361
- return { totalBytes, uniqueBytes, ratio };
1362
- }
1363
- /**
1364
- * Get dedup mirror stats (diagnostic / test helper).
1365
- */
1366
- export function getDedupMirrorStats(db) {
1367
- const row = db
1368
- .prepare(`SELECT COUNT(*) AS cnt,
1369
- COALESCE(SUM(LENGTH(content_bytes)), 0) AS total_bytes,
1370
- COALESCE(AVG(ref_count), 0) AS avg_ref
1371
- FROM dedup_mirror`)
1372
- .get();
1373
- return { rowCount: row.cnt, totalBytes: row.total_bytes, avgRefCount: row.avg_ref };
1374
- }
1375
- /**
1376
- * Update raw_transcript.content_ref to point to dedup_mirror.
1377
- */
1378
- export function updateRawTranscriptRef(db, sessionId, seq, contentHash) {
1379
- db.prepare(`UPDATE raw_transcript SET content_ref = @ref WHERE session_id = @sid AND seq = @seq`).run({
1380
- "@ref": contentHash,
1381
- "@sid": sessionId,
1382
- "@seq": seq,
1383
- });
1384
- }
1385
- const DB_TABLE_NAMES = [
1386
- "context_chunks",
1387
- "session_state",
1388
- "raw_transcript",
1389
- "checkpoint_epochs",
1390
- "dedup_mirror",
1391
- "memories",
1392
- "dedup_stats",
1393
- "daily_log",
1394
- ];
1395
- function fileSizeIfExists(path) {
1396
- try {
1397
- const st = statSync(path);
1398
- return st.size;
1399
- }
1400
- catch {
1401
- return 0;
1402
- }
1403
- }
1404
- /**
1405
- * Gather DB stats for /mega-db-stats: per-table row counts, disk footprint
1406
- * (main + WAL + SHM), page count, freelist, WAL frame count.
1407
- *
1408
- * Read-only: no PRAGMA writes, no VACUUM. Safe to call any time.
1409
- */
1410
- export function getDbStats(stateDir = getStateDir()) {
1411
- const db = openStore(stateDir);
1412
- const tableCounts = {};
1413
- for (const t of DB_TABLE_NAMES) {
1414
- try {
1415
- const row = db.prepare(`SELECT COUNT(*) AS c FROM ${t}`).get();
1416
- if (row)
1417
- tableCounts[t] = row.c;
1418
- }
1419
- catch {
1420
- // Table doesn't exist on this DB (e.g. raw_transcript on a pre-S27 store).
1421
- // Skip silently — /mega-db-stats lists only tables that exist.
1422
- }
1423
- }
1424
- const pageStat = db.prepare("PRAGMA page_count").get();
1425
- const freelistStat = db.prepare("PRAGMA freelist_count").get();
1426
- const pageSizeStat = db.prepare("PRAGMA page_size").get();
1427
- let walFrames = 0;
1428
- try {
1429
- const walInfo = db.prepare("PRAGMA wal_info").get();
1430
- walFrames = walInfo?.frames ?? 0;
1431
- }
1432
- catch {
1433
- // node:sqlite may not expose wal_info on all versions; not fatal.
1434
- }
1435
- const dbPath = join(stateDir, "sqlite.db");
1436
- return {
1437
- tableCounts,
1438
- dbBytes: fileSizeIfExists(dbPath),
1439
- walBytes: fileSizeIfExists(`${dbPath}-wal`),
1440
- shmBytes: fileSizeIfExists(`${dbPath}-shm`),
1441
- pageSize: pageSizeStat?.page_size ?? 0,
1442
- pageCount: pageStat?.page_count ?? 0,
1443
- freelistPages: freelistStat?.freelist_count ?? 0,
1444
- walFrames,
1445
- };
1446
- }
1447
- /**
1448
- * Prune raw_transcript + checkpoint_epochs rows older than `daysOld`.
1449
- * Uses `message_timestamp` (raw_transcript) and `created_at` (epochs), both
1450
- * epoch-ms. Returns the total deleted rows + reclaimed disk bytes.
1451
- *
1452
- * PREVENT-002: parameterized. PREVENT-PI-004: local SQLite only.
1453
- */
1454
- export function pruneOldRows(stateDir = getStateDir(), daysOld = 30) {
1455
- const db = openStore(stateDir);
1456
- const cutoff = Date.now() - daysOld * 86_400_000;
1457
- const beforeBytes = fileSizeIfExists(join(stateDir, "sqlite.db"));
1458
- // raw_transcript: message_timestamp may be NULL (pre-S27 rows); those use
1459
- // the row's insertion order implicitly via seq, so we prune NULL-ts rows
1460
- // only when the whole session is older than the cutoff (join via session_id
1461
- // to checkpoint_epochs.created_at). Simpler: prune NULL-ts rows older than
1462
- // cutoff by falling back to the MIN(created_at) of their epoch.
1463
- // Delete raw_transcript rows whose message_timestamp is older than cutoff,
1464
- // OR whose message_timestamp is NULL and the session's latest epoch is older.
1465
- const delRt = db.prepare(`DELETE FROM raw_transcript
1466
- WHERE message_timestamp IS NOT NULL AND message_timestamp < ?
1467
- OR (message_timestamp IS NULL
1468
- AND session_id IN (
1469
- SELECT session_id FROM checkpoint_epochs
1470
- GROUP BY session_id HAVING MAX(created_at) < ?
1471
- ))`).run(cutoff, cutoff);
1472
- const rtDeleted = delRt?.changes ?? 0;
1473
- // checkpoint_epochs: created_at is NOT NULL.
1474
- const delEp = db.prepare(`DELETE FROM checkpoint_epochs WHERE created_at < ?`).run(cutoff);
1475
- const epDeleted = delEp?.changes ?? 0;
1476
- // dedup_mirror: cascade-delete orphan rows whose ref_count has dropped to 0
1477
- // after the raw_transcript deletes. Safe even if FK is off (raw_transcript has
1478
- // no FK to dedup_mirror; ref_count is maintained by the dedup pipeline).
1479
- const delDedup = db.prepare(`DELETE FROM dedup_mirror WHERE ref_count <= 0`).run();
1480
- const dedupDeleted = delDedup?.changes ?? 0;
1481
- const afterBytes = fileSizeIfExists(join(stateDir, "sqlite.db"));
1482
- const total = rtDeleted + epDeleted + dedupDeleted;
1483
- return {
1484
- affected: total,
1485
- reclaimedBytes: Math.max(0, beforeBytes - afterBytes),
1486
- summary: `pruned ${rtDeleted} raw_transcript + ${epDeleted} epochs + ${dedupDeleted} dedup_mirror rows older than ${daysOld}d`,
1487
- };
1488
- }
1489
- /**
1490
- * Force a WAL checkpoint (TRUNCATE mode) so the -wal sidecar is reclaimed.
1491
- * Returns the WAL bytes reclaimed (pre-wal size minus post-wal size).
1492
- */
1493
- export function checkpointWal(stateDir = getStateDir()) {
1494
- const db = openStore(stateDir);
1495
- const dbPath = join(stateDir, "sqlite.db");
1496
- const beforeWal = fileSizeIfExists(`${dbPath}-wal`);
1497
- // PRAGMA wal_checkpoint(TRUNCATE) blocks until all frames are folded into the
1498
- // main db and the WAL file is truncated to 0 bytes.
1499
- const res = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
1500
- const afterWal = fileSizeIfExists(`${dbPath}-wal`);
1501
- const reclaimed = Math.max(0, beforeWal - afterWal);
1502
- return {
1503
- affected: res?.checkpointed ?? 0,
1504
- reclaimedBytes: reclaimed,
1505
- summary: `wal_checkpoint(TRUNCATE): ${res?.checkpointed ?? 0} frames folded, WAL ${beforeWal}→${afterWal} bytes${res?.busy ? " (busy: " + res.busy + ")" : ""}`,
1506
- };
1507
- }
1508
- /**
1509
- * VACUUM the main DB file (rebuilds pages, reclaims freelist space).
1510
- * Heavy: briefly doubles disk usage. Run only when freelist is large or the
1511
- * user explicitly invokes /mega-db-vacuum.
1512
- */
1513
- export function vacuumDb(stateDir = getStateDir()) {
1514
- const db = openStore(stateDir);
1515
- const dbPath = join(stateDir, "sqlite.db");
1516
- const beforeBytes = fileSizeIfExists(dbPath);
1517
- db.exec("VACUUM"); // VACUUM cannot be parameterized; it rewrites the whole DB.
1518
- const afterBytes = fileSizeIfExists(dbPath);
1519
- const reclaimed = Math.max(0, beforeBytes - afterBytes);
1520
- return {
1521
- affected: 0,
1522
- reclaimedBytes: reclaimed,
1523
- summary: `VACUUM: db ${beforeBytes}→${afterBytes} bytes (reclaimed ${reclaimed})`,
1524
- };
1525
- }
1526
- /**
1527
- * Run `PRAGMA integrity_check` and return the result lines.
1528
- * Returns ["ok"] when the DB is healthy; otherwise returns the error lines.
1529
- */
1530
- export function integrityCheck(stateDir = getStateDir()) {
1531
- const db = openStore(stateDir);
1532
- const rows = db.prepare("PRAGMA integrity_check").all();
1533
- return (rows ?? []).map((r) => r.integrity_check);
1534
- }
1535
- /**
1536
- * Reconcile dedup_mirror vs raw_transcript after pruning or crashes:
1537
- * 1. Recompute ref_count = COUNT(raw_transcript rows pointing at this hash).
1538
- * 2. Delete orphan dedup_mirror rows whose recomputed ref_count is 0.
1539
- * 3. Backfill raw_transcript.content_ref for rows still storing inline bytes.
1540
- *
1541
- * Idempotent. Read-modify-write within a single transaction (withTx).
1542
- */
1543
- export function reconcileDedupMirror(stateDir = getStateDir()) {
1544
- const db = openStore(stateDir);
1545
- const result = { fixedRefCount: 0, orphansDeleted: 0, refsBackfilled: 0 };
1546
- withTx(db, () => {
1547
- // 1. Recompute ref_count for every dedup_mirror row from the actual
1548
- // raw_transcript references.
1549
- const recompute = db.prepare(`UPDATE dedup_mirror AS dm
1550
- SET ref_count = COALESCE((
1551
- SELECT COUNT(*) FROM raw_transcript rt WHERE rt.content_ref = dm.content_hash
1552
- ), 0)
1553
- WHERE dm.ref_count != COALESCE((
1554
- SELECT COUNT(*) FROM raw_transcript rt WHERE rt.content_ref = dm.content_hash
1555
- ), 0)`).run();
1556
- result.fixedRefCount = recompute?.changes ?? 0;
1557
- // 2. Delete orphan dedup_mirror rows (no raw_transcript refs).
1558
- const delOrphans = db.prepare(`DELETE FROM dedup_mirror
1559
- WHERE content_hash NOT IN (SELECT DISTINCT content_ref FROM raw_transcript WHERE content_ref IS NOT NULL)`).run();
1560
- result.orphansDeleted = delOrphans?.changes ?? 0;
1561
- // 3. Backfill content_ref for rows still storing inline content_bytes (no
1562
- // ref yet). Only safe when a matching dedup_mirror row exists; otherwise
1563
- // we'd need to insert one, which is the dedup pipeline's job, not the
1564
- // reconciler's.
1565
- const backfill = db.prepare(`UPDATE raw_transcript AS rt
1566
- SET content_ref = (
1567
- SELECT dm.content_hash FROM dedup_mirror dm WHERE dm.content_bytes = rt.content_bytes
1568
- )
1569
- WHERE rt.content_ref IS NULL
1570
- AND EXISTS (SELECT 1 FROM dedup_mirror dm WHERE dm.content_bytes = rt.content_bytes)`).run();
1571
- result.refsBackfilled = backfill?.changes ?? 0;
1572
- });
1573
- return result;
1574
- }
1575
- /**
1576
- * One-shot auto-maintenance pass for the session_start hook: prune old rows,
1577
- * checkpoint the WAL if it's grown large, and (only if the DB is huge) VACUUM.
1578
- * Best-effort: swallows errors so a session never fails to start over a
1579
- * housekeeping hiccup. Returns a short summary for the diagnostic log.
1580
- */
1581
- export function autoMaintain(stateDir = getStateDir()) {
1582
- try {
1583
- const stats = getDbStats(stateDir);
1584
- const parts = [];
1585
- // Prune rows older than 30d (default retention).
1586
- const prune = pruneOldRows(stateDir, 30);
1587
- if (prune.affected > 0)
1588
- parts.push(`pruned ${prune.affected}`);
1589
- // Checkpoint the WAL if it's over 10 MB (avoid pathological WAL growth).
1590
- if (stats.walBytes > 10 * 1024 * 1024) {
1591
- const ck = checkpointWal(stateDir);
1592
- if (ck.reclaimedBytes > 0)
1593
- parts.push(`wal -${ck.reclaimedBytes}B`);
1594
- }
1595
- // VACUUM only if the DB is over 100 MB AND freelist is >20% of pages.
1596
- if (stats.dbBytes > 100 * 1024 * 1024 &&
1597
- stats.pageCount > 0 &&
1598
- stats.freelistPages / stats.pageCount > 0.2) {
1599
- const v = vacuumDb(stateDir);
1600
- if (v.reclaimedBytes > 0)
1601
- parts.push(`vacuum -${v.reclaimedBytes}B`);
1602
- }
1603
- return parts.length ? `auto-maintain: ${parts.join(", ")}` : "auto-maintain: nothing to do";
1604
- }
1605
- catch (err) {
1606
- // Never block session start over housekeeping.
1607
- return `auto-maintain: skipped (${err.message})`;
1608
- }
1609
- }
4
+ * The implementation has been split into focused submodules under
5
+ * `src/store/sqlite/`; this barrel preserves every existing export so ALL
6
+ * consumer import paths (`from "../store/sqlite.js"` etc.) keep working with
7
+ * zero changes. Do not add new code here — add it to the relevant submodule.
8
+ */
9
+ export * from "./sqlite/utils.js";
10
+ export * from "./sqlite/schema.js";
11
+ export * from "./sqlite/meta.js";
12
+ export * from "./sqlite/global-index.js";
13
+ export * from "./sqlite/foundation.js";
14
+ export * from "./sqlite/memories.js";
15
+ export * from "./sqlite/checkpoints.js";
16
+ export * from "./sqlite/session-state.js";
17
+ export * from "./sqlite/stats.js";
18
+ export * from "./sqlite/model-snapshots.js";
19
+ export * from "./sqlite/raptor.js";
20
+ export * from "./sqlite/raw-transcript.js";
21
+ export * from "./sqlite/dedup-mirror.js";
22
+ export * from "./sqlite/maintenance.js";