pi-mega-compact 0.7.8 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +11 -12
  2. package/dist/extensions/dashboard-server/html.js +1023 -0
  3. package/dist/extensions/dashboard-server/html.test.js +41 -0
  4. package/dist/extensions/dashboard-server/index-reader.js +133 -0
  5. package/dist/extensions/dashboard-server/server.js +530 -0
  6. package/dist/extensions/dashboard-server/server.test.js +120 -0
  7. package/dist/extensions/dashboard-server/snapshot.js +43 -0
  8. package/dist/extensions/dashboard-server/state.js +30 -0
  9. package/dist/extensions/dashboard-server/types.js +5 -0
  10. package/dist/extensions/dashboard-server-s32.test.js +181 -0
  11. package/dist/extensions/dashboard-server.js +7 -1315
  12. package/dist/extensions/mega-commands.js +162 -134
  13. package/dist/extensions/mega-compact.js +3 -0
  14. package/dist/extensions/mega-compact.test.js +90 -21
  15. package/dist/extensions/mega-conflict-cmds.js +5 -1
  16. package/dist/extensions/mega-dashboard-cmds.js +29 -22
  17. package/dist/extensions/mega-db-cmds.js +11 -2
  18. package/dist/extensions/mega-events/agent-handlers.js +222 -0
  19. package/dist/extensions/mega-events/compact-handlers.js +162 -0
  20. package/dist/extensions/mega-events/context-handler.js +249 -0
  21. package/dist/extensions/mega-events/register.js +21 -0
  22. package/dist/extensions/mega-events/session-handlers.js +142 -0
  23. package/dist/extensions/mega-events.js +15 -699
  24. package/dist/extensions/mega-game-cmds.js +106 -0
  25. package/dist/extensions/mega-game-cmds.test.js +113 -0
  26. package/dist/extensions/mega-pipeline/compact.js +324 -0
  27. package/dist/extensions/mega-pipeline/memory-review.js +38 -0
  28. package/dist/extensions/mega-pipeline/recall.js +147 -0
  29. package/dist/extensions/mega-pipeline.js +9 -480
  30. package/dist/extensions/mega-runtime/helpers.js +40 -0
  31. package/dist/extensions/mega-runtime/query.js +29 -0
  32. package/dist/extensions/mega-runtime/state.js +877 -0
  33. package/dist/extensions/mega-runtime/state.test.js +171 -0
  34. package/dist/extensions/mega-runtime/widget.js +270 -0
  35. package/dist/extensions/mega-runtime/widget.test.js +160 -0
  36. package/dist/extensions/mega-runtime.js +15 -947
  37. package/dist/src/config/themes.js +84 -0
  38. package/dist/src/config/themes.test.js +94 -0
  39. package/dist/src/game/scoring.js +105 -0
  40. package/dist/src/game/scoring.test.js +98 -0
  41. package/dist/src/store/sqlite/checkpoints.js +145 -0
  42. package/dist/src/store/sqlite/dedup-mirror.js +64 -0
  43. package/dist/src/store/sqlite/foundation.js +38 -0
  44. package/dist/src/store/sqlite/game-achievements.js +111 -0
  45. package/dist/src/store/sqlite/game-achievements.test.js +67 -0
  46. package/dist/src/store/sqlite/game-scores.js +105 -0
  47. package/dist/src/store/sqlite/game-scores.test.js +106 -0
  48. package/dist/src/store/sqlite/game-state.js +54 -0
  49. package/dist/src/store/sqlite/game-state.test.js +76 -0
  50. package/dist/src/store/sqlite/global-index.js +224 -0
  51. package/dist/src/store/sqlite/maintenance.js +235 -0
  52. package/dist/src/store/sqlite/memories.js +164 -0
  53. package/dist/src/store/sqlite/meta.js +82 -0
  54. package/dist/src/store/sqlite/model-snapshots.js +47 -0
  55. package/dist/src/store/sqlite/raptor.js +57 -0
  56. package/dist/src/store/sqlite/raw-transcript.js +134 -0
  57. package/dist/src/store/sqlite/schema.js +294 -0
  58. package/dist/src/store/sqlite/session-state.js +28 -0
  59. package/dist/src/store/sqlite/stats.js +66 -0
  60. package/dist/src/store/sqlite/utils.js +120 -0
  61. package/dist/src/store/sqlite.js +23 -1607
  62. package/extensions/dashboard-server/html.test.ts +50 -0
  63. package/extensions/dashboard-server/html.ts +1026 -0
  64. package/extensions/dashboard-server/index-reader.ts +130 -0
  65. package/extensions/dashboard-server/server.test.ts +131 -0
  66. package/extensions/dashboard-server/server.ts +505 -0
  67. package/extensions/dashboard-server/snapshot.ts +44 -0
  68. package/extensions/dashboard-server/state.ts +33 -0
  69. package/extensions/dashboard-server/types.ts +134 -0
  70. package/extensions/dashboard-server-s32.test.ts +195 -0
  71. package/extensions/dashboard-server.ts +7 -1431
  72. package/extensions/mega-commands.ts +33 -10
  73. package/extensions/mega-compact.test.ts +198 -43
  74. package/extensions/mega-compact.ts +3 -0
  75. package/extensions/mega-conflict-cmds.ts +6 -2
  76. package/extensions/mega-dashboard-cmds.ts +30 -23
  77. package/extensions/mega-db-cmds.ts +11 -3
  78. package/extensions/mega-events/agent-handlers.ts +262 -0
  79. package/extensions/mega-events/compact-handlers.ts +192 -0
  80. package/extensions/mega-events/context-handler.ts +290 -0
  81. package/extensions/mega-events/register.ts +37 -0
  82. package/extensions/mega-events/session-handlers.ts +165 -0
  83. package/extensions/mega-events.ts +15 -780
  84. package/extensions/mega-game-cmds.test.ts +137 -0
  85. package/extensions/mega-game-cmds.ts +122 -0
  86. package/extensions/mega-pipeline/compact.ts +366 -0
  87. package/extensions/mega-pipeline/memory-review.ts +46 -0
  88. package/extensions/mega-pipeline/recall.ts +165 -0
  89. package/extensions/mega-pipeline.ts +9 -537
  90. package/extensions/mega-runtime/helpers.ts +68 -0
  91. package/extensions/mega-runtime/query.ts +29 -0
  92. package/extensions/mega-runtime/state.test.ts +171 -0
  93. package/extensions/mega-runtime/state.ts +967 -0
  94. package/extensions/mega-runtime/widget.test.ts +185 -0
  95. package/extensions/mega-runtime/widget.ts +359 -0
  96. package/extensions/mega-runtime.ts +15 -1093
  97. package/package.json +4 -3
  98. package/src/config/themes.test.ts +116 -0
  99. package/src/config/themes.ts +124 -0
  100. package/src/game/scoring.test.ts +103 -0
  101. package/src/game/scoring.ts +158 -0
  102. package/src/store/sqlite/checkpoints.ts +204 -0
  103. package/src/store/sqlite/dedup-mirror.ts +114 -0
  104. package/src/store/sqlite/foundation.ts +63 -0
  105. package/src/store/sqlite/game-achievements.test.ts +80 -0
  106. package/src/store/sqlite/game-achievements.ts +147 -0
  107. package/src/store/sqlite/game-scores.test.ts +132 -0
  108. package/src/store/sqlite/game-scores.ts +168 -0
  109. package/src/store/sqlite/game-state.test.ts +89 -0
  110. package/src/store/sqlite/game-state.ts +87 -0
  111. package/src/store/sqlite/global-index.ts +305 -0
  112. package/src/store/sqlite/maintenance.ts +294 -0
  113. package/src/store/sqlite/memories.ts +217 -0
  114. package/src/store/sqlite/meta.ts +108 -0
  115. package/src/store/sqlite/model-snapshots.ts +83 -0
  116. package/src/store/sqlite/raptor.ts +107 -0
  117. package/src/store/sqlite/raw-transcript.ts +221 -0
  118. package/src/store/sqlite/schema.ts +305 -0
  119. package/src/store/sqlite/session-state.ts +38 -0
  120. package/src/store/sqlite/stats.ts +127 -0
  121. package/src/store/sqlite/utils.ts +125 -0
  122. package/src/store/sqlite.ts +23 -2204
@@ -0,0 +1,224 @@
1
+ /**
2
+ * global-index.ts — machine-wide index DB (repo registry + injected-set).
3
+ *
4
+ * A single SQLite DB, separate from every per-repo store, that aggregates one
5
+ * row per repo this machine has run on. The multi-repo dashboard (Summary /
6
+ * All-repos tabs) reads it so ONE dashboard can show every repo's checkpoints,
7
+ * tokens saved, and active model — instead of a per-repo dashboard that only
8
+ * ever sees the repo it was launched from.
9
+ *
10
+ * Written by every pi process on repo-switch (bindRepo) + model capture; read by
11
+ * the dashboard server. Concurrency across 10+ pi processes is handled by WAL +
12
+ * infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
13
+ */
14
+ import { DatabaseSync } from "node:sqlite";
15
+ import { existsSync, mkdirSync } from "node:fs";
16
+ import { homedir, tmpdir } from "node:os";
17
+ import { join } from "node:path";
18
+ /** Resolve the machine-wide index directory (env-overridable). */
19
+ export function getIndexDir() {
20
+ const override = process.env.MEGACOMPACT_INDEX_DIR;
21
+ if (override && override.trim() !== "")
22
+ return override;
23
+ // homedir() can throw in exotic sandboxes; fall back to tmpdir.
24
+ try {
25
+ return join(homedir(), ".mega-compact-index");
26
+ }
27
+ catch {
28
+ return join(tmpdir(), ".mega-compact-index");
29
+ }
30
+ }
31
+ let indexCache;
32
+ let indexCacheDir;
33
+ /** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
34
+ export function openIndexStore(indexDir = getIndexDir()) {
35
+ if (indexCache && indexCacheDir === indexDir)
36
+ return indexCache;
37
+ if (!existsSync(indexDir))
38
+ mkdirSync(indexDir, { recursive: true });
39
+ const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
40
+ iddb.exec("PRAGMA journal_mode = WAL");
41
+ iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
42
+ iddb.exec(`
43
+ CREATE TABLE IF NOT EXISTS repo_registry (
44
+ repo_root TEXT PRIMARY KEY,
45
+ display_name TEXT,
46
+ state_dir TEXT NOT NULL,
47
+ first_seen INTEGER,
48
+ last_seen INTEGER,
49
+ last_compacted_at INTEGER,
50
+ checkpoint_count INTEGER DEFAULT 0,
51
+ tokens_saved INTEGER DEFAULT 0,
52
+ compressed_original_bytes INTEGER DEFAULT 0,
53
+ provider TEXT,
54
+ provider_name TEXT,
55
+ model_name TEXT,
56
+ input_rate REAL,
57
+ output_rate REAL,
58
+ model_captured_at INTEGER
59
+ );
60
+ CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
61
+ -- S18: machine-wide injected-set. A foreign checkpoint injected in repo A is
62
+ -- recorded here so repo B's recall never re-injects it. Keyed by checkpoint
63
+ -- + session (a checkpoint may be injected once per session); repo_id is the
64
+ -- source repo (the foreign repo's stateDir) for tracking/source labels.
65
+ -- PRAMETERIZED queries (PREVENT-002); local node:sqlite (PREVENT-PI-004).
66
+ CREATE TABLE IF NOT EXISTS injected_global (
67
+ checkpoint_id TEXT NOT NULL,
68
+ repo_id TEXT NOT NULL,
69
+ session_id TEXT NOT NULL,
70
+ injected_at INTEGER NOT NULL,
71
+ PRIMARY KEY (checkpoint_id, session_id)
72
+ );
73
+ CREATE INDEX IF NOT EXISTS idx_injected_global_cid ON injected_global(checkpoint_id);
74
+ `);
75
+ indexCache = iddb;
76
+ indexCacheDir = indexDir;
77
+ return iddb;
78
+ }
79
+ /**
80
+ * Upsert a repo's aggregate stats into the global index. Called on repo-switch
81
+ * (infrequent). Preserves first_seen + the model columns on update (model is
82
+ * written separately by recordRepoModel so we never clobber it here with nulls).
83
+ */
84
+ export function upsertRepoRegistry(row, indexDir = getIndexDir()) {
85
+ const db = openIndexStore(indexDir);
86
+ const now = Date.now();
87
+ db.prepare(`INSERT INTO repo_registry
88
+ (repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
89
+ checkpoint_count, tokens_saved, compressed_original_bytes,
90
+ provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
91
+ VALUES (@repo_root, @display_name, @state_dir, @first_seen, @last_seen, @last_compacted_at,
92
+ @checkpoint_count, @tokens_saved, @compressed_original_bytes,
93
+ @provider, @provider_name, @model_name, @input_rate, @output_rate, @model_captured_at)
94
+ ON CONFLICT(repo_root) DO UPDATE SET
95
+ display_name = excluded.display_name,
96
+ state_dir = excluded.state_dir,
97
+ last_seen = COALESCE(excluded.last_seen, @now),
98
+ last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
99
+ checkpoint_count = excluded.checkpoint_count,
100
+ tokens_saved = excluded.tokens_saved,
101
+ compressed_original_bytes = excluded.compressed_original_bytes,
102
+ provider = COALESCE(excluded.provider, repo_registry.provider),
103
+ provider_name = COALESCE(excluded.provider_name, repo_registry.provider_name),
104
+ model_name = COALESCE(excluded.model_name, repo_registry.model_name),
105
+ input_rate = COALESCE(excluded.input_rate, repo_registry.input_rate),
106
+ output_rate = COALESCE(excluded.output_rate, repo_registry.output_rate),
107
+ model_captured_at = COALESCE(excluded.model_captured_at, repo_registry.model_captured_at)`).run({
108
+ repo_root: row.repoRoot,
109
+ display_name: row.displayName,
110
+ state_dir: row.stateDir,
111
+ now,
112
+ first_seen: row.firstSeen ?? null,
113
+ last_seen: row.lastSeen ?? null,
114
+ last_compacted_at: row.lastCompactedAt ?? null,
115
+ checkpoint_count: row.checkpointCount,
116
+ tokens_saved: row.tokensSaved,
117
+ compressed_original_bytes: row.compressedOriginalBytes,
118
+ provider: row.provider ?? null,
119
+ provider_name: row.providerName ?? null,
120
+ model_name: row.modelName ?? null,
121
+ input_rate: row.inputRate ?? null,
122
+ output_rate: row.outputRate ?? null,
123
+ model_captured_at: row.modelCapturedAt ?? null,
124
+ });
125
+ }
126
+ /**
127
+ * Record the active model/provider for a repo in the global index (denormalized
128
+ * so the All-repos table shows model without opening each repo's DB). Upserts a
129
+ * bare registry row if the repo isn't registered yet.
130
+ */
131
+ export function recordRepoModel(repoRoot, model, indexDir = getIndexDir()) {
132
+ const db = openIndexStore(indexDir);
133
+ const now = Date.now();
134
+ db.prepare(`INSERT INTO repo_registry
135
+ (repo_root, display_name, state_dir, first_seen, last_seen,
136
+ provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
137
+ VALUES (@repo_root, @display_name, @state_dir, @now, @now,
138
+ @provider, @provider_name, @model_name, @input_rate, @output_rate, @now)
139
+ ON CONFLICT(repo_root) DO UPDATE SET
140
+ last_seen = excluded.last_seen,
141
+ provider = excluded.provider,
142
+ provider_name = excluded.provider_name,
143
+ model_name = excluded.model_name,
144
+ input_rate = excluded.input_rate,
145
+ output_rate = excluded.output_rate,
146
+ model_captured_at = excluded.model_captured_at`).run({
147
+ repo_root: repoRoot,
148
+ display_name: model.displayName,
149
+ state_dir: model.stateDir,
150
+ now,
151
+ provider: model.provider,
152
+ provider_name: model.providerName,
153
+ model_name: model.modelName,
154
+ input_rate: model.inputRate,
155
+ output_rate: model.outputRate,
156
+ });
157
+ }
158
+ function mapRegistryRow(row) {
159
+ return {
160
+ repoRoot: row.repo_root,
161
+ displayName: row.display_name ?? "",
162
+ stateDir: row.state_dir,
163
+ firstSeen: row.first_seen ?? 0,
164
+ lastSeen: row.last_seen ?? 0,
165
+ lastCompactedAt: row.last_compacted_at ?? null,
166
+ checkpointCount: row.checkpoint_count ?? 0,
167
+ tokensSaved: row.tokens_saved ?? 0,
168
+ compressedOriginalBytes: row.compressed_original_bytes ?? 0,
169
+ provider: row.provider ?? null,
170
+ providerName: row.provider_name ?? null,
171
+ modelName: row.model_name ?? null,
172
+ inputRate: row.input_rate ?? null,
173
+ outputRate: row.output_rate ?? null,
174
+ modelCapturedAt: row.model_captured_at ?? null,
175
+ };
176
+ }
177
+ /** All registered repos, most-recently-seen first. */
178
+ export function listRepoRegistry(indexDir = getIndexDir()) {
179
+ const db = openIndexStore(indexDir);
180
+ const rows = db.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC").all();
181
+ return rows.map(mapRegistryRow);
182
+ }
183
+ /** A single repo's registry row, or undefined. */
184
+ export function getRepoRegistry(repoRoot, indexDir = getIndexDir()) {
185
+ const db = openIndexStore(indexDir);
186
+ const row = db.prepare("SELECT * FROM repo_registry WHERE repo_root = ?").get(repoRoot);
187
+ return row ? mapRegistryRow(row) : undefined;
188
+ }
189
+ /** Close the cached index connection (test teardown only). */
190
+ export function closeIndexStore() {
191
+ if (indexCache) {
192
+ indexCache.close();
193
+ indexCache = undefined;
194
+ indexCacheDir = undefined;
195
+ }
196
+ }
197
+ // ---------------------------------------------------------------------------
198
+ // S18: machine-wide injected-set (cross-repo dedup markers)
199
+ //
200
+ // A foreign checkpoint injected in repo A is recorded here so repo B's recall
201
+ // never re-injects it (a stronger, machine-wide version of the per-session
202
+ // injected-set in the local store). Keyed by (checkpoint_id, session_id); the
203
+ // session_id here is the RECEIVING session, so the same foreign checkpoint can
204
+ // be injected into different sessions but never twice into the same one.
205
+ // PRAMETERIZED queries (PREVENT-002); local node:sqlite + WAL (PREVENT-PI-004),
206
+ // multi-process safe.
207
+ // ---------------------------------------------------------------------------
208
+ /** Record that a (foreign) checkpoint was injected into `sessionId`. Idempotent. */
209
+ export function markInjectedGlobal(checkpointId, repoId, sessionId, indexDir = getIndexDir()) {
210
+ const db = openIndexStore(indexDir);
211
+ db.prepare("INSERT OR IGNORE INTO injected_global (checkpoint_id, repo_id, session_id, injected_at) VALUES ($cid, $rid, $sid, $ts)").run({ $cid: checkpointId, $rid: repoId, $sid: sessionId, $ts: Date.now() });
212
+ }
213
+ /** True when a checkpoint was already injected into `sessionId` (machine-wide). */
214
+ export function wasInjectedGlobal(checkpointId, sessionId, indexDir = getIndexDir()) {
215
+ const db = openIndexStore(indexDir);
216
+ const row = db.prepare("SELECT 1 FROM injected_global WHERE checkpoint_id = $cid AND session_id = $sid LIMIT 1").get({ $cid: checkpointId, $sid: sessionId });
217
+ return row !== undefined;
218
+ }
219
+ /** Count of cross-repo injections recorded (for /mega-status stats). */
220
+ export function countInjectedGlobal(indexDir = getIndexDir()) {
221
+ const db = openIndexStore(indexDir);
222
+ const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get();
223
+ return row?.n ?? 0;
224
+ }
@@ -0,0 +1,235 @@
1
+ /**
2
+ * maintenance.ts — S27 Task 10 DB maintenance / housekeeping primitives.
3
+ *
4
+ * All pi-agnostic, all parameterized (PREVENT-002), all local (PREVENT-PI-004).
5
+ * Exposed via the /mega-db-* slash commands in extensions/mega-db-cmds.ts.
6
+ */
7
+ import { statSync } from "node:fs";
8
+ import { join } from "node:path";
9
+ import { getStateDir } from "../../store.js";
10
+ import { openStore, withTx } from "./utils.js";
11
+ const DB_TABLE_NAMES = [
12
+ "context_chunks",
13
+ "session_state",
14
+ "raw_transcript",
15
+ "checkpoint_epochs",
16
+ "dedup_mirror",
17
+ "memories",
18
+ "dedup_stats",
19
+ "daily_log",
20
+ ];
21
+ function fileSizeIfExists(path) {
22
+ try {
23
+ const st = statSync(path);
24
+ return st.size;
25
+ }
26
+ catch {
27
+ return 0;
28
+ }
29
+ }
30
+ /**
31
+ * Gather DB stats for /mega-db-stats: per-table row counts, disk footprint
32
+ * (main + WAL + SHM), page count, freelist, WAL frame count.
33
+ *
34
+ * Read-only: no PRAGMA writes, no VACUUM. Safe to call any time.
35
+ */
36
+ export function getDbStats(stateDir = getStateDir()) {
37
+ const db = openStore(stateDir);
38
+ const tableCounts = {};
39
+ for (const t of DB_TABLE_NAMES) {
40
+ try {
41
+ const row = db.prepare(`SELECT COUNT(*) AS c FROM ${t}`).get();
42
+ if (row)
43
+ tableCounts[t] = row.c;
44
+ }
45
+ catch {
46
+ // Table doesn't exist on this DB (e.g. raw_transcript on a pre-S27 store).
47
+ // Skip silently — /mega-db-stats lists only tables that exist.
48
+ }
49
+ }
50
+ const pageStat = db.prepare("PRAGMA page_count").get();
51
+ const freelistStat = db.prepare("PRAGMA freelist_count").get();
52
+ const pageSizeStat = db.prepare("PRAGMA page_size").get();
53
+ let walFrames = 0;
54
+ try {
55
+ const walInfo = db.prepare("PRAGMA wal_info").get();
56
+ walFrames = walInfo?.frames ?? 0;
57
+ }
58
+ catch {
59
+ // node:sqlite may not expose wal_info on all versions; not fatal.
60
+ }
61
+ const dbPath = join(stateDir, "sqlite.db");
62
+ return {
63
+ tableCounts,
64
+ dbBytes: fileSizeIfExists(dbPath),
65
+ walBytes: fileSizeIfExists(`${dbPath}-wal`),
66
+ shmBytes: fileSizeIfExists(`${dbPath}-shm`),
67
+ pageSize: pageSizeStat?.page_size ?? 0,
68
+ pageCount: pageStat?.page_count ?? 0,
69
+ freelistPages: freelistStat?.freelist_count ?? 0,
70
+ walFrames,
71
+ };
72
+ }
73
+ /**
74
+ * Prune raw_transcript + checkpoint_epochs rows older than `daysOld`.
75
+ * Uses `message_timestamp` (raw_transcript) and `created_at` (epochs), both
76
+ * epoch-ms. Returns the total deleted rows + reclaimed disk bytes.
77
+ *
78
+ * PREVENT-002: parameterized. PREVENT-PI-004: local SQLite only.
79
+ */
80
+ export function pruneOldRows(stateDir = getStateDir(), daysOld = 30) {
81
+ const db = openStore(stateDir);
82
+ const cutoff = Date.now() - daysOld * 86_400_000;
83
+ const beforeBytes = fileSizeIfExists(join(stateDir, "sqlite.db"));
84
+ // raw_transcript: message_timestamp may be NULL (pre-S27 rows); those use
85
+ // the row's insertion order implicitly via seq, so we prune NULL-ts rows
86
+ // only when the whole session is older than the cutoff (join via session_id
87
+ // to checkpoint_epochs.created_at). Simpler: prune NULL-ts rows older than
88
+ // cutoff by falling back to the MIN(created_at) of their epoch.
89
+ // Delete raw_transcript rows whose message_timestamp is older than cutoff,
90
+ // OR whose message_timestamp is NULL and the session's latest epoch is older.
91
+ const delRt = db.prepare(`DELETE FROM raw_transcript
92
+ WHERE message_timestamp IS NOT NULL AND message_timestamp < ?
93
+ OR (message_timestamp IS NULL
94
+ AND session_id IN (
95
+ SELECT session_id FROM checkpoint_epochs
96
+ GROUP BY session_id HAVING MAX(created_at) < ?
97
+ ))`).run(cutoff, cutoff);
98
+ const rtDeleted = delRt?.changes ?? 0;
99
+ // checkpoint_epochs: created_at is NOT NULL.
100
+ const delEp = db.prepare(`DELETE FROM checkpoint_epochs WHERE created_at < ?`).run(cutoff);
101
+ const epDeleted = delEp?.changes ?? 0;
102
+ // dedup_mirror: cascade-delete orphan rows whose ref_count has dropped to 0
103
+ // after the raw_transcript deletes. Safe even if FK is off (raw_transcript has
104
+ // no FK to dedup_mirror; ref_count is maintained by the dedup pipeline).
105
+ const delDedup = db.prepare(`DELETE FROM dedup_mirror WHERE ref_count <= 0`).run();
106
+ const dedupDeleted = delDedup?.changes ?? 0;
107
+ const afterBytes = fileSizeIfExists(join(stateDir, "sqlite.db"));
108
+ const total = rtDeleted + epDeleted + dedupDeleted;
109
+ return {
110
+ affected: total,
111
+ reclaimedBytes: Math.max(0, beforeBytes - afterBytes),
112
+ summary: `pruned ${rtDeleted} raw_transcript + ${epDeleted} epochs + ${dedupDeleted} dedup_mirror rows older than ${daysOld}d`,
113
+ };
114
+ }
115
+ /**
116
+ * Force a WAL checkpoint (TRUNCATE mode) so the -wal sidecar is reclaimed.
117
+ * Returns the WAL bytes reclaimed (pre-wal size minus post-wal size).
118
+ */
119
+ export function checkpointWal(stateDir = getStateDir()) {
120
+ const db = openStore(stateDir);
121
+ const dbPath = join(stateDir, "sqlite.db");
122
+ const beforeWal = fileSizeIfExists(`${dbPath}-wal`);
123
+ // PRAGMA wal_checkpoint(TRUNCATE) blocks until all frames are folded into the
124
+ // main db and the WAL file is truncated to 0 bytes.
125
+ const res = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
126
+ const afterWal = fileSizeIfExists(`${dbPath}-wal`);
127
+ const reclaimed = Math.max(0, beforeWal - afterWal);
128
+ return {
129
+ affected: res?.checkpointed ?? 0,
130
+ reclaimedBytes: reclaimed,
131
+ summary: `wal_checkpoint(TRUNCATE): ${res?.checkpointed ?? 0} frames folded, WAL ${beforeWal}→${afterWal} bytes${res?.busy ? " (busy: " + res.busy + ")" : ""}`,
132
+ };
133
+ }
134
+ /**
135
+ * VACUUM the main DB file (rebuilds pages, reclaims freelist space).
136
+ * Heavy: briefly doubles disk usage. Run only when freelist is large or the
137
+ * user explicitly invokes /mega-db-vacuum.
138
+ */
139
+ export function vacuumDb(stateDir = getStateDir()) {
140
+ const db = openStore(stateDir);
141
+ const dbPath = join(stateDir, "sqlite.db");
142
+ const beforeBytes = fileSizeIfExists(dbPath);
143
+ db.exec("VACUUM"); // VACUUM cannot be parameterized; it rewrites the whole DB.
144
+ const afterBytes = fileSizeIfExists(dbPath);
145
+ const reclaimed = Math.max(0, beforeBytes - afterBytes);
146
+ return {
147
+ affected: 0,
148
+ reclaimedBytes: reclaimed,
149
+ summary: `VACUUM: db ${beforeBytes}→${afterBytes} bytes (reclaimed ${reclaimed})`,
150
+ };
151
+ }
152
+ /**
153
+ * Run `PRAGMA integrity_check` and return the result lines.
154
+ * Returns ["ok"] when the DB is healthy; otherwise returns the error lines.
155
+ */
156
+ export function integrityCheck(stateDir = getStateDir()) {
157
+ const db = openStore(stateDir);
158
+ const rows = db.prepare("PRAGMA integrity_check").all();
159
+ return (rows ?? []).map((r) => r.integrity_check);
160
+ }
161
+ /**
162
+ * Reconcile dedup_mirror vs raw_transcript after pruning or crashes:
163
+ * 1. Recompute ref_count = COUNT(raw_transcript rows pointing at this hash).
164
+ * 2. Delete orphan dedup_mirror rows whose recomputed ref_count is 0.
165
+ * 3. Backfill raw_transcript.content_ref for rows still storing inline bytes.
166
+ *
167
+ * Idempotent. Read-modify-write within a single transaction (withTx).
168
+ */
169
+ export function reconcileDedupMirror(stateDir = getStateDir()) {
170
+ const db = openStore(stateDir);
171
+ const result = { fixedRefCount: 0, orphansDeleted: 0, refsBackfilled: 0 };
172
+ withTx(db, () => {
173
+ // 1. Recompute ref_count for every dedup_mirror row from the actual
174
+ // raw_transcript references.
175
+ const recompute = db.prepare(`UPDATE dedup_mirror AS dm
176
+ SET ref_count = COALESCE((
177
+ SELECT COUNT(*) FROM raw_transcript rt WHERE rt.content_ref = dm.content_hash
178
+ ), 0)
179
+ WHERE dm.ref_count != COALESCE((
180
+ SELECT COUNT(*) FROM raw_transcript rt WHERE rt.content_ref = dm.content_hash
181
+ ), 0)`).run();
182
+ result.fixedRefCount = recompute?.changes ?? 0;
183
+ // 2. Delete orphan dedup_mirror rows (no raw_transcript refs).
184
+ const delOrphans = db.prepare(`DELETE FROM dedup_mirror
185
+ WHERE content_hash NOT IN (SELECT DISTINCT content_ref FROM raw_transcript WHERE content_ref IS NOT NULL)`).run();
186
+ result.orphansDeleted = delOrphans?.changes ?? 0;
187
+ // 3. Backfill content_ref for rows still storing inline content_bytes (no
188
+ // ref yet). Only safe when a matching dedup_mirror row exists; otherwise
189
+ // we'd need to insert one, which is the dedup pipeline's job, not the
190
+ // reconciler's.
191
+ const backfill = db.prepare(`UPDATE raw_transcript AS rt
192
+ SET content_ref = (
193
+ SELECT dm.content_hash FROM dedup_mirror dm WHERE dm.content_bytes = rt.content_bytes
194
+ )
195
+ WHERE rt.content_ref IS NULL
196
+ AND EXISTS (SELECT 1 FROM dedup_mirror dm WHERE dm.content_bytes = rt.content_bytes)`).run();
197
+ result.refsBackfilled = backfill?.changes ?? 0;
198
+ });
199
+ return result;
200
+ }
201
+ /**
202
+ * One-shot auto-maintenance pass for the session_start hook: prune old rows,
203
+ * checkpoint the WAL if it's grown large, and (only if the DB is huge) VACUUM.
204
+ * Best-effort: swallows errors so a session never fails to start over a
205
+ * housekeeping hiccup. Returns a short summary for the diagnostic log.
206
+ */
207
+ export function autoMaintain(stateDir = getStateDir()) {
208
+ try {
209
+ const stats = getDbStats(stateDir);
210
+ const parts = [];
211
+ // Prune rows older than 30d (default retention).
212
+ const prune = pruneOldRows(stateDir, 30);
213
+ if (prune.affected > 0)
214
+ parts.push(`pruned ${prune.affected}`);
215
+ // Checkpoint the WAL if it's over 10 MB (avoid pathological WAL growth).
216
+ if (stats.walBytes > 10 * 1024 * 1024) {
217
+ const ck = checkpointWal(stateDir);
218
+ if (ck.reclaimedBytes > 0)
219
+ parts.push(`wal -${ck.reclaimedBytes}B`);
220
+ }
221
+ // VACUUM only if the DB is over 100 MB AND freelist is >20% of pages.
222
+ if (stats.dbBytes > 100 * 1024 * 1024 &&
223
+ stats.pageCount > 0 &&
224
+ stats.freelistPages / stats.pageCount > 0.2) {
225
+ const v = vacuumDb(stateDir);
226
+ if (v.reclaimedBytes > 0)
227
+ parts.push(`vacuum -${v.reclaimedBytes}B`);
228
+ }
229
+ return parts.length ? `auto-maintain: ${parts.join(", ")}` : "auto-maintain: nothing to do";
230
+ }
231
+ catch (err) {
232
+ // Never block session start over housekeeping.
233
+ return `auto-maintain: skipped (${err.message})`;
234
+ }
235
+ }
@@ -0,0 +1,164 @@
1
+ /**
2
+ * memories.ts — durable "save to memory" store (taken over from memory extensions).
3
+ *
4
+ * One SQLite store for user-saved memories, scoped by repo. Mirrors the
5
+ * lessons/sessions pattern: all state lives in SQLite from day one.
6
+ */
7
+ import { getStateDir } from "../../store.js";
8
+ import { openStore } from "./utils.js";
9
+ // S24 storage hardening: keep each memory row bounded so the durable store can
10
+ // never blow a downstream consumer's per-entry buffer (e.g. pi's native
11
+ // file-backed memory caps a single entry at ~5k chars). We truncate content at
12
+ // MEMORY_MAX_CHARS and evict the least-recently-referenced rows past
13
+ // MEMORY_MAX_ROWS per repo via LRU. Both are SQLite-only (PREVENT-PI-004): no
14
+ // file-backed memory is written anywhere. Defaults are overridable via env
15
+ // (MEGACOMPACT_MEMORY_MAX_CHARS / MEGACOMPACT_MEMORY_MAX_ROWS).
16
+ export const MEMORY_MAX_CHARS = 4000;
17
+ export const MEMORY_MAX_ROWS = 500;
18
+ /** Read an env override as a positive int, falling back to `fallback`. */
19
+ function envInt(name, fallback) {
20
+ const v = process.env[name];
21
+ if (v == null || v === "")
22
+ return fallback;
23
+ const n = Number(v);
24
+ return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
25
+ }
26
+ /** Effective per-entry char cap (env-overridable, default MEMORY_MAX_CHARS). */
27
+ export function memoryMaxChars() {
28
+ return envInt("MEGACOMPACT_MEMORY_MAX_CHARS", MEMORY_MAX_CHARS);
29
+ }
30
+ /** Effective per-repo row cap (env-overridable, default MEMORY_MAX_ROWS). */
31
+ export function memoryMaxRows() {
32
+ return envInt("MEGACOMPACT_MEMORY_MAX_ROWS", MEMORY_MAX_ROWS);
33
+ }
34
+ /** Truncate memory content to the per-entry cap, preserving a trailing marker. */
35
+ function capMemoryContent(content) {
36
+ const cap = memoryMaxChars();
37
+ if (content.length <= cap)
38
+ return content;
39
+ return content.slice(0, cap) + "…[truncated]";
40
+ }
41
+ /**
42
+ * Evict the least-recently-referenced rows for a repo past MEMORY_MAX_ROWS.
43
+ * LRU key = COALESCE(last_referenced, last_recalled_at, created_at) so a memory
44
+ * that is recalled/referenced survives over a stale one. Best-effort: any error
45
+ * is swallowed by the caller. Repo-scoped so one noisy repo can't evict another.
46
+ */
47
+ function evictMemoryLru(repo, stateDir) {
48
+ const db = openStore(stateDir);
49
+ const maxRows = memoryMaxRows();
50
+ // SQLite `= NULL` is never true, so the null-repo scope (memories are
51
+ // stateDir-scoped when repo is null — the applyMemoryOps path) needs `IS NULL`.
52
+ const where = repo == null ? "repo IS NULL" : "repo = ?";
53
+ const countRow = repo == null
54
+ ? db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get()
55
+ : db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get(repo);
56
+ const count = countRow.n;
57
+ const over = count - maxRows;
58
+ if (over <= 0)
59
+ return;
60
+ // Delete the `over` least-recently-used rows. ORDER BY the LRU key ASC, id ASC
61
+ // (id ASC breaks ties deterministically — oldest created first). The `where`
62
+ // clause is a code-controlled constant (never user input) → PREVENT-002 OK.
63
+ const sql = `DELETE FROM memories WHERE ${where} AND id IN (
64
+ SELECT id FROM memories WHERE ${where}
65
+ ORDER BY COALESCE(last_referenced, last_recalled_at, created_at) ASC, id ASC
66
+ LIMIT ?
67
+ )`;
68
+ if (repo == null)
69
+ db.prepare(sql).run(over);
70
+ else
71
+ db.prepare(sql).run(repo, repo, over);
72
+ }
73
+ /** Save a memory to the current repo's store. Returns the new row id.
74
+ * S24 hardening: content is truncated to MEMORY_MAX_CHARS and, once the per-repo
75
+ * row count exceeds MEMORY_MAX_ROWS, the least-recently-used rows are evicted
76
+ * (LRU) so the store stays bounded. */
77
+ export function addMemory(memory, repo, stateDir = getStateDir()) {
78
+ const db = openStore(stateDir);
79
+ const now = Math.floor(Date.now() / 1000);
80
+ const res = db
81
+ .prepare(`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at, category, target, source_turn)
82
+ VALUES(?, ?, ?, ?, ?, NULL, ?, ?, ?)`)
83
+ .run(repo ?? null, memory.kind ?? "note", capMemoryContent(memory.content), JSON.stringify(memory.tags ?? []), now, memory.category ?? null, memory.target ?? null, memory.sourceTurn ?? null);
84
+ try {
85
+ evictMemoryLru(repo, stateDir);
86
+ }
87
+ catch {
88
+ /* non-fatal: eviction must never fail an add */
89
+ }
90
+ return Number(res.lastInsertRowid);
91
+ }
92
+ /** List recent memories for a repo (or all repos when repo is null). */
93
+ export function listMemories(repo, limit = 50, stateDir = getStateDir()) {
94
+ const db = openStore(stateDir);
95
+ const rows = repo
96
+ ? db.prepare("SELECT * FROM memories WHERE repo = ? ORDER BY created_at DESC LIMIT ?").all(repo, limit)
97
+ : db.prepare("SELECT * FROM memories ORDER BY created_at DESC LIMIT ?").all(limit);
98
+ return rows.map(mapMemoryRow);
99
+ }
100
+ /** Substring search across content + tags. */
101
+ export function searchMemories(query, repo = null, limit = 50, stateDir = getStateDir()) {
102
+ const db = openStore(stateDir);
103
+ const like = `%${query}%`;
104
+ const rows = repo
105
+ ? db.prepare("SELECT * FROM memories WHERE repo = ? AND (content LIKE ? OR tags LIKE ?) ORDER BY created_at DESC LIMIT ?").all(repo, like, like, limit)
106
+ : db.prepare("SELECT * FROM memories WHERE content LIKE ? OR tags LIKE ? ORDER BY created_at DESC LIMIT ?").all(like, like, limit);
107
+ return rows.map(mapMemoryRow);
108
+ }
109
+ /** Mark a memory as recalled (updates last_recalled_at). Returns true if found. */
110
+ export function recallMemory(id, stateDir = getStateDir()) {
111
+ const db = openStore(stateDir);
112
+ const now = Math.floor(Date.now() / 1000);
113
+ const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
114
+ return res.changes > 0;
115
+ }
116
+ /** Mark a memory as referenced (updates last_referenced). Returns true if found. */
117
+ export function referenceMemory(id, stateDir = getStateDir()) {
118
+ const db = openStore(stateDir);
119
+ const now = Math.floor(Date.now() / 1000);
120
+ const res = db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(now, id);
121
+ return res.changes > 0;
122
+ }
123
+ /** Replace a memory's mutable fields by id. Returns true if a row was updated. */
124
+ export function replaceMemory(id, patch, stateDir = getStateDir()) {
125
+ const db = openStore(stateDir);
126
+ const res = db
127
+ .prepare(`UPDATE memories
128
+ SET kind = COALESCE(?, kind),
129
+ content = COALESCE(?, content),
130
+ tags = COALESCE(?, tags),
131
+ category = COALESCE(?, category),
132
+ target = COALESCE(?, target),
133
+ source_turn = COALESCE(?, source_turn)
134
+ WHERE id = ?`)
135
+ .run(patch.kind ?? null, patch.content != null ? capMemoryContent(patch.content) : null, patch.tags ? JSON.stringify(patch.tags) : null, "category" in patch ? (patch.category ?? null) : null, "target" in patch ? (patch.target ?? null) : null, "sourceTurn" in patch ? (patch.sourceTurn ?? null) : null, id);
136
+ return res.changes > 0;
137
+ }
138
+ /** Remove a memory by id. Returns true if a row was deleted. */
139
+ export function removeMemory(id, stateDir = getStateDir()) {
140
+ const db = openStore(stateDir);
141
+ const res = db.prepare("DELETE FROM memories WHERE id = ?").run(id);
142
+ return res.changes > 0;
143
+ }
144
+ /** Look up a single memory by id (or undefined). */
145
+ export function getMemory(id, stateDir = getStateDir()) {
146
+ const db = openStore(stateDir);
147
+ const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
148
+ return row ? mapMemoryRow(row) : undefined;
149
+ }
150
+ function mapMemoryRow(row) {
151
+ return {
152
+ id: row.id,
153
+ repo: row.repo ?? null,
154
+ kind: row.kind ?? "note",
155
+ content: row.content ?? "",
156
+ tags: row.tags ? JSON.parse(row.tags) : [],
157
+ createdAt: row.created_at ?? 0,
158
+ lastRecalledAt: row.last_recalled_at ?? null,
159
+ category: row.category ?? null,
160
+ target: row.target ?? null,
161
+ lastReferenced: row.last_referenced ?? null,
162
+ sourceTurn: row.source_turn ?? null,
163
+ };
164
+ }