pi-mega-compact 0.7.8 → 0.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -12
- package/dist/extensions/dashboard-server/helpers.js +37 -0
- package/dist/extensions/dashboard-server/html/all-repos-tab.js +26 -0
- package/dist/extensions/dashboard-server/html/body-open.js +23 -0
- package/dist/extensions/dashboard-server/html/current-repo-tab.js +130 -0
- package/dist/extensions/dashboard-server/html/head-open.js +16 -0
- package/dist/extensions/dashboard-server/html/high-score-tab.js +25 -0
- package/dist/extensions/dashboard-server/html/repo-detail-modal.js +26 -0
- package/dist/extensions/dashboard-server/html/script.js +259 -0
- package/dist/extensions/dashboard-server/html/styles.js +103 -0
- package/dist/extensions/dashboard-server/html/summary-tab.js +19 -0
- package/dist/extensions/dashboard-server/html-template.js +41 -0
- package/dist/extensions/dashboard-server/html.js +756 -0
- package/dist/extensions/dashboard-server/index-reader.js +133 -0
- package/dist/extensions/dashboard-server/server.js +370 -0
- package/dist/extensions/dashboard-server/snapshot.js +43 -0
- package/dist/extensions/dashboard-server/state.js +30 -0
- package/dist/extensions/dashboard-server/types.js +5 -0
- package/dist/extensions/dashboard-server.js +7 -1315
- package/dist/extensions/mega-commands.js +162 -134
- package/dist/extensions/mega-compact.test.js +90 -21
- package/dist/extensions/mega-conflict-cmds.js +5 -1
- package/dist/extensions/mega-dashboard-cmds.js +29 -22
- package/dist/extensions/mega-db-cmds.js +11 -2
- package/dist/extensions/mega-events/agent-handlers.js +173 -0
- package/dist/extensions/mega-events/compact-handlers.js +133 -0
- package/dist/extensions/mega-events/context-handler.js +249 -0
- package/dist/extensions/mega-events/register.js +21 -0
- package/dist/extensions/mega-events/session-handlers.js +142 -0
- package/dist/extensions/mega-events.js +15 -699
- package/dist/extensions/mega-pipeline/compact.js +324 -0
- package/dist/extensions/mega-pipeline/memory-review.js +38 -0
- package/dist/extensions/mega-pipeline/recall.js +147 -0
- package/dist/extensions/mega-pipeline.js +9 -480
- package/dist/extensions/mega-runtime/helpers.js +40 -0
- package/dist/extensions/mega-runtime/query.js +29 -0
- package/dist/extensions/mega-runtime/state.js +711 -0
- package/dist/extensions/mega-runtime/widget.js +197 -0
- package/dist/extensions/mega-runtime.js +15 -947
- package/dist/src/store/sqlite/checkpoints.js +145 -0
- package/dist/src/store/sqlite/connection.js +35 -0
- package/dist/src/store/sqlite/dedup-mirror.js +64 -0
- package/dist/src/store/sqlite/foundation.js +38 -0
- package/dist/src/store/sqlite/global-index.js +224 -0
- package/dist/src/store/sqlite/index-store.js +167 -0
- package/dist/src/store/sqlite/maintenance.js +235 -0
- package/dist/src/store/sqlite/memories.js +164 -0
- package/dist/src/store/sqlite/memory.js +54 -0
- package/dist/src/store/sqlite/meta.js +82 -0
- package/dist/src/store/sqlite/minhash-lsh.js +47 -0
- package/dist/src/store/sqlite/model-snapshots.js +47 -0
- package/dist/src/store/sqlite/raptor.js +57 -0
- package/dist/src/store/sqlite/raw-transcript.js +134 -0
- package/dist/src/store/sqlite/schema.js +250 -0
- package/dist/src/store/sqlite/session-state.js +28 -0
- package/dist/src/store/sqlite/sessions.js +39 -0
- package/dist/src/store/sqlite/stats.js +66 -0
- package/dist/src/store/sqlite/transaction.js +19 -0
- package/dist/src/store/sqlite/utils.js +120 -0
- package/dist/src/store/sqlite.js +20 -1607
- package/dist/src/vectorStore/add.js +260 -0
- package/dist/src/vectorStore/dedup.js +52 -0
- package/dist/src/vectorStore/index.js +10 -0
- package/dist/src/vectorStore/queries.js +83 -0
- package/dist/src/vectorStore/search.js +95 -0
- package/dist/src/vectorStore/session.js +19 -0
- package/dist/src/vectorStore/store.js +105 -0
- package/dist/src/vectorStore/types.js +6 -0
- package/dist/src/vectorStore/utils.js +23 -0
- package/extensions/dashboard-server/html.ts +758 -0
- package/extensions/dashboard-server/index-reader.ts +130 -0
- package/extensions/dashboard-server/server.ts +358 -0
- package/extensions/dashboard-server/snapshot.ts +44 -0
- package/extensions/dashboard-server/state.ts +33 -0
- package/extensions/dashboard-server/types.ts +134 -0
- package/extensions/dashboard-server.ts +7 -1431
- package/extensions/mega-commands.ts +33 -10
- package/extensions/mega-compact.test.ts +198 -43
- package/extensions/mega-conflict-cmds.ts +6 -2
- package/extensions/mega-dashboard-cmds.ts +30 -23
- package/extensions/mega-db-cmds.ts +11 -3
- package/extensions/mega-events/agent-handlers.ts +214 -0
- package/extensions/mega-events/compact-handlers.ts +164 -0
- package/extensions/mega-events/context-handler.ts +290 -0
- package/extensions/mega-events/register.ts +37 -0
- package/extensions/mega-events/session-handlers.ts +165 -0
- package/extensions/mega-events.ts +15 -780
- package/extensions/mega-pipeline/compact.ts +366 -0
- package/extensions/mega-pipeline/memory-review.ts +46 -0
- package/extensions/mega-pipeline/recall.ts +165 -0
- package/extensions/mega-pipeline.ts +9 -537
- package/extensions/mega-runtime/helpers.ts +68 -0
- package/extensions/mega-runtime/query.ts +29 -0
- package/extensions/mega-runtime/state.ts +797 -0
- package/extensions/mega-runtime/widget.ts +258 -0
- package/extensions/mega-runtime.ts +15 -1093
- package/package.json +4 -3
- package/src/store/sqlite/checkpoints.ts +204 -0
- package/src/store/sqlite/dedup-mirror.ts +114 -0
- package/src/store/sqlite/foundation.ts +63 -0
- package/src/store/sqlite/global-index.ts +305 -0
- package/src/store/sqlite/maintenance.ts +294 -0
- package/src/store/sqlite/memories.ts +217 -0
- package/src/store/sqlite/meta.ts +108 -0
- package/src/store/sqlite/model-snapshots.ts +83 -0
- package/src/store/sqlite/raptor.ts +107 -0
- package/src/store/sqlite/raw-transcript.ts +221 -0
- package/src/store/sqlite/schema.ts +258 -0
- package/src/store/sqlite/session-state.ts +38 -0
- package/src/store/sqlite/stats.ts +127 -0
- package/src/store/sqlite/utils.ts +125 -0
- package/src/store/sqlite.ts +20 -2204
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* global-index.ts — machine-wide index DB (repo registry + injected-set).
|
|
3
|
+
*
|
|
4
|
+
* A single SQLite DB, separate from every per-repo store, that aggregates one
|
|
5
|
+
* row per repo this machine has run on. The multi-repo dashboard (Summary /
|
|
6
|
+
* All-repos tabs) reads it so ONE dashboard can show every repo's checkpoints,
|
|
7
|
+
* tokens saved, and active model — instead of a per-repo dashboard that only
|
|
8
|
+
* ever sees the repo it was launched from.
|
|
9
|
+
*
|
|
10
|
+
* Written by every pi process on repo-switch (bindRepo) + model capture; read by
|
|
11
|
+
* the dashboard server. Concurrency across 10+ pi processes is handled by WAL +
|
|
12
|
+
* infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
|
|
13
|
+
*/
|
|
14
|
+
import { DatabaseSync } from "node:sqlite";
|
|
15
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
16
|
+
import { homedir, tmpdir } from "node:os";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
|
|
19
|
+
/** Resolve the machine-wide index directory (env-overridable). */
|
|
20
|
+
export function getIndexDir(): string {
|
|
21
|
+
const override = process.env.MEGACOMPACT_INDEX_DIR;
|
|
22
|
+
if (override && override.trim() !== "") return override;
|
|
23
|
+
// homedir() can throw in exotic sandboxes; fall back to tmpdir.
|
|
24
|
+
try {
|
|
25
|
+
return join(homedir(), ".mega-compact-index");
|
|
26
|
+
} catch {
|
|
27
|
+
return join(tmpdir(), ".mega-compact-index");
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
let indexCache: DatabaseSync | undefined;
|
|
32
|
+
let indexCacheDir: string | undefined;
|
|
33
|
+
|
|
34
|
+
/** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
|
|
35
|
+
export function openIndexStore(indexDir: string = getIndexDir()): DatabaseSync {
|
|
36
|
+
if (indexCache && indexCacheDir === indexDir) return indexCache;
|
|
37
|
+
if (!existsSync(indexDir)) mkdirSync(indexDir, { recursive: true });
|
|
38
|
+
const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
|
|
39
|
+
iddb.exec("PRAGMA journal_mode = WAL");
|
|
40
|
+
iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
|
|
41
|
+
iddb.exec(`
|
|
42
|
+
CREATE TABLE IF NOT EXISTS repo_registry (
|
|
43
|
+
repo_root TEXT PRIMARY KEY,
|
|
44
|
+
display_name TEXT,
|
|
45
|
+
state_dir TEXT NOT NULL,
|
|
46
|
+
first_seen INTEGER,
|
|
47
|
+
last_seen INTEGER,
|
|
48
|
+
last_compacted_at INTEGER,
|
|
49
|
+
checkpoint_count INTEGER DEFAULT 0,
|
|
50
|
+
tokens_saved INTEGER DEFAULT 0,
|
|
51
|
+
compressed_original_bytes INTEGER DEFAULT 0,
|
|
52
|
+
provider TEXT,
|
|
53
|
+
provider_name TEXT,
|
|
54
|
+
model_name TEXT,
|
|
55
|
+
input_rate REAL,
|
|
56
|
+
output_rate REAL,
|
|
57
|
+
model_captured_at INTEGER
|
|
58
|
+
);
|
|
59
|
+
CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
|
|
60
|
+
-- S18: machine-wide injected-set. A foreign checkpoint injected in repo A is
|
|
61
|
+
-- recorded here so repo B's recall never re-injects it. Keyed by checkpoint
|
|
62
|
+
-- + session (a checkpoint may be injected once per session); repo_id is the
|
|
63
|
+
-- source repo (the foreign repo's stateDir) for tracking/source labels.
|
|
64
|
+
-- PRAMETERIZED queries (PREVENT-002); local node:sqlite (PREVENT-PI-004).
|
|
65
|
+
CREATE TABLE IF NOT EXISTS injected_global (
|
|
66
|
+
checkpoint_id TEXT NOT NULL,
|
|
67
|
+
repo_id TEXT NOT NULL,
|
|
68
|
+
session_id TEXT NOT NULL,
|
|
69
|
+
injected_at INTEGER NOT NULL,
|
|
70
|
+
PRIMARY KEY (checkpoint_id, session_id)
|
|
71
|
+
);
|
|
72
|
+
CREATE INDEX IF NOT EXISTS idx_injected_global_cid ON injected_global(checkpoint_id);
|
|
73
|
+
`);
|
|
74
|
+
indexCache = iddb;
|
|
75
|
+
indexCacheDir = indexDir;
|
|
76
|
+
return iddb;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** One row of the global repo registry (multi-repo dashboard source). */
|
|
80
|
+
export interface RepoRegistryRow {
|
|
81
|
+
repoRoot: string;
|
|
82
|
+
displayName: string;
|
|
83
|
+
stateDir: string;
|
|
84
|
+
firstSeen: number;
|
|
85
|
+
lastSeen: number;
|
|
86
|
+
lastCompactedAt: number | null;
|
|
87
|
+
checkpointCount: number;
|
|
88
|
+
tokensSaved: number;
|
|
89
|
+
compressedOriginalBytes: number;
|
|
90
|
+
provider: string | null;
|
|
91
|
+
providerName: string | null;
|
|
92
|
+
modelName: string | null;
|
|
93
|
+
inputRate: number | null;
|
|
94
|
+
outputRate: number | null;
|
|
95
|
+
modelCapturedAt: number | null;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Upsert a repo's aggregate stats into the global index. Called on repo-switch
|
|
100
|
+
* (infrequent). Preserves first_seen + the model columns on update (model is
|
|
101
|
+
* written separately by recordRepoModel so we never clobber it here with nulls).
|
|
102
|
+
*/
|
|
103
|
+
export function upsertRepoRegistry(
|
|
104
|
+
row: {
|
|
105
|
+
repoRoot: string;
|
|
106
|
+
displayName: string;
|
|
107
|
+
stateDir: string;
|
|
108
|
+
checkpointCount: number;
|
|
109
|
+
tokensSaved: number;
|
|
110
|
+
compressedOriginalBytes: number;
|
|
111
|
+
lastCompactedAt?: number | null;
|
|
112
|
+
// The fields below are optional passthroughs so test fixtures and the
|
|
113
|
+
// /api/repos active-window filter can seed them directly. They're also
|
|
114
|
+
// written by other paths (recordRepoModel, registry refresh) — passing
|
|
115
|
+
// them here is harmless because the ON CONFLICT clause keeps first_seen
|
|
116
|
+
// and the model columns from being clobbered.
|
|
117
|
+
firstSeen?: number;
|
|
118
|
+
lastSeen?: number;
|
|
119
|
+
provider?: string | null;
|
|
120
|
+
providerName?: string | null;
|
|
121
|
+
modelName?: string | null;
|
|
122
|
+
inputRate?: number | null;
|
|
123
|
+
outputRate?: number | null;
|
|
124
|
+
modelCapturedAt?: number | null;
|
|
125
|
+
},
|
|
126
|
+
indexDir: string = getIndexDir(),
|
|
127
|
+
): void {
|
|
128
|
+
const db = openIndexStore(indexDir);
|
|
129
|
+
const now = Date.now();
|
|
130
|
+
db.prepare(
|
|
131
|
+
`INSERT INTO repo_registry
|
|
132
|
+
(repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
|
|
133
|
+
checkpoint_count, tokens_saved, compressed_original_bytes,
|
|
134
|
+
provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
|
|
135
|
+
VALUES (@repo_root, @display_name, @state_dir, @first_seen, @last_seen, @last_compacted_at,
|
|
136
|
+
@checkpoint_count, @tokens_saved, @compressed_original_bytes,
|
|
137
|
+
@provider, @provider_name, @model_name, @input_rate, @output_rate, @model_captured_at)
|
|
138
|
+
ON CONFLICT(repo_root) DO UPDATE SET
|
|
139
|
+
display_name = excluded.display_name,
|
|
140
|
+
state_dir = excluded.state_dir,
|
|
141
|
+
last_seen = COALESCE(excluded.last_seen, @now),
|
|
142
|
+
last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
|
|
143
|
+
checkpoint_count = excluded.checkpoint_count,
|
|
144
|
+
tokens_saved = excluded.tokens_saved,
|
|
145
|
+
compressed_original_bytes = excluded.compressed_original_bytes,
|
|
146
|
+
provider = COALESCE(excluded.provider, repo_registry.provider),
|
|
147
|
+
provider_name = COALESCE(excluded.provider_name, repo_registry.provider_name),
|
|
148
|
+
model_name = COALESCE(excluded.model_name, repo_registry.model_name),
|
|
149
|
+
input_rate = COALESCE(excluded.input_rate, repo_registry.input_rate),
|
|
150
|
+
output_rate = COALESCE(excluded.output_rate, repo_registry.output_rate),
|
|
151
|
+
model_captured_at = COALESCE(excluded.model_captured_at, repo_registry.model_captured_at)`,
|
|
152
|
+
).run({
|
|
153
|
+
repo_root: row.repoRoot,
|
|
154
|
+
display_name: row.displayName,
|
|
155
|
+
state_dir: row.stateDir,
|
|
156
|
+
now,
|
|
157
|
+
first_seen: row.firstSeen ?? null,
|
|
158
|
+
last_seen: row.lastSeen ?? null,
|
|
159
|
+
last_compacted_at: row.lastCompactedAt ?? null,
|
|
160
|
+
checkpoint_count: row.checkpointCount,
|
|
161
|
+
tokens_saved: row.tokensSaved,
|
|
162
|
+
compressed_original_bytes: row.compressedOriginalBytes,
|
|
163
|
+
provider: row.provider ?? null,
|
|
164
|
+
provider_name: row.providerName ?? null,
|
|
165
|
+
model_name: row.modelName ?? null,
|
|
166
|
+
input_rate: row.inputRate ?? null,
|
|
167
|
+
output_rate: row.outputRate ?? null,
|
|
168
|
+
model_captured_at: row.modelCapturedAt ?? null,
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Record the active model/provider for a repo in the global index (denormalized
|
|
174
|
+
* so the All-repos table shows model without opening each repo's DB). Upserts a
|
|
175
|
+
* bare registry row if the repo isn't registered yet.
|
|
176
|
+
*/
|
|
177
|
+
export function recordRepoModel(
|
|
178
|
+
repoRoot: string,
|
|
179
|
+
model: {
|
|
180
|
+
provider: string;
|
|
181
|
+
providerName: string | null;
|
|
182
|
+
modelName: string | null;
|
|
183
|
+
inputRate: number;
|
|
184
|
+
outputRate: number;
|
|
185
|
+
stateDir: string;
|
|
186
|
+
displayName: string;
|
|
187
|
+
},
|
|
188
|
+
indexDir: string = getIndexDir(),
|
|
189
|
+
): void {
|
|
190
|
+
const db = openIndexStore(indexDir);
|
|
191
|
+
const now = Date.now();
|
|
192
|
+
db.prepare(
|
|
193
|
+
`INSERT INTO repo_registry
|
|
194
|
+
(repo_root, display_name, state_dir, first_seen, last_seen,
|
|
195
|
+
provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
|
|
196
|
+
VALUES (@repo_root, @display_name, @state_dir, @now, @now,
|
|
197
|
+
@provider, @provider_name, @model_name, @input_rate, @output_rate, @now)
|
|
198
|
+
ON CONFLICT(repo_root) DO UPDATE SET
|
|
199
|
+
last_seen = excluded.last_seen,
|
|
200
|
+
provider = excluded.provider,
|
|
201
|
+
provider_name = excluded.provider_name,
|
|
202
|
+
model_name = excluded.model_name,
|
|
203
|
+
input_rate = excluded.input_rate,
|
|
204
|
+
output_rate = excluded.output_rate,
|
|
205
|
+
model_captured_at = excluded.model_captured_at`,
|
|
206
|
+
).run({
|
|
207
|
+
repo_root: repoRoot,
|
|
208
|
+
display_name: model.displayName,
|
|
209
|
+
state_dir: model.stateDir,
|
|
210
|
+
now,
|
|
211
|
+
provider: model.provider,
|
|
212
|
+
provider_name: model.providerName,
|
|
213
|
+
model_name: model.modelName,
|
|
214
|
+
input_rate: model.inputRate,
|
|
215
|
+
output_rate: model.outputRate,
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function mapRegistryRow(row: any): RepoRegistryRow {
|
|
220
|
+
return {
|
|
221
|
+
repoRoot: row.repo_root,
|
|
222
|
+
displayName: row.display_name ?? "",
|
|
223
|
+
stateDir: row.state_dir,
|
|
224
|
+
firstSeen: row.first_seen ?? 0,
|
|
225
|
+
lastSeen: row.last_seen ?? 0,
|
|
226
|
+
lastCompactedAt: row.last_compacted_at ?? null,
|
|
227
|
+
checkpointCount: row.checkpoint_count ?? 0,
|
|
228
|
+
tokensSaved: row.tokens_saved ?? 0,
|
|
229
|
+
compressedOriginalBytes: row.compressed_original_bytes ?? 0,
|
|
230
|
+
provider: row.provider ?? null,
|
|
231
|
+
providerName: row.provider_name ?? null,
|
|
232
|
+
modelName: row.model_name ?? null,
|
|
233
|
+
inputRate: row.input_rate ?? null,
|
|
234
|
+
outputRate: row.output_rate ?? null,
|
|
235
|
+
modelCapturedAt: row.model_captured_at ?? null,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** All registered repos, most-recently-seen first. */
|
|
240
|
+
export function listRepoRegistry(indexDir: string = getIndexDir()): RepoRegistryRow[] {
|
|
241
|
+
const db = openIndexStore(indexDir);
|
|
242
|
+
const rows = db.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC").all() as any[];
|
|
243
|
+
return rows.map(mapRegistryRow);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** A single repo's registry row, or undefined. */
|
|
247
|
+
export function getRepoRegistry(repoRoot: string, indexDir: string = getIndexDir()): RepoRegistryRow | undefined {
|
|
248
|
+
const db = openIndexStore(indexDir);
|
|
249
|
+
const row = db.prepare("SELECT * FROM repo_registry WHERE repo_root = ?").get(repoRoot) as any;
|
|
250
|
+
return row ? mapRegistryRow(row) : undefined;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Close the cached index connection (test teardown only). */
|
|
254
|
+
export function closeIndexStore(): void {
|
|
255
|
+
if (indexCache) {
|
|
256
|
+
indexCache.close();
|
|
257
|
+
indexCache = undefined;
|
|
258
|
+
indexCacheDir = undefined;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// ---------------------------------------------------------------------------
|
|
263
|
+
// S18: machine-wide injected-set (cross-repo dedup markers)
|
|
264
|
+
//
|
|
265
|
+
// A foreign checkpoint injected in repo A is recorded here so repo B's recall
|
|
266
|
+
// never re-injects it (a stronger, machine-wide version of the per-session
|
|
267
|
+
// injected-set in the local store). Keyed by (checkpoint_id, session_id); the
|
|
268
|
+
// session_id here is the RECEIVING session, so the same foreign checkpoint can
|
|
269
|
+
// be injected into different sessions but never twice into the same one.
|
|
270
|
+
// PRAMETERIZED queries (PREVENT-002); local node:sqlite + WAL (PREVENT-PI-004),
|
|
271
|
+
// multi-process safe.
|
|
272
|
+
// ---------------------------------------------------------------------------
|
|
273
|
+
|
|
274
|
+
/** Record that a (foreign) checkpoint was injected into `sessionId`. Idempotent. */
|
|
275
|
+
export function markInjectedGlobal(
|
|
276
|
+
checkpointId: string,
|
|
277
|
+
repoId: string,
|
|
278
|
+
sessionId: string,
|
|
279
|
+
indexDir: string = getIndexDir(),
|
|
280
|
+
): void {
|
|
281
|
+
const db = openIndexStore(indexDir);
|
|
282
|
+
db.prepare(
|
|
283
|
+
"INSERT OR IGNORE INTO injected_global (checkpoint_id, repo_id, session_id, injected_at) VALUES ($cid, $rid, $sid, $ts)",
|
|
284
|
+
).run({ $cid: checkpointId, $rid: repoId, $sid: sessionId, $ts: Date.now() });
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/** True when a checkpoint was already injected into `sessionId` (machine-wide). */
|
|
288
|
+
export function wasInjectedGlobal(
|
|
289
|
+
checkpointId: string,
|
|
290
|
+
sessionId: string,
|
|
291
|
+
indexDir: string = getIndexDir(),
|
|
292
|
+
): boolean {
|
|
293
|
+
const db = openIndexStore(indexDir);
|
|
294
|
+
const row = db.prepare(
|
|
295
|
+
"SELECT 1 FROM injected_global WHERE checkpoint_id = $cid AND session_id = $sid LIMIT 1",
|
|
296
|
+
).get({ $cid: checkpointId, $sid: sessionId }) as { "1": number } | undefined;
|
|
297
|
+
return row !== undefined;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** Count of cross-repo injections recorded (for /mega-status stats). */
|
|
301
|
+
export function countInjectedGlobal(indexDir: string = getIndexDir()): number {
|
|
302
|
+
const db = openIndexStore(indexDir);
|
|
303
|
+
const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get() as { n: number } | undefined;
|
|
304
|
+
return row?.n ?? 0;
|
|
305
|
+
}
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* maintenance.ts — S27 Task 10 DB maintenance / housekeeping primitives.
|
|
3
|
+
*
|
|
4
|
+
* All pi-agnostic, all parameterized (PREVENT-002), all local (PREVENT-PI-004).
|
|
5
|
+
* Exposed via the /mega-db-* slash commands in extensions/mega-db-cmds.ts.
|
|
6
|
+
*/
|
|
7
|
+
import { statSync } from "node:fs";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
import { getStateDir } from "../../store.js";
|
|
10
|
+
import { openStore, withTx } from "./utils.js";
|
|
11
|
+
|
|
12
|
+
/** Per-table row counts + DB file sizes for the /mega-db-stats command. */
|
|
13
|
+
export interface DbStats {
|
|
14
|
+
/** Row count per table (keys are table names that exist in this DB). */
|
|
15
|
+
tableCounts: Record<string, number>;
|
|
16
|
+
/** Bytes used by the main DB file on disk. */
|
|
17
|
+
dbBytes: number;
|
|
18
|
+
/** Bytes used by the -wal sidecar file (0 if absent). */
|
|
19
|
+
walBytes: number;
|
|
20
|
+
/** Bytes used by the -shm sidecar file (0 if absent). */
|
|
21
|
+
shmBytes: number;
|
|
22
|
+
/** SQLite page size in bytes. */
|
|
23
|
+
pageSize: number;
|
|
24
|
+
/** Total pages (freelist + in-use). */
|
|
25
|
+
pageCount: number;
|
|
26
|
+
/** Freelist pages (reusable by VACUUM). */
|
|
27
|
+
freelistPages: number;
|
|
28
|
+
/** WAL frame count from PRAGMA wal_info (best-effort; 0 if unsupported). */
|
|
29
|
+
walFrames: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const DB_TABLE_NAMES = [
|
|
33
|
+
"context_chunks",
|
|
34
|
+
"session_state",
|
|
35
|
+
"raw_transcript",
|
|
36
|
+
"checkpoint_epochs",
|
|
37
|
+
"dedup_mirror",
|
|
38
|
+
"memories",
|
|
39
|
+
"dedup_stats",
|
|
40
|
+
"daily_log",
|
|
41
|
+
] as const;
|
|
42
|
+
|
|
43
|
+
function fileSizeIfExists(path: string): number {
|
|
44
|
+
try {
|
|
45
|
+
const st = statSync(path);
|
|
46
|
+
return st.size;
|
|
47
|
+
} catch {
|
|
48
|
+
return 0;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Gather DB stats for /mega-db-stats: per-table row counts, disk footprint
|
|
54
|
+
* (main + WAL + SHM), page count, freelist, WAL frame count.
|
|
55
|
+
*
|
|
56
|
+
* Read-only: no PRAGMA writes, no VACUUM. Safe to call any time.
|
|
57
|
+
*/
|
|
58
|
+
export function getDbStats(stateDir: string = getStateDir()): DbStats {
|
|
59
|
+
const db = openStore(stateDir);
|
|
60
|
+
const tableCounts: Record<string, number> = {};
|
|
61
|
+
for (const t of DB_TABLE_NAMES) {
|
|
62
|
+
try {
|
|
63
|
+
const row = db.prepare(`SELECT COUNT(*) AS c FROM ${t}`).get() as { c: number } | undefined;
|
|
64
|
+
if (row) tableCounts[t] = row.c;
|
|
65
|
+
} catch {
|
|
66
|
+
// Table doesn't exist on this DB (e.g. raw_transcript on a pre-S27 store).
|
|
67
|
+
// Skip silently — /mega-db-stats lists only tables that exist.
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const pageStat = db.prepare("PRAGMA page_count").get() as { page_count?: number } | undefined;
|
|
71
|
+
const freelistStat = db.prepare("PRAGMA freelist_count").get() as { freelist_count?: number } | undefined;
|
|
72
|
+
const pageSizeStat = db.prepare("PRAGMA page_size").get() as { page_size?: number } | undefined;
|
|
73
|
+
let walFrames = 0;
|
|
74
|
+
try {
|
|
75
|
+
const walInfo = db.prepare("PRAGMA wal_info").get() as { frames?: number } | undefined;
|
|
76
|
+
walFrames = walInfo?.frames ?? 0;
|
|
77
|
+
} catch {
|
|
78
|
+
// node:sqlite may not expose wal_info on all versions; not fatal.
|
|
79
|
+
}
|
|
80
|
+
const dbPath = join(stateDir, "sqlite.db");
|
|
81
|
+
return {
|
|
82
|
+
tableCounts,
|
|
83
|
+
dbBytes: fileSizeIfExists(dbPath),
|
|
84
|
+
walBytes: fileSizeIfExists(`${dbPath}-wal`),
|
|
85
|
+
shmBytes: fileSizeIfExists(`${dbPath}-shm`),
|
|
86
|
+
pageSize: pageSizeStat?.page_size ?? 0,
|
|
87
|
+
pageCount: pageStat?.page_count ?? 0,
|
|
88
|
+
freelistPages: freelistStat?.freelist_count ?? 0,
|
|
89
|
+
walFrames,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Result of a prune / VACUUM / checkpoint operation (reclaimed bytes). */
|
|
94
|
+
export interface MaintenanceResult {
|
|
95
|
+
/** Rows deleted (prune) or pages reclaimed (VACUUM / checkpoint). */
|
|
96
|
+
affected: number;
|
|
97
|
+
/** Bytes reclaimed on disk (best-effort: post-op size minus pre-op size). */
|
|
98
|
+
reclaimedBytes: number;
|
|
99
|
+
/** Human-readable summary line for the command output. */
|
|
100
|
+
summary: string;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Prune raw_transcript + checkpoint_epochs rows older than `daysOld`.
|
|
105
|
+
* Uses `message_timestamp` (raw_transcript) and `created_at` (epochs), both
|
|
106
|
+
* epoch-ms. Returns the total deleted rows + reclaimed disk bytes.
|
|
107
|
+
*
|
|
108
|
+
* PREVENT-002: parameterized. PREVENT-PI-004: local SQLite only.
|
|
109
|
+
*/
|
|
110
|
+
export function pruneOldRows(stateDir: string = getStateDir(), daysOld = 30): MaintenanceResult {
|
|
111
|
+
const db = openStore(stateDir);
|
|
112
|
+
const cutoff = Date.now() - daysOld * 86_400_000;
|
|
113
|
+
const beforeBytes = fileSizeIfExists(join(stateDir, "sqlite.db"));
|
|
114
|
+
// raw_transcript: message_timestamp may be NULL (pre-S27 rows); those use
|
|
115
|
+
// the row's insertion order implicitly via seq, so we prune NULL-ts rows
|
|
116
|
+
// only when the whole session is older than the cutoff (join via session_id
|
|
117
|
+
// to checkpoint_epochs.created_at). Simpler: prune NULL-ts rows older than
|
|
118
|
+
// cutoff by falling back to the MIN(created_at) of their epoch.
|
|
119
|
+
// Delete raw_transcript rows whose message_timestamp is older than cutoff,
|
|
120
|
+
// OR whose message_timestamp is NULL and the session's latest epoch is older.
|
|
121
|
+
const delRt = db.prepare(
|
|
122
|
+
`DELETE FROM raw_transcript
|
|
123
|
+
WHERE message_timestamp IS NOT NULL AND message_timestamp < ?
|
|
124
|
+
OR (message_timestamp IS NULL
|
|
125
|
+
AND session_id IN (
|
|
126
|
+
SELECT session_id FROM checkpoint_epochs
|
|
127
|
+
GROUP BY session_id HAVING MAX(created_at) < ?
|
|
128
|
+
))`,
|
|
129
|
+
).run(cutoff, cutoff) as { changes?: number } | undefined;
|
|
130
|
+
const rtDeleted = delRt?.changes ?? 0;
|
|
131
|
+
// checkpoint_epochs: created_at is NOT NULL.
|
|
132
|
+
const delEp = db.prepare(`DELETE FROM checkpoint_epochs WHERE created_at < ?`).run(cutoff) as {
|
|
133
|
+
changes?: number;
|
|
134
|
+
} | undefined;
|
|
135
|
+
const epDeleted = delEp?.changes ?? 0;
|
|
136
|
+
// dedup_mirror: cascade-delete orphan rows whose ref_count has dropped to 0
|
|
137
|
+
// after the raw_transcript deletes. Safe even if FK is off (raw_transcript has
|
|
138
|
+
// no FK to dedup_mirror; ref_count is maintained by the dedup pipeline).
|
|
139
|
+
const delDedup = db.prepare(`DELETE FROM dedup_mirror WHERE ref_count <= 0`).run() as {
|
|
140
|
+
changes?: number;
|
|
141
|
+
} | undefined;
|
|
142
|
+
const dedupDeleted = delDedup?.changes ?? 0;
|
|
143
|
+
const afterBytes = fileSizeIfExists(join(stateDir, "sqlite.db"));
|
|
144
|
+
const total = rtDeleted + epDeleted + dedupDeleted;
|
|
145
|
+
return {
|
|
146
|
+
affected: total,
|
|
147
|
+
reclaimedBytes: Math.max(0, beforeBytes - afterBytes),
|
|
148
|
+
summary: `pruned ${rtDeleted} raw_transcript + ${epDeleted} epochs + ${dedupDeleted} dedup_mirror rows older than ${daysOld}d`,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Force a WAL checkpoint (TRUNCATE mode) so the -wal sidecar is reclaimed.
|
|
154
|
+
* Returns the WAL bytes reclaimed (pre-wal size minus post-wal size).
|
|
155
|
+
*/
|
|
156
|
+
export function checkpointWal(stateDir: string = getStateDir()): MaintenanceResult {
|
|
157
|
+
const db = openStore(stateDir);
|
|
158
|
+
const dbPath = join(stateDir, "sqlite.db");
|
|
159
|
+
const beforeWal = fileSizeIfExists(`${dbPath}-wal`);
|
|
160
|
+
// PRAGMA wal_checkpoint(TRUNCATE) blocks until all frames are folded into the
|
|
161
|
+
// main db and the WAL file is truncated to 0 bytes.
|
|
162
|
+
const res = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get() as {
|
|
163
|
+
busy?: number;
|
|
164
|
+
log?: number;
|
|
165
|
+
checkpointed?: number;
|
|
166
|
+
} | undefined;
|
|
167
|
+
const afterWal = fileSizeIfExists(`${dbPath}-wal`);
|
|
168
|
+
const reclaimed = Math.max(0, beforeWal - afterWal);
|
|
169
|
+
return {
|
|
170
|
+
affected: res?.checkpointed ?? 0,
|
|
171
|
+
reclaimedBytes: reclaimed,
|
|
172
|
+
summary: `wal_checkpoint(TRUNCATE): ${res?.checkpointed ?? 0} frames folded, WAL ${beforeWal}→${afterWal} bytes${res?.busy ? " (busy: " + res.busy + ")" : ""}`,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* VACUUM the main DB file (rebuilds pages, reclaims freelist space).
|
|
178
|
+
* Heavy: briefly doubles disk usage. Run only when freelist is large or the
|
|
179
|
+
* user explicitly invokes /mega-db-vacuum.
|
|
180
|
+
*/
|
|
181
|
+
export function vacuumDb(stateDir: string = getStateDir()): MaintenanceResult {
|
|
182
|
+
const db = openStore(stateDir);
|
|
183
|
+
const dbPath = join(stateDir, "sqlite.db");
|
|
184
|
+
const beforeBytes = fileSizeIfExists(dbPath);
|
|
185
|
+
db.exec("VACUUM"); // VACUUM cannot be parameterized; it rewrites the whole DB.
|
|
186
|
+
const afterBytes = fileSizeIfExists(dbPath);
|
|
187
|
+
const reclaimed = Math.max(0, beforeBytes - afterBytes);
|
|
188
|
+
return {
|
|
189
|
+
affected: 0,
|
|
190
|
+
reclaimedBytes: reclaimed,
|
|
191
|
+
summary: `VACUUM: db ${beforeBytes}→${afterBytes} bytes (reclaimed ${reclaimed})`,
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Run `PRAGMA integrity_check` and return the result lines.
|
|
197
|
+
* Returns ["ok"] when the DB is healthy; otherwise returns the error lines.
|
|
198
|
+
*/
|
|
199
|
+
export function integrityCheck(stateDir: string = getStateDir()): string[] {
|
|
200
|
+
const db = openStore(stateDir);
|
|
201
|
+
const rows = db.prepare("PRAGMA integrity_check").all() as Array<{ integrity_check: string }> | undefined;
|
|
202
|
+
return (rows ?? []).map((r) => r.integrity_check);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Reconcile drift in dedup_mirror.ref_count vs actual raw_transcript refs. */
|
|
206
|
+
export interface DedupReconcileResult {
|
|
207
|
+
/** Rows whose ref_count was corrected. */
|
|
208
|
+
fixedRefCount: number;
|
|
209
|
+
/** Orphan dedup_mirror rows (content_hash with 0 raw_transcript refs) deleted. */
|
|
210
|
+
orphansDeleted: number;
|
|
211
|
+
/** raw_transcript rows whose content_ref was NULL but now set (backfill). */
|
|
212
|
+
refsBackfilled: number;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Reconcile dedup_mirror vs raw_transcript after pruning or crashes:
|
|
217
|
+
* 1. Recompute ref_count = COUNT(raw_transcript rows pointing at this hash).
|
|
218
|
+
* 2. Delete orphan dedup_mirror rows whose recomputed ref_count is 0.
|
|
219
|
+
* 3. Backfill raw_transcript.content_ref for rows still storing inline bytes.
|
|
220
|
+
*
|
|
221
|
+
* Idempotent. Read-modify-write within a single transaction (withTx).
|
|
222
|
+
*/
|
|
223
|
+
export function reconcileDedupMirror(stateDir: string = getStateDir()): DedupReconcileResult {
|
|
224
|
+
const db = openStore(stateDir);
|
|
225
|
+
const result: DedupReconcileResult = { fixedRefCount: 0, orphansDeleted: 0, refsBackfilled: 0 };
|
|
226
|
+
withTx(db, () => {
|
|
227
|
+
// 1. Recompute ref_count for every dedup_mirror row from the actual
|
|
228
|
+
// raw_transcript references.
|
|
229
|
+
const recompute = db.prepare(
|
|
230
|
+
`UPDATE dedup_mirror AS dm
|
|
231
|
+
SET ref_count = COALESCE((
|
|
232
|
+
SELECT COUNT(*) FROM raw_transcript rt WHERE rt.content_ref = dm.content_hash
|
|
233
|
+
), 0)
|
|
234
|
+
WHERE dm.ref_count != COALESCE((
|
|
235
|
+
SELECT COUNT(*) FROM raw_transcript rt WHERE rt.content_ref = dm.content_hash
|
|
236
|
+
), 0)`,
|
|
237
|
+
).run() as { changes?: number } | undefined;
|
|
238
|
+
result.fixedRefCount = recompute?.changes ?? 0;
|
|
239
|
+
// 2. Delete orphan dedup_mirror rows (no raw_transcript refs).
|
|
240
|
+
const delOrphans = db.prepare(
|
|
241
|
+
`DELETE FROM dedup_mirror
|
|
242
|
+
WHERE content_hash NOT IN (SELECT DISTINCT content_ref FROM raw_transcript WHERE content_ref IS NOT NULL)`,
|
|
243
|
+
).run() as { changes?: number } | undefined;
|
|
244
|
+
result.orphansDeleted = delOrphans?.changes ?? 0;
|
|
245
|
+
// 3. Backfill content_ref for rows still storing inline content_bytes (no
|
|
246
|
+
// ref yet). Only safe when a matching dedup_mirror row exists; otherwise
|
|
247
|
+
// we'd need to insert one, which is the dedup pipeline's job, not the
|
|
248
|
+
// reconciler's.
|
|
249
|
+
const backfill = db.prepare(
|
|
250
|
+
`UPDATE raw_transcript AS rt
|
|
251
|
+
SET content_ref = (
|
|
252
|
+
SELECT dm.content_hash FROM dedup_mirror dm WHERE dm.content_bytes = rt.content_bytes
|
|
253
|
+
)
|
|
254
|
+
WHERE rt.content_ref IS NULL
|
|
255
|
+
AND EXISTS (SELECT 1 FROM dedup_mirror dm WHERE dm.content_bytes = rt.content_bytes)`,
|
|
256
|
+
).run() as { changes?: number } | undefined;
|
|
257
|
+
result.refsBackfilled = backfill?.changes ?? 0;
|
|
258
|
+
});
|
|
259
|
+
return result;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* One-shot auto-maintenance pass for the session_start hook: prune old rows,
|
|
264
|
+
* checkpoint the WAL if it's grown large, and (only if the DB is huge) VACUUM.
|
|
265
|
+
* Best-effort: swallows errors so a session never fails to start over a
|
|
266
|
+
* housekeeping hiccup. Returns a short summary for the diagnostic log.
|
|
267
|
+
*/
|
|
268
|
+
export function autoMaintain(stateDir: string = getStateDir()): string {
|
|
269
|
+
try {
|
|
270
|
+
const stats = getDbStats(stateDir);
|
|
271
|
+
const parts: string[] = [];
|
|
272
|
+
// Prune rows older than 30d (default retention).
|
|
273
|
+
const prune = pruneOldRows(stateDir, 30);
|
|
274
|
+
if (prune.affected > 0) parts.push(`pruned ${prune.affected}`);
|
|
275
|
+
// Checkpoint the WAL if it's over 10 MB (avoid pathological WAL growth).
|
|
276
|
+
if (stats.walBytes > 10 * 1024 * 1024) {
|
|
277
|
+
const ck = checkpointWal(stateDir);
|
|
278
|
+
if (ck.reclaimedBytes > 0) parts.push(`wal -${ck.reclaimedBytes}B`);
|
|
279
|
+
}
|
|
280
|
+
// VACUUM only if the DB is over 100 MB AND freelist is >20% of pages.
|
|
281
|
+
if (
|
|
282
|
+
stats.dbBytes > 100 * 1024 * 1024 &&
|
|
283
|
+
stats.pageCount > 0 &&
|
|
284
|
+
stats.freelistPages / stats.pageCount > 0.2
|
|
285
|
+
) {
|
|
286
|
+
const v = vacuumDb(stateDir);
|
|
287
|
+
if (v.reclaimedBytes > 0) parts.push(`vacuum -${v.reclaimedBytes}B`);
|
|
288
|
+
}
|
|
289
|
+
return parts.length ? `auto-maintain: ${parts.join(", ")}` : "auto-maintain: nothing to do";
|
|
290
|
+
} catch (err) {
|
|
291
|
+
// Never block session start over housekeeping.
|
|
292
|
+
return `auto-maintain: skipped (${(err as Error).message})`;
|
|
293
|
+
}
|
|
294
|
+
}
|