pi-mega-compact 0.7.7 → 0.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +11 -12
  2. package/dist/extensions/dashboard-server/helpers.js +37 -0
  3. package/dist/extensions/dashboard-server/html/all-repos-tab.js +26 -0
  4. package/dist/extensions/dashboard-server/html/body-open.js +23 -0
  5. package/dist/extensions/dashboard-server/html/current-repo-tab.js +130 -0
  6. package/dist/extensions/dashboard-server/html/head-open.js +16 -0
  7. package/dist/extensions/dashboard-server/html/high-score-tab.js +25 -0
  8. package/dist/extensions/dashboard-server/html/repo-detail-modal.js +26 -0
  9. package/dist/extensions/dashboard-server/html/script.js +259 -0
  10. package/dist/extensions/dashboard-server/html/styles.js +103 -0
  11. package/dist/extensions/dashboard-server/html/summary-tab.js +19 -0
  12. package/dist/extensions/dashboard-server/html-template.js +41 -0
  13. package/dist/extensions/dashboard-server/html.js +756 -0
  14. package/dist/extensions/dashboard-server/index-reader.js +133 -0
  15. package/dist/extensions/dashboard-server/server.js +370 -0
  16. package/dist/extensions/dashboard-server/snapshot.js +43 -0
  17. package/dist/extensions/dashboard-server/state.js +30 -0
  18. package/dist/extensions/dashboard-server/types.js +5 -0
  19. package/dist/extensions/dashboard-server.js +7 -1315
  20. package/dist/extensions/mega-commands.js +162 -134
  21. package/dist/extensions/mega-compact.test.js +292 -24
  22. package/dist/extensions/mega-config.js +10 -0
  23. package/dist/extensions/mega-conflict-cmds.js +5 -1
  24. package/dist/extensions/mega-dashboard-cmds.js +29 -22
  25. package/dist/extensions/mega-db-cmds.js +11 -2
  26. package/dist/extensions/mega-events/agent-handlers.js +173 -0
  27. package/dist/extensions/mega-events/compact-handlers.js +133 -0
  28. package/dist/extensions/mega-events/context-handler.js +249 -0
  29. package/dist/extensions/mega-events/register.js +21 -0
  30. package/dist/extensions/mega-events/session-handlers.js +142 -0
  31. package/dist/extensions/mega-events.js +15 -652
  32. package/dist/extensions/mega-pipeline/compact.js +324 -0
  33. package/dist/extensions/mega-pipeline/memory-review.js +38 -0
  34. package/dist/extensions/mega-pipeline/recall.js +147 -0
  35. package/dist/extensions/mega-pipeline.js +9 -480
  36. package/dist/extensions/mega-runtime/helpers.js +40 -0
  37. package/dist/extensions/mega-runtime/query.js +29 -0
  38. package/dist/extensions/mega-runtime/state.js +711 -0
  39. package/dist/extensions/mega-runtime/widget.js +197 -0
  40. package/dist/extensions/mega-runtime.js +15 -932
  41. package/dist/src/store/sqlite/checkpoints.js +145 -0
  42. package/dist/src/store/sqlite/connection.js +35 -0
  43. package/dist/src/store/sqlite/dedup-mirror.js +64 -0
  44. package/dist/src/store/sqlite/foundation.js +38 -0
  45. package/dist/src/store/sqlite/global-index.js +224 -0
  46. package/dist/src/store/sqlite/index-store.js +167 -0
  47. package/dist/src/store/sqlite/maintenance.js +235 -0
  48. package/dist/src/store/sqlite/memories.js +164 -0
  49. package/dist/src/store/sqlite/memory.js +54 -0
  50. package/dist/src/store/sqlite/meta.js +82 -0
  51. package/dist/src/store/sqlite/minhash-lsh.js +47 -0
  52. package/dist/src/store/sqlite/model-snapshots.js +47 -0
  53. package/dist/src/store/sqlite/raptor.js +57 -0
  54. package/dist/src/store/sqlite/raw-transcript.js +134 -0
  55. package/dist/src/store/sqlite/schema.js +250 -0
  56. package/dist/src/store/sqlite/session-state.js +28 -0
  57. package/dist/src/store/sqlite/sessions.js +39 -0
  58. package/dist/src/store/sqlite/stats.js +66 -0
  59. package/dist/src/store/sqlite/transaction.js +19 -0
  60. package/dist/src/store/sqlite/utils.js +120 -0
  61. package/dist/src/store/sqlite.js +20 -1607
  62. package/dist/src/vectorStore/add.js +260 -0
  63. package/dist/src/vectorStore/dedup.js +52 -0
  64. package/dist/src/vectorStore/index.js +10 -0
  65. package/dist/src/vectorStore/queries.js +83 -0
  66. package/dist/src/vectorStore/search.js +95 -0
  67. package/dist/src/vectorStore/session.js +19 -0
  68. package/dist/src/vectorStore/store.js +105 -0
  69. package/dist/src/vectorStore/types.js +6 -0
  70. package/dist/src/vectorStore/utils.js +23 -0
  71. package/extensions/dashboard-server/html.ts +758 -0
  72. package/extensions/dashboard-server/index-reader.ts +130 -0
  73. package/extensions/dashboard-server/server.ts +358 -0
  74. package/extensions/dashboard-server/snapshot.ts +44 -0
  75. package/extensions/dashboard-server/state.ts +33 -0
  76. package/extensions/dashboard-server/types.ts +134 -0
  77. package/extensions/dashboard-server.ts +7 -1431
  78. package/extensions/mega-commands.ts +33 -10
  79. package/extensions/mega-compact.test.ts +453 -37
  80. package/extensions/mega-config.ts +22 -0
  81. package/extensions/mega-conflict-cmds.ts +6 -2
  82. package/extensions/mega-dashboard-cmds.ts +30 -23
  83. package/extensions/mega-db-cmds.ts +11 -3
  84. package/extensions/mega-events/agent-handlers.ts +214 -0
  85. package/extensions/mega-events/compact-handlers.ts +164 -0
  86. package/extensions/mega-events/context-handler.ts +290 -0
  87. package/extensions/mega-events/register.ts +37 -0
  88. package/extensions/mega-events/session-handlers.ts +165 -0
  89. package/extensions/mega-events.ts +15 -732
  90. package/extensions/mega-pipeline/compact.ts +366 -0
  91. package/extensions/mega-pipeline/memory-review.ts +46 -0
  92. package/extensions/mega-pipeline/recall.ts +165 -0
  93. package/extensions/mega-pipeline.ts +9 -537
  94. package/extensions/mega-runtime/helpers.ts +68 -0
  95. package/extensions/mega-runtime/query.ts +29 -0
  96. package/extensions/mega-runtime/state.ts +797 -0
  97. package/extensions/mega-runtime/widget.ts +258 -0
  98. package/extensions/mega-runtime.ts +15 -1076
  99. package/package.json +4 -3
  100. package/src/store/sqlite/checkpoints.ts +204 -0
  101. package/src/store/sqlite/dedup-mirror.ts +114 -0
  102. package/src/store/sqlite/foundation.ts +63 -0
  103. package/src/store/sqlite/global-index.ts +305 -0
  104. package/src/store/sqlite/maintenance.ts +294 -0
  105. package/src/store/sqlite/memories.ts +217 -0
  106. package/src/store/sqlite/meta.ts +108 -0
  107. package/src/store/sqlite/model-snapshots.ts +83 -0
  108. package/src/store/sqlite/raptor.ts +107 -0
  109. package/src/store/sqlite/raw-transcript.ts +221 -0
  110. package/src/store/sqlite/schema.ts +258 -0
  111. package/src/store/sqlite/session-state.ts +38 -0
  112. package/src/store/sqlite/stats.ts +127 -0
  113. package/src/store/sqlite/utils.ts +125 -0
  114. package/src/store/sqlite.ts +20 -2204
@@ -0,0 +1,145 @@
1
+ import { getStateDir, normalizeSessionId } from "../../store.js";
2
+ import { openStore, withTx, jsonText, encodeEmbedding, rowToCheckpoint, } from "./utils.js";
3
+ /** Insert or replace a checkpoint (idempotent by id). */
4
+ export function upsertCheckpoint(cp, stateDir = getStateDir()) {
5
+ const db = openStore(stateDir);
6
+ const sid = normalizeSessionId(cp.sessionId);
7
+ withTx(db, () => {
8
+ db.prepare(`INSERT INTO context_chunks
9
+ (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
10
+ normalized_text, summary, topic_summary, summary_hash,
11
+ key_decisions, next_steps, files_modified, embedding_blob,
12
+ token_estimate, original_token_estimate, timestamp, dedup_status, compressed_original)
13
+ VALUES (@id, @sid, @region_hash, @content_hash, @content_hash2, @content_hash_version,
14
+ @normalized_text, @summary, @topic_summary, @summary_hash,
15
+ @key_decisions, @next_steps, @files_modified, @embedding_blob,
16
+ @token_estimate, @original_token_estimate, @timestamp, @dedup_status, @compressed_original)
17
+ ON CONFLICT(session_id, id) DO UPDATE SET
18
+ summary=excluded.summary,
19
+ topic_summary=excluded.topic_summary,
20
+ summary_hash=excluded.summary_hash,
21
+ key_decisions=excluded.key_decisions,
22
+ next_steps=excluded.next_steps,
23
+ files_modified=excluded.files_modified,
24
+ embedding_blob=excluded.embedding_blob,
25
+ token_estimate=excluded.token_estimate,
26
+ original_token_estimate=excluded.original_token_estimate,
27
+ timestamp=excluded.timestamp,
28
+ dedup_status=excluded.dedup_status,
29
+ compressed_original=excluded.compressed_original`).run({
30
+ "@id": cp.checkpointId,
31
+ "@sid": sid,
32
+ "@region_hash": cp.regionHash ?? null,
33
+ "@content_hash": cp.contentHash ?? null,
34
+ "@content_hash2": cp.contentHash2 ?? null,
35
+ "@content_hash_version": cp.contentHashVersion ?? null,
36
+ "@normalized_text": cp.normalizedText ?? null,
37
+ "@summary": cp.summary ?? "",
38
+ "@topic_summary": cp.topicSummary ?? null,
39
+ "@summary_hash": cp.summaryHash ?? null,
40
+ "@key_decisions": jsonText(cp.keyDecisions),
41
+ "@next_steps": jsonText(cp.nextSteps),
42
+ "@files_modified": jsonText(cp.filesModified),
43
+ "@embedding_blob": encodeEmbedding(cp.embedding ?? []),
44
+ "@token_estimate": cp.tokenEstimate ?? 0,
45
+ "@original_token_estimate": cp.originalTokenEstimate ?? null,
46
+ "@timestamp": cp.timestamp ?? 0,
47
+ "@dedup_status": "active",
48
+ "@compressed_original": cp.compressedOriginal ?? null,
49
+ });
50
+ // FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
51
+ // Store normalized_text (the L1 verify key); fall back to summary for rows
52
+ // that predate normalized_text population.
53
+ db.prepare("DELETE FROM context_chunks_trgm WHERE id = ?").run(cp.checkpointId);
54
+ db.prepare("INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)").run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
55
+ });
56
+ }
57
+ // --- Sprint 11: MinHash signatures + LSH buckets --------------------------
58
+ /** Persist a checkpoint's MinHash signature (idempotent by chunk_id + version). */
59
+ export function upsertMinhashSignature(chunkId, sessionId, signatureVersion, signatures, stateDir = getStateDir()) {
60
+ const db = openStore(stateDir);
61
+ const sid = normalizeSessionId(sessionId);
62
+ db.prepare(`INSERT INTO minhash_signatures(chunk_id, session_id, signature_version, signatures)
63
+ VALUES(?, ?, ?, ?)
64
+ ON CONFLICT(chunk_id, signature_version) DO UPDATE SET
65
+ session_id=excluded.session_id, signatures=excluded.signatures`).run(chunkId, sid, signatureVersion, JSON.stringify(signatures));
66
+ }
67
+ /** Persist LSH bucket memberships for a chunk (one row per bucket key). */
68
+ export function insertLshBuckets(chunkId, sessionId, signatureVersion, bucketKeys, stateDir = getStateDir()) {
69
+ const db = openStore(stateDir);
70
+ const sid = normalizeSessionId(sessionId);
71
+ const del = db.prepare("DELETE FROM dedup_lsh_buckets WHERE chunk_id = ?");
72
+ const ins = db.prepare("INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)");
73
+ withTx(db, () => {
74
+ del.run(chunkId);
75
+ for (const key of bucketKeys)
76
+ ins.run(key, chunkId, sid, signatureVersion);
77
+ });
78
+ }
79
+ /**
80
+ * Candidate chunk_ids sharing any LSH bucket with `bucketKeys`, scoped to the
81
+ * session, capped at `limit`. Single query (no N loops) — QA #15 amplification
82
+ * guard. Returns DISTINCT chunk_ids excluding `excludeChunkId` (the new row).
83
+ */
84
+ export function lshCandidateChunks(bucketKeys, sessionId, excludeChunkId, stateDir = getStateDir(), limit = 100) {
85
+ if (bucketKeys.length === 0)
86
+ return [];
87
+ const db = openStore(stateDir);
88
+ const sid = normalizeSessionId(sessionId);
89
+ const placeholders = bucketKeys.map(() => "?").join(",");
90
+ const rows = db
91
+ .prepare(`SELECT DISTINCT chunk_id FROM dedup_lsh_buckets
92
+ WHERE bucket_key IN (${placeholders}) AND session_id = ? AND chunk_id != ?
93
+ LIMIT ?`)
94
+ .all(...bucketKeys, sid, excludeChunkId, limit);
95
+ return rows.map((r) => r.chunk_id);
96
+ }
97
+ /** All checkpoints for a session, sorted by id. */
98
+ export function listCheckpoints(sessionId, stateDir = getStateDir()) {
99
+ const db = openStore(stateDir);
100
+ const sid = normalizeSessionId(sessionId);
101
+ const rows = db
102
+ .prepare("SELECT * FROM context_chunks WHERE session_id = ? ORDER BY id ASC")
103
+ .all(sid);
104
+ return rows.map(rowToCheckpoint);
105
+ }
106
+ /** S25: the newest checkpoint timestamp for a session, or 0 when none. Used by
107
+ * the RAPTOR freshness guard to reject a tree older than the live checkpoints. */
108
+ export function maxCheckpointTimestamp(sessionId, stateDir = getStateDir()) {
109
+ const db = openStore(stateDir);
110
+ const row = db
111
+ .prepare("SELECT MAX(timestamp) AS mx FROM context_chunks WHERE session_id = ?")
112
+ .get(normalizeSessionId(sessionId));
113
+ return Number(row?.mx ?? 0);
114
+ }
115
+ /** Next sequential checkpoint id (chkpt_001 …) for a session. */
116
+ export function nextCheckpointId(sessionId, stateDir = getStateDir()) {
117
+ const db = openStore(stateDir);
118
+ const sid = normalizeSessionId(sessionId);
119
+ const row = db
120
+ .prepare("SELECT MAX(CAST(SUBSTR(id, 7) AS INTEGER)) AS n FROM context_chunks WHERE session_id = ?")
121
+ .get(sid);
122
+ const next = (row.n ?? 0) + 1;
123
+ return `chkpt_${String(next).padStart(3, "0")}`;
124
+ }
125
+ /** True if a checkpoint id already exists for a session. */
126
+ export function hasCheckpoint(sessionId, checkpointId, stateDir = getStateDir()) {
127
+ const db = openStore(stateDir);
128
+ const row = db
129
+ .prepare("SELECT 1 FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
130
+ .get(normalizeSessionId(sessionId), checkpointId);
131
+ return row !== undefined;
132
+ }
133
+ /** Fetch a single checkpoint by (session, id), or undefined if absent. */
134
+ export function getCheckpoint(sessionId, checkpointId, stateDir = getStateDir()) {
135
+ const db = openStore(stateDir);
136
+ const row = db
137
+ .prepare("SELECT * FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
138
+ .get(normalizeSessionId(sessionId), checkpointId);
139
+ return row ? rowToCheckpoint(row) : undefined;
140
+ }
141
+ /** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
142
+ export function setDedupStatus(checkpointId, sessionId, status, stateDir = getStateDir()) {
143
+ const db = openStore(stateDir);
144
+ db.prepare("UPDATE context_chunks SET dedup_status = ? WHERE id = ? AND session_id = ?").run(status, checkpointId, normalizeSessionId(sessionId));
145
+ }
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Per-repo SQLite connection management (open / reuse / close).
3
+ *
4
+ * In-process cache so the same stateDir reuses one connection (and so a fresh
5
+ * VectorStore over the same dir shares the open DB). Cross-process durability
6
+ * comes from reopening the same file path — proven by the integration test.
7
+ */
8
+ import { DatabaseSync } from "node:sqlite";
9
+ import { existsSync, mkdirSync } from "node:fs";
10
+ import { join } from "node:path";
11
+ import { getStateDir } from "../../store.js";
12
+ import { initSchema } from "./schema.js";
13
+ const cache = new Map();
14
+ /** Open (or reuse) the SQLite store for a state dir. */
15
+ export function openStore(stateDir = getStateDir()) {
16
+ const existing = cache.get(stateDir);
17
+ if (existing)
18
+ return existing;
19
+ if (!existsSync(stateDir))
20
+ mkdirSync(stateDir, { recursive: true });
21
+ const db = new DatabaseSync(join(stateDir, "sqlite.db"));
22
+ db.exec("PRAGMA journal_mode = WAL");
23
+ db.exec("PRAGMA foreign_keys = ON");
24
+ initSchema(db);
25
+ cache.set(stateDir, db);
26
+ return db;
27
+ }
28
+ /** Close and evict a cached connection (test teardown only). */
29
+ export function closeStore(stateDir) {
30
+ const db = cache.get(stateDir);
31
+ if (db) {
32
+ db.close();
33
+ cache.delete(stateDir);
34
+ }
35
+ }
@@ -0,0 +1,64 @@
1
+ /**
2
+ * Upsert a row into dedup_mirror. If the hash already exists, increment ref_count.
3
+ * Returns true if this was a NEW unique content (first insert), false if it was a duplicate.
4
+ */
5
+ export function upsertDedupMirror(db, contentHash, contentBytes, seq) {
6
+ const now = Date.now();
7
+ const existing = db
8
+ .prepare(`SELECT content_hash FROM dedup_mirror WHERE content_hash = @hash`)
9
+ .get({ "@hash": contentHash });
10
+ if (existing) {
11
+ db.prepare(`UPDATE dedup_mirror SET ref_count = ref_count + 1 WHERE content_hash = @hash`).run({
12
+ "@hash": contentHash,
13
+ });
14
+ return false;
15
+ }
16
+ db.prepare(`INSERT INTO dedup_mirror (content_hash, content_bytes, ref_count, first_seen_seq, created_at)
17
+ VALUES (@hash, @bytes, 1, @seq, @now)`).run({
18
+ "@hash": contentHash,
19
+ "@bytes": contentBytes,
20
+ "@seq": seq,
21
+ "@now": now,
22
+ });
23
+ return true;
24
+ }
25
+ /**
26
+ * Get dedup ratio for a session: total bytes vs unique bytes.
27
+ */
28
+ export function getDedupRatio(db, sessionId) {
29
+ const totalRow = db
30
+ .prepare(`SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS total
31
+ FROM raw_transcript
32
+ WHERE session_id = @session_id`)
33
+ .get({ "@session_id": sessionId });
34
+ const uniqueRow = db
35
+ .prepare(`SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS unique_bytes
36
+ FROM dedup_mirror`)
37
+ .get();
38
+ const totalBytes = totalRow.total;
39
+ const uniqueBytes = uniqueRow.unique_bytes;
40
+ const ratio = uniqueBytes > 0 ? totalBytes / uniqueBytes : 1;
41
+ return { totalBytes, uniqueBytes, ratio };
42
+ }
43
+ /**
44
+ * Get dedup mirror stats (diagnostic / test helper).
45
+ */
46
+ export function getDedupMirrorStats(db) {
47
+ const row = db
48
+ .prepare(`SELECT COUNT(*) AS cnt,
49
+ COALESCE(SUM(LENGTH(content_bytes)), 0) AS total_bytes,
50
+ COALESCE(AVG(ref_count), 0) AS avg_ref
51
+ FROM dedup_mirror`)
52
+ .get();
53
+ return { rowCount: row.cnt, totalBytes: row.total_bytes, avgRefCount: row.avg_ref };
54
+ }
55
+ /**
56
+ * Update raw_transcript.content_ref to point to dedup_mirror.
57
+ */
58
+ export function updateRawTranscriptRef(db, sessionId, seq, contentHash) {
59
+ db.prepare(`UPDATE raw_transcript SET content_ref = @ref WHERE session_id = @sid AND seq = @seq`).run({
60
+ "@ref": contentHash,
61
+ "@sid": sessionId,
62
+ "@seq": seq,
63
+ });
64
+ }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * foundation.ts — future-feature foundation (resume sessions / daily log / lessons).
3
+ *
4
+ * Scaffolded tables + minimal helpers so all store data lives in SQLite from
5
+ * day one. Full UI/recall for these lands in later sprints.
6
+ */
7
+ import { getStateDir, normalizeSessionId } from "../../store.js";
8
+ import { openStore } from "./utils.js";
9
+ /** Upsert a `sessions` row (resume + per-repo session history). */
10
+ export function touchSession(sessionId, repo, stateDir = getStateDir()) {
11
+ const db = openStore(stateDir);
12
+ const sid = normalizeSessionId(sessionId);
13
+ const existing = db
14
+ .prepare("SELECT started_at FROM sessions WHERE session_id = ?")
15
+ .get(sid);
16
+ const now = Math.floor(Date.now() / 1000);
17
+ if (!existing) {
18
+ db.prepare(`INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
19
+ VALUES(?, ?, ?, ?, 'active')`).run(sid, repo ?? null, now, now);
20
+ }
21
+ else {
22
+ db.prepare("UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?").run(now, repo ?? null, sid);
23
+ }
24
+ }
25
+ /** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
26
+ export function logDaily(sessionId, event, detail, tokensSaved, stateDir = getStateDir()) {
27
+ const db = openStore(stateDir);
28
+ const day = new Date().toISOString().slice(0, 10);
29
+ const now = Math.floor(Date.now() / 1000);
30
+ db.prepare(`INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
31
+ VALUES(?, ?, ?, ?, ?, ?)`).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
32
+ }
33
+ /** Append a `lessons` entry (future lessons-learned browse/recall). */
34
+ export function addLesson(sessionId, repo, lesson, stateDir = getStateDir()) {
35
+ const db = openStore(stateDir);
36
+ const now = Math.floor(Date.now() / 1000);
37
+ db.prepare(`INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
38
+ }
@@ -0,0 +1,224 @@
1
+ /**
2
+ * global-index.ts — machine-wide index DB (repo registry + injected-set).
3
+ *
4
+ * A single SQLite DB, separate from every per-repo store, that aggregates one
5
+ * row per repo this machine has run on. The multi-repo dashboard (Summary /
6
+ * All-repos tabs) reads it so ONE dashboard can show every repo's checkpoints,
7
+ * tokens saved, and active model — instead of a per-repo dashboard that only
8
+ * ever sees the repo it was launched from.
9
+ *
10
+ * Written by every pi process on repo-switch (bindRepo) + model capture; read by
11
+ * the dashboard server. Concurrency across 10+ pi processes is handled by WAL +
12
+ * infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
13
+ */
14
+ import { DatabaseSync } from "node:sqlite";
15
+ import { existsSync, mkdirSync } from "node:fs";
16
+ import { homedir, tmpdir } from "node:os";
17
+ import { join } from "node:path";
18
+ /** Resolve the machine-wide index directory (env-overridable). */
19
+ export function getIndexDir() {
20
+ const override = process.env.MEGACOMPACT_INDEX_DIR;
21
+ if (override && override.trim() !== "")
22
+ return override;
23
+ // homedir() can throw in exotic sandboxes; fall back to tmpdir.
24
+ try {
25
+ return join(homedir(), ".mega-compact-index");
26
+ }
27
+ catch {
28
+ return join(tmpdir(), ".mega-compact-index");
29
+ }
30
+ }
31
+ let indexCache;
32
+ let indexCacheDir;
33
+ /** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
34
+ export function openIndexStore(indexDir = getIndexDir()) {
35
+ if (indexCache && indexCacheDir === indexDir)
36
+ return indexCache;
37
+ if (!existsSync(indexDir))
38
+ mkdirSync(indexDir, { recursive: true });
39
+ const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
40
+ iddb.exec("PRAGMA journal_mode = WAL");
41
+ iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
42
+ iddb.exec(`
43
+ CREATE TABLE IF NOT EXISTS repo_registry (
44
+ repo_root TEXT PRIMARY KEY,
45
+ display_name TEXT,
46
+ state_dir TEXT NOT NULL,
47
+ first_seen INTEGER,
48
+ last_seen INTEGER,
49
+ last_compacted_at INTEGER,
50
+ checkpoint_count INTEGER DEFAULT 0,
51
+ tokens_saved INTEGER DEFAULT 0,
52
+ compressed_original_bytes INTEGER DEFAULT 0,
53
+ provider TEXT,
54
+ provider_name TEXT,
55
+ model_name TEXT,
56
+ input_rate REAL,
57
+ output_rate REAL,
58
+ model_captured_at INTEGER
59
+ );
60
+ CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
61
+ -- S18: machine-wide injected-set. A foreign checkpoint injected in repo A is
62
+ -- recorded here so repo B's recall never re-injects it. Keyed by checkpoint
63
+ -- + session (a checkpoint may be injected once per session); repo_id is the
64
+ -- source repo (the foreign repo's stateDir) for tracking/source labels.
65
+ -- PRAMETERIZED queries (PREVENT-002); local node:sqlite (PREVENT-PI-004).
66
+ CREATE TABLE IF NOT EXISTS injected_global (
67
+ checkpoint_id TEXT NOT NULL,
68
+ repo_id TEXT NOT NULL,
69
+ session_id TEXT NOT NULL,
70
+ injected_at INTEGER NOT NULL,
71
+ PRIMARY KEY (checkpoint_id, session_id)
72
+ );
73
+ CREATE INDEX IF NOT EXISTS idx_injected_global_cid ON injected_global(checkpoint_id);
74
+ `);
75
+ indexCache = iddb;
76
+ indexCacheDir = indexDir;
77
+ return iddb;
78
+ }
79
+ /**
80
+ * Upsert a repo's aggregate stats into the global index. Called on repo-switch
81
+ * (infrequent). Preserves first_seen + the model columns on update (model is
82
+ * written separately by recordRepoModel so we never clobber it here with nulls).
83
+ */
84
+ export function upsertRepoRegistry(row, indexDir = getIndexDir()) {
85
+ const db = openIndexStore(indexDir);
86
+ const now = Date.now();
87
+ db.prepare(`INSERT INTO repo_registry
88
+ (repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
89
+ checkpoint_count, tokens_saved, compressed_original_bytes,
90
+ provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
91
+ VALUES (@repo_root, @display_name, @state_dir, @first_seen, @last_seen, @last_compacted_at,
92
+ @checkpoint_count, @tokens_saved, @compressed_original_bytes,
93
+ @provider, @provider_name, @model_name, @input_rate, @output_rate, @model_captured_at)
94
+ ON CONFLICT(repo_root) DO UPDATE SET
95
+ display_name = excluded.display_name,
96
+ state_dir = excluded.state_dir,
97
+ last_seen = COALESCE(excluded.last_seen, @now),
98
+ last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
99
+ checkpoint_count = excluded.checkpoint_count,
100
+ tokens_saved = excluded.tokens_saved,
101
+ compressed_original_bytes = excluded.compressed_original_bytes,
102
+ provider = COALESCE(excluded.provider, repo_registry.provider),
103
+ provider_name = COALESCE(excluded.provider_name, repo_registry.provider_name),
104
+ model_name = COALESCE(excluded.model_name, repo_registry.model_name),
105
+ input_rate = COALESCE(excluded.input_rate, repo_registry.input_rate),
106
+ output_rate = COALESCE(excluded.output_rate, repo_registry.output_rate),
107
+ model_captured_at = COALESCE(excluded.model_captured_at, repo_registry.model_captured_at)`).run({
108
+ repo_root: row.repoRoot,
109
+ display_name: row.displayName,
110
+ state_dir: row.stateDir,
111
+ now,
112
+ first_seen: row.firstSeen ?? null,
113
+ last_seen: row.lastSeen ?? null,
114
+ last_compacted_at: row.lastCompactedAt ?? null,
115
+ checkpoint_count: row.checkpointCount,
116
+ tokens_saved: row.tokensSaved,
117
+ compressed_original_bytes: row.compressedOriginalBytes,
118
+ provider: row.provider ?? null,
119
+ provider_name: row.providerName ?? null,
120
+ model_name: row.modelName ?? null,
121
+ input_rate: row.inputRate ?? null,
122
+ output_rate: row.outputRate ?? null,
123
+ model_captured_at: row.modelCapturedAt ?? null,
124
+ });
125
+ }
126
+ /**
127
+ * Record the active model/provider for a repo in the global index (denormalized
128
+ * so the All-repos table shows model without opening each repo's DB). Upserts a
129
+ * bare registry row if the repo isn't registered yet.
130
+ */
131
+ export function recordRepoModel(repoRoot, model, indexDir = getIndexDir()) {
132
+ const db = openIndexStore(indexDir);
133
+ const now = Date.now();
134
+ db.prepare(`INSERT INTO repo_registry
135
+ (repo_root, display_name, state_dir, first_seen, last_seen,
136
+ provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
137
+ VALUES (@repo_root, @display_name, @state_dir, @now, @now,
138
+ @provider, @provider_name, @model_name, @input_rate, @output_rate, @now)
139
+ ON CONFLICT(repo_root) DO UPDATE SET
140
+ last_seen = excluded.last_seen,
141
+ provider = excluded.provider,
142
+ provider_name = excluded.provider_name,
143
+ model_name = excluded.model_name,
144
+ input_rate = excluded.input_rate,
145
+ output_rate = excluded.output_rate,
146
+ model_captured_at = excluded.model_captured_at`).run({
147
+ repo_root: repoRoot,
148
+ display_name: model.displayName,
149
+ state_dir: model.stateDir,
150
+ now,
151
+ provider: model.provider,
152
+ provider_name: model.providerName,
153
+ model_name: model.modelName,
154
+ input_rate: model.inputRate,
155
+ output_rate: model.outputRate,
156
+ });
157
+ }
158
+ function mapRegistryRow(row) {
159
+ return {
160
+ repoRoot: row.repo_root,
161
+ displayName: row.display_name ?? "",
162
+ stateDir: row.state_dir,
163
+ firstSeen: row.first_seen ?? 0,
164
+ lastSeen: row.last_seen ?? 0,
165
+ lastCompactedAt: row.last_compacted_at ?? null,
166
+ checkpointCount: row.checkpoint_count ?? 0,
167
+ tokensSaved: row.tokens_saved ?? 0,
168
+ compressedOriginalBytes: row.compressed_original_bytes ?? 0,
169
+ provider: row.provider ?? null,
170
+ providerName: row.provider_name ?? null,
171
+ modelName: row.model_name ?? null,
172
+ inputRate: row.input_rate ?? null,
173
+ outputRate: row.output_rate ?? null,
174
+ modelCapturedAt: row.model_captured_at ?? null,
175
+ };
176
+ }
177
+ /** All registered repos, most-recently-seen first. */
178
+ export function listRepoRegistry(indexDir = getIndexDir()) {
179
+ const db = openIndexStore(indexDir);
180
+ const rows = db.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC").all();
181
+ return rows.map(mapRegistryRow);
182
+ }
183
+ /** A single repo's registry row, or undefined. */
184
+ export function getRepoRegistry(repoRoot, indexDir = getIndexDir()) {
185
+ const db = openIndexStore(indexDir);
186
+ const row = db.prepare("SELECT * FROM repo_registry WHERE repo_root = ?").get(repoRoot);
187
+ return row ? mapRegistryRow(row) : undefined;
188
+ }
189
+ /** Close the cached index connection (test teardown only). */
190
+ export function closeIndexStore() {
191
+ if (indexCache) {
192
+ indexCache.close();
193
+ indexCache = undefined;
194
+ indexCacheDir = undefined;
195
+ }
196
+ }
197
+ // ---------------------------------------------------------------------------
198
+ // S18: machine-wide injected-set (cross-repo dedup markers)
199
+ //
200
+ // A foreign checkpoint injected in repo A is recorded here so repo B's recall
201
+ // never re-injects it (a stronger, machine-wide version of the per-session
202
+ // injected-set in the local store). Keyed by (checkpoint_id, session_id); the
203
+ // session_id here is the RECEIVING session, so the same foreign checkpoint can
204
+ // be injected into different sessions but never twice into the same one.
205
+ // PRAMETERIZED queries (PREVENT-002); local node:sqlite + WAL (PREVENT-PI-004),
206
+ // multi-process safe.
207
+ // ---------------------------------------------------------------------------
208
+ /** Record that a (foreign) checkpoint was injected into `sessionId`. Idempotent. */
209
+ export function markInjectedGlobal(checkpointId, repoId, sessionId, indexDir = getIndexDir()) {
210
+ const db = openIndexStore(indexDir);
211
+ db.prepare("INSERT OR IGNORE INTO injected_global (checkpoint_id, repo_id, session_id, injected_at) VALUES ($cid, $rid, $sid, $ts)").run({ $cid: checkpointId, $rid: repoId, $sid: sessionId, $ts: Date.now() });
212
+ }
213
+ /** True when a checkpoint was already injected into `sessionId` (machine-wide). */
214
+ export function wasInjectedGlobal(checkpointId, sessionId, indexDir = getIndexDir()) {
215
+ const db = openIndexStore(indexDir);
216
+ const row = db.prepare("SELECT 1 FROM injected_global WHERE checkpoint_id = $cid AND session_id = $sid LIMIT 1").get({ $cid: checkpointId, $sid: sessionId });
217
+ return row !== undefined;
218
+ }
219
+ /** Count of cross-repo injections recorded (for /mega-status stats). */
220
+ export function countInjectedGlobal(indexDir = getIndexDir()) {
221
+ const db = openIndexStore(indexDir);
222
+ const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get();
223
+ return row?.n ?? 0;
224
+ }
@@ -0,0 +1,167 @@
1
+ /**
2
+ * Global machine-wide index store (Phase 5b).
3
+ *
4
+ * A single SQLite DB, separate from every per-repo store, that aggregates one
5
+ * row per repo this machine has run on. The multi-repo dashboard (Summary /
6
+ * All-repos tabs) reads it so ONE dashboard can show every repo's checkpoints,
7
+ * tokens saved, and active model — instead of a per-repo dashboard that only
8
+ * ever sees the repo it was launched from.
9
+ *
10
+ * Written by every pi process on repo-switch (bindRepo) + model capture; read by
11
+ * the dashboard server. Concurrency across 10+ pi processes is handled by WAL +
12
+ * infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
13
+ */
14
+ import { DatabaseSync } from "node:sqlite";
15
+ import { existsSync, mkdirSync } from "node:fs";
16
+ import { homedir, tmpdir } from "node:os";
17
+ import { join } from "node:path";
18
+ /** Resolve the machine-wide index directory (env-overridable). */
19
+ export function getIndexDir() {
20
+ const override = process.env.MEGACOMPACT_INDEX_DIR;
21
+ if (override && override.trim() !== "")
22
+ return override;
23
+ // homedir() can throw in exotic sandboxes; fall back to tmpdir.
24
+ try {
25
+ return join(homedir(), ".mega-compact-index");
26
+ }
27
+ catch {
28
+ return join(tmpdir(), ".mega-compact-index");
29
+ }
30
+ }
31
+ let indexCache;
32
+ let indexCacheDir;
33
+ /** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
34
+ export function openIndexStore(indexDir = getIndexDir()) {
35
+ if (indexCache && indexCacheDir === indexDir)
36
+ return indexCache;
37
+ if (!existsSync(indexDir))
38
+ mkdirSync(indexDir, { recursive: true });
39
+ const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
40
+ iddb.exec("PRAGMA journal_mode = WAL");
41
+ iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
42
+ iddb.exec(`
43
+ CREATE TABLE IF NOT EXISTS repo_registry (
44
+ repo_root TEXT PRIMARY KEY,
45
+ display_name TEXT,
46
+ state_dir TEXT NOT NULL,
47
+ first_seen INTEGER,
48
+ last_seen INTEGER,
49
+ last_compacted_at INTEGER,
50
+ checkpoint_count INTEGER DEFAULT 0,
51
+ tokens_saved INTEGER DEFAULT 0,
52
+ compressed_original_bytes INTEGER DEFAULT 0,
53
+ provider TEXT,
54
+ provider_name TEXT,
55
+ model_name TEXT,
56
+ input_rate REAL,
57
+ output_rate REAL,
58
+ model_captured_at INTEGER
59
+ );
60
+ CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
61
+ `);
62
+ indexCache = iddb;
63
+ indexCacheDir = indexDir;
64
+ return iddb;
65
+ }
66
+ /**
67
+ * Upsert a repo's aggregate stats into the global index. Called on repo-switch
68
+ * (infrequent). Preserves first_seen + the model columns on update (model is
69
+ * written separately by recordRepoModel so we never clobber it here with nulls).
70
+ */
71
+ export function upsertRepoRegistry(row, indexDir = getIndexDir()) {
72
+ const db = openIndexStore(indexDir);
73
+ const now = Date.now();
74
+ db.prepare(`INSERT INTO repo_registry
75
+ (repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
76
+ checkpoint_count, tokens_saved, compressed_original_bytes)
77
+ VALUES (@repo_root, @display_name, @state_dir, @now, @now, @last_compacted_at,
78
+ @checkpoint_count, @tokens_saved, @compressed_original_bytes)
79
+ ON CONFLICT(repo_root) DO UPDATE SET
80
+ display_name = excluded.display_name,
81
+ state_dir = excluded.state_dir,
82
+ last_seen = excluded.last_seen,
83
+ last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
84
+ checkpoint_count = excluded.checkpoint_count,
85
+ tokens_saved = excluded.tokens_saved,
86
+ compressed_original_bytes = excluded.compressed_original_bytes`).run({
87
+ repo_root: row.repoRoot,
88
+ display_name: row.displayName,
89
+ state_dir: row.stateDir,
90
+ now,
91
+ last_compacted_at: row.lastCompactedAt ?? null,
92
+ checkpoint_count: row.checkpointCount,
93
+ tokens_saved: row.tokensSaved,
94
+ compressed_original_bytes: row.compressedOriginalBytes,
95
+ });
96
+ }
97
+ /**
98
+ * Record the active model/provider for a repo in the global index (denormalized
99
+ * so the All-repos table shows model without opening each repo's DB). Upserts a
100
+ * bare registry row if the repo isn't registered yet.
101
+ */
102
+ export function recordRepoModel(repoRoot, model, indexDir = getIndexDir()) {
103
+ const db = openIndexStore(indexDir);
104
+ const now = Date.now();
105
+ db.prepare(`INSERT INTO repo_registry
106
+ (repo_root, display_name, state_dir, first_seen, last_seen,
107
+ provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
108
+ VALUES (@repo_root, @display_name, @state_dir, @now, @now,
109
+ @provider, @provider_name, @model_name, @input_rate, @output_rate, @now)
110
+ ON CONFLICT(repo_root) DO UPDATE SET
111
+ last_seen = excluded.last_seen,
112
+ provider = excluded.provider,
113
+ provider_name = excluded.provider_name,
114
+ model_name = excluded.model_name,
115
+ input_rate = excluded.input_rate,
116
+ output_rate = excluded.output_rate,
117
+ model_captured_at = excluded.model_captured_at`).run({
118
+ repo_root: repoRoot,
119
+ display_name: model.displayName,
120
+ state_dir: model.stateDir,
121
+ now,
122
+ provider: model.provider,
123
+ provider_name: model.providerName,
124
+ model_name: model.modelName,
125
+ input_rate: model.inputRate,
126
+ output_rate: model.outputRate,
127
+ });
128
+ }
129
+ function mapRegistryRow(row) {
130
+ return {
131
+ repoRoot: row.repo_root,
132
+ displayName: row.display_name ?? "",
133
+ stateDir: row.state_dir,
134
+ firstSeen: row.first_seen ?? 0,
135
+ lastSeen: row.last_seen ?? 0,
136
+ lastCompactedAt: row.last_compacted_at ?? null,
137
+ checkpointCount: row.checkpoint_count ?? 0,
138
+ tokensSaved: row.tokens_saved ?? 0,
139
+ compressedOriginalBytes: row.compressed_original_bytes ?? 0,
140
+ provider: row.provider ?? null,
141
+ providerName: row.provider_name ?? null,
142
+ modelName: row.model_name ?? null,
143
+ inputRate: row.input_rate ?? null,
144
+ outputRate: row.output_rate ?? null,
145
+ modelCapturedAt: row.model_captured_at ?? null,
146
+ };
147
+ }
148
+ /** All registered repos, most-recently-seen first. */
149
+ export function listRepoRegistry(indexDir = getIndexDir()) {
150
+ const db = openIndexStore(indexDir);
151
+ const rows = db.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC").all();
152
+ return rows.map(mapRegistryRow);
153
+ }
154
+ /** A single repo's registry row, or undefined. */
155
+ export function getRepoRegistry(repoRoot, indexDir = getIndexDir()) {
156
+ const db = openIndexStore(indexDir);
157
+ const row = db.prepare("SELECT * FROM repo_registry WHERE repo_root = ?").get(repoRoot);
158
+ return row ? mapRegistryRow(row) : undefined;
159
+ }
160
+ /** Close the cached index connection (test teardown only). */
161
+ export function closeIndexStore() {
162
+ if (indexCache) {
163
+ indexCache.close();
164
+ indexCache = undefined;
165
+ indexCacheDir = undefined;
166
+ }
167
+ }