pi-mega-compact 0.11.0 → 0.11.2

package/extensions/dashboard-server/routes-maintenance.ts

@@ -0,0 +1,345 @@
+/**
+ * dashboard-server/routes-maintenance.ts — S49B Maintenance tab API routes.
+ *
+ * Handlers for:
+ *   GET  /api/maintenance            — DB stats (table row counts + storage)
+ *   GET  /api/maintenance/schema-health — SCHEMA_VERSION + integrity audit
+ *   POST /api/maintenance/action     — Trigger a maintenance action
+ *
+ * All pi-agnostic; lazy-requires maintenance primitives to avoid a top-level
+ * import of the store from the dashboard server process. PREVENT-PI-004: local
+ * SQLite only; zero network calls.
+ */
+
+import { createRequire } from "node:module";
+import type { IncomingMessage, ServerResponse } from "node:http";
+
+import type { RouteContext } from "./routes-core.js";
+
+// ---------------------------------------------------------------------------
+// Lazy imports (via createRequire)
+// ---------------------------------------------------------------------------
+
+const lazyMaintenance = (() => {
+	let mod: typeof import("../../src/store/sqlite/maintenance.js") | null = null;
+	return () => {
+		if (!mod)
+			mod = createRequire(import.meta.url)(
+				"../../src/store/sqlite/maintenance.js",
+			) as typeof import("../../src/store/sqlite/maintenance.js");
+		return mod;
+	};
+})();
+
+const lazyStoreUtils = (() => {
+	let mod: typeof import("../../src/store/sqlite/utils.js") | null = null;
+	return () => {
+		if (!mod)
+			mod = createRequire(import.meta.url)(
+				"../../src/store/sqlite/utils.js",
+			) as typeof import("../../src/store/sqlite/utils.js");
+		return mod;
+	};
+})();
+
+// ---------------------------------------------------------------------------
+// Types for JSON deserialisation
+// ---------------------------------------------------------------------------
+
+interface MaintenanceActionBody {
+	action: string;
+	daysOld?: number | string;
+}
+
+// ---------------------------------------------------------------------------
+// GET /api/maintenance and sub-paths
+// ---------------------------------------------------------------------------
+
+export function handleMaintenance(
+	req: IncomingMessage,
+	res: ServerResponse,
+	ctx: RouteContext,
+): boolean {
+	const url = req.url ?? "";
+	if (!url.startsWith("/api/maintenance")) return false;
+
+	const method = (req.method ?? "GET").toUpperCase();
+
+	// ── GET /api/maintenance/schema-health ────────────────────────────
+	if (method === "GET" && url === "/api/maintenance/schema-health") {
+		sendSchemaHealth(res, ctx);
+		return true;
+	}
+
+	// ── POST /api/maintenance/action ──────────────────────────────────
+	if (method === "POST" && url === "/api/maintenance/action") {
+		handleMaintenanceAction(req, res, ctx);
+		return true;
+	}
+
+	// ── GET /api/maintenance ──────────────────────────────────────────
+	if (method === "GET" && url === "/api/maintenance") {
+		sendDbStats(res, ctx);
+		return true;
+	}
+
+	// ── Unknown sub-path ──────────────────────────────────────────────
+	sendJson(res, 404, { error: "not found" });
+	return true;
+}
+
+// ---------------------------------------------------------------------------
+// GET /api/maintenance — DB stats
+// ---------------------------------------------------------------------------
+
+function sendDbStats(res: ServerResponse, ctx: RouteContext): void {
+	try {
+		const { getDbStats } = lazyMaintenance();
+		const raw = getDbStats(ctx.stateDir);
+
+		const tables = Object.entries(raw.tableCounts).map(([table, rowCount]) => ({
+			table,
+			rowCount,
+		}));
+
+		sendJson(res, 200, {
+			type: "db-stats" as const,
+			tables,
+			storage: {
+				files: {
+					dbBytes: raw.dbBytes,
+					walBytes: raw.walBytes,
+					shmBytes: raw.shmBytes,
+				},
+				pageSize: raw.pageSize,
+				pageCount: raw.pageCount,
+				freelistPages: raw.freelistPages,
+			},
+		});
+	} catch (err) {
+		sendJson(res, 500, { error: (err as Error).message });
+	}
+}
+
+// ---------------------------------------------------------------------------
+// GET /api/maintenance/schema-health — schema version + integrity audit
+// ---------------------------------------------------------------------------
+
+function sendSchemaHealth(res: ServerResponse, ctx: RouteContext): void {
+	try {
+		const { integrityCheck } = lazyMaintenance();
+		const integrity = integrityCheck(ctx.stateDir);
+
+		const { openStore } = lazyStoreUtils();
+		const db = openStore(ctx.stateDir);
+
+		// SCHEMA_VERSION from meta table
+		const metaRow = db
+			.prepare("SELECT value FROM meta WHERE key = ?")
+			.get("schema_version") as { value: string } | undefined;
+		const schemaVersion = metaRow ? Number(metaRow.value) : 0;
+
+		// PRAGMA foreign_key_check
+		const fkRows = db
+			.prepare("PRAGMA foreign_key_check")
+			.all() as Array<Record<string, unknown>>;
+		const fkCheck = fkRows.map((r) => JSON.stringify(r));
+
+		// Per-column audit: every ensureColumn migration column declared in
+		// schema.ts, checked via PRAGMA table_info (read-only).
+		const expectedColumns: Array<{
+			table: string;
+			column: string;
+			decl: string;
+		}> = [
+			{ table: "context_chunks", column: "original_token_estimate", decl: "INTEGER" },
+			{ table: "raw_transcript", column: "content_ref", decl: "TEXT" },
+			{ table: "memories", column: "category", decl: "TEXT" },
+			{ table: "memories", column: "target", decl: "TEXT" },
+			{ table: "memories", column: "last_referenced", decl: "INTEGER" },
+			{ table: "memories", column: "source_turn", decl: "INTEGER" },
+			{ table: "raptor_nodes", column: "built_at", decl: "INTEGER" },
+			{ table: "raw_transcript", column: "turn_index", decl: "INTEGER" },
+			{ table: "session_state", column: "conversation_id", decl: "TEXT" },
+			{ table: "session_state", column: "last_turn_id", decl: "INTEGER" },
+		];
+
+		const columns = expectedColumns.map((ec) => {
+			const cols = db
+				.prepare(`PRAGMA table_info(${ec.table})`)
+				.all() as Array<{ name: string }>;
+			const present = cols.some((c) => c.name === ec.column);
+			return {
+				table: ec.table,
+				column: ec.column,
+				present,
+				expectedDecl: ec.decl,
+			};
+		});
+
+		const healthy =
+			integrity.length === 1 &&
+			integrity[0] === "ok" &&
+			columns.every((c) => c.present);
+
+		sendJson(res, 200, {
+			type: "schema-health" as const,
+			schemaVersion,
+			integrity,
+			fkCheck,
+			columns,
+			healthy,
+		});
+	} catch (err) {
+		sendJson(res, 500, { error: (err as Error).message });
+	}
+}
+
+// ---------------------------------------------------------------------------
+// POST /api/maintenance/action — Trigger a maintenance action
+// ---------------------------------------------------------------------------
+
+function handleMaintenanceAction(
+	req: IncomingMessage,
+	res: ServerResponse,
+	ctx: RouteContext,
+): void {
+	let body = "";
+	req.setEncoding("utf8");
+	req.on("data", (chunk: string) => {
+		body += chunk;
+		if (body.length > 10_000) req.destroy();
+	});
+	req.on("end", () => {
+		let payload: MaintenanceActionBody;
+		try {
+			payload = JSON.parse(body) as MaintenanceActionBody;
+			if (!payload || !payload.action) {
+				sendJson(res, 400, { error: "missing 'action' field" });
+				return;
+			}
+		} catch {
+			sendJson(res, 400, { error: "invalid JSON" });
+			return;
+		}
+
+			try {
+			const m = lazyMaintenance();
+			const u = lazyStoreUtils();
+			// All destructive actions: temp backup first
+			if (payload.action !== "integrity-check") {
+				// Non-blocking — backup failure is non-fatal
+				m.backupDb(ctx.stateDir);
+			}
+
+			switch (payload.action) {
+				case "vacuum": {
+					const r = m.vacuumDb(ctx.stateDir);
+					sendJson(res, 200, {
+						operation: "vacuum",
+						success: true,
+						affected: r.affected,
+						reclaimedBytes: r.reclaimedBytes,
+						summary: r.summary,
+					});
+					return;
+				}
+				case "checkpoint": {
+					const r = m.checkpointWal(ctx.stateDir);
+					sendJson(res, 200, {
+						operation: "checkpoint",
+						success: true,
+						affected: r.affected,
+						reclaimedBytes: r.reclaimedBytes,
+						summary: r.summary,
+					});
+					return;
+				}
+				case "reindex": {
+					const db = u.openStore(ctx.stateDir);
+					db.exec("REINDEX");
+					sendJson(res, 200, {
+						operation: "reindex",
+						success: true,
+						affected: 0,
+						reclaimedBytes: 0,
+						summary: "REINDEX completed",
+					});
+					return;
+				}
+				case "fts5-rebuild": {
+					const db = u.openStore(ctx.stateDir);
+					db.exec(
+						"INSERT INTO context_chunks_trgm(context_chunks_trgm) VALUES('rebuild')",
+					);
+					sendJson(res, 200, {
+						operation: "fts5-rebuild",
+						success: true,
+						affected: 0,
+						reclaimedBytes: 0,
+						summary: "FTS5 rebuild triggered",
+					});
+					return;
+				}
+				case "reconcile-dedup": {
+					const r = m.reconcileDedupMirror(ctx.stateDir);
+					const lines: string[] = [];
+					if (r.fixedRefCount) lines.push(`${r.fixedRefCount} ref_counts corrected`);
+					if (r.orphansDeleted) lines.push(`${r.orphansDeleted} orphans deleted`);
+					if (r.refsBackfilled) lines.push(`${r.refsBackfilled} refs backfilled`);
+					sendJson(res, 200, {
+						operation: "reconcile-dedup",
+						success: true,
+						affected: r.fixedRefCount + r.orphansDeleted + r.refsBackfilled,
+						reclaimedBytes: 0,
+						summary: lines.length ? lines.join(", ") : "nothing to fix",
+					});
+					return;
+				}
+				case "prune": {
+					const days = Number(payload.daysOld) || 30;
+					const r = m.pruneOldRows(ctx.stateDir, days);
+					sendJson(res, 200, {
+						operation: "prune",
+						success: true,
+						affected: r.affected,
+						reclaimedBytes: r.reclaimedBytes,
+						summary: r.summary,
+					});
+					return;
+				}
+				case "integrity-check": {
+					const lines = m.integrityCheck(ctx.stateDir);
+					sendJson(res, 200, {
+						operation: "integrity-check",
+						success: lines.length === 1 && lines[0] === "ok",
+						affected: 0,
+						reclaimedBytes: 0,
+						summary: lines.join("; "),
+						detail: lines,
+					});
+					return;
+				}
+				default:
+					sendJson(res, 400, {
+						error: `unknown action: ${payload.action}`,
+					});
+			}
+		} catch (err) {
+			sendJson(res, 500, { error: (err as Error).message });
+		}
+	});
+}
+
+// ---------------------------------------------------------------------------
+// JSON helper
+// ---------------------------------------------------------------------------
+
+function sendJson(res: ServerResponse, status: number, data: unknown): void {
+	const body = JSON.stringify(data);
+	res.writeHead(status, {
+		"Content-Type": "application/json",
+		"Content-Length": Buffer.byteLength(body),
+	});
+	res.end(body);
+}

package/extensions/dashboard-server/routes.ts

@@ -18,3 +18,4 @@ export {
 export { handleEvents, handleSessions } from "./routes-sessions.js";
 export { handleTopics } from "./routes-topics.js";
 export { handleTurns } from "./routes-turns.js";
+export { handleMaintenance } from "./routes-maintenance.js";

package/extensions/dashboard-server/server.ts

@@ -36,6 +36,7 @@ import {
 	handleSessions,
 	handleTopics,
 	handleTurns,
+	handleMaintenance,
 	handleStatic,
 } from "./routes.js";
 
@@ -223,6 +224,7 @@ export async function launchDashboardServer(
 		if (handleSessions(req, res, ctx)) return;
 		if (handleTopics(req, res, ctx)) return;
 		if (handleTurns(req, res, ctx)) return;
+		if (handleMaintenance(req, res, ctx)) return;
 		handleStatic(req, res, ctx);
 	});
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.11.0",
+  "version": "0.11.2",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/store/sqlite/maintenance.ts

@@ -4,8 +4,9 @@
  * All pi-agnostic, all parameterized (PREVENT-002), all local (PREVENT-PI-004).
  * Exposed via the /mega-db-* slash commands in extensions/mega-db-cmds.ts.
  */
-import { statSync } from "node:fs";
+import { statSync, mkdirSync, existsSync } from "node:fs";
 import { join } from "node:path";
+import { tmpdir } from "node:os";
 import { getStateDir } from "../../store.js";
 import { openStore, withTx } from "./utils.js";
 
@@ -100,6 +101,24 @@ export interface MaintenanceResult {
   summary: string;
 }
 
+/**
+ * Lightweight temp backup of the main sqlite.db via VACUUM INTO.
+ * 
+ * Creates a one-off backup in the OS tmpdir. Returns the backup path.
+ * VACUUM INTO produces a clean, defragmented copy — also serves as a
+ * pre-op safeguard for any destructive maintenance action.
+ */
+export function backupDb(stateDir: string = getStateDir()): string {
+  const backupDir = join(tmpdir(), "pi-mega-compact-backups");
+  if (!existsSync(backupDir)) mkdirSync(backupDir, { recursive: true });
+  const ts = Date.now();
+  const backupPath = join(backupDir, `sqlite-${ts}.db`);
+  const db = openStore(stateDir);
+  // guardrails-disable-next-line PREVENT-002: path built from tmpdir + timestamp, not user input
+  db.exec(`VACUUM INTO '${backupPath.replace(/'/g, "''")}'`);
+  return backupPath;
+}
+
 /**
  * Prune raw_transcript + checkpoint_epochs rows older than `daysOld`.
  * Uses `message_timestamp` (raw_transcript) and `created_at` (epochs), both

package/src/store/sqlite/schema.ts

@@ -1,7 +1,7 @@
 /**
  * schema.ts — table creation, migrations, `ensureColumn`, PRAGMA setup.
  */
-import { DatabaseSync } from "node:sqlite";
+import type { DatabaseSync } from "node:sqlite";
 import { ACHIEVEMENT_DEFS } from "../../game/scoring.js";
 
 const SCHEMA_VERSION = 2;
@@ -13,14 +13,21 @@ const SCHEMA_VERSION = 2;
  * input), so the unavoidable identifier interpolation here does not violate
  * PREVENT-002 (no external data reaches this SQL).
  */
-export function ensureColumn(db: DatabaseSync, table: string, column: string, decl: string): void {
-  const cols = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
-  if (cols.some((c) => c.name === column)) return;
-  db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${decl}`);
+export function ensureColumn(
+	db: DatabaseSync,
+	table: string,
+	column: string,
+	decl: string,
+): void {
+	const cols = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{
+		name: string;
+	}>;
+	if (cols.some((c) => c.name === column)) return;
+	db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${decl}`);
 }
 
 export function initSchema(db: DatabaseSync): void {
-  db.exec(`
+	db.exec(`
     CREATE TABLE IF NOT EXISTS context_chunks (
       id                  TEXT NOT NULL,
       session_id         TEXT NOT NULL,
@@ -111,10 +118,6 @@ export function initSchema(db: DatabaseSync): void {
     );
     CREATE INDEX IF NOT EXISTS idx_raptor_session ON raptor_nodes(session_id);
     CREATE INDEX IF NOT EXISTS idx_raptor_parent ON raptor_nodes(parent_id);
-    -- S42D/QA perf: session-scoped built_at ordering so the per-search cache
-    -- invalidation MAX(built_at) is index-satisfied.
-    CREATE INDEX IF NOT EXISTS idx_raptor_session_built
-      ON raptor_nodes(session_id, built_at DESC);
 
     -- S42D: structured RAPTOR build history. One row per tree build; the
     -- freshness check (buildHistory.ts) uses completed_at + leaf_count to skip
@@ -364,45 +367,55 @@ export function initSchema(db: DatabaseSync): void {
     CREATE INDEX IF NOT EXISTS idx_perf_samples_ts ON perf_samples(ts);
     CREATE INDEX IF NOT EXISTS idx_perf_samples_kind_ts ON perf_samples(kind, ts);
   `);
-  // Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
-  // pre-existing table, so new columns added to context_chunks after a store was
-  // first created (e.g. original_token_estimate in v0.4.2) must be ALTERed in for
-  // databases created by an older version — otherwise repoStats()/upsert crash
-  // with "no such column" and the extension fails to load. Additive only.
-  ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
-  // S27 Task 6: content_ref column in raw_transcript for dedup_mirror references.
-  ensureColumn(db, "raw_transcript", "content_ref", "TEXT");
-  // S20 memory-RAG extension: additive columns for auto-review ops. Idempotent —
-  // only alters DBs created by an older version that lack these columns.
-  ensureColumn(db, "memories", "category", "TEXT");
-  ensureColumn(db, "memories", "target", "TEXT");
-  ensureColumn(db, "memories", "last_referenced", "INTEGER");
-  ensureColumn(db, "memories", "source_turn", "INTEGER");
-  // S25: RAPTOR freshness-guard timestamp. Additive; old DBs have NULL → 0 →
-  // treated as stale → flat fallback (safe).
-  ensureColumn(db, "raptor_nodes", "built_at", "INTEGER");
-  // S43: turn_index on raw_transcript so a message points directly at its
-  // conversation turn (otherwise it must be inferred from seq ordering). NULL
-  // for legacy rows — turns written before S43 have no turn link.
-  ensureColumn(db, "raw_transcript", "turn_index", "INTEGER");
-  // S43: conversation_id + last_turn_id on session_state (legacy DBs have NULL).
-  ensureColumn(db, "session_state", "conversation_id", "TEXT");
-  ensureColumn(db, "session_state", "last_turn_id", "INTEGER");
-  // S35: idempotent seed of the 9 achievement rows. ON CONFLICT(id) DO
-  // NOTHING so a re-open never clobbers an already-unlocked row's
-  // unlocked_at. No user input reaches this SQL (PREVENT-002 safe).
-  const seedAch = db.prepare(
-    `INSERT INTO game_achievements (id, title, description, hidden, icon)
+	// Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
+	// pre-existing table, so new columns added to context_chunks after a store was
+	// first created (e.g. original_token_estimate in v0.4.2) must be ALTERed in for
+	// databases created by an older version — otherwise repoStats()/upsert crash
+	// with "no such column" and the extension fails to load. Additive only.
+	ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
+	// S27 Task 6: content_ref column in raw_transcript for dedup_mirror references.
+	ensureColumn(db, "raw_transcript", "content_ref", "TEXT");
+	// S20 memory-RAG extension: additive columns for auto-review ops. Idempotent —
+	// only alters DBs created by an older version that lack these columns.
+	ensureColumn(db, "memories", "category", "TEXT");
+	ensureColumn(db, "memories", "target", "TEXT");
+	ensureColumn(db, "memories", "last_referenced", "INTEGER");
+	ensureColumn(db, "memories", "source_turn", "INTEGER");
+	// S25: RAPTOR freshness-guard timestamp. Additive; old DBs have NULL → 0 →
+	// treated as stale → flat fallback (safe).
+	ensureColumn(db, "raptor_nodes", "built_at", "INTEGER");
+	// S42D/QA perf: session-scoped built_at ordering. Must follow ensureColumn
+	// above — old DBs already have raptor_nodes without built_at, so CREATE TABLE
+	// IF NOT EXISTS is a no-op that doesn't add the column, and moving this index
+	// inside the DDL block caused "no such column: built_at" on upgrade.
+	db.exec(
+		"CREATE INDEX IF NOT EXISTS idx_raptor_session_built ON raptor_nodes(session_id, built_at DESC)",
+	);
+	// S43: turn_index on raw_transcript so a message points directly at its
+	// conversation turn (otherwise it must be inferred from seq ordering). NULL
+	// for legacy rows — turns written before S43 have no turn link.
+	ensureColumn(db, "raw_transcript", "turn_index", "INTEGER");
+	// S43: conversation_id + last_turn_id on session_state (legacy DBs have NULL).
+	ensureColumn(db, "session_state", "conversation_id", "TEXT");
+	ensureColumn(db, "session_state", "last_turn_id", "INTEGER");
+	// S35: idempotent seed of the 9 achievement rows. ON CONFLICT(id) DO
+	// NOTHING so a re-open never clobbers an already-unlocked row's
+	// unlocked_at. No user input reaches this SQL (PREVENT-002 safe).
+	const seedAch = db.prepare(
+		`INSERT INTO game_achievements (id, title, description, hidden, icon)
      VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO NOTHING`,
-  );
-  for (const d of ACHIEVEMENT_DEFS) {
-    seedAch.run(d.id, d.title, d.description, d.hidden ? 1 : 0, d.icon);
-  }
+	);
+	for (const d of ACHIEVEMENT_DEFS) {
+		seedAch.run(d.id, d.title, d.description, d.hidden ? 1 : 0, d.icon);
+	}
 
-  const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get() as
-    | { value: string }
-    | undefined;
-  if (!v) {
-    db.prepare("INSERT INTO meta(key, value) VALUES(?, ?)").run("schema_version", String(SCHEMA_VERSION));
-  }
+	const v = db
+		.prepare("SELECT value FROM meta WHERE key='schema_version'")
+		.get() as { value: string } | undefined;
+	if (!v) {
+		db.prepare("INSERT INTO meta(key, value) VALUES(?, ?)").run(
+			"schema_version",
+			String(SCHEMA_VERSION),
+		);
+	}
 }