pi-mega-compact 0.9.2 → 0.10.1

package/src/store/turns/migrations.ts

@@ -0,0 +1,203 @@
+/**
+ * migrations.ts — S49B one-time move of turn tables main-db → turns.db.
+ *
+ * S48 stored turns/turn_recall/conversation_branches inside the main sqlite.db.
+ * S49 isolates them into turns.db with a contract-first schema (master
+ * reconciliation): `conversation_branches` → `conversation_forks`, `model_id` →
+ * `model`, strict `role`/`source`/`pressure_band` enums, and `ended_at` NOT NULL.
+ *
+ * On first open with the flag ON, existing rows are COPIED across (ATTACH +
+ * explicit transformed INSERT) then the legacy tables are DROPPED. Idempotent
+ * (a turns_meta marker) and reversible (legacy tables remain in the main
+ * schema for one release so flag-OFF never loses history). Copy+drop is
+ * wrapped in a single transaction: a crash mid-copy rolls back and retries on
+ * next open. Non-fatal: any failure logs and leaves legacy intact.
+ *
+ * PREVENT-002: parameterized queries; ATTACH path is bound, not concatenated.
+ * PREVENT-PI-004: pure node:sqlite, no network.
+ */
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import type { DatabaseSync } from "node:sqlite";
+import { withTx } from "./connection.js";
+
+/** Main memory DB filename (must match src/store/sqlite/utils.ts). */
+const MAIN_DB_FILE = "sqlite.db";
+
+/**
+ * Legacy tables dropped from the main db after a successful copy. The
+ * DROPs below use these as literal identifiers (no interpolation) so there
+ * is no injection surface.
+ */
+
+const MIGRATED_KEY = "migrated_from_main";
+
+function isMigrated(db: DatabaseSync): boolean {
+	const row = db
+		.prepare("SELECT value FROM turns_meta WHERE key = ?")
+		.get(MIGRATED_KEY) as { value: string } | undefined;
+	return row?.value === "1";
+}
+
+function markMigrated(db: DatabaseSync): void {
+	db.prepare(
+		"INSERT OR REPLACE INTO turns_meta (key, value) VALUES (?, '1')",
+	).run(MIGRATED_KEY);
+}
+
+/**
+ * Map a legacy recall `source` value onto the contract enum
+ * (`checkpoint` | `cluster_summary` | `memory`). Legacy free-text values:
+ *  - `flat` / `checkpoint`        → `checkpoint`
+ *  - `raptor` / `cluster_summary` → `cluster_summary`
+ *  - `cross-repo` / `memory` / * → `memory`
+ */
+function mapSource(legacy: string): string {
+	switch (legacy) {
+		case "flat":
+		case "checkpoint":
+			return "checkpoint";
+		case "raptor":
+		case "cluster_summary":
+			return "cluster_summary";
+		default:
+			return "memory";
+	}
+}
+
+/**
+ * Move legacy turn tables out of the main db into turns.db, once. Transforms
+ * rows to the unified contract schema (column renames + enum coercion + NULL
+ * backfill). Idempotent + non-fatal.
+ *
+ * @param turnDb   the open isolated turns.db handle
+ * @param stateDir the state dir containing sqlite.db
+ * @param log      optional logger (defaults to console.warn on failure)
+ */
+export function migrateTurnTablesIfNeeded(
+	turnDb: DatabaseSync,
+	stateDir: string,
+	log: (msg: string) => void = (m) => console.warn(`[turns-migrate] ${m}`),
+): void {
+	if (isMigrated(turnDb)) return; // already done — idempotent no-op.
+
+	const mainPath = join(stateDir, MAIN_DB_FILE);
+	if (!existsSync(mainPath)) {
+		// No legacy main db (fresh install / test dir) — nothing to move.
+		markMigrated(turnDb);
+		return;
+	}
+
+	try {
+		turnDb.prepare("ATTACH DATABASE ? AS legacy").run(mainPath);
+		try {
+			const hasTurns = turnDb
+				.prepare(
+					"SELECT name FROM legacy.sqlite_master WHERE type = 'table' AND name = 'turns'",
+				)
+				.get() as { name: string } | undefined;
+			if (!hasTurns) {
+				// Main db predates S48 turn tables — nothing to move.
+				markMigrated(turnDb);
+				return;
+			}
+
+			withTx(turnDb, () => {
+				// turns: rename model_id→model, coerce role/ended_at non-null, clamp
+				// pressure_band to the enum. Idempotent on (conversation_id, turn_index).
+				turnDb.exec(`
+          INSERT OR IGNORE INTO main.turns
+            (id, conversation_id, session_id, turn_index, role, started_at, ended_at,
+             ctx_tokens, ctx_percent, pressure_band, model, epoch_id)
+          SELECT
+            id, conversation_id, session_id, turn_index,
+            COALESCE(NULLIF(role, ''), 'user') AS role,
+            started_at,
+            COALESCE(ended_at, started_at, 0) AS ended_at,
+            ctx_tokens, ctx_percent,
+            CASE pressure_band
+              WHEN 'green'  THEN 'green'
+              WHEN 'yellow' THEN 'yellow'
+              WHEN 'red'    THEN 'red'
+              ELSE NULL
+            END AS pressure_band,
+            model_id AS model,
+            epoch_id
+          FROM legacy.turns
+        `);
+
+				// turn_recall: coerce source onto the contract enum. Idempotent on
+				// (turn_id, checkpoint_id).
+				turnDb.exec(`
+          INSERT OR IGNORE INTO main.turn_recall
+            (id, turn_id, checkpoint_id, score, source, raptor_level)
+          SELECT
+            id, turn_id, checkpoint_id, score,
+            CASE source
+              WHEN 'flat'            THEN 'checkpoint'
+              WHEN 'checkpoint'      THEN 'checkpoint'
+              WHEN 'raptor'           THEN 'cluster_summary'
+              WHEN 'cluster_summary' THEN 'cluster_summary'
+              ELSE 'memory'
+            END AS source,
+            raptor_level
+          FROM legacy.turn_recall
+        `);
+
+				// conversation_branches → conversation_forks (rename + column map).
+				// Legacy: (conversation_id PK, parent_conversation_id, fork_turn_id, created_at).
+				// Contract: (id, parent_conversation_id, child_conversation_id, fork_turn_index, created_at).
+				turnDb.exec(`
+          INSERT OR IGNORE INTO main.conversation_forks
+            (parent_conversation_id, child_conversation_id, fork_turn_index, created_at)
+          SELECT
+            parent_conversation_id,
+            conversation_id AS child_conversation_id,
+            fork_turn_id AS fork_turn_index,
+            created_at
+          FROM legacy.conversation_branches
+        `);
+
+				// session_conversations: hydrate from legacy session_state if it has a
+				// conversation_id column (S48 stored the active conversation there).
+				const hasSessionState = turnDb
+					.prepare(
+						"SELECT name FROM legacy.sqlite_master WHERE type = 'table' AND name = 'session_state'",
+					)
+					.get() as { name: string } | undefined;
+				if (hasSessionState) {
+					const cols = turnDb
+						.prepare("PRAGMA legacy.table_info(session_state)")
+						.all() as Array<{ name: string }>;
+					if (cols.some((c) => c.name === "conversation_id")) {
+						turnDb.exec(`
+              INSERT OR IGNORE INTO main.session_conversations (session_id, conversation_id)
+              SELECT session_id, conversation_id
+              FROM legacy.session_state
+              WHERE conversation_id IS NOT NULL
+            `);
+					}
+				}
+
+				// Drop legacy tables from the main db only after a successful copy.
+				// These are literal identifiers from the fixed LEGACY_TABLES allowlist
+				// (not user input) — SQLite cannot parameterize DDL identifiers, so a
+				// literal per table is the provably-injection-free form.
+				turnDb.exec("DROP TABLE IF EXISTS legacy.turns");
+				turnDb.exec("DROP TABLE IF EXISTS legacy.turn_recall");
+				turnDb.exec("DROP TABLE IF EXISTS legacy.conversation_branches");
+				markMigrated(turnDb);
+			});
+		} finally {
+			turnDb.exec("DETACH DATABASE legacy");
+		}
+	} catch (e) {
+		// Non-fatal: leave legacy tables intact so nothing is lost; retry next open.
+		log(
+			`migration failed (will retry on next open): ${e instanceof Error ? e.message : String(e)}`,
+		);
+	}
+}
+
+// Exported for tests that need to inspect the source mapping.
+export { mapSource };

package/src/store/turns/schema.ts

@@ -0,0 +1,158 @@
+/**
+ * schema.ts — S49 unified turn-store schema (isolated turns.db).
+ *
+ * RECONCILIATION (s49-turn-db ∪ master): the contract-first shapes from
+ * master's SqliteTurnStore (`turns` / `turn_recall` / `conversation_forks` /
+ * `session_conversations` / `turns_meta`, with the strict enums the contract
+ * `TurnEntry`/`TurnRecallEntry` requires) PLUS the additive shells from this
+ * branch's program: `turns.epoch_id` (S50B epoch linking), `pending_fork`
+ * (S52 rewind handshake), and `topics` / `memory_topics` (S51 wiki). No other
+ * sprint needs a new migration — everything is pre-created here.
+ *
+ * Idempotent (all `IF NOT EXISTS`). Single statement per `exec` so a failure
+ * names the exact table. Pi-agnostic (PREVENT-PI-004: pure node:sqlite).
+ * PREVENT-002: DDL only; all runtime queries are parameterized elsewhere.
+ */
+import type { DatabaseSync } from "node:sqlite";
+
+/** Schema version stamp (written once into turns_meta). */
+const SCHEMA_VERSION = 1;
+
+/** Create the unified turn-store schema if absent. Idempotent. */
+export function initTurnSchema(db: DatabaseSync): void {
+	// ── Contract tables (master's SqliteTurnStore shape) ──────────────────
+	db.exec(`
+    CREATE TABLE IF NOT EXISTS turns_meta (
+      key   TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    )
+  `);
+
+	// One row per turn_end. role/source/pressure_band use the contract enums.
+	// epoch_id is the S50B additive column (links a turn to the compact epoch).
+	db.exec(`
+    CREATE TABLE IF NOT EXISTS turns (
+      id              INTEGER PRIMARY KEY AUTOINCREMENT,
+      conversation_id TEXT    NOT NULL,
+      session_id      TEXT    NOT NULL,
+      turn_index      INTEGER NOT NULL,
+      role            TEXT    NOT NULL CHECK(role IN ('user','assistant','system','tool')),
+      started_at      INTEGER,
+      ended_at        INTEGER NOT NULL,
+      ctx_tokens      INTEGER,
+      ctx_percent     REAL,
+      pressure_band   TEXT    CHECK(pressure_band IS NULL OR pressure_band IN ('green','yellow','red')),
+      model           TEXT,
+      epoch_id        TEXT,
+      UNIQUE(conversation_id, turn_index)
+    )
+  `);
+	db.exec(
+		`CREATE INDEX IF NOT EXISTS idx_turns_conversation ON turns(conversation_id, turn_index)`,
+	);
+	db.exec(`CREATE INDEX IF NOT EXISTS idx_turns_session ON turns(session_id)`);
+	db.exec(`CREATE INDEX IF NOT EXISTS idx_turns_ended_at ON turns(ended_at)`);
+	db.exec(
+		`CREATE INDEX IF NOT EXISTS idx_turns_pressure ON turns(pressure_band) WHERE pressure_band IS NOT NULL`,
+	);
+	db.exec(
+		`CREATE INDEX IF NOT EXISTS idx_turns_epoch ON turns(epoch_id) WHERE epoch_id IS NOT NULL`,
+	);
+
+	// Recall provenance — which checkpoints / cluster summaries were injected
+	// at a turn, with score + source path. source uses the contract enum.
+	db.exec(`
+    CREATE TABLE IF NOT EXISTS turn_recall (
+      id            INTEGER PRIMARY KEY AUTOINCREMENT,
+      turn_id       INTEGER NOT NULL REFERENCES turns(id) ON DELETE CASCADE,
+      checkpoint_id TEXT    NOT NULL,
+      score         REAL    NOT NULL,
+      source        TEXT    NOT NULL CHECK(source IN ('checkpoint','cluster_summary','memory')),
+      raptor_level  INTEGER,
+      UNIQUE(turn_id, checkpoint_id)
+    )
+  `);
+	db.exec(
+		`CREATE INDEX IF NOT EXISTS idx_turn_recall_turn ON turn_recall(turn_id)`,
+	);
+	db.exec(
+		`CREATE INDEX IF NOT EXISTS idx_turn_recall_cp ON turn_recall(checkpoint_id)`,
+	);
+
+	// Conversation fork registry (contract: ConversationFork).
+	db.exec(`
+    CREATE TABLE IF NOT EXISTS conversation_forks (
+      id                     INTEGER PRIMARY KEY AUTOINCREMENT,
+      parent_conversation_id TEXT NOT NULL,
+      child_conversation_id  TEXT NOT NULL,
+      fork_turn_index        INTEGER NOT NULL,
+      created_at             INTEGER NOT NULL,
+      UNIQUE(child_conversation_id)
+    )
+  `);
+	db.exec(
+		`CREATE INDEX IF NOT EXISTS idx_forks_parent ON conversation_forks(parent_conversation_id)`,
+	);
+
+	// Session → active conversation map (contract: one current conversation per session).
+	db.exec(`
+    CREATE TABLE IF NOT EXISTS session_conversations (
+      session_id      TEXT PRIMARY KEY,
+      conversation_id TEXT NOT NULL
+    )
+  `);
+
+	// ── Program additive shells (this branch) ───────────────────────────
+	// S52: rewind-and-fork intents written by an external surface (dashboard)
+	// and consumed by the host at a safe lifecycle point. Pre-created → S52
+	// adds no migration.
+	db.exec(`
+    CREATE TABLE IF NOT EXISTS pending_fork (
+      id                     INTEGER PRIMARY KEY AUTOINCREMENT,
+      target_conversation_id TEXT NOT NULL,
+      target_turn_id         INTEGER NOT NULL,
+      requested_at           INTEGER NOT NULL,
+      consumed_at            INTEGER
+    )
+  `);
+	db.exec(
+		`CREATE INDEX IF NOT EXISTS idx_pending_fork_unconsumed ON pending_fork(consumed_at) WHERE consumed_at IS NULL`,
+	);
+
+	// S51: auto-categorizing wiki topics (k-means over real embeddings + TF-IDF
+	// labels — see docs/specs/s47-auto-categorizing-wiki.md). Pre-created → S51
+	// adds no migration. Derived at rebuild time; no seed data.
+	db.exec(`
+    CREATE TABLE IF NOT EXISTS topics (
+      id                     TEXT PRIMARY KEY,
+      label                  TEXT NOT NULL,
+      term_scores            TEXT,
+      memory_count           INTEGER DEFAULT 0,
+      last_updated           INTEGER,
+      cluster_model_built_at INTEGER
+    )
+  `);
+	db.exec(`
+    CREATE TABLE IF NOT EXISTS memory_topics (
+      memory_id   TEXT NOT NULL,
+      topic_id    TEXT NOT NULL REFERENCES topics(id),
+      confidence  REAL,
+      assigned_at INTEGER,
+      method      TEXT CHECK(method IN ('kmeans+tfidf')),
+      PRIMARY KEY (memory_id, topic_id)
+    )
+  `);
+	db.exec(
+		`CREATE INDEX IF NOT EXISTS idx_memory_topics_topic ON memory_topics(topic_id)`,
+	);
+
+	// Stamp schema version once.
+	const existing = db
+		.prepare("SELECT value FROM turns_meta WHERE key = 'schema_version'")
+		.get() as { value: string } | undefined;
+	if (!existing) {
+		db.prepare(
+			"INSERT INTO turns_meta (key, value) VALUES ('schema_version', ?)",
+		).run(String(SCHEMA_VERSION));
+	}
+}

package/src/store/turns/sqlite-store.test.ts

@@ -0,0 +1,161 @@
+/**
+ * sqlite-store.test.ts — SqliteTurnStore compliance + SQLite-specific tests.
+ *
+ * Compliance: shared suite run in-memory (fast).
+ * SQLite-specific: file-backed tests verifying:
+ *   - Separate turns.db (not sqlite.db)
+ *   - WAL mode active
+ *   - Connection caching (same path → same connection)
+ *   - dbSizeBytes reports file size
+ *   - File persistence across store instances
+ */
+
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, statSync, rmSync, mkdtempSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { SqliteTurnStore } from "./sqlite-store.js";
+import { runComplianceSuite } from "./contract-compliance.test.js";
+import { closeAllTurnDbs } from "./connection.js";
+import type { TurnEntry } from "./types.js";
+
+// ── Compliance suite (in-memory, fast) ──────────────────────────
+
+runComplianceSuite(
+	"SqliteTurnStore",
+	(options) => new SqliteTurnStore(options),
+	{ stateDir: join(tmpdir(), "turns-compliance-sqlite"), inMemory: true },
+);
+
+// ── SQLite-specific file-backed tests ───────────────────────────
+
+describe("SqliteTurnStore — file-backed", () => {
+	let stateDir: string;
+	let store: SqliteTurnStore;
+
+	beforeEach(() => {
+		closeAllTurnDbs(); // clear any cached connections
+		stateDir = mkdtempSync(join(tmpdir(), "turns-file-"));
+		store = new SqliteTurnStore({ stateDir });
+	});
+
+	afterEach(() => {
+		try {
+			store.close();
+		} catch {
+			// best-effort
+		}
+		closeAllTurnDbs();
+		try {
+			rmSync(stateDir, { recursive: true, force: true });
+		} catch {
+			// best-effort
+		}
+	});
+
+	it("creates turns.db separate from sqlite.db", () => {
+		const turnsDbPath = join(stateDir, "turns.db");
+		assert.ok(existsSync(turnsDbPath), "turns.db should exist");
+
+		// sqlite.db should NOT be created by the turns store
+		const sqliteDbPath = join(stateDir, "sqlite.db");
+		assert.ok(!existsSync(sqliteDbPath), "sqlite.db should NOT be created");
+	});
+
+	it("uses WAL journal mode", () => {
+		const row = (store as unknown as { db: { prepare: (s: string) => { get: () => unknown } } })
+			.db.prepare("PRAGMA journal_mode")
+			.get() as { journal_mode: string };
+		assert.equal(row.journal_mode, "wal");
+	});
+
+	it("caches connections for the same path", () => {
+		// Creating a second store with the same stateDir should reuse
+		// the cached connection (not create a new file lock conflict)
+		const store2 = new SqliteTurnStore({ stateDir });
+		try {
+			// Both should be able to write without locking errors
+			const entry: TurnEntry = {
+				conversationId: "conv_cache",
+				sessionId: "sess_cache",
+				turnIndex: 0,
+				role: "assistant",
+				endedAt: Date.now(),
+			};
+			store.asWriter().appendTurn(entry);
+			store2.asWriter().appendTurn({
+				...entry,
+				turnIndex: 1,
+			});
+
+			assert.equal(store.asReader().countTurns("conv_cache"), 2);
+		} finally {
+			store2.close();
+		}
+	});
+
+	it("reports dbSizeBytes for file-backed store", () => {
+		const entry: TurnEntry = {
+			conversationId: "conv_size",
+			sessionId: "sess_size",
+			turnIndex: 0,
+			role: "assistant",
+			endedAt: Date.now(),
+		};
+		store.asWriter().appendTurn(entry);
+
+		// prune returns a PruneReport with freedBytes; we can also
+		// verify the file exists and has content
+		const turnsDbPath = join(stateDir, "turns.db");
+		const size = statSync(turnsDbPath).size;
+		assert.ok(size > 0, "turns.db should have non-zero size after writes");
+	});
+
+	it("persists data across store instances", () => {
+		const entry: TurnEntry = {
+			conversationId: "conv_persist",
+			sessionId: "sess_persist",
+			turnIndex: 0,
+			role: "assistant",
+			endedAt: Date.now(),
+			model: "test-model",
+		};
+		store.asWriter().appendTurn(entry);
+		store.close();
+		closeAllTurnDbs();
+
+		// Re-open the same stateDir
+		const store2 = new SqliteTurnStore({ stateDir });
+		try {
+			const turns = store2.asReader().query({ conversationId: "conv_persist" });
+			assert.equal(turns.length, 1);
+			assert.equal(turns[0].model, "test-model");
+		} finally {
+			store2.close();
+			closeAllTurnDbs();
+		}
+	});
+
+	it("supports custom dbPath", () => {
+		const customPath = join(stateDir, "custom-turns.db");
+		const customStore = new SqliteTurnStore({
+			stateDir,
+			dbPath: customPath,
+		});
+		try {
+			assert.ok(existsSync(customPath), "custom db path should be created");
+			customStore.asWriter().appendTurn({
+				conversationId: "conv_custom",
+				sessionId: "sess_custom",
+				turnIndex: 0,
+				role: "user",
+				endedAt: Date.now(),
+			});
+			assert.equal(customStore.asReader().countTurns("conv_custom"), 1);
+		} finally {
+			customStore.close();
+			closeAllTurnDbs();
+		}
+	});
+});