pi-mega-compact 0.9.2 → 0.11.0

package/src/store/turns/index.ts

@@ -0,0 +1,65 @@
+/**
+ * index.ts — Barrel for src/store/turns/ (S49 reconciled).
+ *
+ * Re-exports the contract-first factory + all contract types (master's design)
+ * and the two backend constructors (SqliteTurnStore / InMemoryTurnStore), plus
+ * this branch's reconciled connection layer + migration helper. Hosts import
+ * only this file + types.ts.
+ *
+ * PREVENT-PI-004: no network. Reuse-clean: no pi imports.
+ */
+
+export type {
+	TurnId,
+	ConversationId,
+	SessionId,
+	TurnEntry,
+	TurnRecallEntry,
+	ConversationFork,
+	TurnFilter,
+	PruneReport,
+	RetentionPolicy,
+	StoreSnapshot,
+	ConversationStats,
+	TurnReader,
+	TurnWriter,
+	TurnAdmin,
+	TurnStore,
+	TurnStoreOptions,
+	TurnStoreFactory,
+} from "./types.js";
+
+export { SqliteTurnStore } from "./sqlite-store.js";
+export { InMemoryTurnStore } from "./memory-store.js";
+
+// Reconciled connection layer (master shape + this branch's migration +
+// closed-handle eviction + env override + openTurnStore back-compat alias).
+export {
+	openTurnDb,
+	openTurnStore,
+	closeTurnDb,
+	closeTurnStore,
+	closeAllTurnDbs,
+	turnDbPath,
+	TURNS_DB_FILE,
+	withTx,
+} from "./connection.js";
+export { initTurnSchema } from "./schema.js";
+export { migrateTurnTablesIfNeeded } from "./migrations.js";
+
+import { SqliteTurnStore } from "./sqlite-store.js";
+import { InMemoryTurnStore } from "./memory-store.js";
+import type { TurnStore, TurnStoreOptions } from "./types.js";
+
+/**
+ * Factory: create a TurnStore from options.
+ *
+ * - Default: SqliteTurnStore backed by turns.db in stateDir
+ * - inMemory: InMemoryTurnStore (no file I/O)
+ */
+export function createTurnStore(options: TurnStoreOptions): TurnStore {
+	if (options.inMemory) {
+		return new InMemoryTurnStore(options);
+	}
+	return new SqliteTurnStore(options);
+}

package/src/store/turns/memory-store.test.ts

@@ -0,0 +1,15 @@
+/**
+ * memory-store.test.ts — InMemoryTurnStore compliance.
+ */
+
+import { InMemoryTurnStore } from "./memory-store.js";
+import { runComplianceSuite } from "./contract-compliance.test.js";
+
+// Run the shared compliance suite against the in-memory backend
+runComplianceSuite(
+	"InMemoryTurnStore",
+	(options) => {
+		return new InMemoryTurnStore(options);
+	},
+	{ stateDir: "/tmp/turns-compliance-memory", inMemory: true },
+);

package/src/store/turns/memory-store.ts

@@ -0,0 +1,469 @@
+/**
+ * memory-store.ts — InMemoryTurnStore: test harness + embeddable backend.
+ *
+ * Same contract as SqliteTurnStore, backed by Maps. No file I/O.
+ * Used for tests and for hosts that don't want SQLite.
+ *
+ * PREVENT-PI-004: no network. PREVENT-002: no SQL here.
+ */
+
+import { randomBytes } from "node:crypto";
+import type {
+	TurnStore,
+	TurnReader,
+	TurnWriter,
+	TurnAdmin,
+	TurnId,
+	ConversationId,
+	SessionId,
+	TurnEntry,
+	TurnRecallEntry,
+	ConversationFork,
+	TurnFilter,
+	PruneReport,
+	RetentionPolicy,
+	StoreSnapshot,
+	ConversationStats,
+	TurnStoreOptions,
+} from "./types.js";
+
+function newConversationId(): ConversationId {
+	return `conv_${randomBytes(8).toString("hex")}`;
+}
+
+function normalizeSessionId(sid: SessionId): SessionId {
+	return sid.replace(/-.*$/, "");
+}
+
+// Auto-increment counter
+let nextId = 1;
+function allocId(): number {
+	return nextId++;
+}
+function resetIdCounter(): void {
+	nextId = 1;
+}
+
+// Internal row with numeric id
+interface TurnRow {
+	id: number;
+	entry: TurnEntry;
+	epochId?: string;
+}
+
+interface RecallRow {
+	turnId: number;
+	entry: TurnRecallEntry;
+}
+
+export class InMemoryTurnStore implements TurnStore {
+	private turns: Map<number, TurnRow> = new Map();
+	private recall: RecallRow[] = [];
+	private forks: ConversationFork[] = [];
+	// conversation_id → turn ids (for fast lookup)
+	private convIndex: Map<ConversationId, number[]> = new Map();
+	// session_id → conversation_id
+	private sessionConv: Map<SessionId, ConversationId> = new Map();
+
+	constructor(_options?: TurnStoreOptions) {
+		resetIdCounter();
+	}
+
+	// ── TurnReader ──────────────────────────────────────────────
+
+	query(filter: TurnFilter): TurnEntry[] {
+		let results = [...this.turns.values()].map((r) => r.entry);
+
+		if (filter.conversationId !== undefined) {
+			results = results.filter(
+				(t) => t.conversationId === filter.conversationId,
+			);
+		}
+		if (filter.sessionId !== undefined) {
+			results = results.filter(
+				(t) =>
+					normalizeSessionId(t.sessionId) ===
+					normalizeSessionId(filter.sessionId!),
+			);
+		}
+		if (filter.sinceMs !== undefined) {
+			results = results.filter((t) => t.endedAt >= filter.sinceMs!);
+		}
+		if (filter.untilMs !== undefined) {
+			results = results.filter((t) => t.endedAt <= filter.untilMs!);
+		}
+		if (filter.pressureBand !== undefined) {
+			results = results.filter((t) => t.pressureBand === filter.pressureBand);
+		}
+
+		results.sort((a, b) => a.endedAt - b.endedAt);
+
+		const offset = filter.offset ?? 0;
+		const limit = filter.limit ?? 1000;
+		return results.slice(offset, offset + limit);
+	}
+
+	getTurn(turnId: TurnId): TurnEntry | undefined {
+		const row = this.turns.get(Number(turnId));
+		return row ? { ...row.entry, epochId: row.epochId } : undefined;
+	}
+
+	getTurnByIndex(
+		conversationId: ConversationId,
+		turnIndex: number,
+	): TurnEntry | undefined {
+		const ids = this.convIndex.get(conversationId) ?? [];
+		for (const id of ids) {
+			const row = this.turns.get(id);
+			if (row && row.entry.turnIndex === turnIndex) {
+				return { ...row.entry, epochId: row.epochId };
+			}
+		}
+		return undefined;
+	}
+
+	listRecall(turnId: TurnId): TurnRecallEntry[] {
+		return this.recall
+			.filter((r) => r.turnId === Number(turnId))
+			.map((r) => r.entry)
+			.sort((a, b) => b.score - a.score);
+	}
+
+	listRecallByIndex(
+		conversationId: ConversationId,
+		turnIndex: number,
+	): TurnRecallEntry[] {
+		const turn = this.getTurnByIndex(conversationId, turnIndex);
+		if (!turn) return [];
+		// Resolve the numeric row id for this turn via the convIndex.
+		const ids = this.convIndex.get(conversationId) ?? [];
+		for (const id of ids) {
+			const row = this.turns.get(id);
+			if (row && row.entry.turnIndex === turnIndex) {
+				return this.listRecall(String(id));
+			}
+		}
+		return [];
+	}
+
+	listForks(conversationId: ConversationId): ConversationFork[] {
+		return this.forks
+			.filter((f) => f.parentConversationId === conversationId)
+			.sort((a, b) => a.createdAt - b.createdAt);
+	}
+
+	countTurns(conversationId: ConversationId): number {
+		const ids = this.convIndex.get(conversationId);
+		return ids?.length ?? 0;
+	}
+
+	conversationStats(conversationId: ConversationId): ConversationStats {
+		const ids = this.convIndex.get(conversationId) ?? [];
+		const entries = ids
+			.map((id) => this.turns.get(id)?.entry)
+			.filter((e): e is TurnEntry => e !== undefined);
+
+		if (entries.length === 0) {
+			return {
+				turnCount: 0,
+				firstTurnAt: 0,
+				lastTurnAt: 0,
+				avgCtxPercent: 0,
+				pressureBands: {},
+			};
+		}
+
+		let ctxSum = 0;
+		let ctxCount = 0;
+		const bands: Record<string, number> = {};
+
+		for (const e of entries) {
+			if (e.ctxPercent !== undefined) {
+				ctxSum += e.ctxPercent;
+				ctxCount++;
+			}
+			if (e.pressureBand)
+				bands[e.pressureBand] = (bands[e.pressureBand] ?? 0) + 1;
+		}
+
+		return {
+			turnCount: entries.length,
+			firstTurnAt: entries[0].endedAt,
+			lastTurnAt: entries[entries.length - 1].endedAt,
+			avgCtxPercent: ctxCount > 0 ? ctxSum / ctxCount : 0,
+			pressureBands: bands,
+		};
+	}
+
+	// ── TurnWriter ──────────────────────────────────────────────
+
+	appendTurn(entry: TurnEntry): TurnId {
+		const id = allocId();
+		const sid = normalizeSessionId(entry.sessionId);
+		const normalized = { ...entry, sessionId: sid };
+		this.turns.set(id, { id, entry: normalized });
+
+		// Update indices
+		const convIds = this.convIndex.get(entry.conversationId) ?? [];
+		convIds.push(id);
+		this.convIndex.set(entry.conversationId, convIds);
+
+		this.sessionConv.set(sid, entry.conversationId);
+
+		return String(id);
+	}
+
+	appendRecall(entry: TurnRecallEntry): void {
+		// De-dup by (turn_id, checkpoint_id)
+		const exists = this.recall.some(
+			(r) =>
+				r.turnId === Number(entry.turnId) &&
+				r.entry.checkpointId === entry.checkpointId,
+		);
+		if (!exists) {
+			this.recall.push({ turnId: Number(entry.turnId), entry });
+		}
+	}
+
+	ensureConversationId(sessionId: SessionId): ConversationId {
+		const sid = normalizeSessionId(sessionId);
+		const existing = this.sessionConv.get(sid);
+		if (existing) return existing;
+		// Generate and store a new conversation ID for this session
+		const convId = newConversationId();
+		this.sessionConv.set(sid, convId);
+		return convId;
+	}
+
+	forkConversation(
+		parentId: ConversationId,
+		forkTurnIndex: number,
+	): ConversationId {
+		const childId = newConversationId();
+		const now = Date.now();
+
+		// 1. Record the fork lineage
+		this.forks.push({
+			parentConversationId: parentId,
+			childConversationId: childId,
+			forkTurnIndex,
+			createdAt: now,
+		});
+
+		// 2. Find the parent's turn at forkTurnIndex
+		const parentTurnIds = this.convIndex.get(parentId) ?? [];
+		const parentTurnRow = parentTurnIds
+			.map((id) => this.turns.get(id))
+			.find((r) => r?.entry.turnIndex === forkTurnIndex);
+
+		if (!parentTurnRow) return childId; // no parent turn — nothing to seed
+
+		// 3. Create a seed turn in the child conversation
+		const seedId = allocId();
+		const seedEntry: TurnEntry = {
+			conversationId: childId,
+			sessionId: `fork_${childId}`,
+			turnIndex: 0,
+			role: "system",
+			endedAt: now,
+		};
+		this.turns.set(seedId, { id: seedId, entry: seedEntry });
+		this.convIndex.set(childId, [seedId]);
+
+		// 4. Copy the parent's recall entries into the child's seed turn
+		const parentRecall = this.recall.filter(
+			(r) => r.turnId === parentTurnRow.id,
+		);
+		for (const r of parentRecall) {
+			this.recall.push({
+				turnId: seedId,
+				entry: {
+					turnId: String(seedId),
+					checkpointId: r.entry.checkpointId,
+					score: r.entry.score,
+					source: r.entry.source,
+					raptorLevel: r.entry.raptorLevel,
+				},
+			});
+		}
+
+		return childId;
+	}
+
+	// ── TurnAdmin ───────────────────────────────────────────────
+
+	prune(policy: RetentionPolicy): PruneReport {
+		const cutoff = Date.now() - policy.maxTurnAgeMs;
+		let turnsRemoved = 0;
+		let recallRemoved = 0;
+
+		for (const [convId, turnIds] of this.convIndex.entries()) {
+			const total = turnIds.length;
+			const canDelete = Math.max(0, total - policy.keepMinPerConversation);
+			if (canDelete === 0) continue;
+
+			// Find old turns to delete
+			const oldIds = turnIds
+				.filter((id) => {
+					const row = this.turns.get(id);
+					return row && row.entry.endedAt < cutoff;
+				})
+				.slice(0, canDelete);
+
+			for (const id of oldIds) {
+				// Remove recall
+				const before = this.recall.length;
+				this.recall = this.recall.filter((r) => r.turnId !== id);
+				recallRemoved += before - this.recall.length;
+
+				this.turns.delete(id);
+				turnsRemoved++;
+			}
+
+			// Update convIndex
+			this.convIndex.set(
+				convId,
+				turnIds.filter((id) => this.turns.has(id)),
+			);
+		}
+
+		const branchesPreserved = this.forks.length;
+
+		return {
+			turnsRemoved,
+			recallRemoved,
+			branchesPreserved,
+			freedBytes: 0, // in-memory — no file size to measure
+		};
+	}
+
+	vacuum(): void {
+		// No-op for in-memory store
+	}
+
+	checkpoint(): StoreSnapshot {
+		// Emit turns in a stable order (by internal ID / insertion order)
+		const sorted = [...this.turns.values()].sort((a, b) => a.id - b.id);
+		const turns = sorted.map((r) => r.entry);
+
+		// Build mapping: internal ID → 1-based position
+		const idToPos = new Map<number, number>();
+		for (let i = 0; i < sorted.length; i++) {
+			idToPos.set(sorted[i].id, i + 1);
+		}
+
+		// Replace recall's turnId with the stable 1-based position
+		const recall = this.recall.map((r) => ({
+			turnId: String(idToPos.get(r.turnId) ?? r.turnId),
+			checkpointId: r.entry.checkpointId,
+			score: r.entry.score,
+			source: r.entry.source,
+			raptorLevel: r.entry.raptorLevel,
+		}));
+
+		return {
+			version: 1,
+			exportedAt: Date.now(),
+			turns,
+			recall,
+			forks: [...this.forks],
+		};
+	}
+
+	restore(from: StoreSnapshot): void {
+		this.clear();
+
+		// Map: 1-based position → new internal ID
+		const posToNewId = new Map<number, number>();
+
+		for (let i = 0; i < from.turns.length; i++) {
+			const t = from.turns[i];
+			const newId = allocId();
+			const sid = normalizeSessionId(t.sessionId);
+			const normalized = { ...t, sessionId: sid };
+			this.turns.set(newId, { id: newId, entry: normalized });
+
+			// Update indices
+			const convIds = this.convIndex.get(t.conversationId) ?? [];
+			convIds.push(newId);
+			this.convIndex.set(t.conversationId, convIds);
+			this.sessionConv.set(sid, t.conversationId);
+
+			posToNewId.set(i + 1, newId);
+		}
+
+		// Re-attach recall with position-based TurnId → new internal ID mapping
+		for (const r of from.recall) {
+			const targetInternalId = posToNewId.get(Number(r.turnId));
+			if (targetInternalId !== undefined) {
+				this.recall.push({
+					turnId: targetInternalId,
+					entry: r,
+				});
+			}
+		}
+
+		this.forks = [...from.forks];
+	}
+
+	clear(): void {
+		this.turns.clear();
+		this.recall = [];
+		this.forks = [];
+		this.convIndex.clear();
+		this.sessionConv.clear();
+		resetIdCounter();
+	}
+
+	stampTurnsEpoch(sessionId: SessionId, epochId: string): number {
+		const sid = normalizeSessionId(sessionId);
+		let stamped = 0;
+		for (const row of this.turns.values()) {
+			if (row.epochId === undefined && row.entry.sessionId === sid) {
+				row.epochId = epochId;
+				stamped++;
+			}
+		}
+		return stamped;
+	}
+
+	// ── Capability gating ───────────────────────────────────────
+
+	asReader(): TurnReader {
+		return {
+			query: (f) => this.query(f),
+			getTurn: (id) => this.getTurn(id),
+			getTurnByIndex: (c, t) => this.getTurnByIndex(c, t),
+			listRecall: (id) => this.listRecall(id),
+			listRecallByIndex: (c, t) => this.listRecallByIndex(c, t),
+			listForks: (id) => this.listForks(id),
+			countTurns: (id) => this.countTurns(id),
+			conversationStats: (id) => this.conversationStats(id),
+		};
+	}
+
+	asWriter(): TurnWriter {
+		return {
+			appendTurn: (e) => this.appendTurn(e),
+			appendRecall: (e) => this.appendRecall(e),
+			ensureConversationId: (s) => this.ensureConversationId(s),
+			forkConversation: (p, t) => this.forkConversation(p, t),
+		};
+	}
+
+	asAdmin(): TurnAdmin {
+		return {
+			prune: (p) => this.prune(p),
+			vacuum: () => this.vacuum(),
+			checkpoint: () => this.checkpoint(),
+			restore: (s) => this.restore(s),
+			clear: () => this.clear(),
+			stampTurnsEpoch: (s, e) => this.stampTurnsEpoch(s, e),
+		};
+	}
+
+	close(): void {
+		// No-op for in-memory store
+		this.clear();
+	}
+}

package/src/store/turns/migrations.test.ts

@@ -0,0 +1,161 @@
+/**
+ * migrations.test.ts — S49B main-db → turns.db migration tests.
+ *
+ * Seeds a legacy main sqlite.db (with S48-era turn tables + session_state
+ * conversation pointers), then opens the turns store and asserts rows moved,
+ * legacy dropped, and the move is idempotent + flag-gated. No network.
+ */
+
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, mkdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { openTurnStore, closeTurnStore } from "./connection.js";
+import { TurnsConfig } from "../../config/turns.js";
+
+let tmpDir: string;
+let counter = 0;
+
+beforeEach(() => {
+	tmpDir = mkdtempSync(join(tmpdir(), "mc-migrate-"));
+});
+
+afterEach(() => {
+	rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function stateDir(): string {
+	return join(tmpDir, `run-${counter++}`);
+}
+
+/** Seed a legacy main db with S48-era turn tables + a conversation pointer. */
+function seedLegacyMainDb(dir: string): void {
+	mkdirSync(dir, { recursive: true });
+	const db = new DatabaseSync(join(dir, "sqlite.db"));
+	db.exec(`
+    CREATE TABLE turns (
+      id INTEGER PRIMARY KEY AUTOINCREMENT, conversation_id TEXT NOT NULL,
+      session_id TEXT NOT NULL, turn_index INTEGER NOT NULL, role TEXT,
+      started_at INTEGER NOT NULL, ended_at INTEGER, ctx_tokens INTEGER,
+      ctx_percent REAL, pressure_band TEXT, model_id TEXT, epoch_id TEXT,
+      UNIQUE(session_id, turn_index)
+    );
+    CREATE TABLE turn_recall (
+      id INTEGER PRIMARY KEY AUTOINCREMENT, turn_id INTEGER NOT NULL,
+      checkpoint_id TEXT NOT NULL, score REAL NOT NULL, source TEXT NOT NULL,
+      raptor_level INTEGER, UNIQUE(turn_id, checkpoint_id)
+    );
+    CREATE TABLE conversation_branches (
+      conversation_id TEXT PRIMARY KEY, parent_conversation_id TEXT NOT NULL,
+      fork_turn_id INTEGER NOT NULL, created_at INTEGER NOT NULL
+    );
+    CREATE TABLE session_state (
+      session_id TEXT PRIMARY KEY, injected_checkpoint_ids TEXT,
+      stored_region_hashes TEXT, conversation_id TEXT, last_turn_id INTEGER
+    );
+    INSERT INTO turns (conversation_id, session_id, turn_index, started_at, ended_at, ctx_tokens)
+      VALUES ('conv_leg', 'sess_leg', 0, 100, 200, 500);
+    INSERT INTO turn_recall (turn_id, checkpoint_id, score, source)
+      VALUES (1, 'cp_leg', 0.6, 'flat');
+    INSERT INTO conversation_branches (conversation_id, parent_conversation_id, fork_turn_id, created_at)
+      VALUES ('conv_child', 'conv_leg', 1, 300);
+    INSERT INTO session_state (session_id, conversation_id) VALUES ('sess_leg', 'conv_leg');
+  `);
+	db.close();
+}
+
+function tableExists(db: DatabaseSync, name: string): boolean {
+	return (
+		db
+			.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?")
+			.get(name) !== undefined
+	);
+}
+
+test("fresh dir (no main db) → no-op, marks migrated", () => {
+	const dir = stateDir();
+	const db = openTurnStore(dir);
+	// openTurnStore ran the migration hook; with no sqlite.db it should just mark.
+	const row = db
+		.prepare("SELECT value FROM turns_meta WHERE key='migrated_from_main'")
+		.get() as { value: string } | undefined;
+	assert.equal(row?.value, "1");
+	closeTurnStore(dir);
+});
+
+test("legacy rows moved, legacy tables dropped, conversation pointer migrated", () => {
+	const dir = stateDir();
+	seedLegacyMainDb(dir);
+	const db = openTurnStore(dir); // triggers migration
+
+	// Rows present in turns.db.
+	const turns = db.prepare("SELECT * FROM turns").all() as Array<
+		Record<string, unknown>
+	>;
+	assert.equal(turns.length, 1);
+	assert.equal(turns[0].conversation_id, "conv_leg");
+	const recall = db.prepare("SELECT * FROM turn_recall").all() as Array<
+		Record<string, unknown>
+	>;
+	assert.equal(recall.length, 1);
+	assert.equal(recall[0].checkpoint_id, "cp_leg");
+	const branches = db
+		.prepare("SELECT * FROM conversation_forks")
+		.all() as Array<Record<string, unknown>>;
+	assert.equal(branches.length, 1);
+	assert.equal(branches[0].parent_conversation_id, "conv_leg");
+	assert.equal(branches[0].child_conversation_id, "conv_child");
+	assert.equal(branches[0].fork_turn_index, 1);
+	// Conversation pointer migrated to session_conversations (session_id → conversation_id).
+	const ptr = db
+		.prepare(
+			"SELECT conversation_id FROM session_conversations WHERE session_id='sess_leg'",
+		)
+		.get() as { conversation_id: string } | undefined;
+	assert.equal(ptr?.conversation_id, "conv_leg");
+
+	closeTurnStore(dir);
+
+	// Legacy tables dropped from the main db.
+	const main = new DatabaseSync(join(dir, "sqlite.db"));
+	assert.ok(!tableExists(main, "turns"));
+	assert.ok(!tableExists(main, "turn_recall"));
+	assert.ok(!tableExists(main, "conversation_branches"));
+	assert.ok(tableExists(main, "session_state")); // left intact
+	main.close();
+});
+
+test("idempotent: re-open does not re-copy or re-drop", () => {
+	const dir = stateDir();
+	seedLegacyMainDb(dir);
+	openTurnStore(dir);
+	closeTurnStore(dir);
+	// Second open — marker set, should be a no-op even though main db now lacks turn tables.
+	const db = openTurnStore(dir);
+	const turns = db.prepare("SELECT * FROM turns").all() as Array<
+		Record<string, unknown>
+	>;
+	assert.equal(turns.length, 1); // still exactly the migrated row, not duplicated
+	closeTurnStore(dir);
+});
+
+test("flag OFF → legacy tables untouched, no marker write from hook", () => {
+	const dir = stateDir();
+	seedLegacyMainDb(dir);
+	const original = TurnsConfig.TURNS_DB_ENABLED;
+	TurnsConfig.TURNS_DB_ENABLED = false;
+	try {
+		const db = openTurnStore(dir);
+		closeTurnStore(dir);
+		void db;
+	} finally {
+		TurnsConfig.TURNS_DB_ENABLED = original;
+	}
+	// Main db turn tables must be intact (migration hook was gated off).
+	const main = new DatabaseSync(join(dir, "sqlite.db"));
+	assert.ok(tableExists(main, "turns"));
+	assert.ok(tableExists(main, "conversation_branches"));
+	main.close();
+});