pi-mega-compact 0.13.1 → 0.13.2

package/src/store/turns/sqlite-store/admin.ts

@@ -0,0 +1,249 @@
+/**
+ * admin.ts — TurnAdmin method bodies (extracted from sqlite-store.ts). All
+ * queries use bound parameters (PREVENT-002). DELETE/UPDATE only here (the
+ * append-only rule applies to turns/turn_recall writes outside migrations).
+ */
+import { statSync } from "node:fs";
+import { join } from "node:path";
+import type {
+	SessionId,
+	PruneReport,
+	RetentionPolicy,
+	StoreSnapshot,
+	TurnRecallEntry,
+} from "../types.js";
+import type { SqliteTurnStoreCtx } from "./ctx.js";
+import { normalizeSessionId, rowToEntry, rowToFork } from "./rows.js";
+
+export function prune(ctx: SqliteTurnStoreCtx, policy: RetentionPolicy): PruneReport {
+	const { db } = ctx;
+	const beforeSize = dbSizeBytes(ctx);
+	const cutoff = Date.now() - policy.maxTurnAgeMs;
+
+	// Find conversations with turns to potentially prune
+	const convRows = db
+		.prepare("SELECT DISTINCT conversation_id FROM turns")
+		.all() as Array<{ conversation_id: string }>;
+
+	let turnsRemoved = 0;
+	let recallRemoved = 0;
+
+	for (const { conversation_id } of convRows) {
+		// Count total turns in this conversation
+		const total = db
+			.prepare("SELECT COUNT(*) as cnt FROM turns WHERE conversation_id = ?")
+			.get(conversation_id) as { cnt: number };
+
+		// Keep at least keepMinPerConversation — delete only the oldest
+		const canDelete = Math.max(0, total.cnt - policy.keepMinPerConversation);
+		if (canDelete === 0) continue;
+
+		// Find old turns to delete
+		const oldTurns = db
+			.prepare(
+				`SELECT id FROM turns WHERE conversation_id = ? AND ended_at < ?
+           ORDER BY ended_at ASC LIMIT ?`,
+			)
+			.all(conversation_id, cutoff, canDelete) as Array<{
+			id: number;
+		}>;
+
+		if (oldTurns.length === 0) continue;
+
+		const ids = oldTurns.map((t) => t.id);
+		const placeholders = ids.map(() => "?").join(",");
+
+		// Delete recall for those turns
+		const delRecall = db
+			.prepare(`DELETE FROM turn_recall WHERE turn_id IN (${placeholders})`)
+			.run(...ids);
+		recallRemoved += Number(delRecall.changes);
+
+		// Delete the turns
+		const delTurns = db
+			.prepare(`DELETE FROM turns WHERE id IN (${placeholders})`)
+			.run(...ids);
+		turnsRemoved += Number(delTurns.changes);
+	}
+
+	// Count preserved branches
+	const branchesPreserved = (
+		db
+			.prepare("SELECT COUNT(*) as cnt FROM conversation_forks")
+			.get() as { cnt: number }
+	).cnt;
+
+	if (policy.vacuumAfterPrune && (turnsRemoved > 0 || recallRemoved > 0)) {
+		vacuum(ctx);
+	}
+
+	const afterSize = dbSizeBytes(ctx);
+
+	return {
+		turnsRemoved,
+		recallRemoved,
+		branchesPreserved,
+		freedBytes: Math.max(0, beforeSize - afterSize),
+	};
+}
+
+export function vacuum(ctx: SqliteTurnStoreCtx): void {
+	ctx.db.exec("VACUUM");
+}
+
+export function checkpoint(ctx: SqliteTurnStoreCtx): StoreSnapshot {
+	const { db } = ctx;
+	// Query turns in row-id order (ended_at ASC with unique constraint)
+	const turnRows = db
+		.prepare("SELECT * FROM turns ORDER BY ended_at ASC")
+		.all() as Array<Record<string, unknown>>;
+	const turns = turnRows.map(rowToEntry);
+
+	// Build mapping: row_id → 1-based position in turns array
+	const rowIdToPos = new Map<number, number>();
+	for (let i = 0; i < turnRows.length; i++) {
+		rowIdToPos.set(turnRows[i].id as number, i + 1);
+	}
+
+	const recallRows = db
+		.prepare("SELECT * FROM turn_recall ORDER BY turn_id ASC")
+		.all() as Array<Record<string, unknown>>;
+
+	// Replace recall's turnId with the 1-based position of the referenced turn
+	// in the turns array. This makes the TurnId stable across restore.
+	const recall = recallRows.map((r) => {
+		const originalTurnId = r.turn_id as number;
+		const stableTurnId = rowIdToPos.get(originalTurnId) ?? originalTurnId;
+		return {
+			turnId: String(stableTurnId),
+			checkpointId: r.checkpoint_id as string,
+			score: r.score as number,
+			source: r.source as TurnRecallEntry["source"],
+			raptorLevel: (r.raptor_level as number | null) ?? undefined,
+		};
+	});
+
+	const forks = (
+		db
+			.prepare("SELECT * FROM conversation_forks ORDER BY created_at ASC")
+			.all() as Array<Record<string, unknown>>
+	).map(rowToFork);
+
+	return {
+		version: 1,
+		exportedAt: Date.now(),
+		turns,
+		recall,
+		forks,
+	};
+}
+
+export function restore(ctx: SqliteTurnStoreCtx, from: StoreSnapshot): void {
+	const { db } = ctx;
+	clear(ctx);
+	const insertTurn = db.prepare(
+		`INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
+	                               ctx_tokens, ctx_percent, pressure_band, model, epoch_id)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	);
+	const insertRecall = db.prepare(
+		`INSERT INTO turn_recall (turn_id, checkpoint_id, score, source, raptor_level)
+       VALUES (?, ?, ?, ?, ?)`,
+	);
+	const insertFork = db.prepare(
+		`INSERT INTO conversation_forks (parent_conversation_id, child_conversation_id, fork_turn_index, created_at)
+       VALUES (?, ?, ?, ?)`,
+	);
+
+	db.exec("BEGIN TRANSACTION");
+	try {
+		// Map: 1-based position → new auto-increment row ID
+		const posToNewId = new Map<number, number>();
+
+		for (let i = 0; i < from.turns.length; i++) {
+			const t = from.turns[i];
+			insertTurn.run(
+				t.conversationId,
+				normalizeSessionId(t.sessionId),
+				t.turnIndex,
+				t.role,
+				t.endedAt,
+				t.ctxTokens ?? null,
+				t.ctxPercent ?? null,
+				t.pressureBand ?? null,
+				t.model ?? null,
+				t.epochId ?? null,
+			);
+			const row = db
+				.prepare(
+					"SELECT id FROM turns WHERE conversation_id = ? AND turn_index = ?",
+				)
+				.get(t.conversationId, t.turnIndex) as { id: number } | undefined;
+			if (row) posToNewId.set(i + 1, row.id);
+		}
+
+		// Recall's turnId is now the 1-based position from checkpoint().
+		// Map it to the new row ID.
+		for (const r of from.recall) {
+			const targetTurnId =
+				posToNewId.get(Number(r.turnId)) ?? Number(r.turnId);
+			insertRecall.run(
+				targetTurnId,
+				r.checkpointId,
+				r.score,
+				r.source,
+				r.raptorLevel ?? null,
+			);
+		}
+
+		for (const f of from.forks) {
+			insertFork.run(
+				f.parentConversationId,
+				f.childConversationId,
+				f.forkTurnIndex,
+				f.createdAt,
+			);
+		}
+		db.exec("COMMIT");
+	} catch (err) {
+		db.exec("ROLLBACK");
+		throw err;
+	}
+}
+
+export function clear(ctx: SqliteTurnStoreCtx): void {
+	const { db } = ctx;
+	db.exec("DELETE FROM turn_recall");
+	db.exec("DELETE FROM conversation_forks");
+	db.exec("DELETE FROM turns");
+	db.exec("DELETE FROM session_conversations");
+	// Reset autoincrement
+	db.exec(
+		"DELETE FROM sqlite_sequence WHERE name IN ('turns','turn_recall','conversation_forks')",
+	);
+}
+
+export function stampTurnsEpoch(
+	ctx: SqliteTurnStoreCtx,
+	sessionId: SessionId,
+	epochId: string,
+): number {
+	const { db } = ctx;
+	const sid = normalizeSessionId(sessionId);
+	const res = db
+		.prepare(
+			"UPDATE turns SET epoch_id = ? WHERE session_id = ? AND epoch_id IS NULL",
+		)
+		.run(epochId, sid);
+	return Number(res.changes ?? 0);
+}
+
+function dbSizeBytes(ctx: SqliteTurnStoreCtx): number {
+	if (ctx.opts.inMemory) return 0;
+	try {
+		const path = ctx.opts.dbPath ?? join(ctx.stateDir, "turns.db");
+		return statSync(path).size;
+	} catch {
+		return 0;
+	}
+}

package/src/store/turns/sqlite-store/ctx.ts

@@ -0,0 +1,14 @@
+/**
+ * ctx.ts — internal context interface for SqliteTurnStore method bodies
+ * (extracted from sqlite-store.ts). Each free function in this directory
+ * operates on a SqliteTurnStoreCtx so the shell class can delegate without
+ * exposing its private fields.
+ */
+import type { DatabaseSync } from "node:sqlite";
+import type { TurnStoreOptions } from "../types.js";
+
+export interface SqliteTurnStoreCtx {
+	db: DatabaseSync;
+	stateDir: string;
+	opts: TurnStoreOptions;
+}

package/src/store/turns/sqlite-store/read.ts

@@ -0,0 +1,174 @@
+/**
+ * read.ts — TurnReader method bodies (extracted from sqlite-store.ts). All
+ * queries use bound parameters (PREVENT-002). Append-only reads.
+ */
+import type {
+	ConversationId,
+	ConversationStats,
+	TurnEntry,
+	TurnFilter,
+	TurnId,
+	TurnRecallEntry,
+	ConversationFork,
+} from "../types.js";
+import type { SqliteTurnStoreCtx } from "./ctx.js";
+import { normalizeSessionId, rowToEntry, rowToRecall, rowToFork } from "./rows.js";
+
+export function query(ctx: SqliteTurnStoreCtx, filter: TurnFilter): TurnEntry[] {
+	const { db } = ctx;
+	const clauses: string[] = [];
+	const params: (string | number)[] = [];
+
+	if (filter.conversationId !== undefined) {
+		clauses.push("conversation_id = ?");
+		params.push(filter.conversationId);
+	}
+	if (filter.sessionId !== undefined) {
+		clauses.push("session_id = ?");
+		params.push(normalizeSessionId(filter.sessionId));
+	}
+	if (filter.sinceMs !== undefined) {
+		clauses.push("ended_at >= ?");
+		params.push(filter.sinceMs);
+	}
+	if (filter.untilMs !== undefined) {
+		clauses.push("ended_at <= ?");
+		params.push(filter.untilMs);
+	}
+	if (filter.pressureBand !== undefined) {
+		clauses.push("pressure_band = ?");
+		params.push(filter.pressureBand);
+	}
+
+	const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
+	const limit = filter.limit ?? 1000;
+	const offset = filter.offset ?? 0;
+
+	const rows = db
+		.prepare(
+			`SELECT * FROM turns ${where} ORDER BY ended_at ASC LIMIT ? OFFSET ?`,
+		)
+		.all(...params, limit, offset) as Array<Record<string, unknown>>;
+
+	return rows.map(rowToEntry);
+}
+
+export function getTurn(
+	ctx: SqliteTurnStoreCtx,
+	turnId: TurnId,
+): TurnEntry | undefined {
+	const { db } = ctx;
+	const row = db
+		.prepare("SELECT * FROM turns WHERE id = ?")
+		.get(Number(turnId)) as Record<string, unknown> | undefined;
+	return row ? rowToEntry(row) : undefined;
+}
+
+export function getTurnByIndex(
+	ctx: SqliteTurnStoreCtx,
+	conversationId: ConversationId,
+	turnIndex: number,
+): TurnEntry | undefined {
+	const { db } = ctx;
+	const row = db
+		.prepare(
+			"SELECT * FROM turns WHERE conversation_id = ? AND turn_index = ?",
+		)
+		.get(conversationId, turnIndex) as Record<string, unknown> | undefined;
+	return row ? rowToEntry(row) : undefined;
+}
+
+export function listRecall(
+	ctx: SqliteTurnStoreCtx,
+	turnId: TurnId,
+): TurnRecallEntry[] {
+	const { db } = ctx;
+	const rows = db
+		.prepare(
+			"SELECT * FROM turn_recall WHERE turn_id = ? ORDER BY score DESC",
+		)
+		.all(Number(turnId)) as Array<Record<string, unknown>>;
+	return rows.map(rowToRecall);
+}
+
+export function listRecallByIndex(
+	ctx: SqliteTurnStoreCtx,
+	conversationId: ConversationId,
+	turnIndex: number,
+): TurnRecallEntry[] {
+	const { db } = ctx;
+	const row = db
+		.prepare(
+			"SELECT id FROM turns WHERE conversation_id = ? AND turn_index = ?",
+		)
+		.get(conversationId, turnIndex) as { id: number } | undefined;
+	if (!row) return [];
+	return listRecall(ctx, String(row.id));
+}
+
+export function listForks(
+	ctx: SqliteTurnStoreCtx,
+	conversationId: ConversationId,
+): ConversationFork[] {
+	const { db } = ctx;
+	const rows = db
+		.prepare(
+			"SELECT * FROM conversation_forks WHERE parent_conversation_id = ? ORDER BY created_at ASC",
+		)
+		.all(conversationId) as Array<Record<string, unknown>>;
+	return rows.map(rowToFork);
+}
+
+export function countTurns(
+	ctx: SqliteTurnStoreCtx,
+	conversationId: ConversationId,
+): number {
+	const { db } = ctx;
+	const row = db
+		.prepare("SELECT COUNT(*) as cnt FROM turns WHERE conversation_id = ?")
+		.get(conversationId) as { cnt: number };
+	return row.cnt;
+}
+
+export function conversationStats(
+	ctx: SqliteTurnStoreCtx,
+	conversationId: ConversationId,
+): ConversationStats {
+	const { db } = ctx;
+	const rows = db
+		.prepare(
+			"SELECT ended_at, ctx_percent, pressure_band FROM turns WHERE conversation_id = ? ORDER BY ended_at ASC",
+		)
+		.all(conversationId) as Array<Record<string, unknown>>;
+
+	if (rows.length === 0) {
+		return {
+			turnCount: 0,
+			firstTurnAt: 0,
+			lastTurnAt: 0,
+			avgCtxPercent: 0,
+			pressureBands: {},
+		};
+	}
+
+	let ctxSum = 0;
+	let ctxCount = 0;
+	const bands: Record<string, number> = {};
+
+	for (const r of rows) {
+		if (r.ctx_percent !== null) {
+			ctxSum += r.ctx_percent as number;
+			ctxCount++;
+		}
+		const band = r.pressure_band as string | null;
+		if (band) bands[band] = (bands[band] ?? 0) + 1;
+	}
+
+	return {
+		turnCount: rows.length,
+		firstTurnAt: rows[0].ended_at as number,
+		lastTurnAt: rows[rows.length - 1].ended_at as number,
+		avgCtxPercent: ctxCount > 0 ? ctxSum / ctxCount : 0,
+		pressureBands: bands,
+	};
+}

package/src/store/turns/sqlite-store/rows.ts

@@ -0,0 +1,55 @@
+/**
+ * rows.ts — row ↔ entry mappers + id helpers (extracted from sqlite-store.ts).
+ */
+import { randomBytes } from "node:crypto";
+import type {
+	ConversationId,
+	SessionId,
+	TurnEntry,
+	TurnRecallEntry,
+	ConversationFork,
+} from "../types.js";
+
+export function newConversationId(): ConversationId {
+	return `conv_${randomBytes(8).toString("hex")}`;
+}
+
+export function normalizeSessionId(sid: SessionId): SessionId {
+	// Matches the existing normalizeSessionId in src/store.ts
+	return sid.replace(/-.*$/, "");
+}
+
+export function rowToEntry(r: Record<string, unknown>): TurnEntry {
+	return {
+		conversationId: r.conversation_id as string,
+		sessionId: r.session_id as string,
+		turnIndex: r.turn_index as number,
+		role: r.role as TurnEntry["role"],
+		endedAt: r.ended_at as number,
+		ctxTokens: (r.ctx_tokens as number | null) ?? undefined,
+		ctxPercent: (r.ctx_percent as number | null) ?? undefined,
+		pressureBand:
+			(r.pressure_band as "green" | "yellow" | "red" | null) ?? undefined,
+		model: (r.model as string | null) ?? undefined,
+		epochId: (r.epoch_id as string | null) ?? undefined,
+	};
+}
+
+export function rowToRecall(r: Record<string, unknown>): TurnRecallEntry {
+	return {
+		turnId: String(r.turn_id),
+		checkpointId: r.checkpoint_id as string,
+		score: r.score as number,
+		source: r.source as TurnRecallEntry["source"],
+		raptorLevel: (r.raptor_level as number | null) ?? undefined,
+	};
+}
+
+export function rowToFork(r: Record<string, unknown>): ConversationFork {
+	return {
+		parentConversationId: r.parent_conversation_id as string,
+		childConversationId: r.child_conversation_id as string,
+		forkTurnIndex: r.fork_turn_index as number,
+		createdAt: r.created_at as number,
+	};
+}

package/src/store/turns/sqlite-store/write.ts

@@ -0,0 +1,193 @@
+/**
+ * write.ts — TurnWriter method bodies (extracted from sqlite-store.ts). All
+ * queries use bound parameters (PREVENT-002). Append-only: no UPDATE on
+ * turns/turn_recall after INSERT (only in migration).
+ */
+import type {
+	ConversationId,
+	SessionId,
+	TurnEntry,
+	TurnId,
+	TurnRecallEntry,
+} from "../types.js";
+import { DuplicateTurnError } from "../types.js";
+import type { SqliteTurnStoreCtx } from "./ctx.js";
+import { newConversationId, normalizeSessionId } from "./rows.js";
+
+export function appendTurn(ctx: SqliteTurnStoreCtx, entry: TurnEntry): TurnId {
+	const { db } = ctx;
+	const sid = normalizeSessionId(entry.sessionId);
+	persistSessionConv(ctx, sid, entry.conversationId);
+	try {
+		db
+			.prepare(
+				`INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
+	                           ctx_tokens, ctx_percent, pressure_band, model, epoch_id)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+			)
+			.run(
+				entry.conversationId,
+				sid,
+				entry.turnIndex,
+				entry.role,
+				entry.endedAt,
+				entry.ctxTokens ?? null,
+				entry.ctxPercent ?? null,
+				entry.pressureBand ?? null,
+				entry.model ?? null,
+				entry.epochId ?? null,
+			);
+	} catch (e) {
+		if (
+			e instanceof Error &&
+			e.message.includes("UNIQUE constraint") &&
+			(e as { code?: string }).code === "ERR_SQLITE_ERROR"
+		) {
+			throw new DuplicateTurnError(entry.conversationId, entry.turnIndex);
+		}
+		throw e;
+	}
+	const row = db
+		.prepare(
+			"SELECT id FROM turns WHERE conversation_id = ? AND turn_index = ?",
+		)
+		.get(entry.conversationId, entry.turnIndex) as { id: number } | undefined;
+	return String(row!.id);
+}
+
+/** Persist the session → conversation mapping when a turn is written. */
+function persistSessionConv(
+	ctx: SqliteTurnStoreCtx,
+	sid: string,
+	conversationId: string,
+): void {
+	ctx.db
+		.prepare(
+			"INSERT OR IGNORE INTO session_conversations (session_id, conversation_id) VALUES (?, ?)",
+		)
+		.run(sid, conversationId);
+}
+
+export function appendRecall(
+	ctx: SqliteTurnStoreCtx,
+	entry: TurnRecallEntry,
+): void {
+	const { db } = ctx;
+	db
+		.prepare(
+			`INSERT OR IGNORE INTO turn_recall (turn_id, checkpoint_id, score, source, raptor_level)
+         VALUES (?, ?, ?, ?, ?)`,
+		)
+		.run(
+			Number(entry.turnId),
+			entry.checkpointId,
+			entry.score,
+			entry.source,
+			entry.raptorLevel ?? null,
+		);
+}
+
+export function ensureConversationId(
+	ctx: SqliteTurnStoreCtx,
+	sessionId: SessionId,
+): ConversationId {
+	const { db } = ctx;
+	const sid = normalizeSessionId(sessionId);
+	// Check session_conversations table first (persistent mapping)
+	const row = db
+		.prepare(
+			"SELECT conversation_id FROM session_conversations WHERE session_id = ?",
+		)
+		.get(sid) as { conversation_id: string } | undefined;
+	if (row) return row.conversation_id;
+
+	// Check if any turn exists for this session (fallback)
+	const turnRow = db
+		.prepare(
+			"SELECT conversation_id FROM turns WHERE session_id = ? ORDER BY ended_at DESC LIMIT 1",
+		)
+		.get(sid) as { conversation_id: string } | undefined;
+	if (turnRow) {
+		// Persist the mapping so future calls are consistent
+		db
+			.prepare(
+				"INSERT OR IGNORE INTO session_conversations (session_id, conversation_id) VALUES (?, ?)",
+			)
+			.run(sid, turnRow.conversation_id);
+		return turnRow.conversation_id;
+	}
+
+	// No turns yet — generate and persist a new conversation id
+	const convId = newConversationId();
+	db
+		.prepare(
+			"INSERT OR IGNORE INTO session_conversations (session_id, conversation_id) VALUES (?, ?)",
+		)
+		.run(sid, convId);
+	return convId;
+}
+
+export function forkConversation(
+	ctx: SqliteTurnStoreCtx,
+	parentId: ConversationId,
+	forkTurnIndex: number,
+): ConversationId {
+	const { db } = ctx;
+	const childId = newConversationId();
+	const now = Date.now();
+
+	// 1. Record the fork lineage
+	db
+		.prepare(
+			`INSERT INTO conversation_forks (parent_conversation_id, child_conversation_id, fork_turn_index, created_at)
+         VALUES (?, ?, ?, ?)`,
+		)
+		.run(parentId, childId, forkTurnIndex, now);
+
+	// 2. Find the parent's turn at forkTurnIndex
+	const parentTurn = db
+		.prepare(
+			"SELECT id FROM turns WHERE conversation_id = ? AND turn_index = ?",
+		)
+		.get(parentId, forkTurnIndex) as { id: number } | undefined;
+
+	if (!parentTurn) return childId; // no parent turn at that index — nothing to seed
+
+	// 3. Create a seed turn in the child conversation (turnIndex 0, role system)
+	db
+		.prepare(
+			`INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at)
+         VALUES (?, ?, 0, 'system', ?)`,
+		)
+		.run(childId, `fork_${childId}`, now);
+
+	const childTurnRow = db
+		.prepare(
+			"SELECT id FROM turns WHERE conversation_id = ? AND turn_index = 0",
+		)
+		.get(childId) as { id: number } | undefined;
+	if (!childTurnRow) return childId;
+
+	// 4. Copy the parent's recall entries into the child's seed turn
+	const parentRecall = db
+		.prepare(
+			"SELECT checkpoint_id, score, source, raptor_level FROM turn_recall WHERE turn_id = ?",
+		)
+		.all(parentTurn.id) as Array<Record<string, unknown>>;
+
+	const insertRecall = db.prepare(
+		`INSERT OR IGNORE INTO turn_recall (turn_id, checkpoint_id, score, source, raptor_level)
+         VALUES (?, ?, ?, ?, ?)`,
+	);
+	for (const r of parentRecall) {
+		insertRecall.run(
+			childTurnRow.id,
+			r.checkpoint_id as string,
+			r.score as number,
+			r.source as string,
+			(r.raptor_level as number | null) ?? null,
+		);
+	}
+
+	return childId;
+}