pi-mega-compact 0.8.24 → 0.8.26

package/src/store/sqlite/turns.ts

@@ -0,0 +1,302 @@
+/**
+ * turns.ts — S43 per-turn vector + conversation tracking.
+ *
+ * The relational spine for per-turn + per-conversation memories:
+ *   - `turns`: one row per turn_end (metrics + epoch link + conversation id)
+ *   - `turn_recall`: which checkpoints/cluster summaries were injected at a turn
+ *   - `conversation_branches`: fork registry (parent → child at a turn point)
+ *
+ * `conversationId` (on SessionState) groups turns across pi session resumes;
+ * `/clear` or any fresh root generates a new one via `newConversationId()`.
+ *
+ * `forkConversation(parent, forkTurnId)` copies a parent conversation's
+ * recall state at `forkTurnId` into a fresh conversation's injected-set, so a
+ * forked session starts with exactly the context conversation X had at turn N
+ * (a recall-fork, not a live-window replay — see docs/specs/s48-per-turn-vector-tracking.md).
+ *
+ * No network. Pure SQLite (PREVENT-PI-004). All queries parameterized (PREVENT-002).
+ */
+
+import { randomBytes } from "node:crypto";
+import { openStore, withTx } from "./utils.js";
+import { getStateDir, normalizeSessionId } from "../../store.js";
+import { loadSessionState, saveSessionState } from "./session-state.js";
+import type { DatabaseSync } from "node:sqlite";
+
+/** A row in `turns`. */
+export interface TurnRow {
+	id: number;
+	conversationId: string;
+	sessionId: string;
+	turnIndex: number;
+	role: string | null;
+	startedAt: number;
+	endedAt: number | null;
+	ctxTokens: number | null;
+	ctxPercent: number | null;
+	pressureBand: string | null;
+	modelId: string | null;
+	epochId: string | null;
+}
+
+/** A row in `turn_recall`. */
+export interface TurnRecallRow {
+	id: number;
+	turnId: number;
+	checkpointId: string;
+	score: number;
+	source: string;
+	raptorLevel: number | null;
+}
+
+/** Where a recalled hit came from (recorded on turn_recall.source). */
+export type RecallSource = "flat" | "raptor" | "cross-repo" | "memory";
+
+/** Generate a new conversation id (`conv_` + 16 hex). */
+export function newConversationId(): string {
+	return `conv_${randomBytes(8).toString("hex")}`;
+}
+
+function rowToTurn(r: Record<string, unknown>): TurnRow {
+	return {
+		id: r.id as number,
+		conversationId: r.conversation_id as string,
+		sessionId: r.session_id as string,
+		turnIndex: r.turn_index as number,
+		role: (r.role as string | null) ?? null,
+		startedAt: r.started_at as number,
+		endedAt: (r.ended_at as number | null) ?? null,
+		ctxTokens: (r.ctx_tokens as number | null) ?? null,
+		ctxPercent: (r.ctx_percent as number | null) ?? null,
+		pressureBand: (r.pressure_band as string | null) ?? null,
+		modelId: (r.model_id as string | null) ?? null,
+		epochId: (r.epoch_id as string | null) ?? null,
+	};
+}
+
+/**
+ * Upsert a turn row at turn_end. Returns the turn id. `startedAt` defaults to
+ * now (the turn_start write can update it; turn_end sets ended_at + metrics).
+ * Idempotent on (session_id, turn_index) — re-upserting overwrites metrics.
+ */
+export function recordTurn(
+	input: {
+		conversationId: string;
+		sessionId: string;
+		turnIndex: number;
+		role?: string;
+		startedAt?: number;
+		endedAt?: number;
+		ctxTokens?: number;
+		ctxPercent?: number;
+		pressureBand?: string;
+		modelId?: string;
+		epochId?: string;
+	},
+	stateDir: string = getStateDir(),
+): number {
+	const db = openStore(stateDir);
+	const sid = normalizeSessionId(input.sessionId);
+	const startedAt = input.startedAt ?? Date.now();
+	db.prepare(
+		`INSERT INTO turns (conversation_id, session_id, turn_index, role, started_at,
+                         ended_at, ctx_tokens, ctx_percent, pressure_band, model_id, epoch_id)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(session_id, turn_index) DO UPDATE SET
+       conversation_id = excluded.conversation_id,
+       role = COALESCE(excluded.role, role),
+       started_at = COALESCE(excluded.started_at, started_at),
+       ended_at = COALESCE(excluded.ended_at, ended_at),
+       ctx_tokens = COALESCE(excluded.ctx_tokens, ctx_tokens),
+       ctx_percent = COALESCE(excluded.ctx_percent, ctx_percent),
+       pressure_band = COALESCE(excluded.pressure_band, pressure_band),
+       model_id = COALESCE(excluded.model_id, model_id),
+       epoch_id = COALESCE(excluded.epoch_id, epoch_id)`,
+	).run(
+		input.conversationId,
+		sid,
+		input.turnIndex,
+		input.role ?? null,
+		startedAt,
+		input.endedAt ?? null,
+		input.ctxTokens ?? null,
+		input.ctxPercent ?? null,
+		input.pressureBand ?? null,
+		input.modelId ?? null,
+		input.epochId ?? null,
+	);
+	const row = db
+		.prepare("SELECT id FROM turns WHERE session_id = ? AND turn_index = ?")
+		.get(sid, input.turnIndex) as { id: number };
+	return row.id;
+}
+
+/**
+ * Record what was recalled at a turn. Called from recallAndInline with the
+ * resolved toInject list — each hit becomes a turn_recall row with its source
+ * path + score. RAPTOR cluster hits carry raptorLevel. Best-effort + non-fatal.
+ */
+export function recordTurnRecall(
+	turnId: number,
+	hits: {
+		checkpointId: string;
+		score: number;
+		source: RecallSource;
+		raptorLevel?: number;
+	}[],
+	stateDir: string = getStateDir(),
+): void {
+	if (hits.length === 0) return;
+	const db = openStore(stateDir);
+	const stmt = db.prepare(
+		`INSERT INTO turn_recall (turn_id, checkpoint_id, score, source, raptor_level)
+     VALUES (?, ?, ?, ?, ?)
+     ON CONFLICT(turn_id, checkpoint_id) DO UPDATE SET
+       score = excluded.score, source = excluded.source,
+       raptor_level = excluded.raptor_level`,
+	);
+	withTx(db, () => {
+		for (const h of hits) {
+			stmt.run(
+				turnId,
+				h.checkpointId,
+				h.score,
+				h.source,
+				h.raptorLevel ?? null,
+			);
+		}
+	});
+}
+
+/** Get a turn by conversation id + turn index (the lookup a fork uses). */
+export function getTurn(
+	conversationId: string,
+	turnIndex: number,
+	stateDir: string = getStateDir(),
+): TurnRow | null {
+	const db = openStore(stateDir);
+	const row = db
+		.prepare(`SELECT * FROM turns WHERE conversation_id = ? AND turn_index = ?`)
+		.get(conversationId, turnIndex) as Record<string, unknown> | undefined;
+	return row ? rowToTurn(row) : null;
+}
+
+/** Get a turn by its global id. */
+export function getTurnById(
+	turnId: number,
+	stateDir: string = getStateDir(),
+): TurnRow | null {
+	const db = openStore(stateDir);
+	const row = db.prepare("SELECT * FROM turns WHERE id = ?").get(turnId) as
+		| Record<string, unknown>
+		| undefined;
+	return row ? rowToTurn(row) : null;
+}
+
+/** All turn_recall rows for a turn (what was injected at that turn). */
+export function listTurnRecall(
+	turnId: number,
+	stateDir: string = getStateDir(),
+): TurnRecallRow[] {
+	const db = openStore(stateDir);
+	const rows = db
+		.prepare(
+			`SELECT id, turn_id, checkpoint_id, score, source, raptor_level
+       FROM turn_recall WHERE turn_id = ? ORDER BY score DESC`,
+		)
+		.all(turnId) as Array<Record<string, unknown>>;
+	return rows.map((r) => ({
+		id: r.id as number,
+		turnId: r.turn_id as number,
+		checkpointId: r.checkpoint_id as string,
+		score: r.score as number,
+		source: r.source as string,
+		raptorLevel: (r.raptor_level as number | null) ?? null,
+	}));
+}
+
+/** All turns in a conversation, ascending by turn_index. */
+export function listConversationTurns(
+	conversationId: string,
+	stateDir: string = getStateDir(),
+): TurnRow[] {
+	const db = openStore(stateDir);
+	const rows = db
+		.prepare(
+			`SELECT * FROM turns WHERE conversation_id = ? ORDER BY turn_index ASC`,
+		)
+		.all(conversationId) as Array<Record<string, unknown>>;
+	return rows.map(rowToTurn);
+}
+
+/** Resolve a session's conversation id, generating + persisting one if none.
+ *  A resumed session inherits its existing conversationId from session_state. */
+export function ensureConversationId(
+	sessionId: string,
+	stateDir: string = getStateDir(),
+): string {
+	const st = loadSessionState(sessionId, stateDir);
+	if (st.conversationId) return st.conversationId;
+	const conv = newConversationId();
+	saveSessionState(
+		sessionId,
+		{
+			...st,
+			conversationId: conv,
+		},
+		stateDir,
+	);
+	return conv;
+}
+
+/**
+ * Fork a conversation at `forkTurnId`: create a new conversation id, record the
+ * branch lineage, and return the parent's recall set at that turn (the
+ * checkpoint_ids + scores that were injected) so the caller can seed the forked
+ * session's injected-set with exactly that context.
+ *
+ * Returns the new conversation id + the recall-set rows to replay. The caller
+ * (the engine/extension) is responsible for marking those checkpoint_ids as
+ * injected in the new session's session_state so they're not re-recalled.
+ */
+export function forkConversation(
+	parentConversationId: string,
+	forkTurnId: number,
+	stateDir: string = getStateDir(),
+): { conversationId: string; recalled: TurnRecallRow[] } {
+	const childId = newConversationId();
+	const db: DatabaseSync = openStore(stateDir);
+	withTx(db, () => {
+		db.prepare(
+			`INSERT INTO conversation_branches
+         (conversation_id, parent_conversation_id, fork_turn_id, created_at)
+       VALUES (?, ?, ?, ?)
+       ON CONFLICT(conversation_id) DO NOTHING`,
+		).run(childId, parentConversationId, forkTurnId, Date.now());
+	});
+	// Replay set: the parent's injected checkpoints at the fork turn.
+	const recalled = listTurnRecall(forkTurnId, stateDir);
+	return { conversationId: childId, recalled };
+}
+
+/** Clear turn tracking rows for a session (tests / DR). */
+export function clearTurns(
+	sessionId: string,
+	stateDir: string = getStateDir(),
+): void {
+	const db: DatabaseSync = openStore(stateDir);
+	const sid = normalizeSessionId(sessionId);
+	withTx(db, () => {
+		const turnIds = db
+			.prepare("SELECT id FROM turns WHERE session_id = ?")
+			.all(sid) as Array<{ id: number }>;
+		const ids = turnIds.map((t) => t.id);
+		if (ids.length > 0) {
+			const placeholders = ids.map(() => "?").join(",");
+			db.prepare(
+				`DELETE FROM turn_recall WHERE turn_id IN (${placeholders})`,
+			).run(...ids);
+		}
+		db.prepare("DELETE FROM turns WHERE session_id = ?").run(sid);
+	});
+}

package/src/store/sqlite/utils.ts

@@ -41,6 +41,16 @@ export function jsonText(v: unknown): string {
   return JSON.stringify(v ?? []);
 }
 
+/** Safe JSON parse — returns `fallback` on null/undefined/corrupt input. */
+export function safeJson<T>(s: string | null | undefined, fallback: T): T {
+  if (!s) return fallback;
+  try {
+    return JSON.parse(s) as T;
+  } catch {
+    return fallback;
+  }
+}
+
 // In-process cache so the same stateDir reuses one connection (and so a fresh
 // VectorStore over the same dir shares the open DB). Cross-process durability
 // comes from reopening the same file path — proven by the integration test.
@@ -54,7 +64,7 @@ export function openStore(stateDir: string = getStateDir()): DatabaseSync {
     // instead of closeStore) would surface as "database is not open" on the
     // next reuse. Detect and evict so callers never see a dead handle.
     try {
-      existing.prepare("SELECT 1");
+      existing.exec("SELECT 1");
       return existing;
     } catch {
       cache.delete(stateDir);
@@ -105,9 +115,9 @@ export function rowToCheckpoint(row: any): StoredCheckpoint {
     summary: row.summary ?? "",
     topicSummary: row.topic_summary ?? undefined,
     summaryHash: row.summary_hash ?? undefined,
-    keyDecisions: row.key_decisions ? JSON.parse(row.key_decisions) : [],
-    nextSteps: row.next_steps ? JSON.parse(row.next_steps) : [],
-    filesModified: row.files_modified ? JSON.parse(row.files_modified) : [],
+    keyDecisions: safeJson<string[]>(row.key_decisions, []),
+    nextSteps: safeJson<string[]>(row.next_steps, []),
+    filesModified: safeJson<string[]>(row.files_modified, []),
     tokenEstimate: row.token_estimate ?? 0,
     originalTokenEstimate: row.original_token_estimate ?? undefined,
     regionHash: row.region_hash ?? "",

package/src/store/sqlite.ts

@@ -24,3 +24,4 @@ export * from "./sqlite/game-state.js";
 export * from "./sqlite/game-scores.js";
 export * from "./sqlite/game-achievements.js";
 export * from "./sqlite/perf-samples.js";
+export * from "./sqlite/turns.js";

package/src/store.ts

@@ -41,7 +41,7 @@ export function normalizeSessionId(sessionId: string | undefined | null): string
   if (!sessionId) return `sess_${randomBytes(8).toString("hex")}`;
   if (sessionId.startsWith("sess_")) return sessionId;
   if (sessionId.length >= 32 && sessionId.includes("-")) {
-    return `sess_${sessionId.replace(/-/g, "").slice(0, 16)}`;
+    return `sess_${sessionId.replace(/-/g, "").toLowerCase().slice(0, 16)}`;
   }
   return `sess_${sessionId}`;
 }
@@ -113,7 +113,7 @@ export function writeGzJson(path: string, data: unknown): void {
 
 /** Append a checkpoint to the per-session checkpoint file (gzipped). */
 export function appendCheckpoint(cp: StoredCheckpoint, stateDir: string = getStateDir()): void {
-  const file = join(stateDir, `${cp.sessionId}.checkpoints.json.gz`);
+  const file = join(stateDir, `${normalizeSessionId(cp.sessionId)}.checkpoints.json.gz`);
   const existing = readGzJson<StoredCheckpoint[]>(file, []);
   existing.push(cp);
   writeGzJson(file, existing);
@@ -157,6 +157,13 @@ export interface SessionState {
   injectedCheckpointIds: string[];
   /** regionHashes already represented (for sentinel dedup). */
   storedRegionHashes: string[];
+  /** S43: conversation id grouping turns across pi session resumes. A resumed
+   *  session inherits its parent's; /clear (or any fresh root) generates a new
+   *  one. A fork copies the parent's turn-N injected-set and starts a new id. */
+  conversationId?: string;
+  /** S43: the global turn id of the most recent turn_end in this conversation.
+   *  Carried across resumes so fork-turn references stay stable. */
+  lastTurnId?: number;
 }
 
 /** Load mutable session state (created on demand). */

package/src/vector-search-cache.test.ts

@@ -0,0 +1,190 @@
+/**
+ * vector-search-cache.test.ts — QA perf/correctness regression tests for the
+ * RAPTOR serve path and the per-session raptorCache.
+ *
+ * Guards two invariants the S42B/S25 wiring introduced:
+ *   (a) Enabling RAPTOR (multilevel ON) never returns FEWER relevant hits than
+ *       flat-only at the same k — no cache-hit loss / MMR starvation.
+ *   (b) The per-search listCheckpoints is shared (not double-loaded): the
+ *       warm-search latency at 200 checkpoints stays under the QA budget.
+ *
+ * No network. Real stores, temp state dirs.
+ */
+
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore, vectorList, vectorSearch } from "./vectorStore.js";
+import { runRaptor } from "./dedup/raptor/index.js";
+import { compactSession } from "./engine.js";
+import { loadDedupConfig } from "./config/dedup.js";
+import { normalizeSessionId } from "./store.js";
+import type { EngineMessage } from "./types.js";
+import type { DedupConfigShape } from "./config/dedup.js";
+
+let tmpDir: string;
+let counter = 0;
+
+beforeEach(() => {
+	tmpDir = mkdtempSync(join(tmpdir(), "mc-vc-"));
+});
+
+afterEach(() => {
+	rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function stateDir(): string {
+	return join(tmpDir, `run-${counter++}`);
+}
+
+function cfg(overrides?: Record<string, unknown>): DedupConfigShape {
+	return {
+		...loadDedupConfig(),
+		RAPTOR_ENABLED: true,
+		L0_ENABLED: false,
+		L1_ENABLED: false,
+		L2_ENABLED: true,
+		...overrides,
+	};
+}
+
+function msg(text: string, toolName?: string): EngineMessage {
+	return toolName
+		? { role: "assistant", text, toolName, input: text, output: text }
+		: { role: "user", text };
+}
+
+function seedTwoTopics(store: VectorStore, sid: string, perTopic: number) {
+	const topics = [
+		"database connection pool postgres query optimizer",
+		"user interface button render react component",
+	];
+	for (let i = 1; i <= perTopic * 2; i++) {
+		const t = topics[i % 2];
+		compactSession(
+			{
+				sessionId: sid,
+				messages: [msg(`${t} checkpoint ${i} alpha beta gamma`), msg(`ack ${i}`, "Edit")],
+				keepFrom: 2,
+				timestamp: i,
+			},
+			store,
+		);
+	}
+}
+
+// ─── (a) RAPTOR ON never loses hits vs OFF ──────────────────────────────────
+
+test("QA cache: RAPTOR-ON returns >= flat-only hits at same k (no MMR starvation)", () => {
+	const k = 5;
+	// Build the same session twice (isolated state dirs) so the comparison is
+	// exact: same checkpoints, same embeddings, only the RAPTOR flag differs.
+	const queries = ["database connection pool", "user interface button", "alpha beta gamma"];
+
+	for (const q of queries) {
+		let offHits = 0;
+		let onHits = 0;
+		for (const raptorOn of [false, true]) {
+			const sd = stateDir();
+			const s = new VectorStore({
+				dedupSim: 0.9,
+				stateDir: sd,
+				config: raptorOn
+					? cfg({ RAPTOR_MULTILEVEL_ENABLED: true, RAPTOR_LEAF_EXPANSION: true })
+					: cfg({ RAPTOR_ENABLED: false }),
+			});
+			const sid = `cache-${q.length}-${raptorOn ? 1 : 0}`;
+			seedTwoTopics(s, sid, 8); // 16 checkpoints, 2 topics
+			if (raptorOn) {
+				const nsid = normalizeSessionId(sid);
+				runRaptor(
+					vectorList(s, nsid).map((cp) => ({
+						id: cp.checkpointId,
+						messages: [],
+						sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+						embedding: cp.embedding,
+					})),
+					{ stateDir: sd, sessionId: nsid },
+				);
+			}
+			process.env.RAPTOR_SHADOW_MODE = raptorOn ? "false" : "true";
+			const hits = vectorSearch(s, sid, q, k);
+			if (raptorOn) onHits = hits.length;
+			else offHits = hits.length;
+		}
+		assert.ok(
+			onHits >= offHits,
+			`RAPTOR ON (${onHits}) >= OFF (${offHits}) for query "${q}"`,
+		);
+	}
+});
+
+// ─── (b) Warm-search latency budget at 200 checkpoints ─────────────────────
+
+test("QA perf: warm RAPTOR search at 200 checkpoints stays under budget (no double scan)", () => {
+	const sd = stateDir();
+	const s = new VectorStore({
+		dedupSim: 0.9,
+		stateDir: sd,
+		config: cfg({ RAPTOR_MULTILEVEL_ENABLED: true }),
+	});
+	const sid = "perf";
+	seedTwoTopics(s, sid, 100); // 200 checkpoints
+	const nsid = normalizeSessionId(sid);
+	runRaptor(
+		vectorList(s, nsid).map((cp) => ({
+			id: cp.checkpointId,
+			messages: [],
+			sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+			embedding: cp.embedding,
+		})),
+		{ stateDir: sd, sessionId: nsid },
+	);
+	process.env.RAPTOR_SHADOW_MODE = "false";
+	// Warm the cache + embedder.
+	vectorSearch(s, sid, "database connection pool", 5);
+	vectorSearch(s, sid, "database connection pool", 5);
+	// Time a burst of warm searches. The double-scan fix keeps this well under
+	// the pre-fix cost. Budget is generous (10ms/search) to avoid CI flake;
+	// the regression it guards is the ~2x blowup from listCheckpoints running
+	// twice per search.
+	const ITERS = 50;
+	const t0 = Date.now();
+	for (let i = 0; i < ITERS; i++)
+		vectorSearch(s, sid, "database connection pool", 5);
+	const perSearch = (Date.now() - t0) / ITERS;
+	assert.ok(
+		perSearch < 10,
+		`warm search ${perSearch.toFixed(2)}ms < 10ms budget (double-scan regression guard)`,
+	);
+});
+
+// ─── (c) No duplicate checkpointIds in merged results ──────────────────────
+
+test("QA cache: RAPTOR merge produces no duplicate checkpointIds", () => {
+	const sd = stateDir();
+	const s = new VectorStore({
+		dedupSim: 0.9,
+		stateDir: sd,
+		config: cfg({ RAPTOR_MULTILEVEL_ENABLED: true, RAPTOR_LEAF_EXPANSION: false }),
+	});
+	const sid = "dup";
+	seedTwoTopics(s, sid, 8);
+	const nsid = normalizeSessionId(sid);
+	runRaptor(
+		vectorList(s, nsid).map((cp) => ({
+			id: cp.checkpointId,
+			messages: [],
+			sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+			embedding: cp.embedding,
+		})),
+		{ stateDir: sd, sessionId: nsid },
+	);
+	process.env.RAPTOR_SHADOW_MODE = "false";
+	const hits = vectorSearch(s, sid, "database connection pool", 8);
+	const ids = hits.map((h) => h.checkpoint.checkpointId);
+	const dupes = ids.filter((id, i) => ids.indexOf(id) !== i);
+	assert.equal(dupes.length, 0, "no duplicate checkpointIds in merged results");
+});