pi-mega-compact 0.13.5 → 0.13.7

package/src/store/turns/sqlite-store/write.ts

@@ -22,8 +22,12 @@ export function appendTurn(ctx: SqliteTurnStoreCtx, entry: TurnEntry): TurnId {
 		db
 			.prepare(
 				`INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
-	                           ctx_tokens, ctx_percent, pressure_band, model, epoch_id)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	                           ctx_tokens, ctx_percent, pressure_band, model, epoch_id,
+	                           hyde_ran, hyde_doc, hyde_raw_count, hyde_hyde_count,
+	                           hyde_fused_count, hyde_lift, hyde_generation_ms, hyde_reason,
+	                           recall_score, recall_pass, recall_relevance, recall_coverage,
+	                           recall_diversity, recall_specificity)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 			)
 			.run(
 				entry.conversationId,
@@ -36,6 +40,20 @@ export function appendTurn(ctx: SqliteTurnStoreCtx, entry: TurnEntry): TurnId {
 				entry.pressureBand ?? null,
 				entry.model ?? null,
 				entry.epochId ?? null,
+				entry.hyde?.ran == null ? null : entry.hyde.ran ? 1 : 0,
+				entry.hyde?.hypotheticalDoc ?? null,
+				entry.hyde?.rawHitCount ?? null,
+				entry.hyde?.hydeHitCount ?? null,
+				entry.hyde?.fusedHitCount ?? null,
+				entry.hyde?.lift ?? null,
+				entry.hyde?.generationMs ?? null,
+				entry.hyde?.reason ?? null,
+				entry.recallMetrics?.score ?? null,
+				entry.recallMetrics?.pass == null ? null : entry.recallMetrics.pass ? 1 : 0,
+				entry.recallMetrics?.relevance ?? null,
+				entry.recallMetrics?.coverage ?? null,
+				entry.recallMetrics?.diversity ?? null,
+				entry.recallMetrics?.specificity ?? null,
 			);
 	} catch (e) {
 		if (

package/src/store/turns/types.ts

@@ -59,6 +59,39 @@ export interface TurnEntry {
 	model?: string; // model used for this turn
 	/** S50B: the compact epoch that superseded this turn (stamped post-hoc). */
 	epochId?: string;
+	/** H1: HyDE invocation telemetry for this turn (persisted to hyde_* columns). */
+	hyde?: TurnHydeTelemetry;
+	/** H1: recall-quality snapshot for this turn (persisted to recall_* columns). */
+	recallMetrics?: TurnRecallTelemetry;
+}
+
+/**
+ * H1: HyDE telemetry for a turn — a structural subset of the recall-core
+ * `HydeInvocationInfo` that maps onto the turns `hyde_*` columns.
+ */
+export interface TurnHydeTelemetry {
+	ran: boolean;
+	/** Skip reason when HyDE didn't run: "disabled" | "no-llm" | "generation-failed". */
+	reason: string;
+	hypotheticalDoc: string;
+	rawHitCount: number;
+	hydeHitCount: number;
+	fusedHitCount: number;
+	lift: number;
+	generationMs: number;
+}
+
+/**
+ * H1: recall-quality snapshot for a turn — a structural subset of the recall-core
+ * `RecallMetricsSnapshot` that maps onto the turns `recall_*` columns.
+ */
+export interface TurnRecallTelemetry {
+	score: number;
+	pass: boolean;
+	relevance: number;
+	coverage: number;
+	diversity: number;
+	specificity: number;
 }
 
 /** A recall hit recorded during a turn — an immutable, append-only fact. */

package/src/topics/durability.ts

@@ -0,0 +1,231 @@
+/**
+ * durability.ts — W5 full user-override replay over the topic tables.
+ *
+ * `replaceTopicModel` rebuilds `topics` + `memory_topics` from scratch, which
+ * would silently discard user curation (rename / merge / split). The small
+ * `applyOverridesAfterRebuild` in store.ts only replays label overrides; this
+ * module is the comprehensive replay — it replays label, merge, AND split
+ * overrides in chronological (`overridden_at`) order so the topic model stays
+ * durable across rebuilds and the incremental-assignment path (wiki/incremental.ts).
+ *
+ * Host-agnostic (no pi imports), pure local node:sqlite (PREVENT-PI-004), all
+ * SQL parameterized (PREVENT-002). Every override step is wrapped in try/catch
+ * and best-effort — a single bad row never aborts the rest of the replay.
+ */
+import type { DatabaseSync } from "node:sqlite";
+import { withTx } from "../store/turns/connection.js";
+import {
+	moveMemory,
+	recordEvolution,
+} from "../wiki/curation-helpers.js";
+
+/** One `topic_overrides` row ordered by override time. */
+interface OverrideRow {
+	topic_id: string;
+	kind: "label" | "merge" | "split";
+	custom_label: string | null;
+	merged_into: string | null;
+	split_from: string | null;
+	split_memory_ids: string | null;
+	merged_memory_ids: string | null;
+}
+
+/** Read all overrides in chronological order (oldest first). */
+function loadOverrides(db: DatabaseSync): OverrideRow[] {
+	return db
+		.prepare(
+			`SELECT topic_id, kind, custom_label, merged_into, split_from, split_memory_ids, merged_memory_ids
+			 FROM topic_overrides ORDER BY overridden_at ASC`,
+		)
+		.all() as unknown as OverrideRow[];
+}
+
+/** Does a topic row with this id exist? */
+function topicExists(db: DatabaseSync, topicId: string): boolean {
+	return !!db.prepare("SELECT 1 FROM topics WHERE id = ?").get(topicId);
+}
+
+/** All existing topic ids (for nearest-by-label-overlap fallback). */
+function allTopicIds(db: DatabaseSync): string[] {
+	const rows = db
+		.prepare("SELECT id FROM topics")
+		.all() as Array<{ id: string }>;
+	return rows.map((r) => r.id);
+}
+
+/**
+ * When a curated target topic dissolved in a rebuild, fall back to whichever
+ * surviving topic shares the most label tokens, so merged memories still land
+ * somewhere sensible. Returns "" when no overlap candidate exists.
+ */
+function nearestByLabelOverlap(db: DatabaseSync, label: string): string {
+	const tokens = new Set(
+		label.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 0),
+	);
+	if (tokens.size === 0) return "";
+	let bestId = "";
+	let bestOverlap = 0;
+	for (const id of allTopicIds(db)) {
+		const r = db
+			.prepare("SELECT label FROM topics WHERE id = ?")
+			.get(id) as { label: string } | undefined;
+		if (!r) continue;
+		const labelTokens = new Set(
+			r.label.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 0),
+		);
+		let overlap = 0;
+		for (const t of tokens) if (labelTokens.has(t)) overlap++;
+		if (overlap > bestOverlap) {
+			bestOverlap = overlap;
+			bestId = id;
+		}
+	}
+	return bestOverlap > 0 ? bestId : "";
+}
+
+/**
+ * Replay a single merge override: force the source topic's merged members back
+ * into the target topic (or its nearest-by-label-overlap survivor if the target
+ * dissolved). Member ids are read from `merged_memory_ids` on the override row
+ * (W5 — survives `replaceTopicModel` because `topic_overrides` is never wiped).
+ * Falls back to `topic_evolution` for pre-W5 merges (only works when evolution
+ * rows are intact, i.e. the incremental path or no rebuild has fired since).
+ */
+function replayMerge(
+	db: DatabaseSync,
+	row: OverrideRow,
+): void {
+	const target = row.merged_into;
+	if (!target) return;
+	// Primary source: merged_memory_ids on the override row itself.
+	let memoryIds: string[] = [];
+	if (row.merged_memory_ids) {
+		try {
+			const parsed = JSON.parse(row.merged_memory_ids) as unknown;
+			if (Array.isArray(parsed)) {
+				memoryIds = parsed.filter(
+					(m): m is string => typeof m === "string",
+				);
+			}
+		} catch {
+			/* non-fatal: malformed member list falls through to evolution */
+		}
+	}
+	// Fallback: topic_evolution rows (only intact when no rebuild has fired).
+	let members: Array<{ memory_id: string; session_id: string | null }> = [];
+	if (memoryIds.length === 0) {
+		members = db
+			.prepare(
+				`SELECT memory_id, session_id FROM topic_evolution
+				 WHERE topic_id = ? AND method = 'merge' ORDER BY memory_id ASC`,
+			)
+			.all(target) as Array<{ memory_id: string; session_id: string | null }>;
+		if (members.length === 0) return;
+	} else {
+		// Resolve session_id for each member from its current assignment (if any).
+		for (const mid of memoryIds) {
+			const r = db
+				.prepare(
+					"SELECT session_id FROM memory_topics WHERE memory_id = ? LIMIT 1",
+				)
+				.get(mid) as { session_id: string | null } | undefined;
+			members.push({
+				memory_id: mid,
+				session_id: r?.session_id ?? null,
+			});
+		}
+	}
+	let dest = target;
+	if (!topicExists(db, target)) {
+		dest = nearestByLabelOverlap(db, target);
+		if (!dest) return;
+	}
+	withTx(db, () => {
+		const t = Date.now();
+		for (const m of members) {
+			moveMemory(db, m.memory_id, dest);
+			recordEvolution(db, dest, m.memory_id, m.session_id, t, "merge");
+		}
+	});
+}
+
+/**
+ * Replay a single split override: recreate the split topic (if the source
+ * still exists or the memories are somewhere in the model) and move the listed
+ * memory ids into it.
+ */
+function replaySplit(db: DatabaseSync, row: OverrideRow): void {
+	let memoryIds: string[] = [];
+	if (row.split_memory_ids) {
+		try {
+			const parsed = JSON.parse(row.split_memory_ids) as unknown;
+			if (Array.isArray(parsed)) {
+				memoryIds = parsed.filter(
+					(m): m is string => typeof m === "string",
+				);
+			}
+		} catch {
+			/* non-fatal: malformed member list is skipped */
+		}
+	}
+	if (memoryIds.length === 0 || topicExists(db, row.topic_id)) return;
+	// Recreate the split topic reusing its original id + source label.
+	const sourceLabel =
+		row.split_from && topicExists(db, row.split_from)
+			? (db.prepare("SELECT label FROM topics WHERE id = ?")
+				.get(row.split_from) as { label: string }).label
+			: row.topic_id;
+	withTx(db, () => {
+		db.prepare(
+			`INSERT INTO topics (id, label, term_scores, memory_count, last_updated, cluster_model_built_at)
+			 VALUES (?, ?, NULL, 0, ?, NULL)`,
+		).run(row.topic_id, `${sourceLabel} (split)`, Date.now());
+		const t = Date.now();
+		// Only memories that still exist in the model move; drop the rest.
+		for (const memoryId of memoryIds) {
+			const src = db
+				.prepare(
+					"SELECT session_id FROM memory_topics WHERE memory_id = ? LIMIT 1",
+				)
+				.get(memoryId) as { session_id: string | null } | undefined;
+			if (!src) continue;
+			moveMemory(db, memoryId, row.topic_id);
+			recordEvolution(db, row.topic_id, memoryId, src.session_id, t, "split");
+		}
+	});
+}
+
+/**
+ * Apply all user overrides (label / merge / split) after a topic-model rebuild,
+ * in `overridden_at` order. Non-fatal end-to-end: each override is isolated so a
+ * single corruption never aborts the replay. This is the W5 high-fidelity replay (feature A).
+ */
+export function applyFullOverridesAfterRebuild(db: DatabaseSync): void {
+	if (!db) return;
+	let rows: OverrideRow[];
+	try {
+		rows = loadOverrides(db);
+	} catch {
+		return; // non-fatal: no overrides table / unreadable → nothing to replay
+	}
+	for (const row of rows) {
+		try {
+			if (row.kind === "label") {
+				// Same fast path as store.applyOverridesAfterRebuild, inline here
+				// so the full replay owns label + merge + split in one pass.
+				if (row.custom_label && row.custom_label.trim() !== "") {
+					db.prepare("UPDATE topics SET label = ? WHERE id = ?").run(
+						row.custom_label,
+						row.topic_id,
+					);
+				}
+			} else if (row.kind === "merge") {
+				replayMerge(db, row);
+			} else if (row.kind === "split") {
+				replaySplit(db, row);
+			}
+		} catch {
+			/* non-fatal: skip this override, keep replaying the rest */
+		}
+	}
+}

package/src/topics/index.ts

@@ -7,7 +7,13 @@ export {
 	DEFAULT_CLUSTER_CONFIG,
 	type WikiClusterConfig,
 } from "./cluster.js";
-export { createTopicStore, getWikiCompactCounter, bumpWikiCompactCounter } from "./store.js";
+export {
+	createTopicStore,
+	getWikiCompactCounter,
+	bumpWikiCompactCounter,
+	applyOverridesAfterRebuild,
+} from "./store.js";
+export { applyFullOverridesAfterRebuild } from "./durability.js";
 export type { TopicStore, StoredTopic } from "./store.js";
 export { tokenize, tfidfScores, labelFromScores, membershipConfidence } from "./labels.js";
 export type {

package/src/topics/store.ts

@@ -55,6 +55,7 @@ interface AssignmentRow {
 	confidence: number | null;
 	assigned_at: number | null;
 	method: string | null;
+	session_id: string | null;
 }
 
 function parseTermScores(
@@ -85,10 +86,7 @@ function rowToTopic(r: TopicRow): StoredTopic {
 function rowToAssignment(r: AssignmentRow): TopicAssignment {
 	return {
 		memoryId: r.memory_id,
-		// memory_topics has no session_id column; the chunk id is globally unique
-		// enough for the wiki view (context_chunks PK is (session_id, id)).
-		// sessionId omitted — memory_topics has no session_id column;
-		// the TopicAssignment type marks it optional for this reason.
+		sessionId: r.session_id ?? "",
 		topicId: r.topic_id,
 		confidence: r.confidence ?? 0,
 		assignedAt: r.assigned_at ?? 0,
@@ -119,11 +117,17 @@ export function createTopicStore(stateDir?: string): TopicStore {
 				);
 			}
 			const insAssign = db.prepare(
-				`INSERT OR REPLACE INTO memory_topics (memory_id, topic_id, confidence, assigned_at, method)
-         VALUES (?, ?, ?, ?, 'kmeans+tfidf')`,
+				`INSERT OR REPLACE INTO memory_topics (memory_id, topic_id, confidence, assigned_at, method, session_id)
+         VALUES (?, ?, ?, ?, 'kmeans+tfidf', ?)`,
 			);
 			for (const a of model.assignments) {
-				insAssign.run(a.memoryId, a.topicId, a.confidence, a.assignedAt);
+				insAssign.run(
+					a.memoryId,
+					a.topicId,
+					a.confidence,
+					a.assignedAt,
+					a.sessionId ?? null,
+				);
 			}
 		});
 	}
@@ -145,7 +149,7 @@ export function createTopicStore(stateDir?: string): TopicStore {
 		): TopicAssignment[] {
 			const rows = db
 				.prepare(
-					`SELECT memory_id, topic_id, confidence, assigned_at, method
+					`SELECT memory_id, topic_id, confidence, assigned_at, method, session_id
            FROM memory_topics WHERE topic_id = ?
            ORDER BY confidence DESC, memory_id ASC LIMIT ? OFFSET ?`,
 				)
@@ -156,7 +160,7 @@ export function createTopicStore(stateDir?: string): TopicStore {
 		getTopicForMemory(memoryId: string): TopicAssignment | null {
 			const r = db
 				.prepare(
-					`SELECT memory_id, topic_id, confidence, assigned_at, method
+					`SELECT memory_id, topic_id, confidence, assigned_at, method, session_id
            FROM memory_topics WHERE memory_id = ? LIMIT 1`,
 				)
 				.get(memoryId) as unknown as AssignmentRow | undefined;
@@ -186,6 +190,28 @@ export function createTopicStore(stateDir?: string): TopicStore {
 	};
 }
 
+/**
+ * Re-apply user label overrides after a topic-model rebuild. `replaceTopicModel`
+ * deletes + recreates `topics`, so any `label` override rows that resolved to a
+ * custom label would otherwise be lost — this copies them back onto the fresh
+ * rows. Non-fatal: a failure never breaks a rebuild.
+ */
+export function applyOverridesAfterRebuild(db: DatabaseSync): void {
+	try {
+		const rows = db
+			.prepare("SELECT topic_id, custom_label FROM topic_overrides WHERE kind = 'label'")
+			.all() as Array<{ topic_id: string; custom_label: string | null }>;
+		const upd = db.prepare("UPDATE topics SET label = ? WHERE id = ?");
+		for (const r of rows) {
+			if (r.custom_label && r.custom_label.trim() !== "") {
+				upd.run(r.custom_label, r.topic_id);
+			}
+		}
+	} catch {
+		/* non-fatal: override re-apply never breaks a rebuild */
+	}
+}
+
 /** Turns-meta counter key for the every-Nth-compaction rebuild trigger. */
 const WIKI_COUNTER_KEY = "wiki_compact_counter";
 

package/src/wiki/curation-helpers.ts

@@ -0,0 +1,140 @@
+/**
+ * curation-helpers.ts — private DB helpers for the wiki curation store.
+ *
+ * Pure node:sqlite primitives shared by the rename/merge/split transactions in
+ * curation.ts (the factory shell). Parameterized (PREVENT-002), host-agnostic.
+ */
+import type { DatabaseSync } from "node:sqlite";
+
+interface LabelRow {
+	label: string;
+}
+interface CountRow {
+	c: number;
+}
+
+/** Read a topic's current label, or null when the topic does not exist. */
+export function getTopicLabel(db: DatabaseSync, topicId: string): string | null {
+	const r = db
+		.prepare("SELECT label FROM topics WHERE id = ?")
+		.get(topicId) as LabelRow | undefined;
+	return r?.label ?? null;
+}
+
+/** Recompute memory_count for a topic from its live member rows. */
+export function recountTopic(db: DatabaseSync, topicId: string): void {
+	const c = db
+		.prepare("SELECT COUNT(*) AS c FROM memory_topics WHERE topic_id = ?")
+		.get(topicId) as unknown as CountRow;
+	db.prepare("UPDATE topics SET memory_count = ? WHERE id = ?").run(c.c, topicId);
+}
+
+/** Look up a custom label override; empty/blank means "no override". */
+export function customLabelOverride(
+	db: DatabaseSync,
+	topicId: string,
+): string | null {
+	const r = db
+		.prepare(
+			`SELECT custom_label FROM topic_overrides
+			 WHERE topic_id = ? AND kind = 'label'`,
+		)
+		.get(topicId) as { custom_label: string | null } | undefined;
+	const label = r?.custom_label;
+	return label && label.trim() !== "" ? label : null;
+}
+
+/** Write (or clear) a label override for a topic. */
+export function writeLabelOverride(
+	db: DatabaseSync,
+	topicId: string,
+	label: string,
+	now: number,
+): void {
+	if (label.trim() === "") {
+		db.prepare(
+			"DELETE FROM topic_overrides WHERE topic_id = ? AND kind = 'label'",
+		).run(topicId);
+		return;
+	}
+	db.prepare(
+		`INSERT OR REPLACE INTO topic_overrides
+		   (topic_id, kind, custom_label, overridden_at)
+		 VALUES (?, 'label', ?, ?)`,
+	).run(topicId, label, now);
+	// Keep the live topics.label in sync with the override so reads resolve directly.
+	db.prepare("UPDATE topics SET label = ? WHERE id = ?").run(label, topicId);
+}
+
+/** Copy a source assignment into the target topic (idempotent, atomic caller). */
+export function moveMemory(
+	db: DatabaseSync,
+	memoryId: string,
+	toTopicId: string,
+): void {
+	const src = db
+		.prepare(
+			`SELECT topic_id, confidence, assigned_at, session_id FROM memory_topics
+			 WHERE memory_id = ? LIMIT 1`,
+		)
+		.get(memoryId) as {
+		topic_id: string;
+		confidence: number | null;
+		assigned_at: number | null;
+		session_id: string | null;
+	} | undefined;
+	// Delete the source row first so a memory already present in the target just stays.
+	db.prepare("DELETE FROM memory_topics WHERE memory_id = ?").run(memoryId);
+	if (!src) return;
+	db.prepare(
+		`INSERT OR REPLACE INTO memory_topics
+		   (memory_id, topic_id, confidence, assigned_at, method, session_id)
+		 VALUES (?, ?, ?, ?, 'kmeans+tfidf', ?)`,
+	).run(
+		memoryId,
+		toTopicId,
+		src.confidence ?? 0,
+		src.assigned_at ?? 0,
+		src.session_id ?? null,
+	);
+}
+
+/** Record a member's provenance evolution row (best-effort, non-fatal). */
+export function recordEvolution(
+	db: DatabaseSync,
+	topicId: string,
+	memoryId: string,
+	sessionId: string | null,
+	assignedAt: number,
+	method: "merge" | "split",
+): void {
+	db.prepare(
+		`INSERT OR REPLACE INTO topic_evolution
+		   (topic_id, memory_id, session_id, assigned_at, method)
+		 VALUES (?, ?, ?, ?, ?)`,
+	).run(topicId, memoryId, sessionId, assignedAt, method);
+}
+
+/** Create a new split topic, returning its id. */
+export function createSplitTopic(
+	db: DatabaseSync,
+	sourceId: string,
+	sourceLabel: string,
+	now: number,
+): string {
+	// Two splits in the same millisecond collide on the base id; append a
+	// numeric suffix until the id is unused.
+	let newId = `topic_${sourceId}_split_${now}`;
+	let suffix = 2;
+	let exists = db.prepare("SELECT 1 FROM topics WHERE id = ?").get(newId);
+	while (exists) {
+		newId = `topic_${sourceId}_split_${now}_${suffix}`;
+		exists = db.prepare("SELECT 1 FROM topics WHERE id = ?").get(newId);
+		suffix++;
+	}
+	db.prepare(
+		`INSERT INTO topics (id, label, term_scores, memory_count, last_updated, cluster_model_built_at)
+		 VALUES (?, ?, NULL, 0, ?, NULL)`,
+	).run(newId, `${sourceLabel} (split)`, now);
+	return newId;
+}