pi-mega-compact 0.9.2 → 0.10.1

package/src/store/turns/types.ts

@@ -0,0 +1,181 @@
+/**
+ * types.ts — S49 contract module (the source of truth).
+ *
+ * Every implementation must satisfy these interfaces. Hosts import only this
+ * file + the factory from index.ts. SQL schemas are private to implementations.
+ *
+ * Design principles (from docs/specs/s49-rev1-architecture-upgrade.md):
+ *   1. Contract-first — the interface IS the spec.
+ *   2. Append-only — TurnWriter.append* are the only write methods; no UPDATE.
+ *   3. Capability-gated — asReader/asWriter/asAdmin return subset views.
+ *   4. Ledger protocol — host PUSHES facts, PULLS views. Store never initiates.
+ *   5. StoreSnapshot — checkpoint/restore for backup, migration, test seeding.
+ *
+ * PREVENT-PI-004: zero network. PREVENT-002: no SQL here (private to impls).
+ */
+
+// ─── Domain types ───────────────────────────────────────────────────
+
+/** Unique turn identifier (string to stay backend-agnostic). */
+export type TurnId = string;
+
+/** Unique conversation identifier. */
+export type ConversationId = string;
+
+/** Unique session identifier. */
+export type SessionId = string;
+
+/** A single turn record — an immutable, append-only fact. */
+export interface TurnEntry {
+	conversationId: ConversationId;
+	sessionId: SessionId;
+	turnIndex: number;
+	role: "user" | "assistant" | "system" | "tool";
+	endedAt: number; // epoch ms
+	ctxTokens?: number; // context window tokens at end of turn
+	ctxPercent?: number; // context window utilization 0-1
+	pressureBand?: "green" | "yellow" | "red";
+	model?: string; // model used for this turn
+	/** S50B: the compact epoch that superseded this turn (stamped post-hoc). */
+	epochId?: string;
+}
+
+/** A recall hit recorded during a turn — an immutable, append-only fact. */
+export interface TurnRecallEntry {
+	turnId: TurnId;
+	checkpointId: string;
+	score: number;
+	source: "checkpoint" | "cluster_summary" | "memory";
+	raptorLevel?: number;
+}
+
+/** A conversation fork — an immutable, append-only fact. */
+export interface ConversationFork {
+	parentConversationId: ConversationId;
+	childConversationId: ConversationId;
+	forkTurnIndex: number; // turn in the parent where the fork happened
+	createdAt: number;
+}
+
+/** Filters for querying turns. All fields optional (AND-combined). */
+export interface TurnFilter {
+	conversationId?: ConversationId;
+	sessionId?: SessionId;
+	sinceMs?: number; // epoch ms lower bound
+	untilMs?: number; // epoch ms upper bound
+	pressureBand?: string;
+	limit?: number;
+	offset?: number;
+}
+
+/** What a prune operation removed. */
+export interface PruneReport {
+	turnsRemoved: number;
+	recallRemoved: number;
+	branchesPreserved: number;
+	freedBytes: number; // approximate, from file-size delta
+}
+
+/** Retention policy — what the admin capability allows. */
+export interface RetentionPolicy {
+	maxTurnAgeMs: number; // delete turns older than this
+	keepMinPerConversation: number; // always keep at least N turns per conversation
+	vacuumAfterPrune: boolean; // run VACUUM after pruning
+}
+
+/** A complete snapshot for backup/migration/test-seeding. */
+export interface StoreSnapshot {
+	version: 1;
+	exportedAt: number; // epoch ms
+	turns: TurnEntry[];
+	recall: TurnRecallEntry[];
+	forks: ConversationFork[];
+}
+
+/** Aggregate stats for a conversation (materialized view, derived on read). */
+export interface ConversationStats {
+	turnCount: number;
+	firstTurnAt: number;
+	lastTurnAt: number;
+	avgCtxPercent: number;
+	pressureBands: Record<string, number>;
+}
+
+// ─── Capability interfaces ──────────────────────────────────────────
+
+/** Read-only view — dashboards, TUI, analytics. Cannot write. */
+export interface TurnReader {
+	query(filter: TurnFilter): TurnEntry[];
+	getTurn(turnId: TurnId): TurnEntry | undefined;
+	/** Resolve a turn by its (conversationId, turnIndex) coordinate (fork + recall replay). */
+	getTurnByIndex(
+		conversationId: ConversationId,
+		turnIndex: number,
+	): TurnEntry | undefined;
+	listRecall(turnId: TurnId): TurnRecallEntry[];
+	/** Recall hits recorded for the turn at (conversationId, turnIndex) — the
+	 *  replay set a fork rehydrates. Avoids needing the opaque TurnId. */
+	listRecallByIndex(
+		conversationId: ConversationId,
+		turnIndex: number,
+	): TurnRecallEntry[];
+	listForks(conversationId: ConversationId): ConversationFork[];
+	countTurns(conversationId: ConversationId): number;
+	conversationStats(conversationId: ConversationId): ConversationStats;
+}
+
+/** Append-only writer — compaction engine, event handlers. Cannot prune. */
+export interface TurnWriter {
+	appendTurn(entry: TurnEntry): TurnId;
+	appendRecall(entry: TurnRecallEntry): void;
+	ensureConversationId(sessionId: SessionId): ConversationId;
+	forkConversation(
+		parentId: ConversationId,
+		forkTurnIndex: number,
+	): ConversationId;
+}
+
+/** Admin operations — prune command, DR, migration. */
+export interface TurnAdmin {
+	prune(policy: RetentionPolicy): PruneReport;
+	vacuum(): void;
+	checkpoint(): StoreSnapshot;
+	restore(from: StoreSnapshot): void;
+	clear(): void; // test-only; wipes all data
+	/** S50B: stamp `epoch_id` on this session's turns that have none yet (links
+	 *  a turn to the compact epoch that superseded it). Backfill UPDATE, so it
+	 *  lives on the admin capability (the writer is append-only). Returns the
+	 *  number of turns stamped. */
+	stampTurnsEpoch(sessionId: SessionId, epochId: string): number;
+}
+
+/** The composed store — hosts get a capability-gated view. */
+export interface TurnStore extends TurnReader, TurnWriter, TurnAdmin {
+	/** Return a read-only view (for dashboards, TUI, analytics). */
+	asReader(): TurnReader;
+	/** Return an append-only view (for event handlers, compaction). */
+	asWriter(): TurnWriter;
+	/** Return an admin view (for prune, DR, migration). */
+	asAdmin(): TurnAdmin;
+	/** Close the underlying connection. For tests + graceful shutdown. */
+	close(): void;
+}
+
+// ─── Factory ───────────────────────────────────────────────────────
+
+/** Options for creating a TurnStore. */
+export interface TurnStoreOptions {
+	stateDir: string;
+	/** Override DB path (for tests / DR). Default: join(stateDir, "turns.db") */
+	dbPath?: string;
+	/** In-memory mode (for tests). Default: false. */
+	inMemory?: boolean;
+}
+
+/**
+ * Factory: create a TurnStore from a state directory.
+ *
+ * By default returns a SqliteTurnStore backed by turns.db.
+ * When options.inMemory is true, returns an InMemoryTurnStore.
+ */
+export type TurnStoreFactory = (options: TurnStoreOptions) => TurnStore;

package/src/topics/cluster.test.ts

@@ -0,0 +1,176 @@
+/**
+ * cluster.test.ts — S51A clustering + labeling tests. No network, temp dirs.
+ * Includes the honest-boundary grep-asserts (no ollama/llm/fetch, no fabricated
+ * keyword literals) required by the s47/s51 acceptance criteria.
+ */
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, readFileSync, readdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { openStore } from "../store/sqlite/utils.js";
+import { upsertCheckpoint } from "../store/sqlite/checkpoints.js";
+import type { StoredCheckpoint } from "../store.js";
+import { loadEmbeddings, buildTopicModel, DEFAULT_CLUSTER_CONFIG } from "./cluster.js";
+import { tfidfScores, membershipConfidence } from "./labels.js";
+import type { EmbeddedChunk } from "./types.js";
+
+let tmpDir: string;
+let counter = 0;
+
+beforeEach(() => {
+	tmpDir = mkdtempSync(join(tmpdir(), "mc-topics-"));
+});
+
+afterEach(() => {
+	rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function stateDir(): string {
+	return join(tmpDir, `run-${counter++}`);
+}
+
+/** Unit vector along axis `axis` (dim = total dims). */
+function axisVec(axis: number, dim: number): number[] {
+	const v = new Array<number>(dim).fill(0);
+	v[axis] = 1;
+	return v;
+}
+
+function cp(id: string, text: string, embedding: number[], session = "sess_t"): StoredCheckpoint {
+	return {
+		checkpointId: id,
+		sessionId: session,
+		summary: text,
+		normalizedText: text,
+		keyDecisions: [],
+		nextSteps: [],
+		filesModified: [],
+		tokenEstimate: 0,
+		regionHash: "r",
+		embedding,
+		timestamp: 1,
+	};
+}
+
+/** Seed N clusters of M chunks each: cluster i sits near axis i with distinct terms. */
+function seedClusters(dir: string, clusters: number, perCluster: number, dim: number): void {
+	openStore(dir);
+	const terms = ["sqlite", "wal", "checkpoint", "recovery", "backup", "replay", "fork", "merge"];
+	for (let c = 0; c < clusters; c++) {
+		for (let m = 0; m < perCluster; m++) {
+			// Near-axis vector with slight per-member jitter (stays closest to its axis).
+			const v = axisVec(c, dim).map((x, i) => x + (i === c ? 0 : (m + 1) * 1e-4 * (i + 1)));
+			const term = terms[(c * 2) % terms.length];
+			const term2 = terms[(c * 2 + 1) % terms.length];
+			upsertCheckpoint(
+				cp(`chkpt_c${c}_${m}`, `${term} ${term2} work item ${m} details`, v),
+				dir,
+			);
+		}
+	}
+}
+
+test("loadEmbeddings returns only chunks with embeddings + text", () => {
+	const dir = stateDir();
+	openStore(dir);
+	upsertCheckpoint(cp("a", "hello world", axisVec(0, 4)), dir);
+	upsertCheckpoint(cp("b", "", axisVec(1, 4)), dir); // empty text → skipped
+	upsertCheckpoint(cp("c", "no embedding here", [], "sess_t"), dir); // empty embedding → skipped
+	const chunks = loadEmbeddings(openStore(dir));
+	assert.equal(chunks.length, 1);
+	assert.equal(chunks[0].chunkId, "a");
+	assert.equal(chunks[0].text, "hello world");
+});
+
+test("buildTopicModel finds 3 well-separated clusters (silhouette picks k=3)", () => {
+	const dir = stateDir();
+	seedClusters(dir, 3, 6, 8); // 18 chunks, 3 tight clusters
+	const model = buildTopicModel(openStore(dir), { ...DEFAULT_CLUSTER_CONFIG, kRange: [2, 5] });
+	assert.equal(model.totalChunks, 18);
+	assert.equal(model.k, 3);
+	assert.equal(model.topics.length, 3);
+	// Every chunk assigned; assignment count == chunk count.
+	assert.equal(model.assignments.length, 18);
+	// Labels reflect the seeded discriminative terms (sqlite/wal, checkpoint/recovery, backup/replay).
+	const labels = model.topics.map((t) => t.label).join(" | ");
+	assert.ok(/sqlite|checkpoint|backup/.test(labels), `labels should carry seeded terms: ${labels}`);
+});
+
+test("corpus too small → single general cluster, no crash", () => {
+	const dir = stateDir();
+	openStore(dir);
+	upsertCheckpoint(cp("only", "lone chunk about sqlite", axisVec(0, 4)), dir);
+	const model = buildTopicModel(openStore(dir), DEFAULT_CLUSTER_CONFIG); // kRange[0]=3 > 1 chunk
+	assert.equal(model.k, 1);
+	assert.equal(model.topics.length, 1);
+	assert.equal(model.topics[0].memoryCount, 1);
+	assert.equal(model.silhouetteScore, null);
+});
+
+test("empty corpus → zero topics, no crash", () => {
+	const dir = stateDir();
+	openStore(dir);
+	const model = buildTopicModel(openStore(dir), DEFAULT_CLUSTER_CONFIG);
+	assert.equal(model.k, 0);
+	assert.equal(model.topics.length, 0);
+	assert.equal(model.assignments.length, 0);
+});
+
+test("tfidfScores surfaces discriminative terms, down-weights common ones", () => {
+	const mk = (text: string): EmbeddedChunk => ({
+		chunkId: text, sessionId: "s", vec: [1], text,
+	});
+	const corpus = [
+		mk("sqlite wal sqlite pragma"),
+		mk("sqlite wal journal"),
+		mk("react component render"),
+		mk("react hooks state"),
+	];
+	const members = corpus.slice(0, 2); // the sqlite cluster
+	const scores = tfidfScores(members, corpus);
+	const top = scores[0]?.term;
+	assert.ok(top === "sqlite" || top === "wal", `top term should be discriminative, got ${top}`);
+	// 'react' never appears in members → not scored for this cluster.
+	assert.ok(!scores.some((s) => s.term === "react"));
+});
+
+test("membershipConfidence maps cosine [-1,1] → [0,1]", () => {
+	assert.equal(membershipConfidence(1), 1);
+	assert.equal(membershipConfidence(-1), 0);
+	assert.equal(membershipConfidence(0), 0.5);
+});
+
+test("buildTopicModel is deterministic for a fixed seed", () => {
+	const dir = stateDir();
+	seedClusters(dir, 3, 5, 8);
+	const db = openStore(dir);
+	const a = buildTopicModel(db, { ...DEFAULT_CLUSTER_CONFIG, kRange: [2, 5] });
+	const b = buildTopicModel(db, { ...DEFAULT_CLUSTER_CONFIG, kRange: [2, 5] });
+	assert.equal(a.k, b.k);
+	assert.deepEqual(
+		a.topics.map((t) => t.label).sort(),
+		b.topics.map((t) => t.label).sort(),
+	);
+});
+
+test("honest boundary: no LLM/Ollama/network/keyword-literals in src/topics", () => {
+const here = dirname(fileURLToPath(import.meta.url));
+const srcTopics = join(here, "..", "..", "..", "src", "topics"); // dist/src/topics → src/topics
+const files = readdirSync(srcTopics).filter((f) => f.endsWith(".ts") && !f.endsWith(".test.ts"));
+const banned = [/ollama/i, /\bllm\b/i, /\bfetch\s*\(/, /\bhybrid\b/i];
+const fabricated = ["graphql", "docker", "stacktrace", "bottleneck", "trade-off"];
+for (const f of files) {
+const text = readFileSync(join(srcTopics, f), "utf8");
+for (const re of banned) {
+assert.ok(!re.test(text), `${f} must not contain ${re} (PREVENT-PI-004 / no-LLM boundary)`);
+}
+for (const word of fabricated) {
+assert.ok(
+!text.toLowerCase().includes(`"${word}"`),
+`${f} must not hardcode fabricated taxonomy term "${word}"`,
+);
+}
+}
+});

package/src/topics/cluster.ts

@@ -0,0 +1,191 @@
+/**
+ * cluster.ts — S51A k-means topic clustering over real stored embeddings.
+ *
+ * Host-agnostic (no pi imports), pure local math (PREVENT-PI-004): clusters the
+ * real `context_chunks.embedding_blob` vectors with k-means++ (reused from
+ * src/dedup/raptor/kmeans.ts), picks k by a real criterion (elbow on WCSS, or
+ * silhouette when the corpus is large enough), and labels each cluster with the
+ * top TF-IDF terms of its real member text. Pure math only — no model calls,
+ * no network, no remote service (PREVENT-PI-004).
+ *
+ * The topic view is derived + read-only over `context_chunks` — building a model
+ * never mutates any memory (PREVENT-PI-001/002). Parameterized SQL (PREVENT-002).
+ */
+
+import type { DatabaseSync } from "node:sqlite";
+import { cosineSimilarity } from "../embedder.js";
+import { decodeEmbedding } from "../store/sqlite/utils.js";
+import {
+	tfidfScores,
+	labelFromScores,
+	membershipConfidence,
+} from "./labels.js";
+import { selectK, type Candidate } from "./kselection.js";
+import type {
+	ClusterModel,
+	EmbeddedChunk,
+	Topic,
+	TopicAssignment,
+} from "./types.js";
+
+/** Config surface for clustering (host supplies; see src/config/dedup.ts in S51B). */
+export interface WikiClusterConfig {
+	/** [minK, maxK] search space for k selection. Default [3, 15]. */
+	kRange: [number, number];
+	/** How many TF-IDF terms form a label. Default 5. */
+	labelTopTerms: number;
+	/** k-means restarts per k (keep lowest WCSS). Default 5. */
+	restarts: number;
+	/** PRNG seed base for deterministic clustering. Default fixed. */
+	seed: number;
+}
+
+export const DEFAULT_CLUSTER_CONFIG: WikiClusterConfig = {
+	kRange: [3, 15],
+	labelTopTerms: 5,
+	restarts: 5,
+	seed: 0x9e3779b9,
+};
+
+interface ChunkRow {
+	id: string;
+	session_id: string;
+	embedding_blob: Uint8Array | null;
+	text: string | null;
+}
+
+/**
+ * Load chunks that have a usable embedding. Text is the best-available surface
+ * (normalized_text → summary → topic_summary). Skips rows with no embedding or
+ * no text. Returns in stable (session_id, id) order for determinism.
+ */
+export function loadEmbeddings(mainDb: DatabaseSync): EmbeddedChunk[] {
+	const rows = mainDb
+		.prepare(
+			`SELECT id, session_id, embedding_blob,
+              COALESCE(normalized_text, summary, topic_summary) AS text
+       FROM context_chunks
+       WHERE embedding_blob IS NOT NULL
+       ORDER BY session_id ASC, id ASC`,
+		)
+		.all() as unknown as ChunkRow[];
+	const out: EmbeddedChunk[] = [];
+	for (const r of rows) {
+		const vec = decodeEmbedding(r.embedding_blob);
+		if (vec.length === 0) continue;
+		const text = (r.text ?? "").trim();
+		if (text.length === 0) continue;
+		out.push({ chunkId: r.id, sessionId: r.session_id, vec, text });
+	}
+	return out;
+}
+
+/**
+ * Build a topic model from the real stored chunks. Picks k by silhouette when
+ * computable, else elbow; degenerate/small corpus → single `general` cluster.
+ * Never throws on degenerate input — returns the safe single-cluster model.
+ */
+export function buildTopicModel(
+	mainDb: DatabaseSync,
+	cfg: WikiClusterConfig = DEFAULT_CLUSTER_CONFIG,
+): ClusterModel {
+	const builtAt = Date.now();
+	const chunks = loadEmbeddings(mainDb);
+	const totalChunks = chunks.length;
+
+	// Too small to cluster meaningfully → single general cluster.
+	if (totalChunks < Math.max(2, cfg.kRange[0])) {
+		return singleClusterModel(chunks, builtAt, cfg);
+	}
+
+	const points = chunks.map((c) => c.vec);
+	const [minK, maxK] = cfg.kRange;
+	const hi = Math.min(maxK, Math.max(minK, Math.floor(totalChunks / 2)));
+	const ks: number[] = [];
+	for (let k = minK; k <= hi; k++) ks.push(k);
+	if (ks.length === 0) ks.push(minK);
+
+	const { candidate: chosen, criterion } = selectK(points, ks, cfg);
+	const meanSil = chosen.silhouette;
+	return assembleModel(chunks, chosen, criterion, meanSil, builtAt, cfg);
+}
+
+/** Single `general` cluster holding every chunk (degenerate/small corpus). */
+function singleClusterModel(
+	chunks: EmbeddedChunk[],
+	builtAt: number,
+	cfg: WikiClusterConfig,
+): ClusterModel {
+	const topic: Topic = {
+		id: "topic_0",
+		label: labelFromScores(tfidfScores(chunks, chunks), cfg.labelTopTerms),
+		termScores: tfidfScores(chunks, chunks),
+		memoryCount: chunks.length,
+		lastUpdated: builtAt,
+	};
+	const assignments: TopicAssignment[] = chunks.map((c) => ({
+		memoryId: c.chunkId,
+		sessionId: c.sessionId,
+		topicId: "topic_0",
+		confidence: 1,
+		assignedAt: builtAt,
+		method: "kmeans+tfidf",
+	}));
+	return {
+		topics: chunks.length > 0 ? [topic] : [],
+		assignments,
+		k: chunks.length > 0 ? 1 : 0,
+		criterion: "elbow",
+		silhouetteScore: null,
+		totalChunks: chunks.length,
+		builtAt,
+	};
+}
+
+/** Assemble topics + assignments from a chosen clustering. */
+function assembleModel(
+	chunks: EmbeddedChunk[],
+	chosen: Candidate,
+	criterion: "elbow" | "silhouette",
+	meanSil: number | null,
+	builtAt: number,
+	cfg: WikiClusterConfig,
+): ClusterModel {
+	const topics: Topic[] = [];
+	const assignments: TopicAssignment[] = [];
+	for (let c = 0; c < chosen.k; c++) {
+		const members = chunks.filter((_, i) => chosen.assignments[i] === c);
+		if (members.length === 0) continue;
+		const id = `topic_${topics.length}`;
+		const scores = tfidfScores(members, chunks);
+		topics.push({
+			id,
+			label: labelFromScores(scores, cfg.labelTopTerms),
+			termScores: scores,
+			memoryCount: members.length,
+			lastUpdated: builtAt,
+		});
+	}
+	for (let i = 0; i < chunks.length; i++) {
+		const cluster = chosen.assignments[i];
+		const centroid = chosen.centroids[cluster];
+		const sim = centroid ? cosineSimilarity(chunks[i].vec, centroid) : 0;
+		assignments.push({
+			memoryId: chunks[i].chunkId,
+			sessionId: chunks[i].sessionId,
+			topicId: `topic_${cluster}`,
+			confidence: membershipConfidence(sim),
+			assignedAt: builtAt,
+			method: "kmeans+tfidf",
+		});
+	}
+	return {
+		topics,
+		assignments,
+		k: topics.length,
+		criterion,
+		silhouetteScore: meanSil,
+		totalChunks: chunks.length,
+		builtAt,
+	};
+}

package/src/topics/index.ts

@@ -0,0 +1,18 @@
+/**
+ * index.ts — S51 topics barrel. Re-export only; no logic.
+ */
+export {
+	loadEmbeddings,
+	buildTopicModel,
+	DEFAULT_CLUSTER_CONFIG,
+	type WikiClusterConfig,
+} from "./cluster.js";
+export { createTopicStore, getWikiCompactCounter, bumpWikiCompactCounter } from "./store.js";
+export type { TopicStore, StoredTopic } from "./store.js";
+export { tokenize, tfidfScores, labelFromScores, membershipConfidence } from "./labels.js";
+export type {
+	Topic,
+	TopicAssignment,
+	ClusterModel,
+	EmbeddedChunk,
+} from "./types.js";