pi-mega-compact 0.9.2 → 0.10.1

package/src/topics/kselection.ts

@@ -0,0 +1,182 @@
+/**
+ * kselection.ts — S51A k-selection criterion math for topic clustering.
+ *
+ * Pure local math (PREVENT-PI-004): within-cluster sum of squares (cosine
+ * distance), mean silhouette score, and the elbow heuristic. k-means++ is
+ * reused from src/dedup/raptor/kmeans.ts. No model calls, no network.
+ *
+ * Extracted from cluster.ts so each module stays under the project's 300-line
+ * file-size cap with a single responsibility: this module owns the
+ * "which k is best" criterion; cluster.ts owns load + orchestration.
+ *
+ * Host-agnostic (no pi imports). Parameterized only by Vector arrays.
+ */
+
+import {
+	cosineDistance,
+	kmeanspp,
+	meanVector,
+} from "../dedup/raptor/kmeans.js";
+import type { Vector } from "../embedder.js";
+import type { WikiClusterConfig } from "./cluster.js";
+
+/** One k-means candidate result for a single k, kept for criterion scoring. */
+export interface Candidate {
+	k: number;
+	assignments: number[];
+	centroids: Vector[];
+	wcss: number;
+	silhouette: number | null;
+}
+
+/** Within-cluster sum of squares (cosine distance) for one clustering. */
+export function wcss(
+	points: Vector[],
+	assignments: number[],
+	centroids: Vector[],
+): number {
+	let s = 0;
+	for (let i = 0; i < points.length; i++) {
+		const d = cosineDistance(points[i], centroids[assignments[i]]);
+		s += d * d;
+	}
+	return s;
+}
+
+/**
+ * Mean silhouette score across all points (−1..1); null when undefined
+ * (k<=1 or k>=n, or every cluster is a singleton).
+ */
+export function silhouette(
+	points: Vector[],
+	assignments: number[],
+	k: number,
+): number | null {
+	const n = points.length;
+	if (k <= 1 || k >= n) return null;
+	// Precompute cluster sizes once.
+	const sizes = new Map<number, number>();
+	for (const a of assignments) sizes.set(a, (sizes.get(a) ?? 0) + 1);
+	let sum = 0;
+	let counted = 0;
+	for (let i = 0; i < n; i++) {
+		const a = assignments[i];
+		const ownSize = sizes.get(a) ?? 0;
+		if (ownSize <= 1) continue; // singleton cluster → contribution 0 (skip)
+		let aSum = 0;
+		const bSum = new Map<number, number>();
+		for (let j = 0; j < n; j++) {
+			if (i === j) continue;
+			const d = cosineDistance(points[i], points[j]);
+			if (assignments[j] === a) {
+				aSum += d;
+			} else {
+				bSum.set(assignments[j], (bSum.get(assignments[j]) ?? 0) + d);
+			}
+		}
+		const aMean = aSum / (ownSize - 1); // exclude self
+		let b = Infinity;
+		for (const [cluster, total] of bSum) {
+			const cnt = sizes.get(cluster) ?? 0;
+			if (cnt > 0) b = Math.min(b, total / cnt);
+		}
+		if (!Number.isFinite(b)) continue;
+		const denom = Math.max(aMean, b);
+		if (denom > 0) {
+			sum += (b - aMean) / denom;
+			counted++;
+		}
+	}
+	return counted > 0 ? sum / counted : null;
+}
+
+/** Elbow: index of max curvature on the WCSS-vs-k curve. */
+export function elbowIndex(ks: number[], wcssVals: number[]): number {
+	if (ks.length <= 2) return 0;
+	// Normalize and find the point farthest from the line joining the endpoints.
+	const x0 = ks[0];
+	const y0 = wcssVals[0];
+	const x1 = ks[ks.length - 1];
+	const y1 = wcssVals[wcssVals.length - 1];
+	const dx = x1 - x0;
+	const dy = y1 - y0;
+	const len = Math.hypot(dx, dy) || 1;
+	let best = 0;
+	let bestDist = -1;
+	for (let i = 0; i < ks.length; i++) {
+		const dist =
+			Math.abs(dy * ks[i] - dx * wcssVals[i] + x1 * y0 - y1 * x0) / len;
+		if (dist > bestDist) {
+			bestDist = dist;
+			best = i;
+		}
+	}
+	return best;
+}
+
+/** Run k-means with restarts for one k; keep the lowest-WCSS result. */
+export function bestForK(
+	points: Vector[],
+	k: number,
+	cfg: WikiClusterConfig,
+): Candidate {
+	let best: Candidate | null = null;
+	for (let r = 0; r < cfg.restarts; r++) {
+		const res = kmeanspp(points, k, { seed: cfg.seed + r });
+		if (res.k === 0) continue;
+		const w = wcss(points, res.assignments, res.centroids);
+		if (!best || w < best.wcss) {
+			best = {
+				k: res.k,
+				assignments: res.assignments,
+				centroids: res.centroids,
+				wcss: w,
+				silhouette: null,
+			};
+		}
+	}
+	// Degenerate: kmeans collapsed everything to < k clusters.
+	if (!best) {
+		const mean = meanVector(points);
+		best = {
+			k: 1,
+			assignments: points.map(() => 0),
+			centroids: [mean],
+			wcss: 0,
+			silhouette: null,
+		};
+	}
+	best.silhouette = silhouette(points, best.assignments, best.k);
+	return best;
+}
+
+/** The chosen candidate + which criterion selected it. */
+export interface SelectedK {
+	candidate: Candidate;
+	criterion: "elbow" | "silhouette";
+}
+
+/**
+ * Run k-means across the k search space and pick the best candidate:
+ * prefer silhouette when any candidate produced a computable score, else
+ * the elbow on the WCSS curve. `points` must be non-empty and `ks` non-empty.
+ */
+export function selectK(
+	points: Vector[],
+	ks: number[],
+	cfg: WikiClusterConfig,
+): SelectedK {
+	const candidates = ks.map((k) => bestForK(points, k, cfg));
+	const withSil = candidates.filter((c) => c.silhouette !== null);
+	if (withSil.length > 0) {
+		const chosen = withSil.reduce((a, b) =>
+			(b.silhouette ?? -1) > (a.silhouette ?? -1) ? b : a,
+		);
+		return { candidate: chosen, criterion: "silhouette" };
+	}
+	const idx = elbowIndex(
+		candidates.map((c) => c.k),
+		candidates.map((c) => c.wcss),
+	);
+	return { candidate: candidates[idx], criterion: "elbow" };
+}

package/src/topics/labels.ts

@@ -0,0 +1,67 @@
+/**
+ * labels.ts — S51A TF-IDF cluster labeling + membership confidence.
+ *
+ * Pure local math over real stored text (PREVENT-PI-004). NO fabricated keyword
+ * taxonomy, NO model: a cluster's label is the top TF-IDF terms of its real member
+ * chunks (real TF × real IDF over the whole corpus), so common words are
+ * naturally down-weighted by IDF rather than by a hand-written stopword list.
+ */
+
+import type { EmbeddedChunk } from "./types.js";
+
+/** Tokenize text: lowercase, split on non-alphanumeric, drop empties/1-char. */
+export function tokenize(text: string): string[] {
+	return text
+		.toLowerCase()
+		.split(/[^a-z0-9]+/)
+		.filter((t) => t.length > 1);
+}
+
+/**
+ * Compute TF-IDF term scores for one cluster's member chunks against the whole
+ * corpus. TF = term count across the cluster's members; IDF = log(1 + N/df)
+ * where N = total chunks and df = chunks containing the term. Returns the full
+ * list sorted by score desc.
+ */
+export function tfidfScores(
+	memberChunks: EmbeddedChunk[],
+	corpus: EmbeddedChunk[],
+): Array<{ term: string; score: number }> {
+	const tf = new Map<string, number>();
+	for (const c of memberChunks) {
+		for (const term of tokenize(c.text)) tf.set(term, (tf.get(term) ?? 0) + 1);
+	}
+	if (tf.size === 0) return [];
+
+	// Document frequency over the whole corpus (one vote per chunk per term).
+	const df = new Map<string, number>();
+	for (const c of corpus) {
+		for (const term of new Set(tokenize(c.text))) df.set(term, (df.get(term) ?? 0) + 1);
+	}
+	const n = Math.max(1, corpus.length);
+
+	const out: Array<{ term: string; score: number }> = [];
+	for (const [term, f] of tf) {
+		const idf = Math.log(1 + n / (df.get(term) ?? 1));
+		out.push({ term, score: f * idf });
+	}
+	out.sort((a, b) => b.score - a.score || a.term.localeCompare(b.term));
+	return out;
+}
+
+/** Join the top `topTerms` TF-IDF terms into a human label. */
+export function labelFromScores(
+	scores: Array<{ term: string; score: number }>,
+	topTerms: number,
+): string {
+	const top = scores.slice(0, Math.max(1, topTerms)).map((s) => s.term);
+	return top.length > 0 ? top.join(" · ") : "general";
+}
+
+/**
+ * Normalized cosine membership confidence in [0,1]. Cosine is in [-1,1]; map to
+ * [0,1] via (1 + cos)/2. A chunk sitting exactly on its centroid → 1.
+ */
+export function membershipConfidence(cosineSimilarity: number): number {
+	return Math.max(0, Math.min(1, (1 + cosineSimilarity) / 2));
+}

package/src/topics/store.test.ts

@@ -0,0 +1,132 @@
+/**
+ * store.test.ts — S51B topic persistence tests. Temp dirs; node:sqlite only.
+ * Verifies the topics/memory_topics shells (S49) are written atomically, CRUD
+ * round-trips, and the every-Nth-compaction counter behaves.
+ */
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+	createTopicStore,
+	getWikiCompactCounter,
+	bumpWikiCompactCounter,
+} from "./store.js";
+import { openTurnStore, closeTurnStore } from "../store/turns/connection.js";
+import type { ClusterModel } from "./types.js";
+
+let tmpDir: string;
+let counter = 0;
+
+beforeEach(() => {
+	tmpDir = mkdtempSync(join(tmpdir(), "mc-topicstore-"));
+});
+
+afterEach(() => {
+	rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function stateDir(): string {
+	return join(tmpDir, `run-${counter++}`);
+}
+
+function makeModel(builtAt: number, topics: number): ClusterModel {
+	const tops = [];
+	const assigns = [];
+	for (let i = 0; i < topics; i++) {
+		tops.push({
+			id: `topic_${i}`,
+			label: `label ${i}`,
+			termScores: [{ term: `t${i}`, score: i + 1 }],
+			memoryCount: 2,
+			lastUpdated: builtAt,
+		});
+		for (let m = 0; m < 2; m++) {
+			assigns.push({
+				memoryId: `mem_${i}_${m}`,
+				sessionId: "s",
+				topicId: `topic_${i}`,
+				confidence: 0.9 - m * 0.1,
+				assignedAt: builtAt,
+				method: "kmeans+tfidf" as const,
+			});
+		}
+	}
+	return {
+		topics: tops,
+		assignments: assigns,
+		k: topics,
+		criterion: "silhouette",
+		silhouetteScore: 0.5,
+		totalChunks: topics * 2,
+		builtAt,
+	};
+}
+
+test("replaceTopicModel inserts topics + assignments; CRUD round-trips", () => {
+	const dir = stateDir();
+	const store = createTopicStore(dir);
+	store.replaceTopicModel(makeModel(1000, 2));
+	const topics = store.getTopics();
+	assert.equal(topics.length, 2);
+	assert.equal(topics[0].label, "label 0");
+	assert.equal(topics[0].clusterModelBuiltAt, 1000);
+	assert.deepEqual(topics[0].termScores, [{ term: "t0", score: 1 }]);
+	const members = store.getMemoriesForTopic("topic_0");
+	assert.equal(members.length, 2);
+	assert.equal(members[0].memoryId, "mem_0_0"); // confidence DESC
+	const a = store.getTopicForMemory("mem_1_1");
+	assert.equal(a?.topicId, "topic_1");
+	assert.ok(Math.abs((a?.confidence ?? 0) - 0.8) < 1e-9);
+	closeTurnStore(dir);
+});
+
+test("replaceTopicModel is atomic — old model fully replaced", () => {
+	const dir = stateDir();
+	const store = createTopicStore(dir);
+	store.replaceTopicModel(makeModel(1000, 3)); // 3 topics, 6 assignments
+	store.replaceTopicModel(makeModel(2000, 1)); // 1 topic, 2 assignments
+	const stats = store.getTopicStats();
+	assert.equal(stats.totalTopics, 1);
+	assert.equal(stats.totalAssigned, 2);
+	assert.equal(stats.lastRebuildAt, 2000);
+	// Old topic ids gone.
+	assert.equal(store.getTopicForMemory("mem_2_0"), null);
+	closeTurnStore(dir);
+});
+
+test("getTopics sorted by memory_count DESC; stats roll up", () => {
+	const dir = stateDir();
+	const store = createTopicStore(dir);
+	const model = makeModel(500, 2);
+	model.topics[1].memoryCount = 99; // make topic_1 biggest
+	// give topic_1 more assignments to match memoryCount semantics loosely
+	store.replaceTopicModel(model);
+	const topics = store.getTopics();
+	assert.equal(topics[0].id, "topic_1"); // highest count first
+	const stats = store.getTopicStats();
+	assert.equal(stats.totalTopics, 2);
+	closeTurnStore(dir);
+});
+
+test("wiki compact counter increments + persists; default 0", () => {
+	const dir = stateDir();
+	const db = openTurnStore(dir);
+	assert.equal(getWikiCompactCounter(db), 0);
+	assert.equal(bumpWikiCompactCounter(db), 1);
+	assert.equal(bumpWikiCompactCounter(db), 2);
+	// Reopen handle (same cached conn) — value persists.
+	assert.equal(getWikiCompactCounter(openTurnStore(dir)), 2);
+	closeTurnStore(dir);
+});
+
+test("topics + memory_topics shells exist with no seed data (S49 schema)", () => {
+	const dir = stateDir();
+	const db = openTurnStore(dir);
+	const topics = db.prepare("SELECT COUNT(*) AS c FROM topics").get() as { c: number };
+	const assigns = db.prepare("SELECT COUNT(*) AS c FROM memory_topics").get() as { c: number };
+	assert.equal(topics.c, 0);
+	assert.equal(assigns.c, 0);
+	closeTurnStore(dir);
+});

package/src/topics/store.ts

@@ -0,0 +1,208 @@
+/**
+ * store.ts — S51B topic persistence over the turns.db `topics`/`memory_topics`
+ * shells (reserved in S49's schema, so NO migration / no schema drift).
+ *
+ * Host-agnostic (no pi imports). Pure local node:sqlite (PREVENT-PI-004), all
+ * queries parameterized (PREVENT-002). This is a derived, read-mostly view:
+ * `replaceTopicModel` rewrites ONLY the two topic tables in one transaction and
+ * never touches turns/memories/checkpoints (PREVENT-PI-001/002).
+ */
+
+import type { DatabaseSync } from "node:sqlite";
+import { openTurnStore, withTx } from "../store/turns/connection.js";
+import { getStateDir } from "../store.js";
+import type { ClusterModel, Topic, TopicAssignment } from "./types.js";
+
+/** A topic row joined with its computed member count. */
+export interface StoredTopic extends Topic {
+	/** Epoch ms the owning model was built (cluster_model_built_at column). */
+	clusterModelBuiltAt: number | null;
+}
+
+export interface TopicStore {
+	/** Atomically replace the entire topic model (clears old topics + assignments). */
+	replaceTopicModel(model: ClusterModel): void;
+	/** All topics, sorted by memory_count DESC. */
+	getTopics(): StoredTopic[];
+	/** Member memory ids for a topic (paginated). */
+	getMemoriesForTopic(
+		topicId: string,
+		limit?: number,
+		offset?: number,
+	): TopicAssignment[];
+	/** The assignment for one memory, or null. */
+	getTopicForMemory(memoryId: string): TopicAssignment | null;
+	/** Rollup stats for the wiki index header. */
+	getTopicStats(): {
+		totalTopics: number;
+		totalAssigned: number;
+		lastRebuildAt: number | null;
+	};
+}
+
+interface TopicRow {
+	id: string;
+	label: string;
+	term_scores: string | null;
+	memory_count: number;
+	last_updated: number | null;
+	cluster_model_built_at: number | null;
+}
+
+interface AssignmentRow {
+	memory_id: string;
+	topic_id: string;
+	confidence: number | null;
+	assigned_at: number | null;
+	method: string | null;
+}
+
+function parseTermScores(
+	json: string | null,
+): Array<{ term: string; score: number }> {
+	if (!json) return [];
+	try {
+		const v = JSON.parse(json) as unknown;
+		return Array.isArray(v)
+			? (v as Array<{ term: string; score: number }>)
+			: [];
+	} catch {
+		return [];
+	}
+}
+
+function rowToTopic(r: TopicRow): StoredTopic {
+	return {
+		id: r.id,
+		label: r.label,
+		termScores: parseTermScores(r.term_scores),
+		memoryCount: r.memory_count,
+		lastUpdated: r.last_updated ?? 0,
+		clusterModelBuiltAt: r.cluster_model_built_at,
+	};
+}
+
+function rowToAssignment(r: AssignmentRow): TopicAssignment {
+	return {
+		memoryId: r.memory_id,
+		// memory_topics has no session_id column; the chunk id is globally unique
+		// enough for the wiki view (context_chunks PK is (session_id, id)).
+		// sessionId omitted — memory_topics has no session_id column;
+		// the TopicAssignment type marks it optional for this reason.
+		topicId: r.topic_id,
+		confidence: r.confidence ?? 0,
+		assignedAt: r.assigned_at ?? 0,
+		method: "kmeans+tfidf",
+	};
+}
+
+/** Open a TopicStore over the same turns.db the S49 TurnStore uses (shared connection cache). */
+export function createTopicStore(stateDir?: string): TopicStore {
+	const db: DatabaseSync = openTurnStore(stateDir ?? getStateDir());
+
+	function replaceTopicModel(model: ClusterModel): void {
+		withTx(db, () => {
+			db.prepare("DELETE FROM memory_topics").run();
+			db.prepare("DELETE FROM topics").run();
+			const insTopic = db.prepare(
+				`INSERT INTO topics (id, label, term_scores, memory_count, last_updated, cluster_model_built_at)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+			);
+			for (const t of model.topics) {
+				insTopic.run(
+					t.id,
+					t.label,
+					JSON.stringify(t.termScores),
+					t.memoryCount,
+					t.lastUpdated,
+					model.builtAt,
+				);
+			}
+			const insAssign = db.prepare(
+				`INSERT OR REPLACE INTO memory_topics (memory_id, topic_id, confidence, assigned_at, method)
+         VALUES (?, ?, ?, ?, 'kmeans+tfidf')`,
+			);
+			for (const a of model.assignments) {
+				insAssign.run(a.memoryId, a.topicId, a.confidence, a.assignedAt);
+			}
+		});
+	}
+
+	return {
+		replaceTopicModel,
+
+		getTopics(): StoredTopic[] {
+			const rows = db
+				.prepare("SELECT * FROM topics ORDER BY memory_count DESC, id ASC")
+				.all() as unknown as TopicRow[];
+			return rows.map(rowToTopic);
+		},
+
+		getMemoriesForTopic(
+			topicId: string,
+			limit = 50,
+			offset = 0,
+		): TopicAssignment[] {
+			const rows = db
+				.prepare(
+					`SELECT memory_id, topic_id, confidence, assigned_at, method
+           FROM memory_topics WHERE topic_id = ?
+           ORDER BY confidence DESC, memory_id ASC LIMIT ? OFFSET ?`,
+				)
+				.all(topicId, limit, offset) as unknown as AssignmentRow[];
+			return rows.map(rowToAssignment);
+		},
+
+		getTopicForMemory(memoryId: string): TopicAssignment | null {
+			const r = db
+				.prepare(
+					`SELECT memory_id, topic_id, confidence, assigned_at, method
+           FROM memory_topics WHERE memory_id = ? LIMIT 1`,
+				)
+				.get(memoryId) as unknown as AssignmentRow | undefined;
+			return r ? rowToAssignment(r) : null;
+		},
+
+		getTopicStats(): {
+			totalTopics: number;
+			totalAssigned: number;
+			lastRebuildAt: number | null;
+		} {
+			const t = db.prepare("SELECT COUNT(*) AS c FROM topics").get() as {
+				c: number;
+			};
+			const a = db.prepare("SELECT COUNT(*) AS c FROM memory_topics").get() as {
+				c: number;
+			};
+			const r = db
+				.prepare("SELECT MAX(cluster_model_built_at) AS m FROM topics")
+				.get() as { m: number | null };
+			return {
+				totalTopics: t.c,
+				totalAssigned: a.c,
+				lastRebuildAt: r.m ?? null,
+			};
+		},
+	};
+}
+
+/** Turns-meta counter key for the every-Nth-compaction rebuild trigger. */
+const WIKI_COUNTER_KEY = "wiki_compact_counter";
+
+/** Read the rebuild counter (0 when unset). */
+export function getWikiCompactCounter(db: DatabaseSync): number {
+	const r = db
+		.prepare("SELECT value FROM turns_meta WHERE key = ?")
+		.get(WIKI_COUNTER_KEY) as { value: string } | undefined;
+	const n = r ? Number(r.value) : 0;
+	return Number.isFinite(n) ? n : 0;
+}
+
+/** Increment + persist the rebuild counter; returns the new value. */
+export function bumpWikiCompactCounter(db: DatabaseSync): number {
+	const next = getWikiCompactCounter(db) + 1;
+	db.prepare(
+		"INSERT OR REPLACE INTO turns_meta (key, value) VALUES (?, ?)",
+	).run(WIKI_COUNTER_KEY, String(next));
+	return next;
+}

package/src/topics/types.ts

@@ -0,0 +1,64 @@
+/**
+ * types.ts — S51A auto-categorizing wiki: shared types.
+ *
+ * Host-agnostic (no pi imports). These describe a derived, read-only topic view
+ * over stored memory chunks — see docs/specs/s51-auto-categorizing-wiki.md and
+ * the extended s47 spec. No model calls, no network (PREVENT-PI-004); pure local math.
+ */
+
+/** One derived topic cluster. */
+export interface Topic {
+	/** Stable cluster id (e.g. "topic_0"). */
+	id: string;
+	/** Human label = top TF-IDF terms joined (e.g. "sqlite · wal · checkpoint"). */
+	label: string;
+	/** Full sorted TF-IDF term scores for this cluster (label is the top slice). */
+	termScores: Array<{ term: string; score: number }>;
+	/** Number of member chunks assigned to this topic. */
+	memoryCount: number;
+	/** Epoch ms when this topic was (re)built. */
+	lastUpdated: number;
+}
+
+/** Assignment of one memory chunk to a topic. */
+export interface TopicAssignment {
+	/** `context_chunks.id` (per-session). */
+	memoryId: string;
+	/** Owning session — `context_chunks` PK is (session_id, id).
+	 *  Empty string when reconstructed from `memory_topics` (no session_id column there). */
+	sessionId?: string;
+	topicId: string;
+	/** Normalized cosine membership in [0,1] (closer to centroid → higher). */
+	confidence: number;
+	/** Epoch ms when assigned. */
+	assignedAt: number;
+	/** Constant provenance tag. */
+	method: "kmeans+tfidf";
+}
+
+/** The full derived model: k clusters + assignments + the criterion used. */
+export interface ClusterModel {
+	topics: Topic[];
+	assignments: TopicAssignment[];
+	/** Chosen cluster count (1 for a degenerate/small corpus). */
+	k: number;
+	/** Real criterion used to pick k. */
+	criterion: "elbow" | "silhouette";
+	/** Mean silhouette across clusters; null when the corpus is too small. */
+	silhouetteScore: number | null;
+	/** Chunks with usable embeddings that were clustered. */
+	totalChunks: number;
+	/** Epoch ms when built. */
+	builtAt: number;
+}
+
+/** A loaded chunk ready for clustering: its embedding + best-available text. */
+export interface EmbeddedChunk {
+	/** `context_chunks.id`. */
+	chunkId: string;
+	sessionId: string;
+	/** Decoded float32 embedding (L2-normalized by the embedder). */
+	vec: number[];
+	/** COALESCE(normalized_text, summary, topic_summary) — TF-IDF + summary source. */
+	text: string;
+}