pi-mega-compact 0.9.2 → 0.10.1

package/src/wiki.test.ts

@@ -0,0 +1,217 @@
+/**
+ * wiki.test.ts — S51C extractive wiki page tests. No network; temp dirs.
+ * Includes the honest-boundary grep-assert (no ollama/llm/fetch) required by s47/s51.
+ */
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, readFileSync, readdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { extractiveSummary, generateWikiPage, buildWikiIndex } from "./wiki.js";
+import type { Topic, TopicAssignment } from "./topics/types.js";
+
+let tmpDir: string;
+
+beforeEach(() => {
+	tmpDir = mkdtempSync(join(tmpdir(), "mc-wiki-"));
+});
+
+afterEach(() => {
+	rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function mkTopic(
+	id: string,
+	label: string,
+	termScores: Array<{ term: string; score: number }>,
+): Topic {
+	return { id, label, termScores, memoryCount: 0, lastUpdated: 0 };
+}
+
+function mkAssignment(
+	memId: string,
+	topicId: string,
+	confidence = 0.8,
+): TopicAssignment {
+	return {
+		memoryId: memId,
+		sessionId: "s",
+		topicId,
+		confidence,
+		assignedAt: 0,
+		method: "kmeans+tfidf",
+	};
+}
+
+test("extractiveSummary returns top sentences joined", () => {
+	const topic = mkTopic("t1", "sqlite", [{ term: "sqlite", score: 1 }]);
+	const assignments = [mkAssignment("a", "t1"), mkAssignment("b", "t1")];
+	const getContent = (id: string): string => {
+		if (id === "a")
+			return "SQLite uses WAL mode. SQLite uses WAL mode. SQLite uses WAL mode.";
+		if (id === "b") return "Postgres uses WAL mode. Postgres uses WAL mode.";
+		return "";
+	};
+	const summary = extractiveSummary(topic, assignments, getContent);
+	// Should be non-empty and contain at least one sentence
+	assert.ok(summary.length > 0);
+	assert.ok(summary.includes("WAL"));
+});
+
+test("extractiveSummary returns empty string when topic has no members", () => {
+	const topic = mkTopic("t1", "empty", []);
+	const summary = extractiveSummary(topic, [], () => "");
+	assert.equal(summary, "");
+});
+
+test("generateWikiPage produces page with summary + key/recent memories", () => {
+	const topic = mkTopic("t1", "sqlite", [{ term: "sqlite", score: 1 }]);
+	const assignments = [
+		mkAssignment("a", "t1", 0.9),
+		mkAssignment("b", "t1", 0.5),
+	];
+	const getMemoryInfo = (
+		id: string,
+	): { content: string; timestamp: number; importance: number } | null => {
+		if (id === "a")
+			return {
+				content: "First memory content",
+				timestamp: 100,
+				importance: 0.9,
+			};
+		if (id === "b")
+			return {
+				content: "Second memory content",
+				timestamp: 200,
+				importance: 0.5,
+			};
+		return null;
+	};
+	const allTopics = [topic];
+
+	const page = generateWikiPage(topic, assignments, getMemoryInfo, allTopics);
+	assert.equal(page.topic.id, "t1");
+	assert.equal(page.keyMemories.length, 2);
+	// Key memories sorted by importance DESC
+	assert.equal(page.keyMemories[0].importance, 0.9);
+	// Recent memories sorted by timestamp DESC
+	assert.equal(page.recentMemories[0].timestamp, 200);
+	assert.equal(page.relatedTopics.length, 0);
+	assert.ok(page.generatedAt > 0);
+});
+
+test("generateWikiPage finds related topics via co-occurrence", () => {
+	const topicA = mkTopic("A", "topic-a", []);
+	const topicB = mkTopic("B", "topic-b", []);
+	const topicC = mkTopic("C", "topic-c", []);
+
+	// Memory "shared" appears in both topicA and topicB → A and B are related.
+	// Memory "a-only" only in topicA.
+	// Memory "c-only" only in topicC (no overlap with A).
+	const assignments = [
+		mkAssignment("shared", "A"),
+		mkAssignment("shared", "B"),
+		mkAssignment("a-only", "A"),
+		mkAssignment("c-only", "C"),
+	];
+	const getMemoryInfo = (
+		id: string,
+	): { content: string; timestamp: number; importance: number } | null => ({
+		content: `content for ${id}`,
+		timestamp: 100,
+		importance: 0.5,
+	});
+
+	const page = generateWikiPage(topicA, assignments, getMemoryInfo, [
+		topicA,
+		topicB,
+		topicC,
+	]);
+	// topicB should be related (shares "shared" with topicA); topicC should not.
+	const relatedIds = page.relatedTopics.map((t) => t.id);
+	assert.ok(
+		relatedIds.includes("B"),
+		`expected B in related topics, got ${relatedIds.join(",")}`,
+	);
+	assert.ok(
+		!relatedIds.includes("C"),
+		`C has no co-occurrence with A, should not be related`,
+	);
+});
+
+test("buildWikiIndex aggregates topics + memory counts", () => {
+	const topics = [mkTopic("t1", "sqlite", []), mkTopic("t2", "wal", [])];
+	topics[0].memoryCount = 5;
+	topics[1].memoryCount = 3;
+	topics[0].lastUpdated = 1000;
+	topics[1].lastUpdated = 2000;
+
+	const idx = buildWikiIndex(topics);
+	assert.equal(idx.totalTopics, 2);
+	assert.equal(idx.totalMemories, 8);
+	assert.equal(idx.topics.length, 2);
+	assert.ok(idx.lastRebuildAt !== null);
+});
+
+test("buildWikiIndex handles empty topics", () => {
+	const idx = buildWikiIndex([]);
+	assert.equal(idx.totalTopics, 0);
+	assert.equal(idx.totalMemories, 0);
+	assert.equal(idx.lastRebuildAt, null);
+});
+
+test("honest boundary: no LLM/network/keyword-literals in src/wiki.ts", () => {
+	const here = dirname(fileURLToPath(import.meta.url));
+	const wikiPath = join(here, "..", "..", "src", "wiki.ts");
+	const text = readFileSync(wikiPath, "utf8");
+	const banned = [/ollama/i, /\bllm\b/i, /\bfetch\s*\(/, /\bhybrid\b/i];
+	const fabricated = [
+		"graphql",
+		"docker",
+		"stacktrace",
+		"bottleneck",
+		"trade-off",
+	];
+	for (const re of banned) {
+		assert.ok(
+			!re.test(text),
+			`wiki.ts must not contain ${re} (PREVENT-PI-004 / no-model boundary)`,
+		);
+	}
+	for (const word of fabricated) {
+		assert.ok(
+			!text.toLowerCase().includes(`"${word}"`),
+			`wiki.ts must not hardcode fabricated taxonomy term "${word}"`,
+		);
+	}
+});
+
+test("honest boundary: no LLM/Ollama/network in src/topics/* (non-test)", () => {
+	const here = dirname(fileURLToPath(import.meta.url));
+	// dist/src/wiki.test.js → ../../src/topics
+	const topicsDir = join(here, "..", "..", "src", "topics");
+	const files = readdirSync(topicsDir).filter(
+		(f) => f.endsWith(".ts") && !f.endsWith(".test.ts"),
+	);
+	const banned = [/ollama/i, /\bllm\b/i, /\bfetch\s*\(/, /\bhybrid\b/i];
+	const fabricated = [
+		"graphql",
+		"docker",
+		"stacktrace",
+		"bottleneck",
+		"trade-off",
+	];
+	for (const f of files) {
+		const text = readFileSync(join(topicsDir, f), "utf8");
+		for (const re of banned) {
+			assert.ok(!re.test(text), `${f} must not contain ${re}`);
+		}
+		for (const word of fabricated) {
+			assert.ok(
+				!text.toLowerCase().includes(`"${word}"`),
+				`${f} must not hardcode fabricated taxonomy term "${word}"`,
+			);
+		}
+	}
+});

package/src/wiki.ts

@@ -0,0 +1,198 @@
+/**
+ * wiki.ts — S51C extractive wiki page generation from topic models.
+ *
+ * Host-agnostic (no pi imports). Pure local math + string processing
+ * (PREVENT-PI-004); read-only over the topic store + memory data. No
+ * model/network calls.
+ *
+ * Extractive summary uses a local TF-IDF sentence ranker (advances the dedup
+ * `src/dedup/raptor` patterns).
+ */
+
+import type { Topic, TopicAssignment, EmbeddedChunk } from "./topics/types.js";
+import { tfidfScores } from "./topics/labels.js";
+
+/** A wiki page for a single topic. */
+export interface WikiPage {
+	topic: Topic;
+	/** Extractive summary: top-scoring sentences from topic members, in original order. */
+	summary: string;
+	/** Memories with importance for the key-memories panel. */
+	keyMemories: Array<{
+		content: string;
+		timestamp: number;
+		importance: number;
+	}>;
+	/** Recently added or used memories (ordered by timestamp DESC). */
+	recentMemories: Array<{
+		content: string;
+		timestamp: number;
+		importance: number;
+	}>;
+	/** Topics that share members with this one (co-occurrence), limited to top few. */
+	relatedTopics: Topic[];
+	/** Generated at epoch ms. */
+	generatedAt: number;
+}
+
+/** Index shown on the wiki landing page. */
+export interface WikiIndex {
+	topics: Array<{
+		id: string;
+		label: string;
+		memoryCount: number;
+		lastUpdated: number;
+	}>;
+	totalTopics: number;
+	totalMemories: number;
+	lastRebuildAt: number | null;
+}
+
+/** Split text into sentences (very simple: period + newline + question mark + exclamation). */
+function splitSentences(text: string): string[] {
+	return text
+		.split(/[.\n!?]/)
+		.map((s) => s.trim())
+		.filter((s) => s.length > 10);
+}
+
+/** Wrap a text string as a pseudo EmbeddedChunk for tfidfScores reuse. */
+function pseudoChunk(text: string): EmbeddedChunk {
+	return { chunkId: "", sessionId: "", vec: [], text };
+}
+
+interface MemoryContent {
+	content: string;
+	timestamp: number;
+	importance: number;
+}
+
+/**
+ * Generate an extractive summary for a topic. Takes the topic's member texts,
+ * ranks sentences by TF-IDF, and returns the top-scoring ones joined.
+ * Pure local math; no model/network calls.
+ */
+export function extractiveSummary(
+	topic: Topic,
+	assignments: TopicAssignment[],
+	getContent: (memId: string) => string,
+): string {
+	// Join all member texts for the corpus.
+	const members: string[] = [];
+	for (const a of assignments) {
+		if (a.topicId === topic.id) {
+			const txt = getContent(a.memoryId);
+			if (txt) members.push(txt);
+		}
+	}
+	const corpusText = members.join(" ");
+	if (corpusText.length === 0) return "";
+	// Extractively summarize: rank sentences by TF-IDF, pick top 3.
+	const sentences = splitSentences(corpusText);
+	if (sentences.length === 0) return "";
+
+	// Reuse tfidfScores from labels.ts: each sentence is a pseudo-chunk scored
+	// against the full corpus. Sum the top-N term scores for each sentence.
+	const corpusChunks = [pseudoChunk(corpusText)];
+	const scored = sentences.map((s, i) => {
+		const sentScores = tfidfScores([pseudoChunk(s)], corpusChunks);
+		const totalScore = sentScores.reduce((sum, ts) => sum + ts.score, 0);
+		return { text: s, score: totalScore, idx: i };
+	});
+	scored.sort((a, b) => b.score - a.score || a.idx - b.idx);
+	const top = Math.min(3, sentences.length);
+	const chosen = scored.slice(0, top).sort((a, b) => a.idx - b.idx);
+	return chosen.map((s) => s.text).join(". ");
+}
+
+/**
+ * Find other topics (not `selfId`) that a memory also belongs to. Returns
+ * distinct topic ids where the memory has assignments for both `selfId` and
+ * that other topic.
+ */
+function coTopicsForMemory(
+	memId: string,
+	selfId: string,
+	allAssignments: TopicAssignment[],
+): string[] {
+	const memAssignments = allAssignments.filter((a) => a.memoryId === memId);
+	const topicSet = new Set(memAssignments.map((a) => a.topicId));
+	// Memory must belong to selfId + at least one other for co-occurrence.
+	if (!topicSet.has(selfId)) return [];
+	topicSet.delete(selfId);
+	return [...topicSet];
+}
+
+/**
+ * Generate a full wiki page for a topic.
+ *
+ * @param topic The topic to render.
+ * @param assignments All topic assignments to pull member memories.
+ * @param getMemoryInfo Function returning content/timestamp/importance for a memoryId.
+ * @param allTopics All topics (for co-occurrence computation).
+ */
+export function generateWikiPage(
+	topic: Topic,
+	assignments: TopicAssignment[],
+	getMemoryInfo: (memId: string) => MemoryContent | null,
+	allTopics: Topic[],
+): WikiPage {
+	const topicAssignments = assignments.filter((a) => a.topicId === topic.id);
+
+	const keyMemories = topicAssignments
+		.map((a) => getMemoryInfo(a.memoryId))
+		.filter((m): m is MemoryContent => m !== null)
+		.sort((a, b) => b.importance - a.importance)
+		.slice(0, 10);
+
+	const recentMemories = topicAssignments
+		.map((a) => getMemoryInfo(a.memoryId))
+		.filter((m): m is MemoryContent => m !== null)
+		.sort((a, b) => b.timestamp - a.timestamp)
+		.slice(0, 5);
+
+	// Co-occurrence: scan ALL assignments (not just this topic's) and count how
+	// many memories also belong to each other topic. Topics sharing member ids
+	// with this one are "related". A memory that belongs to topicA + topicB is
+	// evidence that A and B are related.
+	const topicIdCount = new Map<string, number>();
+	for (const a of assignments) {
+		for (const other of coTopicsForMemory(a.memoryId, topic.id, assignments)) {
+			topicIdCount.set(other, (topicIdCount.get(other) ?? 0) + 1);
+		}
+	}
+	const relatedTopics: Topic[] = [];
+	const ordered = [...topicIdCount.entries()].sort((a, b) => b[1] - a[1]);
+	for (const [tid] of ordered.slice(0, 3)) {
+		const t = allTopics.find((x) => x.id === tid);
+		if (t) relatedTopics.push(t);
+	}
+
+	const getContent = (memId: string): string =>
+		getMemoryInfo(memId)?.content ?? "";
+	const summary = extractiveSummary(topic, assignments, getContent);
+
+	return {
+		topic,
+		summary,
+		keyMemories,
+		recentMemories,
+		relatedTopics,
+		generatedAt: Date.now(),
+	};
+}
+
+/** Build the wiki index for the landing page. */
+export function buildWikiIndex(topics: Topic[]): WikiIndex {
+	return {
+		topics: topics.map((t) => ({
+			id: t.id,
+			label: t.label,
+			memoryCount: t.memoryCount,
+			lastUpdated: t.lastUpdated,
+		})),
+		totalTopics: topics.length,
+		totalMemories: topics.reduce((s, t) => s + t.memoryCount, 0),
+		lastRebuildAt: topics.length > 0 ? topics[0].lastUpdated : null,
+	};
+}