pi-mega-compact 0.11.4 → 0.11.5

package/src/supersede.test.ts

@@ -1,42 +0,0 @@
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import type { EngineMessage } from "./types.js";
-import { findSuperseded, supersede } from "./supersede.js";
-
-function msg(role: EngineMessage["role"], text: string): EngineMessage { return { role, text }; }
-
-test("read superseded by later write to same path is pruned", () => {
-  const messages = [
-    msg("assistant", "read src/server.ts"),
-    msg("user", "now change it"),
-    msg("assistant", "write src/server.ts with the fix"),
-  ];
-  assert.deepEqual(findSuperseded(messages), [0]);
-});
-
-test("older read superseded by newer read of same path (keep latest)", () => {
-  const messages = [
-    msg("assistant", "read src/a.ts"),
-    msg("assistant", "read src/a.ts again"),
-  ];
-  assert.deepEqual(findSuperseded(messages), [0]);
-});
-
-test("unrelated reads are kept", () => {
-  const messages = [
-    msg("assistant", "read src/a.ts"),
-    msg("assistant", "read src/b.ts"),
-  ];
-  assert.deepEqual(findSuperseded(messages), []);
-});
-
-test("supersede() drops the obsolete read and preserves order", () => {
-  const messages = [
-    msg("assistant", "read src/server.ts"),
-    msg("user", "change it"),
-    msg("assistant", "write src/server.ts done"),
-  ];
-  const out = supersede(messages);
-  assert.equal(out.length, 2);
-  assert.equal(out[0].text, "change it");
-});

package/src/topics/cluster.test.ts

@@ -1,176 +0,0 @@
-/**
- * cluster.test.ts — S51A clustering + labeling tests. No network, temp dirs.
- * Includes the honest-boundary grep-asserts (no ollama/llm/fetch, no fabricated
- * keyword literals) required by the s47/s51 acceptance criteria.
- */
-import { test, beforeEach, afterEach } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync, readFileSync, readdirSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-import { openStore } from "../store/sqlite/utils.js";
-import { upsertCheckpoint } from "../store/sqlite/checkpoints.js";
-import type { StoredCheckpoint } from "../store.js";
-import { loadEmbeddings, buildTopicModel, DEFAULT_CLUSTER_CONFIG } from "./cluster.js";
-import { tfidfScores, membershipConfidence } from "./labels.js";
-import type { EmbeddedChunk } from "./types.js";
-
-let tmpDir: string;
-let counter = 0;
-
-beforeEach(() => {
-	tmpDir = mkdtempSync(join(tmpdir(), "mc-topics-"));
-});
-
-afterEach(() => {
-	rmSync(tmpDir, { recursive: true, force: true });
-});
-
-function stateDir(): string {
-	return join(tmpDir, `run-${counter++}`);
-}
-
-/** Unit vector along axis `axis` (dim = total dims). */
-function axisVec(axis: number, dim: number): number[] {
-	const v = new Array<number>(dim).fill(0);
-	v[axis] = 1;
-	return v;
-}
-
-function cp(id: string, text: string, embedding: number[], session = "sess_t"): StoredCheckpoint {
-	return {
-		checkpointId: id,
-		sessionId: session,
-		summary: text,
-		normalizedText: text,
-		keyDecisions: [],
-		nextSteps: [],
-		filesModified: [],
-		tokenEstimate: 0,
-		regionHash: "r",
-		embedding,
-		timestamp: 1,
-	};
-}
-
-/** Seed N clusters of M chunks each: cluster i sits near axis i with distinct terms. */
-function seedClusters(dir: string, clusters: number, perCluster: number, dim: number): void {
-	openStore(dir);
-	const terms = ["sqlite", "wal", "checkpoint", "recovery", "backup", "replay", "fork", "merge"];
-	for (let c = 0; c < clusters; c++) {
-		for (let m = 0; m < perCluster; m++) {
-			// Near-axis vector with slight per-member jitter (stays closest to its axis).
-			const v = axisVec(c, dim).map((x, i) => x + (i === c ? 0 : (m + 1) * 1e-4 * (i + 1)));
-			const term = terms[(c * 2) % terms.length];
-			const term2 = terms[(c * 2 + 1) % terms.length];
-			upsertCheckpoint(
-				cp(`chkpt_c${c}_${m}`, `${term} ${term2} work item ${m} details`, v),
-				dir,
-			);
-		}
-	}
-}
-
-test("loadEmbeddings returns only chunks with embeddings + text", () => {
-	const dir = stateDir();
-	openStore(dir);
-	upsertCheckpoint(cp("a", "hello world", axisVec(0, 4)), dir);
-	upsertCheckpoint(cp("b", "", axisVec(1, 4)), dir); // empty text → skipped
-	upsertCheckpoint(cp("c", "no embedding here", [], "sess_t"), dir); // empty embedding → skipped
-	const chunks = loadEmbeddings(openStore(dir));
-	assert.equal(chunks.length, 1);
-	assert.equal(chunks[0].chunkId, "a");
-	assert.equal(chunks[0].text, "hello world");
-});
-
-test("buildTopicModel finds 3 well-separated clusters (silhouette picks k=3)", () => {
-	const dir = stateDir();
-	seedClusters(dir, 3, 6, 8); // 18 chunks, 3 tight clusters
-	const model = buildTopicModel(openStore(dir), { ...DEFAULT_CLUSTER_CONFIG, kRange: [2, 5] });
-	assert.equal(model.totalChunks, 18);
-	assert.equal(model.k, 3);
-	assert.equal(model.topics.length, 3);
-	// Every chunk assigned; assignment count == chunk count.
-	assert.equal(model.assignments.length, 18);
-	// Labels reflect the seeded discriminative terms (sqlite/wal, checkpoint/recovery, backup/replay).
-	const labels = model.topics.map((t) => t.label).join(" | ");
-	assert.ok(/sqlite|checkpoint|backup/.test(labels), `labels should carry seeded terms: ${labels}`);
-});
-
-test("corpus too small → single general cluster, no crash", () => {
-	const dir = stateDir();
-	openStore(dir);
-	upsertCheckpoint(cp("only", "lone chunk about sqlite", axisVec(0, 4)), dir);
-	const model = buildTopicModel(openStore(dir), DEFAULT_CLUSTER_CONFIG); // kRange[0]=3 > 1 chunk
-	assert.equal(model.k, 1);
-	assert.equal(model.topics.length, 1);
-	assert.equal(model.topics[0].memoryCount, 1);
-	assert.equal(model.silhouetteScore, null);
-});
-
-test("empty corpus → zero topics, no crash", () => {
-	const dir = stateDir();
-	openStore(dir);
-	const model = buildTopicModel(openStore(dir), DEFAULT_CLUSTER_CONFIG);
-	assert.equal(model.k, 0);
-	assert.equal(model.topics.length, 0);
-	assert.equal(model.assignments.length, 0);
-});
-
-test("tfidfScores surfaces discriminative terms, down-weights common ones", () => {
-	const mk = (text: string): EmbeddedChunk => ({
-		chunkId: text, sessionId: "s", vec: [1], text,
-	});
-	const corpus = [
-		mk("sqlite wal sqlite pragma"),
-		mk("sqlite wal journal"),
-		mk("react component render"),
-		mk("react hooks state"),
-	];
-	const members = corpus.slice(0, 2); // the sqlite cluster
-	const scores = tfidfScores(members, corpus);
-	const top = scores[0]?.term;
-	assert.ok(top === "sqlite" || top === "wal", `top term should be discriminative, got ${top}`);
-	// 'react' never appears in members → not scored for this cluster.
-	assert.ok(!scores.some((s) => s.term === "react"));
-});
-
-test("membershipConfidence maps cosine [-1,1] → [0,1]", () => {
-	assert.equal(membershipConfidence(1), 1);
-	assert.equal(membershipConfidence(-1), 0);
-	assert.equal(membershipConfidence(0), 0.5);
-});
-
-test("buildTopicModel is deterministic for a fixed seed", () => {
-	const dir = stateDir();
-	seedClusters(dir, 3, 5, 8);
-	const db = openStore(dir);
-	const a = buildTopicModel(db, { ...DEFAULT_CLUSTER_CONFIG, kRange: [2, 5] });
-	const b = buildTopicModel(db, { ...DEFAULT_CLUSTER_CONFIG, kRange: [2, 5] });
-	assert.equal(a.k, b.k);
-	assert.deepEqual(
-		a.topics.map((t) => t.label).sort(),
-		b.topics.map((t) => t.label).sort(),
-	);
-});
-
-test("honest boundary: no LLM/Ollama/network/keyword-literals in src/topics", () => {
-const here = dirname(fileURLToPath(import.meta.url));
-const srcTopics = join(here, "..", "..", "..", "src", "topics"); // dist/src/topics → src/topics
-const files = readdirSync(srcTopics).filter((f) => f.endsWith(".ts") && !f.endsWith(".test.ts"));
-const banned = [/ollama/i, /\bllm\b/i, /\bfetch\s*\(/, /\bhybrid\b/i];
-const fabricated = ["graphql", "docker", "stacktrace", "bottleneck", "trade-off"];
-for (const f of files) {
-const text = readFileSync(join(srcTopics, f), "utf8");
-for (const re of banned) {
-assert.ok(!re.test(text), `${f} must not contain ${re} (PREVENT-PI-004 / no-LLM boundary)`);
-}
-for (const word of fabricated) {
-assert.ok(
-!text.toLowerCase().includes(`"${word}"`),
-`${f} must not hardcode fabricated taxonomy term "${word}"`,
-);
-}
-}
-});

package/src/topics/store.test.ts

@@ -1,132 +0,0 @@
-/**
- * store.test.ts — S51B topic persistence tests. Temp dirs; node:sqlite only.
- * Verifies the topics/memory_topics shells (S49) are written atomically, CRUD
- * round-trips, and the every-Nth-compaction counter behaves.
- */
-import { test, beforeEach, afterEach } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import {
-	createTopicStore,
-	getWikiCompactCounter,
-	bumpWikiCompactCounter,
-} from "./store.js";
-import { openTurnStore, closeTurnStore } from "../store/turns/connection.js";
-import type { ClusterModel } from "./types.js";
-
-let tmpDir: string;
-let counter = 0;
-
-beforeEach(() => {
-	tmpDir = mkdtempSync(join(tmpdir(), "mc-topicstore-"));
-});
-
-afterEach(() => {
-	rmSync(tmpDir, { recursive: true, force: true });
-});
-
-function stateDir(): string {
-	return join(tmpDir, `run-${counter++}`);
-}
-
-function makeModel(builtAt: number, topics: number): ClusterModel {
-	const tops = [];
-	const assigns = [];
-	for (let i = 0; i < topics; i++) {
-		tops.push({
-			id: `topic_${i}`,
-			label: `label ${i}`,
-			termScores: [{ term: `t${i}`, score: i + 1 }],
-			memoryCount: 2,
-			lastUpdated: builtAt,
-		});
-		for (let m = 0; m < 2; m++) {
-			assigns.push({
-				memoryId: `mem_${i}_${m}`,
-				sessionId: "s",
-				topicId: `topic_${i}`,
-				confidence: 0.9 - m * 0.1,
-				assignedAt: builtAt,
-				method: "kmeans+tfidf" as const,
-			});
-		}
-	}
-	return {
-		topics: tops,
-		assignments: assigns,
-		k: topics,
-		criterion: "silhouette",
-		silhouetteScore: 0.5,
-		totalChunks: topics * 2,
-		builtAt,
-	};
-}
-
-test("replaceTopicModel inserts topics + assignments; CRUD round-trips", () => {
-	const dir = stateDir();
-	const store = createTopicStore(dir);
-	store.replaceTopicModel(makeModel(1000, 2));
-	const topics = store.getTopics();
-	assert.equal(topics.length, 2);
-	assert.equal(topics[0].label, "label 0");
-	assert.equal(topics[0].clusterModelBuiltAt, 1000);
-	assert.deepEqual(topics[0].termScores, [{ term: "t0", score: 1 }]);
-	const members = store.getMemoriesForTopic("topic_0");
-	assert.equal(members.length, 2);
-	assert.equal(members[0].memoryId, "mem_0_0"); // confidence DESC
-	const a = store.getTopicForMemory("mem_1_1");
-	assert.equal(a?.topicId, "topic_1");
-	assert.ok(Math.abs((a?.confidence ?? 0) - 0.8) < 1e-9);
-	closeTurnStore(dir);
-});
-
-test("replaceTopicModel is atomic — old model fully replaced", () => {
-	const dir = stateDir();
-	const store = createTopicStore(dir);
-	store.replaceTopicModel(makeModel(1000, 3)); // 3 topics, 6 assignments
-	store.replaceTopicModel(makeModel(2000, 1)); // 1 topic, 2 assignments
-	const stats = store.getTopicStats();
-	assert.equal(stats.totalTopics, 1);
-	assert.equal(stats.totalAssigned, 2);
-	assert.equal(stats.lastRebuildAt, 2000);
-	// Old topic ids gone.
-	assert.equal(store.getTopicForMemory("mem_2_0"), null);
-	closeTurnStore(dir);
-});
-
-test("getTopics sorted by memory_count DESC; stats roll up", () => {
-	const dir = stateDir();
-	const store = createTopicStore(dir);
-	const model = makeModel(500, 2);
-	model.topics[1].memoryCount = 99; // make topic_1 biggest
-	// give topic_1 more assignments to match memoryCount semantics loosely
-	store.replaceTopicModel(model);
-	const topics = store.getTopics();
-	assert.equal(topics[0].id, "topic_1"); // highest count first
-	const stats = store.getTopicStats();
-	assert.equal(stats.totalTopics, 2);
-	closeTurnStore(dir);
-});
-
-test("wiki compact counter increments + persists; default 0", () => {
-	const dir = stateDir();
-	const db = openTurnStore(dir);
-	assert.equal(getWikiCompactCounter(db), 0);
-	assert.equal(bumpWikiCompactCounter(db), 1);
-	assert.equal(bumpWikiCompactCounter(db), 2);
-	// Reopen handle (same cached conn) — value persists.
-	assert.equal(getWikiCompactCounter(openTurnStore(dir)), 2);
-	closeTurnStore(dir);
-});
-
-test("topics + memory_topics shells exist with no seed data (S49 schema)", () => {
-	const dir = stateDir();
-	const db = openTurnStore(dir);
-	const topics = db.prepare("SELECT COUNT(*) AS c FROM topics").get() as { c: number };
-	const assigns = db.prepare("SELECT COUNT(*) AS c FROM memory_topics").get() as { c: number };
-	assert.equal(topics.c, 0);
-	assert.equal(assigns.c, 0);
-	closeTurnStore(dir);
-});

package/src/types.test.ts

@@ -1,10 +0,0 @@
-import { test } from "node:test";
-import assert from "node:assert/strict";
-
-test("EngineMessage roles are constrained to the pi message contract (no system role)", () => {
-  const roles = ["user", "assistant", "tool", "custom"] as const;
-  // pi Message = user|assistant|tool (+ custom for markers). There is no system role.
-  const probe: string = "system";
-  assert.equal((roles as readonly string[]).includes(probe), false);
-  assert.equal(roles.length, 4);
-});

package/src/vector-search-cache.test.ts

@@ -1,190 +0,0 @@
-/**
- * vector-search-cache.test.ts — QA perf/correctness regression tests for the
- * RAPTOR serve path and the per-session raptorCache.
- *
- * Guards two invariants the S42B/S25 wiring introduced:
- *   (a) Enabling RAPTOR (multilevel ON) never returns FEWER relevant hits than
- *       flat-only at the same k — no cache-hit loss / MMR starvation.
- *   (b) The per-search listCheckpoints is shared (not double-loaded): the
- *       warm-search latency at 200 checkpoints stays under the QA budget.
- *
- * No network. Real stores, temp state dirs.
- */
-
-import { test, beforeEach, afterEach } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { VectorStore, vectorList, vectorSearch } from "./vectorStore.js";
-import { runRaptor } from "./dedup/raptor/index.js";
-import { compactSession } from "./engine.js";
-import { loadDedupConfig } from "./config/dedup.js";
-import { normalizeSessionId } from "./store.js";
-import type { EngineMessage } from "./types.js";
-import type { DedupConfigShape } from "./config/dedup.js";
-
-let tmpDir: string;
-let counter = 0;
-
-beforeEach(() => {
-	tmpDir = mkdtempSync(join(tmpdir(), "mc-vc-"));
-});
-
-afterEach(() => {
-	rmSync(tmpDir, { recursive: true, force: true });
-});
-
-function stateDir(): string {
-	return join(tmpDir, `run-${counter++}`);
-}
-
-function cfg(overrides?: Record<string, unknown>): DedupConfigShape {
-	return {
-		...loadDedupConfig(),
-		RAPTOR_ENABLED: true,
-		L0_ENABLED: false,
-		L1_ENABLED: false,
-		L2_ENABLED: true,
-		...overrides,
-	};
-}
-
-function msg(text: string, toolName?: string): EngineMessage {
-	return toolName
-		? { role: "assistant", text, toolName, input: text, output: text }
-		: { role: "user", text };
-}
-
-function seedTwoTopics(store: VectorStore, sid: string, perTopic: number) {
-	const topics = [
-		"database connection pool postgres query optimizer",
-		"user interface button render react component",
-	];
-	for (let i = 1; i <= perTopic * 2; i++) {
-		const t = topics[i % 2];
-		compactSession(
-			{
-				sessionId: sid,
-				messages: [msg(`${t} checkpoint ${i} alpha beta gamma`), msg(`ack ${i}`, "Edit")],
-				keepFrom: 2,
-				timestamp: i,
-			},
-			store,
-		);
-	}
-}
-
-// ─── (a) RAPTOR ON never loses hits vs OFF ──────────────────────────────────
-
-test("QA cache: RAPTOR-ON returns >= flat-only hits at same k (no MMR starvation)", () => {
-	const k = 5;
-	// Build the same session twice (isolated state dirs) so the comparison is
-	// exact: same checkpoints, same embeddings, only the RAPTOR flag differs.
-	const queries = ["database connection pool", "user interface button", "alpha beta gamma"];
-
-	for (const q of queries) {
-		let offHits = 0;
-		let onHits = 0;
-		for (const raptorOn of [false, true]) {
-			const sd = stateDir();
-			const s = new VectorStore({
-				dedupSim: 0.9,
-				stateDir: sd,
-				config: raptorOn
-					? cfg({ RAPTOR_MULTILEVEL_ENABLED: true, RAPTOR_LEAF_EXPANSION: true })
-					: cfg({ RAPTOR_ENABLED: false }),
-			});
-			const sid = `cache-${q.length}-${raptorOn ? 1 : 0}`;
-			seedTwoTopics(s, sid, 8); // 16 checkpoints, 2 topics
-			if (raptorOn) {
-				const nsid = normalizeSessionId(sid);
-				runRaptor(
-					vectorList(s, nsid).map((cp) => ({
-						id: cp.checkpointId,
-						messages: [],
-						sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
-						embedding: cp.embedding,
-					})),
-					{ stateDir: sd, sessionId: nsid },
-				);
-			}
-			process.env.RAPTOR_SHADOW_MODE = raptorOn ? "false" : "true";
-			const hits = vectorSearch(s, sid, q, k);
-			if (raptorOn) onHits = hits.length;
-			else offHits = hits.length;
-		}
-		assert.ok(
-			onHits >= offHits,
-			`RAPTOR ON (${onHits}) >= OFF (${offHits}) for query "${q}"`,
-		);
-	}
-});
-
-// ─── (b) Warm-search latency budget at 200 checkpoints ─────────────────────
-
-test("QA perf: warm RAPTOR search at 200 checkpoints stays under budget (no double scan)", () => {
-	const sd = stateDir();
-	const s = new VectorStore({
-		dedupSim: 0.9,
-		stateDir: sd,
-		config: cfg({ RAPTOR_MULTILEVEL_ENABLED: true }),
-	});
-	const sid = "perf";
-	seedTwoTopics(s, sid, 100); // 200 checkpoints
-	const nsid = normalizeSessionId(sid);
-	runRaptor(
-		vectorList(s, nsid).map((cp) => ({
-			id: cp.checkpointId,
-			messages: [],
-			sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
-			embedding: cp.embedding,
-		})),
-		{ stateDir: sd, sessionId: nsid },
-	);
-	process.env.RAPTOR_SHADOW_MODE = "false";
-	// Warm the cache + embedder.
-	vectorSearch(s, sid, "database connection pool", 5);
-	vectorSearch(s, sid, "database connection pool", 5);
-	// Time a burst of warm searches. The double-scan fix keeps this well under
-	// the pre-fix cost. Budget is generous (10ms/search) to avoid CI flake;
-	// the regression it guards is the ~2x blowup from listCheckpoints running
-	// twice per search.
-	const ITERS = 50;
-	const t0 = Date.now();
-	for (let i = 0; i < ITERS; i++)
-		vectorSearch(s, sid, "database connection pool", 5);
-	const perSearch = (Date.now() - t0) / ITERS;
-	assert.ok(
-		perSearch < 10,
-		`warm search ${perSearch.toFixed(2)}ms < 10ms budget (double-scan regression guard)`,
-	);
-});
-
-// ─── (c) No duplicate checkpointIds in merged results ──────────────────────
-
-test("QA cache: RAPTOR merge produces no duplicate checkpointIds", () => {
-	const sd = stateDir();
-	const s = new VectorStore({
-		dedupSim: 0.9,
-		stateDir: sd,
-		config: cfg({ RAPTOR_MULTILEVEL_ENABLED: true, RAPTOR_LEAF_EXPANSION: false }),
-	});
-	const sid = "dup";
-	seedTwoTopics(s, sid, 8);
-	const nsid = normalizeSessionId(sid);
-	runRaptor(
-		vectorList(s, nsid).map((cp) => ({
-			id: cp.checkpointId,
-			messages: [],
-			sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
-			embedding: cp.embedding,
-		})),
-		{ stateDir: sd, sessionId: nsid },
-	);
-	process.env.RAPTOR_SHADOW_MODE = "false";
-	const hits = vectorSearch(s, sid, "database connection pool", 8);
-	const ids = hits.map((h) => h.checkpoint.checkpointId);
-	const dupes = ids.filter((id, i) => ids.indexOf(id) !== i);
-	assert.equal(dupes.length, 0, "no duplicate checkpointIds in merged results");
-});