pi-recollect 1.0.0

package/src/store/search.ts

@@ -0,0 +1,222 @@
+import type Database from "better-sqlite3";
+
+// ── Types ──────────────────────────────────────────────────────────────────────
+
+export interface SearchResult {
+	id: string;
+	score: number;
+	title: string;
+	content: string;
+	category: string;
+}
+
+export interface SearchOptions {
+	maxResults?: number;
+	rrfK?: number;
+	proximityBoost?: number;
+	scope?: string;
+	detail?: "compact" | "medium" | "full";
+}
+
+// ── Porter stemmer search ──────────────────────────────────────────────────────
+
+export function porterSearch(
+	db: Database.Database,
+	query: string,
+	limit: number,
+): SearchResult[] {
+	const sql = `
+		SELECT source_id, title, content, category, rank
+		FROM chunks
+		WHERE chunks MATCH ?
+		ORDER BY bm25(chunks)
+		LIMIT ?
+	`;
+	try {
+		const rows = db.prepare(sql).all(query, limit) as Array<{
+			source_id: string;
+			title: string;
+			content: string;
+			category: string;
+			rank: number;
+		}>;
+		return rows.map((r) => ({
+			id: r.source_id,
+			score: -r.rank,
+			title: r.title ?? "",
+			content: r.content ?? "",
+			category: r.category ?? "",
+		}));
+	} catch {
+		return [];
+	}
+}
+
+// ── Trigram search ─────────────────────────────────────────────────────────────
+
+export function trigramSearch(
+	db: Database.Database,
+	query: string,
+	limit: number,
+): SearchResult[] {
+	// Trigram FTS5 requires wrapping query in quotes for phrase matching
+	const trigramQuery = `"${query.replace(/"/g, '""')}"`;
+	const sql = `
+		SELECT source_id, content, rank
+		FROM chunks_trigram
+		WHERE chunks_trigram MATCH ?
+		ORDER BY bm25(chunks_trigram)
+		LIMIT ?
+	`;
+	try {
+		const rows = db.prepare(sql).all(trigramQuery, limit) as Array<{
+			source_id: string;
+			content: string;
+			rank: number;
+		}>;
+		return rows.map((r) => ({
+			id: r.source_id,
+			score: -r.rank,
+			title: "",
+			content: r.content ?? "",
+			category: "",
+		}));
+	} catch {
+		return [];
+	}
+}
+
+// ── Reciprocal Rank Fusion ─────────────────────────────────────────────────────
+
+export function rrf(
+	porterResults: SearchResult[],
+	trigramResults: SearchResult[],
+	K: number = 60,
+): SearchResult[] {
+	const scores = new Map<string, { score: number; title: string; content: string; category: string }>();
+
+	for (const [rank, result] of porterResults.entries()) {
+		const existing = scores.get(result.id);
+		const addedScore = 1 / (K + rank + 1);
+		if (existing) {
+			existing.score += addedScore;
+		} else {
+			scores.set(result.id, {
+				score: addedScore,
+				title: result.title,
+				content: result.content,
+				category: result.category,
+			});
+		}
+	}
+
+	for (const [rank, result] of trigramResults.entries()) {
+		const existing = scores.get(result.id);
+		const addedScore = 1 / (K + rank + 1);
+		if (existing) {
+			existing.score += addedScore;
+			// Keep porter title if available
+			if (!existing.title && result.title) existing.title = result.title;
+		} else {
+			scores.set(result.id, {
+				score: addedScore,
+				title: result.title,
+				content: result.content,
+				category: result.category,
+			});
+		}
+	}
+
+	return [...scores.entries()]
+		.sort((a, b) => b[1].score - a[1].score)
+		.map(([id, data]) => ({
+			id,
+			score: data.score,
+			title: data.title,
+			content: data.content,
+			category: data.category,
+		}));
+}
+
+// ── Proximity reranking ────────────────────────────────────────────────────────
+
+export function proximityRerank(
+	results: SearchResult[],
+	query: string,
+	boost: number = 1.5,
+): SearchResult[] {
+	const terms = query.toLowerCase().split(/\s+/).filter((t) => t.length > 0);
+	if (terms.length <= 1) return results;
+
+	return results.map((result) => {
+		const lower = result.content.toLowerCase();
+		const positions: number[][] = terms.map((term) => {
+			const pos: number[] = [];
+			let idx = 0;
+			while (true) {
+				const found = lower.indexOf(term, idx);
+				if (found === -1) break;
+				pos.push(found);
+				idx = found + 1;
+			}
+			return pos;
+		});
+
+		// If any term not found, no proximity bonus
+		if (positions.some((p) => p.length === 0)) return result;
+
+		// Find minimum span covering one occurrence of each term
+		let minSpan = Infinity;
+		for (const pos0 of positions[0]!) {
+			const current: number[] = [pos0];
+			for (let t = 1; t < positions.length; t++) {
+				// Find closest position in term t to the last added position
+				const lastPos = current[current.length - 1]!;
+				let closest = positions[t]![0]!;
+				for (const p of positions[t]!) {
+					if (Math.abs(p - lastPos) < Math.abs(closest - lastPos)) {
+						closest = p;
+					}
+				}
+				current.push(closest);
+			}
+			const span = Math.max(...current) - Math.min(...current);
+			minSpan = Math.min(minSpan, span);
+		}
+
+		// Boost inversely proportional to span (closer terms = higher boost)
+		const contentLength = Math.max(result.content.length, 1);
+		const proximityScore = minSpan < contentLength ? boost * (1 - minSpan / contentLength) : 0;
+
+		return {
+			...result,
+			score: result.score + proximityScore,
+		};
+	}).sort((a, b) => b.score - a.score);
+}
+
+// ── Unified search ─────────────────────────────────────────────────────────────
+
+export function search(
+	db: Database.Database,
+	query: string,
+	opts: SearchOptions = {},
+): SearchResult[] {
+	const maxResults = opts.maxResults ?? 5;
+	const rrfK = opts.rrfK ?? 60;
+	const boost = opts.proximityBoost ?? 1.5;
+	const fetchLimit = maxResults * 4; // fetch more for reranking
+
+	const porter = porterSearch(db, query, fetchLimit);
+	const trigram = trigramSearch(db, query, fetchLimit);
+
+	let fused = rrf(porter, trigram, rrfK);
+	fused = proximityRerank(fused, query, boost);
+
+	// Scope filter
+	if (opts.scope && opts.scope !== "all") {
+		fused = fused.filter((r) => r.category === opts.scope);
+	}
+
+	return fused.slice(0, maxResults);
+}

package/src/store/sqlite.ts

@@ -0,0 +1,53 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import BetterSqlite3 from "better-sqlite3";
+import type Database from "better-sqlite3";
+import { initSchema } from "./schema.ts";
+
+/**
+ * Manages the SQLite database connection for pi-recall.
+ * Database is stored at <cwd>/.pi-recall/memory.db.
+ */
+export class MemoryDB {
+	public readonly cwd: string;
+	public readonly dbPath: string;
+	private db: Database.Database | null = null;
+
+	constructor(cwd: string) {
+		this.cwd = cwd;
+		const dir = path.join(cwd, ".pi-recall");
+		fs.mkdirSync(dir, { recursive: true });
+		this.dbPath = path.join(dir, "memory.db");
+	}
+
+	/** Open (or return existing) database connection with full schema. */
+	open(): Database.Database {
+		if (this.db) return this.db;
+		this.db = new BetterSqlite3(this.dbPath) as unknown as Database.Database;
+		initSchema(this.db);
+		return this.db;
+	}
+
+	/** Get the current open connection (throws if not open). */
+	getConnection(): Database.Database {
+		if (!this.db) throw new Error("MemoryDB not opened. Call open() first.");
+		return this.db;
+	}
+
+	/** Close the database connection. */
+	close(): void {
+		if (this.db) {
+			try {
+				this.db.close();
+			} catch {
+				// Ignore close errors
+			}
+			this.db = null;
+		}
+	}
+
+	/** Check if the database is currently open. */
+	get isOpen(): boolean {
+		return this.db !== null;
+	}
+}

package/src/store/vocabulary.ts

@@ -0,0 +1,34 @@
+import type Database from "better-sqlite3";
+
+export interface VocabStats {
+	docCount: number;
+	totalCount: number;
+}
+
+/**
+ * Update vocabulary term frequencies for a list of terms.
+ * Increments doc_count (unique docs) and total_count (total occurrences).
+ */
+export function updateVocabulary(db: Database.Database, terms: string[]): void {
+	const upsert = db.prepare(`
+		INSERT INTO vocabulary (term, doc_count, total_count)
+		VALUES (?, 1, 1)
+		ON CONFLICT(term) DO UPDATE SET
+			doc_count = doc_count + 1,
+			total_count = total_count + 1
+	`);
+	for (const term of terms) {
+		upsert.run(term.toLowerCase());
+	}
+}
+
+/**
+ * Get vocabulary statistics for a specific term.
+ */
+export function getVocabStats(db: Database.Database, term: string): VocabStats | null {
+	const row = db.prepare(
+		`SELECT doc_count, total_count FROM vocabulary WHERE term = ?`,
+	).get(term.toLowerCase()) as { doc_count: number; total_count: number } | undefined;
+	if (!row) return null;
+	return { docCount: row.doc_count, totalCount: row.total_count };
+}

package/src/tools/memory-recall.ts

@@ -0,0 +1,17 @@
+import type Database from "better-sqlite3";
+import { recallMemories, wrapInAntiFeedbackTags } from "../memory/recall.ts";
+
+export interface MemoryRecallInput {
+	context: string;
+	budget?: number;
+}
+
+/**
+ * Handle the memory_recall tool invocation.
+ */
+export function handleMemoryRecall(db: Database.Database, input: MemoryRecallInput): string {
+	const budget = input.budget ?? 2048;
+	const result = recallMemories(db, input.context, budget);
+	const wrapped = wrapInAntiFeedbackTags(result.content);
+	return `Recalled ${result.resultCount} memories (detail: ${result.detailLevel}, budget: ${budget} bytes):\n\n${wrapped}`;
+}

package/src/tools/memory-search.ts

@@ -0,0 +1,56 @@
+import type Database from "better-sqlite3";
+import { search, type SearchOptions } from "../store/search.ts";
+
+export interface MemorySearchInput {
+	query: string;
+	maxResults?: number;
+	scope?: "all" | "solutions" | "decisions" | "gotchas" | "conventions";
+	detail?: "compact" | "medium" | "full";
+}
+
+export interface MemorySearchResult {
+	results: Array<{
+		title: string;
+		content: string;
+		category: string;
+		score: number;
+	}>;
+	totalCount: number;
+}
+
+function formatResult(title: string, content: string, category: string, score: number, detail: string): string {
+	switch (detail) {
+		case "compact":
+			return `${title} (${category}, score: ${score.toFixed(3)})`;
+		case "medium":
+			return `${title} (${category}): ${content.slice(0, 200)}`;
+		case "full":
+			return `## ${title}\nCategory: ${category}\n\n${content}`;
+		default:
+			return `${title}: ${content.slice(0, 200)}`;
+	}
+}
+
+/**
+ * Handle the memory_search tool invocation.
+ */
+export function handleMemorySearch(db: Database.Database, input: MemorySearchInput): string {
+	const opts: SearchOptions = {
+		maxResults: input.maxResults ?? 5,
+		scope: input.scope ?? "all",
+		detail: input.detail ?? "compact",
+	};
+
+	const results = search(db, input.query, opts);
+
+	if (results.length === 0) {
+		return `No results found for "${input.query}".`;
+	}
+
+	const detail = input.detail ?? "compact";
+	const formatted = results
+		.map((r) => formatResult(r.title, r.content, r.category, r.score, detail))
+		.join("\n\n");
+
+	return `Found ${results.length} result(s) for "${input.query}":\n\n${formatted}`;
+}

package/src/tools/memory-status.ts

@@ -0,0 +1,113 @@
+import * as fs from "node:fs";
+import type Database from "better-sqlite3";
+
+export interface MemoryStatusInput {
+	action: "status" | "stats" | "reindex" | "compact" | "export";
+	format?: "text" | "json";
+}
+
+export interface MemoryStats {
+	solutions: number;
+	sources: number;
+	events: number;
+	mentalModels: number;
+	dbSize: number;
+	lastUpdated: string | null;
+}
+
+function getStats(db: Database.Database, dbPath: string): MemoryStats {
+	const solutions = (db.prepare(`SELECT COUNT(*) as c FROM solutions`).get() as { c: number }).c;
+	const sources = (db.prepare(`SELECT COUNT(*) as c FROM sources`).get() as { c: number }).c;
+	const events = (db.prepare(`SELECT COUNT(*) as c FROM events`).get() as { c: number }).c;
+	const mentalModels = (db.prepare(`SELECT COUNT(*) as c FROM mental_models`).get() as { c: number }).c;
+
+	let dbSize = 0;
+	try {
+		const stat = fs.statSync(dbPath);
+		dbSize = stat.size;
+	} catch { /* ignore */ }
+
+	const lastSolution = db.prepare(
+		`SELECT updated_at FROM solutions ORDER BY updated_at DESC LIMIT 1`,
+	).get() as { updated_at: string } | undefined;
+
+	return {
+		solutions,
+		sources,
+		events,
+		mentalModels,
+		dbSize,
+		lastUpdated: lastSolution?.updated_at ?? null,
+	};
+}
+
+function formatBytes(bytes: number): string {
+	if (bytes < 1024) return `${bytes}B`;
+	if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
+	return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+}
+
+/**
+ * Handle the memory_status tool invocation.
+ */
+export function handleMemoryStatus(db: Database.Database, dbPath: string, input: MemoryStatusInput): string {
+	const format = input.format ?? "text";
+
+	switch (input.action) {
+		case "status":
+		case "stats": {
+			const stats = getStats(db, dbPath);
+			if (format === "json") {
+				return JSON.stringify(stats, null, 2);
+			}
+			return [
+				`pi-recall Status:`,
+				`  Solutions stored: ${stats.solutions}`,
+				`  Sources indexed: ${stats.sources}`,
+				`  Session events: ${stats.events}`,
+				`  Mental models: ${stats.mentalModels}`,
+				`  Index size: ${formatBytes(stats.dbSize)}`,
+				`  Last updated: ${stats.lastUpdated ?? "never"}`,
+			].join("\n");
+		}
+		case "reindex": {
+			// Delete and rebuild FTS5 indexes from sources
+			db.exec(`DELETE FROM chunks`);
+			db.exec(`DELETE FROM chunks_trigram`);
+			const sources = db.prepare(`SELECT id, title, category, metadata FROM sources`).all() as Array<{
+				id: string;
+				title: string;
+				category: string;
+				metadata: string;
+			}>;
+			// Re-insert content from solutions and sources
+			const insertChunk = db.prepare(
+				`INSERT INTO chunks (source_id, title, content, category) VALUES (?, ?, ?, ?)`,
+			);
+			const insertTrigram = db.prepare(
+				`INSERT INTO chunks_trigram (source_id, content) VALUES (?, ?)`,
+			);
+			for (const source of sources) {
+				const content = source.title; // Use title as content for reindex
+				insertChunk.run(source.id, source.title, content, source.category);
+				insertTrigram.run(source.id, content);
+			}
+			return `Reindexed ${sources.length} sources.`;
+		}
+		case "compact": {
+			// VACUUM the database to reclaim space
+			db.exec(`VACUUM`);
+			const stats = getStats(db, dbPath);
+			return `Database compacted. Current size: ${formatBytes(stats.dbSize)}.`;
+		}
+		case "export": {
+			const stats = getStats(db, dbPath);
+			if (format === "json") {
+				return JSON.stringify({ stats }, null, 2);
+			}
+			return `Export:\n${JSON.stringify(stats, null, 2)}`;
+		}
+		default:
+			return `Unknown action: ${input.action}`;
+	}
+}

package/src/tools/memory-store.ts

@@ -0,0 +1,30 @@
+import type Database from "better-sqlite3";
+import { storeMemory, type StoreMemoryOpts } from "../memory/retain.ts";
+
+export interface MemoryStoreInput {
+	category: "gotcha" | "convention" | "decision" | "pattern" | "architecture";
+	title: string;
+	content: string;
+	metadata?: {
+		files?: string[];
+		tags?: string[];
+		severity?: "low" | "medium" | "high" | "critical";
+	};
+}
+
+/**
+ * Handle the memory_store tool invocation.
+ */
+export function handleMemoryStore(db: Database.Database, cwd: string, input: MemoryStoreInput): string {
+	const opts: StoreMemoryOpts = {
+		category: input.category,
+		title: input.title,
+		content: input.content,
+		files: input.metadata?.files,
+		tags: input.metadata?.tags,
+		severity: input.metadata?.severity,
+	};
+
+	const id = storeMemory(db, cwd, opts);
+	return `Stored memory "${input.title}" (${input.category}) with id ${id}.`;
+}

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "strict": true,
+    "noImplicitAny": true,
+    "exactOptionalPropertyTypes": false,
+    "skipLibCheck": true,
+    "allowImportingTsExtensions": true,
+    "noEmit": true,
+    "types": ["node"]
+  },
+  "include": [
+    "*.ts",
+    "src/**/*.ts",
+    "test/**/*.ts"
+  ]
+}