pi-context-engine 0.5.3 → 0.7.0

package/src/engine.ts

@@ -20,7 +20,7 @@ import { SessionCapture } from "./memory/session-capture.js";
 import { drainOne, enqueueSessionRollup, enqueueTurn, type CompressionDeps } from "./memory/compressor.js";
 import { compress, expand, compressWithCounts, DEFAULT_LEVEL } from "./memory/grammar.js";
 import { cosine } from "./memory/extractive.js";
-import { AutomaticIndexFileLimitError, runIndexing } from "./indexer/pipeline.js";
+import { AutomaticIndexFileLimitError, runIndexing, withPipelineLock } from "./indexer/pipeline.js";
 import { getModelPricing } from "./pricing.js";
 import type { Chunk } from "./types.js";
 
@@ -125,8 +125,10 @@ export interface SessionStats {
 export interface EngineOptions {
 	projectDir: string;
 	config?: Config;
-	/** Test seam: reuse an existing backend/embedder/reranker instead of auto-detecting. */
-	runtime?: { backend: LocalBackend; embedder: Embedder; reranker?: LocalReranker };
+	/** Test seam: reuse an existing backend/embedder/reranker instead of
+	 * auto-detecting. `backend` may be nulled by resetIndex — the pipeline
+	 * then constructs a fresh one. */
+	runtime?: { backend?: LocalBackend | null; embedder: Embedder; reranker?: LocalReranker };
 }
 
 /** Issue #11 (upstream warmup): what /context-warmup resolved + cached. */
@@ -161,8 +163,8 @@ export class ContextEngine {
 	readonly projectName: string;
 	readonly storageBase: string;
 	config: Config;
-	private readonly injectedRuntime:
-		| { backend: LocalBackend; embedder: Embedder; reranker?: LocalReranker }
+	private injectedRuntime:
+		| { backend?: LocalBackend | null; embedder: Embedder; reranker?: LocalReranker }
 		| undefined;
 	private backendInstance: LocalBackend | null = null;
 	private embedderInstance: Embedder | null = null;
@@ -188,7 +190,7 @@ export class ContextEngine {
 		this.config = options.config ?? loadConfigForProject(this.projectDir);
 		this.injectedRuntime = options.runtime;
 		if (options.runtime) {
-			this.backendInstance = options.runtime.backend;
+			this.backendInstance = options.runtime.backend ?? null;
 			this.embedderInstance = options.runtime.embedder;
 		}
 		this.storageBase = projectStorageDir(this.config.storagePath, this.projectDir);
@@ -358,6 +360,93 @@ export class ContextEngine {
 		return this.backend.countChunks();
 	}
 
+	/**
+	 * Delete this project's index (manifest, vector/FTS/graph stores, embedding
+	 * cache) while keeping memory, sessions and the savings ledger — those are
+	 * user history, not index state. Handles are closed BEFORE deletion:
+	 * Windows refuses to delete files with open sqlite handles, which is why
+	 * manually deleting the storage dir fails while a session is running.
+	 * Queues behind in-flight and queued pipeline runs, and holds the same lock
+	 * during deletion so a later reindex cannot race the reset.
+	 * Errors are reported per target (e.g. another session holding a handle)
+	 * instead of failing the whole reset.
+	 */
+	async resetIndex(): Promise<{ deletedFiles: number; freedBytes: number; errors: string[] }> {
+		return withPipelineLock(this.storageBase, () => this.resetIndexLocked());
+	}
+
+	private async resetIndexLocked(): Promise<{ deletedFiles: number; freedBytes: number; errors: string[] }> {
+		try {
+			this.backendInstance?.close();
+		} catch {
+			// already closed
+		}
+		this.backendInstance = null;
+		this.retrieverInstance = null;
+		// Mutate the runtime object rather than replacing it: reindex calls that
+		// were queued behind this reset captured the same object and must see the
+		// closed backend dropped before they execute. Embedder and reranker
+		// survive: they hold no files.
+		if (this.injectedRuntime) this.injectedRuntime.backend = null;
+		// A reset index may be rebuilt by the next automatic kick.
+		this.autoIndexDone = false;
+		let deletedFiles = 0;
+		let freedBytes = 0;
+		const errors: string[] = [];
+		const countAndDelete = (target: string): void => {
+			let stat: fs.Stats;
+			try {
+				stat = fs.statSync(target);
+			} catch {
+				return; // not there — nothing to free
+			}
+			let files = 0;
+			let bytes = 0;
+			const measure = (p: string, s: fs.Stats): void => {
+				if (s.isDirectory()) {
+					for (const entry of fs.readdirSync(p)) {
+						const child = path.join(p, entry);
+						try {
+							measure(child, fs.statSync(child));
+						} catch {
+							// raced away between readdir and stat
+						}
+					}
+					return;
+				}
+				files++;
+				bytes += s.size;
+			};
+			try {
+				measure(target, stat);
+				fs.rmSync(target, { recursive: true, force: true });
+				// Only report bytes actually released; a locked target must not
+				// inflate the success report when rmSync fails.
+				deletedFiles += files;
+				freedBytes += bytes;
+			} catch (err) {
+				// Locked file (another session?) — report instead of failing the reset.
+				errors.push(`${path.relative(this.storageBase, target) || path.basename(target)}: ${(err as Error).message.slice(0, 80)}`);
+			}
+		};
+		const graphDb = path.join(this.storageBase, "graph.db");
+		const embeddingCache = path.join(this.storageBase, "embedding_cache.db");
+		for (const target of [
+			path.join(this.storageBase, "manifest.json"),
+			path.join(this.storageBase, "vectors"),
+			path.join(this.storageBase, "fts"),
+			graphDb,
+			`${graphDb}-shm`,
+			`${graphDb}-wal`,
+			embeddingCache,
+			`${embeddingCache}-shm`,
+			`${embeddingCache}-wal`,
+		]) {
+			countAndDelete(target);
+		}
+		return { deletedFiles, freedBytes, errors };
+	}
+
 	async reindex(options: { full?: boolean; targetPath?: string | null; automatic?: boolean; onProgress?: (msg: string) => void } = {}): Promise<{
 		indexed: number;
 		chunks: number;

package/src/extension.ts

@@ -10,10 +10,12 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
 import fs from "node:fs";
 import path from "node:path";
+import { spawn } from "node:child_process";
 import { ContextEngine, isPceConfig } from "./engine.js";
 import { applyContextSearchGuidance } from "./guidance.js";
 import { ProjectWatcher } from "./indexer/watcher.js";
 import { AutomaticIndexFileLimitError } from "./indexer/pipeline.js";
+import { DashboardServer } from "./dashboard/server.js";
 
 /** Tools whose payloads are worth capturing for session timelines. */
 const CAPTURED_TOOLS = new Set(["bash", "read", "edit", "write", "grep", "find", "ls"]);
@@ -66,6 +68,14 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 	let watcherKickPending = false;
 	let watcherNotify: ((msg: string, level: "info" | "warning") => void) | undefined;
 
+	/**
+	 * Read-only web dashboard (config `dashboard.port`, 0 = free port). Owned
+	 * by this closure like the watcher: it must outlive individual command
+	 * invocations, reference the LIVE engine on every request (engine
+	 * instances are replaced per session), and stop on session_shutdown.
+	 */
+	let dashboard: DashboardServer | null = null;
+
 	/**
 	 * Watcher-triggered incremental reindex. Unlike `kickBackgroundIndex` this
 	 * calls `reindex({ automatic: true })` directly — the watcher must re-fire
@@ -245,6 +255,20 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		return engine;
 	};
 
+	/** Human-readable byte count for the reset report (explicit user action —
+	 * the automatic guard never measures bytes by design). */
+	const formatBytes = (n: number): string => {
+		if (n < 1024) return `${n} B`;
+		const units = ["KiB", "MiB", "GiB"];
+		let v = n / 1024;
+		let u = 0;
+		while (v >= 1024 && u < units.length - 1) {
+			v /= 1024;
+			u++;
+		}
+		return `${v.toFixed(v >= 10 ? 0 : 1)} ${units[u]}`;
+	};
+
 	const text = (body: string, details?: unknown) => ({
 		content: [{ type: "text" as const, text: body }],
 		details: details ?? {},
@@ -495,9 +519,46 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 	/* ── Commands ────────────────────────────────────────────────────────── */
 
 	pi.registerCommand("context-index", {
-		description: "Build or refresh the semantic code index: /context-index [--full]",
+		description: "Build or refresh the semantic code index: /context-index [--full] [--reset]",
 		handler: async (args, ctx) => {
 			const eng = getEngine(ctx.cwd);
+			// --reset: delete the index through the engine (handles closed first, so
+			// it works while THIS session runs — manual deletion hits locked-file
+			// errors). Memory, sessions and savings survive. No auto-rebuild: the
+			// next search or /context-index decides, so an oversized workspace is
+			// not ambushed with a multi-minute build it just escaped.
+			if (args.includes("--reset")) {
+				if (indexInFlight) {
+					ctx.ui.notify(
+						"Context index: indexing is in progress — run /context-index --reset again once it finishes.",
+						"warning",
+					);
+					return;
+				}
+				// Reset also cancels the delayed startup kick. Otherwise its timer
+				// would rebuild the index immediately after the explicit delete.
+				if (autoIndexTimer) {
+					clearTimeout(autoIndexTimer);
+					autoIndexTimer = null;
+				}
+				const report = await eng.resetIndex();
+				// Free the lazy kick so the next search can rebuild (guarded by the
+				// same file budget as every automatic run).
+				lazyIndexStarted = false;
+				if (report.errors.length > 0) {
+					ctx.ui.notify(
+						`⚠ Index reset incomplete: ${report.errors.length} target(s) locked (${report.errors[0]}). Close other sessions on this project and run /context-index --reset again.`,
+						"warning",
+					);
+				} else {
+					ctx.ui.notify(
+						`✓ Index deleted: ${report.deletedFiles} file(s), ${formatBytes(report.freedBytes)} freed — memory, sessions and savings kept. ` +
+							"Run /context-index to build a fresh index.",
+						"info",
+					);
+				}
+				return;
+			}
 			const full = args.includes("--full");
 			ctx.ui.notify(full ? "Re-indexing everything…" : "Indexing (incremental)…", "info");
 			const result = await eng.reindex({ full });
@@ -553,6 +614,46 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		},
 	});
 
+	pi.registerCommand("context-dashboard", {
+		description: "Open the read-only web dashboard (index health, files, sessions, memory, savings) in your browser",
+		handler: async (_args, ctx) => {
+			const openBrowser = (url: string): void => {
+				const cmd = process.platform === "win32" ? "cmd.exe" : process.platform === "darwin" ? "open" : "xdg-open";
+				const args = process.platform === "win32" ? ["/c", "start", "", url] : [url];
+				try {
+					spawn(cmd, args, { detached: true, stdio: "ignore" }).unref();
+				} catch {
+					// best-effort: the URL is printed either way
+				}
+			};
+			let url = dashboard?.url ?? null;
+			if (!dashboard || !url) {
+				const eng = getEngine(ctx.cwd);
+				dashboard = new DashboardServer();
+				try {
+					url = await dashboard.start({
+						engine: () => engine,
+						indexing: () => indexInFlight,
+						port: eng.config.dashboardPort,
+					});
+				} catch (err) {
+					dashboard = null;
+					ctx.ui.notify(
+						`⚠ Dashboard failed to start: ${(err as Error).message}. ` +
+							`If dashboard.port is set, another process may be using it.`,
+						"warning",
+					);
+					return;
+				}
+			}
+			openBrowser(url);
+			ctx.ui.notify(
+				`✓ Dashboard: ${url} (read-only, localhost only — stops when this session ends)`,
+				"info",
+			);
+		},
+	});
+
 	/* ── Lifecycle hooks (replace upstream's curl-based hooks) ───────────── */
 
 	pi.on("session_start", async (_event, ctx) => {
@@ -654,6 +755,11 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		stopWatcher();
 		watcherNotify = undefined;
 		automaticIndexBlock = null;
+		if (dashboard) {
+			const d = dashboard;
+			dashboard = null;
+			await d.stop().catch(() => undefined);
+		}
 		await engine?.shutdown().catch(() => undefined);
 		engine = null;
 	});

package/src/guidance.ts

@@ -100,6 +100,7 @@ If the first search misses:
 - If a search says the index is empty, a background index is already running - retry shortly, or run \`/context-index\` for a synchronous build
 - After large refactors, \`reindex\` (or \`/context-index\`) keeps results current; \`index_status\` reports freshness and measured token savings
 - Incremental re-index only re-reads changed files, so it is cheap
+- To wipe the index (stale or corrupted results), \`/context-index --reset\` deletes it safely mid-session - memory and sessions survive; the next search rebuilds
 `;
 
 /**

package/src/indexer/pipeline.ts

@@ -407,6 +407,22 @@ export async function indexCommits(
  */
 const pipelineLocks = new Map<string, Promise<unknown>>();
 
+/**
+ * Queue work behind every indexing run already registered for this storage
+ * dir, and keep later runs queued until the work finishes. Reset uses this
+ * instead of a one-shot idle wait: a reindex queued during that wait must not
+ * open the stores while deletion is in progress.
+ */
+export async function withPipelineLock<T>(storageBase: string, work: () => T | Promise<T>): Promise<T> {
+	const previous = pipelineLocks.get(storageBase) ?? Promise.resolve();
+	const current = previous.then(() => work());
+	pipelineLocks.set(
+		storageBase,
+		current.catch(() => undefined),
+	);
+	return current;
+}
+
 export async function runIndexing(
 	config: Config,
 	projectDirInput: string,
@@ -416,23 +432,18 @@ export async function runIndexing(
 		/** Automatic full-project runs only. Stop before migrations when the workspace exceeds this budget. */
 		maxAutoFiles?: number;
 		callbacks?: IndexCallbacks;
-		/** Reuse an already-constructed backend/embedder (the long-lived server path). */
-		runtime?: { backend: LocalBackend; embedder: Embedder };
+		/** Reuse an already-constructed backend/embedder (the long-lived server path);
+		 * a null backend falls through to a fresh LocalBackend (used by resetIndex). */
+		runtime?: { backend?: LocalBackend | null; embedder: Embedder };
 	} = {},
 ): Promise<IndexResult> {
 	const projectDir = fs.realpathSync(path.resolve(projectDirInput));
 	const storageBase = projectStorageDir(config.storagePath, projectDir);
 	fs.mkdirSync(storageBase, { recursive: true });
 
-	const previous = pipelineLocks.get(storageBase) ?? Promise.resolve();
-	const current = previous.then(() =>
+	return withPipelineLock(storageBase, () =>
 		runIndexingLocked(config, projectDir, storageBase, options, options.runtime),
 	);
-	pipelineLocks.set(
-		storageBase,
-		current.catch(() => undefined),
-	);
-	return current;
 }
 
 async function runIndexingLocked(
@@ -440,13 +451,16 @@ async function runIndexingLocked(
 	projectDir: string,
 	storageBase: string,
 	options: { full?: boolean; targetPath?: string | null; maxAutoFiles?: number; callbacks?: IndexCallbacks },
-	runtime?: { backend: LocalBackend; embedder: Embedder },
+	runtime?: { backend?: LocalBackend | null; embedder: Embedder },
 ): Promise<IndexResult> {
 	let { full = false, targetPath = null } = options;
 	const { callbacks = {} } = options;
 	const { log, progress, phase } = callbacks;
 
-	const ownsBackend = runtime === undefined;
+	// The caller only owns the backend it actually passed in; a null backend
+	// (resetIndex dropped its closed handle) means the pipeline built its own
+	// and must close it — otherwise its sqlite handles leak until exit.
+	const ownsBackend = !runtime?.backend;
 	const chunker = new Chunker();
 	const manifest = new Manifest(path.join(storageBase, "manifest.json"));
 	const result: IndexResult = {

package/src/memory/db.ts

@@ -571,6 +571,168 @@ CREATE VIRTUAL TABLE IF NOT EXISTS turn_summaries_vec USING vec0(embedding float
 		return out;
 	}
 
+	// ── Read-only views (dashboard) ───────────────────────────────────────
+	// These never write and never touch the savings ledger — they exist so the
+	// web dashboard can render history without going through the text renderers
+	// (which record progressive_disclosure savings as a side effect).
+
+	/** Recent sessions, newest first, with prompt/turn counts. */
+	listSessions(limit = 50): {
+		id: string;
+		project: string;
+		started_at: string;
+		ended_at: string | null;
+		status: string;
+		promptCount: number;
+		rollup: string | null;
+	}[] {
+		try {
+			const rows = this.db
+				.prepare(
+					`SELECT s.id, s.project, s.started_at, s.ended_at, s.status, s.rollup_summary,
+						(SELECT COUNT(*) FROM prompts p WHERE p.session_id = s.id) AS prompt_count
+					 FROM sessions s ORDER BY s.started_at_epoch DESC LIMIT ?`,
+				)
+				.all(Math.max(1, Math.min(limit, 500))) as {
+				id: string;
+				project: string;
+				started_at: string;
+				ended_at: string | null;
+				status: string;
+				rollup_summary: string | null;
+				prompt_count: number | null;
+			}[];
+			return rows.map((r) => ({
+				id: r.id,
+				project: r.project,
+				started_at: r.started_at,
+				ended_at: r.ended_at,
+				status: r.status,
+				promptCount: Number(r.prompt_count ?? 0),
+				rollup: r.rollup_summary,
+			}));
+		} catch {
+			return [];
+		}
+	}
+
+	/** Turn summaries + tool-event activity for one session, in turn order. */
+	sessionDetail(
+		sessionId: string,
+		turnLimit = 200,
+	): {
+		turns: { promptNumber: number; tier: string; summary: string }[];
+		events: { id: number; tool: string; promptNumber: number; createdAt: string; hasPayload: boolean }[];
+	} {
+		const empty = { turns: [], events: [] as { id: number; tool: string; promptNumber: number; createdAt: string; hasPayload: boolean }[] };
+		try {
+			const turns = this.db
+				.prepare(
+					"SELECT prompt_number, tier, summary FROM turn_summaries WHERE session_id = ? ORDER BY prompt_number ASC LIMIT ?",
+				)
+				.all(sessionId, Math.max(1, Math.min(turnLimit, 500))) as {
+				prompt_number: number;
+				tier: string;
+				summary: string;
+			}[];
+			const events = this.db
+				.prepare(
+					`SELECT te.id, te.tool_name, te.prompt_number, te.created_at, (te.payload_id IS NOT NULL) AS has_payload
+					 FROM tool_events te WHERE te.session_id = ? ORDER BY te.id ASC LIMIT ?`,
+				)
+				.all(sessionId, Math.max(1, Math.min(turnLimit * 20, 2000))) as {
+				id: number;
+				tool_name: string;
+				prompt_number: number;
+				created_at: string;
+				has_payload: number;
+			}[];
+			return {
+				turns: turns.map((t) => ({ promptNumber: t.prompt_number, tier: t.tier, summary: t.summary })),
+				events: events.map((e) => ({
+					id: e.id,
+					tool: e.tool_name,
+					promptNumber: e.prompt_number,
+					createdAt: e.created_at,
+					hasPayload: Number(e.has_payload) === 1,
+				})),
+			};
+		} catch {
+			return empty;
+		}
+	}
+
+	/** Recent decisions, newest first. */
+	listDecisions(limit = 100): { id: number; decision: string; reason: string; source: string; createdAt: string }[] {
+		try {
+			const rows = this.db
+				.prepare("SELECT id, decision, reason, source, created_at FROM decisions ORDER BY created_at_epoch DESC LIMIT ?")
+				.all(Math.max(1, Math.min(limit, 500))) as {
+				id: number;
+				decision: string;
+				reason: string;
+				source: string;
+				created_at: string;
+			}[];
+			return rows.map((r) => ({ id: r.id, decision: r.decision, reason: r.reason, source: r.source, createdAt: r.created_at }));
+		} catch {
+			return [];
+		}
+	}
+
+	/** Recent code areas, newest first. */
+	listCodeAreas(limit = 100): { id: number; filePath: string; description: string; source: string; createdAt: string }[] {
+		try {
+			const rows = this.db
+				.prepare("SELECT id, file_path, description, source, created_at FROM code_areas ORDER BY created_at_epoch DESC LIMIT ?")
+				.all(Math.max(1, Math.min(limit, 500))) as {
+				id: number;
+				file_path: string;
+				description: string;
+				source: string;
+				created_at: string;
+			}[];
+			return rows.map((r) => ({ id: r.id, filePath: r.file_path, description: r.description, source: r.source, createdAt: r.created_at }));
+		} catch {
+			return [];
+		}
+	}
+
+	/** Raw savings events, newest first. */
+	recentSavings(limit = 200): { bucket: string; baseline: number; served: number; ts: number }[] {
+		try {
+			const rows = this.db
+				.prepare("SELECT bucket, baseline, served, ts FROM savings_log ORDER BY ts DESC, id DESC LIMIT ?")
+				.all(Math.max(1, Math.min(limit, 2000))) as { bucket: string; baseline: number; served: number; ts: number }[];
+			return rows.map((r) => ({ bucket: r.bucket, baseline: r.baseline, served: r.served, ts: r.ts }));
+		} catch {
+			return [];
+		}
+	}
+
+	/** Savings per local calendar day (oldest → newest) for trend charts. */
+	savingsByDay(days = 30): { day: string; baseline: number; served: number }[] {
+		const out: { day: string; baseline: number; served: number }[] = [];
+		try {
+			const cutoff = epochSeconds() - Math.max(1, Math.min(days, 365)) * 86400;
+			const rows = this.db
+				.prepare("SELECT ts, baseline, served FROM savings_log WHERE ts >= ? ORDER BY ts ASC")
+				.all(cutoff) as { ts: number; baseline: number; served: number }[];
+			const byDay = new Map<string, { baseline: number; served: number }>();
+			for (const r of rows) {
+				const day = new Date(r.ts * 1000).toISOString().slice(0, 10);
+				const acc = byDay.get(day) ?? { baseline: 0, served: 0 };
+				acc.baseline += r.baseline;
+				acc.served += r.served;
+				byDay.set(day, acc);
+			}
+			for (const [day, v] of [...byDay.entries()].sort()) out.push({ day, ...v });
+		} catch {
+			// empty series
+		}
+		return out;
+	}
+
 	// ── Retention ─────────────────────────────────────────────────────────
 
 	pruneOldPayloads(days = 30): { payloadsPruned: number; bytesFreedEstimate: number } {