pi-context-engine 0.5.2 → 0.6.0

package/README.md

@@ -56,7 +56,7 @@ Ten tools the agent can call, plus four slash commands.
 - **`index_status`**: chunk count and measured savings.
 - **`reindex`**: refresh a path or the whole project.
 
-Commands: `/context-index [--full]`, `/context-status`, `/context-savings`,
+Commands: `/context-index [--full] [--reset]`, `/context-status`, `/context-savings`,
 `/context-warmup`.
 
 **Shipped guidance.** The package also injects a ~900-token guidance section
@@ -109,13 +109,24 @@ unchanged repo took 237 ms.
   not add a full directory scan. Periodic yields keep the check responsive.
   This prevents an accidental parent-directory
   session from starting an unrequested multi-minute workload. `/context-index`
-  remains available as an explicit manual build.
+  remains available as an explicit manual build. While the pause is active,
+  every `context_search` result carries a stale-note and `/context-status`
+  leads with the pause line, so outdated results are never served silently;
+  all messages name the config file that raises the limit.
 - **Resume after interruption**: the manifest checkpoints completed batches,
   starting with the first batch and then at most once every five seconds. An
   interrupted build can skip completed files instead of starting from zero.
   Migration runs (schema reset, embedding-dimension change) are exempt: their
   manifest stays untouched on disk until the run completes, so an interrupted
   migration resumes cleanly instead of stranding stale rows.
+- **One-command reset**: `/context-index --reset` deletes the index through the
+  engine while the session is running: sqlite handles are closed first, so it
+  succeeds where manual deletion of the storage directory fails with locked
+  file errors (EPERM on Windows). The report shows how many files and bytes
+  were freed, and memory, sessions, and the savings ledger survive. The
+  embedding cache is deleted too, so a rebuild re-embeds from scratch. The
+  next search rebuilds via the lazy kick, guarded by the same file budget as
+  every automatic run.
 - **Cold-start visibility**: the first build on an empty index announces its
   start (`building the first index…`) and always closes with a result line
   (`updated: N file(s)` or `no files to index`). This is the one background

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-context-engine",
-  "version": "0.5.2",
+  "version": "0.6.0",
   "description": "Semantic code context for pi: tree-sitter index + hybrid vector/BM25 retrieval + cross-session memory, all local. Port of code-context-engine (MIT, elara-labs) as a native pi extension.",
   "keywords": [
     "pi-package",

package/src/engine.ts

@@ -20,7 +20,7 @@ import { SessionCapture } from "./memory/session-capture.js";
 import { drainOne, enqueueSessionRollup, enqueueTurn, type CompressionDeps } from "./memory/compressor.js";
 import { compress, expand, compressWithCounts, DEFAULT_LEVEL } from "./memory/grammar.js";
 import { cosine } from "./memory/extractive.js";
-import { AutomaticIndexFileLimitError, runIndexing } from "./indexer/pipeline.js";
+import { AutomaticIndexFileLimitError, runIndexing, withPipelineLock } from "./indexer/pipeline.js";
 import { getModelPricing } from "./pricing.js";
 import type { Chunk } from "./types.js";
 
@@ -125,8 +125,10 @@ export interface SessionStats {
 export interface EngineOptions {
 	projectDir: string;
 	config?: Config;
-	/** Test seam: reuse an existing backend/embedder/reranker instead of auto-detecting. */
-	runtime?: { backend: LocalBackend; embedder: Embedder; reranker?: LocalReranker };
+	/** Test seam: reuse an existing backend/embedder/reranker instead of
+	 * auto-detecting. `backend` may be nulled by resetIndex — the pipeline
+	 * then constructs a fresh one. */
+	runtime?: { backend?: LocalBackend | null; embedder: Embedder; reranker?: LocalReranker };
 }
 
 /** Issue #11 (upstream warmup): what /context-warmup resolved + cached. */
@@ -161,8 +163,8 @@ export class ContextEngine {
 	readonly projectName: string;
 	readonly storageBase: string;
 	config: Config;
-	private readonly injectedRuntime:
-		| { backend: LocalBackend; embedder: Embedder; reranker?: LocalReranker }
+	private injectedRuntime:
+		| { backend?: LocalBackend | null; embedder: Embedder; reranker?: LocalReranker }
 		| undefined;
 	private backendInstance: LocalBackend | null = null;
 	private embedderInstance: Embedder | null = null;
@@ -188,7 +190,7 @@ export class ContextEngine {
 		this.config = options.config ?? loadConfigForProject(this.projectDir);
 		this.injectedRuntime = options.runtime;
 		if (options.runtime) {
-			this.backendInstance = options.runtime.backend;
+			this.backendInstance = options.runtime.backend ?? null;
 			this.embedderInstance = options.runtime.embedder;
 		}
 		this.storageBase = projectStorageDir(this.config.storagePath, this.projectDir);
@@ -358,6 +360,93 @@ export class ContextEngine {
 		return this.backend.countChunks();
 	}
 
+	/**
+	 * Delete this project's index (manifest, vector/FTS/graph stores, embedding
+	 * cache) while keeping memory, sessions and the savings ledger — those are
+	 * user history, not index state. Handles are closed BEFORE deletion:
+	 * Windows refuses to delete files with open sqlite handles, which is why
+	 * manually deleting the storage dir fails while a session is running.
+	 * Queues behind in-flight and queued pipeline runs, and holds the same lock
+	 * during deletion so a later reindex cannot race the reset.
+	 * Errors are reported per target (e.g. another session holding a handle)
+	 * instead of failing the whole reset.
+	 */
+	async resetIndex(): Promise<{ deletedFiles: number; freedBytes: number; errors: string[] }> {
+		return withPipelineLock(this.storageBase, () => this.resetIndexLocked());
+	}
+
+	private async resetIndexLocked(): Promise<{ deletedFiles: number; freedBytes: number; errors: string[] }> {
+		try {
+			this.backendInstance?.close();
+		} catch {
+			// already closed
+		}
+		this.backendInstance = null;
+		this.retrieverInstance = null;
+		// Mutate the runtime object rather than replacing it: reindex calls that
+		// were queued behind this reset captured the same object and must see the
+		// closed backend dropped before they execute. Embedder and reranker
+		// survive: they hold no files.
+		if (this.injectedRuntime) this.injectedRuntime.backend = null;
+		// A reset index may be rebuilt by the next automatic kick.
+		this.autoIndexDone = false;
+		let deletedFiles = 0;
+		let freedBytes = 0;
+		const errors: string[] = [];
+		const countAndDelete = (target: string): void => {
+			let stat: fs.Stats;
+			try {
+				stat = fs.statSync(target);
+			} catch {
+				return; // not there — nothing to free
+			}
+			let files = 0;
+			let bytes = 0;
+			const measure = (p: string, s: fs.Stats): void => {
+				if (s.isDirectory()) {
+					for (const entry of fs.readdirSync(p)) {
+						const child = path.join(p, entry);
+						try {
+							measure(child, fs.statSync(child));
+						} catch {
+							// raced away between readdir and stat
+						}
+					}
+					return;
+				}
+				files++;
+				bytes += s.size;
+			};
+			try {
+				measure(target, stat);
+				fs.rmSync(target, { recursive: true, force: true });
+				// Only report bytes actually released; a locked target must not
+				// inflate the success report when rmSync fails.
+				deletedFiles += files;
+				freedBytes += bytes;
+			} catch (err) {
+				// Locked file (another session?) — report instead of failing the reset.
+				errors.push(`${path.relative(this.storageBase, target) || path.basename(target)}: ${(err as Error).message.slice(0, 80)}`);
+			}
+		};
+		const graphDb = path.join(this.storageBase, "graph.db");
+		const embeddingCache = path.join(this.storageBase, "embedding_cache.db");
+		for (const target of [
+			path.join(this.storageBase, "manifest.json"),
+			path.join(this.storageBase, "vectors"),
+			path.join(this.storageBase, "fts"),
+			graphDb,
+			`${graphDb}-shm`,
+			`${graphDb}-wal`,
+			embeddingCache,
+			`${embeddingCache}-shm`,
+			`${embeddingCache}-wal`,
+		]) {
+			countAndDelete(target);
+		}
+		return { deletedFiles, freedBytes, errors };
+	}
+
 	async reindex(options: { full?: boolean; targetPath?: string | null; automatic?: boolean; onProgress?: (msg: string) => void } = {}): Promise<{
 		indexed: number;
 		chunks: number;

package/src/extension.ts

@@ -26,6 +26,26 @@ const CAPTURED_TOOLS = new Set(["bash", "read", "edit", "write", "grep", "find",
 const AUTO_INDEX_DELAY_MS = 2000;
 
 export default function contextEngineExtension(pi: ExtensionAPI) {
+	const MAX_AUTO_FILES_HINT =
+		"raise indexer.max_auto_files (~/.pi-context-engine/config.yaml or the project's .context-engine.yaml)";
+
+	/** One-line pause notice for /context-status and the index_status tool: the
+	 * most likely question there is "why is my index not updating?" */
+	const pauseNotice = (): string | null =>
+		automaticIndexBlock
+			? `Automatic indexing paused: workspace exceeds ${automaticIndexBlock.limit.toLocaleString()} indexable files — the index may be stale. ` +
+				`Run /context-index, or ${MAX_AUTO_FILES_HINT}.`
+			: null;
+
+	/** Bracket note appended to every context_search result while the pause is
+	 * active and a (possibly outdated) index is served. Without it the agent
+	 * cannot tell fresh results from stale ones. */
+	const staleIndexNote = (): string | null =>
+		automaticIndexBlock
+			? `[note: automatic indexing is paused — workspace exceeds ${automaticIndexBlock.limit.toLocaleString()} indexable files; these results may be stale. ` +
+				`Run /context-index, or ${MAX_AUTO_FILES_HINT} to refresh]`
+			: null;
+
 	let engine: ContextEngine | null = null;
 	let currentCwd = process.cwd();
 	let promptNumber = 0;
@@ -114,7 +134,7 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		notify?.(
 			`Context index: automatic indexing${eng.config.indexerWatch ? " and watcher" : ""} paused. ` +
 				`Workspace exceeds ${err.limit.toLocaleString()} indexable files. ` +
-				"Run /context-index to index anyway, or raise indexer.max_auto_files.",
+				`Run /context-index to index anyway, or ${MAX_AUTO_FILES_HINT}.`,
 			"warning",
 		);
 	};
@@ -225,6 +245,20 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		return engine;
 	};
 
+	/** Human-readable byte count for the reset report (explicit user action —
+	 * the automatic guard never measures bytes by design). */
+	const formatBytes = (n: number): string => {
+		if (n < 1024) return `${n} B`;
+		const units = ["KiB", "MiB", "GiB"];
+		let v = n / 1024;
+		let u = 0;
+		while (v >= 1024 && u < units.length - 1) {
+			v /= 1024;
+			u++;
+		}
+		return `${v.toFixed(v >= 10 ? 0 : 1)} ${units[u]}`;
+	};
+
 	const text = (body: string, details?: unknown) => ({
 		content: [{ type: "text" as const, text: body }],
 		details: details ?? {},
@@ -267,7 +301,7 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 					return text(
 						`Automatic indexing is paused for ${eng.projectName}: the workspace exceeds ` +
 							`${automaticIndexBlock.limit.toLocaleString()} indexable files. ` +
-							"Run /context-index to index manually, or raise indexer.max_auto_files.",
+							`Run /context-index to index manually, or ${MAX_AUTO_FILES_HINT}.`,
 					);
 				}
 				if (indexInFlight) {
@@ -293,6 +327,8 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 			let body = result.body;
 			const nudge = eng.buildNudge();
 			if (nudge) body += nudge;
+			const stale = staleIndexNote();
+			if (stale) body += `\n${stale}`;
 			if (result.droppedLowValue > 0) {
 				body +=
 					"\n[note: lower-confidence results omitted — lower retrieval.marginal_ratio or retrieval.confidence_threshold to include them]";
@@ -431,7 +467,9 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		description: "Check index health, chunk count, and token savings for this project.",
 		parameters: Type.Object({}),
 		async execute(_toolCallId, _params, _signal, _onUpdate, ctx) {
-			return text(await getEngine(ctx.cwd).statusText());
+			const status = await getEngine(ctx.cwd).statusText();
+			const pause = pauseNotice();
+			return text(pause ? `${pause}\n${status}` : status);
 		},
 	});
 
@@ -471,9 +509,46 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 	/* ── Commands ────────────────────────────────────────────────────────── */
 
 	pi.registerCommand("context-index", {
-		description: "Build or refresh the semantic code index: /context-index [--full]",
+		description: "Build or refresh the semantic code index: /context-index [--full] [--reset]",
 		handler: async (args, ctx) => {
 			const eng = getEngine(ctx.cwd);
+			// --reset: delete the index through the engine (handles closed first, so
+			// it works while THIS session runs — manual deletion hits locked-file
+			// errors). Memory, sessions and savings survive. No auto-rebuild: the
+			// next search or /context-index decides, so an oversized workspace is
+			// not ambushed with a multi-minute build it just escaped.
+			if (args.includes("--reset")) {
+				if (indexInFlight) {
+					ctx.ui.notify(
+						"Context index: indexing is in progress — run /context-index --reset again once it finishes.",
+						"warning",
+					);
+					return;
+				}
+				// Reset also cancels the delayed startup kick. Otherwise its timer
+				// would rebuild the index immediately after the explicit delete.
+				if (autoIndexTimer) {
+					clearTimeout(autoIndexTimer);
+					autoIndexTimer = null;
+				}
+				const report = await eng.resetIndex();
+				// Free the lazy kick so the next search can rebuild (guarded by the
+				// same file budget as every automatic run).
+				lazyIndexStarted = false;
+				if (report.errors.length > 0) {
+					ctx.ui.notify(
+						`⚠ Index reset incomplete: ${report.errors.length} target(s) locked (${report.errors[0]}). Close other sessions on this project and run /context-index --reset again.`,
+						"warning",
+					);
+				} else {
+					ctx.ui.notify(
+						`✓ Index deleted: ${report.deletedFiles} file(s), ${formatBytes(report.freedBytes)} freed — memory, sessions and savings kept. ` +
+							"Run /context-index to build a fresh index.",
+						"info",
+					);
+				}
+				return;
+			}
 			const full = args.includes("--full");
 			ctx.ui.notify(full ? "Re-indexing everything…" : "Indexing (incremental)…", "info");
 			const result = await eng.reindex({ full });
@@ -488,7 +563,9 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 	pi.registerCommand("context-status", {
 		description: "Show index health and token savings for this project",
 		handler: async (_args, ctx) => {
-			ctx.ui.notify(await getEngine(ctx.cwd).statusText(), "info");
+			const status = await getEngine(ctx.cwd).statusText();
+			const pause = pauseNotice();
+			ctx.ui.notify(pause ? `${pause}\n${status}` : status, "info");
 		},
 	});
 

package/src/guidance.ts

@@ -100,6 +100,7 @@ If the first search misses:
 - If a search says the index is empty, a background index is already running - retry shortly, or run \`/context-index\` for a synchronous build
 - After large refactors, \`reindex\` (or \`/context-index\`) keeps results current; \`index_status\` reports freshness and measured token savings
 - Incremental re-index only re-reads changed files, so it is cheap
+- To wipe the index (stale or corrupted results), \`/context-index --reset\` deletes it safely mid-session - memory and sessions survive; the next search rebuilds
 `;
 
 /**

package/src/indexer/pipeline.ts

@@ -407,6 +407,22 @@ export async function indexCommits(
  */
 const pipelineLocks = new Map<string, Promise<unknown>>();
 
+/**
+ * Queue work behind every indexing run already registered for this storage
+ * dir, and keep later runs queued until the work finishes. Reset uses this
+ * instead of a one-shot idle wait: a reindex queued during that wait must not
+ * open the stores while deletion is in progress.
+ */
+export async function withPipelineLock<T>(storageBase: string, work: () => T | Promise<T>): Promise<T> {
+	const previous = pipelineLocks.get(storageBase) ?? Promise.resolve();
+	const current = previous.then(() => work());
+	pipelineLocks.set(
+		storageBase,
+		current.catch(() => undefined),
+	);
+	return current;
+}
+
 export async function runIndexing(
 	config: Config,
 	projectDirInput: string,
@@ -416,23 +432,18 @@ export async function runIndexing(
 		/** Automatic full-project runs only. Stop before migrations when the workspace exceeds this budget. */
 		maxAutoFiles?: number;
 		callbacks?: IndexCallbacks;
-		/** Reuse an already-constructed backend/embedder (the long-lived server path). */
-		runtime?: { backend: LocalBackend; embedder: Embedder };
+		/** Reuse an already-constructed backend/embedder (the long-lived server path);
+		 * a null backend falls through to a fresh LocalBackend (used by resetIndex). */
+		runtime?: { backend?: LocalBackend | null; embedder: Embedder };
 	} = {},
 ): Promise<IndexResult> {
 	const projectDir = fs.realpathSync(path.resolve(projectDirInput));
 	const storageBase = projectStorageDir(config.storagePath, projectDir);
 	fs.mkdirSync(storageBase, { recursive: true });
 
-	const previous = pipelineLocks.get(storageBase) ?? Promise.resolve();
-	const current = previous.then(() =>
+	return withPipelineLock(storageBase, () =>
 		runIndexingLocked(config, projectDir, storageBase, options, options.runtime),
 	);
-	pipelineLocks.set(
-		storageBase,
-		current.catch(() => undefined),
-	);
-	return current;
 }
 
 async function runIndexingLocked(
@@ -440,13 +451,16 @@ async function runIndexingLocked(
 	projectDir: string,
 	storageBase: string,
 	options: { full?: boolean; targetPath?: string | null; maxAutoFiles?: number; callbacks?: IndexCallbacks },
-	runtime?: { backend: LocalBackend; embedder: Embedder },
+	runtime?: { backend?: LocalBackend | null; embedder: Embedder },
 ): Promise<IndexResult> {
 	let { full = false, targetPath = null } = options;
 	const { callbacks = {} } = options;
 	const { log, progress, phase } = callbacks;
 
-	const ownsBackend = runtime === undefined;
+	// The caller only owns the backend it actually passed in; a null backend
+	// (resetIndex dropped its closed handle) means the pipeline built its own
+	// and must close it — otherwise its sqlite handles leak until exit.
+	const ownsBackend = !runtime?.backend;
 	const chunker = new Chunker();
 	const manifest = new Manifest(path.join(storageBase, "manifest.json"));
 	const result: IndexResult = {