pi-context-engine 0.3.0 → 0.4.0

package/README.md

@@ -95,8 +95,13 @@ unchanged repo took 237 ms.
   it. You get a notification only when something changed.
 - **Lazy fallback**: if a search still lands on an empty index, indexing kicks
   off on the spot and the tool asks you to retry in a few seconds.
-- **One switch for both**: `indexer.auto_index: false` disables both behaviors;
-  build the index manually with `/context-index` instead.
+- **Live reindex while you work**: a recursive file watcher re-indexes on
+  edits, saves and branch switches (debounced, so a `git checkout` collapses
+  into one run). The agent's own edits are searchable right away instead of
+  after the next session.
+- **Switches**: `indexer.auto_index: false` disables the session-start and
+  lazy kicks; `indexer.watch: false` disables the file watcher. Use
+  `/context-index` for manual builds.
 - **Why it matters**: incremental runs are cheap (a few hundred ms on an
   unchanged repo), so this removes the classic cold-start failure of searching
   an empty index, without ever blocking the UI.
@@ -167,6 +172,7 @@ indexer:
   chunking: legacy        # legacy (default) | ast-density — cAST-style budgeted split-merge (experimental)
   chunk_budget_chars: 1200 # non-whitespace budget per ast-density chunk
   auto_index: true        # incremental background reindex shortly after session start
+  watch: true             # reindex on file changes while the session runs (debounced, incremental)
 memory:
   redact_pii: true
 pricing:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-context-engine",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Semantic code context for pi: tree-sitter index + hybrid vector/BM25 retrieval + cross-session memory, all local. Port of code-context-engine (MIT, elara-labs) as a native pi extension.",
   "keywords": [
     "pi-package",
@@ -46,7 +46,6 @@
   },
   "dependencies": {
     "better-sqlite3": "^13.0.3",
-    "chokidar": "^5.0.0",
     "sqlite-vec": "^0.1.9",
     "tree-sitter-wasms": "^0.1.13",
     "web-tree-sitter": "^0.22.6",

package/src/config.ts

@@ -95,6 +95,12 @@ export interface Config {
 	indexerContextHeaders: boolean;
 	/** Run an incremental reindex in the background shortly after session start. */
 	indexerAutoIndex: boolean;
+	/**
+	 * Watch the project tree while a session runs and reindex on changes
+	 * (debounced, incremental). Independent of `indexerAutoIndex`, which only
+	 * gates the session-start kick.
+	 */
+	indexerWatch: boolean;
 	indexerRedactSecrets: boolean;
 	memoryRedactPii: boolean;
 	pricingModel: string;
@@ -128,6 +134,7 @@ export function defaultConfig(): Config {
 		indexerIgnore: [...DEFAULT_IGNORE],
 		indexerContextHeaders: true,
 		indexerAutoIndex: true,
+		indexerWatch: true,
 		indexerRedactSecrets: true,
 		memoryRedactPii: true,
 		pricingModel: "opus",
@@ -162,6 +169,7 @@ const MAPPING: Record<string, keyof Config> = {
 	"indexer.ignore": "indexerIgnore",
 	"indexer.context_headers": "indexerContextHeaders",
 	"indexer.auto_index": "indexerAutoIndex",
+	"indexer.watch": "indexerWatch",
 	"indexer.redact_secrets": "indexerRedactSecrets",
 	"memory.redact_pii": "memoryRedactPii",
 	"storage.path": "storagePath",

package/src/extension.ts

@@ -12,6 +12,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { ContextEngine, isPceConfig } from "./engine.js";
 import { applyContextSearchGuidance } from "./guidance.js";
+import { ProjectWatcher } from "./indexer/watcher.js";
 
 /** Tools whose payloads are worth capturing for session timelines. */
 const CAPTURED_TOOLS = new Set(["bash", "read", "edit", "write", "grep", "find", "ls"]);
@@ -32,6 +33,100 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 	let autoIndexTimer: NodeJS.Timeout | null = null;
 	let indexInFlight = false;
 
+	/**
+	 * File watcher (config `indexer.watch`, default true): a debounced change
+	 * signal over the project tree. Unlike the session-start auto-index this
+	 * is re-kickable — one kick per settled change burst, for the whole
+	 * session. Owned by this closure, like `autoIndexTimer`: engine instances
+	 * are replaced per `session_start` without a shutdown event.
+	 */
+	let watcher: ProjectWatcher | null = null;
+	let watcherKickPending = false;
+	let watcherNotify: ((msg: string, level: "info" | "warning") => void) | undefined;
+
+	/**
+	 * Watcher-triggered incremental reindex. Unlike `kickBackgroundIndex` this
+	 * calls `reindex()` directly — the watcher must re-fire all session, not
+	 * consume a one-shot. Shares `indexInFlight` with the other kicks so runs
+	 * never stack; a change that arrives mid-run coalesces into exactly one
+	 * trailing kick, so edits made *while indexing* still land. Successful
+	 * runs stay silent (the user's own edit caused them); failures — both
+	 * rejections AND per-file errors resolved in `result.errors` — notify.
+	 */
+	const kickWatcherIndex = (eng: ContextEngine, notify?: (msg: string, level: "info" | "warning") => void) => {
+		if (engine !== eng) return; // session moved on
+		if (indexInFlight) {
+			watcherKickPending = true;
+			return;
+		}
+		indexInFlight = true;
+		void eng
+			.reindex({})
+			.then((result) => {
+				// Per-file chunking/read failures RESOLVE in result.errors — mirroring
+				// kickBackgroundIndex, surface them or affected files go stale silently.
+				if (engine !== eng || !notify) return; // session moved on
+				if (result.errors.length > 0) {
+					notify(`Context index: watcher reindex hit ${result.errors.length} error(s)`, "warning");
+				}
+			})
+			.catch((err: unknown) => {
+				if (engine === eng && notify) {
+					const detail = (err instanceof Error ? err.message : String(err)).slice(0, 120);
+					notify(`Context index: watcher reindex failed (${detail})`, "warning");
+				}
+			})
+			.finally(() => {
+				indexInFlight = false;
+				drainWatcherKick();
+			});
+	};
+
+	/** Deliver a watcher change that arrived while a background run was in flight. */
+	const drainWatcherKick = () => {
+		if (!watcherKickPending) return;
+		watcherKickPending = false;
+		if (engine && watcher) kickWatcherIndex(engine, watcherNotify);
+	};
+
+	const stopWatcher = () => {
+		watcher?.stop();
+		watcher = null;
+		watcherKickPending = false;
+	};
+
+	/**
+	 * (Re)point the file watcher at the live engine's project dir. Idempotent
+	 * per dir (called from every `getEngine`), so a mid-session project switch
+	 * re-arms the watcher exactly like the auto-index timer re-points its kick.
+	 */
+	const attachWatcher = (eng: ContextEngine) => {
+		if (watcher && watcher.dir === eng.projectDir) return;
+		stopWatcher();
+		if (!eng.config.indexerWatch) return;
+		// An in-project storage dir (storage.path inside the checkout) is written
+		// on EVERY reindex run (manifest.save()), so its events must not be
+		// accepted — otherwise each kick schedules the next one forever.
+		const storageRel = path.relative(eng.projectDir, eng.storageBase);
+		const storageInsideProject = storageRel !== "" && !storageRel.startsWith("..") && !path.isAbsolute(storageRel);
+		const w = new ProjectWatcher(eng.projectDir, {
+			// Same sets the walker enforces: any-depth + root-only semantics,
+			// user additions to indexer.ignore included, case-sensitive.
+			ignoreSegments: eng.config.indexerIgnore,
+			ignorePaths: storageInsideProject ? [eng.storageBase] : [],
+			onChange: () => {
+				if (engine) kickWatcherIndex(engine, watcherNotify);
+			},
+			onError: (err) => {
+				// E.g. Linux inotify exhaustion: degrade to session-start/lazy
+				// indexing and say so once. Never crash the session.
+				stopWatcher();
+				watcherNotify?.(`Context index: file watcher disabled (${err.message.slice(0, 100)}) — back to session-start indexing only`, "warning");
+			},
+		});
+		if (w.start()) watcher = w;
+	};
+
 	/**
 	 * Kick the one-shot background index (session-start auto-index and the
 	 * first-search lazy path share this). Never stacks: `lazyIndexStarted`
@@ -69,6 +164,7 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 			})
 			.finally(() => {
 				indexInFlight = false;
+				drainWatcherKick(); // a watcher change may have landed mid-run
 			});
 	};
 
@@ -82,6 +178,7 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		// Always track the live engine so `session_shutdown` closes it even when
 		// a tool ran before `session_start` (tests, mid-reload edge cases).
 		engine = new ContextEngine({ projectDir: dir });
+		attachWatcher(engine);
 		return engine;
 	};
 
@@ -372,6 +469,12 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		engine.startBackgroundWork();
 		lazyIndexStarted = false;
 		bootstrapSent = false;
+		watcherNotify = ctx.hasUI ? (msg, level) => ctx.ui.notify(msg, level) : undefined;
+		// (Re)arm the file watcher for this session: fresh notify channel, and
+		// config may have changed since the last session. No-op when
+		// `indexer.watch` is off.
+		stopWatcher();
+		attachWatcher(engine);
 		if (autoIndexTimer) {
 			clearTimeout(autoIndexTimer);
 			autoIndexTimer = null;
@@ -455,6 +558,8 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 			clearTimeout(autoIndexTimer);
 			autoIndexTimer = null;
 		}
+		stopWatcher();
+		watcherNotify = undefined;
 		await engine?.shutdown().catch(() => undefined);
 		engine = null;
 	});

package/src/indexer/pipeline.ts

@@ -366,7 +366,10 @@ export async function indexCommits(
 	return { chunks, nodes, edges, headSha };
 }
 
-/** Per-storage-dir indexing lock so watcher + manual reindex can't interleave. */
+/**
+ * Per-storage-dir indexing lock so concurrent reindex runs (auto-index kick,
+ * lazy search kick, file watcher, manual /context-index) can't interleave.
+ */
 const pipelineLocks = new Map<string, Promise<unknown>>();
 
 export async function runIndexing(
@@ -531,7 +534,8 @@ async function runIndexingLocked(
 			const underScope = (p: string): boolean => (rel === "" ? true : p === rel || p.startsWith(`${rel}/`));
 			purgeIndexed(Object.keys(manifest.fileEntries).filter((p) => underScope(p) && !yielded.has(p)));
 		} else {
-			// Watcher deletion path: prune instead of erroring out.
+			// Target vanished between runs (deleted file, or a branch switch removed it):
+// prune instead of erroring out.
 			const rel = path.relative(projectDir, target).replace(/\\/g, "/");
 			if (manifest.getHash(rel) !== undefined) {
 				backend.deleteByFiles([rel]);

package/src/indexer/watcher.ts

@@ -0,0 +1,195 @@
+/**
+ * Real-time reindex trigger — issue #12. A thin debounced wrapper around
+ * Node's built-in `fs.watch(dir, { recursive: true })` (one OS handle on
+ * Windows/macOS, inotify tree on Linux since Node 19.1).
+ *
+ * The watcher only produces a *change signal*: the incremental pipeline
+ * re-walks and re-hashes anyway, so false positives cost one cheap walk and
+ * only missed events matter. That is why the built-in suffices and neither
+ * chokidar (per-file stat-confirmed events, Node >= 22.22) nor
+ * @parcel/watcher (native addon, snapshot diffs) are needed.
+ *
+ * Filter semantics deliberately MIRROR the pipeline's inclusion policy
+ * (`targetExcluded` / `iterProjectFiles` in pipeline.ts), because a dropped
+ * event is the only unrecoverable failure mode: a wrongly accepted event just
+ * costs one cheap incremental run.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { DEFAULT_IGNORE, ROOT_ONLY_IGNORE } from "../config.js";
+
+/**
+ * Trailing-edge debounce for watcher events: long enough that a burst
+ * (editor save, `git checkout` rewriting many files) collapses into one
+ * kick, short enough that retrieval catches up quickly after an edit.
+ */
+export const WATCH_DEBOUNCE_MS = 1500;
+
+const ROOT_ONLY_SET = new Set(ROOT_ONLY_IGNORE);
+
+export interface ProjectWatcherOptions {
+	/** Called once per debounced change burst. Keep it cheap and sync. */
+	onChange: () => void;
+	/** Fatal watcher problem (e.g. Linux ENOSPC inotify exhaustion). The watcher closes itself; the caller falls back to session-start/lazy indexing. */
+	onError?: (err: Error) => void;
+	/** Defaults to {@link WATCH_DEBOUNCE_MS}. */
+	debounceMs?: number;
+	/**
+	 * Segment names ignored at ANY depth — defaults to `DEFAULT_IGNORE`; the
+	 * extension passes `config.indexerIgnore` so user additions apply too
+	 * (same set the walker checks via `ignoreSet.has(seg)`).
+	 */
+	ignoreSegments?: readonly string[];
+	/**
+	 * Absolute directories whose events are dropped entirely. The extension
+	 * passes the engine's storage dir when it points INSIDE the checkout:
+	 * every reindex rewrites `manifest.json` there unconditionally, so
+	 * without this exclusion a watcher kick would re-trigger itself forever
+	 * after source edits stop (PR #15 review, #12).
+	 */
+	ignorePaths?: readonly string[];
+}
+
+export class ProjectWatcher {
+	private watcher: fs.FSWatcher | null = null;
+	private debounceTimer: NodeJS.Timeout | null = null;
+	private stopped = false;
+	private watchRoot: string | null = null;
+	private readonly ignoreSet: Set<string>;
+	private ignorePaths: string[] = [];
+
+	constructor(
+		readonly dir: string,
+		private readonly options: ProjectWatcherOptions,
+	) {
+		this.ignoreSet = new Set(options.ignoreSegments ?? DEFAULT_IGNORE);
+	}
+
+	/** True while an OS watch handle is held. */
+	get watching(): boolean {
+		return this.watcher !== null;
+	}
+
+	/** Start watching. Returns false (after reporting via `onError`) when the watch could not be established — never throws. */
+	start(): boolean {
+		if (this.watcher) return true;
+		let watcher: fs.FSWatcher;
+		try {
+			// Canonicalize 8.3 short names first: on Windows,
+			// fs.watch({recursive}) hits an uncatchable libuv assertion
+			// (fs-event.c: `filename` vs watched dir prefix mismatch) that ABORTS
+			// the whole host process when the watched path has short-name
+			// components (common for os.tmpdir() and some tool-set cwds) and an
+			// event then reports the long name. realpathSync.native returns the
+			// canonical long path and avoids the mismatch entirely.
+			const dir = fs.realpathSync.native(this.dir);
+			this.watchRoot = dir;
+			this.ignorePaths = (this.options.ignorePaths ?? []).map((p) => {
+				try {
+					return fs.realpathSync.native(p);
+				} catch {
+					return path.resolve(p);
+				}
+			});
+			watcher = fs.watch(dir, { recursive: true }, (_event, filename) => this.onEvent(filename));
+		} catch (err) {
+			this.options.onError?.(err as Error);
+			return false;
+		}
+		this.watcher = watcher;
+		watcher.on("error", (err) => {
+			// E.g. inotify watch exhaustion on huge Linux trees. Close and let
+			// the caller degrade to session-start/lazy indexing — a watcher
+			// problem must never crash or stall the session.
+			this.closeWatcher();
+			if (!this.stopped) this.options.onError?.(err);
+		});
+		// Never keep the host process alive just for indexing.
+		watcher.unref?.();
+		return true;
+	}
+
+	private onEvent(filename: string | Buffer | null): void {
+		if (this.stopped || !this.watcher) return;
+		// No filename (happens, e.g. Linux inotify): cannot filter — a missed
+		// event is the unrecoverable failure mode, so accept and debounce it.
+		if (filename === null) {
+			this.scheduleKick();
+			return;
+		}
+		const name = Buffer.isBuffer(filename) ? filename.toString("utf-8") : filename;
+		if (name !== "" && this.isNoise(name)) return;
+		this.scheduleKick();
+	}
+
+	/**
+	 * Mirrors the walker's inclusion policy (see header): case-sensitive
+	 * segment match at any depth, root-only names only for the FIRST segment,
+	 * and explicit storage-dir path exclusion. Returns "noise" = drop event.
+	 */
+	private isNoise(filename: string): boolean {
+		const rel = this.relativeToRoot(filename);
+		if (rel === null) return false; // absolute path outside the watch root: be conservative, kick
+		const segments = rel.split(/[\\/]/).filter((s) => s !== "");
+		if (segments.length === 0) return false;
+		// Root-only names (`storage`, `logs`, `tmp`, ...) count only at the
+		// project root — nested `src/storage/` is real source the pipeline
+		// indexes, so its events must kick (PR #15 review).
+		if (ROOT_ONLY_SET.has(segments[0]!)) return true;
+		// Case-SENSITIVE like `ignoreSet.has(seg)`: the pipeline indexes
+		// `src/Build/`, so its events must not be dropped (PR #15 review).
+		if (segments.some((s) => this.ignoreSet.has(s))) return true;
+		return this.isUnderIgnoredPath(rel);
+	}
+
+	/** Relative path from the watch root, or null when outside/unknown. */
+	private relativeToRoot(filename: string): string | null {
+		if (!path.isAbsolute(filename)) return filename;
+		const rel = path.relative(this.watchRoot ?? this.dir, filename);
+		if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) return null;
+		return rel;
+	}
+
+	/** True when the event's resolved path lives under an ignored dir (e.g. an in-project storage dir). */
+	private isUnderIgnoredPath(rel: string): boolean {
+		if (this.ignorePaths.length === 0 || this.watchRoot === null) return false;
+		const abs = path.resolve(this.watchRoot, rel);
+		const fold = (p: string) => (process.platform === "win32" ? p.toLowerCase() : p);
+		const absFolded = fold(abs);
+		return this.ignorePaths.some((p) => {
+			const ignored = fold(p);
+			return absFolded === ignored || absFolded.startsWith(ignored + path.sep);
+		});
+	}
+
+	private scheduleKick(): void {
+		if (this.debounceTimer) clearTimeout(this.debounceTimer);
+		this.debounceTimer = setTimeout(() => {
+			this.debounceTimer = null;
+			// `stopped` mid-debounce, or the watcher died (error path) while
+			// the timer was pending: no kick.
+			if (this.stopped || !this.watcher) return;
+			this.options.onChange();
+		}, this.options.debounceMs ?? WATCH_DEBOUNCE_MS);
+		this.debounceTimer.unref?.();
+	}
+
+	/** Stop watching and cancel any pending debounced kick. Idempotent. */
+	stop(): void {
+		this.stopped = true;
+		if (this.debounceTimer) {
+			clearTimeout(this.debounceTimer);
+			this.debounceTimer = null;
+		}
+		this.closeWatcher();
+	}
+
+	private closeWatcher(): void {
+		try {
+			this.watcher?.close();
+		} catch {
+			// already closed by the runtime
+		}
+		this.watcher = null;
+	}
+}