pi-context-engine 0.1.2 → 0.1.4

package/src/indexer/embedder.ts

@@ -40,13 +40,37 @@ interface TransformersModule {
 	env: Record<string, unknown>;
 }
 
+export type TransformersLoader = () => Promise<TransformersModule>;
+
+const defaultLoadTransformers: TransformersLoader = async () =>
+	(await import("@huggingface/transformers")) as unknown as TransformersModule;
+
+/**
+ * The package ships `@huggingface/transformers` as an optional dependency, so a
+ * plain npm install has it, but installs done with `--omit=optional` do not.
+ * Detect that case so we can tell the user exactly how to fix it instead of
+ * crashing with a bare "Cannot find module".
+ */
+function isModuleNotFoundError(err: unknown): boolean {
+	const code = (err as NodeJS.ErrnoException | undefined)?.code;
+	if (code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND") return true;
+	const message = (err as Error | undefined)?.message ?? "";
+	return message.includes("Cannot find module") || message.includes("Cannot find package");
+}
+
+const LOCAL_BACKEND_INSTALL_HINT =
+	"Local embedding backend unavailable: @huggingface/transformers is not installed. " +
+	"It ships as an optional dependency, so a plain install includes it. " +
+	"Fix: reinstall pi-context-engine (pi update npm:pi-context-engine), " +
+	"or force the Ollama backend with PCE_EMBED_BACKEND=ollama.";
+
 export class LocalBackend implements EmbeddingBackend {
 	readonly name = "local";
 	readonly modelName: string;
 	dimension = 0;
 	private pipelinePromise: Promise<(inputs: string | string[]) => Promise<number[][]>> | null = null;
 
-	constructor(modelName: string = DEFAULT_LOCAL_MODEL) {
+	constructor(modelName: string = DEFAULT_LOCAL_MODEL, private readonly loadTransformers: TransformersLoader = defaultLoadTransformers) {
 		this.modelName = modelName;
 	}
 
@@ -55,7 +79,16 @@ export class LocalBackend implements EmbeddingBackend {
 			this.pipelinePromise = (async () => {
 				const cacheDir = resolveModelCacheDir();
 				fs.mkdirSync(cacheDir, { recursive: true });
-				const transformers = (await import("@huggingface/transformers")) as unknown as TransformersModule;
+				let transformers: TransformersModule;
+				try {
+					transformers = await this.loadTransformers();
+				} catch (err) {
+					// Drop the memoized failure so a retry (after installing the
+					// package) gets a fresh chance instead of replaying the error.
+					this.pipelinePromise = null;
+					if (isModuleNotFoundError(err)) throw new Error(LOCAL_BACKEND_INSTALL_HINT);
+					throw err;
+				}
 				transformers.env.cacheDir = cacheDir;
 				const extractor = await transformers.pipeline("feature-extraction", this.modelName, {
 					pooling: "mean",

package/src/indexer/pipeline.ts

@@ -139,6 +139,40 @@ function resolveWithin(projectDir: string, target: string): string {
 	return resolved;
 }
 
+/** A directory target is excluded when it or any ancestor matches a dir rule (upstream #165b). */
+function cceignoreMatchesDir(relPosix: string, patterns: string[]): boolean {
+	const parts = relPosix.split("/");
+	for (let i = 1; i <= parts.length; i++) {
+		if (matchesAny(parts.slice(0, i).join("/"), true, patterns)) return true;
+	}
+	return false;
+}
+
+/**
+ * Target-path policy gate, mirroring what the full walker enforces while
+ * descending: every path segment must pass `indexer.ignore`, files must pass
+ * the secret-filename check, and `.cceignore` is matched against the
+ * project-relative path (dir rules for directories, file + ancestor rules for
+ * files). The project root (`rel === ""`) is exempt — rules apply to entries
+ * below the root, not to the root itself (a checkout named `build` must still
+ * reindex via `path: "."`).
+ */
+function targetExcluded(
+	relPosix: string,
+	opts: { isDir: boolean; ignoreSet: Set<string>; redactSecrets: boolean; patterns: string[] },
+): boolean {
+	if (relPosix === "") return false;
+	const segments = relPosix.split("/");
+	for (const seg of segments) {
+		if (opts.ignoreSet.has(seg)) return true;
+	}
+	// The walker only classifies secret names on file entries; a secret-named
+	// directory's contents are still gated per file.
+	if (!opts.isDir && opts.redactSecrets && isSecretFile(segments[segments.length - 1]!)) return true;
+	if (opts.patterns.length === 0) return false;
+	return opts.isDir ? cceignoreMatchesDir(relPosix, opts.patterns) : cceignoreMatchesFile(relPosix, opts.patterns);
+}
+
 function safeRead(filePath: string): string | null {
 	try {
 		if (fs.statSync(filePath).size > MAX_FILE_BYTES) return null;
@@ -149,7 +183,9 @@ function safeRead(filePath: string): string | null {
 }
 
 function* iterProjectFiles(
-	root: string,
+	walkRoot: string,
+	/** Project root: `.cceignore` patterns are matched against project-relative paths even when walking from a target. */
+	relRoot: string,
 	ignoreSet: Set<string>,
 	opts: { redactSecrets: boolean; cceignorePatterns: string[] },
 ): Generator<string> {
@@ -175,7 +211,7 @@ function* iterProjectFiles(
 			}
 			if (seen.has(resolved)) continue;
 			seen.add(resolved);
-			const rel = path.relative(root, full).replace(/\\/g, "/");
+			const rel = path.relative(relRoot, full).replace(/\\/g, "/");
 			if (opts.cceignorePatterns.length > 0 && matchesAny(rel, entry.isDirectory(), opts.cceignorePatterns)) {
 				continue;
 			}
@@ -188,7 +224,7 @@ function* iterProjectFiles(
 			}
 		}
 	};
-	yield* walk(root);
+	yield* walk(walkRoot);
 }
 
 /** `{embeddings, cacheHits, cacheMisses}` for a batch, using the content cache. */
@@ -385,32 +421,77 @@ async function runIndexingLocked(
 	if (cceignorePatterns.length > 0) log?.(`  [.cceignore] ${cceignorePatterns.length} pattern(s) loaded`);
 
 	// ── Determine the file set ─────────────────────────────────────────────
+	/**
+	 * A file the inclusion policy now rejects must not leave stale chunks
+	 * behind (upstream #127): purge its manifest entry and indexed rows so a
+	 * targeted reindex is enough to enforce a new secret/ignore rule.
+	 */
+	const purgeIndexed = (rels: string[]): void => {
+		const indexed = rels.filter((rel) => manifest.getHash(rel) !== undefined);
+		if (indexed.length === 0) return;
+		backend.deleteByFiles(indexed);
+		for (const rel of indexed) {
+			manifest.remove(rel);
+			result.deletedFiles.push(rel);
+			log?.(`  [delete] ${rel} (excluded by policy)`);
+		}
+		manifest.save();
+	};
 	let fileList: string[];
 	if (targetPath) {
 		const target = resolveWithin(projectDir, targetPath);
 		if (fs.existsSync(target) && fs.statSync(target).isFile()) {
 			const rel = path.relative(projectDir, target).replace(/\\/g, "/");
 			if (SKIP_EXTENSIONS.has(path.extname(target).toLowerCase())) {
+				purgeIndexed([rel]);
 				fileList = [];
-			} else if (ignoreSet.has(path.basename(target))) {
-				result.skippedFiles.push(rel);
-				fileList = [];
-			} else if (config.indexerRedactSecrets && isSecretFile(target)) {
-				result.skippedFiles.push(rel);
-				fileList = [];
-			} else if (cceignorePatterns.length > 0 && cceignoreMatchesFile(rel, cceignorePatterns)) {
+			} else if (
+				targetExcluded(rel, {
+					isDir: false,
+					ignoreSet,
+					redactSecrets: config.indexerRedactSecrets,
+					patterns: cceignorePatterns,
+				})
+			) {
+				// Ancestor segments included: the full walker would have pruned
+				// the path before reaching this file (upstream #165b parity).
 				result.skippedFiles.push(rel);
+				purgeIndexed([rel]);
 				fileList = [];
 			} else {
 				fileList = [target];
 			}
 		} else if (fs.existsSync(target) && fs.statSync(target).isDirectory()) {
-			fileList = [
-				...iterProjectFiles(target, ignoreSet, {
-					redactSecrets: config.indexerRedactSecrets,
-					cceignorePatterns,
-				}),
-			];
+			// Upstream #165b: the walk starts inside the target, so the target
+			// path itself must first pass the same policy the full walker
+			// enforces while descending — every segment vs `indexer.ignore`
+			// (e.g. `dist/nested` is rejected because `dist` is), plus
+			// `.cceignore` dir rules for the target and its ancestors.
+			const rel = path.relative(projectDir, target).replace(/\\/g, "/");
+			const excluded = targetExcluded(rel, {
+				isDir: true,
+				ignoreSet,
+				redactSecrets: config.indexerRedactSecrets,
+				patterns: cceignorePatterns,
+			});
+			if (excluded) {
+				result.skippedFiles.push(rel);
+				fileList = [];
+			} else {
+				fileList = [
+					...iterProjectFiles(target, projectDir, ignoreSet, {
+						redactSecrets: config.indexerRedactSecrets,
+						cceignorePatterns,
+					}),
+				];
+			}
+			// Targeted runs skip the full-run prune below, so purge manifest
+			// entries under this subtree the walk no longer yields: newly
+			// excluded files, secret files, and deleted files (upstream
+			// #127/#165b). Scoped to the target so siblings are untouched.
+			const yielded = new Set(fileList.map((f) => path.relative(projectDir, f).replace(/\\/g, "/")));
+			const underScope = (p: string): boolean => (rel === "" ? true : p === rel || p.startsWith(`${rel}/`));
+			purgeIndexed(Object.keys(manifest.fileEntries).filter((p) => underScope(p) && !yielded.has(p)));
 		} else {
 			// Watcher deletion path: prune instead of erroring out.
 			const rel = path.relative(projectDir, target).replace(/\\/g, "/");
@@ -426,7 +507,7 @@ async function runIndexingLocked(
 		}
 	} else {
 		fileList = [
-			...iterProjectFiles(projectDir, ignoreSet, {
+			...iterProjectFiles(projectDir, projectDir, ignoreSet, {
 				redactSecrets: config.indexerRedactSecrets,
 				cceignorePatterns,
 			}),

package/src/memory/db.ts

@@ -309,9 +309,31 @@ CREATE VIRTUAL TABLE IF NOT EXISTS turn_summaries_vec USING vec0(embedding float
 		this.writeVecRow("turn_summaries_vec", turnId, vec);
 	}
 
+	/**
+	 * Count rows still missing a vec entry, without touching an embedder.
+	 * Lets callers skip the backfill (and its model init) in the steady state.
+	 */
+	vecBackfillPending(): { decisions: number; turnSummaries: number } {
+		if (!this.hasVecTables()) return { decisions: 0, turnSummaries: 0 };
+		try {
+			const d = this.db
+				.prepare("SELECT COUNT(*) AS n FROM decisions d WHERE NOT EXISTS (SELECT 1 FROM decisions_vec v WHERE v.rowid = d.id)")
+				.get() as { n: number };
+			const t = this.db
+				.prepare("SELECT COUNT(*) AS n FROM turn_summaries t WHERE NOT EXISTS (SELECT 1 FROM turn_summaries_vec v WHERE v.rowid = t.id)")
+				.get() as { n: number };
+			return { decisions: Number(d?.n ?? 0), turnSummaries: Number(t?.n ?? 0) };
+		} catch {
+			return { decisions: 0, turnSummaries: 0 };
+		}
+	}
+
 	async backfillVecTables(embedder: Embedder): Promise<{ decisions: number; turnSummaries: number }> {
 		const counts = { decisions: 0, turnSummaries: 0 };
 		if (!this.hasVecTables()) return counts;
+		// Steady state: nothing missing — skip the embedder entirely.
+		const pending = this.vecBackfillPending();
+		if (pending.decisions + pending.turnSummaries === 0) return counts;
 		try {
 			this.ensureVecDim((await embedder.embedQuery("dimension probe")).length);
 		} catch {
@@ -348,6 +370,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS turn_summaries_vec USING vec0(embedding float
 		} catch {
 			return [];
 		}
+		await this.healVecTables(embedder, vec);
 		return this.vecSearch("decisions_vec", vec, k, maxDistance);
 	}
 
@@ -359,9 +382,32 @@ CREATE VIRTUAL TABLE IF NOT EXISTS turn_summaries_vec USING vec0(embedding float
 		} catch {
 			return [];
 		}
+		await this.healVecTables(embedder, vec);
 		return this.vecSearch("turn_summaries_vec", vec, k, maxDistance);
 	}
 
+	/**
+	 * Self-heal at the point of recall (upstream #137), covering both failure
+	 * shapes a dim-mismatched or partially-empty vec table produces:
+	 *
+	 * 1. Tables built by another embedder: a dim-mismatched MATCH errors and
+	 *    silently returns nothing → recreate at the query's dim.
+	 * 2. Rows missing for any reason (e.g. a mid-session `recordDecisionVec`
+	 *    wiped the tables via `ensureVecDim` after the engine's one-shot
+	 *    backfill already ran) → restore them here, where the embedder is
+	 *    already in hand.
+	 *
+	 * Steady state costs two COUNT queries; the embedder is only touched when
+	 * something is actually wrong.
+	 */
+	private async healVecTables(embedder: Embedder, vec: readonly number[]): Promise<void> {
+		if (!this.hasVecTables()) return;
+		const declared = this.declaredVecDim();
+		if (declared !== null && declared !== vec.length) this.ensureVecDim(vec.length);
+		const pending = this.vecBackfillPending();
+		if (pending.decisions + pending.turnSummaries > 0) await this.backfillVecTables(embedder);
+	}
+
 	private vecSearch(table: string, vec: number[], k: number, maxDistance: number): number[] {
 		try {
 			const rows = this.db