pi-context-engine 0.1.2 → 0.1.3

package/README.md

@@ -116,7 +116,6 @@ embedding:
   model: local           # or a specific transformers.js model id
   ollama_model: nomic-embed-text
 indexer:
-  watch: true
   redact_secrets: true
 memory:
   redact_pii: true

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pi-context-engine",
-	"version": "0.1.2",
+	"version": "0.1.3",
 	"description": "Semantic code context for pi: tree-sitter index + hybrid vector/BM25 retrieval + cross-session memory, all local. Port of code-context-engine (MIT, elara-labs) as a native pi extension.",
 	"keywords": [
 		"pi-package",

package/src/config.ts

@@ -60,13 +60,9 @@ export interface Config {
 	retrievalConfidenceThreshold: number;
 	retrievalTopK: number;
 	retrievalMarginalRatio: number;
-	bootstrapMaxTokens: number;
-	indexerWatch: boolean;
-	indexerDebounceMs: number;
 	indexerIgnore: string[];
 	indexerRedactSecrets: boolean;
 	memoryRedactPii: boolean;
-	auditLogEnabled: boolean;
 	pricingModel: string;
 	pricingInput: number | null;
 	pricingOutput: number | null;
@@ -84,13 +80,9 @@ export function defaultConfig(): Config {
 		retrievalConfidenceThreshold: 0.2,
 		retrievalTopK: 20,
 		retrievalMarginalRatio: 0.75,
-		bootstrapMaxTokens: 10000,
-		indexerWatch: true,
-		indexerDebounceMs: 500,
 		indexerIgnore: [...DEFAULT_IGNORE],
 		indexerRedactSecrets: true,
 		memoryRedactPii: true,
-		auditLogEnabled: false,
 		pricingModel: "opus",
 		pricingInput: null,
 		pricingOutput: null,
@@ -109,13 +101,9 @@ const MAPPING: Record<string, keyof Config> = {
 	"retrieval.confidence_threshold": "retrievalConfidenceThreshold",
 	"retrieval.top_k": "retrievalTopK",
 	"retrieval.marginal_ratio": "retrievalMarginalRatio",
-	"retrieval.bootstrap_max_tokens": "bootstrapMaxTokens",
-	"indexer.watch": "indexerWatch",
-	"indexer.debounce_ms": "indexerDebounceMs",
 	"indexer.ignore": "indexerIgnore",
 	"indexer.redact_secrets": "indexerRedactSecrets",
 	"memory.redact_pii": "memoryRedactPii",
-	"audit.enabled": "auditLogEnabled",
 	"storage.path": "storagePath",
 	"pricing.model": "pricingModel",
 	"pricing.input": "pricingInput",

package/src/engine.ts

@@ -7,7 +7,7 @@ import fs from "node:fs";
 import path from "node:path";
 import type { Config } from "./config.js";
 import { loadConfigForProject, resolveOllamaUrl } from "./config.js";
-import { epochSeconds, humaniseRelativeTime, projectStorageDir, sha256Hex } from "./utils.js";
+import { atomicWriteText, epochSeconds, humaniseRelativeTime, projectStorageDir, sha256Hex } from "./utils.js";
 import { LocalBackend } from "./storage/local-backend.js";
 import { Embedder, selectBackend } from "./indexer/embedder.js";
 import { HybridRetriever } from "./retrieval/retriever.js";
@@ -146,6 +146,7 @@ export class ContextEngine {
 	draining = false;
 	private searchesSinceDecision = 0;
 	private hasRecordedDecision = false;
+	private vecBackfillStarted = false;
 
 	constructor(options: EngineOptions) {
 		this.projectDir = fs.realpathSync(path.resolve(options.projectDir));
@@ -247,7 +248,7 @@ export class ContextEngine {
 
 	private saveStats(): void {
 		try {
-			fs.writeFileSync(this.statsPath, JSON.stringify(this.stats));
+			atomicWriteText(this.statsPath, JSON.stringify(this.stats));
 		} catch {
 			// stats are best-effort
 		}
@@ -426,6 +427,25 @@ export class ContextEngine {
 			void this.drainCompressionQueue();
 		}, 5000);
 		this.drainTimer.unref?.();
+		void this.backfillMemoryVec().catch(() => undefined);
+	}
+
+	/**
+	 * One-shot semantic-memory backfill (upstream #137 lesson). After an
+	 * embedder change the vec tables are recreated empty; without this, old
+	 * decisions silently lose semantic recall (FTS still works, so the loss is
+	 * easy to miss). Guarded by a cheap missing-row count so the steady state
+	 * never initialises an embedder for this. Recall also self-heals via
+	 * `MemoryDb.healVecTables`, covering wipes that happen after this runs.
+	 */
+	private async backfillMemoryVec(): Promise<void> {
+		if (this.vecBackfillStarted) return;
+		this.vecBackfillStarted = true;
+		const memory = this.getMemory();
+		if (!memory || !memory.vecAvailable || !memory.hasVecTables()) return;
+		const pending = memory.vecBackfillPending();
+		if (pending.decisions + pending.turnSummaries === 0) return;
+		await memory.backfillVecTables(await this.getEmbedder());
 	}
 
 	async shutdown(): Promise<void> {

package/src/indexer/pipeline.ts

@@ -139,6 +139,40 @@ function resolveWithin(projectDir: string, target: string): string {
 	return resolved;
 }
 
+/** A directory target is excluded when it or any ancestor matches a dir rule (upstream #165b). */
+function cceignoreMatchesDir(relPosix: string, patterns: string[]): boolean {
+	const parts = relPosix.split("/");
+	for (let i = 1; i <= parts.length; i++) {
+		if (matchesAny(parts.slice(0, i).join("/"), true, patterns)) return true;
+	}
+	return false;
+}
+
+/**
+ * Target-path policy gate, mirroring what the full walker enforces while
+ * descending: every path segment must pass `indexer.ignore`, files must pass
+ * the secret-filename check, and `.cceignore` is matched against the
+ * project-relative path (dir rules for directories, file + ancestor rules for
+ * files). The project root (`rel === ""`) is exempt — rules apply to entries
+ * below the root, not to the root itself (a checkout named `build` must still
+ * reindex via `path: "."`).
+ */
+function targetExcluded(
+	relPosix: string,
+	opts: { isDir: boolean; ignoreSet: Set<string>; redactSecrets: boolean; patterns: string[] },
+): boolean {
+	if (relPosix === "") return false;
+	const segments = relPosix.split("/");
+	for (const seg of segments) {
+		if (opts.ignoreSet.has(seg)) return true;
+	}
+	// The walker only classifies secret names on file entries; a secret-named
+	// directory's contents are still gated per file.
+	if (!opts.isDir && opts.redactSecrets && isSecretFile(segments[segments.length - 1]!)) return true;
+	if (opts.patterns.length === 0) return false;
+	return opts.isDir ? cceignoreMatchesDir(relPosix, opts.patterns) : cceignoreMatchesFile(relPosix, opts.patterns);
+}
+
 function safeRead(filePath: string): string | null {
 	try {
 		if (fs.statSync(filePath).size > MAX_FILE_BYTES) return null;
@@ -149,7 +183,9 @@ function safeRead(filePath: string): string | null {
 }
 
 function* iterProjectFiles(
-	root: string,
+	walkRoot: string,
+	/** Project root: `.cceignore` patterns are matched against project-relative paths even when walking from a target. */
+	relRoot: string,
 	ignoreSet: Set<string>,
 	opts: { redactSecrets: boolean; cceignorePatterns: string[] },
 ): Generator<string> {
@@ -175,7 +211,7 @@ function* iterProjectFiles(
 			}
 			if (seen.has(resolved)) continue;
 			seen.add(resolved);
-			const rel = path.relative(root, full).replace(/\\/g, "/");
+			const rel = path.relative(relRoot, full).replace(/\\/g, "/");
 			if (opts.cceignorePatterns.length > 0 && matchesAny(rel, entry.isDirectory(), opts.cceignorePatterns)) {
 				continue;
 			}
@@ -188,7 +224,7 @@ function* iterProjectFiles(
 			}
 		}
 	};
-	yield* walk(root);
+	yield* walk(walkRoot);
 }
 
 /** `{embeddings, cacheHits, cacheMisses}` for a batch, using the content cache. */
@@ -385,32 +421,77 @@ async function runIndexingLocked(
 	if (cceignorePatterns.length > 0) log?.(`  [.cceignore] ${cceignorePatterns.length} pattern(s) loaded`);
 
 	// ── Determine the file set ─────────────────────────────────────────────
+	/**
+	 * A file the inclusion policy now rejects must not leave stale chunks
+	 * behind (upstream #127): purge its manifest entry and indexed rows so a
+	 * targeted reindex is enough to enforce a new secret/ignore rule.
+	 */
+	const purgeIndexed = (rels: string[]): void => {
+		const indexed = rels.filter((rel) => manifest.getHash(rel) !== undefined);
+		if (indexed.length === 0) return;
+		backend.deleteByFiles(indexed);
+		for (const rel of indexed) {
+			manifest.remove(rel);
+			result.deletedFiles.push(rel);
+			log?.(`  [delete] ${rel} (excluded by policy)`);
+		}
+		manifest.save();
+	};
 	let fileList: string[];
 	if (targetPath) {
 		const target = resolveWithin(projectDir, targetPath);
 		if (fs.existsSync(target) && fs.statSync(target).isFile()) {
 			const rel = path.relative(projectDir, target).replace(/\\/g, "/");
 			if (SKIP_EXTENSIONS.has(path.extname(target).toLowerCase())) {
+				purgeIndexed([rel]);
 				fileList = [];
-			} else if (ignoreSet.has(path.basename(target))) {
-				result.skippedFiles.push(rel);
-				fileList = [];
-			} else if (config.indexerRedactSecrets && isSecretFile(target)) {
-				result.skippedFiles.push(rel);
-				fileList = [];
-			} else if (cceignorePatterns.length > 0 && cceignoreMatchesFile(rel, cceignorePatterns)) {
+			} else if (
+				targetExcluded(rel, {
+					isDir: false,
+					ignoreSet,
+					redactSecrets: config.indexerRedactSecrets,
+					patterns: cceignorePatterns,
+				})
+			) {
+				// Ancestor segments included: the full walker would have pruned
+				// the path before reaching this file (upstream #165b parity).
 				result.skippedFiles.push(rel);
+				purgeIndexed([rel]);
 				fileList = [];
 			} else {
 				fileList = [target];
 			}
 		} else if (fs.existsSync(target) && fs.statSync(target).isDirectory()) {
-			fileList = [
-				...iterProjectFiles(target, ignoreSet, {
-					redactSecrets: config.indexerRedactSecrets,
-					cceignorePatterns,
-				}),
-			];
+			// Upstream #165b: the walk starts inside the target, so the target
+			// path itself must first pass the same policy the full walker
+			// enforces while descending — every segment vs `indexer.ignore`
+			// (e.g. `dist/nested` is rejected because `dist` is), plus
+			// `.cceignore` dir rules for the target and its ancestors.
+			const rel = path.relative(projectDir, target).replace(/\\/g, "/");
+			const excluded = targetExcluded(rel, {
+				isDir: true,
+				ignoreSet,
+				redactSecrets: config.indexerRedactSecrets,
+				patterns: cceignorePatterns,
+			});
+			if (excluded) {
+				result.skippedFiles.push(rel);
+				fileList = [];
+			} else {
+				fileList = [
+					...iterProjectFiles(target, projectDir, ignoreSet, {
+						redactSecrets: config.indexerRedactSecrets,
+						cceignorePatterns,
+					}),
+				];
+			}
+			// Targeted runs skip the full-run prune below, so purge manifest
+			// entries under this subtree the walk no longer yields: newly
+			// excluded files, secret files, and deleted files (upstream
+			// #127/#165b). Scoped to the target so siblings are untouched.
+			const yielded = new Set(fileList.map((f) => path.relative(projectDir, f).replace(/\\/g, "/")));
+			const underScope = (p: string): boolean => (rel === "" ? true : p === rel || p.startsWith(`${rel}/`));
+			purgeIndexed(Object.keys(manifest.fileEntries).filter((p) => underScope(p) && !yielded.has(p)));
 		} else {
 			// Watcher deletion path: prune instead of erroring out.
 			const rel = path.relative(projectDir, target).replace(/\\/g, "/");
@@ -426,7 +507,7 @@ async function runIndexingLocked(
 		}
 	} else {
 		fileList = [
-			...iterProjectFiles(projectDir, ignoreSet, {
+			...iterProjectFiles(projectDir, projectDir, ignoreSet, {
 				redactSecrets: config.indexerRedactSecrets,
 				cceignorePatterns,
 			}),

package/src/memory/db.ts

@@ -309,9 +309,31 @@ CREATE VIRTUAL TABLE IF NOT EXISTS turn_summaries_vec USING vec0(embedding float
 		this.writeVecRow("turn_summaries_vec", turnId, vec);
 	}
 
+	/**
+	 * Count rows still missing a vec entry, without touching an embedder.
+	 * Lets callers skip the backfill (and its model init) in the steady state.
+	 */
+	vecBackfillPending(): { decisions: number; turnSummaries: number } {
+		if (!this.hasVecTables()) return { decisions: 0, turnSummaries: 0 };
+		try {
+			const d = this.db
+				.prepare("SELECT COUNT(*) AS n FROM decisions d WHERE NOT EXISTS (SELECT 1 FROM decisions_vec v WHERE v.rowid = d.id)")
+				.get() as { n: number };
+			const t = this.db
+				.prepare("SELECT COUNT(*) AS n FROM turn_summaries t WHERE NOT EXISTS (SELECT 1 FROM turn_summaries_vec v WHERE v.rowid = t.id)")
+				.get() as { n: number };
+			return { decisions: Number(d?.n ?? 0), turnSummaries: Number(t?.n ?? 0) };
+		} catch {
+			return { decisions: 0, turnSummaries: 0 };
+		}
+	}
+
 	async backfillVecTables(embedder: Embedder): Promise<{ decisions: number; turnSummaries: number }> {
 		const counts = { decisions: 0, turnSummaries: 0 };
 		if (!this.hasVecTables()) return counts;
+		// Steady state: nothing missing — skip the embedder entirely.
+		const pending = this.vecBackfillPending();
+		if (pending.decisions + pending.turnSummaries === 0) return counts;
 		try {
 			this.ensureVecDim((await embedder.embedQuery("dimension probe")).length);
 		} catch {
@@ -348,6 +370,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS turn_summaries_vec USING vec0(embedding float
 		} catch {
 			return [];
 		}
+		await this.healVecTables(embedder, vec);
 		return this.vecSearch("decisions_vec", vec, k, maxDistance);
 	}
 
@@ -359,9 +382,32 @@ CREATE VIRTUAL TABLE IF NOT EXISTS turn_summaries_vec USING vec0(embedding float
 		} catch {
 			return [];
 		}
+		await this.healVecTables(embedder, vec);
 		return this.vecSearch("turn_summaries_vec", vec, k, maxDistance);
 	}
 
+	/**
+	 * Self-heal at the point of recall (upstream #137), covering both failure
+	 * shapes a dim-mismatched or partially-empty vec table produces:
+	 *
+	 * 1. Tables built by another embedder: a dim-mismatched MATCH errors and
+	 *    silently returns nothing → recreate at the query's dim.
+	 * 2. Rows missing for any reason (e.g. a mid-session `recordDecisionVec`
+	 *    wiped the tables via `ensureVecDim` after the engine's one-shot
+	 *    backfill already ran) → restore them here, where the embedder is
+	 *    already in hand.
+	 *
+	 * Steady state costs two COUNT queries; the embedder is only touched when
+	 * something is actually wrong.
+	 */
+	private async healVecTables(embedder: Embedder, vec: readonly number[]): Promise<void> {
+		if (!this.hasVecTables()) return;
+		const declared = this.declaredVecDim();
+		if (declared !== null && declared !== vec.length) this.ensureVecDim(vec.length);
+		const pending = this.vecBackfillPending();
+		if (pending.decisions + pending.turnSummaries > 0) await this.backfillVecTables(embedder);
+	}
+
 	private vecSearch(table: string, vec: number[], k: number, maxDistance: number): number[] {
 		try {
 			const rows = this.db