pi-context-engine 0.5.0 → 0.5.2

package/README.md

@@ -103,6 +103,25 @@ unchanged repo took 237 ms.
 - **Switches**: `indexer.auto_index: false` disables the session-start and
   lazy kicks; `indexer.watch: false` disables the file watcher. Use
   `/context-index` for manual builds.
+- **Large-workspace guard**: automatic indexing and the watcher pause with a
+  warning above `indexer.max_auto_files` (default 10,000). The check uses the
+  normal directory walk and stops at the first file above the limit, so it does
+  not add a full directory scan. Periodic yields keep the check responsive.
+  This prevents an accidental parent-directory
+  session from starting an unrequested multi-minute workload. `/context-index`
+  remains available as an explicit manual build.
+- **Resume after interruption**: the manifest checkpoints completed batches,
+  starting with the first batch and then at most once every five seconds. An
+  interrupted build can skip completed files instead of starting from zero.
+  Migration runs (schema reset, embedding-dimension change) are exempt: their
+  manifest stays untouched on disk until the run completes, so an interrupted
+  migration resumes cleanly instead of stranding stale rows.
+- **Cold-start visibility**: the first build on an empty index announces its
+  start (`building the first index…`) and always closes with a result line
+  (`updated: N file(s)` or `no files to index`). This is the one background
+  run that can take minutes, so a pause reads as "indexing", not as a hung
+  session. Warm incremental runs stay quiet on start: they finish in a few
+  hundred ms, and a toast per session would be noise.
 - **Why it matters**: incremental runs are cheap (a few hundred ms on an
   unchanged repo), so this removes the classic cold-start failure of searching
   an empty index, without ever blocking the UI.
@@ -186,6 +205,7 @@ indexer:
   chunking: legacy        # legacy (default) | ast-density — cAST-style budgeted split-merge (experimental)
   chunk_budget_chars: 1200 # non-whitespace budget per ast-density chunk
   auto_index: true        # incremental background reindex shortly after session start
+  max_auto_files: 10000   # automatic full-project file budget; 0 disables the guard
   watch: true             # reindex on file changes while the session runs (debounced, incremental)
 memory:
   redact_pii: true

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-context-engine",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "Semantic code context for pi: tree-sitter index + hybrid vector/BM25 retrieval + cross-session memory, all local. Port of code-context-engine (MIT, elara-labs) as a native pi extension.",
   "keywords": [
     "pi-package",

package/src/config.ts

@@ -95,6 +95,8 @@ export interface Config {
 	indexerContextHeaders: boolean;
 	/** Run an incremental reindex in the background shortly after session start. */
 	indexerAutoIndex: boolean;
+	/** Maximum files for automatic full-project indexing. Zero disables the guard; manual runs always bypass it. */
+	indexerMaxAutoFiles: number;
 	/**
 	 * Watch the project tree while a session runs and reindex on changes
 	 * (debounced, incremental). Independent of `indexerAutoIndex`, which only
@@ -134,6 +136,7 @@ export function defaultConfig(): Config {
 		indexerIgnore: [...DEFAULT_IGNORE],
 		indexerContextHeaders: true,
 		indexerAutoIndex: true,
+		indexerMaxAutoFiles: 10_000,
 		indexerWatch: true,
 		indexerRedactSecrets: true,
 		memoryRedactPii: true,
@@ -169,6 +172,7 @@ const MAPPING: Record<string, keyof Config> = {
 	"indexer.ignore": "indexerIgnore",
 	"indexer.context_headers": "indexerContextHeaders",
 	"indexer.auto_index": "indexerAutoIndex",
+	"indexer.max_auto_files": "indexerMaxAutoFiles",
 	"indexer.watch": "indexerWatch",
 	"indexer.redact_secrets": "indexerRedactSecrets",
 	"memory.redact_pii": "memoryRedactPii",
@@ -212,6 +216,11 @@ function applyDict(config: Config, data: any): void {
 				if (typeof item === "string" && !merged.includes(item)) merged.push(item);
 			}
 			config.indexerIgnore = merged;
+		} else if (attr === "indexerMaxAutoFiles") {
+			if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
+				throw new Error(`config ${dotted} must be a non-negative integer`);
+			}
+			config.indexerMaxAutoFiles = value;
 		} else if (attr === "pricingInput" || attr === "pricingOutput") {
 			if (value !== null && typeof value !== "number") throw new Error(`config ${dotted} must be a number or null`);
 			(config as any)[attr] = value;

package/src/engine.ts

@@ -20,7 +20,7 @@ import { SessionCapture } from "./memory/session-capture.js";
 import { drainOne, enqueueSessionRollup, enqueueTurn, type CompressionDeps } from "./memory/compressor.js";
 import { compress, expand, compressWithCounts, DEFAULT_LEVEL } from "./memory/grammar.js";
 import { cosine } from "./memory/extractive.js";
-import { runIndexing } from "./indexer/pipeline.js";
+import { AutomaticIndexFileLimitError, runIndexing } from "./indexer/pipeline.js";
 import { getModelPricing } from "./pricing.js";
 import type { Chunk } from "./types.js";
 
@@ -358,7 +358,7 @@ export class ContextEngine {
 		return this.backend.countChunks();
 	}
 
-	async reindex(options: { full?: boolean; targetPath?: string | null; onProgress?: (msg: string) => void } = {}): Promise<{
+	async reindex(options: { full?: boolean; targetPath?: string | null; automatic?: boolean; onProgress?: (msg: string) => void } = {}): Promise<{
 		indexed: number;
 		chunks: number;
 		deleted: number;
@@ -368,6 +368,10 @@ export class ContextEngine {
 		const result = await runIndexing(this.config, this.projectDir, {
 			full: options.full,
 			targetPath: options.targetPath ?? null,
+			maxAutoFiles:
+				options.automatic && !options.targetPath && this.config.indexerMaxAutoFiles > 0
+					? this.config.indexerMaxAutoFiles
+					: undefined,
 			callbacks: { phase: options.onProgress },
 			runtime: this.injectedRuntime,
 		});
@@ -393,7 +397,8 @@ export class ContextEngine {
 	 * next kick retries — the index is still empty, after all. The pipeline
 	 * lock in runIndexing serializes this with a manual /context-index or a
 	 * lazy search-triggered run, so races are safe — just wasteful, which the
-	 * one-shot guard prevents.
+	 * one-shot guard prevents. Transient rejections un-consume the one-shot;
+	 * a workspace-size refusal stays consumed because retrying cannot help.
 	 */
 	async autoIndexOnce(): Promise<{
 		indexed: number;
@@ -405,9 +410,11 @@ export class ContextEngine {
 		if (!this.config.indexerAutoIndex || this.autoIndexDone) return null;
 		this.autoIndexDone = true;
 		try {
-			return await this.reindex({});
+			return await this.reindex({ automatic: true });
 		} catch (err) {
-			this.autoIndexDone = false;
+			// A workspace-size refusal is a stable policy decision, not a
+			// transient embedding failure. Do not retry it on every lazy search.
+			if (!(err instanceof AutomaticIndexFileLimitError)) this.autoIndexDone = false;
 			throw err;
 		}
 	}

package/src/extension.ts

@@ -13,6 +13,7 @@ import path from "node:path";
 import { ContextEngine, isPceConfig } from "./engine.js";
 import { applyContextSearchGuidance } from "./guidance.js";
 import { ProjectWatcher } from "./indexer/watcher.js";
+import { AutomaticIndexFileLimitError } from "./indexer/pipeline.js";
 
 /** Tools whose payloads are worth capturing for session timelines. */
 const CAPTURED_TOOLS = new Set(["bash", "read", "edit", "write", "grep", "find", "ls"]);
@@ -32,6 +33,7 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 	let bootstrapSent = false;
 	let autoIndexTimer: NodeJS.Timeout | null = null;
 	let indexInFlight = false;
+	let automaticIndexBlock: AutomaticIndexFileLimitError | null = null;
 
 	/**
 	 * File watcher (config `indexer.watch`, default true): a debounced change
@@ -46,22 +48,23 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 
 	/**
 	 * Watcher-triggered incremental reindex. Unlike `kickBackgroundIndex` this
-	 * calls `reindex()` directly — the watcher must re-fire all session, not
-	 * consume a one-shot. Shares `indexInFlight` with the other kicks so runs
+	 * calls `reindex({ automatic: true })` directly — the watcher must re-fire
+	 * all session, not consume a one-shot. The automatic flag enforces the same
+	 * workspace file budget as session-start indexing. Shares `indexInFlight` with the other kicks so runs
 	 * never stack; a change that arrives mid-run coalesces into exactly one
 	 * trailing kick, so edits made *while indexing* still land. Successful
 	 * runs stay silent (the user's own edit caused them); failures — both
 	 * rejections AND per-file errors resolved in `result.errors` — notify.
 	 */
 	const kickWatcherIndex = (eng: ContextEngine, notify?: (msg: string, level: "info" | "warning") => void) => {
-		if (engine !== eng) return; // session moved on
+		if (engine !== eng || automaticIndexBlock) return; // session moved on, or the workspace exceeds the auto budget
 		if (indexInFlight) {
 			watcherKickPending = true;
 			return;
 		}
 		indexInFlight = true;
 		void eng
-			.reindex({})
+			.reindex({ automatic: true })
 			.then((result) => {
 				// Per-file chunking/read failures RESOLVE in result.errors — mirroring
 				// kickBackgroundIndex, surface them or affected files go stale silently.
@@ -71,7 +74,10 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 				}
 			})
 			.catch((err: unknown) => {
-				if (engine === eng && notify) {
+				if (engine !== eng) return;
+				if (err instanceof AutomaticIndexFileLimitError) {
+					pauseAutomaticIndexing(eng, err, notify);
+				} else if (notify) {
 					const detail = (err instanceof Error ? err.message : String(err)).slice(0, 120);
 					notify(`Context index: watcher reindex failed (${detail})`, "warning");
 				}
@@ -95,6 +101,24 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		watcherKickPending = false;
 	};
 
+	/** A stable size refusal pauses every automatic path until the next session. */
+	const pauseAutomaticIndexing = (
+		eng: ContextEngine,
+		err: AutomaticIndexFileLimitError,
+		notify?: (msg: string, level: "info" | "warning") => void,
+	) => {
+		if (engine !== eng) return;
+		automaticIndexBlock = err;
+		lazyIndexStarted = true; // a lazy search must not immediately retry the same refusal
+		stopWatcher();
+		notify?.(
+			`Context index: automatic indexing${eng.config.indexerWatch ? " and watcher" : ""} paused. ` +
+				`Workspace exceeds ${err.limit.toLocaleString()} indexable files. ` +
+				"Run /context-index to index anyway, or raise indexer.max_auto_files.",
+			"warning",
+		);
+	};
+
 	/**
 	 * (Re)point the file watcher at the live engine's project dir. Idempotent
 	 * per dir (called from every `getEngine`), so a mid-session project switch
@@ -135,8 +159,17 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 	 * itself one-shot per engine instance.
 	 */
 	const kickBackgroundIndex = (eng: ContextEngine, notify?: (msg: string, level: "info" | "warning") => void) => {
+		if (engine !== eng || automaticIndexBlock) return;
 		lazyIndexStarted = true;
 		indexInFlight = true;
+		// Cold-start visibility: the first build on an empty index is the only
+		// background run that can take minutes (every file gets chunked and
+		// embedded), and it used to stay invisible until the completion line.
+		// Announce the start so a long pause reads as "indexing", not as a hung
+		// session. Warm incremental runs are NOT announced: they finish in a few
+		// hundred ms and a toast on every session start would be noise.
+		const announcedStart = eng.chunkCount() === 0 && notify !== undefined;
+		if (announcedStart) notify?.(`Context index: building the first index for ${eng.projectName}…`, "info");
 		void eng
 			.autoIndexOnce()
 			.then((result) => {
@@ -145,6 +178,11 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 					notify(`Context index: background index hit ${result.errors.length} error(s)`, "warning");
 				} else if (result.indexed > 0) {
 					notify(`Context index updated: ${result.indexed} file(s), ${result.chunks} chunk(s)`, "info");
+				} else if (announcedStart) {
+					// The start toast promised feedback — close the loop even when
+					// there is nothing to index (empty project), instead of leaving
+					// "building…" hanging without a result.
+					notify(`Context index: no files to index for ${eng.projectName}`, "info");
 				}
 			})
 			.catch((err: unknown) => {
@@ -154,12 +192,15 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 				// the lazy path burned on an empty index. Free the lazy path so a
 				// later search retries (autoIndexOnce un-consumes its one-shot),
 				// and surface the failure when we have a notify channel.
-				if (engine === eng) {
-					lazyIndexStarted = false;
-					if (notify) {
-						const detail = (err instanceof Error ? err.message : String(err)).slice(0, 120);
-						notify(`Context index: background indexing failed — run /context-index for details (${detail})`, "warning");
-					}
+				if (engine !== eng) return;
+				if (err instanceof AutomaticIndexFileLimitError) {
+					pauseAutomaticIndexing(eng, err, notify);
+					return;
+				}
+				lazyIndexStarted = false;
+				if (notify) {
+					const detail = (err instanceof Error ? err.message : String(err)).slice(0, 120);
+					notify(`Context index: background indexing failed — run /context-index for details (${detail})`, "warning");
 				}
 			})
 			.finally(() => {
@@ -175,6 +216,8 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 			// Best-effort: a cross-project switch must not leak sqlite handles.
 			void engine.shutdown().catch(() => undefined);
 		}
+		automaticIndexBlock = null;
+		lazyIndexStarted = false;
 		// Always track the live engine so `session_shutdown` closes it even when
 		// a tool ran before `session_start` (tests, mid-reload edge cases).
 		engine = new ContextEngine({ projectDir: dir });
@@ -220,6 +263,13 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 			// `indexer.auto_index: false` disables this lazy kick too — one switch
 			// for all automatic indexing.
 			if (eng.chunkCount() === 0) {
+				if (automaticIndexBlock) {
+					return text(
+						`Automatic indexing is paused for ${eng.projectName}: the workspace exceeds ` +
+							`${automaticIndexBlock.limit.toLocaleString()} indexable files. ` +
+							"Run /context-index to index manually, or raise indexer.max_auto_files.",
+					);
+				}
 				if (indexInFlight) {
 					return text(
 						`Index for ${eng.projectName} is still building in the background — retry this search in a few seconds.`,
@@ -481,6 +531,7 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 
 	pi.on("session_start", async (_event, ctx) => {
 		currentCwd = ctx.cwd;
+		automaticIndexBlock = null;
 		engine = new ContextEngine({ projectDir: ctx.cwd });
 		engine.startBackgroundWork();
 		lazyIndexStarted = false;
@@ -576,6 +627,7 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
 		}
 		stopWatcher();
 		watcherNotify = undefined;
+		automaticIndexBlock = null;
 		await engine?.shutdown().catch(() => undefined);
 		engine = null;
 	});

package/src/indexer/pipeline.ts

@@ -112,9 +112,24 @@ const LANGUAGE_MAP: Record<string, string> = {
 /** Skip any single file larger than this — protects from OOM on log dumps. */
 const MAX_FILE_BYTES = 2 * 1024 * 1024;
 const BATCH = 50;
+const MANIFEST_CHECKPOINT_MS = 5000;
+const WALK_YIELD_FILES = 250;
+const WALK_YIELD_MS = 50;
 
 export class PathOutsideProjectError extends Error {}
 
+/** Automatic full-project runs refuse workspaces above their configured file budget. */
+export class AutomaticIndexFileLimitError extends Error {
+	constructor(
+		readonly limit: number,
+		/** Exact manifest size, or limit + 1 when the capped walk stopped early. */
+		readonly observedFiles: number,
+	) {
+		super(`Automatic indexing exceeds the ${limit}-file limit (found at least ${observedFiles})`);
+		this.name = "AutomaticIndexFileLimitError";
+	}
+}
+
 export interface IndexResult {
 	indexedFiles: string[];
 	skippedFiles: string[];
@@ -235,6 +250,26 @@ function* iterProjectFiles(
 	yield* walk(walkRoot);
 }
 
+/**
+ * Collect a directory walk without starving pi's input/render loop. The file
+ * count covers cheap SSD walks; the time bound also covers slow filesystems.
+ */
+async function collectProjectFiles(files: Iterable<string>, maxAutoFiles?: number): Promise<string[]> {
+	const collected: string[] = [];
+	let lastYield = Date.now();
+	for (const filePath of files) {
+		if (maxAutoFiles !== undefined && collected.length >= maxAutoFiles) {
+			throw new AutomaticIndexFileLimitError(maxAutoFiles, collected.length + 1);
+		}
+		collected.push(filePath);
+		if (collected.length % WALK_YIELD_FILES === 0 || Date.now() - lastYield >= WALK_YIELD_MS) {
+			await yieldToEventLoop();
+			lastYield = Date.now();
+		}
+	}
+	return collected;
+}
+
 /** `{embeddings, cacheHits, cacheMisses}` for a batch, using the content cache. */
 async function embedChunks(
 	chunks: Chunk[],
@@ -378,6 +413,8 @@ export async function runIndexing(
 	options: {
 		full?: boolean;
 		targetPath?: string | null;
+		/** Automatic full-project runs only. Stop before migrations when the workspace exceeds this budget. */
+		maxAutoFiles?: number;
 		callbacks?: IndexCallbacks;
 		/** Reuse an already-constructed backend/embedder (the long-lived server path). */
 		runtime?: { backend: LocalBackend; embedder: Embedder };
@@ -402,7 +439,7 @@ async function runIndexingLocked(
 	config: Config,
 	projectDir: string,
 	storageBase: string,
-	options: { full?: boolean; targetPath?: string | null; callbacks?: IndexCallbacks },
+	options: { full?: boolean; targetPath?: string | null; maxAutoFiles?: number; callbacks?: IndexCallbacks },
 	runtime?: { backend: LocalBackend; embedder: Embedder },
 ): Promise<IndexResult> {
 	let { full = false, targetPath = null } = options;
@@ -410,7 +447,6 @@ async function runIndexingLocked(
 	const { log, progress, phase } = callbacks;
 
 	const ownsBackend = runtime === undefined;
-	const backend = runtime?.backend ?? new LocalBackend(storageBase);
 	const chunker = new Chunker();
 	const manifest = new Manifest(path.join(storageBase, "manifest.json"));
 	const result: IndexResult = {
@@ -423,11 +459,49 @@ async function runIndexingLocked(
 		cacheMisses: 0,
 	};
 
+	const ignoreSet = new Set(config.indexerIgnore);
+	const rootIgnoreSet = new Set(ROOT_ONLY_IGNORE);
+	const cceignorePatterns = loadIgnorePatterns(projectDir);
+	if (cceignorePatterns.length > 0) log?.(`  [.cceignore] ${cceignorePatterns.length} pattern(s) loaded`);
+
 	// PR #7 review fix: migrations below call manifest.clearEntries(), which
 	// also erases the file list pruneFilesNotIn() needs to detect files that
 	// vanished BEFORE this run. Snapshot the paths first so the prune step
 	// can still delete their indexed chunks instead of orphaning them.
 	const preMigrationPaths = Object.keys(manifest.fileEntries);
+	// Checkpoints (below) persist the in-memory manifest mid-run. A run that
+	// wiped or invalidated existing entries must NOT do that: the on-disk
+	// manifest is what a resumed run derives its migration context from
+	// (pre-migration paths, old schema version, git pointer). Checkpointing
+	// after clearEntries() would persist the emptied state, so an interrupted
+	// migration would resume as a plain incremental run and orphan FTS/graph
+	// rows of vanished files or skip the git-history rebuild (Augment review,
+	// PR #21). Migration runs save once at the end, exactly as before
+	// checkpoints existed — losing resume progress there is fine; correctness
+	// is not.
+	let migrationRun = false;
+
+	// The automatic file budget rides on the walk we already need. No second
+	// directory scan or per-file stat pass is required. Reuse the completed
+	// walk below, and stop at limit + 1 before any store migration can wipe an
+	// existing index. A large manifest is an even cheaper early refusal.
+	let precheckedFileList: string[] | null = null;
+	if (options.maxAutoFiles !== undefined && targetPath === null) {
+		if (preMigrationPaths.length > options.maxAutoFiles) {
+			throw new AutomaticIndexFileLimitError(options.maxAutoFiles, preMigrationPaths.length);
+		}
+		precheckedFileList = await collectProjectFiles(
+			iterProjectFiles(projectDir, projectDir, ignoreSet, rootIgnoreSet, {
+				redactSecrets: config.indexerRedactSecrets,
+				cceignorePatterns,
+			}),
+			options.maxAutoFiles,
+		);
+	}
+
+	// Open stores only after the automatic guard has passed, so a refusal
+	// cannot leak SQLite handles on the session's long-lived process.
+	const backend = runtime?.backend ?? new LocalBackend(storageBase);
 
 	// Refuse unrecognized index formats outright (no compat reading): a
 	// manifest we can't recognize — pre-versioning, corrupt, or a future
@@ -452,13 +526,9 @@ async function runIndexingLocked(
 			targetPath = null;
 		}
 		full = true;
+		migrationRun = true;
 	}
 
-	const ignoreSet = new Set(config.indexerIgnore);
-	const rootIgnoreSet = new Set(ROOT_ONLY_IGNORE);
-	const cceignorePatterns = loadIgnorePatterns(projectDir);
-	if (cceignorePatterns.length > 0) log?.(`  [.cceignore] ${cceignorePatterns.length} pattern(s) loaded`);
-
 	// ── Determine the file set ─────────────────────────────────────────────
 	/**
 	 * A file the inclusion policy now rejects must not leave stale chunks
@@ -519,12 +589,12 @@ async function runIndexingLocked(
 				result.skippedFiles.push(rel);
 				fileList = [];
 			} else {
-				fileList = [
-					...iterProjectFiles(target, projectDir, ignoreSet, rootIgnoreSet, {
+				fileList = await collectProjectFiles(
+					iterProjectFiles(target, projectDir, ignoreSet, rootIgnoreSet, {
 						redactSecrets: config.indexerRedactSecrets,
 						cceignorePatterns,
 					}),
-				];
+				);
 			}
 			// Targeted runs skip the full-run prune below, so purge manifest
 			// entries under this subtree the walk no longer yields: newly
@@ -548,12 +618,12 @@ async function runIndexingLocked(
 			return result;
 		}
 	} else {
-		fileList = [
-			...iterProjectFiles(projectDir, projectDir, ignoreSet, rootIgnoreSet, {
+		fileList = precheckedFileList ?? (await collectProjectFiles(
+			iterProjectFiles(projectDir, projectDir, ignoreSet, rootIgnoreSet, {
 				redactSecrets: config.indexerRedactSecrets,
 				cceignorePatterns,
 			}),
-		];
+		));
 	}
 
 	// Warm up every AST grammar we're about to need (async, once).
@@ -616,6 +686,7 @@ async function runIndexingLocked(
 				// The dim swap dropped stored vectors (git history included); reset
 				// the pointer so the next full run re-fetches those commits too.
 				manifest.lastGitSha = null;
+				migrationRun = true;
 			}
 		}
 
@@ -624,6 +695,19 @@ async function runIndexingLocked(
 		let announced = false;
 		let totalCacheHits = 0;
 		let totalCacheMisses = 0;
+		// The first successful batch checkpoints immediately; later writes are
+		// rate-limited. An interrupted run can therefore skip completed batches
+		// without paying a manifest write for every file. Migration runs are
+		// exempt (see migrationRun above): their manifest stays untouched on
+		// disk until the run completes.
+		let lastManifestCheckpoint = 0;
+		const checkpointManifest = () => {
+			if (migrationRun) return;
+			const now = Date.now();
+			if (now - lastManifestCheckpoint < MANIFEST_CHECKPOINT_MS) return;
+			manifest.save();
+			lastManifestCheckpoint = now;
+		};
 
 		for (let batchIndex = 0; batchIndex < fileList.length; batchIndex += BATCH) {
 			// Cooperative yield: the loops below do sync I/O, tree-sitter parsing
@@ -798,6 +882,7 @@ async function runIndexingLocked(
 
 			if (batchChunks.length > 0) {
 				const { embedder: emb, cache: cch } = await ensureEmbedder();
+				manifest.embeddingDim = emb.dimension;
 				if (phase && !announced) {
 					phase(`Embedding ${batchChunks.length.toLocaleString()} chunks${totalBatches > 1 ? ` (batch ${Math.floor(batchIndex / BATCH) + 1}/${totalBatches})` : ""}…`);
 					announced = true;
@@ -817,6 +902,7 @@ async function runIndexingLocked(
 				for (const [rel, hash] of batchManifestUpdates) manifest.update(rel, hash);
 				result.indexedFiles.push(...batchIndexed);
 			}
+			if (batchManifestUpdates.length > 0) checkpointManifest();
 			result.skippedFiles.push(...toChunk.filter((t) => !batchIndexed.includes(t.relPath)).map((t) => t.relPath));
 		}
 
@@ -884,6 +970,7 @@ async function runIndexingLocked(
 				const git = await indexCommits(projectDir, manifest.lastGitSha);
 				if (git.chunks.length > 0) {
 					const { embedder: emb, cache: cch } = await ensureEmbedder();
+					manifest.embeddingDim = emb.dimension;
 					phase?.(`Embedding ${git.chunks.length} git commits…`);
 					const stats = await embedChunks(git.chunks, emb, cch);
 					totalCacheHits += stats.hits;