pi-context-engine 0.1.4 → 0.2.0

package/src/indexer/chunker.ts

@@ -14,11 +14,13 @@ const require = createRequire(import.meta.url);
 const Parser: any = require("web-tree-sitter");
 
 const FUNCTION_TYPES = new Set([
-	"function_definition", "function_declaration",
-	"method_definition", "method_declaration",
-	"arrow_function",
-	"function_item",
-	"local_function_statement",
+	"function_definition", "function_declaration", "generator_function_declaration",
+	"method_definition", "method_declaration", "generator_method",
+	"arrow_function", "function_expression",
+	"function_item", "local_function_statement",
+	// Constructors: a Java/C# class's constructor is a definition users search
+	// for by name; missing it left holes exactly like the generator gap.
+	"constructor_declaration", "compact_constructor_declaration",
 ]);
 const CLASS_TYPES = new Set([
 	"class_definition", "class_declaration",
@@ -34,6 +36,12 @@ const IMPORT_TYPES = new Set([
 	"using_directive",
 ]);
 
+/** Child node types that carry a symbol name (any language we parse). */
+const NAME_TYPES = new Set([
+	"identifier", "property_identifier", "type_identifier",
+	"field_identifier", "name",
+]);
+
 /** tree-sitter-wasms file stems, keyed by our language id. */
 const WASM_BY_LANGUAGE: Record<string, string> = {
 	python: "tree-sitter-python",
@@ -60,6 +68,30 @@ interface TsNode {
 	text: string;
 }
 
+/** Budget for the R6 module chunk: embed/FTS cap is 5000 chars, leave room for the header. */
+const MODULE_CHUNK_BUDGET_CHARS = 3000;
+
+/** R12: default non-whitespace budget per ast-density chunk (cAST-style ~1200). */
+export const DEFAULT_CHUNK_BUDGET_CHARS = 1200;
+
+export interface ChunkingOptions {
+	/** R12: `legacy` (default) walks function/class nodes; `ast-density` tiles the file into budgeted, self-contained chunks. */
+	chunking?: "legacy" | "ast-density";
+	/** R12: non-whitespace char budget for `ast-density`. */
+	budgetChars?: number;
+}
+
+/** Internal build unit for ast-density tiling. */
+interface DensityUnit {
+	start: number;
+	end: number;
+	text: string;
+	kind: ChunkType;
+	unit: string | null;
+	parent: string | null;
+	segment: number;
+}
+
 let parserInitPromise: Promise<void> | undefined;
 const languageCache = new Map<string, Promise<unknown>>();
 
@@ -125,55 +157,170 @@ export class Chunker {
 		}
 	}
 
-	chunkWithImports(source: string, filePath: string, language: string): { chunks: Chunk[]; imports: string[] } {
-		const chunks = this.chunk(source, filePath, language);
-		const imports = this.extractImports(source, language);
-		return { chunks, imports };
+	chunkWithImports(
+		source: string,
+		filePath: string,
+		language: string,
+		options: ChunkingOptions = {},
+	): { chunks: Chunk[]; imports: string[]; importSpecs: string[] } {
+		const chunks =
+			options.chunking === "ast-density"
+				? this.chunkAstDensity(source, filePath, language, options.budgetChars ?? DEFAULT_CHUNK_BUDGET_CHARS)
+				: this.chunk(source, filePath, language);
+		const extracted = this.extractImports(source, language);
+		const imports = extracted.imports;
+		const importSpecs = extracted.rawSpecs;
+		// Always attempt the module chunk, even with zero children: a file of
+		// pure top-level code (config tables, scripts) still needs a chunk, and
+		// this makes the pipeline's whole-file fallback a rare safety net.
+		const moduleChunk = this.buildModuleChunk(source, filePath, language, chunks);
+		if (moduleChunk) chunks.push(moduleChunk);
+		return { chunks, imports, importSpecs };
 	}
 
-	private walk(node: TsNode, source: string, filePath: string, language: string, chunks: Chunk[]): void {
+	/**
+	 * R6: one `module` chunk per AST file covering what the function/class walk
+	 * leaves out — imports, top-level consts, type declarations, comments.
+	 *
+	 * Without it, module-level content (`DEFAULT_IGNORE = [...]`, config keys,
+	 * prompt strings) was retrievable by NO path: eval queries about defaults
+	 * scored exactly zero. Chunks carry `chunkType: "module"` which doubles as
+	 * the parent marker: the retriever exempts module chunks from overlap
+	 * dedupe (their span covers the whole file) and serves them as the
+	 * small-to-big parent context of a hit.
+	 */
+	private buildModuleChunk(source: string, filePath: string, language: string, children: Chunk[]): Chunk | null {
+		const lines = source.split("\n");
+		const covered = new Array<boolean>(lines.length).fill(false);
+		for (const c of children) {
+			for (let i = Math.max(0, c.startLine - 1); i < Math.min(lines.length, c.endLine); i++) covered[i] = true;
+		}
+		const uncovered = lines
+			.map((line, i) => ({ line, i }))
+			.filter(({ line, i }) => !covered[i] && line.trim().length > 0)
+			.map(({ line }) => line);
+
+		const defines = children
+			.map((c) => (typeof c.metadata["unit"] === "string" ? (c.metadata["unit"] as string) : null))
+			.filter((u): u is string => !!u && !u.includes("."));
+		const parts = [...uncovered];
+		if (defines.length > 0) parts.push(`defines: ${[...new Set(defines)].slice(0, 30).join(", ")}`);
+		if (parts.length === 0) return null;
+
+		let body = parts.join("\n");
+		if (body.length > MODULE_CHUNK_BUDGET_CHARS) {
+			// Keep the head: imports and first declarations carry the module shape.
+			let cut = 0;
+			let used = 0;
+			for (const part of parts) {
+				if (used + part.length + 1 > MODULE_CHUNK_BUDGET_CHARS) break;
+				used += part.length + 1;
+				cut++;
+			}
+			body = `${parts.slice(0, Math.max(1, cut)).join("\n")}\n…[module context truncated]`;
+		}
+
+		const endLine = Math.max(1, lines.length - (lines[lines.length - 1] === "" ? 1 : 0));
+		const id = sha256Hex(`${filePath}:module`).slice(0, 16);
+		const base = filePath.split("/").pop()!.replace(/\.[^.]+$/, "");
+		return makeChunk({
+			id,
+			content: body,
+			chunkType: "module",
+			filePath,
+			startLine: 1,
+			endLine,
+			language,
+			metadata: { unit: base, tier: "parent" },
+		});
+	}
+
+	private walk(node: TsNode, source: string, filePath: string, language: string, chunks: Chunk[], ancestors: string[] = []): void {
+		let unit: string | null = null;
 		if (FUNCTION_TYPES.has(node.type)) {
-			chunks.push(this.nodeToChunk(node, source, filePath, language, "function"));
+			unit = nodeUnit(node, ancestors);
+			chunks.push(this.nodeToChunk(node, source, filePath, language, "function", unit));
 		} else if (CLASS_TYPES.has(node.type)) {
-			chunks.push(this.nodeToChunk(node, source, filePath, language, "class"));
+			unit = nodeUnit(node, ancestors);
+			chunks.push(this.nodeToChunk(node, source, filePath, language, "class", unit));
 		}
+		// Descendants inherit the enclosing chain: `PaymentGateway.refund`.
+		const name = unit?.split(".").pop();
+		const chain = name ? [...ancestors, name] : ancestors;
 		for (const child of node.children) {
-			this.walk(child, source, filePath, language, chunks);
+			this.walk(child, source, filePath, language, chunks, chain);
 		}
 	}
 
-	private nodeToChunk(node: TsNode, source: string, filePath: string, language: string, chunkType: ChunkType): Chunk {
+	private nodeToChunk(node: TsNode, source: string, filePath: string, language: string, chunkType: ChunkType, unit: string | null = null): Chunk {
 		// node.text is sliced correctly by the binding (UTF-8 aware).
 		void source;
 		const content = node.text;
 		const startLine = node.startPosition.row + 1;
 		const endLine = node.endPosition.row + 1;
 		const id = sha256Hex(`${filePath}:${startLine}:${endLine}:${content.slice(0, 100)}`).slice(0, 16);
-		return makeChunk({ id, content, chunkType, filePath, startLine, endLine, language });
+		return makeChunk({ id, content, chunkType, filePath, startLine, endLine, language, metadata: unit ? { unit } : {} });
 	}
 
-	private extractImports(source: string, language: string): string[] {
+	/**
+	 * R12: cAST-style split-merge. Tiles the file into top-level segments,
+	 * splits oversized ones at syntactic children (line windows only at
+	 * leaves), then greedily merges small adjacent siblings up to the budget.
+	 * Concatenating the returned chunks in order (excluding the module chunk)
+	 * reproduces the source byte-for-byte.
+	 */
+	private chunkAstDensity(source: string, filePath: string, language: string, budget: number): Chunk[] {
 		const lang = syncLanguageCache.get(language);
-		if (!lang) return [];
+		if (!lang) {
+			throw new Error(`language ${language} not warmed up — call await warmupLanguages() first`);
+		}
+		const parser = this.acquireParser();
+		try {
+			(parser as { setLanguage(l: unknown): void }).setLanguage(lang);
+			const tree = (parser as { parse(s: string): { rootNode: TsNode } }).parse(source);
+			const units = splitNode(tree.rootNode, source, budget, 0, source.length, null);
+			const merged = mergeUnits(units, source, budget);
+			const chunks: Chunk[] = merged.map((u, i) => {
+				const startLine = offsetToLine(source, u.start);
+				const endLine = offsetToLine(source, Math.max(u.start, u.end - 1));
+				const id = sha256Hex(`${filePath}:${u.start}:${u.end}:${i}:${u.text.slice(0, 100)}`).slice(0, 16);
+				const metadata: Record<string, unknown> = {};
+				if (u.unit) metadata["unit"] = u.unit;
+				if (u.parent) metadata["parent"] = u.parent;
+				if (u.segment > 0) metadata["segment"] = String(u.segment);
+				return makeChunk({ id, content: u.text, chunkType: u.kind, filePath, startLine, endLine, language, metadata });
+			});
+			return dedupeSpans(chunks);
+		} finally {
+			this.releaseParser(parser);
+		}
+	}
+
+	private extractImports(source: string, language: string): { imports: string[]; rawSpecs: string[] } {
+		const lang = syncLanguageCache.get(language);
+		if (!lang) return { imports: [], rawSpecs: [] };
 		const parser = this.acquireParser();
 		try {
 			(parser as { setLanguage(l: unknown): void }).setLanguage(lang);
 			const tree = (parser as { parse(s: string): { rootNode: TsNode } }).parse(source);
 			const imports: string[] = [];
-			this.walkImports(tree.rootNode, language, imports);
-			return [...new Set(imports)];
+			const rawSpecs: string[] = [];
+			this.walkImports(tree.rootNode, language, imports, rawSpecs);
+			return { imports: [...new Set(imports)], rawSpecs: [...new Set(rawSpecs)] };
 		} finally {
 			this.releaseParser(parser);
 		}
 	}
 
-	private walkImports(node: TsNode, language: string, imports: string[]): void {
+	private walkImports(node: TsNode, language: string, imports: string[], rawSpecs: string[]): void {
 		if (IMPORT_TYPES.has(node.type)) {
 			const module = parseImportModule(node, language);
 			if (module) imports.push(module);
+			const raw = parseImportModuleRaw(node);
+			if (raw) rawSpecs.push(raw);
 		}
 		for (const child of node.children) {
-			this.walkImports(child, language, imports);
+			this.walkImports(child, language, imports, rawSpecs);
 		}
 	}
 }
@@ -185,6 +332,42 @@ function firstChildOfType(node: TsNode, types: string[]): TsNode | null {
 	return null;
 }
 
+/**
+ * `Class.method` / `functionName` for a chunk, from the first name-bearing
+ * child plus the enclosing chain. Used by the R1 context header and by the
+ * symbol path in retrieval (exact identifier hits rank by name, not vibes).
+ *
+ * The name child is picked by TYPE priority, not document order: for a Java
+ * method with a reference-type return (`Result compute()`), the return type
+ * is a `type_identifier` that appears BEFORE the method name — picking the
+ * first name-bearing child mislabeled the unit as `Result` and broke exact
+ * symbol lookup. Field/property identifiers win, plain identifiers come
+ * next, type identifiers only as a last resort.
+ */
+const NAME_TYPE_PRIORITY: Record<string, number> = {
+	field_identifier: 0,
+	property_identifier: 0,
+	identifier: 1,
+	name: 1,
+	type_identifier: 2,
+};
+
+function nodeUnit(node: TsNode, ancestors: string[]): string | null {
+	let best: { name: string; priority: number } | null = null;
+	for (const child of node.children) {
+		const priority = NAME_TYPE_PRIORITY[child.type];
+		if (priority === undefined) continue;
+		const text = child.text?.trim();
+		if (!text) continue;
+		if (!best || priority < best.priority) best = { name: text, priority };
+	}
+	const name = best?.name;
+	if (name && ancestors.length > 0) return [...ancestors, name].join(".");
+	if (name) return name;
+	if (ancestors.length > 0) return ancestors.join(".");
+	return null;
+}
+
 function firstChild(node: TsNode): TsNode | null {
 	return node.children[0] ?? null;
 }
@@ -201,6 +384,110 @@ function dedupeSpans(chunks: Chunk[]): Chunk[] {
 	return out;
 }
 
+// ── R12: ast-density helpers ─────────────────────────────────────────────────
+
+function nonWsLength(text: string): number {
+	let n = 0;
+	for (const ch of text) if (!/\s/.test(ch)) n++;
+	return n;
+}
+
+function offsetToLine(source: string, offset: number): number {
+	let line = 1;
+	for (let i = 0; i < offset && i < source.length; i++) if (source[i] === "\n") line++;
+	return line;
+}
+
+function classifyNode(node: TsNode): ChunkType {
+	if (FUNCTION_TYPES.has(node.type)) return "function";
+	if (CLASS_TYPES.has(node.type)) return "class";
+	return "doc";
+}
+
+/**
+ * Tile [rangeStart, rangeEnd) of `source` at node children boundaries: the
+ * gap before each child is attached to that child (leading comments travel
+ * with the declaration they precede), so the units cover the range with no
+ * holes. Oversized children recurse; children without useful children fall
+ * back to line windows. The cAST insight: syntax boundaries first, character
+ * windows only where syntax runs out.
+ */
+function splitNode(node: TsNode, source: string, budget: number, rangeStart: number, rangeEnd: number, parentUnit: string | null): DensityUnit[] {
+	const units: DensityUnit[] = [];
+	const kids = node.children;
+	if (kids.length === 0 || rangeEnd <= rangeStart) {
+		return splitByLines(source, rangeStart, rangeEnd, budget, parentUnit);
+	}
+	for (let i = 0; i < kids.length; i++) {
+		const kid = kids[i]!;
+		const segStart = i === 0 ? rangeStart : kids[i - 1]!.endIndex;
+		const segEnd = i === kids.length - 1 ? rangeEnd : kid.endIndex;
+		if (segEnd <= segStart) continue;
+		const text = source.slice(segStart, segEnd);
+		if (nonWsLength(text) <= budget) {
+			const unit = nodeUnit(kid, parentUnit ? [parentUnit] : []);
+			units.push({ start: segStart, end: segEnd, text, kind: classifyNode(kid), unit, parent: parentUnit, segment: 0 });
+		} else {
+			const ownName = nodeUnit(kid, []) ?? parentUnit;
+			units.push(...splitNode(kid, source, budget, segStart, segEnd, ownName));
+		}
+	}
+	// Re-number segments within this range so ids stay unique on sibling spans.
+	let seg = 0;
+	for (const u of units) {
+		if (u.parent === parentUnit) u.segment = seg++;
+	}
+	return units;
+}
+
+/** Leaf fallback: budget-sized line windows over [start, end). */
+function splitByLines(source: string, start: number, end: number, budget: number, parentUnit: string | null): DensityUnit[] {
+	const units: DensityUnit[] = [];
+	let unitStart = start;
+	let used = 0;
+	let pos = start;
+	while (pos < end) {
+		let nl = source.indexOf("\n", pos);
+		if (nl === -1 || nl >= end) nl = end - 1;
+		const lineEnd = nl + 1;
+		const line = source.slice(pos, lineEnd);
+		const lineNw = nonWsLength(line);
+		if (used > 0 && used + lineNw > budget) {
+			units.push({ start: unitStart, end: pos, text: source.slice(unitStart, pos), kind: "doc", unit: parentUnit, parent: parentUnit, segment: units.length });
+			unitStart = pos;
+			used = 0;
+		}
+		used += lineNw;
+		pos = lineEnd;
+	}
+	if (pos > unitStart) {
+		units.push({ start: unitStart, end: pos, text: source.slice(unitStart, pos), kind: "doc", unit: parentUnit, parent: parentUnit, segment: units.length });
+	}
+	return units;
+}
+
+/** Greedy-merge adjacent TRIVIA units while the combination stays within budget. Named units (functions/classes) never merge with siblings: the eval showed that gluing a small function into a budget-sized blob costs the symbol signal (metadata.unit) and dilutes the hash-embedding — R@10 dropped .936 → .772. Trivia (imports, comments, top-level statements) is what needs gluing to become self-contained. */
+function mergeUnits(units: DensityUnit[], source: string, budget: number): DensityUnit[] {
+	const isTrivia = (u: DensityUnit) => u.unit === null && u.kind === "doc";
+	const out: DensityUnit[] = [];
+	for (const u of units) {
+		const last = out[out.length - 1];
+		if (
+			last &&
+			last.end === u.start &&
+			isTrivia(last) &&
+			isTrivia(u) &&
+			nonWsLength(last.text) + nonWsLength(u.text) <= budget
+		) {
+			last.end = u.end;
+			last.text = source.slice(last.start, last.end);
+		} else {
+			out.push({ ...u });
+		}
+	}
+	return out;
+}
+
 /**
  * Normalize a JS/TS module specifier to a stable root segment.
  *
@@ -219,6 +506,7 @@ function normalizeModuleSpecifier(raw: string): string {
 	return segments[0]!;
 }
 
+/** Port of upstream `_parse_import_module`. */
 /** Port of upstream `_parse_import_module`. */
 function parseImportModule(node: TsNode, _language: string): string | null {
 	const textOf = (n: TsNode) => n.text;
@@ -256,6 +544,18 @@ function parseImportModule(node: TsNode, _language: string): string | null {
 	return null;
 }
 
+/**
+ * R7: the RAW specifier of a JS/TS string import (`./b.js`, not `b.js`).
+ * Import resolution in the pipeline needs it; returns null for syntaxes
+ * that carry no path-like specifier (Python dotted names, Rust paths).
+ */
+function parseImportModuleRaw(node: TsNode): string | null {
+	if (node.type !== "import_statement" && node.type !== "import_declaration") return null;
+	const stringChild = firstChildOfType(node, ["string"]);
+	if (!stringChild) return null;
+	return stringChild.text.replace(/^['"]|['"]$/g, "");
+}
+
 /** Sync cache of loaded Language objects (promises resolved during warmup). */
 const syncLanguageCache = new Map<string, unknown>();
 

package/src/indexer/context-header.ts

@@ -0,0 +1,86 @@
+/**
+ * Deterministic context headers for chunks — plan item R1
+ * (docs/IMPLEMENTATION-PLAN.md).
+ *
+ * Why: a chunk used to carry only its own text, so a query like "how does the
+ * retrieval module merge results" had no signal tying a hit to
+ * `src/retrieval/`. Anthropic's contextual retrieval measured failed retrievals
+ * dropping 49% from prepending this kind of path/unit context before embedding
+ * and before FTS ingest. Ours is free: one line, no LLM, built from data the
+ * indexer already has (path, language, enclosing unit, file imports).
+ *
+ * The header is prepended to `chunk.content` as the single source of truth, so
+ * the embedder, the FTS ingest, compression and `expand_chunk` all see it
+ * without four separate code paths. It is also indexed text: contextual BM25
+ * was the larger half of Anthropic's gain.
+ */
+import type { Chunk } from "../types.js";
+
+/** ~120 code tokens at 3.3 chars/token: context, never content. */
+export const HEADER_BUDGET_CHARS = 400;
+const MAX_IMPORTS = 8;
+const HEADER_PREFIX = "// file: ";
+
+export interface ContextHeaderInput {
+	filePath: string;
+	language: string;
+	unit?: string | null;
+	imports?: string[];
+}
+
+export function buildContextHeader(input: ContextHeaderInput): string {
+	const parts = [`file: ${input.filePath}`, `lang: ${input.language}`];
+	if (input.unit) parts.push(`unit: ${input.unit}`);
+	const imports = (input.imports ?? []).filter(Boolean);
+	if (imports.length > 0) {
+		let list = imports.slice(0, MAX_IMPORTS).join(", ");
+		if (imports.length > MAX_IMPORTS) list += `, +${imports.length - MAX_IMPORTS} more`;
+		parts.push(`imports: ${list}`);
+	}
+	let header = `// ${parts.join(" | ")}`;
+	if (header.length > HEADER_BUDGET_CHARS) {
+		// Drop imports first: paths and units carry more signal per character.
+		header = `// ${parts.filter((p) => !p.startsWith("imports:")).join(" | ")}`;
+		if (header.length > HEADER_BUDGET_CHARS) {
+			header = header.slice(0, HEADER_BUDGET_CHARS - 3) + "...";
+		}
+	}
+	return header;
+}
+
+/** Prepend a header. Idempotent so re-processing a chunk cannot double it. */
+export function applyContextHeader(content: string, input: ContextHeaderInput): string {
+	if (content.startsWith(HEADER_PREFIX)) return content;
+	return `${buildContextHeader(input)}\n${content}`;
+}
+
+/** Read the unit back out of a header-prefixed chunk (R2 symbol matching). */
+export function parseContextHeaderUnit(content: string): string | null {
+	if (!content.startsWith(HEADER_PREFIX)) return null;
+	const firstLine = content.slice(0, content.indexOf("\n") < 0 ? undefined : content.indexOf("\n"));
+	const match = firstLine.match(/\| unit: ([^|]+?)(?: \|.*|\s*)$/);
+	return match ? match[1]!.trim() : null;
+}
+
+/** Apply the header to a chunk, reading the unit the chunker recorded. */
+export function applyContextHeaderToChunk(chunk: Chunk, imports: string[]): void {
+	const unit = typeof chunk.metadata["unit"] === "string" ? (chunk.metadata["unit"] as string) : null;
+	chunk.content = applyContextHeader(chunk.content, {
+		filePath: chunk.filePath,
+		language: chunk.language,
+		unit,
+		imports,
+	});
+}
+
+/**
+ * Drop the context header line from stored chunk content (inverse of
+ * `applyContextHeader`). Used when re-deriving call-graph identifiers from
+ * backend-stored chunks, so header tokens (`file`, `unit`, `imports`, path
+ * segments) cannot pollute the identifier set.
+ */
+export function stripContextHeader(content: string): string {
+	if (!content.startsWith(HEADER_PREFIX)) return content;
+	const nl = content.indexOf("\n");
+	return nl === -1 ? "" : content.slice(nl + 1);
+}

package/src/indexer/embedder.ts

@@ -10,6 +10,8 @@ import os from "node:os";
 import { sha256Hex } from "../utils.js";
 
 export const DEFAULT_LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
+/** R11: code-trained local embedder, measured in README (not a default — see eval table). */
+export const RECOMMENDED_CODE_MODEL = "michaelfeil/jina-embeddings-v2-base-code";
 export const DEFAULT_OLLAMA_MODEL = "nomic-embed-text";
 export const DEFAULT_OLLAMA_URL = "http://localhost:11434";
 
@@ -31,6 +33,11 @@ function resolveModelCacheDir(): string {
 	return path.join(os.homedir(), ".pi-context-engine", "models");
 }
 
+/** R10: shared with the reranker so every transformers.js load uses one models dir. */
+export function transformersCacheDir(): string {
+	return resolveModelCacheDir();
+}
+
 interface TransformersModule {
 	pipeline: (
 		task: string,
@@ -94,8 +101,16 @@ export class LocalBackend implements EmbeddingBackend {
 					pooling: "mean",
 					normalize: true,
 				} as Record<string, unknown>);
+				// R11: transformers.js truncates to the MODEL's max length — up to 8192
+				// tokens for long-context embedders, which OOMs a batch on CPU. Cap the
+				// text ourselves; ~2000 chars ≈ 512 BPE tokens covers almost every chunk.
+				const MAX_EMBED_CHARS = 2000;
+				const cap = (s: string) => (s.length > MAX_EMBED_CHARS ? s.slice(0, MAX_EMBED_CHARS) : s);
 				return async (inputs: string | string[]): Promise<number[][]> => {
-					const out = (await extractor(inputs)) as { data: Float32Array; dims: number[] } | { data: Float32Array; dims: number[] }[];
+					const capped = typeof inputs === "string" ? cap(inputs) : inputs.map(cap);
+					const out = (await extractor(capped)) as
+						| { data: Float32Array; dims: number[] }
+						| { data: Float32Array; dims: number[] }[];
 					const single = Array.isArray(out) ? out[0]! : out;
 					const dim = single.dims[single.dims.length - 1]!;
 					const vecs: number[][] = [];

package/src/indexer/manifest.ts

@@ -5,7 +5,7 @@
 import fs from "node:fs";
 import { atomicWriteText } from "../utils.js";
 
-export const CURRENT_SCHEMA_VERSION = 2;
+export const CURRENT_SCHEMA_VERSION = 3;
 
 interface ManifestData {
 	__schema_version?: number;
@@ -16,7 +16,7 @@ interface ManifestData {
 
 export class Manifest {
 	private entries: Record<string, string> = {};
-	private schemaVersion: number = CURRENT_SCHEMA_VERSION;
+	private schemaVersionValue: number = CURRENT_SCHEMA_VERSION;
 	private lastGitShaValue: string | null = null;
 	private embeddingDimValue: number | null = null;
 
@@ -26,13 +26,13 @@ export class Manifest {
 				const loaded = JSON.parse(fs.readFileSync(manifestPath, "utf-8")) as ManifestData;
 				if (loaded && typeof loaded === "object") {
 					if ("__schema_version" in loaded) {
-						this.schemaVersion = loaded.__schema_version ?? 1;
+						this.schemaVersionValue = loaded.__schema_version ?? 1;
 						this.entries = loaded.files ?? {};
 						this.lastGitShaValue = loaded.last_git_sha ?? null;
 						this.embeddingDimValue = loaded.embedding_dim ?? null;
 					} else {
 						// Old plain-dict format (pre-v0.2) — treat as version 1.
-						this.schemaVersion = 1;
+						this.schemaVersionValue = 1;
 						this.entries = loaded as unknown as Record<string, string>;
 					}
 				}
@@ -48,6 +48,10 @@ export class Manifest {
 	set lastGitSha(value: string | null) {
 		this.lastGitShaValue = value;
 	}
+	/** Format version the on-disk manifest was written with (drives migrations). */
+	get schemaVersion(): number {
+		return this.schemaVersionValue;
+	}
 	get embeddingDim(): number | null {
 		return this.embeddingDimValue;
 	}