xindex 1.0.18 → 1.0.20

package/.mcp.json

@@ -1,9 +1,5 @@
 {
   "mcpServers": {
-    "fetch": {
-      "command": "docker",
-      "args": ["run", "-i", "--rm", "mcp/fetch"]
-    },
     "xindex": {
       "command": "xindex-mcp",
       "args": []

package/.xindex.json

@@ -105,5 +105,6 @@
   ],
   "maxLines": 30,
   "maxFileBytes": 100000,
-  "followSymlinks": false
+  "followSymlinks": false,
+  "searchDefaultLimit": 7
 }

package/apps/indexApp.ts

@@ -6,7 +6,7 @@ import {run} from "../packages/streamx/src/index.js";
 import {IWalkFiles} from "../components/walkFiles.js";
 import {IIndexContent} from "../features/indexContent.js";
 import {ILogger} from "../components/logger.js";
-import {INDEXING_BATCH_SIZE} from "../components/config/INDEXING_BATCH_SIZE";
+import {INDEXING_BATCH_SIZE} from "../components/config/INDEXING_BATCH_SIZE.js";
 import {SafeIndexBatch} from "../components/io/safeIndexBatch.js";
 
 export type IIndexApp = (inputs: string[]) => Promise<void>;

package/apps/mcpApp.ts

@@ -50,14 +50,28 @@ export function McpApp({
             inputSchema: z.object({
                 query: z.string()
                     .describe("Natural language search query"),
-                limit: z.number().int().min(1).max(50).default(7)
-                    .describe("Max results to return, 7 by default, 50 max"),
+                limit: z.number().int().min(1).max(50).default(config.searchDefaultLimit)
+                    .describe(`Max results to return default ${config.searchDefaultLimit}, max 50)`),
+                windowLines: z.number().int().min(1).default(config.maxLines).optional()
+                    .describe("Optional line-window size for per-file snippet locating"),
+                includePaths: z.array(z.string()).optional()
+                    .describe("Optional gitignore-style include patterns (e.g. 'src/**', '**/*.ts', '!src/vendor/**')"),
+                excludePaths: z.array(z.string()).optional()
+                    .describe("Optional gitignore-style exclude patterns (e.g. 'node_modules/**', '**/*.test.ts')"),
+                scoreThreshold: z.number().min(0).max(1).optional()
+                    .describe("Optional minimum semantic score threshold (0..1)"),
             }),
             annotations: {readOnlyHint: true},
-        }, async ({query, limit}) => {
+        }, async ({query, limit, scoreThreshold, windowLines, includePaths, excludePaths}) => {
             try {
                 const format = FormatSearchResults();
-                const results = await search(query, limit);
+                const results = await search(query, {
+                    limit,
+                    scoreThreshold,
+                    windowLines,
+                    includePaths,
+                    excludePaths,
+                });
                 const text = await format(query, results);
                 return {content: [{type: "text" as const, text}]};
             } catch (e) {

package/apps/run.mcp.ts

@@ -29,7 +29,7 @@ const indexApp = async (inputs: string[]) => {
     await rawIndexApp(inputs);
     await flush();
 };
-const search = SearchApp({searchContentIndex});
+const search = SearchApp({searchIndex: searchContentIndex, searchDefaultLimit: config.searchDefaultLimit});
 
 const appId = AppId();
 const watcherLock = WatcherLock({

package/apps/run.search.ts

@@ -4,8 +4,8 @@ import {SearchApp} from "./searchApp.js";
 import {FormatSearchResults} from "../components/index/formatSearchResults.js";
 
 const log = BufferedLoggerToStdOut();
-const {searchContentIndex} = await BuildComponents({log});
-const search = SearchApp({searchContentIndex});
+const {searchContentIndex, config} = await BuildComponents({log});
+const search = SearchApp({searchIndex: searchContentIndex, searchDefaultLimit: config.searchDefaultLimit});
 
 const query = process.argv.slice(2).join(" ");
 if (!query) {

package/apps/searchApp.ts

@@ -1,11 +1,23 @@
-import {ISearchIndex, IIndexRecord} from "../features/searchIndex.js";
+import {IIndexRecord, ISearchIndex, ISearchIndexOptions} from "../features/searchIndex.js";
 
-export type ISearchApp = (query: string, limit?: number) => Promise<IIndexRecord[]>;
+export type ISearchApp = (query: string, options?: Partial<ISearchIndexOptions>) => Promise<IIndexRecord[]>;
 
-export function SearchApp({searchContentIndex}: {
-    searchContentIndex: ISearchIndex;
+export function SearchApp({searchIndex, searchDefaultLimit = 7}: {
+    searchIndex: ISearchIndex;
+    searchDefaultLimit?: number;
 }): ISearchApp {
-    return async function search(query, limit = 7) {
-        return searchContentIndex(query, limit);
+    return async function search(query, options = {}) {
+        const normalizedOptions: ISearchIndexOptions = {
+            limit: options.limit ?? searchDefaultLimit,
+            prefetchMultiplier: options.prefetchMultiplier,
+            scoreThreshold: options.scoreThreshold,
+            locateInFiles: options.locateInFiles,
+            windowLines: options.windowLines,
+            useKeywordsSearch: options.useKeywordsSearch,
+            includePaths: options.includePaths ?? [],
+            excludePaths: options.excludePaths ?? [],
+        };
+
+        return searchIndex({query, options: normalizedOptions});
     }
 }

package/apps/watchApp.ts

@@ -10,7 +10,7 @@ import {ILogger} from "../components/logger.js";
 import {IWatcherLock} from "../components/index/watcherLock.js";
 import {WatchFileEventsApp} from "./watchFileEventsApp.js";
 import {IIndexContent} from "../features/indexContent.js";
-import {INDEXING_BATCH_SIZE} from "../components/config/INDEXING_BATCH_SIZE";
+import {INDEXING_BATCH_SIZE} from "../components/config/INDEXING_BATCH_SIZE.js";
 import {SafeIndexBatch} from "../components/io/safeIndexBatch.js";
 
 export type IWatchApp = {

package/components/buildComponents.ts

@@ -23,6 +23,7 @@ export async function BuildComponents({log, watchCoalesceMs = 0, indexingCoalesc
         extractKeywords,
         cleanUpKeywords,
         windowLines: config.maxLines,
+        embedConcurrency: config.searchConcurrency,
         maxFileBytes: config.maxFileBytes,
     });
 
@@ -38,6 +39,7 @@ export async function BuildComponents({log, watchCoalesceMs = 0, indexingCoalesc
         cleanUpKeywords,
         locateInFile,
         scoreThreshold: SCORE_THRESHOLD,
+        searchConcurrency: config.searchConcurrency,
         log,
         indexingWatchCoalesceMs: watchCoalesceMs,
         indexingCoalesceMs,

package/components/config/loadConfig.ts

@@ -3,6 +3,8 @@ import {IXindexConfig} from "./xindexConfig.js";
 import {ILogger} from "../logger.js";
 
 const DEFAULT_MAX_LINES = 30;
+const DEFAULT_SEARCH_DEFAULT_LIMIT = 7;
+const DEFAULT_SEARCH_CONCURRENCY = 4;
 const DEFAULT_MAX_FILE_BYTES = 50_000;
 const DEFAULT_FOLLOW_SYMLINKS = false;
 const DEFAULT_IGNORE_FILES = ['.xindex', 'node_modules'];
@@ -11,6 +13,8 @@ const DEFAULTS: IXindexConfig = {
     ignoreKeywords: [],
     ignoreFiles: DEFAULT_IGNORE_FILES,
     maxLines: DEFAULT_MAX_LINES,
+    searchDefaultLimit: DEFAULT_SEARCH_DEFAULT_LIMIT,
+    searchConcurrency: DEFAULT_SEARCH_CONCURRENCY,
     maxFileBytes: DEFAULT_MAX_FILE_BYTES,
     followSymlinks: DEFAULT_FOLLOW_SYMLINKS,
 };
@@ -45,6 +49,8 @@ export function LoadConfig({configPath, log}: { configPath: string, log: ILogger
             ignoreKeywords: toStrings(parsed.ignoreKeywords),
             ignoreFiles: toStrings(parsed.ignoreFiles),
             maxLines: toNum(parsed.maxLines, DEFAULT_MAX_LINES),
+            searchDefaultLimit: Math.max(1, toNum(parsed.searchDefaultLimit, DEFAULT_SEARCH_DEFAULT_LIMIT)),
+            searchConcurrency: Math.max(1, toNum(parsed.searchConcurrency, DEFAULT_SEARCH_CONCURRENCY)),
             maxFileBytes: toNum(parsed.maxFileBytes, DEFAULT_MAX_FILE_BYTES),
             followSymlinks: typeof parsed.followSymlinks === "boolean" ? parsed.followSymlinks : DEFAULT_FOLLOW_SYMLINKS,
         };

package/components/config/xindexConfig.ts

@@ -2,6 +2,8 @@ export type IXindexConfig = {
     ignoreKeywords: string[];
     ignoreFiles: string[];
     maxLines: number;
+    searchDefaultLimit: number;
+    searchConcurrency: number;
     maxFileBytes: number;
     followSymlinks: boolean;
 };

package/components/index/contentIndexDriver.ts

@@ -23,7 +23,7 @@ export type IContentIndexDriver = Readonly<{
 }>;
 
 export async function ContentIndexDriver({
-    path, embed, extractKeywords, cleanUpKeywords, locateInFile, scoreThreshold, log,
+    path, embed, extractKeywords, cleanUpKeywords, locateInFile, scoreThreshold, searchConcurrency, log,
     indexingWatchCoalesceMs = 0, indexingCoalesceMs = 0,
 }: {
     path: string,
@@ -32,6 +32,7 @@ export async function ContentIndexDriver({
     cleanUpKeywords: ICleanUpKeywords,
     locateInFile: ILocateInFile,
     scoreThreshold: number,
+    searchConcurrency: number,
     log: ILogger,
     indexingWatchCoalesceMs?: number,
     indexingCoalesceMs?: number,
@@ -62,7 +63,7 @@ export async function ContentIndexDriver({
         indexContentWatch: IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi: watchCoalesce, log}),
         indexContentBatch: IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi: batchCoalesce, log}),
         removeContent: RemoveContent({indexApi: drainingApi}),
-        searchContentIndex: SearchIndex({extractKeywords, cleanUpKeywords, embed, index, locateInFile, scoreThreshold}),
+        searchContentIndex: SearchIndex({extractKeywords, cleanUpKeywords, embed, index, locateInFile, scoreThreshold, searchConcurrency}),
         resetIndex: ResetIndex({indexApi: drainingApi}),
         flush: flushAll,
     };

package/components/keywords/cleanUpKeywords.ts

@@ -13,7 +13,16 @@ export function CleanUpKeywords({maxNgrams, minLength, ignoreKeywords = []}: {
     const ignoreSet = new Set(ignoreKeywords.map(normalize));
 
     return function cleanUpKeywords(keywords) {
-        const cleaned = keywords.join(" ").replace(/\W+/gm, " ").trim();
+        const joined = keywords.join(" ");
+        const expanded = joined + " " + splitIdentifiers(joined);
+        const tokens = expanded.replace(/[\W_]+/gm, " ").trim().split(/\s+/);
+        const seenTokens = new Set<string>();
+        const cleaned = tokens.filter(t => {
+            const k = t.toLowerCase();
+            if (!k || seenTokens.has(k)) return false;
+            seenTokens.add(k);
+            return true;
+        }).join(" ");
 
         const extracted: string[] = keyword_extractor.extract(cleaned, {
             language: "english",
@@ -25,14 +34,25 @@ export function CleanUpKeywords({maxNgrams, minLength, ignoreKeywords = []}: {
 
         const seen = new Set<string>();
 
-        return extracted
+        const output = extracted
             .map(normalize)
             .filter((kw: string) => {
-                if (kw.length <= minLength || !/[a-z]/i.test(kw)) return false;
+                // if (kw.length <= minLength || !/[a-z]/i.test(kw)) return false;
+                if (kw.length <= minLength) return false;
                 if (ignoreSet.has(kw)) return false;
                 if (seen.has(kw)) return false;
                 seen.add(kw);
                 return true;
             });
+
+        // console.log(`CleanUpKeywords: input=${JSON.stringify(keywords, null, 2)} output=${JSON.stringify(output, null, 2)}`);
+        return output;
     }
 }
+
+function splitIdentifiers(text: string): string {
+    return text
+        .replace(/[_\-.]+/g, " ")
+        .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+        .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+}

package/components/locate/locateInFile.ts

@@ -6,20 +6,20 @@ import {IIndexRecord} from "../../features/searchIndex.js";
 import {IInMemoryIndex, InMemoryIndex} from "./inMemoryIndex.js";
 import {IWindow, windowsOf} from "./windowsOf.js";
 import {Bm25, IBm25Doc, tokenizeForBm25} from "./bm25.js";
+import {DEFAULT_LOCATE_BATCH_SIZE} from "../config/DEFAULT_LOCATE_BATCH_SIZE.js";
 import {from} from "../../packages/streamx/src/from.js";
 import {filter} from "../../packages/streamx/src/filter.js";
 import {map} from "../../packages/streamx/src/map.js";
 import {flatMap} from "../../packages/streamx/src/flatMap.js";
-import {tap} from "../../packages/streamx/src/tap.js";
-import {run} from "../../packages/streamx/src/index.js";
-import {DEFAULT_LOCATE_BATCH_SIZE} from "../config/DEFAULT_LOCATE_BATCH_SIZE";
-import {scaleSync} from "../../packages/streamx/src/scaleSync";
+import {scaleSync} from "../../packages/streamx/src/scaleSync.js";
+import {toArray} from "../../packages/streamx/src/toArray.js";
 
 export type ILocateInFile = (
     query: string,
     queryVector: number[],
     candidates: IIndexRecord[],
     limit: number,
+    windowLinesOverride?: number,
 ) => Promise<IIndexRecord[]>;
 
 export function LocateInFile({
@@ -41,67 +41,70 @@ export function LocateInFile({
     type IWindowWithKeywords = IWindow & { keywords: string };
     type IWindowWithVector = IWindowWithKeywords & { vector: number[] };
 
-    return async function locateInFile(query, queryVector, candidates, limit) {
+    return async function locateInFile(query, queryVector, candidates, limit, windowLinesOverride) {
+        const effectiveWindowLines = windowLinesOverride ?? windowLines;
+        const effectiveConcurrency = Math.max(1, embedConcurrency);
         const memIndex: IInMemoryIndex = InMemoryIndex({dimensions: queryVector.length});
 
         try {
-            const ids = candidates.map(c => c.id);
+            const ids: string[] = await toArray(
+                from<IIndexRecord>(candidates)
+                    .pipe(scaleSync<IIndexRecord, string | null>(effectiveConcurrency, async (c) => {
+                        try {
+                            const s = await stat(c.id);
+                            if (!s.isFile()) return null;
+                            if (s.size > maxFileBytes) return null;
+                            return c.id;
+                        } catch {
+                            return null;
+                        }
+                    }))
+                    .pipe(filter((id: string | null) => id !== null))
+                    .pipe(map<string | null, string>((id) => id as string)),
+            );
             const bm25Docs: IBm25Doc[] = [];
             const metaById = new Map<string, { fileId: string; startLine: number; endLine: number; snippet: string }>();
 
-            const SCALE_FILE_READS = embedConcurrency;
-            const windows = from<string>(ids)
-                .pipe(filter(async (id: string) => {
-                    try {
-                        const s = await stat(id);
-                        if (!s.isFile()) return false;
-                        if (s.size > maxFileBytes) return false;
-                        return true;
-                    } catch {
-                        return false;
-                    }
-                }))
-                .pipe(scaleSync(SCALE_FILE_READS, async (id: string): Promise<{ id: string, text: string } | null> => {
-                    try {
-                        const text = await readFile(id, "utf8");
-                        return {id, text};
-                    } catch {
-                        return null;
-                    }
-                }))
-                .pipe(filter(async (r: { id: string, text: string } | null) => r !== null))
-                .pipe(flatMap((r: { id: string, text: string } | null): IWindow[] => {
-                    return windowsOf({text: r!.text, id: r!.id, windowLines});
-                }));
+            const withVectors: IWindowWithVector[] = await toArray(
+                from<string>(ids)
+                    .pipe(scaleSync<string, {id: string, text: string} | null>(effectiveConcurrency, async (id) => {
+                        try {
+                            const text = await readFile(id, "utf8");
+                            return {id, text};
+                        } catch {
+                            return null;
+                        }
+                    }))
+                    .pipe(filter((r: {id: string, text: string} | null) => r !== null))
+                    .pipe(flatMap((r: {id: string, text: string} | null): IWindow[] =>
+                        windowsOf({text: r!.text, id: r!.id, windowLines: effectiveWindowLines})))
+                    .pipe(map<IWindow, IWindowWithKeywords>((w: IWindow) => {
+                        const kw = cleanUpKeywords(extractKeywords(w.snippet)).join(", ");
+                        return {...w, keywords: kw || w.snippet.slice(0, 200)};
+                    }))
+                    .pipe(scaleSync<IWindowWithKeywords, IWindowWithVector>(effectiveConcurrency, async (w) => {
+                        const vector = await embed(w.keywords);
+                        return {...w, vector};
+                    })),
+            );
 
-            const withVectors = from<IWindow>(windows)
-                .pipe(map<IWindow, IWindowWithKeywords>(w => {
-                    const kw = cleanUpKeywords(extractKeywords(w.snippet)).join(", ");
-                    return {...w, keywords: kw || w.snippet.slice(0, 200)};
-                }))
-                .pipe(scaleSync<IWindowWithKeywords, IWindowWithVector>(embedConcurrency, async (w) => {
-                    const vector = await embed(w.keywords);
-                    return {...w, vector};
-                }))
-                .pipe(tap(async (w: IWindowWithVector) => {
-                    const id = `${w.fileId}:${w.startLine}:${w.endLine}`;
-                    await memIndex.upsertItem(
-                        id,
-                        w.vector,
-                        {fileId: w.fileId, startLine: w.startLine, endLine: w.endLine, snippet: w.snippet},
-                    );
-                    bm25Docs.push({id, tokens: tokenizeForBm25(w.snippet)});
-                    metaById.set(id, {
-                        fileId: w.fileId,
-                        startLine: w.startLine,
-                        endLine: w.endLine,
-                        snippet: w.snippet
-                    });
-                }));
-
-            await run(withVectors);
+            for (const w of withVectors) {
+                const id = `${w.fileId}:${w.startLine}:${w.endLine}`;
+                await memIndex.upsertItem(
+                    id,
+                    w.vector,
+                    {fileId: w.fileId, startLine: w.startLine, endLine: w.endLine, snippet: w.snippet},
+                );
+                bm25Docs.push({id, tokens: tokenizeForBm25(w.snippet)});
+                metaById.set(id, {
+                    fileId: w.fileId,
+                    startLine: w.startLine,
+                    endLine: w.endLine,
+                    snippet: w.snippet,
+                });
+            }
 
-            const poolSize = Math.max(limit * 8, 40);
+            const poolSize = Math.min(limit * 10, 100);
             const vecHits = await memIndex.query(queryVector, query, poolSize);
 
             const bm25 = Bm25({docs: bm25Docs});
@@ -151,3 +154,4 @@ export function LocateInFile({
         }
     };
 }
+

package/components/locate/windowsOf.ts

@@ -12,7 +12,9 @@ export function windowsOf({text, id, windowLines}: {
 }): IWindow[] {
     const lines = text.split("\n");
     const windows: IWindow[] = [];
-    const scales = [Math.max(1, Math.floor(windowLines / 2)), windowLines];
+    const a12 = Math.max(1, Math.floor(windowLines / 2));
+    // const a13 = Math.max(1, Math.floor(windowLines / 3));
+    const scales = [a12, windowLines];
     for (const size of scales) {
         const step = Math.max(1, Math.floor(size / 2));
         for (let i = 0; i < lines.length; i += step) {

package/features/searchIndex.ts

@@ -1,8 +1,10 @@
 import {LocalIndex} from "vectra";
+import ignore from "ignore";
 import {IEmbed} from "../components/llm/embed.js";
 import {IExtractKeywords} from "../components/keywords/extractKeywords.js";
 import {ICleanUpKeywords} from "../components/keywords/cleanUpKeywords.js";
 import {ILocateInFile} from "../components/locate/locateInFile.js";
+import {Concurrency} from "../packages/fun/src/concurrency.js";
 
 export type IIndexRecord = {
     score: number;
@@ -15,25 +17,66 @@ export type IIndexRecord = {
     endLine?: number;
 };
 
-export type ISearchIndex = (query: string, limit: number) => Promise<IIndexRecord[]>;
+export type ISearchIndexOptions = {
+    limit: number;
+    prefetchMultiplier?: number;
+    scoreThreshold?: number;
+    locateInFiles?: boolean;
+    windowLines?: number;
+    useKeywordsSearch?: boolean;
+    includePaths?: string[];
+    excludePaths?: string[];
+};
+
+export type ISearchIndexQuery = Readonly<{
+    query: string;
+    options: ISearchIndexOptions;
+}>;
+export type ISearchIndex = (query: ISearchIndexQuery) => Promise<IIndexRecord[]>;
 
-export function SearchIndex({extractKeywords, cleanUpKeywords, embed, index, locateInFile, scoreThreshold = 0.05}: {
+export function SearchIndex({
+                                extractKeywords,
+                                cleanUpKeywords,
+                                embed,
+                                index,
+                                locateInFile,
+                                searchConcurrency = 3,
+                                scoreThreshold = 0.05
+                            }: {
     extractKeywords: IExtractKeywords,
     cleanUpKeywords: ICleanUpKeywords,
     embed: IEmbed,
     index: LocalIndex,
     locateInFile: ILocateInFile,
     scoreThreshold: number
+    searchConcurrency?: number
 }): ISearchIndex {
-    return async function searchContentIndex(query, limit) {
-        const keywords = cleanUpKeywords(extractKeywords(query));
-        const searchText = keywords.length > 0 ? keywords.join(", ") : query;
+    const worker: ISearchIndex = async function searchContentIndex({
+                                                                       query, options: {
+            limit,
+            prefetchMultiplier = 3,
+            scoreThreshold: threshold = scoreThreshold,
+            locateInFiles = true,
+            windowLines,
+            useKeywordsSearch = false,
+            includePaths = [],
+            excludePaths = [],
+        }
+                                                                   }: ISearchIndexQuery) {
+        const keywords = useKeywordsSearch ? cleanUpKeywords(extractKeywords(query)) : [];
+        const searchText = useKeywordsSearch && keywords.length > 0 ? keywords.join(", ") : query;
         const vector = await embed(searchText);
+        const includeMatcher = createGitignoreMatcher(includePaths);
+        const excludeMatcher = createGitignoreMatcher(excludePaths);
 
-        const results = await index.queryItems(vector, searchText, limit * 2);
+        const expectedMultiplier = limit * prefetchMultiplier;
+        const pathPatternBoost = Math.max(1, includePaths.length + excludePaths.length);
+        const results = await index.queryItems(vector, searchText, expectedMultiplier + pathPatternBoost);
 
         const candidates = results
-            .filter(r => r.score >= scoreThreshold)
+            .filter(r => r.score >= threshold)
+            .filter(r => includePaths.length === 0 || includeMatcher(r.item.id))
+            .filter(r => excludePaths.length === 0 || !excludeMatcher(r.item.id))
             .sort((a, b) => b.score - a.score)
             .map(r => ({
                 score: r.score,
@@ -41,10 +84,29 @@ export function SearchIndex({extractKeywords, cleanUpKeywords, embed, index, loc
                 keywords: typeof r.item.metadata?.keywords === "string" ? r.item.metadata.keywords : "",
             }));
 
-        if (!locateInFile || candidates.length === 0) {
+        if (!locateInFile || !locateInFiles || candidates.length === 0) {
             return candidates;
         }
 
-        return locateInFile(query, vector, candidates, limit);
+        return locateInFile(query, vector, candidates, limit, windowLines);
+    };
+
+    const control = Concurrency(searchConcurrency, worker);
+
+    return async (input) => {
+        const resolve = await control(input);
+        return await resolve();
+    };
+}
+
+function createGitignoreMatcher(patterns: string[]): (path: string) => boolean {
+    const ig = ignore();
+    for (const pattern of patterns) {
+        ig.add(pattern);
     }
+    return (path: string) => ig.ignores(normalizePath(path));
+}
+
+function normalizePath(path: string): string {
+    return path.replaceAll("\\", "/").replace(/^\.\/+/, "");
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xindex",
-  "version": "1.0.18",
+  "version": "1.0.20",
   "description": "Local semantic code search — index codebase, search by meaning or keywords",
   "type": "module",
   "main": "xindex.ts",
@@ -21,7 +21,7 @@
     "mcp": "tsx apps/run.mcp.ts",
     "watch": "tsx apps/run.watch.ts",
     "test.functional": "bash test/functional.sh",
-    "test.compilation": "npx -y tsc --ignoreConfig --noEmit --target ES2022 --module ESNext --moduleResolution bundler --esModuleInterop --skipLibCheck --strict false $(git ls-files '*.ts')",
+    "test.compilation": "tsc --ignoreConfig --noEmit --target ES2022 --module ESNext --moduleResolution bundler --esModuleInterop --skipLibCheck --strict false $(git ls-files '*.ts') && tsc -p tsconfig.json --noEmit",
     "test.npx": "docker run --rm -it -w /tmp node:22 bash -c 'npm i -g xindex && xindex-index tsx-0 && xindex-search streamx map | grep \"await mapper\" && which xindex | grep bin/xindex' ",
     "console": "docker run --rm -it -v \"$PWD:/app\" -w /app node:22 bash"
   },

package/packages/fun/src/array-finder.ts

@@ -1,4 +1,4 @@
-import { sortObject } from './sort-object';
+import { sortObject } from './sort-object.js';
 
 export function ArrayFinder<Type, ID>(
   items: Type[] | ReadonlyArray<Type>,

package/packages/fun/src/array-index.ts

@@ -1,4 +1,4 @@
-import { sortObject } from './sort-object';
+import { sortObject } from './sort-object.js';
 
 export function ArrayIndex<Type, ID>(
   items: Type[] | ReadonlyArray<Type>,

package/packages/fun/src/asyncRequest.ts

@@ -1,4 +1,4 @@
-import { Defer, IDefer } from './defer';
+import { Defer, IDefer } from './defer.js';
 
 export type IAsyncRequest<Request = any, Response = void, Cancellation = void> = Omit<
   IDefer<Response>,

package/packages/fun/src/concurrency.ts

@@ -1,7 +1,7 @@
-import { Counter, ICounter } from './counter';
-import { Defer, IDefer } from './defer';
-import { IValue, Value } from './value';
-import { waitForCounter, waitForZeroCounter } from './waitForCounter';
+import { Counter, ICounter } from './counter.js';
+import { Defer, IDefer } from './defer.js';
+import { IValue, Value } from './value.js';
+import { waitForCounter, waitForZeroCounter } from './waitForCounter.js';
 
 export type IStopKeepConcurrency = () => Promise<void>;
 
@@ -168,7 +168,7 @@ export function KeyedConcurrency<Input, Output = any>(
         const keyedControl = Concurrency<Input>(workerConcurrency, worker);
         registry[key] = keyedControl;
 
-        const unsubscribe = keyedControl.quantity.subscribe(value => {
+        const unsubscribe = keyedControl.quantity.subscribe((value: number) => {
           if (value <= 0) {
             unsubscribe();
 

package/packages/fun/src/counter.ts

@@ -1,4 +1,4 @@
-import { IPubSubSubscribe, SyncPubSub } from './pubsub';
+import { IPubSubSubscribe, SyncPubSub } from './pubsub.js';
 
 export type ICounter = {
   increment: (n?: number) => number;

package/packages/fun/src/flatten.ts

@@ -1,11 +1,17 @@
-import { isIterable } from './isIterable';
+import { isIterable } from './isIterable.js';
 
 export function flatten<T>(items: T | T[] | Iterable<T>): T[] {
-  if (Array.isArray(items) || isIterable(items)) {
-    return [...items].reduce((acc, item) => {
+  if (Array.isArray(items)) {
+    return items.reduce<T[]>((acc, item) => {
       return [...acc, ...flatten(item)];
-    }, [] as T[]);
-  } else {
-    return [items];
+    }, []);
   }
+
+  if (isIterable(items)) {
+    return [...items].reduce<T[]>((acc, item) => {
+      return [...acc, ...flatten(item)];
+    }, []);
+  }
+
+  return [items];
 }

package/packages/fun/src/hash128.ts

@@ -1,5 +1,5 @@
-import { hashHex } from './hash';
-import { sortObject } from './sort-object';
+import { hashHex } from './hash.js';
+import { sortObject } from './sort-object.js';
 
 export function hash128(input: any) {
   return hashHex(JSON.stringify(sortObject(input)), 128);

package/packages/fun/src/hash256.ts

@@ -1,5 +1,5 @@
-import { hashHex } from './hash';
-import { sortObject } from './sort-object';
+import { hashHex } from './hash.js';
+import { sortObject } from './sort-object.js';
 
 export function hash256(input: any) {
   return hashHex(JSON.stringify(sortObject(input)), 256);

package/packages/fun/src/hub.ts

@@ -1,4 +1,4 @@
-import { IPubSub, IPubSubSubscriber, IPubSubUnsubscribe, PubSub } from './pubsub';
+import { IPubSub, IPubSubSubscriber, IPubSubUnsubscribe, PubSub } from './pubsub.js';
 
 export type IHubTopicName = string | any;
 

package/packages/fun/src/interval.ts

@@ -1,4 +1,4 @@
-import { Defer, IDefer } from './defer';
+import { Defer, IDefer } from './defer.js';
 
 export type IFinishInterval = () => Promise<void>;
 

package/packages/fun/src/mailbox.ts

@@ -1,4 +1,4 @@
-import { IPubSub, PubSub } from './pubsub';
+import { IPubSub, PubSub } from './pubsub.js';
 
 export type IMailbox<Inbox, Outbox = never> = Readonly<{
   inbox: IPubSub<Inbox>;

package/packages/fun/src/match-left-and-right-arrays.ts

@@ -1,4 +1,4 @@
-import { sortObject } from './sort-object';
+import { sortObject } from './sort-object.js';
 
 export type IMatchLeftAndRightArraysResult<Left, Right> = {
   matched: { left: Left; right: Right }[];

package/packages/fun/src/memos.ts

@@ -1,4 +1,4 @@
-import { IMem, mem } from './mem';
+import { IMem, mem } from './mem.js';
 
 export type IMemos<Input, Output> = ((value: Input) => Output) & {
   reset: () => void;

package/packages/fun/src/pubsub.ts

@@ -1,5 +1,5 @@
-import { Defer } from "./defer";
-import { IType, IUseType } from "./IType";
+import { Defer } from "./defer.js";
+import { IType, IUseType } from "./IType.js";
 
 export type ISyncPubSub<T> = {
   publish: (message: T) => void;

package/packages/fun/src/tick.ts

@@ -1,4 +1,4 @@
-import { Defer } from './defer';
+import { Defer } from './defer.js';
 
 type ITickSyncRunner = () => any;
 type ITickRunner = () => Promise<any> | any;

package/packages/fun/src/time-behavior.ts

@@ -1,4 +1,4 @@
-import { IValue, Value } from './value';
+import { IValue, Value } from './value.js';
 
 export function throttleTrailing<T>(fn: Function, ms: IValue<number>): () => void {
   let scheduled = false;

package/packages/fun/src/timedFallback.ts

@@ -1,4 +1,4 @@
-import { Defer } from './defer';
+import { Defer } from './defer.js';
 
 export type MayBeAsync<Type = any> = Type | Promise<Type>;
 

package/packages/fun/src/value.ts

@@ -1,4 +1,4 @@
-import { ISyncPubSub, SyncPubSub } from './pubsub';
+import { ISyncPubSub, SyncPubSub } from './pubsub.js';
 
 const ValueSymbol = Symbol('ValueSymbol');
 type IValueSymbol = typeof ValueSymbol;

package/packages/fun/src/waitForCounter.ts

@@ -1,5 +1,5 @@
-import { ICounter } from './counter';
-import { waitForMessage } from './pubsub';
+import { ICounter } from './counter.js';
+import { waitForMessage } from './pubsub.js';
 
 export async function waitForCounter(
   counter: ICounter,

package/packages/streamx/src/batch.ts

@@ -1,8 +1,8 @@
-import { StreamX, StreamXMapper } from './index';
+import { StreamX, StreamXMapper } from './index.js';
 
 export function batch<Input>(size: number): StreamXMapper<Input, Input[]> {
   let batched: Input[] = [];
-  return inputStream =>
+  return (inputStream: StreamX<Input>) =>
     (async function* batchedStream(): StreamX<Input[]> {
       for await (const record of inputStream) {
         batched.push(record);

package/packages/streamx/src/batchTimed.ts

@@ -1,7 +1,7 @@
-import { StreamXMapper } from './index';
-import { IRead, read } from './reader';
-import { IWriter, Writer } from './writer';
-import { syncTick } from '../../fun/src/tick';
+import { StreamXMapper } from './index.js';
+import { IRead, read } from './reader.js';
+import { IWriter, Writer } from './writer.js';
+import { syncTick } from '../../fun/src/tick.js';
 import { clearTimeout } from 'timers';
 
 type Milliseconds = number;
@@ -41,7 +41,7 @@ export function batchTimed<Input>(
     }
   }
 
-  return inputStream => {
+  return (inputStream: AsyncIterable<Input>) => {
     return {
       [Symbol.asyncIterator](): AsyncIterator<Input[]> {
         return {

package/packages/streamx/src/buffer.ts

@@ -1,7 +1,7 @@
-import { StreamX, StreamXMapper } from './index';
-import { IRead, read } from './reader';
-import { IWriter, Writer } from './writer';
-import { syncTick } from '../../fun/src/tick';
+import { StreamX, StreamXMapper } from './index.js';
+import { IRead, read } from './reader.js';
+import { IWriter, Writer } from './writer.js';
+import { syncTick } from '../../fun/src/tick.js';
 
 export function buffer<Input>(size: number): StreamXMapper<Input, Input> {
   let outputBuffer: IWriter<Input>;

package/packages/streamx/src/concatenate.ts

@@ -1,4 +1,4 @@
-import { StreamX } from './index';
+import { StreamX } from './index.js';
 
 export function concatenate<T>(...streams: StreamX<T>[]): StreamX<T>;
 

package/packages/streamx/src/filter.ts

@@ -1,9 +1,9 @@
-import { Promised, StreamX, StreamXMapper } from './index';
+import { Promised, StreamX, StreamXMapper } from './index.js';
 
 export function filter<Input>(
   condition: (input: Input) => Promised<boolean | undefined | null>
 ): StreamXMapper<Input, Input> {
-  return inputStream =>
+  return (inputStream: StreamX<Input>) =>
     (async function* filtered(): StreamX<Input> {
       for await (const record of inputStream) {
         if (await condition(record)) {

package/packages/streamx/src/flat.ts

@@ -1,4 +1,4 @@
-import { StreamX, StreamXLike, StreamXMapper } from './index';
+import { StreamX, StreamXLike, StreamXMapper } from './index.js';
 
 export function flat<Type>(): StreamXMapper<Type | StreamXLike<Type>, Type> {
   return inputStream =>

package/packages/streamx/src/flatMap.ts

@@ -1,6 +1,6 @@
-import { flat } from './flat';
-import { map } from './map';
-import { pipe, Promised, StreamXLike, StreamXMapper } from './index';
+import { flat } from './flat.js';
+import { map } from './map.js';
+import { pipe, Promised, StreamXLike, StreamXMapper } from './index.js';
 
 export function flatMap<Input, Output>(
   mapper: (input: Input) => Promised<Output | StreamXLike<Output>>

package/packages/streamx/src/from.ts

@@ -1,4 +1,4 @@
-import { of, StreamX, StreamXLike, StreamXOf } from './index';
+import { of, StreamX, StreamXLike, StreamXOf } from './index.js';
 
 export function isStreamXLike(stream: any): stream is StreamXLike<any> {
   return (

package/packages/streamx/src/interval.ts

@@ -1,6 +1,6 @@
-import { Defer } from '../../fun/src/defer';
-import { reader } from './reader';
-import { StreamX } from './index';
+import { Defer } from '../../fun/src/defer.js';
+import { reader } from './reader.js';
+import { StreamX } from './index.js';
 
 export type IInterval = StreamX<number> & {
   stop: () => void;

package/packages/streamx/src/loop.ts

@@ -1,5 +1,5 @@
-import { Promised, StreamX } from './index';
-import { reader } from './reader';
+import { Promised, StreamX } from './index.js';
+import { reader } from './reader.js';
 
 export function loop(condition: () => Promised<boolean>): StreamX<true> {
   return reader<true>(async () => {

package/packages/streamx/src/map.ts

@@ -1,9 +1,9 @@
-import { Promised, StreamX, StreamXMapper } from './index';
+import { Promised, StreamX, StreamXMapper } from './index.js';
 
 export function map<Input, Output>(
   mapper: (input: Input) => Promised<Output>
 ): StreamXMapper<Input, Output> {
-  return inputStream =>
+  return (inputStream: StreamX<Input>) =>
     (async function* mappedStream(): StreamX<Output> {
       for await (const record of inputStream) {
         yield await mapper(record);

package/packages/streamx/src/merge.ts

@@ -1,7 +1,7 @@
-import { StreamX } from './index';
-import { IRead, read } from './reader';
-import { IWriter, Writer } from './writer';
-import { syncTick } from '../../fun/src/tick';
+import { StreamX } from './index.js';
+import { IRead, read } from './reader.js';
+import { IWriter, Writer } from './writer.js';
+import { syncTick } from '../../fun/src/tick.js';
 
 export function merge<T1, T2, T3, T4>(
   stream1: StreamX<T1>,

package/packages/streamx/src/nodeReadable.ts

@@ -1,4 +1,4 @@
-import { of, StreamXOf } from './index';
+import { of, StreamXOf } from './index.js';
 import { Readable } from 'stream';
 
 export function nodeReadable<Output>(readable: Readable): StreamXOf<Output> {

package/packages/streamx/src/nodeTransform.ts

@@ -1,9 +1,9 @@
-import { of, StreamXMapper } from './index';
+import { of, StreamX, StreamXMapper } from './index.js';
 import { Readable, ReadableOptions, Transform } from 'stream';
 
 export function nodeTransform<Input, Output>(
   transform: Transform,
   options: ReadableOptions = {}
 ): StreamXMapper<Input, Output> {
-  return inputStream => of(Readable.from(inputStream, options).pipe(transform));
+  return (inputStream: StreamX<Input>) => of(Readable.from(inputStream, options).pipe(transform));
 }

package/packages/streamx/src/nodeWritable.ts

@@ -1,12 +1,12 @@
-import { StreamX, StreamXMapper } from './index';
+import { StreamX, StreamXMapper } from './index.js';
 import { Writable } from 'stream';
-import { Defer } from '../../fun/src/defer';
+import { Defer } from '../../fun/src/defer.js';
 
 export function nodeWritable<Type>(
   writable: Writable,
   encoding: BufferEncoding = 'utf-8'
 ): StreamXMapper<Type, Type> {
-  return inputStream =>
+  return (inputStream: StreamX<Type>) =>
     (async function* _nodeWritable(): StreamX<Type> {
       const deferEnd = Defer<void>();
       writable.on('error', error => deferEnd.reject(error));

package/packages/streamx/src/objectReader.ts

@@ -1,5 +1,5 @@
-import { Promised, StreamX } from './index';
-import { reader } from './reader';
+import { Promised, StreamX } from './index.js';
+import { reader } from './reader.js';
 
 export function objectReader<T extends object | object[]>(
   read: () => Promised<T | null | undefined | boolean | number>

package/packages/streamx/src/reader.ts

@@ -1,4 +1,4 @@
-import { Promised, StreamX } from './index';
+import { Promised, StreamX } from './index.js';
 
 const DONE = Symbol('DONE');
 

package/packages/streamx/src/reduce.ts

@@ -1,10 +1,10 @@
-import { Promised, StreamX, StreamXMapper } from './index';
+import { Promised, StreamX, StreamXMapper } from './index.js';
 
 export function reduce<Input, Accumulator>(
   reducer: (accumulator: Accumulator, input: Input) => Promised<Accumulator>,
   initial: Accumulator
 ): StreamXMapper<Input, Accumulator> {
-  return inputStream =>
+  return (inputStream: StreamX<Input>) =>
     (async function* reduced(): StreamX<Accumulator> {
       let finalValue: Accumulator = initial;
       for await (const input of inputStream) {

package/packages/streamx/src/scale.ts

@@ -1,8 +1,8 @@
-import { Promised, StreamXMapper } from './index';
-import { IRead, read } from './reader';
-import { IWriter, Writer } from './writer';
-import { Concurrency, IPublishToConcurrency } from '../../fun/src/concurrency';
-import { syncTick } from '../../fun/src/tick';
+import { Promised, StreamX, StreamXMapper } from './index.js';
+import { IRead, read } from './reader.js';
+import { IWriter, Writer } from './writer.js';
+import { Concurrency, IPublishToConcurrency } from '../../fun/src/concurrency.js';
+import { syncTick } from '../../fun/src/tick.js';
 
 export function scale<Input, Output>(
   max: number,
@@ -24,7 +24,7 @@ export function scale<Input, Output>(
     }
   }
 
-  return inputStream => {
+  return (inputStream: StreamX<Input>) => {
     return {
       [Symbol.asyncIterator](): AsyncIterator<Output> {
         return {
@@ -41,7 +41,7 @@ export function scale<Input, Output>(
               outputBuffer = Writer<Output>();
 
               if (!concurrencyControl) {
-                concurrencyControl = Concurrency<Input>(max, async input => {
+                concurrencyControl = Concurrency<Input>(max, async (input: Input) => {
                   try {
                     const output = await mapper(input);
                     await outputBuffer.write(output);

package/packages/streamx/src/scaleSync.ts

@@ -1,13 +1,13 @@
-import { pipe, Promised, StreamXMapper } from './index';
-import { batch } from './batch';
-import { map } from './map';
-import { flat } from './flat';
+import { pipe, Promised, StreamXMapper } from './index.js';
+import { batch } from './batch.js';
+import { map } from './map.js';
+import { flat } from './flat.js';
 
 export function scaleSync<Input, Output>(
   size: number,
   mapper: (input: Input) => Promised<Output>
 ): StreamXMapper<Input, Output> {
   return pipe(batch<Input>(size))
-    .pipe(map(values => Promise.all(values.map(mapper))))
+    .pipe(map((values: Input[]) => Promise.all(values.map(mapper))))
     .pipe(flat());
 }

package/packages/streamx/src/sequence.ts

@@ -1,4 +1,4 @@
-import { StreamX } from './index';
+import { StreamX } from './index.js';
 
 export async function* sequence(length: number): StreamX<number> {
   for (let i = 0; i < length; i++) {

package/packages/streamx/src/tap.ts

@@ -1,8 +1,8 @@
-import { map } from './map';
-import { Promised, StreamXMapper } from './index';
+import { map } from './map.js';
+import { Promised, StreamXMapper } from './index.js';
 
 export function tap<Input>(fn: (input: Input) => Promised<any>): StreamXMapper<Input, Input> {
-  return map<Input, Input>(async (input): Promise<Input> => {
+  return map<Input, Input>(async (input: Input): Promise<Input> => {
     await fn(input);
     return input;
   });

package/packages/streamx/src/toArray.ts

@@ -1,4 +1,4 @@
-import { StreamX } from './index';
+import { StreamX } from './index.js';
 
 export async function toArray<T>(input: StreamX<T>): Promise<T[]> {
   const values: T[] = [];

package/packages/streamx/src/writer.ts

@@ -1,7 +1,7 @@
-import { Defer, IDefer } from './defer';
-import { of, StreamX } from './index';
-import { reader } from './reader';
-import { flat } from './flat';
+import { Defer, IDefer } from './defer.js';
+import { of, StreamX } from './index.js';
+import { reader } from './reader.js';
+import { flat } from './flat.js';
 
 export type IWriter<T> = {
   stream: StreamX<T>;

package/tsconfig.json

@@ -9,6 +9,6 @@
     "rootDir": ".",
     "skipLibCheck": true
   },
-  "include": ["*.ts"],
+  "include": ["*.ts", "apps/**/*.ts", "components/**/*.ts", "features/**/*.ts", "packages/**/*.ts"],
   "exclude": ["node_modules", "dist"]
 }

package/components/index/documentContentIndexDriver.ts

@@ -1,127 +0,0 @@
-import {rm} from "node:fs/promises";
-import {extname} from "node:path";
-import type {IContentIndexDriver} from "./contentIndexDriver.js";
-import type {IGetIndexStats} from "./getIndexStats.js";
-import type {IIndexContent} from "../../features/indexContent.js";
-import type {IRemoveContent} from "../../features/removeContent.js";
-import type {IResetIndex} from "../../features/resetIndex.js";
-import type {IIndexRecord, ISearchIndex} from "../../features/searchIndex.js";
-import {DocumentIndex} from "./documentIndex.js";
-
-export async function DocumentContentIndexDriver({path, model}: {
-    path: string,
-    model?: string,
-}): Promise<IContentIndexDriver> {
-    const docsPath = `${path}/docs`;
-    let index = await DocumentIndex({path: docsPath, model});
-
-    const indexContent: IIndexContent = async items => {
-        for (const item of items) {
-            await index.upsertDocument(item.id, item.content, docTypeForPath(item.id), {});
-        }
-    };
-
-    const removeContent: IRemoveContent = async ids => {
-        for (const id of ids) {
-            await index.deleteDocument(id);
-        }
-    };
-
-    const searchContentIndex: ISearchIndex = async (query, limit) => {
-        const results = await index.queryDocuments(query, {
-            maxDocuments: limit,
-            maxChunks: 5,
-            isBm25: true,
-        });
-
-        const out: IIndexRecord[] = [];
-
-        for (const r of results) {
-            const text = await r.loadText();
-            const lineStarts = computeLineStarts(text);
-
-            for (const c of r.chunks) {
-                const {startPos, endPos} = c.item.metadata;
-                const snippet = text.slice(startPos, endPos);
-                const startLine = lineOf(lineStarts, startPos);
-                const endLine = lineOf(lineStarts, endPos);
-
-                out.push({
-                    id: r.uri,
-                    score: c.score,
-                    startPos,
-                    endPos,
-                    snippet,
-                    startLine,
-                    endLine,
-                });
-            }
-        }
-
-        return out;
-    };
-
-    const resetIndex: IResetIndex = async () => {
-        /** Clears persisted catalog + chunks on disk; recreates empty index metadata. */
-        await rm(docsPath, {recursive: true, force: true});
-
-        /*
-         * Re-bind `index`: after `rm`, the prior LocalDocumentIndex instance may retain stale
-         * in-memory caches; `DocumentIndex` constructs a fresh instance against the recreated folder.
-         */
-        index = await DocumentIndex({path: docsPath, model});
-    };
-
-    const getIndexStats: IGetIndexStats = async () => {
-        const stats = await index.getCatalogStats();
-
-        return {indexedAmount: stats.documents ?? 0};
-    };
-
-    const flush = async () => {
-        /* LocalDocumentIndex persists updates without an explicit flush. */
-    };
-
-    return {
-        getIndexStats,
-        indexContentWatch: indexContent,
-        indexContentBatch: indexContent,
-        removeContent,
-        searchContentIndex,
-        resetIndex,
-        flush,
-    };
-}
-
-const DOC_TYPE_BY_EXT: Record<string, string> = {
-    ".ts": "ts", ".tsx": "tsx", ".mts": "ts", ".cts": "ts",
-    ".js": "js", ".jsx": "jsx", ".mjs": "js", ".cjs": "js",
-    ".py": "python",
-    ".go": "go",
-    ".java": "java",
-    ".cs": "csharp",
-    ".cpp": "cpp", ".cc": "cpp", ".cxx": "cpp", ".hpp": "cpp", ".h": "cpp",
-    ".md": "markdown",
-};
-
-function docTypeForPath(path: string): string {
-    return DOC_TYPE_BY_EXT[extname(path).toLowerCase()] ?? "text";
-}
-
-function computeLineStarts(text: string): number[] {
-    const starts = [0];
-    for (let i = 0; i < text.length; i++) {
-        if (text.charCodeAt(i) === 10) starts.push(i + 1);
-    }
-    return starts;
-}
-
-function lineOf(lineStarts: number[], pos: number): number {
-    let lo = 0, hi = lineStarts.length - 1;
-    while (lo < hi) {
-        const mid = (lo + hi + 1) >>> 1;
-        if (lineStarts[mid] <= pos) lo = mid;
-        else hi = mid - 1;
-    }
-    return lo + 1;
-}