xindex 1.0.5 → 1.0.7

package/.tmp/watch.log

@@ -0,0 +1,14 @@
+npm warn Unknown env config "version-git-tag". This will stop working in the next major version of npm.
+npm warn Unknown env config "argv". This will stop working in the next major version of npm.
+npm warn Unknown env config "version-commit-hooks". This will stop working in the next major version of npm.
+npm warn Unknown env config "version-git-message". This will stop working in the next major version of npm.
+npm warn Unknown env config "version-tag-prefix". This will stop working in the next major version of npm.
+[xindex-1776991372027-99a3bfc69e73f] started, indexing and watching: .
+indexing: notes.md
+indexing: util.ts
+initial index complete, acquiring watcher lock...
+done. indexed: {"indexedAmount":0}
+watching for changes: .
+index: wombat.md
+remove: functional
+index end failed: Error saving index: Error: ENOENT: no such file or directory, open '.xindex/semantic/index.json'

package/.xindex.json

@@ -1,23 +1,95 @@
 {
   "ignoreKeywords": [
-    "import", "export", "const", "function", "return", "async", "await", "type", "string", "number",
-    "from", "default", "let", "var",
-    "if", "else", "switch", "case", "break", "continue",
-    "for", "of", "in",
-    "try", "catch", "finally", "throw",
-    "new", "delete", "as", "is", "typeof", "instanceof", "void",
-    "this", "super",
-    "interface", "class", "extends", "implements",
-    "public", "private", "protected", "static", "abstract", "readonly",
-    "declare", "namespace", "module", "enum",
-    "boolean", "any", "unknown", "never",
-    "undefined", "null", "object",
-    "record", "array", "readonlyarray", "promise",
-    "true", "false",
-    "src", "packages", "fun", "componets", "streamx", "apps", "run", "rnd", "tsx",
-    "utf8", "length", "map", "slice", "push", "join", "resolve", "stringify",
-    "json", "settimeout", "path", "readfile"
+    "import",
+    "export",
+    "const",
+    "function",
+    "return",
+    "async",
+    "await",
+    "type",
+    "string",
+    "number",
+    "from",
+    "default",
+    "let",
+    "var",
+    "if",
+    "else",
+    "switch",
+    "case",
+    "break",
+    "continue",
+    "for",
+    "of",
+    "in",
+    "try",
+    "catch",
+    "finally",
+    "throw",
+    "new",
+    "delete",
+    "as",
+    "is",
+    "typeof",
+    "instanceof",
+    "void",
+    "this",
+    "super",
+    "interface",
+    "class",
+    "extends",
+    "implements",
+    "public",
+    "private",
+    "protected",
+    "static",
+    "abstract",
+    "readonly",
+    "declare",
+    "namespace",
+    "module",
+    "enum",
+    "boolean",
+    "any",
+    "unknown",
+    "never",
+    "undefined",
+    "null",
+    "object",
+    "record",
+    "array",
+    "readonlyarray",
+    "promise",
+    "true",
+    "false",
+    "src",
+    "packages",
+    "fun",
+    "componets",
+    "streamx",
+    "apps",
+    "run",
+    "rnd",
+    "tsx",
+    "utf8",
+    "length",
+    "map",
+    "slice",
+    "push",
+    "join",
+    "resolve",
+    "stringify",
+    "json",
+    "settimeout",
+    "path",
+    "readfile"
   ],
-  "ignoreFiles": [".xindex", "media"],
-  "maxLines": 12
+  "ignoreFiles": [
+    ".xindex",
+    "media"
+  ],
+  "maxLines": 30,
+  "maxFileBytes": 100000,
+  "followSymlinks": false
 }

package/README.md

@@ -6,6 +6,8 @@ Local semantic code search for your codebase — plus an MCP server so Claude Co
 
 ## Install
 
+Requires **Node ≥22**.
+
 ```bash
 npm i -g xindex
 ```
@@ -47,6 +49,7 @@ Open the project in Claude Code — it picks up the xindex MCP server and can ca
 - **Watch mode** — keeps the index warm while you code
 - **Gitignore-aware** — respects `.gitignore` + custom ignore rules
 - **Zero config** — works with defaults; `.xindex.json` is optional
+- **Tolerant** — tolerates unreadable files, oversize files, empty files, binary files, and symlinks; each is skipped with a log line so the run always finishes
 
 ## Claude Code skills (`@xi`)
 
@@ -171,7 +174,8 @@ Project-root file. All fields optional; unknown keys ignored; missing/empty →
 - **`ignoreKeywords`** — `string[]`, default `[]`. Tokens stripped before embedding — add project slang/boilerplate polluting results. Entries ≤1 char warn.
 - **`ignoreFiles`** — `string[]`, default `[]`. Extra globs excluded during walk/watch, on top of `.gitignore` — add vendored/generated folders.
 - **`maxLines`** — `number`, default `30`. Lines per chunk — tune if chunks feel over/under-sized.
-- **`maxFileBytes`** — `number`, default `5000000`. Skip files over this (5 MB) — lower for faster indexing on huge generated files.
+- **`maxFileBytes`** — `number`, default `50000`. Skip files over this (50 KB) — raise to index larger generated files.
+- **`followSymlinks`** — `boolean`, default `false`. When `false`, symbolic links encountered during walk/watch are skipped with a log line. Set `true` to follow them (cycles are broken via `realpath` dedup).
 
 ```json
 {

package/apps/indexApp.ts

@@ -1,4 +1,3 @@
-import {readFile} from "fs/promises";
 import {from} from "../packages/streamx/src/from.js";
 import {batch} from "../packages/streamx/src/batch.js";
 import {map} from "../packages/streamx/src/map.js";
@@ -8,23 +7,24 @@ import {IWalkFiles} from "../componets/walkFiles.js";
 import {IIndexContent} from "../features/indexContent.js";
 import {ILogger} from "../componets/logger.js";
 import {INDEXING_BATCH_SIZE} from "../componets/config/INDEXING_BATCH_SIZE";
+import {SafeIndexBatch} from "../componets/io/safeIndexBatch.js";
 
 export type IIndexApp = (inputs: string[]) => Promise<void>;
 
-export function IndexApp({walkFiles, indexContent, log}: {
+export function IndexApp({walkFiles, indexContent, log, maxFileBytes}: {
     walkFiles: IWalkFiles,
     indexContent: IIndexContent,
     log: ILogger,
+    maxFileBytes: number,
 }): IIndexApp {
+    const safeIndexBatch = SafeIndexBatch({indexContent, log, maxFileBytes});
     return async function indexApp(inputs) {
         await run(
             from(walkFiles(inputs))
                 .pipe(tap(id => log(`indexing: ${id}`)))
                 .pipe(batch(INDEXING_BATCH_SIZE))
                 .pipe(map<string[], string[]>(async (ids) => {
-                    const texts = await Promise.all(ids.map(id => readFile(id, "utf8")));
-                    const items = ids.map((id, i) => ({id, content: `${texts[i]}. ${id}`}));
-                    await indexContent(items);
+                    await safeIndexBatch(ids);
                     return ids;
                 }))
         );

package/apps/run.index.ts

@@ -3,17 +3,19 @@ import {BufferedLoggerToStdOut} from "../componets/logger.js";
 import {WalkFiles} from "../componets/walkFiles.js";
 import {IndexApp} from "./indexApp.js";
 import {AppId} from "../componets/appId.js";
+import {INDEXING_COALESCE_MS} from "../componets/config/INDEXING_COALESCE_MS.js";
 
 const appId = AppId();
 const cwd = process.cwd();
 const log = BufferedLoggerToStdOut();
-const {indexContent, getIndexStats, config} = await BuildComponents({log});
-const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles});
-const indexApp = IndexApp({walkFiles, indexContent, log});
+const {indexContentBatch, getIndexStats, config, flush} = await BuildComponents({log, indexingCoalesceMs: INDEXING_COALESCE_MS});
+const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles, followSymlinks: config.followSymlinks});
+const indexApp = IndexApp({walkFiles, indexContent: indexContentBatch, log, maxFileBytes: config.maxFileBytes});
 
 const inputs = process.argv.slice(2);
 if (!inputs.length) inputs.push(".");
 
 log(`[${appId}] started, indexing: ${inputs.join(", ")}`);
 await indexApp(inputs);
+await flush();
 log(`[${appId}] done:`, await getIndexStats());

package/apps/run.mcp.ts

@@ -9,6 +9,8 @@ import {McpApp} from "./mcpApp.js";
 import {SearchApp} from "./searchApp.js";
 import {join} from "path";
 import {AppId} from "../componets/appId.js";
+import {INDEXING_COALESCE_MS} from "../componets/config/INDEXING_COALESCE_MS.js";
+import {WATCH_COALESCE_MS} from "../componets/config/WATCH_COALESCE_MS.js";
 
 const args = process.argv.slice(2);
 const watchDirArg = args.find(a => a.startsWith("--watch-dir="));
@@ -16,9 +18,17 @@ const watchDisabled = args.includes("--watch-disabled");
 
 const cwd = process.cwd();
 const log = BufferedLoggerToStdErr();
-const {indexContent, removeContent, getIndexStats, searchContentIndex, resetIndex, config} = await BuildComponents({log});
-const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles});
-const indexApp = IndexApp({walkFiles, indexContent, log});
+const {indexContentWatch, indexContentBatch, removeContent, getIndexStats, searchContentIndex, resetIndex, config, flush} = await BuildComponents({
+    log,
+    watchCoalesceMs: WATCH_COALESCE_MS,
+    indexingCoalesceMs: INDEXING_COALESCE_MS,
+});
+const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles, followSymlinks: config.followSymlinks});
+const rawIndexApp = IndexApp({walkFiles, indexContent: indexContentBatch, log, maxFileBytes: config.maxFileBytes});
+const indexApp = async (inputs: string[]) => {
+    await rawIndexApp(inputs);
+    await flush();
+};
 const search = SearchApp({searchContentIndex});
 
 const appId = AppId();
@@ -31,18 +41,18 @@ const watcherLock = WatcherLock({
 const watch = watchDisabled ? undefined : {
     watchFiles: WatchFiles({cwd, log, ignoreFiles: config.ignoreFiles}),
     watchDir: watchDirArg ? watchDirArg.split("=")[1] : ".",
-    handleFileEvents: HandleFileEvents({indexContent, removeContent, log}),
+    handleFileEvents: HandleFileEvents({indexContent: indexContentWatch, removeContent, log, maxFileBytes: config.maxFileBytes}),
     watcherLock,
 };
 
 process.on("SIGINT", async () => {
-    log(`[${appId}] shutting down — stopping heartbeat...`);
+    log(`shutting down — stopping heartbeat...`);
     watcherLock.stopHeartbeat();
-    log(`[${appId}] releasing lock...`);
+    log(`releasing lock...`);
     await watcherLock.release();
-    log(`[${appId}] waiting 7s for another watcher to take over...`);
+    log(`waiting 7s for another watcher to take over...`);
     await new Promise(r => setTimeout(r, 7000));
-    log(`[${appId}] exiting`);
+    log(`exiting`);
     process.exit(0);
 });
 

package/apps/run.watch.ts

@@ -7,13 +7,19 @@ import {WatcherLock} from "../componets/index/watcherLock.js";
 import {WatchApp} from "./watchApp.js";
 import {AppId} from "../componets/appId.js";
 import {join} from "path";
+import {WATCH_COALESCE_MS} from "../componets/config/WATCH_COALESCE_MS.js";
+import {INDEXING_COALESCE_MS} from "../componets/config/INDEXING_COALESCE_MS.js";
 
 const cwd = process.cwd();
 const log = BufferedLoggerToStdOut();
-const {indexContent, removeContent, getIndexStats, config} = await BuildComponents({log});
-const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles});
+const {indexContentWatch, indexContentBatch, removeContent, getIndexStats, config} = await BuildComponents({
+    log,
+    watchCoalesceMs: WATCH_COALESCE_MS,
+    indexingCoalesceMs: INDEXING_COALESCE_MS,
+});
+const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles, followSymlinks: config.followSymlinks});
 const watchFiles = WatchFiles({cwd, log, ignoreFiles: config.ignoreFiles});
-const handleFileEvents = HandleFileEvents({indexContent, removeContent, log});
+const handleFileEvents = HandleFileEvents({indexContent: indexContentWatch, removeContent, log, maxFileBytes: config.maxFileBytes});
 
 const appId = AppId();
 const watcherLock = WatcherLock({
@@ -22,7 +28,7 @@ const watcherLock = WatcherLock({
     log,
 });
 
-const app = WatchApp({walkFiles, watchFiles, handleFileEvents, indexContent, log, watcherLock});
+const app = WatchApp({walkFiles, watchFiles, handleFileEvents, indexContent: indexContentBatch, log, watcherLock, maxFileBytes: config.maxFileBytes});
 
 const inputs = process.argv.slice(2);
 if (!inputs.length) inputs.push(".");

package/apps/watchApp.ts

@@ -3,7 +3,6 @@ import {batch} from "../packages/streamx/src/batch.js";
 import {map} from "../packages/streamx/src/map.js";
 import {tap} from "../packages/streamx/src/tap.js";
 import {run} from "../packages/streamx/src/index.js";
-import {readFile} from "fs/promises";
 import {IWalkFiles} from "../componets/walkFiles.js";
 import {IWatchFiles} from "../componets/watchFiles.js";
 import {IHandleFileEvents} from "../componets/index/handleFileEvent.js";
@@ -12,20 +11,23 @@ import {IWatcherLock} from "../componets/index/watcherLock.js";
 import {WatchFileEventsApp} from "./watchFileEventsApp.js";
 import {IIndexContent} from "../features/indexContent.js";
 import {INDEXING_BATCH_SIZE} from "../componets/config/INDEXING_BATCH_SIZE";
+import {SafeIndexBatch} from "../componets/io/safeIndexBatch.js";
 
 export type IWatchApp = {
     run: (inputs: string[]) => Promise<void>;
     stop: () => void;
 };
 
-export function WatchApp({walkFiles, watchFiles, handleFileEvents, indexContent, log, watcherLock}: {
+export function WatchApp({walkFiles, watchFiles, handleFileEvents, indexContent, log, watcherLock, maxFileBytes}: {
     walkFiles: IWalkFiles,
     watchFiles: IWatchFiles,
     handleFileEvents: IHandleFileEvents,
     indexContent: IIndexContent,
     log: ILogger,
     watcherLock: IWatcherLock,
+    maxFileBytes: number,
 }): IWatchApp {
+    const safeIndexBatch = SafeIndexBatch({indexContent, log, maxFileBytes});
     return {
         async run(inputs) {
             // Phase 1: initial index — walk all files
@@ -34,9 +36,7 @@ export function WatchApp({walkFiles, watchFiles, handleFileEvents, indexContent,
                     .pipe(tap(id => log(`indexing: ${id}`)))
                     .pipe(batch(INDEXING_BATCH_SIZE))
                     .pipe(map<string[], string[]>(async (ids) => {
-                        const texts = await Promise.all(ids.map(id => readFile(id, "utf8")));
-                        const items = ids.map((id, i) => ({id, content: `${texts[i]}. ${id}`}));
-                        await indexContent(items);
+                        await safeIndexBatch(ids);
                         return ids;
                     }))
             );

package/componets/appId.ts

@@ -1,3 +1,3 @@
 export function AppId() {
-    return `xindex-${Date.now()}-${Math.random().toString(16).slice(2)}`;
+    return `xindex-${Math.random().toString(16).slice(2)}`;
 }

package/componets/buildComponents.ts

@@ -6,7 +6,11 @@ import {LoadConfig} from "./config/loadConfig.js";
 import {ILogger} from "./logger.js";
 import {LocateInFile} from "./locate/locateInFile.js";
 
-export async function BuildComponents({log}: { log: ILogger }) {
+export async function BuildComponents({log, watchCoalesceMs = 0, indexingCoalesceMs = 0}: {
+    log: ILogger,
+    watchCoalesceMs?: number,
+    indexingCoalesceMs?: number,
+}) {
     const loadConfig = LoadConfig({configPath: ".xindex.json", log});
     const config = await loadConfig();
 
@@ -26,18 +30,24 @@ export async function BuildComponents({log}: { log: ILogger }) {
 
     const SCORE_THRESHOLD = 0.01;
 
-    const {indexContent, removeContent, getIndexStats, searchContentIndex, resetIndex}
+    const {indexContentWatch, indexContentBatch, removeContent, getIndexStats, searchContentIndex, resetIndex, flush}
         = await ContentIndexDriver({
         path: DEFAULT_INDEX_PATH,
         embed,
         extractKeywords,
         cleanUpKeywords,
         locateInFile,
-        scoreThreshold: SCORE_THRESHOLD
+        scoreThreshold: SCORE_THRESHOLD,
+        log,
+        indexingWatchCoalesceMs: watchCoalesceMs,
+        indexingCoalesceMs,
     });
 
     return {
-        extractKeywords, cleanUpKeywords, indexContent, removeContent, getIndexStats,
-        searchContentIndex, resetIndex, locateInFile, config
+        extractKeywords, cleanUpKeywords,
+        indexContentWatch, indexContentBatch,
+        removeContent, getIndexStats,
+        searchContentIndex, resetIndex,
+        locateInFile, config, flush,
     };
 }

package/componets/config/DEFAULT_LOCATE_BATCH_SIZE.ts

@@ -1 +1 @@
-export const DEFAULT_LOCATE_BATCH_SIZE = 3;
+export const DEFAULT_LOCATE_BATCH_SIZE = 8;

package/componets/config/INDEXING_BATCH_SIZE.ts

@@ -1 +1 @@
-export const INDEXING_BATCH_SIZE = 5;
+export const INDEXING_BATCH_SIZE = 16;

package/componets/config/INDEXING_COALESCE_MAX_ITEMS.ts

@@ -0,0 +1 @@
+export const INDEXING_COALESCE_MAX_ITEMS = 1000;

package/componets/config/INDEXING_COALESCE_MS.ts

@@ -0,0 +1 @@
+export const INDEXING_COALESCE_MS = 1000;

package/componets/config/WATCH_COALESCE_MS.ts

@@ -0,0 +1 @@
+export const WATCH_COALESCE_MS = 100;

package/componets/config/WATCH_FLUSH_MS.ts

@@ -1 +1 @@
-export const WATCH_FLUSH_MS = 500;
+export const WATCH_FLUSH_MS = 300;

package/componets/config/loadConfig.ts

@@ -3,13 +3,16 @@ import {IXindexConfig} from "./xindexConfig.js";
 import {ILogger} from "../logger.js";
 
 const DEFAULT_MAX_LINES = 30;
-const DEFAULT_MAX_FILE_BYTES = 5_000_000;
+const DEFAULT_MAX_FILE_BYTES = 50_000;
+const DEFAULT_FOLLOW_SYMLINKS = false;
+const DEFAULT_IGNORE_FILES = ['.xindex', 'node_modules'];
 
 const DEFAULTS: IXindexConfig = {
     ignoreKeywords: [],
-    ignoreFiles: [],
+    ignoreFiles: DEFAULT_IGNORE_FILES,
     maxLines: DEFAULT_MAX_LINES,
     maxFileBytes: DEFAULT_MAX_FILE_BYTES,
+    followSymlinks: DEFAULT_FOLLOW_SYMLINKS,
 };
 
 export type ILoadConfig = () => Promise<IXindexConfig>;
@@ -41,11 +44,18 @@ export function LoadConfig({configPath, log}: { configPath: string, log: ILogger
             ignoreFiles: toStrings(parsed.ignoreFiles),
             maxLines: toNum(parsed.maxLines, DEFAULT_MAX_LINES),
             maxFileBytes: toNum(parsed.maxFileBytes, DEFAULT_MAX_FILE_BYTES),
+            followSymlinks: typeof parsed.followSymlinks === "boolean" ? parsed.followSymlinks : DEFAULT_FOLLOW_SYMLINKS,
         };
 
         for (const kw of config.ignoreKeywords) {
             if (kw.length <= 1) log(`warning: ignoreKeywords entry "${kw}" is <=1 char`);
         }
+        if (config.maxFileBytes < 1024) {
+            log(`warning: maxFileBytes (${config.maxFileBytes}) < 1024 — likely a typo`);
+        }
+        if (parsed.followSymlinks !== undefined && typeof parsed.followSymlinks !== "boolean") {
+            log(`warning: followSymlinks must be boolean — using default (${DEFAULT_FOLLOW_SYMLINKS})`);
+        }
 
         return config;
     };

package/componets/config/xindexConfig.ts

@@ -3,4 +3,5 @@ export type IXindexConfig = {
     ignoreFiles: string[];
     maxLines: number;
     maxFileBytes: number;
+    followSymlinks: boolean;
 };

package/componets/index/coalesceIndexApi.ts

@@ -0,0 +1,74 @@
+import {IIndexApi, IndexCommandType} from "./indexApi.js";
+import {ILogger} from "../logger.js";
+import {INDEXING_COALESCE_MAX_ITEMS} from "../config/INDEXING_COALESCE_MAX_ITEMS.js";
+import {Defer, IDefer} from "../../packages/fun/src/defer.js";
+
+type Item = {id: string, vector: number[], keywords: string};
+
+export function CoalesceIndexApi({inner, windowMs, maxItems = INDEXING_COALESCE_MAX_ITEMS, log}: {
+    inner: IIndexApi,
+    windowMs: number,
+    maxItems?: number,
+    log: ILogger,
+}): IIndexApi {
+    if (windowMs <= 0) return inner;
+
+    let pending: Item[] = [];
+    let delivery: IDefer | null = null;
+    let timer: ReturnType<typeof setTimeout> | null = null;
+
+    async function deliver() {
+        if (delivery) { await delivery.promise; return; }
+        if (timer) { clearTimeout(timer); timer = null; }
+        if (pending.length === 0) return;
+
+        const d = Defer();
+        delivery = d;
+        const items = pending;
+        pending = [];
+        try {
+            await inner({type: IndexCommandType.index, items});
+        } catch (e: any) {
+            log(`coalesced flush failed (${items.length} items): ${e?.message ?? e}`);
+        } finally {
+            delivery = null;
+            d.resolve();
+        }
+
+        if (pending.length >= maxItems) {
+            await deliver();
+        } else if (pending.length > 0 && !timer) {
+            timer = setTimeout(deliver, windowMs);
+        }
+    }
+
+    const api = (async function coalescedIndexApi(msg) {
+        switch (msg.type) {
+            case IndexCommandType.index: {
+                // if (delivery) await delivery.promise;
+                pending.push(...msg.items);
+
+                if (!timer) {
+                    timer = setTimeout(deliver, windowMs);
+                }
+
+                if (pending.length >= maxItems) {
+                    await deliver();
+                }
+                return;
+            }
+            case IndexCommandType.delete:
+            case IndexCommandType.reset: {
+                while (pending.length > 0 || delivery) await deliver();
+                return inner(msg);
+            }
+        }
+    }) as IIndexApi;
+
+    api.flush = async () => {
+        while (pending.length > 0 || delivery) await deliver();
+        await inner.flush();
+    };
+
+    return api;
+}

package/componets/index/contentIndexDriver.ts

@@ -7,35 +7,63 @@ import {SearchIndex, ISearchIndex} from "../../features/searchIndex.js";
 import {RemoveContent, IRemoveContent} from "../../features/removeContent.js";
 import {ResetIndex, IResetIndex} from "../../features/resetIndex.js";
 import {VectraIndex} from "./vectraIndex.js";
-import {IndexApi} from "./indexApi.js";
+import {IndexApi, IIndexApi} from "./indexApi.js";
+import {CoalesceIndexApi} from "./coalesceIndexApi.js";
 import {ILocateInFile} from "../locate/locateInFile.js";
+import {ILogger} from "../logger.js";
 
 export type IContentIndexDriver = Readonly<{
     getIndexStats: IGetIndexStats,
-    indexContent: IIndexContent,
+    indexContentWatch: IIndexContent,
+    indexContentBatch: IIndexContent,
     removeContent: IRemoveContent,
     searchContentIndex: ISearchIndex,
     resetIndex: IResetIndex,
     flush: () => Promise<void>,
 }>;
 
-export async function ContentIndexDriver({path, embed, extractKeywords, cleanUpKeywords, locateInFile, scoreThreshold}: {
+export async function ContentIndexDriver({
+    path, embed, extractKeywords, cleanUpKeywords, locateInFile, scoreThreshold, log,
+    indexingWatchCoalesceMs = 0, indexingCoalesceMs = 0,
+}: {
     path: string,
     embed: IEmbed,
     extractKeywords: IExtractKeywords,
     cleanUpKeywords: ICleanUpKeywords,
     locateInFile: ILocateInFile,
     scoreThreshold: number,
+    log: ILogger,
+    indexingWatchCoalesceMs?: number,
+    indexingCoalesceMs?: number,
 }): Promise<IContentIndexDriver> {
     const index = await VectraIndex(path + "/semantic");
-    const indexApi = IndexApi({index});
+    const rawIndexApi = IndexApi({index, log});
+    const watchCoalesce = CoalesceIndexApi({inner: rawIndexApi, windowMs: indexingWatchCoalesceMs, log});
+    const batchCoalesce = CoalesceIndexApi({inner: rawIndexApi, windowMs: indexingCoalesceMs, log});
+
+    const flushAll = async () => {
+        await Promise.all([watchCoalesce.flush(), batchCoalesce.flush()]);
+        await rawIndexApi.flush();
+    };
+
+    const drainThen = (inner: IIndexApi): IIndexApi => {
+        const api = (async (msg) => {
+            await flushAll();
+            return inner(msg);
+        }) as IIndexApi;
+        api.flush = flushAll;
+        return api;
+    };
+
+    const drainingApi = drainThen(rawIndexApi);
 
     return {
         getIndexStats: GetIndexStats({index}),
-        indexContent: IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi}),
-        removeContent: RemoveContent({indexApi}),
+        indexContentWatch: IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi: watchCoalesce, log}),
+        indexContentBatch: IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi: batchCoalesce, log}),
+        removeContent: RemoveContent({indexApi: drainingApi}),
         searchContentIndex: SearchIndex({extractKeywords, cleanUpKeywords, embed, index, locateInFile, scoreThreshold}),
-        resetIndex: ResetIndex({indexApi}),
-        flush: () => indexApi.flush(),
+        resetIndex: ResetIndex({indexApi: drainingApi}),
+        flush: flushAll,
     };
 }

package/componets/index/documentContentIndexDriver.ts

@@ -84,7 +84,8 @@ export async function DocumentContentIndexDriver({path, model}: {
 
     return {
         getIndexStats,
-        indexContent,
+        indexContentWatch: indexContent,
+        indexContentBatch: indexContent,
         removeContent,
         searchContentIndex,
         resetIndex,

package/componets/index/handleFileEvent.ts

@@ -1,22 +1,22 @@
-import {readFile} from "fs/promises";
 import {IIndexContent} from "../../features/indexContent.js";
 import {IRemoveContent} from "../../features/removeContent.js";
 import {ILogger} from "../logger.js";
 import {FileEventType, IFileEvent} from "../watchFiles.js";
+import {SafeIndexBatch} from "../io/safeIndexBatch.js";
 
 export type IHandleFileEvent = (event: IFileEvent) => Promise<void>;
 export type IHandleFileEvents = (events: IFileEvent[]) => Promise<void>;
 
-export function HandleFileEvent({indexContent, removeContent, log}: {
+export function HandleFileEvent({indexContent, removeContent, log, maxFileBytes}: {
     indexContent: IIndexContent,
     removeContent: IRemoveContent,
     log: ILogger,
+    maxFileBytes: number,
 }): IHandleFileEvent {
+    const safeIndexBatch = SafeIndexBatch({indexContent, log, maxFileBytes});
     return async function handleFileEvent(event) {
         if (event.type === FileEventType.index) {
-            const text = await readFile(event.path, "utf8");
-            await indexContent([{id: event.path, content: `${text}. ${event.path}`}]);
-            log(`index: ${event.path}`);
+            await safeIndexBatch([event.path]);
         } else {
             try { await removeContent([event.path]); } catch (e) { log(`remove failed: ${event.path} — ${(e as any)?.message ?? e}`); }
             log(`remove: ${event.path}`);
@@ -24,42 +24,22 @@ export function HandleFileEvent({indexContent, removeContent, log}: {
     };
 }
 
-export function HandleFileEvents({indexContent, removeContent, log}: {
+export function HandleFileEvents({indexContent, removeContent, log, maxFileBytes}: {
     indexContent: IIndexContent,
     removeContent: IRemoveContent,
     log: ILogger,
+    maxFileBytes: number,
 }): IHandleFileEvents {
+    const safeIndexBatch = SafeIndexBatch({indexContent, log, maxFileBytes});
     return async function handleFileEvents(events) {
-        const indexEvents = events.filter((event) => event.type === FileEventType.index);
-        const removeEvents = events.filter((event) => event.type === FileEventType.remove);
+        const indexPaths = events.filter(e => e.type === FileEventType.index).map(e => e.path);
+        const removePaths = events.filter(e => e.type === FileEventType.remove).map(e => e.path);
 
-        if (indexEvents.length > 0) {
-            const indexItems = (await Promise.all(indexEvents.map(async (event) => {
-                log(`index: ${event.path}`);
-                try {
-                    const text = await readFile(event.path, "utf8");
-                    return {id: event.path, content: `${text}. ${event.path}`};
-                } catch (e) {
-                    log(`index failed: ${event.path} — ${(e as any)?.message ?? e}`);
-                    return undefined;
-                }
-            }))).filter((item): item is { id: string, content: string } => !!item);
-
-            if (indexItems.length > 0) {
-                try {
-                    await indexContent(indexItems);
-                } catch (e) {
-                    log(`index batch failed: ${(e as any)?.message ?? e}`);
-                }
-            }
-        }
-
-        if (removeEvents.length > 0) {
-            const removePaths = removeEvents.map((event) => {
-                log(`remove: ${event.path}`);
-                return event.path;
-            });
+        for (const p of indexPaths) log(`index: ${p}`);
+        if (indexPaths.length > 0) await safeIndexBatch(indexPaths);
 
+        if (removePaths.length > 0) {
+            for (const p of removePaths) log(`remove: ${p}`);
             try {
                 await removeContent(removePaths);
             } catch (e) {

package/componets/index/indexApi.ts

@@ -2,6 +2,7 @@ import {LocalIndex} from "vectra";
 import {IType} from "../IType.js";
 import {ISerial, Serial} from "../../packages/fun/src/serial.js";
 import {caseNever} from "../../packages/fun/src/case-never.js";
+import {ILogger} from "../logger.js";
 
 export enum IndexCommandType {
     index = 'index',
@@ -16,17 +17,29 @@ export type IIndexCommand =
 
 export type IIndexApi = ISerial<IIndexCommand, void>;
 
-export function IndexApi({index}: { index: LocalIndex }): IIndexApi {
+export function IndexApi({index, log}: { index: LocalIndex, log: ILogger }): IIndexApi {
     return Serial<IIndexCommand, void>(async msg => {
         switch (msg.type) {
             case IndexCommandType.delete: {
-                await index.beginUpdate();
                 try {
-                    for (const id of msg.ids) {
-                        await index.deleteItem(id);
+                    await index.beginUpdate();
+                    try {
+                        for (const id of msg.ids) {
+                            try {
+                                await index.deleteItem(id);
+                            } catch (e: any) {
+                                log(`delete failed for ${id}: ${e?.message ?? e}`);
+                            }
+                        }
+                    } finally {
+                        try {
+                            await index.endUpdate();
+                        } catch (e: any) {
+                            log(`index end failed: ${e?.message ?? e}`);
+                        }
                     }
-                } finally {
-                    await index.endUpdate();
+                } catch (e: any) {
+                    log(`index begin failed: ${e?.message ?? e}`);
                 }
                 break;
             }
@@ -35,35 +48,51 @@ export function IndexApi({index}: { index: LocalIndex }): IIndexApi {
                     await index.batchInsertItems(msg.items.map(item => ({
                         id: item.id,
                         vector: item.vector,
-                        metadata: {id: item.id, keywords: item.keywords},
+                        metadata: {},
                     })));
-                } catch (error) {
-                    const errorMessage = String((error as Error)?.message ?? error).toLowerCase();
+                } catch (error: any) {
+                    const errorMessage = String(error?.message ?? error).toLowerCase();
                     const isDuplicateIdError =
                         errorMessage.includes("already exists")
                         || errorMessage.includes("duplicate");
                     if (!isDuplicateIdError) {
-                        throw error;
+                        log(`index write failed (${msg.items.length} items): ${error?.message ?? error}`);
+                        break;
                     }
-
-                    await index.beginUpdate();
                     try {
-                        for (const item of msg.items) {
-                            await index.upsertItem({
-                                id: item.id,
-                                vector: item.vector,
-                                metadata: {id: item.id, keywords: item.keywords},
-                            });
+                        await index.beginUpdate();
+                        try {
+                            for (const item of msg.items) {
+                                try {
+                                    await index.upsertItem({
+                                        id: item.id,
+                                        vector: item.vector,
+                                        metadata: {},
+                                    });
+                                } catch (e: any) {
+                                    log(`index write failed for ${item.id}: ${e?.message ?? e}`);
+                                }
+                            }
+                        } finally {
+                            try {
+                                await index.endUpdate();
+                            } catch (e: any) {
+                                log(`index end failed: ${e?.message ?? e}`);
+                            }
                         }
-                    } finally {
-                        await index.endUpdate();
+                    } catch (e: any) {
+                        log(`index begin failed: ${e?.message ?? e}`);
                     }
                 }
                 break;
             }
             case IndexCommandType.reset: {
-                await index.deleteIndex();
-                await index.createIndex();
+                try {
+                    await index.deleteIndex();
+                    await index.createIndex();
+                } catch (e: any) {
+                    log(`reset failed: ${e?.message ?? e}`);
+                }
                 break;
             }
             default:

package/componets/io/safeIndexBatch.ts

@@ -0,0 +1,30 @@
+import {safeReadFile, SafeReadResultType} from "./safeReadFile.js";
+import {IIndexContent} from "../../features/indexContent.js";
+import {ILogger} from "../logger.js";
+
+export type ISafeIndexBatch = (ids: string[]) => Promise<void>;
+
+export function SafeIndexBatch({indexContent, log, maxFileBytes}: {
+    indexContent: IIndexContent,
+    log: ILogger,
+    maxFileBytes: number,
+}): ISafeIndexBatch {
+    return async function safeIndexBatch(ids) {
+        const results = await Promise.all(ids.map(id => safeReadFile(id, maxFileBytes)));
+        const ok: {id: string, content: string}[] = [];
+        for (const r of results) {
+            if (r.type === SafeReadResultType.ok) {
+                ok.push({id: r.id, content: `${r.content}. ${r.id}`});
+            } else {
+                log(`skip ${r.id}: ${r.reason}`);
+            }
+        }
+        if (ok.length > 0) {
+            try {
+                await indexContent(ok);
+            } catch (e: any) {
+                log(`index batch failed: ${e?.message ?? e}`);
+            }
+        }
+    };
+}

package/componets/io/safeReadFile.ts

@@ -0,0 +1,36 @@
+import {stat, readFile} from "fs/promises";
+import {IType} from "../IType.js";
+
+export enum SafeReadResultType {
+    ok = "ok",
+    skip = "skip",
+    error = "error",
+}
+
+export enum SafeReadSkipReason {
+    tooLarge = "tooLarge",
+    empty = "empty",
+    binary = "binary",
+}
+
+export type ISafeReadResult =
+    | IType<{ type: SafeReadResultType.ok, id: string, content: string }>
+    | IType<{ type: SafeReadResultType.skip, id: string, reason: SafeReadSkipReason }>
+    | IType<{ type: SafeReadResultType.error, id: string, reason: string }>;
+
+export async function safeReadFile(id: string, maxBytes: number): Promise<ISafeReadResult> {
+    try {
+        const s = await stat(id);
+        if (s.size > maxBytes) return {type: SafeReadResultType.skip, id, reason: SafeReadSkipReason.tooLarge};
+        const buf = await readFile(id);
+        const probe = buf.subarray(0, Math.min(10, buf.length));
+        for (let i = 0; i < probe.length; i++) {
+            if (probe[i] === 0) return {type: SafeReadResultType.skip, id, reason: SafeReadSkipReason.binary};
+        }
+        const content = buf.toString("utf8");
+        if (!content.trim()) return {type: SafeReadResultType.skip, id, reason: SafeReadSkipReason.empty};
+        return {type: SafeReadResultType.ok, id, content};
+    } catch (e: any) {
+        return {type: SafeReadResultType.error, id, reason: e?.code ?? e?.message ?? String(e)};
+    }
+}

package/componets/llm/embed.ts

@@ -1,18 +1,24 @@
 import {pipeline} from "@huggingface/transformers";
 
-const embedder = await pipeline(
-    "feature-extraction",
-    "sentence-transformers/all-MiniLM-L6-v2"
-);
+const MODEL = "sentence-transformers/all-MiniLM-L6-v2";
 
 export type IEmbed = (text: string) => Promise<number[]>;
 
 type Pooling = "none" | "mean" | "cls" | "first_token" | "eos" | "last_token";
 
 export function Embed({pooling, normalize}: {pooling: Pooling, normalize: boolean}): IEmbed {
+    let embedderPromise: Promise<any> | null = null;
+
+    function getEmbedder() {
+        if (!embedderPromise) {
+            embedderPromise = pipeline("feature-extraction", MODEL);
+        }
+        return embedderPromise;
+    }
+
     return async function embed(text) {
+        const embedder = await getEmbedder();
         const result = await embedder(text, {pooling, normalize});
         return Array.from(result.data as Float32Array);
     }
 }
-

package/componets/locate/locateInFile.ts

@@ -11,9 +11,9 @@ import {filter} from "../../packages/streamx/src/filter.js";
 import {map} from "../../packages/streamx/src/map.js";
 import {flatMap} from "../../packages/streamx/src/flatMap.js";
 import {tap} from "../../packages/streamx/src/tap.js";
-import {scaleSync} from "../../packages/streamx/src/scaleSync.js";
 import {run} from "../../packages/streamx/src/index.js";
 import {DEFAULT_LOCATE_BATCH_SIZE} from "../config/DEFAULT_LOCATE_BATCH_SIZE";
+import {scaleSync} from "../../packages/streamx/src/scaleSync";
 
 export type ILocateInFile = (
     query: string,
@@ -61,12 +61,17 @@ export function LocateInFile({
                         return false;
                     }
                 }))
-                .pipe(scaleSync(SCALE_FILE_READS, async (id: string) => {
-                    const text = await readFile(id, "utf8");
-                    return {id, text};
+                .pipe(scaleSync(SCALE_FILE_READS, async (id: string): Promise<{ id: string, text: string } | null> => {
+                    try {
+                        const text = await readFile(id, "utf8");
+                        return {id, text};
+                    } catch {
+                        return null;
+                    }
                 }))
-                .pipe(flatMap(({id, text}: { id: string, text: string }): IWindow[] => {
-                    return windowsOf({text, id, windowLines});
+                .pipe(filter(async (r: { id: string, text: string } | null) => r !== null))
+                .pipe(flatMap((r: { id: string, text: string } | null): IWindow[] => {
+                    return windowsOf({text: r!.text, id: r!.id, windowLines});
                 }));
 
             const withVectors = from<IWindow>(windows)

package/componets/logger.ts

@@ -1,34 +1,37 @@
 export type ILogger = (...args: any[]) => void;
 
-export function Logger(fn: (...args: any[]) => void): ILogger {
-    return function log(...args) {
-        fn(...args);
-    }
-}
-
 const DEFAULT_FLUSH_MS = 100;
+const DEFAULT_FLUSH_LINES = 15;
 
-export function BufferedLoggerToStdOut(flushMs = DEFAULT_FLUSH_MS): ILogger {
-    return BufferedLogger(s => process.stdout.write(s), flushMs);
+export function BufferedLoggerToStdOut(flushMs = DEFAULT_FLUSH_MS, flushLines = DEFAULT_FLUSH_LINES): ILogger {
+    return BufferedLogger(s => process.stdout.write(s), flushMs, flushLines);
 }
 
-export function BufferedLoggerToStdErr(flushMs = DEFAULT_FLUSH_MS): ILogger {
-    return BufferedLogger(s => process.stderr.write(s), flushMs);
+export function BufferedLoggerToStdErr(flushMs = DEFAULT_FLUSH_MS, flushLines = DEFAULT_FLUSH_LINES): ILogger {
+    return BufferedLogger(s => process.stderr.write(s), flushMs, flushLines);
 }
 
-export function BufferedLogger(write: (text: string) => void, flushMs = DEFAULT_FLUSH_MS): ILogger {
+export function BufferedLogger(
+    write: (text: string) => void,
+    flushMs = DEFAULT_FLUSH_MS,
+    flushLines = DEFAULT_FLUSH_LINES,
+): ILogger {
     let buffer: string[] = [];
     let timer: ReturnType<typeof setTimeout> | null = null;
 
     function flush() {
+        if (timer) { clearTimeout(timer); timer = null; }
         if (buffer.length === 0) return;
         write(buffer.join("\n") + "\n");
         buffer = [];
-        timer = null;
     }
 
     return function log(...args) {
         buffer.push(args.map(a => typeof a === "object" ? JSON.stringify(a) : String(a)).join(" "));
+        if (buffer.length >= flushLines) {
+            flush();
+            return;
+        }
         if (!timer) timer = setTimeout(flush, flushMs);
     }
 }

package/componets/walkFiles.ts

@@ -1,11 +1,16 @@
-import {readdir, stat, readFile} from "fs/promises";
+import {readdir, stat, readFile, realpath} from "fs/promises";
 import {join, relative} from "path";
 import ignore from "ignore";
 import {ILogger} from "./logger.js";
 
 export type IWalkFiles = (inputs: string[]) => AsyncIterable<string>;
 
-export function WalkFiles({cwd, log, ignoreFiles = []}: {cwd: string, log: ILogger, ignoreFiles?: string[]}): IWalkFiles {
+export function WalkFiles({cwd, log, ignoreFiles = [], followSymlinks = false}: {
+    cwd: string,
+    log: ILogger,
+    ignoreFiles?: string[],
+    followSymlinks?: boolean,
+}): IWalkFiles {
 
     async function tryReadGitignore(dir: string): Promise<string> {
         try {
@@ -15,6 +20,8 @@ export function WalkFiles({cwd, log, ignoreFiles = []}: {cwd: string, log: ILogg
         }
     }
 
+    const visited = new Set<string>();
+
     async function* walk(dir: string, parentRules: string[]): AsyncIterable<string> {
         const localGitignore = await tryReadGitignore(dir);
         const rules = localGitignore ? [...parentRules, localGitignore] : parentRules;
@@ -27,11 +34,35 @@ export function WalkFiles({cwd, log, ignoreFiles = []}: {cwd: string, log: ILogg
             const abs = join(dir, entry.name);
             const rel = relative(cwd, abs);
 
+            if (entry.isSymbolicLink()) {
+                if (!followSymlinks) {
+                    log(`skip symlink ${rel}`);
+                    continue;
+                }
+                try {
+                    const real = await realpath(abs);
+                    if (visited.has(real)) continue;
+                    visited.add(real);
+                    const s = await stat(real);
+                    if (s.isDirectory()) {
+                        if (entry.name.startsWith(".")) continue;
+                        if (ig.ignores(rel + "/")) continue;
+                        yield* walk(abs, rules);
+                    } else if (s.isFile()) {
+                        if (ig.ignores(rel)) continue;
+                        yield rel;
+                    }
+                } catch (e: any) {
+                    log(`skip symlink ${rel}: ${e?.code ?? e?.message ?? e}`);
+                }
+                continue;
+            }
+
             if (entry.isDirectory()) {
                 if (entry.name.startsWith(".")) continue;
                 if (ig.ignores(rel + "/")) continue;
                 yield* walk(abs, rules);
-            } else {
+            } else if (entry.isFile()) {
                 if (ig.ignores(rel)) continue;
                 yield rel;
             }

package/features/indexContent.ts

@@ -2,22 +2,47 @@ import {IndexCommandType, IIndexApi} from "../componets/index/indexApi.js";
 import {IExtractKeywords} from "../componets/keywords/extractKeywords.js";
 import {ICleanUpKeywords} from "../componets/keywords/cleanUpKeywords.js";
 import {IEmbed} from "../componets/llm/embed.js";
+import {ILogger} from "../componets/logger.js";
 
 export type IIndexContent = (items: Array<{id: string, content: string}>) => Promise<void>;
 
-export function IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi}: {
+const SLOW_WARN_MS = 3000;
+
+export function IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi, log}: {
     extractKeywords: IExtractKeywords,
     cleanUpKeywords: ICleanUpKeywords,
     embed: IEmbed,
     indexApi: IIndexApi,
+    log: ILogger,
 }): IIndexContent {
     return async function indexContent(items) {
-        const indexItems = await Promise.all(items.map(async (item) => {
-            const keywords = cleanUpKeywords(extractKeywords(item.content)).join(", ");
-            const vector = await embed(keywords);
-            return {id: item.id, vector, keywords};
-        }));
+        const indexItems = (await Promise.all(items.map(async (item) => {
+            const warn = setTimeout(
+                () => log(`slow: still embedding ${item.id} after ${SLOW_WARN_MS}ms`),
+                SLOW_WARN_MS,
+            );
+            try {
+                const keywords = cleanUpKeywords(extractKeywords(item.content)).join(", ");
+                const vector = await embed(keywords);
+                return {id: item.id, vector, keywords};
+            } catch (e: any) {
+                log(`skip ${item.id}: keywords/embed failed — ${e?.code ?? e?.message ?? e}`);
+                return null;
+            } finally {
+                clearTimeout(warn);
+            }
+        }))).filter((x): x is NonNullable<typeof x> => !!x);
 
-        await indexApi({type: IndexCommandType.index, items: indexItems});
+        if (indexItems.length > 0) {
+            const warn = setTimeout(
+                () => log(`slow: still writing index after ${SLOW_WARN_MS}ms — ${indexItems.map(i => i.id).join(", ")}`),
+                SLOW_WARN_MS,
+            );
+            try {
+                await indexApi({type: IndexCommandType.index, items: indexItems});
+            } finally {
+                clearTimeout(warn);
+            }
+        }
     }
 }

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "xindex",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "Local semantic code search — index codebase, search by meaning or keywords",
   "type": "module",
   "main": "xindex.ts",
   "bin": {
+    "xindex": "bin/xindex-mcp",
     "xindex-index": "bin/xindex-index",
     "xindex-search": "bin/xindex-search",
     "xindex-mcp": "bin/xindex-mcp",
@@ -15,11 +16,16 @@
     "index": "tsx apps/run.index.ts",
     "search": "tsx apps/run.search.ts",
     "reset": "tsx apps/run.reset.ts",
+    "test": "npm run test.functional && npm run test.compilation",
     "mcp": "tsx apps/run.mcp.ts",
     "watch": "tsx apps/run.watch.ts",
+    "test.functional": "bash test/functional.sh",
     "test.compilation": "npx -y tsc --ignoreConfig --noEmit --target ES2022 --module ESNext --moduleResolution bundler --esModuleInterop --skipLibCheck --strict false $(git ls-files '*.ts')"
   },
   "private": false,
+  "engines": {
+    "node": ">=22"
+  },
   "keywords": [
     "semantic-search",
     "code-search",