xindex 1.0.6 → 1.0.7

package/.tmp/watch.log

@@ -1,9 +1,14 @@
-[xindex-1776976173089-3ab4dd18cfbfc8] started, indexing and watching: .
+npm warn Unknown env config "version-git-tag". This will stop working in the next major version of npm.
+npm warn Unknown env config "argv". This will stop working in the next major version of npm.
+npm warn Unknown env config "version-commit-hooks". This will stop working in the next major version of npm.
+npm warn Unknown env config "version-git-message". This will stop working in the next major version of npm.
+npm warn Unknown env config "version-tag-prefix". This will stop working in the next major version of npm.
+[xindex-1776991372027-99a3bfc69e73f] started, indexing and watching: .
 indexing: notes.md
 indexing: util.ts
 initial index complete, acquiring watcher lock...
-done. indexed: {"indexedAmount":2}
+done. indexed: {"indexedAmount":0}
 watching for changes: .
 index: wombat.md
 remove: functional
-remove batch failed: Error saving index: Error: ENOENT: no such file or directory, open '.xindex/semantic/index.json'
+index end failed: Error saving index: Error: ENOENT: no such file or directory, open '.xindex/semantic/index.json'

package/.xindex.json

@@ -1,23 +1,95 @@
 {
   "ignoreKeywords": [
-    "import", "export", "const", "function", "return", "async", "await", "type", "string", "number",
-    "from", "default", "let", "var",
-    "if", "else", "switch", "case", "break", "continue",
-    "for", "of", "in",
-    "try", "catch", "finally", "throw",
-    "new", "delete", "as", "is", "typeof", "instanceof", "void",
-    "this", "super",
-    "interface", "class", "extends", "implements",
-    "public", "private", "protected", "static", "abstract", "readonly",
-    "declare", "namespace", "module", "enum",
-    "boolean", "any", "unknown", "never",
-    "undefined", "null", "object",
-    "record", "array", "readonlyarray", "promise",
-    "true", "false",
-    "src", "packages", "fun", "componets", "streamx", "apps", "run", "rnd", "tsx",
-    "utf8", "length", "map", "slice", "push", "join", "resolve", "stringify",
-    "json", "settimeout", "path", "readfile"
+    "import",
+    "export",
+    "const",
+    "function",
+    "return",
+    "async",
+    "await",
+    "type",
+    "string",
+    "number",
+    "from",
+    "default",
+    "let",
+    "var",
+    "if",
+    "else",
+    "switch",
+    "case",
+    "break",
+    "continue",
+    "for",
+    "of",
+    "in",
+    "try",
+    "catch",
+    "finally",
+    "throw",
+    "new",
+    "delete",
+    "as",
+    "is",
+    "typeof",
+    "instanceof",
+    "void",
+    "this",
+    "super",
+    "interface",
+    "class",
+    "extends",
+    "implements",
+    "public",
+    "private",
+    "protected",
+    "static",
+    "abstract",
+    "readonly",
+    "declare",
+    "namespace",
+    "module",
+    "enum",
+    "boolean",
+    "any",
+    "unknown",
+    "never",
+    "undefined",
+    "null",
+    "object",
+    "record",
+    "array",
+    "readonlyarray",
+    "promise",
+    "true",
+    "false",
+    "src",
+    "packages",
+    "fun",
+    "componets",
+    "streamx",
+    "apps",
+    "run",
+    "rnd",
+    "tsx",
+    "utf8",
+    "length",
+    "map",
+    "slice",
+    "push",
+    "join",
+    "resolve",
+    "stringify",
+    "json",
+    "settimeout",
+    "path",
+    "readfile"
   ],
-  "ignoreFiles": [".xindex", "media"],
-  "maxLines": 12
+  "ignoreFiles": [
+    ".xindex",
+    "media"
+  ],
+  "maxLines": 30,
+  "maxFileBytes": 100000,
+  "followSymlinks": false
 }

package/README.md

@@ -174,7 +174,7 @@ Project-root file. All fields optional; unknown keys ignored; missing/empty →
 - **`ignoreKeywords`** — `string[]`, default `[]`. Tokens stripped before embedding — add project slang/boilerplate polluting results. Entries ≤1 char warn.
 - **`ignoreFiles`** — `string[]`, default `[]`. Extra globs excluded during walk/watch, on top of `.gitignore` — add vendored/generated folders.
 - **`maxLines`** — `number`, default `30`. Lines per chunk — tune if chunks feel over/under-sized.
-- **`maxFileBytes`** — `number`, default `1000000`. Skip files over this (1 MB) — raise to index larger generated files.
+- **`maxFileBytes`** — `number`, default `50000`. Skip files over this (50 KB) — raise to index larger generated files.
 - **`followSymlinks`** — `boolean`, default `false`. When `false`, symbolic links encountered during walk/watch are skipped with a log line. Set `true` to follow them (cycles are broken via `realpath` dedup).
 
 ```json

package/apps/run.index.ts

@@ -3,17 +3,19 @@ import {BufferedLoggerToStdOut} from "../componets/logger.js";
 import {WalkFiles} from "../componets/walkFiles.js";
 import {IndexApp} from "./indexApp.js";
 import {AppId} from "../componets/appId.js";
+import {INDEXING_COALESCE_MS} from "../componets/config/INDEXING_COALESCE_MS.js";
 
 const appId = AppId();
 const cwd = process.cwd();
 const log = BufferedLoggerToStdOut();
-const {indexContent, getIndexStats, config} = await BuildComponents({log});
+const {indexContentBatch, getIndexStats, config, flush} = await BuildComponents({log, indexingCoalesceMs: INDEXING_COALESCE_MS});
 const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles, followSymlinks: config.followSymlinks});
-const indexApp = IndexApp({walkFiles, indexContent, log, maxFileBytes: config.maxFileBytes});
+const indexApp = IndexApp({walkFiles, indexContent: indexContentBatch, log, maxFileBytes: config.maxFileBytes});
 
 const inputs = process.argv.slice(2);
 if (!inputs.length) inputs.push(".");
 
 log(`[${appId}] started, indexing: ${inputs.join(", ")}`);
 await indexApp(inputs);
+await flush();
 log(`[${appId}] done:`, await getIndexStats());

package/apps/run.mcp.ts

@@ -9,6 +9,8 @@ import {McpApp} from "./mcpApp.js";
 import {SearchApp} from "./searchApp.js";
 import {join} from "path";
 import {AppId} from "../componets/appId.js";
+import {INDEXING_COALESCE_MS} from "../componets/config/INDEXING_COALESCE_MS.js";
+import {WATCH_COALESCE_MS} from "../componets/config/WATCH_COALESCE_MS.js";
 
 const args = process.argv.slice(2);
 const watchDirArg = args.find(a => a.startsWith("--watch-dir="));
@@ -16,9 +18,17 @@ const watchDisabled = args.includes("--watch-disabled");
 
 const cwd = process.cwd();
 const log = BufferedLoggerToStdErr();
-const {indexContent, removeContent, getIndexStats, searchContentIndex, resetIndex, config} = await BuildComponents({log});
+const {indexContentWatch, indexContentBatch, removeContent, getIndexStats, searchContentIndex, resetIndex, config, flush} = await BuildComponents({
+    log,
+    watchCoalesceMs: WATCH_COALESCE_MS,
+    indexingCoalesceMs: INDEXING_COALESCE_MS,
+});
 const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles, followSymlinks: config.followSymlinks});
-const indexApp = IndexApp({walkFiles, indexContent, log, maxFileBytes: config.maxFileBytes});
+const rawIndexApp = IndexApp({walkFiles, indexContent: indexContentBatch, log, maxFileBytes: config.maxFileBytes});
+const indexApp = async (inputs: string[]) => {
+    await rawIndexApp(inputs);
+    await flush();
+};
 const search = SearchApp({searchContentIndex});
 
 const appId = AppId();
@@ -31,18 +41,18 @@ const watcherLock = WatcherLock({
 const watch = watchDisabled ? undefined : {
     watchFiles: WatchFiles({cwd, log, ignoreFiles: config.ignoreFiles}),
     watchDir: watchDirArg ? watchDirArg.split("=")[1] : ".",
-    handleFileEvents: HandleFileEvents({indexContent, removeContent, log, maxFileBytes: config.maxFileBytes}),
+    handleFileEvents: HandleFileEvents({indexContent: indexContentWatch, removeContent, log, maxFileBytes: config.maxFileBytes}),
     watcherLock,
 };
 
 process.on("SIGINT", async () => {
-    log(`[${appId}] shutting down — stopping heartbeat...`);
+    log(`shutting down — stopping heartbeat...`);
     watcherLock.stopHeartbeat();
-    log(`[${appId}] releasing lock...`);
+    log(`releasing lock...`);
     await watcherLock.release();
-    log(`[${appId}] waiting 7s for another watcher to take over...`);
+    log(`waiting 7s for another watcher to take over...`);
     await new Promise(r => setTimeout(r, 7000));
-    log(`[${appId}] exiting`);
+    log(`exiting`);
     process.exit(0);
 });
 

package/apps/run.watch.ts

@@ -7,13 +7,19 @@ import {WatcherLock} from "../componets/index/watcherLock.js";
 import {WatchApp} from "./watchApp.js";
 import {AppId} from "../componets/appId.js";
 import {join} from "path";
+import {WATCH_COALESCE_MS} from "../componets/config/WATCH_COALESCE_MS.js";
+import {INDEXING_COALESCE_MS} from "../componets/config/INDEXING_COALESCE_MS.js";
 
 const cwd = process.cwd();
 const log = BufferedLoggerToStdOut();
-const {indexContent, removeContent, getIndexStats, config} = await BuildComponents({log});
+const {indexContentWatch, indexContentBatch, removeContent, getIndexStats, config} = await BuildComponents({
+    log,
+    watchCoalesceMs: WATCH_COALESCE_MS,
+    indexingCoalesceMs: INDEXING_COALESCE_MS,
+});
 const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles, followSymlinks: config.followSymlinks});
 const watchFiles = WatchFiles({cwd, log, ignoreFiles: config.ignoreFiles});
-const handleFileEvents = HandleFileEvents({indexContent, removeContent, log, maxFileBytes: config.maxFileBytes});
+const handleFileEvents = HandleFileEvents({indexContent: indexContentWatch, removeContent, log, maxFileBytes: config.maxFileBytes});
 
 const appId = AppId();
 const watcherLock = WatcherLock({
@@ -22,7 +28,7 @@ const watcherLock = WatcherLock({
     log,
 });
 
-const app = WatchApp({walkFiles, watchFiles, handleFileEvents, indexContent, log, watcherLock, maxFileBytes: config.maxFileBytes});
+const app = WatchApp({walkFiles, watchFiles, handleFileEvents, indexContent: indexContentBatch, log, watcherLock, maxFileBytes: config.maxFileBytes});
 
 const inputs = process.argv.slice(2);
 if (!inputs.length) inputs.push(".");

package/componets/appId.ts

@@ -1,3 +1,3 @@
 export function AppId() {
-    return `xindex-${Date.now()}-${Math.random().toString(16).slice(2)}`;
+    return `xindex-${Math.random().toString(16).slice(2)}`;
 }

package/componets/buildComponents.ts

@@ -6,7 +6,11 @@ import {LoadConfig} from "./config/loadConfig.js";
 import {ILogger} from "./logger.js";
 import {LocateInFile} from "./locate/locateInFile.js";
 
-export async function BuildComponents({log}: { log: ILogger }) {
+export async function BuildComponents({log, watchCoalesceMs = 0, indexingCoalesceMs = 0}: {
+    log: ILogger,
+    watchCoalesceMs?: number,
+    indexingCoalesceMs?: number,
+}) {
     const loadConfig = LoadConfig({configPath: ".xindex.json", log});
     const config = await loadConfig();
 
@@ -26,7 +30,7 @@ export async function BuildComponents({log}: { log: ILogger }) {
 
     const SCORE_THRESHOLD = 0.01;
 
-    const {indexContent, removeContent, getIndexStats, searchContentIndex, resetIndex}
+    const {indexContentWatch, indexContentBatch, removeContent, getIndexStats, searchContentIndex, resetIndex, flush}
         = await ContentIndexDriver({
         path: DEFAULT_INDEX_PATH,
         embed,
@@ -35,10 +39,15 @@ export async function BuildComponents({log}: { log: ILogger }) {
         locateInFile,
         scoreThreshold: SCORE_THRESHOLD,
         log,
+        indexingWatchCoalesceMs: watchCoalesceMs,
+        indexingCoalesceMs,
     });
 
     return {
-        extractKeywords, cleanUpKeywords, indexContent, removeContent, getIndexStats,
-        searchContentIndex, resetIndex, locateInFile, config
+        extractKeywords, cleanUpKeywords,
+        indexContentWatch, indexContentBatch,
+        removeContent, getIndexStats,
+        searchContentIndex, resetIndex,
+        locateInFile, config, flush,
     };
 }

package/componets/config/DEFAULT_LOCATE_BATCH_SIZE.ts

@@ -1 +1 @@
-export const DEFAULT_LOCATE_BATCH_SIZE = 3;
+export const DEFAULT_LOCATE_BATCH_SIZE = 8;

package/componets/config/INDEXING_COALESCE_MAX_ITEMS.ts

@@ -0,0 +1 @@
+export const INDEXING_COALESCE_MAX_ITEMS = 1000;

package/componets/config/INDEXING_COALESCE_MS.ts

@@ -0,0 +1 @@
+export const INDEXING_COALESCE_MS = 1000;

package/componets/config/WATCH_COALESCE_MS.ts

@@ -0,0 +1 @@
+export const WATCH_COALESCE_MS = 100;

package/componets/config/loadConfig.ts

@@ -3,12 +3,13 @@ import {IXindexConfig} from "./xindexConfig.js";
 import {ILogger} from "../logger.js";
 
 const DEFAULT_MAX_LINES = 30;
-const DEFAULT_MAX_FILE_BYTES = 1_000_000;
+const DEFAULT_MAX_FILE_BYTES = 50_000;
 const DEFAULT_FOLLOW_SYMLINKS = false;
+const DEFAULT_IGNORE_FILES = ['.xindex', 'node_modules'];
 
 const DEFAULTS: IXindexConfig = {
     ignoreKeywords: [],
-    ignoreFiles: [],
+    ignoreFiles: DEFAULT_IGNORE_FILES,
     maxLines: DEFAULT_MAX_LINES,
     maxFileBytes: DEFAULT_MAX_FILE_BYTES,
     followSymlinks: DEFAULT_FOLLOW_SYMLINKS,

package/componets/index/coalesceIndexApi.ts

@@ -0,0 +1,74 @@
+import {IIndexApi, IndexCommandType} from "./indexApi.js";
+import {ILogger} from "../logger.js";
+import {INDEXING_COALESCE_MAX_ITEMS} from "../config/INDEXING_COALESCE_MAX_ITEMS.js";
+import {Defer, IDefer} from "../../packages/fun/src/defer.js";
+
+type Item = {id: string, vector: number[], keywords: string};
+
+export function CoalesceIndexApi({inner, windowMs, maxItems = INDEXING_COALESCE_MAX_ITEMS, log}: {
+    inner: IIndexApi,
+    windowMs: number,
+    maxItems?: number,
+    log: ILogger,
+}): IIndexApi {
+    if (windowMs <= 0) return inner;
+
+    let pending: Item[] = [];
+    let delivery: IDefer | null = null;
+    let timer: ReturnType<typeof setTimeout> | null = null;
+
+    async function deliver() {
+        if (delivery) { await delivery.promise; return; }
+        if (timer) { clearTimeout(timer); timer = null; }
+        if (pending.length === 0) return;
+
+        const d = Defer();
+        delivery = d;
+        const items = pending;
+        pending = [];
+        try {
+            await inner({type: IndexCommandType.index, items});
+        } catch (e: any) {
+            log(`coalesced flush failed (${items.length} items): ${e?.message ?? e}`);
+        } finally {
+            delivery = null;
+            d.resolve();
+        }
+
+        if (pending.length >= maxItems) {
+            await deliver();
+        } else if (pending.length > 0 && !timer) {
+            timer = setTimeout(deliver, windowMs);
+        }
+    }
+
+    const api = (async function coalescedIndexApi(msg) {
+        switch (msg.type) {
+            case IndexCommandType.index: {
+                // if (delivery) await delivery.promise;
+                pending.push(...msg.items);
+
+                if (!timer) {
+                    timer = setTimeout(deliver, windowMs);
+                }
+
+                if (pending.length >= maxItems) {
+                    await deliver();
+                }
+                return;
+            }
+            case IndexCommandType.delete:
+            case IndexCommandType.reset: {
+                while (pending.length > 0 || delivery) await deliver();
+                return inner(msg);
+            }
+        }
+    }) as IIndexApi;
+
+    api.flush = async () => {
+        while (pending.length > 0 || delivery) await deliver();
+        await inner.flush();
+    };
+
+    return api;
+}

package/componets/index/contentIndexDriver.ts

@@ -7,20 +7,25 @@ import {SearchIndex, ISearchIndex} from "../../features/searchIndex.js";
 import {RemoveContent, IRemoveContent} from "../../features/removeContent.js";
 import {ResetIndex, IResetIndex} from "../../features/resetIndex.js";
 import {VectraIndex} from "./vectraIndex.js";
-import {IndexApi} from "./indexApi.js";
+import {IndexApi, IIndexApi} from "./indexApi.js";
+import {CoalesceIndexApi} from "./coalesceIndexApi.js";
 import {ILocateInFile} from "../locate/locateInFile.js";
 import {ILogger} from "../logger.js";
 
 export type IContentIndexDriver = Readonly<{
     getIndexStats: IGetIndexStats,
-    indexContent: IIndexContent,
+    indexContentWatch: IIndexContent,
+    indexContentBatch: IIndexContent,
     removeContent: IRemoveContent,
     searchContentIndex: ISearchIndex,
     resetIndex: IResetIndex,
     flush: () => Promise<void>,
 }>;
 
-export async function ContentIndexDriver({path, embed, extractKeywords, cleanUpKeywords, locateInFile, scoreThreshold, log}: {
+export async function ContentIndexDriver({
+    path, embed, extractKeywords, cleanUpKeywords, locateInFile, scoreThreshold, log,
+    indexingWatchCoalesceMs = 0, indexingCoalesceMs = 0,
+}: {
     path: string,
     embed: IEmbed,
     extractKeywords: IExtractKeywords,
@@ -28,16 +33,37 @@ export async function ContentIndexDriver({path, embed, extractKeywords, cleanUpK
     locateInFile: ILocateInFile,
     scoreThreshold: number,
     log: ILogger,
+    indexingWatchCoalesceMs?: number,
+    indexingCoalesceMs?: number,
 }): Promise<IContentIndexDriver> {
     const index = await VectraIndex(path + "/semantic");
-    const indexApi = IndexApi({index});
+    const rawIndexApi = IndexApi({index, log});
+    const watchCoalesce = CoalesceIndexApi({inner: rawIndexApi, windowMs: indexingWatchCoalesceMs, log});
+    const batchCoalesce = CoalesceIndexApi({inner: rawIndexApi, windowMs: indexingCoalesceMs, log});
+
+    const flushAll = async () => {
+        await Promise.all([watchCoalesce.flush(), batchCoalesce.flush()]);
+        await rawIndexApi.flush();
+    };
+
+    const drainThen = (inner: IIndexApi): IIndexApi => {
+        const api = (async (msg) => {
+            await flushAll();
+            return inner(msg);
+        }) as IIndexApi;
+        api.flush = flushAll;
+        return api;
+    };
+
+    const drainingApi = drainThen(rawIndexApi);
 
     return {
         getIndexStats: GetIndexStats({index}),
-        indexContent: IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi, log}),
-        removeContent: RemoveContent({indexApi}),
+        indexContentWatch: IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi: watchCoalesce, log}),
+        indexContentBatch: IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi: batchCoalesce, log}),
+        removeContent: RemoveContent({indexApi: drainingApi}),
         searchContentIndex: SearchIndex({extractKeywords, cleanUpKeywords, embed, index, locateInFile, scoreThreshold}),
-        resetIndex: ResetIndex({indexApi}),
-        flush: () => indexApi.flush(),
+        resetIndex: ResetIndex({indexApi: drainingApi}),
+        flush: flushAll,
     };
 }

package/componets/index/documentContentIndexDriver.ts

@@ -84,7 +84,8 @@ export async function DocumentContentIndexDriver({path, model}: {
 
     return {
         getIndexStats,
-        indexContent,
+        indexContentWatch: indexContent,
+        indexContentBatch: indexContent,
         removeContent,
         searchContentIndex,
         resetIndex,

package/componets/index/indexApi.ts

@@ -2,6 +2,7 @@ import {LocalIndex} from "vectra";
 import {IType} from "../IType.js";
 import {ISerial, Serial} from "../../packages/fun/src/serial.js";
 import {caseNever} from "../../packages/fun/src/case-never.js";
+import {ILogger} from "../logger.js";
 
 export enum IndexCommandType {
     index = 'index',
@@ -16,17 +17,29 @@ export type IIndexCommand =
 
 export type IIndexApi = ISerial<IIndexCommand, void>;
 
-export function IndexApi({index}: { index: LocalIndex }): IIndexApi {
+export function IndexApi({index, log}: { index: LocalIndex, log: ILogger }): IIndexApi {
     return Serial<IIndexCommand, void>(async msg => {
         switch (msg.type) {
             case IndexCommandType.delete: {
-                await index.beginUpdate();
                 try {
-                    for (const id of msg.ids) {
-                        await index.deleteItem(id);
+                    await index.beginUpdate();
+                    try {
+                        for (const id of msg.ids) {
+                            try {
+                                await index.deleteItem(id);
+                            } catch (e: any) {
+                                log(`delete failed for ${id}: ${e?.message ?? e}`);
+                            }
+                        }
+                    } finally {
+                        try {
+                            await index.endUpdate();
+                        } catch (e: any) {
+                            log(`index end failed: ${e?.message ?? e}`);
+                        }
                     }
-                } finally {
-                    await index.endUpdate();
+                } catch (e: any) {
+                    log(`index begin failed: ${e?.message ?? e}`);
                 }
                 break;
             }
@@ -35,35 +48,51 @@ export function IndexApi({index}: { index: LocalIndex }): IIndexApi {
                     await index.batchInsertItems(msg.items.map(item => ({
                         id: item.id,
                         vector: item.vector,
-                        metadata: {id: item.id, keywords: item.keywords},
+                        metadata: {},
                     })));
-                } catch (error) {
-                    const errorMessage = String((error as Error)?.message ?? error).toLowerCase();
+                } catch (error: any) {
+                    const errorMessage = String(error?.message ?? error).toLowerCase();
                     const isDuplicateIdError =
                         errorMessage.includes("already exists")
                         || errorMessage.includes("duplicate");
                     if (!isDuplicateIdError) {
-                        throw error;
+                        log(`index write failed (${msg.items.length} items): ${error?.message ?? error}`);
+                        break;
                     }
-
-                    await index.beginUpdate();
                     try {
-                        for (const item of msg.items) {
-                            await index.upsertItem({
-                                id: item.id,
-                                vector: item.vector,
-                                metadata: {id: item.id, keywords: item.keywords},
-                            });
+                        await index.beginUpdate();
+                        try {
+                            for (const item of msg.items) {
+                                try {
+                                    await index.upsertItem({
+                                        id: item.id,
+                                        vector: item.vector,
+                                        metadata: {},
+                                    });
+                                } catch (e: any) {
+                                    log(`index write failed for ${item.id}: ${e?.message ?? e}`);
+                                }
+                            }
+                        } finally {
+                            try {
+                                await index.endUpdate();
+                            } catch (e: any) {
+                                log(`index end failed: ${e?.message ?? e}`);
+                            }
                         }
-                    } finally {
-                        await index.endUpdate();
+                    } catch (e: any) {
+                        log(`index begin failed: ${e?.message ?? e}`);
                     }
                 }
                 break;
             }
             case IndexCommandType.reset: {
-                await index.deleteIndex();
-                await index.createIndex();
+                try {
+                    await index.deleteIndex();
+                    await index.createIndex();
+                } catch (e: any) {
+                    log(`reset failed: ${e?.message ?? e}`);
+                }
                 break;
             }
             default:

package/componets/llm/embed.ts

@@ -1,18 +1,24 @@
 import {pipeline} from "@huggingface/transformers";
 
-const embedder = await pipeline(
-    "feature-extraction",
-    "sentence-transformers/all-MiniLM-L6-v2"
-);
+const MODEL = "sentence-transformers/all-MiniLM-L6-v2";
 
 export type IEmbed = (text: string) => Promise<number[]>;
 
 type Pooling = "none" | "mean" | "cls" | "first_token" | "eos" | "last_token";
 
 export function Embed({pooling, normalize}: {pooling: Pooling, normalize: boolean}): IEmbed {
+    let embedderPromise: Promise<any> | null = null;
+
+    function getEmbedder() {
+        if (!embedderPromise) {
+            embedderPromise = pipeline("feature-extraction", MODEL);
+        }
+        return embedderPromise;
+    }
+
     return async function embed(text) {
+        const embedder = await getEmbedder();
         const result = await embedder(text, {pooling, normalize});
         return Array.from(result.data as Float32Array);
     }
 }
-

package/componets/logger.ts

@@ -1,34 +1,37 @@
 export type ILogger = (...args: any[]) => void;
 
-export function Logger(fn: (...args: any[]) => void): ILogger {
-    return function log(...args) {
-        fn(...args);
-    }
-}
-
 const DEFAULT_FLUSH_MS = 100;
+const DEFAULT_FLUSH_LINES = 15;
 
-export function BufferedLoggerToStdOut(flushMs = DEFAULT_FLUSH_MS): ILogger {
-    return BufferedLogger(s => process.stdout.write(s), flushMs);
+export function BufferedLoggerToStdOut(flushMs = DEFAULT_FLUSH_MS, flushLines = DEFAULT_FLUSH_LINES): ILogger {
+    return BufferedLogger(s => process.stdout.write(s), flushMs, flushLines);
 }
 
-export function BufferedLoggerToStdErr(flushMs = DEFAULT_FLUSH_MS): ILogger {
-    return BufferedLogger(s => process.stderr.write(s), flushMs);
+export function BufferedLoggerToStdErr(flushMs = DEFAULT_FLUSH_MS, flushLines = DEFAULT_FLUSH_LINES): ILogger {
+    return BufferedLogger(s => process.stderr.write(s), flushMs, flushLines);
 }
 
-export function BufferedLogger(write: (text: string) => void, flushMs = DEFAULT_FLUSH_MS): ILogger {
+export function BufferedLogger(
+    write: (text: string) => void,
+    flushMs = DEFAULT_FLUSH_MS,
+    flushLines = DEFAULT_FLUSH_LINES,
+): ILogger {
     let buffer: string[] = [];
     let timer: ReturnType<typeof setTimeout> | null = null;
 
     function flush() {
+        if (timer) { clearTimeout(timer); timer = null; }
         if (buffer.length === 0) return;
         write(buffer.join("\n") + "\n");
         buffer = [];
-        timer = null;
     }
 
     return function log(...args) {
         buffer.push(args.map(a => typeof a === "object" ? JSON.stringify(a) : String(a)).join(" "));
+        if (buffer.length >= flushLines) {
+            flush();
+            return;
+        }
         if (!timer) timer = setTimeout(flush, flushMs);
     }
 }

package/features/indexContent.ts

@@ -6,6 +6,8 @@ import {ILogger} from "../componets/logger.js";
 
 export type IIndexContent = (items: Array<{id: string, content: string}>) => Promise<void>;
 
+const SLOW_WARN_MS = 3000;
+
 export function IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi, log}: {
     extractKeywords: IExtractKeywords,
     cleanUpKeywords: ICleanUpKeywords,
@@ -15,6 +17,10 @@ export function IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi,
 }): IIndexContent {
     return async function indexContent(items) {
         const indexItems = (await Promise.all(items.map(async (item) => {
+            const warn = setTimeout(
+                () => log(`slow: still embedding ${item.id} after ${SLOW_WARN_MS}ms`),
+                SLOW_WARN_MS,
+            );
             try {
                 const keywords = cleanUpKeywords(extractKeywords(item.content)).join(", ");
                 const vector = await embed(keywords);
@@ -22,11 +28,21 @@ export function IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi,
             } catch (e: any) {
                 log(`skip ${item.id}: keywords/embed failed — ${e?.code ?? e?.message ?? e}`);
                 return null;
+            } finally {
+                clearTimeout(warn);
             }
         }))).filter((x): x is NonNullable<typeof x> => !!x);
 
         if (indexItems.length > 0) {
-            await indexApi({type: IndexCommandType.index, items: indexItems});
+            const warn = setTimeout(
+                () => log(`slow: still writing index after ${SLOW_WARN_MS}ms — ${indexItems.map(i => i.id).join(", ")}`),
+                SLOW_WARN_MS,
+            );
+            try {
+                await indexApi({type: IndexCommandType.index, items: indexItems});
+            } finally {
+                clearTimeout(warn);
+            }
         }
     }
 }

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "xindex",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "Local semantic code search — index codebase, search by meaning or keywords",
   "type": "module",
   "main": "xindex.ts",
   "bin": {
+    "xindex": "bin/xindex-mcp",
     "xindex-index": "bin/xindex-index",
     "xindex-search": "bin/xindex-search",
     "xindex-mcp": "bin/xindex-mcp",