xindex 1.0.12 → 1.0.13

package/.mcp.json

@@ -1,5 +1,9 @@
 {
   "mcpServers": {
+    "fetch": {
+      "command": "docker",
+      "args": ["run", "-i", "--rm", "mcp/fetch"]
+    },
     "xindex": {
       "command": "xindex-mcp",
       "args": []

package/apps/indexApp.ts

@@ -3,11 +3,11 @@ import {batch} from "../packages/streamx/src/batch.js";
 import {map} from "../packages/streamx/src/map.js";
 import {tap} from "../packages/streamx/src/tap.js";
 import {run} from "../packages/streamx/src/index.js";
-import {IWalkFiles} from "../componets/walkFiles.js";
+import {IWalkFiles} from "../components/walkFiles.js";
 import {IIndexContent} from "../features/indexContent.js";
-import {ILogger} from "../componets/logger.js";
-import {INDEXING_BATCH_SIZE} from "../componets/config/INDEXING_BATCH_SIZE";
-import {SafeIndexBatch} from "../componets/io/safeIndexBatch.js";
+import {ILogger} from "../components/logger.js";
+import {INDEXING_BATCH_SIZE} from "../components/config/INDEXING_BATCH_SIZE";
+import {SafeIndexBatch} from "../components/io/safeIndexBatch.js";
 
 export type IIndexApp = (inputs: string[]) => Promise<void>;
 

package/apps/mcpApp.ts

@@ -2,15 +2,15 @@ import {McpServer} from "@modelcontextprotocol/sdk/server/mcp.js";
 import {StdioServerTransport} from "@modelcontextprotocol/sdk/server/stdio.js";
 import {z} from "zod";
 import {IIndexApp} from "./indexApp.js";
-import {IGetIndexStats} from "../componets/index/getIndexStats.js";
+import {IGetIndexStats} from "../components/index/getIndexStats.js";
 import {IResetIndex} from "../features/resetIndex.js";
-import {IWatchFiles} from "../componets/watchFiles.js";
-import {IHandleFileEvents} from "../componets/index/handleFileEvent.js";
-import {ILogger} from "../componets/logger.js";
+import {IWatchFiles} from "../components/watchFiles.js";
+import {IHandleFileEvents} from "../components/index/handleFileEvent.js";
+import {ILogger} from "../components/logger.js";
 import {WatchFileEventsApp} from "./watchFileEventsApp.js";
-import {IWatcherLock} from "../componets/index/watcherLock.js";
-import {IXindexConfig} from "../componets/config/xindexConfig.js";
-import {FormatSearchResults} from "../componets/index/formatSearchResults.js";
+import {IWatcherLock} from "../components/index/watcherLock.js";
+import {IXindexConfig} from "../components/config/xindexConfig.js";
+import {FormatSearchResults} from "../components/index/formatSearchResults.js";
 import {ISearchApp} from "./searchApp.js";
 
 export type IMcpApp = () => Promise<void>;

package/apps/run.index.ts

@@ -1,9 +1,9 @@
-import {BuildComponents} from "../componets/buildComponents.js";
-import {BufferedLoggerToStdOut} from "../componets/logger.js";
-import {WalkFiles} from "../componets/walkFiles.js";
+import {BuildComponents} from "../components/buildComponents.js";
+import {BufferedLoggerToStdOut} from "../components/logger.js";
+import {WalkFiles} from "../components/walkFiles.js";
 import {IndexApp} from "./indexApp.js";
-import {AppId} from "../componets/appId.js";
-import {INDEXING_COALESCE_MS} from "../componets/config/INDEXING_COALESCE_MS.js";
+import {AppId} from "../components/appId.js";
+import {INDEXING_COALESCE_MS} from "../components/config/INDEXING_COALESCE_MS.js";
 
 const appId = AppId();
 const cwd = process.cwd();

package/apps/run.mcp.ts

@@ -1,16 +1,16 @@
-import {BuildComponents} from "../componets/buildComponents.js";
-import {HandleFileEvents} from "../componets/index/handleFileEvent.js";
-import {BufferedLoggerToStdErr} from "../componets/logger.js";
-import {WalkFiles} from "../componets/walkFiles.js";
-import {WatchFiles} from "../componets/watchFiles.js";
-import {WatcherLock} from "../componets/index/watcherLock.js";
+import {BuildComponents} from "../components/buildComponents.js";
+import {HandleFileEvents} from "../components/index/handleFileEvent.js";
+import {BufferedLoggerToStdErr} from "../components/logger.js";
+import {WalkFiles} from "../components/walkFiles.js";
+import {WatchFiles} from "../components/watchFiles.js";
+import {WatcherLock} from "../components/index/watcherLock.js";
 import {IndexApp} from "./indexApp.js";
 import {McpApp} from "./mcpApp.js";
 import {SearchApp} from "./searchApp.js";
 import {join} from "path";
-import {AppId} from "../componets/appId.js";
-import {INDEXING_COALESCE_MS} from "../componets/config/INDEXING_COALESCE_MS.js";
-import {WATCH_COALESCE_MS} from "../componets/config/WATCH_COALESCE_MS.js";
+import {AppId} from "../components/appId.js";
+import {INDEXING_COALESCE_MS} from "../components/config/INDEXING_COALESCE_MS.js";
+import {WATCH_COALESCE_MS} from "../components/config/WATCH_COALESCE_MS.js";
 
 const args = process.argv.slice(2);
 const watchDirArg = args.find(a => a.startsWith("--watch-dir="));

package/apps/run.reset.ts

@@ -1,6 +1,6 @@
-import {BuildComponents} from "../componets/buildComponents.js";
-import {BufferedLoggerToStdOut} from "../componets/logger.js";
-import {AppId} from "../componets/appId.js";
+import {BuildComponents} from "../components/buildComponents.js";
+import {BufferedLoggerToStdOut} from "../components/logger.js";
+import {AppId} from "../components/appId.js";
 
 const appId = AppId();
 const log = BufferedLoggerToStdOut();

package/apps/run.search.ts

@@ -1,7 +1,7 @@
-import {BuildComponents} from "../componets/buildComponents.js";
-import {BufferedLoggerToStdOut} from "../componets/logger.js";
+import {BuildComponents} from "../components/buildComponents.js";
+import {BufferedLoggerToStdOut} from "../components/logger.js";
 import {SearchApp} from "./searchApp.js";
-import {FormatSearchResults} from "../componets/index/formatSearchResults.js";
+import {FormatSearchResults} from "../components/index/formatSearchResults.js";
 
 const log = BufferedLoggerToStdOut();
 const {searchContentIndex} = await BuildComponents({log});

package/apps/run.watch.ts

@@ -1,14 +1,14 @@
-import {BuildComponents} from "../componets/buildComponents.js";
-import {HandleFileEvents} from "../componets/index/handleFileEvent.js";
-import {BufferedLoggerToStdOut} from "../componets/logger.js";
-import {WalkFiles} from "../componets/walkFiles.js";
-import {WatchFiles} from "../componets/watchFiles.js";
-import {WatcherLock} from "../componets/index/watcherLock.js";
+import {BuildComponents} from "../components/buildComponents.js";
+import {HandleFileEvents} from "../components/index/handleFileEvent.js";
+import {BufferedLoggerToStdOut} from "../components/logger.js";
+import {WalkFiles} from "../components/walkFiles.js";
+import {WatchFiles} from "../components/watchFiles.js";
+import {WatcherLock} from "../components/index/watcherLock.js";
 import {WatchApp} from "./watchApp.js";
-import {AppId} from "../componets/appId.js";
+import {AppId} from "../components/appId.js";
 import {join} from "path";
-import {WATCH_COALESCE_MS} from "../componets/config/WATCH_COALESCE_MS.js";
-import {INDEXING_COALESCE_MS} from "../componets/config/INDEXING_COALESCE_MS.js";
+import {WATCH_COALESCE_MS} from "../components/config/WATCH_COALESCE_MS.js";
+import {INDEXING_COALESCE_MS} from "../components/config/INDEXING_COALESCE_MS.js";
 
 const cwd = process.cwd();
 const log = BufferedLoggerToStdOut();

package/apps/watchApp.ts

@@ -3,15 +3,15 @@ import {batch} from "../packages/streamx/src/batch.js";
 import {map} from "../packages/streamx/src/map.js";
 import {tap} from "../packages/streamx/src/tap.js";
 import {run} from "../packages/streamx/src/index.js";
-import {IWalkFiles} from "../componets/walkFiles.js";
-import {IWatchFiles} from "../componets/watchFiles.js";
-import {IHandleFileEvents} from "../componets/index/handleFileEvent.js";
-import {ILogger} from "../componets/logger.js";
-import {IWatcherLock} from "../componets/index/watcherLock.js";
+import {IWalkFiles} from "../components/walkFiles.js";
+import {IWatchFiles} from "../components/watchFiles.js";
+import {IHandleFileEvents} from "../components/index/handleFileEvent.js";
+import {ILogger} from "../components/logger.js";
+import {IWatcherLock} from "../components/index/watcherLock.js";
 import {WatchFileEventsApp} from "./watchFileEventsApp.js";
 import {IIndexContent} from "../features/indexContent.js";
-import {INDEXING_BATCH_SIZE} from "../componets/config/INDEXING_BATCH_SIZE";
-import {SafeIndexBatch} from "../componets/io/safeIndexBatch.js";
+import {INDEXING_BATCH_SIZE} from "../components/config/INDEXING_BATCH_SIZE";
+import {SafeIndexBatch} from "../components/io/safeIndexBatch.js";
 
 export type IWatchApp = {
     run: (inputs: string[]) => Promise<void>;

package/apps/watchFileEventsApp.ts

@@ -1,13 +1,13 @@
-import {IWatchFiles} from "../componets/watchFiles.js";
-import {IHandleFileEvents} from "../componets/index/handleFileEvent.js";
-import {ILogger} from "../componets/logger.js";
-import {IWatcherLock} from "../componets/index/watcherLock.js";
+import {IWatchFiles} from "../components/watchFiles.js";
+import {IHandleFileEvents} from "../components/index/handleFileEvent.js";
+import {ILogger} from "../components/logger.js";
+import {IWatcherLock} from "../components/index/watcherLock.js";
 import {from} from "../packages/streamx/src/from.js";
 import {batchTimed} from "../packages/streamx/src/batchTimed.js";
 import {map} from "../packages/streamx/src/map.js";
 import {run} from "../packages/streamx/src/index.js";
-import {INDEXING_BATCH_SIZE} from "../componets/config/INDEXING_BATCH_SIZE.js";
-import {WATCH_FLUSH_MS} from "../componets/config/WATCH_FLUSH_MS.js";
+import {INDEXING_BATCH_SIZE} from "../components/config/INDEXING_BATCH_SIZE.js";
+import {WATCH_FLUSH_MS} from "../components/config/WATCH_FLUSH_MS.js";
 
 export type IWatchFileEventsApp = () => void;
 

package/{componets → components}/llm/embed.ts

@@ -7,17 +7,12 @@ export type IEmbed = (text: string) => Promise<number[]>;
 type Pooling = "none" | "mean" | "cls" | "first_token" | "eos" | "last_token";
 
 export function Embed({pooling, normalize}: {pooling: Pooling, normalize: boolean}): IEmbed {
-    let embedderPromise: Promise<any> | null = null;
-
-    function getEmbedder() {
-        if (!embedderPromise) {
-            embedderPromise = pipeline("feature-extraction", MODEL);
-        }
-        return embedderPromise;
-    }
+    let embedder: any = undefined
 
     return async function embed(text) {
-        const embedder = await getEmbedder();
+        if (!embedder) {
+            embedder = await pipeline("feature-extraction", MODEL);
+        }
         const result = await embedder(text, {pooling, normalize});
         return Array.from(result.data as Float32Array);
     }

package/components/llm/queryLLM.ts

@@ -0,0 +1,31 @@
+import {pipeline} from "@huggingface/transformers";
+
+
+export type IQueryLLM = (prompt: string, input: string, maxTokens?: number) => Promise<string>;
+
+const MODEL = "HuggingFaceTB/SmolLM2-135M-Instruct";
+
+export function QueryLLM({maxTokens, model = MODEL}: {
+    maxTokens: number,
+    model?: string
+}): IQueryLLM {
+    let generator: any = undefined;
+
+    return async function queryLLM(prompt, input, limit = maxTokens) {
+        if (!generator) {
+            generator = await pipeline(
+                "text-generation",
+                model
+            );
+        }
+
+        const messages = [
+            {role: "system", content: prompt},
+            {role: "user", content: input},
+        ];
+
+        const output = await generator(messages, {max_new_tokens: limit ?? maxTokens});
+        return (output[0].generated_text.at(-1)?.content as string) ?? "";
+    }
+}
+

package/features/indexContent.ts

@@ -1,8 +1,8 @@
-import {IndexCommandType, IIndexApi} from "../componets/index/indexApi.js";
-import {IExtractKeywords} from "../componets/keywords/extractKeywords.js";
-import {ICleanUpKeywords} from "../componets/keywords/cleanUpKeywords.js";
-import {IEmbed} from "../componets/llm/embed.js";
-import {ILogger} from "../componets/logger.js";
+import {IndexCommandType, IIndexApi} from "../components/index/indexApi.js";
+import {IExtractKeywords} from "../components/keywords/extractKeywords.js";
+import {ICleanUpKeywords} from "../components/keywords/cleanUpKeywords.js";
+import {IEmbed} from "../components/llm/embed.js";
+import {ILogger} from "../components/logger.js";
 
 export type IIndexContent = (items: Array<{id: string, content: string}>) => Promise<void>;
 

package/features/removeContent.ts

@@ -1,4 +1,4 @@
-import {IndexCommandType, IIndexApi} from "../componets/index/indexApi.js";
+import {IndexCommandType, IIndexApi} from "../components/index/indexApi.js";
 
 export type IRemoveContent = (ids: string[]) => Promise<void>;
 

package/features/resetIndex.ts

@@ -1,4 +1,4 @@
-import {IndexCommandType, IIndexApi} from "../componets/index/indexApi.js";
+import {IndexCommandType, IIndexApi} from "../components/index/indexApi.js";
 
 export type IResetIndex = () => Promise<void>;
 

package/features/searchIndex.ts

@@ -1,8 +1,8 @@
 import {LocalIndex} from "vectra";
-import {IEmbed} from "../componets/llm/embed.js";
-import {IExtractKeywords} from "../componets/keywords/extractKeywords.js";
-import {ICleanUpKeywords} from "../componets/keywords/cleanUpKeywords.js";
-import {ILocateInFile} from "../componets/locate/locateInFile.js";
+import {IEmbed} from "../components/llm/embed.js";
+import {IExtractKeywords} from "../components/keywords/extractKeywords.js";
+import {ICleanUpKeywords} from "../components/keywords/cleanUpKeywords.js";
+import {ILocateInFile} from "../components/locate/locateInFile.js";
 
 export type IIndexRecord = {
     score: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xindex",
-  "version": "1.0.12",
+  "version": "1.0.13",
   "description": "Local semantic code search — index codebase, search by meaning or keywords",
   "type": "module",
   "main": "xindex.ts",

package/componets/llm/queryLLM.ts

@@ -1,20 +0,0 @@
-import {pipeline} from "@huggingface/transformers";
-
-const generator = await pipeline(
-    "text-generation",
-    "HuggingFaceTB/SmolLM2-135M-Instruct"
-);
-
-export type IQueryLLM = (prompt: string, input: string, maxTokens?: number) => Promise<string>;
-
-export function QueryLLM({maxTokens}: { maxTokens: number }): IQueryLLM {
-    return async function queryLLM(prompt, input, overrideMaxTokens?) {
-        const messages = [
-            {role: "system", content: prompt},
-            {role: "user", content: input},
-        ];
-        const output = await generator(messages, {max_new_tokens: overrideMaxTokens ?? maxTokens});
-        return (output[0].generated_text.at(-1)?.content as string) ?? "";
-    }
-}
-