xindex 1.0.3 → 1.0.4

package/.xindex.json

@@ -18,5 +18,6 @@
     "utf8", "length", "map", "slice", "push", "join", "resolve", "stringify",
     "json", "settimeout", "path", "readfile"
   ],
-  "ignoreFiles": [".xindex"]
+  "ignoreFiles": [".xindex", "media"],
+  "maxLines": 12
 }

package/CLAUDE.md

@@ -9,6 +9,7 @@
 - **Text diagrams** — ASCII flows, hierarchies, tables. Keep minimal.
 - **research/search/ground** — search the Internet using DuckDuckGo MCP
 - **plan dev / go / dev / implement** — start implementation → triggers Pre-implementation check
+- **check types / test compilation** — run `yarn test.compilation` to validate TypeScript compilation without relying on `tsconfig.json`
 - **recover** — find most recent `task.*.md` in `.ai/task/` (by date+mtime, exclude `*.log.md`/`*.report.md`), summarize state and next steps
 - **pull details / expand / flesh out** — enrich task with full detail while preserving shape (see Detail expansion)
 

package/README.md

@@ -233,6 +233,12 @@ yarn install   # or npm install
 npm link       # exposes xindex-* binaries from your working copy
 ```
 
+Check TypeScript compilation:
+
+```bash
+yarn test.compilation
+```
+
 ## License
 
 MIT

package/apps/indexApp.ts

@@ -1,30 +1,31 @@
 import {readFile} from "fs/promises";
 import {from} from "../packages/streamx/src/from.js";
+import {batch} from "../packages/streamx/src/batch.js";
 import {map} from "../packages/streamx/src/map.js";
 import {tap} from "../packages/streamx/src/tap.js";
 import {run} from "../packages/streamx/src/index.js";
 import {IWalkFiles} from "../componets/walkFiles.js";
 import {IIndexContent} from "../features/indexContent.js";
-import {IRemoveContent} from "../features/removeContent.js";
 import {ILogger} from "../componets/logger.js";
+import {INDEXING_BATCH_SIZE} from "../componets/config/INDEXING_BATCH_SIZE";
 
 export type IIndexApp = (inputs: string[]) => Promise<void>;
 
-export function IndexApp({walkFiles, indexContent, removeContent, log}: {
+export function IndexApp({walkFiles, indexContent, log}: {
     walkFiles: IWalkFiles,
     indexContent: IIndexContent,
-    removeContent: IRemoveContent,
     log: ILogger,
 }): IIndexApp {
     return async function indexApp(inputs) {
         await run(
             from(walkFiles(inputs))
                 .pipe(tap(id => log(`indexing: ${id}`)))
-                .pipe(map<string, string>(async (id) => {
-                    try { await removeContent(id); } catch (e) { log(`remove failed: ${id} — ${(e as any)?.message ?? e}`); }
-                    const text = await readFile(id, "utf8");
-                    await indexContent(id, `${text}. ${id}`);
-                    return id;
+                .pipe(batch(INDEXING_BATCH_SIZE))
+                .pipe(map<string[], string[]>(async (ids) => {
+                    const texts = await Promise.all(ids.map(id => readFile(id, "utf8")));
+                    const items = ids.map((id, i) => ({id, content: `${texts[i]}. ${id}`}));
+                    await indexContent(items);
+                    return ids;
                 }))
         );
     }

package/apps/mcpApp.ts

@@ -1,31 +1,31 @@
 import {McpServer} from "@modelcontextprotocol/sdk/server/mcp.js";
 import {StdioServerTransport} from "@modelcontextprotocol/sdk/server/stdio.js";
 import {z} from "zod";
-import {ISearchIndex} from "../features/searchIndex.js";
 import {IIndexApp} from "./indexApp.js";
 import {IGetIndexStats} from "../componets/index/getIndexStats.js";
 import {IResetIndex} from "../features/resetIndex.js";
 import {IWatchFiles} from "../componets/watchFiles.js";
-import {IHandleFileEvent} from "../componets/index/handleFileEvent.js";
+import {IHandleFileEvents} from "../componets/index/handleFileEvent.js";
 import {ILogger} from "../componets/logger.js";
 import {WatchFileEventsApp} from "./watchFileEventsApp.js";
 import {IWatcherLock} from "../componets/index/watcherLock.js";
 import {IXindexConfig} from "../componets/config/xindexConfig.js";
 import {FormatSearchResults} from "../componets/index/formatSearchResults.js";
+import {ISearchApp} from "./searchApp.js";
 
 export type IMcpApp = () => Promise<void>;
 
 export type IMcpWatch = {
     watchFiles: IWatchFiles,
     watchDir: string,
-    handleFileEvent: IHandleFileEvent,
+    handleFileEvents: IHandleFileEvents,
     watcherLock: IWatcherLock,
 };
 
 export function McpApp({
-    searchContentIndex, indexApp, getIndexStats, resetIndex, log, watch, config,
+    search, indexApp, getIndexStats, resetIndex, log, watch, config,
 }: {
-    searchContentIndex: ISearchIndex,
+    search: ISearchApp,
     indexApp: IIndexApp,
     getIndexStats: IGetIndexStats,
     resetIndex: IResetIndex,
@@ -51,7 +51,7 @@ export function McpApp({
         }, async ({query, limit}) => {
             try {
                 const format = FormatSearchResults();
-                const results = await searchContentIndex(query, limit);
+                const results = await search(query, limit);
                 const text = await format(query, results);
                 return {content: [{type: "text" as const, text}]};
             } catch (e) {

package/apps/run.index.ts

@@ -7,9 +7,9 @@ import {AppId} from "../componets/appId.js";
 const appId = AppId();
 const cwd = process.cwd();
 const log = BufferedLoggerToStdOut();
-const {indexContent, removeContent, getIndexStats, config} = await BuildComponents({log});
+const {indexContent, getIndexStats, config} = await BuildComponents({log});
 const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles});
-const indexApp = IndexApp({walkFiles, indexContent, removeContent, log});
+const indexApp = IndexApp({walkFiles, indexContent, log});
 
 const inputs = process.argv.slice(2);
 if (!inputs.length) inputs.push(".");

package/apps/run.mcp.ts

@@ -1,11 +1,12 @@
 import {BuildComponents} from "../componets/buildComponents.js";
-import {HandleFileEvent} from "../componets/index/handleFileEvent.js";
+import {HandleFileEvents} from "../componets/index/handleFileEvent.js";
 import {BufferedLoggerToStdErr} from "../componets/logger.js";
 import {WalkFiles} from "../componets/walkFiles.js";
 import {WatchFiles} from "../componets/watchFiles.js";
 import {WatcherLock} from "../componets/index/watcherLock.js";
 import {IndexApp} from "./indexApp.js";
 import {McpApp} from "./mcpApp.js";
+import {SearchApp} from "./searchApp.js";
 import {join} from "path";
 import {AppId} from "../componets/appId.js";
 
@@ -17,7 +18,8 @@ const cwd = process.cwd();
 const log = BufferedLoggerToStdErr();
 const {indexContent, removeContent, getIndexStats, searchContentIndex, resetIndex, config} = await BuildComponents({log});
 const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles});
-const indexApp = IndexApp({walkFiles, indexContent, removeContent, log});
+const indexApp = IndexApp({walkFiles, indexContent, log});
+const search = SearchApp({searchContentIndex});
 
 const appId = AppId();
 const watcherLock = WatcherLock({
@@ -29,7 +31,7 @@ const watcherLock = WatcherLock({
 const watch = watchDisabled ? undefined : {
     watchFiles: WatchFiles({cwd, log, ignoreFiles: config.ignoreFiles}),
     watchDir: watchDirArg ? watchDirArg.split("=")[1] : ".",
-    handleFileEvent: HandleFileEvent({indexContent, removeContent, log}),
+    handleFileEvents: HandleFileEvents({indexContent, removeContent, log}),
     watcherLock,
 };
 
@@ -45,5 +47,5 @@ process.on("SIGINT", async () => {
 });
 
 log(`[${appId}] started`);
-const mcpApp = McpApp({searchContentIndex, indexApp, getIndexStats, resetIndex, log, watch, config});
+const mcpApp = McpApp({search, indexApp, getIndexStats, resetIndex, log, watch, config});
 await mcpApp();

package/apps/run.search.ts

@@ -4,7 +4,7 @@ import {SearchApp} from "./searchApp.js";
 import {FormatSearchResults} from "../componets/index/formatSearchResults.js";
 
 const log = BufferedLoggerToStdOut();
-const {searchContentIndex, config} = await BuildComponents({log});
+const {searchContentIndex} = await BuildComponents({log});
 const search = SearchApp({searchContentIndex});
 
 const query = process.argv.slice(2).join(" ");

package/apps/run.watch.ts

@@ -1,5 +1,5 @@
 import {BuildComponents} from "../componets/buildComponents.js";
-import {HandleFileEvent} from "../componets/index/handleFileEvent.js";
+import {HandleFileEvents} from "../componets/index/handleFileEvent.js";
 import {BufferedLoggerToStdOut} from "../componets/logger.js";
 import {WalkFiles} from "../componets/walkFiles.js";
 import {WatchFiles} from "../componets/watchFiles.js";
@@ -13,7 +13,7 @@ const log = BufferedLoggerToStdOut();
 const {indexContent, removeContent, getIndexStats, config} = await BuildComponents({log});
 const walkFiles = WalkFiles({cwd, log, ignoreFiles: config.ignoreFiles});
 const watchFiles = WatchFiles({cwd, log, ignoreFiles: config.ignoreFiles});
-const handleFileEvent = HandleFileEvent({indexContent, removeContent, log});
+const handleFileEvents = HandleFileEvents({indexContent, removeContent, log});
 
 const appId = AppId();
 const watcherLock = WatcherLock({
@@ -22,7 +22,7 @@ const watcherLock = WatcherLock({
     log,
 });
 
-const app = WatchApp({walkFiles, watchFiles, handleFileEvent, log, watcherLock});
+const app = WatchApp({walkFiles, watchFiles, handleFileEvents, indexContent, log, watcherLock});
 
 const inputs = process.argv.slice(2);
 if (!inputs.length) inputs.push(".");

package/apps/searchApp.ts

@@ -2,8 +2,10 @@ import {ISearchIndex, IIndexRecord} from "../features/searchIndex.js";
 
 export type ISearchApp = (query: string, limit?: number) => Promise<IIndexRecord[]>;
 
-export function SearchApp({searchContentIndex}: {searchContentIndex: ISearchIndex}): ISearchApp {
+export function SearchApp({searchContentIndex}: {
+    searchContentIndex: ISearchIndex;
+}): ISearchApp {
     return async function search(query, limit = 7) {
-        return await searchContentIndex(query, limit);
+        return searchContentIndex(query, limit);
     }
 }

package/apps/watchApp.ts

@@ -1,23 +1,28 @@
 import {from} from "../packages/streamx/src/from.js";
+import {batch} from "../packages/streamx/src/batch.js";
 import {map} from "../packages/streamx/src/map.js";
 import {tap} from "../packages/streamx/src/tap.js";
 import {run} from "../packages/streamx/src/index.js";
+import {readFile} from "fs/promises";
 import {IWalkFiles} from "../componets/walkFiles.js";
-import {FileEventType, IWatchFiles} from "../componets/watchFiles.js";
-import {IHandleFileEvent} from "../componets/index/handleFileEvent.js";
+import {IWatchFiles} from "../componets/watchFiles.js";
+import {IHandleFileEvents} from "../componets/index/handleFileEvent.js";
 import {ILogger} from "../componets/logger.js";
 import {IWatcherLock} from "../componets/index/watcherLock.js";
 import {WatchFileEventsApp} from "./watchFileEventsApp.js";
+import {IIndexContent} from "../features/indexContent.js";
+import {INDEXING_BATCH_SIZE} from "../componets/config/INDEXING_BATCH_SIZE";
 
 export type IWatchApp = {
     run: (inputs: string[]) => Promise<void>;
     stop: () => void;
 };
 
-export function WatchApp({walkFiles, watchFiles, handleFileEvent, log, watcherLock}: {
+export function WatchApp({walkFiles, watchFiles, handleFileEvents, indexContent, log, watcherLock}: {
     walkFiles: IWalkFiles,
     watchFiles: IWatchFiles,
-    handleFileEvent: IHandleFileEvent,
+    handleFileEvents: IHandleFileEvents,
+    indexContent: IIndexContent,
     log: ILogger,
     watcherLock: IWatcherLock,
 }): IWatchApp {
@@ -27,9 +32,12 @@ export function WatchApp({walkFiles, watchFiles, handleFileEvent, log, watcherLo
             await run(
                 from(walkFiles(inputs))
                     .pipe(tap(id => log(`indexing: ${id}`)))
-                    .pipe(map<string, string>(async (id) => {
-                        await handleFileEvent({type: FileEventType.index, path: id});
-                        return id;
+                    .pipe(batch(INDEXING_BATCH_SIZE))
+                    .pipe(map<string[], string[]>(async (ids) => {
+                        const texts = await Promise.all(ids.map(id => readFile(id, "utf8")));
+                        const items = ids.map((id, i) => ({id, content: `${texts[i]}. ${id}`}));
+                        await indexContent(items);
+                        return ids;
                     }))
             );
 
@@ -39,7 +47,7 @@ export function WatchApp({walkFiles, watchFiles, handleFileEvent, log, watcherLo
             const startWatch = WatchFileEventsApp({
                 watchFiles,
                 watchDir: inputs[0] ?? ".",
-                handleFileEvent,
+                handleFileEvents,
                 log,
                 watcherLock,
             });

package/apps/watchFileEventsApp.ts

@@ -1,14 +1,20 @@
 import {IWatchFiles} from "../componets/watchFiles.js";
-import {IHandleFileEvent} from "../componets/index/handleFileEvent.js";
+import {IHandleFileEvents} from "../componets/index/handleFileEvent.js";
 import {ILogger} from "../componets/logger.js";
 import {IWatcherLock} from "../componets/index/watcherLock.js";
+import {from} from "../packages/streamx/src/from.js";
+import {batchTimed} from "../packages/streamx/src/batchTimed.js";
+import {map} from "../packages/streamx/src/map.js";
+import {run} from "../packages/streamx/src/index.js";
+import {INDEXING_BATCH_SIZE} from "../componets/config/INDEXING_BATCH_SIZE.js";
+import {WATCH_FLUSH_MS} from "../componets/config/WATCH_FLUSH_MS.js";
 
 export type IWatchFileEventsApp = () => void;
 
-export function WatchFileEventsApp({watchFiles, watchDir, handleFileEvent, log, watcherLock}: {
+export function WatchFileEventsApp({watchFiles, watchDir, handleFileEvents, log, watcherLock}: {
     watchFiles: IWatchFiles,
     watchDir: string,
-    handleFileEvent: IHandleFileEvent,
+    handleFileEvents: IHandleFileEvents,
     log: ILogger,
     watcherLock: IWatcherLock,
 }): IWatchFileEventsApp {
@@ -21,7 +27,11 @@ export function WatchFileEventsApp({watchFiles, watchDir, handleFileEvent, log,
                     const watcher = watchFiles([watchDir]);
                     const events = (async () => {
                         try {
-                            for await (const e of watcher.events) await handleFileEvent(e);
+                            await run(
+                                from(watcher.events)
+                                    .pipe(batchTimed(INDEXING_BATCH_SIZE, WATCH_FLUSH_MS))
+                                    .pipe(map(handleFileEvents))
+                            );
                         } catch (e) {
                             log(`watch error: ${(e as any)?.message ?? e}`);
                         }

package/componets/buildComponents.ts

@@ -4,24 +4,40 @@ import {CleanUpKeywords} from "./keywords/cleanUpKeywords.js";
 import {ContentIndexDriver} from "./index/contentIndexDriver.js";
 import {LoadConfig} from "./config/loadConfig.js";
 import {ILogger} from "./logger.js";
+import {LocateInFile} from "./locate/locateInFile.js";
 
-export async function BuildComponents({log}: {log: ILogger}) {
+export async function BuildComponents({log}: { log: ILogger }) {
     const loadConfig = LoadConfig({configPath: ".xindex.json", log});
     const config = await loadConfig();
 
     const embed = Embed({pooling: "mean", normalize: true});
     const extractKeywords = ExtractKeywords();
-    const cleanUpKeywords = CleanUpKeywords({maxNgrams: 2, minLength: 2, ignoreKeywords: config.ignoreKeywords});
+    const cleanUpKeywords = CleanUpKeywords({maxNgrams: 2, minLength: 1, ignoreKeywords: config.ignoreKeywords});
+
+    const locateInFile = LocateInFile({
+        embed,
+        extractKeywords,
+        cleanUpKeywords,
+        windowLines: config.maxLines,
+        maxFileBytes: config.maxFileBytes,
+    });
 
     const DEFAULT_INDEX_PATH = ".xindex";
 
+    const SCORE_THRESHOLD = 0.01;
+
     const {indexContent, removeContent, getIndexStats, searchContentIndex, resetIndex}
         = await ContentIndexDriver({
-            path: DEFAULT_INDEX_PATH,
-            embed,
-            extractKeywords,
-            cleanUpKeywords,
-        });
-    return {extractKeywords, cleanUpKeywords, indexContent, removeContent, getIndexStats,
-            searchContentIndex, resetIndex, config};
+        path: DEFAULT_INDEX_PATH,
+        embed,
+        extractKeywords,
+        cleanUpKeywords,
+        locateInFile,
+        scoreThreshold: SCORE_THRESHOLD
+    });
+
+    return {
+        extractKeywords, cleanUpKeywords, indexContent, removeContent, getIndexStats,
+        searchContentIndex, resetIndex, locateInFile, config
+    };
 }

package/componets/config/DEFAULT_LOCATE_BATCH_SIZE.ts

@@ -0,0 +1 @@
+export const DEFAULT_LOCATE_BATCH_SIZE = 3;

package/componets/config/INDEXING_BATCH_SIZE.ts

@@ -0,0 +1 @@
+export const INDEXING_BATCH_SIZE = 5;

package/componets/config/WATCH_FLUSH_MS.ts

@@ -0,0 +1 @@
+export const WATCH_FLUSH_MS = 500;

package/componets/config/loadConfig.ts

@@ -2,14 +2,19 @@ import {readFile} from "fs/promises";
 import {IXindexConfig} from "./xindexConfig.js";
 import {ILogger} from "../logger.js";
 
+const DEFAULT_MAX_LINES = 30;
+const DEFAULT_MAX_FILE_BYTES = 5_000_000;
+
 const DEFAULTS: IXindexConfig = {
     ignoreKeywords: [],
     ignoreFiles: [],
+    maxLines: DEFAULT_MAX_LINES,
+    maxFileBytes: DEFAULT_MAX_FILE_BYTES,
 };
 
 export type ILoadConfig = () => Promise<IXindexConfig>;
 
-export function LoadConfig({configPath, log}: {configPath: string, log: ILogger}): ILoadConfig {
+export function LoadConfig({configPath, log}: { configPath: string, log: ILogger }): ILoadConfig {
     return async function loadConfig() {
         let raw: string;
         try {
@@ -29,9 +34,13 @@ export function LoadConfig({configPath, log}: {configPath: string, log: ILogger}
         }
 
         const toStrings = (v: unknown) => Array.isArray(v) ? v.filter((e): e is string => typeof e === "string") : [];
+        const toNum = (v: unknown, def: number): number => typeof v === "number" ? v : def;
+
         const config: IXindexConfig = {
             ignoreKeywords: toStrings(parsed.ignoreKeywords),
             ignoreFiles: toStrings(parsed.ignoreFiles),
+            maxLines: toNum(parsed.maxLines, DEFAULT_MAX_LINES),
+            maxFileBytes: toNum(parsed.maxFileBytes, DEFAULT_MAX_FILE_BYTES),
         };
 
         for (const kw of config.ignoreKeywords) {

package/componets/config/xindexConfig.ts

@@ -1,4 +1,6 @@
 export type IXindexConfig = {
     ignoreKeywords: string[];
     ignoreFiles: string[];
+    maxLines: number;
+    maxFileBytes: number;
 };

package/componets/ignore/loadIgnoreChain.ts

@@ -0,0 +1,40 @@
+import {readFile} from "fs/promises";
+import {join, dirname, relative} from "path";
+import ignore from "ignore";
+
+/**
+ * Build an `ignore` instance that accumulates `.gitignore` rules from `cwd`
+ * down to the directory containing `relPath`.
+ *
+ * Mirrors the per-directory parent-chain logic in `walkFiles.ts`.
+ *
+ * @param cwd       Absolute root (same as the `cwd` passed to WalkFiles/WatchFiles)
+ * @param relPath   Path of the FS event, relative to `cwd`
+ * @param ignoreFiles  Additional glob patterns from config (applied on top)
+ */
+export async function loadIgnoreChain(
+    cwd: string,
+    relPath: string,
+    ignoreFiles: string[] = [],
+): Promise<ReturnType<typeof ignore>> {
+    // Segments from cwd down to (but not including) the file itself
+    const fileDir = dirname(relPath); // e.g. "pkg/sub" or "."
+    const segments = fileDir === "." ? [] : fileDir.split("/");
+
+    const ig = ignore();
+    ig.add(".*");
+
+    // Walk from root down: cwd, cwd/seg0, cwd/seg0/seg1, …
+    const dirs = [cwd, ...segments.map((_, i) => join(cwd, ...segments.slice(0, i + 1)))];
+    for (const dir of dirs) {
+        try {
+            const content = await readFile(join(dir, ".gitignore"), "utf8");
+            if (content) ig.add(content);
+        } catch {
+            // no .gitignore in this dir — fine
+        }
+    }
+
+    for (const pattern of ignoreFiles) ig.add(pattern);
+    return ig;
+}

package/componets/index/contentIndexDriver.ts

@@ -8,6 +8,7 @@ import {RemoveContent, IRemoveContent} from "../../features/removeContent.js";
 import {ResetIndex, IResetIndex} from "../../features/resetIndex.js";
 import {VectraIndex} from "./vectraIndex.js";
 import {IndexApi} from "./indexApi.js";
+import {ILocateInFile} from "../locate/locateInFile.js";
 
 export type IContentIndexDriver = Readonly<{
     getIndexStats: IGetIndexStats,
@@ -18,21 +19,22 @@ export type IContentIndexDriver = Readonly<{
     flush: () => Promise<void>,
 }>;
 
-export async function ContentIndexDriver({path, embed, extractKeywords, cleanUpKeywords, scoreThreshold}: {
+export async function ContentIndexDriver({path, embed, extractKeywords, cleanUpKeywords, locateInFile, scoreThreshold}: {
     path: string,
     embed: IEmbed,
     extractKeywords: IExtractKeywords,
     cleanUpKeywords: ICleanUpKeywords,
-    scoreThreshold?: number,
+    locateInFile: ILocateInFile,
+    scoreThreshold: number,
 }): Promise<IContentIndexDriver> {
     const index = await VectraIndex(path + "/semantic");
-    const indexApi = IndexApi({index, embed});
+    const indexApi = IndexApi({index});
 
     return {
         getIndexStats: GetIndexStats({index}),
-        indexContent: IndexContent({extractKeywords, cleanUpKeywords, indexApi}),
+        indexContent: IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi}),
         removeContent: RemoveContent({indexApi}),
-        searchContentIndex: SearchIndex({extractKeywords, cleanUpKeywords, embed, index, scoreThreshold}),
+        searchContentIndex: SearchIndex({extractKeywords, cleanUpKeywords, embed, index, locateInFile, scoreThreshold}),
         resetIndex: ResetIndex({indexApi}),
         flush: () => indexApi.flush(),
     };

package/componets/index/documentContentIndexDriver.ts

@@ -0,0 +1,126 @@
+import {rm} from "node:fs/promises";
+import {extname} from "node:path";
+import type {IContentIndexDriver} from "./contentIndexDriver.js";
+import type {IGetIndexStats} from "./getIndexStats.js";
+import type {IIndexContent} from "../../features/indexContent.js";
+import type {IRemoveContent} from "../../features/removeContent.js";
+import type {IResetIndex} from "../../features/resetIndex.js";
+import type {IIndexRecord, ISearchIndex} from "../../features/searchIndex.js";
+import {DocumentIndex} from "./documentIndex.js";
+
+export async function DocumentContentIndexDriver({path, model}: {
+    path: string,
+    model?: string,
+}): Promise<IContentIndexDriver> {
+    const docsPath = `${path}/docs`;
+    let index = await DocumentIndex({path: docsPath, model});
+
+    const indexContent: IIndexContent = async items => {
+        for (const item of items) {
+            await index.upsertDocument(item.id, item.content, docTypeForPath(item.id), {});
+        }
+    };
+
+    const removeContent: IRemoveContent = async ids => {
+        for (const id of ids) {
+            await index.deleteDocument(id);
+        }
+    };
+
+    const searchContentIndex: ISearchIndex = async (query, limit) => {
+        const results = await index.queryDocuments(query, {
+            maxDocuments: limit,
+            maxChunks: 5,
+            isBm25: true,
+        });
+
+        const out: IIndexRecord[] = [];
+
+        for (const r of results) {
+            const text = await r.loadText();
+            const lineStarts = computeLineStarts(text);
+
+            for (const c of r.chunks) {
+                const {startPos, endPos} = c.item.metadata;
+                const snippet = text.slice(startPos, endPos);
+                const startLine = lineOf(lineStarts, startPos);
+                const endLine = lineOf(lineStarts, endPos);
+
+                out.push({
+                    id: r.uri,
+                    score: c.score,
+                    startPos,
+                    endPos,
+                    snippet,
+                    startLine,
+                    endLine,
+                });
+            }
+        }
+
+        return out;
+    };
+
+    const resetIndex: IResetIndex = async () => {
+        /** Clears persisted catalog + chunks on disk; recreates empty index metadata. */
+        await rm(docsPath, {recursive: true, force: true});
+
+        /*
+         * Re-bind `index`: after `rm`, the prior LocalDocumentIndex instance may retain stale
+         * in-memory caches; `DocumentIndex` constructs a fresh instance against the recreated folder.
+         */
+        index = await DocumentIndex({path: docsPath, model});
+    };
+
+    const getIndexStats: IGetIndexStats = async () => {
+        const stats = await index.getCatalogStats();
+
+        return {indexedAmount: stats.documents ?? 0};
+    };
+
+    const flush = async () => {
+        /* LocalDocumentIndex persists updates without an explicit flush. */
+    };
+
+    return {
+        getIndexStats,
+        indexContent,
+        removeContent,
+        searchContentIndex,
+        resetIndex,
+        flush,
+    };
+}
+
+const DOC_TYPE_BY_EXT: Record<string, string> = {
+    ".ts": "ts", ".tsx": "tsx", ".mts": "ts", ".cts": "ts",
+    ".js": "js", ".jsx": "jsx", ".mjs": "js", ".cjs": "js",
+    ".py": "python",
+    ".go": "go",
+    ".java": "java",
+    ".cs": "csharp",
+    ".cpp": "cpp", ".cc": "cpp", ".cxx": "cpp", ".hpp": "cpp", ".h": "cpp",
+    ".md": "markdown",
+};
+
+function docTypeForPath(path: string): string {
+    return DOC_TYPE_BY_EXT[extname(path).toLowerCase()] ?? "text";
+}
+
+function computeLineStarts(text: string): number[] {
+    const starts = [0];
+    for (let i = 0; i < text.length; i++) {
+        if (text.charCodeAt(i) === 10) starts.push(i + 1);
+    }
+    return starts;
+}
+
+function lineOf(lineStarts: number[], pos: number): number {
+    let lo = 0, hi = lineStarts.length - 1;
+    while (lo < hi) {
+        const mid = (lo + hi + 1) >>> 1;
+        if (lineStarts[mid] <= pos) lo = mid;
+        else hi = mid - 1;
+    }
+    return lo + 1;
+}

package/componets/index/documentIndex.ts

@@ -0,0 +1,26 @@
+import { LocalDocumentIndex, TransformersEmbeddings } from "vectra";
+
+export type IDocumentIndex = LocalDocumentIndex;
+
+export async function DocumentIndex({
+    path,
+    model = "Xenova/all-MiniLM-L6-v2",
+    dtype = "q8",
+}: {
+    path: string;
+    model?: string;
+    dtype?: "fp32" | "fp16" | "q8" | "q4";
+}): Promise<LocalDocumentIndex> {
+    const embeddings = await TransformersEmbeddings.create({ model, dtype });
+    const index = new LocalDocumentIndex({
+        folderPath: path,
+        embeddings,
+        chunkingConfig: { chunkSize: 512, chunkOverlap: 0 },
+    });
+
+    if (!(await index.isIndexCreated())) {
+        await index.createIndex();
+    }
+
+    return index;
+}