xindex 1.0.3 → 1.0.5

package/componets/index/documentContentIndexDriver.ts

@@ -0,0 +1,126 @@
+import {rm} from "node:fs/promises";
+import {extname} from "node:path";
+import type {IContentIndexDriver} from "./contentIndexDriver.js";
+import type {IGetIndexStats} from "./getIndexStats.js";
+import type {IIndexContent} from "../../features/indexContent.js";
+import type {IRemoveContent} from "../../features/removeContent.js";
+import type {IResetIndex} from "../../features/resetIndex.js";
+import type {IIndexRecord, ISearchIndex} from "../../features/searchIndex.js";
+import {DocumentIndex} from "./documentIndex.js";
+
+export async function DocumentContentIndexDriver({path, model}: {
+    path: string,
+    model?: string,
+}): Promise<IContentIndexDriver> {
+    const docsPath = `${path}/docs`;
+    let index = await DocumentIndex({path: docsPath, model});
+
+    const indexContent: IIndexContent = async items => {
+        for (const item of items) {
+            await index.upsertDocument(item.id, item.content, docTypeForPath(item.id), {});
+        }
+    };
+
+    const removeContent: IRemoveContent = async ids => {
+        for (const id of ids) {
+            await index.deleteDocument(id);
+        }
+    };
+
+    const searchContentIndex: ISearchIndex = async (query, limit) => {
+        const results = await index.queryDocuments(query, {
+            maxDocuments: limit,
+            maxChunks: 5,
+            isBm25: true,
+        });
+
+        const out: IIndexRecord[] = [];
+
+        for (const r of results) {
+            const text = await r.loadText();
+            const lineStarts = computeLineStarts(text);
+
+            for (const c of r.chunks) {
+                const {startPos, endPos} = c.item.metadata;
+                const snippet = text.slice(startPos, endPos);
+                const startLine = lineOf(lineStarts, startPos);
+                const endLine = lineOf(lineStarts, endPos);
+
+                out.push({
+                    id: r.uri,
+                    score: c.score,
+                    startPos,
+                    endPos,
+                    snippet,
+                    startLine,
+                    endLine,
+                });
+            }
+        }
+
+        return out;
+    };
+
+    const resetIndex: IResetIndex = async () => {
+        /** Clears persisted catalog + chunks on disk; recreates empty index metadata. */
+        await rm(docsPath, {recursive: true, force: true});
+
+        /*
+         * Re-bind `index`: after `rm`, the prior LocalDocumentIndex instance may retain stale
+         * in-memory caches; `DocumentIndex` constructs a fresh instance against the recreated folder.
+         */
+        index = await DocumentIndex({path: docsPath, model});
+    };
+
+    const getIndexStats: IGetIndexStats = async () => {
+        const stats = await index.getCatalogStats();
+
+        return {indexedAmount: stats.documents ?? 0};
+    };
+
+    const flush = async () => {
+        /* LocalDocumentIndex persists updates without an explicit flush. */
+    };
+
+    return {
+        getIndexStats,
+        indexContent,
+        removeContent,
+        searchContentIndex,
+        resetIndex,
+        flush,
+    };
+}
+
+const DOC_TYPE_BY_EXT: Record<string, string> = {
+    ".ts": "ts", ".tsx": "tsx", ".mts": "ts", ".cts": "ts",
+    ".js": "js", ".jsx": "jsx", ".mjs": "js", ".cjs": "js",
+    ".py": "python",
+    ".go": "go",
+    ".java": "java",
+    ".cs": "csharp",
+    ".cpp": "cpp", ".cc": "cpp", ".cxx": "cpp", ".hpp": "cpp", ".h": "cpp",
+    ".md": "markdown",
+};
+
+function docTypeForPath(path: string): string {
+    return DOC_TYPE_BY_EXT[extname(path).toLowerCase()] ?? "text";
+}
+
+function computeLineStarts(text: string): number[] {
+    const starts = [0];
+    for (let i = 0; i < text.length; i++) {
+        if (text.charCodeAt(i) === 10) starts.push(i + 1);
+    }
+    return starts;
+}
+
+function lineOf(lineStarts: number[], pos: number): number {
+    let lo = 0, hi = lineStarts.length - 1;
+    while (lo < hi) {
+        const mid = (lo + hi + 1) >>> 1;
+        if (lineStarts[mid] <= pos) lo = mid;
+        else hi = mid - 1;
+    }
+    return lo + 1;
+}

package/componets/index/documentIndex.ts

@@ -0,0 +1,26 @@
+import { LocalDocumentIndex, TransformersEmbeddings } from "vectra";
+
+export type IDocumentIndex = LocalDocumentIndex;
+
+export async function DocumentIndex({
+    path,
+    model = "Xenova/all-MiniLM-L6-v2",
+    dtype = "q8",
+}: {
+    path: string;
+    model?: string;
+    dtype?: "fp32" | "fp16" | "q8" | "q4";
+}): Promise<LocalDocumentIndex> {
+    const embeddings = await TransformersEmbeddings.create({ model, dtype });
+    const index = new LocalDocumentIndex({
+        folderPath: path,
+        embeddings,
+        chunkingConfig: { chunkSize: 512, chunkOverlap: 0 },
+    });
+
+    if (!(await index.isIndexCreated())) {
+        await index.createIndex();
+    }
+
+    return index;
+}

package/componets/index/formatSearchResults.ts

@@ -9,8 +9,22 @@ export function FormatSearchResults(): IFormatSearchResults {
         const lines: string[] = [];
         for (let i = 0; i < results.length; i++) {
             const r = results[i];
-            const kw = r.keywords ? ` — ${r.keywords}` : "";
-            lines.push(`${i + 1}. ${r.id}${kw}`);
+            const scoreStr = `[${r.score.toFixed(2)}]`;
+            if (r.snippet !== undefined && r.snippet !== "") {
+                const header =
+                    typeof r.startLine === "number"
+                        ? `${i + 1}. ${r.id}:L${r.startLine}-L${r.endLine ?? r.startLine}  ${scoreStr}`
+                        : `${i + 1}. ${r.id}  ${scoreStr}`;
+                const snippetBlock = r.snippet
+                    .split("\n")
+                    .map((line) => `   ${line}`)
+                    .join("\n");
+                lines.push(`${header}\n\n${snippetBlock}`);
+            } else if (r.keywords) {
+                lines.push(`${i + 1}. ${r.id}  ${scoreStr} — ${r.keywords}`);
+            } else {
+                lines.push(`${i + 1}. ${r.id}  ${scoreStr}`);
+            }
         }
 
         return `\n# Search: "${query}" — ${results.length} result(s)\n\n${lines.join(";\n\n")}\n`;

package/componets/index/handleFileEvent.ts

@@ -5,6 +5,7 @@ import {ILogger} from "../logger.js";
 import {FileEventType, IFileEvent} from "../watchFiles.js";
 
 export type IHandleFileEvent = (event: IFileEvent) => Promise<void>;
+export type IHandleFileEvents = (events: IFileEvent[]) => Promise<void>;
 
 export function HandleFileEvent({indexContent, removeContent, log}: {
     indexContent: IIndexContent,
@@ -13,13 +14,57 @@ export function HandleFileEvent({indexContent, removeContent, log}: {
 }): IHandleFileEvent {
     return async function handleFileEvent(event) {
         if (event.type === FileEventType.index) {
-            try { await removeContent(event.path); } catch (e) { log(`remove failed: ${event.path} — ${(e as any)?.message ?? e}`); }
             const text = await readFile(event.path, "utf8");
-            await indexContent(event.path, `${text}. ${event.path}`);
+            await indexContent([{id: event.path, content: `${text}. ${event.path}`}]);
             log(`index: ${event.path}`);
         } else {
-            try { await removeContent(event.path); } catch (e) { log(`remove failed: ${event.path} — ${(e as any)?.message ?? e}`); }
+            try { await removeContent([event.path]); } catch (e) { log(`remove failed: ${event.path} — ${(e as any)?.message ?? e}`); }
             log(`remove: ${event.path}`);
         }
     };
 }
+
+export function HandleFileEvents({indexContent, removeContent, log}: {
+    indexContent: IIndexContent,
+    removeContent: IRemoveContent,
+    log: ILogger,
+}): IHandleFileEvents {
+    return async function handleFileEvents(events) {
+        const indexEvents = events.filter((event) => event.type === FileEventType.index);
+        const removeEvents = events.filter((event) => event.type === FileEventType.remove);
+
+        if (indexEvents.length > 0) {
+            const indexItems = (await Promise.all(indexEvents.map(async (event) => {
+                log(`index: ${event.path}`);
+                try {
+                    const text = await readFile(event.path, "utf8");
+                    return {id: event.path, content: `${text}. ${event.path}`};
+                } catch (e) {
+                    log(`index failed: ${event.path} — ${(e as any)?.message ?? e}`);
+                    return undefined;
+                }
+            }))).filter((item): item is { id: string, content: string } => !!item);
+
+            if (indexItems.length > 0) {
+                try {
+                    await indexContent(indexItems);
+                } catch (e) {
+                    log(`index batch failed: ${(e as any)?.message ?? e}`);
+                }
+            }
+        }
+
+        if (removeEvents.length > 0) {
+            const removePaths = removeEvents.map((event) => {
+                log(`remove: ${event.path}`);
+                return event.path;
+            });
+
+            try {
+                await removeContent(removePaths);
+            } catch (e) {
+                log(`remove batch failed: ${(e as any)?.message ?? e}`);
+            }
+        }
+    };
+}

package/componets/index/indexApi.ts

@@ -2,7 +2,6 @@ import {LocalIndex} from "vectra";
 import {IType} from "../IType.js";
 import {ISerial, Serial} from "../../packages/fun/src/serial.js";
 import {caseNever} from "../../packages/fun/src/case-never.js";
-import {IEmbed} from "../llm/embed.js";
 
 export enum IndexCommandType {
     index = 'index',
@@ -11,26 +10,55 @@ export enum IndexCommandType {
 }
 
 export type IIndexCommand =
-    | IType<{ type: IndexCommandType.index, id: string, content: string, keywords: string }>
-    | IType<{ type: IndexCommandType.delete, id: string }>
+    | IType<{ type: IndexCommandType.index, items: Array<{id: string, vector: number[], keywords: string}> }>
+    | IType<{ type: IndexCommandType.delete, ids: string[] }>
     | IType<{ type: IndexCommandType.reset }>;
 
 export type IIndexApi = ISerial<IIndexCommand, void>;
 
-export function IndexApi({index, embed}: { index: LocalIndex, embed: IEmbed }): IIndexApi {
+export function IndexApi({index}: { index: LocalIndex }): IIndexApi {
     return Serial<IIndexCommand, void>(async msg => {
         switch (msg.type) {
             case IndexCommandType.delete: {
-                await index.deleteItem(msg.id);
+                await index.beginUpdate();
+                try {
+                    for (const id of msg.ids) {
+                        await index.deleteItem(id);
+                    }
+                } finally {
+                    await index.endUpdate();
+                }
                 break;
             }
             case IndexCommandType.index: {
-                const vector = await embed(msg.content);
-                await index.upsertItem({
-                    id: msg.id,
-                    vector,
-                    metadata: {id: msg.id, keywords: msg.keywords},
-                });
+                try {
+                    await index.batchInsertItems(msg.items.map(item => ({
+                        id: item.id,
+                        vector: item.vector,
+                        metadata: {id: item.id, keywords: item.keywords},
+                    })));
+                } catch (error) {
+                    const errorMessage = String((error as Error)?.message ?? error).toLowerCase();
+                    const isDuplicateIdError =
+                        errorMessage.includes("already exists")
+                        || errorMessage.includes("duplicate");
+                    if (!isDuplicateIdError) {
+                        throw error;
+                    }
+
+                    await index.beginUpdate();
+                    try {
+                        for (const item of msg.items) {
+                            await index.upsertItem({
+                                id: item.id,
+                                vector: item.vector,
+                                metadata: {id: item.id, keywords: item.keywords},
+                            });
+                        }
+                    } finally {
+                        await index.endUpdate();
+                    }
+                }
                 break;
             }
             case IndexCommandType.reset: {

package/componets/locate/bm25.ts

@@ -0,0 +1,50 @@
+export type IBm25Doc = { id: string; tokens: string[] };
+
+export type IBm25 = {
+    score(queryTokens: string[]): Array<{ id: string; score: number }>;
+};
+
+export function Bm25({docs, k1 = 1.5, b = 0.75}: {
+    docs: IBm25Doc[];
+    k1?: number;
+    b?: number;
+}): IBm25 {
+    const N = docs.length;
+    const df = new Map<string, number>();
+    const tfs: Array<Map<string, number>> = [];
+    let totalLen = 0;
+
+    for (const d of docs) {
+        const tf = new Map<string, number>();
+        for (const t of d.tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
+        tfs.push(tf);
+        totalLen += d.tokens.length;
+        for (const t of tf.keys()) df.set(t, (df.get(t) ?? 0) + 1);
+    }
+    const avgdl = totalLen / Math.max(1, N);
+
+    return {
+        score(queryTokens) {
+            const out: Array<{ id: string; score: number }> = [];
+            const uniq = [...new Set(queryTokens)];
+            for (let i = 0; i < docs.length; i++) {
+                const tf = tfs[i];
+                const dl = docs[i].tokens.length;
+                let s = 0;
+                for (const t of uniq) {
+                    const f = tf.get(t);
+                    if (!f) continue;
+                    const n = df.get(t)!;
+                    const idf = Math.log(((N - n + 0.5) / (n + 0.5)) + 1);
+                    s += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * dl / avgdl));
+                }
+                if (s > 0) out.push({id: docs[i].id, score: s});
+            }
+            return out.sort((a, b) => b.score - a.score);
+        },
+    };
+}
+
+export function tokenizeForBm25(text: string): string[] {
+    return text.toLowerCase().split(/[^a-z0-9]+/).filter(t => t.length >= 2);
+}

package/componets/locate/inMemoryIndex.ts

@@ -0,0 +1,48 @@
+import {LocalIndex, VirtualFileStorage} from "vectra";
+
+export type ISnippetMeta = {
+    fileId: string;
+    startLine: number;
+    endLine: number;
+    snippet: string;
+};
+
+export type IInMemoryIndex = {
+    upsertItem(id: string, vector: number[], meta: ISnippetMeta): Promise<void>;
+    query(vector: number[], text: string, limit: number): Promise<Array<{score: number; meta: ISnippetMeta}>>;
+    dispose(): Promise<void>;
+};
+
+export function InMemoryIndex(_opts: {dimensions: number} = {dimensions: 0}): IInMemoryIndex {
+    const storage = new VirtualFileStorage();
+    const index = new LocalIndex("mem://idx", undefined, storage);
+    let created = false;
+
+    async function ensureCreated() {
+        if (created) return;
+        if (!(await index.isIndexCreated())) {
+            await index.createIndex();
+        }
+        created = true;
+    }
+
+    return {
+        async upsertItem(id, vector, meta) {
+            await ensureCreated();
+            await index.upsertItem({id, vector, metadata: meta as unknown as Record<string, any>});
+        },
+
+        async query(vector, text, limit) {
+            await ensureCreated();
+            const results = await index.queryItems(vector, text, limit);
+            return results.map(r => ({
+                score: r.score,
+                meta: r.item.metadata as unknown as ISnippetMeta,
+            }));
+        },
+
+        async dispose() {
+            // No-op: VirtualFileStorage is GC'd with the instance
+        },
+    };
+}

package/componets/locate/locateInFile.ts

@@ -0,0 +1,148 @@
+import {readFile, stat} from "fs/promises";
+import {IEmbed} from "../llm/embed.js";
+import {IExtractKeywords} from "../keywords/extractKeywords.js";
+import {ICleanUpKeywords} from "../keywords/cleanUpKeywords.js";
+import {IIndexRecord} from "../../features/searchIndex.js";
+import {IInMemoryIndex, InMemoryIndex} from "./inMemoryIndex.js";
+import {IWindow, windowsOf} from "./windowsOf.js";
+import {Bm25, IBm25Doc, tokenizeForBm25} from "./bm25.js";
+import {from} from "../../packages/streamx/src/from.js";
+import {filter} from "../../packages/streamx/src/filter.js";
+import {map} from "../../packages/streamx/src/map.js";
+import {flatMap} from "../../packages/streamx/src/flatMap.js";
+import {tap} from "../../packages/streamx/src/tap.js";
+import {scaleSync} from "../../packages/streamx/src/scaleSync.js";
+import {run} from "../../packages/streamx/src/index.js";
+import {DEFAULT_LOCATE_BATCH_SIZE} from "../config/DEFAULT_LOCATE_BATCH_SIZE";
+
+export type ILocateInFile = (
+    query: string,
+    queryVector: number[],
+    candidates: IIndexRecord[],
+    limit: number,
+) => Promise<IIndexRecord[]>;
+
+export function LocateInFile({
+                                 embed,
+                                 extractKeywords,
+                                 cleanUpKeywords,
+                                 windowLines = 15,
+                                 maxFileBytes = 10_000_000,
+                                 embedConcurrency = DEFAULT_LOCATE_BATCH_SIZE,
+                             }: {
+    embed: IEmbed;
+    extractKeywords: IExtractKeywords;
+    cleanUpKeywords: ICleanUpKeywords;
+    windowLines?: number;
+    maxSnippetsPerFile?: number;
+    maxFileBytes?: number;
+    embedConcurrency?: number;
+}): ILocateInFile {
+    type IWindowWithKeywords = IWindow & { keywords: string };
+    type IWindowWithVector = IWindowWithKeywords & { vector: number[] };
+
+    return async function locateInFile(query, queryVector, candidates, limit) {
+        const memIndex: IInMemoryIndex = InMemoryIndex({dimensions: queryVector.length});
+
+        try {
+            const ids = candidates.map(c => c.id);
+            const bm25Docs: IBm25Doc[] = [];
+            const metaById = new Map<string, { fileId: string; startLine: number; endLine: number; snippet: string }>();
+
+            const SCALE_FILE_READS = embedConcurrency;
+            const windows = from<string>(ids)
+                .pipe(filter(async (id: string) => {
+                    try {
+                        const s = await stat(id);
+                        if (!s.isFile()) return false;
+                        if (s.size > maxFileBytes) return false;
+                        return true;
+                    } catch {
+                        return false;
+                    }
+                }))
+                .pipe(scaleSync(SCALE_FILE_READS, async (id: string) => {
+                    const text = await readFile(id, "utf8");
+                    return {id, text};
+                }))
+                .pipe(flatMap(({id, text}: { id: string, text: string }): IWindow[] => {
+                    return windowsOf({text, id, windowLines});
+                }));
+
+            const withVectors = from<IWindow>(windows)
+                .pipe(map<IWindow, IWindowWithKeywords>(w => {
+                    const kw = cleanUpKeywords(extractKeywords(w.snippet)).join(", ");
+                    return {...w, keywords: kw || w.snippet.slice(0, 200)};
+                }))
+                .pipe(scaleSync<IWindowWithKeywords, IWindowWithVector>(embedConcurrency, async (w) => {
+                    const vector = await embed(w.keywords);
+                    return {...w, vector};
+                }))
+                .pipe(tap(async (w: IWindowWithVector) => {
+                    const id = `${w.fileId}:${w.startLine}:${w.endLine}`;
+                    await memIndex.upsertItem(
+                        id,
+                        w.vector,
+                        {fileId: w.fileId, startLine: w.startLine, endLine: w.endLine, snippet: w.snippet},
+                    );
+                    bm25Docs.push({id, tokens: tokenizeForBm25(w.snippet)});
+                    metaById.set(id, {
+                        fileId: w.fileId,
+                        startLine: w.startLine,
+                        endLine: w.endLine,
+                        snippet: w.snippet
+                    });
+                }));
+
+            await run(withVectors);
+
+            const poolSize = Math.max(limit * 8, 40);
+            const vecHits = await memIndex.query(queryVector, query, poolSize);
+
+            const bm25 = Bm25({docs: bm25Docs});
+            const bm25Hits = bm25.score(tokenizeForBm25(query)).slice(0, poolSize);
+
+            const cosById = new Map<string, number>();
+            vecHits.forEach(h => {
+                const id = `${h.meta.fileId}:${h.meta.startLine}:${h.meta.endLine}`;
+                cosById.set(id, h.score);
+            });
+            const bmById = new Map<string, number>();
+            bm25Hits.forEach(h => bmById.set(h.id, h.score));
+
+            const cosMax = Math.max(...vecHits.map(h => h.score), 1e-9);
+            const bmMax = Math.max(...bm25Hits.map(h => h.score), 1e-9);
+
+            const allIds = new Set<string>([...cosById.keys(), ...bmById.keys()]);
+            const ranked = [...allIds]
+                .map(id => {
+                    const cosNorm = (cosById.get(id) ?? 0) / cosMax;
+                    const bmNorm = (bmById.get(id) ?? 0) / bmMax;
+                    return {id, score: Math.max(cosNorm, bmNorm), meta: metaById.get(id)!};
+                })
+                .filter(r => r.meta)
+                .sort((a, b) => b.score - a.score);
+
+            const kept: typeof ranked = [];
+            for (const h of ranked) {
+                const overlaps = kept.some(k =>
+                    k.meta.fileId === h.meta.fileId &&
+                    h.meta.startLine <= k.meta.endLine &&
+                    h.meta.endLine >= k.meta.startLine
+                );
+                if (!overlaps) kept.push(h);
+                if (kept.length >= limit) break;
+            }
+
+            return kept.map(h => ({
+                score: h.score,
+                id: h.meta.fileId,
+                startLine: h.meta.startLine,
+                endLine: h.meta.endLine,
+                snippet: h.meta.snippet,
+            }));
+        } finally {
+            await memIndex.dispose();
+        }
+    };
+}

package/componets/locate/windowsOf.ts

@@ -0,0 +1,29 @@
+export type IWindow = {
+    fileId: string;
+    startLine: number;
+    endLine: number;
+    snippet: string;
+};
+
+export function windowsOf({text, id, windowLines}: {
+    text: string;
+    id: string;
+    windowLines: number;
+}): IWindow[] {
+    const lines = text.split("\n");
+    const windows: IWindow[] = [];
+    const scales = [Math.max(1, Math.floor(windowLines / 2)), windowLines];
+    for (const size of scales) {
+        const step = Math.max(1, Math.floor(size / 2));
+        for (let i = 0; i < lines.length; i += step) {
+            const slice = lines.slice(i, i + size);
+            windows.push({
+                fileId: id,
+                startLine: i + 1,
+                endLine: Math.min(i + size, lines.length),
+                snippet: slice.join("\n"),
+            });
+        }
+    }
+    return windows;
+}

package/componets/watchFiles.ts

@@ -1,8 +1,8 @@
-import {readFile, stat, watch} from "fs/promises";
+import {stat, watch} from "fs/promises";
 import {join, relative} from "path";
-import ignore from "ignore";
 import {ILogger} from "./logger.js";
 import {IType} from "./IType.js";
+import {loadIgnoreChain} from "./ignore/loadIgnoreChain.js";
 
 export enum FileEventType {
     index = 'index',
@@ -19,18 +19,6 @@ export type IWatchFiles = (inputs: string[]) => IWatchFilesResult;
 
 export function WatchFiles({cwd, log, ignoreFiles = []}: { cwd: string, log: ILogger, ignoreFiles?: string[] }): IWatchFiles {
 
-    async function loadGitignore(dir: string): Promise<ReturnType<typeof ignore>> {
-        const ig = ignore();
-        ig.add(".*");
-        try {
-            const content = await readFile(join(dir, ".gitignore"), "utf8");
-            ig.add(content);
-        } catch {
-        }
-        for (const pattern of ignoreFiles) ig.add(pattern);
-        return ig;
-    }
-
     return function watchFiles(inputs) {
         const pending = new Map<string, IFileEvent>();
         let notify: (() => void) | null = null;
@@ -38,7 +26,6 @@ export function WatchFiles({cwd, log, ignoreFiles = []}: { cwd: string, log: ILo
         const abortControllers: AbortController[] = [];
 
         async function startWatching(dir: string) {
-            const ig = await loadGitignore(dir);
             const ac = new AbortController();
             abortControllers.push(ac);
 
@@ -48,9 +35,11 @@ export function WatchFiles({cwd, log, ignoreFiles = []}: { cwd: string, log: ILo
                     if (!event.filename) continue;
 
                     const rel = relative(cwd, join(dir, event.filename));
-                    if (ig.ignores(rel)) continue;
                     if (rel.endsWith("~")) continue;
 
+                    const ig = await loadIgnoreChain(cwd, rel, ignoreFiles);
+                    if (ig.ignores(rel)) continue;
+
                     try {
                         await stat(join(dir, event.filename));
                         pending.set(rel, {type: FileEventType.index, path: rel});

package/features/indexContent.ts

@@ -1,16 +1,23 @@
 import {IndexCommandType, IIndexApi} from "../componets/index/indexApi.js";
 import {IExtractKeywords} from "../componets/keywords/extractKeywords.js";
 import {ICleanUpKeywords} from "../componets/keywords/cleanUpKeywords.js";
+import {IEmbed} from "../componets/llm/embed.js";
 
-export type IIndexContent = (id: string, content: string) => Promise<void>;
+export type IIndexContent = (items: Array<{id: string, content: string}>) => Promise<void>;
 
-export function IndexContent({extractKeywords, cleanUpKeywords, indexApi}: {
+export function IndexContent({extractKeywords, cleanUpKeywords, embed, indexApi}: {
     extractKeywords: IExtractKeywords,
     cleanUpKeywords: ICleanUpKeywords,
+    embed: IEmbed,
     indexApi: IIndexApi,
 }): IIndexContent {
-    return async function indexContent(id, content) {
-        const keywords = cleanUpKeywords(extractKeywords(content)).join(", ");
-        await indexApi({type: IndexCommandType.index, id, content: keywords, keywords});
+    return async function indexContent(items) {
+        const indexItems = await Promise.all(items.map(async (item) => {
+            const keywords = cleanUpKeywords(extractKeywords(item.content)).join(", ");
+            const vector = await embed(keywords);
+            return {id: item.id, vector, keywords};
+        }));
+
+        await indexApi({type: IndexCommandType.index, items: indexItems});
     }
 }