vectra 0.12.2 → 0.14.0

package/src/LocalIndex.ts

@@ -1,5 +1,4 @@
-import * as fs from 'fs/promises';
-import * as path from 'path';
+import { pathUtils as path } from './utils/pathUtils';
 import { v4 } from 'uuid';
 import { ItemSelector } from './ItemSelector';
 import { IndexItem, IndexStats, MetadataFilter, MetadataTypes, QueryResult } from './types';
@@ -8,12 +7,15 @@ import { LocalDocumentIndex } from './LocalDocumentIndex';
 import bm25 from 'wink-bm25-text-search';
 import winkNLP from 'wink-nlp';
 import model from 'wink-eng-lite-web-model';
+import { FileStorage, LocalFileStorage } from './storage';
+import { IndexCodec, JsonCodec } from './codecs';
+
 export interface CreateIndexConfig {
-    version: number;
-    deleteIfExists?: boolean;
-    metadata_config?: {
-        indexed?: string[];
-    };
+  version: number;
+  deleteIfExists?: boolean;
+  metadata_config?: {
+    indexed?: string[];
+  };
 }
 
 /**
@@ -22,468 +24,516 @@ export interface CreateIndexConfig {
  * This class is used to create, update, and query a local vector index.
  * Each index is a folder on disk containing an index.json file and an optional set of metadata files.
  */
-export class LocalIndex<TMetadata extends Record<string,MetadataTypes> = Record<string,MetadataTypes>>{
-    private readonly _folderPath: string;
-    private readonly _indexName: string;
-
-    private _data?: IndexData;
-    private _update?: IndexData;
-    //member fields for BM25
-    private _bm25Engine: any;
-
-    /**
-     * Creates a new instance of LocalIndex.
-     * @param folderPath Path to the index folder.
-     * @param indexName Optional name of the index file. Defaults to index.json.
-     */
-    public constructor(folderPath: string, indexName?: string) {
-        this._folderPath = folderPath;
-        this._indexName = indexName || "index.json";
-    }
-
-    /**
-     * Path to the index folder.
-     */
-    public get folderPath(): string {
-        return this._folderPath;
+export class LocalIndex<TMetadata extends Record<string, MetadataTypes> = Record<string, MetadataTypes>> {
+  private readonly _folderPath: string;
+  private readonly _indexName: string = 'index.json';
+  private readonly _storage: FileStorage;
+  private readonly _codec: IndexCodec;
+  private _data?: IndexData;
+  private _update?: IndexData;
+
+  // member fields for BM25
+  private _bm25Engine: any;
+  private readonly _bm25Factory: () => any;
+  private readonly _docReader: (docId: string) => Promise<string>;
+
+  /**
+   * Creates a new instance of LocalIndex.
+   * @param folderPath Path to the index folder.
+   * @param indexName Optional index file name. Defaults to 'index' + codec.extension.
+   * @param storage Optional file storage instance. Defaults to LocalFileStorage.
+   * @param codec Optional codec for serialization. Defaults to JsonCodec.
+   * @param options Optional constructor options for dependency injection.
+   */
+  public constructor(
+    folderPath: string,
+    indexName?: string,
+    storage?: FileStorage,
+    codec?: IndexCodec,
+    options?: {
+      bm25Factory?: () => any;
+      docReader?: (docId: string) => Promise<string>;
     }
-
-    /**
-     * Optional name of the index file. 
-     */
-    public get indexName(): string {
-        return this._indexName;
+  ) {
+    this._folderPath = folderPath;
+    this._codec = codec || new JsonCodec();
+    if (indexName) {
+      this._indexName = indexName;
+    } else {
+      this._indexName = `index${this._codec.extension}`;
     }
-
-    /**
-     * Begins an update to the index.
-     * @remarks
-     * This method loads the index into memory and prepares it for updates.
-     */
-    public async beginUpdate(): Promise<void> {
-        if (this._update) {
-            throw new Error('Update already in progress');
-        }
-
-        await this.loadIndexData();
-        this._update = structuredClone(this._data);
-    }
-
-    /**
-     * Cancels an update to the index.
-     * @remarks
-     * This method discards any changes made to the index since the update began.
-     */
-    public cancelUpdate(): void {
-        this._update = undefined;
-    }
-
-    /**
-     * Creates a new index.
-     * @remarks
-     * This method creates a new folder on disk containing an index.json file.
-     * @param config Index configuration.
-     */
-    public async createIndex(config: CreateIndexConfig = {version: 1}): Promise<void> {
-        // Delete if exists
-        if (await this.isIndexCreated()) {
-            if (config.deleteIfExists) {
-                await this.deleteIndex();
-            } else {
-                throw new Error('Index already exists');
-            }
-        }
-
-        try {
-            // Create folder for index
-            await fs.mkdir(this._folderPath, { recursive: true });
-
-            // Initialize index.json file
-            this._data = {
-                version: config.version,
-                metadata_config: config.metadata_config ?? {},
-                items: []
-            };
-
-            await fs.writeFile(path.join(this._folderPath, this._indexName), JSON.stringify(this._data));
-        } catch (err: unknown) {
-            await this.deleteIndex();
-            throw new Error('Error creating index');
-        }
+    this._storage = storage || new LocalFileStorage();
+    this._bm25Factory = options?.bm25Factory || (() => bm25());
+    this._docReader = options?.docReader || (async (docId: string) => {
+      const doc = new LocalDocument((this as unknown) as LocalDocumentIndex, docId, '');
+      return await doc.loadText();
+    });
+  }
+
+  /** Path to the index folder. */
+  public get folderPath(): string {
+    return this._folderPath;
+  }
+
+  /** Name of the index file. */
+  public get indexName(): string {
+    return this._indexName;
+  }
+
+  /** Storage provider used to store the index. */
+  public get storage(): FileStorage {
+    return this._storage;
+  }
+
+  /** Codec used for serialization. */
+  public get codec(): IndexCodec {
+    return this._codec;
+  }
+
+  /**
+   * Begins an update to the index.
+   * @remarks
+   * This method loads the index into memory and prepares it for updates.
+   */
+  public async beginUpdate(): Promise<void> {
+    if (this._update) {
+      throw new Error('Update already in progress');
     }
-
-    /**
-     * Deletes the index.
-     * @remarks
-     * This method deletes the index folder from disk.
-     */
-    public deleteIndex(): Promise<void> {
-        this._data = undefined;
-        return fs.rm(this._folderPath, {
-            recursive: true,
-            maxRetries: 3
-        });
+    await this.loadIndexData();
+    this._update = structuredClone(this._data);
+  }
+
+  /**
+   * Cancels an update to the index.
+   * @remarks
+   * This method discards any changes made to the index since the update began.
+   */
+  public cancelUpdate(): void {
+    this._update = undefined;
+  }
+
+  /**
+   * Creates a new index.
+   * @remarks
+   * This method creates a new folder on disk containing an index.json file.
+   * @param config Index configuration.
+   */
+  public async createIndex(config: CreateIndexConfig = { version: 1 }): Promise<void> {
+    // Delete if exists
+    if (await this.isIndexCreated()) {
+      if (config.deleteIfExists) {
+        await this.deleteIndex();
+      } else {
+        throw new Error('Index already exists');
+      }
     }
 
-    /**
-     * Deletes an item from the index.
-     * @param id ID of item to delete.
-     */
-    public async deleteItem(id: string): Promise<void> {
-        if (this._update) {
-            const index = this._update.items.findIndex(i => i.id === id);
-            if (index >= 0) {
-                this._update.items.splice(index, 1);
-            }
-        } else {
-            await this.beginUpdate();
-            const index = this._update!.items.findIndex(i => i.id === id);
-            if (index >= 0) {
-                this._update!.items.splice(index, 1);
-            }
-            await this.endUpdate();
-        }
+    try {
+      // Create folder for index
+      await this.storage.createFolder(this._folderPath);
+
+      // Initialize index.json file
+      this._data = {
+        version: config.version,
+        metadata_config: config.metadata_config ?? {},
+        items: []
+      };
+      await this.storage.upsertFile(path.join(this._folderPath, this._indexName), this._codec.serializeIndex(this._data));
+    } catch {
+      await this.deleteIndex();
+      throw new Error('Error creating index');
     }
-
-    /**
-     * Ends an update to the index.
-     * @remarks
-     * This method saves the index to disk.
-     */
-    public async endUpdate(): Promise<void> {
-        if (!this._update) {
-            throw new Error('No update in progress');
-        }
-
-        try {
-            // Save index
-            await fs.writeFile(path.join(this._folderPath, this._indexName), JSON.stringify(this._update));
-            this._data = this._update;
-            this._update = undefined;
-        } catch(err: unknown) {
-            throw new Error(`Error saving index: ${(err as any).toString()}`);
-        }
+  }
+
+  /**
+   * Deletes the index.
+   * @remarks
+   * This method deletes the index folder from disk.
+   */
+  public async deleteIndex(): Promise<void> {
+    this._data = undefined;
+    return await this.storage.deleteFolder(this._folderPath);
+  }
+
+  /**
+   * Deletes an item from the index.
+   * @param id ID of item to delete.
+   */
+  public async deleteItem(id: string): Promise<void> {
+    if (this._update) {
+      const index = this._update.items.findIndex(i => i.id === id);
+      if (index >= 0) {
+        this._update.items.splice(index, 1);
+      }
+    } else {
+      await this.beginUpdate();
+      const index = this._update!.items.findIndex(i => i.id === id);
+      if (index >= 0) {
+        this._update!.items.splice(index, 1);
+      }
+      await this.endUpdate();
     }
-
-    /**
-     * Loads an index from disk and returns its stats.
-     * @returns Index stats.
-     */
-    public async getIndexStats(): Promise<IndexStats> {
-        await this.loadIndexData();
-        return {
-            version: this._data!.version,
-            metadata_config: this._data!.metadata_config,
-            items: this._data!.items.length
-        };
+  }
+
+  /**
+   * Ends an update to the index.
+   * @remarks
+   * This method saves the index to disk.
+   */
+  public async endUpdate(): Promise<void> {
+    if (!this._update) {
+      throw new Error('No update in progress');
     }
 
-    /**
-     * Returns an item from the index given its ID.
-     * @param id ID of the item to retrieve.
-     * @returns Item or undefined if not found.
-     */
-    public async getItem<TItemMetadata extends TMetadata = TMetadata>(id: string): Promise<IndexItem<TItemMetadata> | undefined> {
-        await this.loadIndexData();
-        return this._data!.items.find(i => i.id === id) as any | undefined;
+    try {
+      // Save index
+      await this.storage.upsertFile(path.join(this._folderPath, this._indexName), this._codec.serializeIndex(this._update));
+      this._data = this._update;
+      this._update = undefined;
+    } catch (err: unknown) {
+      throw new Error(`Error saving index: ${(err as any).toString()}`);
     }
-
-    /**
-     * Adds an item to the index.
-     * @remarks
-     * A new update is started if one is not already in progress. If an item with the same ID
-     * already exists, an error will be thrown.
-     * @param item Item to insert.
-     * @returns Inserted item.
-     */
-    public async insertItem<TItemMetadata extends TMetadata = TMetadata>(item: Partial<IndexItem<TItemMetadata>>): Promise<IndexItem<TItemMetadata>> {
-        if (this._update) {
-            return await this.addItemToUpdate(item, true) as any;
-        } else {
-            await this.beginUpdate();
-            const newItem = await this.addItemToUpdate(item, true);
-            await this.endUpdate();
-            return newItem as any;
-        }
+  }
+
+  /**
+   * Loads an index from disk and returns its stats.
+   * @returns Index stats.
+   */
+  public async getIndexStats(): Promise<IndexStats> {
+    await this.loadIndexData();
+    return {
+      version: this._data!.version,
+      metadata_config: this._data!.metadata_config,
+      items: this._data!.items.length
+    };
+  }
+
+  /**
+   * Returns an item from the index given its ID.
+   * @param id ID of the item to retrieve.
+   * @returns Item or undefined if not found.
+   */
+  public async getItem<TItemMetadata extends TMetadata = TMetadata>(id: string): Promise<IndexItem<TItemMetadata> | undefined> {
+    await this.loadIndexData();
+    return this._data!.items.find(i => i.id === id) as any | undefined;
+  }
+
+  /**
+   * Adds an item to the index.
+   * @remarks
+   * A new update is started if one is not already in progress. If an item with the same ID
+   * already exists, an error will be thrown.
+   * @param item Item to insert.
+   * @returns Inserted item.
+   */
+  public async insertItem<TItemMetadata extends TMetadata = TMetadata>(item: Partial<IndexItem<TItemMetadata>>): Promise<IndexItem<TItemMetadata>> {
+    if (this._update) {
+      return await this.addItemToUpdate(item, true) as any;
+    } else {
+      await this.beginUpdate();
+      const newItem = await this.addItemToUpdate(item, true);
+      await this.endUpdate();
+      return newItem as any;
     }
-
-    /**
-     * Adds a batch of items to the index.
-     * @remarks
-     * Batch update requires no update to be in progress. This is necessary so that if any one
-     * insert operation fails, the entire update can be safely cancelled. This prevents partial
-     * updates from being applied to the local index.
-     * @param items Items to insert.
-     * @returns Inserted items.
-     */
-    public async batchInsertItems<TItemMetadata extends TMetadata = TMetadata>(items: Partial<IndexItem<TItemMetadata>>[]): Promise<IndexItem[]> {
-        await this.beginUpdate();
-        try {
-            const newItems: IndexItem[] = [];
-            for (const item of items) {
-                const newItem = await this.addItemToUpdate(item, true);
-                newItems.push(newItem);
-            }
-            await this.endUpdate();
-            return newItems;
-        } catch (e) {
-            // cancels this update to prevent partial batch updates. allows error to bubble up.
-            await this.cancelUpdate();
-            throw e;
-        }
+  }
+
+  /**
+   * Adds a batch of items to the index.
+   * @remarks
+   * Batch update requires no update to be in progress. This is necessary so that if any one
+   * insert operation fails, the entire update can be safely cancelled. This prevents partial
+   * updates from being applied to the local index.
+   * @param items Items to insert.
+   * @returns Inserted items.
+   */
+  public async batchInsertItems<TItemMetadata extends TMetadata = TMetadata>(items: Partial<IndexItem<TItemMetadata>>[]): Promise<IndexItem[]> {
+    await this.beginUpdate();
+    try {
+      const newItems: IndexItem[] = [];
+      for (const item of items) {
+        const newItem = await this.addItemToUpdate(item, true);
+        newItems.push(newItem);
+      }
+      await this.endUpdate();
+      return newItems;
+    } catch (e) {
+      // cancels this update to prevent partial batch updates. allows error to bubble up.
+      await this.cancelUpdate();
+      throw e;
     }
-
-    /**
-     * Returns true if the index exists.
-     */
-    public async isIndexCreated(): Promise<boolean> {
-        try {
-            await fs.access(path.join(this._folderPath, this.indexName));
-            return true;
-        } catch (err: unknown) {
-            return false;
-        }
+  }
+
+  /** Returns true if the index exists. */
+  public async isIndexCreated(): Promise<boolean> {
+    return await this.storage.pathExists(path.join(this._folderPath, this._indexName));
+  }
+
+  /**
+   * Returns all items in the index.
+   * @remarks
+   * This method loads the index into memory and returns all its items. A copy of the items
+   * array is returned so no modifications should be made to the array.
+   * @returns Array of all items in the index.
+   */
+  public async listItems<TItemMetadata extends TMetadata = TMetadata>(): Promise<IndexItem<TItemMetadata>[]> {
+    await this.loadIndexData();
+    return this._data!.items.slice() as any;
+  }
+
+  /**
+   * Returns all items in the index matching the filter.
+   * @remarks
+   * This method loads the index into memory and returns all its items matching the filter.
+   * @param filter Filter to apply.
+   * @returns Array of items matching the filter.
+   */
+  public async listItemsByMetadata<TItemMetadata extends TMetadata = TMetadata>(filter: MetadataFilter): Promise<IndexItem<TItemMetadata>[]> {
+    await this.loadIndexData();
+    return this._data!.items.filter(i => ItemSelector.select(i.metadata, filter)) as any;
+  }
+
+  /**
+   * Finds the top k items in the index that are most similar to the vector.
+   * @remarks
+   * This method loads the index into memory and returns the top k items that are most similar.
+   * An optional filter can be applied to the metadata of the items.
+   * @param vector Vector to query against.
+   * @param query Query string (used when isBm25=true).
+   * @param topK Number of items to return.
+   * @param filter Optional. Filter to apply.
+   * @param isBm25 Optional. If true, append BM25 keyword results to semantic results.
+   * @returns Similar items to the vector that match the supplied filter.
+   */
+  public async queryItems<TItemMetadata extends TMetadata = TMetadata>(
+    vector: number[],
+    query: string,
+    topK: number,
+    filter?: MetadataFilter,
+    isBm25?: boolean
+  ): Promise<QueryResult<TItemMetadata>[]> {
+    await this.loadIndexData();
+
+    // Filter items
+    let items = this._data!.items;
+    if (filter) {
+      items = items.filter(i => ItemSelector.select(i.metadata, filter));
     }
 
-    /**
-     * Returns all items in the index.
-     * @remarks
-     * This method loads the index into memory and returns all its items. A copy of the items
-     * array is returned so no modifications should be made to the array.
-     * @returns Array of all items in the index.
-     */
-    public async listItems<TItemMetadata extends TMetadata = TMetadata>(): Promise<IndexItem<TItemMetadata>[]> {
-        await this.loadIndexData();
-        return this._data!.items.slice() as any;
+    // Calculate distances
+    const norm = ItemSelector.normalize(vector);
+    const distances: { index: number; distance: number }[] = [];
+    for (let i = 0; i < items.length; i++) {
+      const item = items[i];
+      const distance = ItemSelector.normalizedCosineSimilarity(vector, norm, item.vector, item.norm);
+      distances.push({ index: i, distance: distance });
     }
 
-    /**
-     * Returns all items in the index matching the filter.
-     * @remarks
-     * This method loads the index into memory and returns all its items matching the filter.
-     * @param filter Filter to apply.
-     * @returns Array of items matching the filter.
-     */
-    public async listItemsByMetadata<TItemMetadata extends TMetadata = TMetadata>(filter: MetadataFilter): Promise<IndexItem<TItemMetadata>[]> {
-        await this.loadIndexData();
-        return this._data!.items.filter(i => ItemSelector.select(i.metadata, filter)) as any;
+    // Sort by distance DESCENDING
+    distances.sort((a, b) => b.distance - a.distance);
+
+    // Find top k
+    const top: QueryResult<TItemMetadata>[] = distances.slice(0, topK).map(d => {
+      return {
+        item: Object.assign({}, items[d.index]) as any,
+        score: d.distance
+      };
+    });
+
+    // Load external metadata
+    for (const item of top) {
+      if (item.item.metadataFile) {
+        const metadataPath = path.join(this._folderPath, item.item.metadataFile);
+        const metadataBuffer = await this.storage.readFile(metadataPath);
+        item.item.metadata = this._codec.deserializeMetadata(metadataBuffer) as any;
+      }
     }
 
-    /**
-     * Finds the top k items in the index that are most similar to the vector.
-     * @remarks
-     * This method loads the index into memory and returns the top k items that are most similar.
-     * An optional filter can be applied to the metadata of the items.
-     * @param vector Vector to query against.
-     * @param topK Number of items to return.
-     * @param filter Optional. Filter to apply.
-     * @returns Similar items to the vector that matche the supplied filter.
-     */
-    public async queryItems<TItemMetadata extends TMetadata = TMetadata>(vector: number[], query: string, topK: number, filter?: MetadataFilter, isBm25?: boolean): Promise<QueryResult<TItemMetadata>[]> {
-        await this.loadIndexData();
-
-        // Filter items
-        let items = this._data!.items;
-        if (filter) {
-            items = items.filter(i => ItemSelector.select(i.metadata, filter));
+    // Perform bm25 search only if enabled. Avoid duplicate chunks that are already selected during semantic search.
+    if (isBm25) {
+      const itemSet = new Set<string>();
+      for (const r of top) itemSet.add(r.item.id);
+
+      // Set up BM25 engine
+      await this.setupBm25();
+
+      // Add docs if we have necessary metadata; guard everything to avoid crashes
+      for (let i = 0; i < items.length; i++) {
+        if (!itemSet.has(items[i].id)) {
+          const item = items[i] as any;
+          const md = item.metadata || {};
+          if (md.documentId != undefined && md.startPos != undefined && md.endPos != undefined) {
+            try {
+              const currDocTxt = await this._docReader(String(md.documentId));
+              const startPos = Number(md.startPos);
+              const endPos = Number(md.endPos);
+              const chunkText = currDocTxt.substring(startPos, endPos + 1);
+              this._bm25Engine.addDoc?.({ body: chunkText }, i);
+            } catch {
+              // Ignore load or engine errors for BM25 doc prep
+            }
+          }
         }
+      }
 
-        // Calculate distances
-        const norm = ItemSelector.normalize(vector);
-        const distances: { index: number, distance: number }[] = [];
-        for (let i = 0; i < items.length; i++) {
-            const item = items[i];
-            const distance = ItemSelector.normalizedCosineSimilarity(vector, norm, item.vector, item.norm);
-            distances.push({ index: i, distance: distance });
-        }
+      this._bm25Engine.consolidate?.();
 
-        // Sort by distance DESCENDING
-        distances.sort((a, b) => b.distance - a.distance);
-
-        // Find top k
-        const top: QueryResult<TItemMetadata>[] = distances.slice(0, topK).map(d => {
-            return {
-                item: Object.assign({}, items[d.index]) as any,
-                score: d.distance
-            };
-        });
-
-        // Load external metadata
-        for (const item of top) {
-            if (item.item.metadataFile) {
-                const metadataPath = path.join(this._folderPath, item.item.metadataFile);
-                const metadata = await fs.readFile(metadataPath);
-                item.item.metadata = JSON.parse(metadata.toString());
-            }
-        }
+      const results: any[] = await this.bm25Search(query, items, topK);
 
-        //Peform bm25 search only if enabled. Avoid duplicate chunks, which are already selected during semantic search.
-        if (isBm25) {
-            const itemSet = new Set();
-            for (const item of top) itemSet.add(item.item.id);
-            
-            this.setupbm25();
-
-            let currDoc;
-            let currDocTxt;
-            for (let i = 0; i < items.length; i++) {
-                if (!itemSet.has(items[i].id)) {
-                    const item = items[i];
-                    currDoc = new LocalDocument((this as unknown) as LocalDocumentIndex, item.metadata.documentId.toString(), '');
-                    currDocTxt = await currDoc.loadText();
-                    const startPos = item.metadata.startPos;
-                    const endPos = item.metadata.endPos;
-                    const chunkText = currDocTxt.substring(Number(startPos), Number(endPos) + 1);
-                    this._bm25Engine.addDoc({body: chunkText}, i);
-                }
-            }
-            this._bm25Engine.consolidate();
-            var results = await this.bm25Search(query, items, topK);
-            results.forEach((res: any) => {
-                top.push({
-                    item: Object.assign({}, {...items[res[0]], metadata: {...items[res[0]].metadata, isBm25: true}}) as any,
-                    score: res[1]
-                });
+      results.forEach((res: any) => {
+        // Support both [index, score] tuples and { item, score } objects
+        if (Array.isArray(res)) {
+          const idx = res[0];
+          const score = res[1];
+          if (items[idx]) {
+            top.push({
+              item: Object.assign({}, { ...items[idx], metadata: { ...items[idx].metadata, isBm25: true } }) as any,
+              score
             });
-            
+          }
+        } else if (res && typeof res === 'object' && 'item' in res && 'score' in res) {
+          const objItem = Object.assign({}, { ...(res.item || {}), metadata: { ...(res.item?.metadata || {}), isBm25: true } }) as any;
+          top.push({ item: objItem, score: res.score });
         }
-        return top;
+      });
     }
 
-    /**
-     * Adds or replaces an item in the index.
-     * @remarks
-     * A new update is started if one is not already in progress. If an item with the same ID
-     * already exists, it will be replaced.
-     * @param item Item to insert or replace.
-     * @returns Upserted item.
-     */
-    public async upsertItem<TItemMetadata extends TMetadata = TMetadata>(item: Partial<IndexItem<TItemMetadata>>): Promise<IndexItem<TItemMetadata>> {
-        if (this._update) {
-            return await this.addItemToUpdate(item, false) as any;
-        } else {
-            await this.beginUpdate();
-            const newItem = await this.addItemToUpdate(item, false);
-            await this.endUpdate();
-            return newItem as any;
-        }
+    return top;
+  }
+
+  /**
+   * Adds or replaces an item in the index.
+   * @remarks
+   * A new update is started if one is not already in progress. If an item with the same ID
+   * already exists, it will be replaced.
+   * @param item Item to insert or replace.
+   * @returns Upserted item.
+   */
+  public async upsertItem<TItemMetadata extends TMetadata = TMetadata>(item: Partial<IndexItem<TItemMetadata>>): Promise<IndexItem<TItemMetadata>> {
+    if (this._update) {
+      return await this.addItemToUpdate(item, false) as any;
+    } else {
+      await this.beginUpdate();
+      const newItem = await this.addItemToUpdate(item, false);
+      await this.endUpdate();
+      return newItem as any;
     }
+  }
 
-    /**
-     * Ensures that the index has been loaded into memory.
-     */
-    protected async loadIndexData(): Promise<void> {
-        if (this._data) {
-            return;
-        }
-
-        if (!await this.isIndexCreated()) {
-            throw new Error('Index does not exist');
-        }
-
-        const data = await fs.readFile(path.join(this._folderPath, this.indexName));
-        this._data = JSON.parse(data.toString());
+  /** Ensures that the index has been loaded into memory. */
+  protected async loadIndexData(): Promise<void> {
+    if (this._data) {
+      return;
+    }
+    if (!await this.isIndexCreated()) {
+      throw new Error('Index does not exist');
     }
 
-    private async addItemToUpdate(item: Partial<IndexItem<any>>, unique: boolean): Promise<IndexItem> {
-        // Ensure vector is provided
-        if (!item.vector) {
-            throw new Error('Vector is required');
-        }
-
-        // Ensure unique
-        const id = item.id ?? v4();
-        if (unique) {
-            const existing = this._update!.items.find(i => i.id === id);
-            if (existing) {
-                throw new Error(`Item with id ${id} already exists`);
-            }
-        }
-
-        // Check for indexed metadata
-        let metadata: Record<string,any> = {};
-        let metadataFile: string | undefined;
-        if (this._update!.metadata_config.indexed && this._update!.metadata_config.indexed.length > 0 && item.metadata) {
-            // Copy only indexed metadata
-            for (const key of this._update!.metadata_config.indexed) {
-                if (item.metadata && item.metadata[key]) {
-                    metadata[key] = item.metadata[key];
-                }
-            }
+    const data = await this.storage.readFile(path.join(this._folderPath, this.indexName));
+    this._data = this._codec.deserializeIndex(data);
+  }
 
-            // Save remaining metadata to disk
-            metadataFile = `${v4()}.json`;
-            const metadataPath = path.join(this._folderPath, metadataFile);
-            await fs.writeFile(metadataPath, JSON.stringify(item.metadata));
-        } else if (item.metadata) {
-            metadata = item.metadata;
-        }
+  private async addItemToUpdate(item: Partial<IndexItem<any>>, unique: boolean): Promise<IndexItem> {
+    // Ensure vector is provided
+    if (!item.vector) {
+      throw new Error('Vector is required');
+    }
 
-        // Create new item
-        const newItem: IndexItem = {
-            id: id,
-            metadata: metadata,
-            vector: item.vector,
-            norm: ItemSelector.normalize(item.vector)
-        };
-        if (metadataFile) {
-            newItem.metadataFile = metadataFile;
-        }
+    // Ensure unique
+    const id = item.id ?? v4();
+    if (unique) {
+      const existing = this._update!.items.find(i => i.id === id);
+      if (existing) {
+        throw new Error(`Item with id ${id} already exists`);
+      }
+    }
 
-        // Add item to index
-        if (!unique) {
-            const existing = this._update!.items.find(i => i.id === id);
-            if (existing) {
-                existing.metadata = newItem.metadata;
-                existing.vector = newItem.vector;
-                existing.metadataFile = newItem.metadataFile;
-                return existing;
-            } else {
-                this._update!.items.push(newItem);
-                return newItem;
-            }
-        } else {
-            this._update!.items.push(newItem);
-            return newItem;
+    // Check for indexed metadata
+    let metadata: Record<string, any> = {};
+    let metadataFile: string | undefined;
+    const indexedKeys = this._update!.metadata_config.indexed ?? [];
+    if (indexedKeys.length > 0 && item.metadata) {
+      // Copy only indexed metadata
+      const indexedOnly: Record<string, any> = {};
+      for (const key of indexedKeys) {
+        if (Object.prototype.hasOwnProperty.call(item.metadata, key)) {
+          indexedOnly[key] = (item.metadata as any)[key];
         }
+      }
+
+      // Determine if there are any non-indexed keys
+      const hasNonIndexed = Object.keys(item.metadata).some(k => !indexedKeys.includes(k));
+
+      // Always store only indexed keys in the index
+      metadata = indexedOnly;
+
+      // Write full metadata externally only if there are non-indexed keys present
+      if (hasNonIndexed) {
+        metadataFile = `${v4()}${this._codec.extension}`;
+        const metadataPath = path.join(this._folderPath, metadataFile);
+        await this.storage.upsertFile(metadataPath, this._codec.serializeMetadata(item.metadata as Record<string, MetadataTypes>));
+      }
+    } else if (item.metadata) {
+      metadata = item.metadata;
     }
 
-    private async setupbm25(): Promise<any> {
-        this._bm25Engine = bm25();
-        const nlp = winkNLP( model );
-        const its = nlp.its;
-
-        const prepTask = function ( text: string ) {
-            const tokens: any[] = [];
-            nlp.readDoc(text)
-                .tokens()
-                // Use only words ignoring punctuations etc and from them remove stop words
-                .filter( (t: any) => ( t.out(its.type) === 'word' && !t.out(its.stopWordFlag) ) )
-                // Handle negation and extract stem of the word
-                .each( (t: any) => tokens.push( (t.out(its.negationFlag)) ? '!' + t.out(its.stem) : t.out(its.stem) ) );
-
-            return tokens;
-        };
-
-        this._bm25Engine.defineConfig( { fldWeights: { body: 1 } } );
-        // Step II: Define PrepTasks pipe.
-        this._bm25Engine.definePrepTasks( [ prepTask ] );
+    // Create new item
+    const newItem: IndexItem = {
+      id: id,
+      metadata: metadata,
+      vector: item.vector,
+      norm: ItemSelector.normalize(item.vector)
+    };
+    if (metadataFile) {
+      newItem.metadataFile = metadataFile;
     }
 
-    private async bm25Search(searchQuery: string, items: any, topK: number): Promise<any> {
-        var query = searchQuery;
-        // `results` is an array of [ doc-id, score ], sorted by score
-        var results = this._bm25Engine.search( query );
-
-        return results.slice(0, topK);
+    // Add item to index
+    if (!unique) {
+      const existing = this._update!.items.find(i => i.id === id);
+      if (existing) {
+        existing.metadata = newItem.metadata;
+        existing.vector = newItem.vector;
+        existing.metadataFile = newItem.metadataFile;
+        return existing;
+      } else {
+        this._update!.items.push(newItem);
+        return newItem;
+      }
+    } else {
+      this._update!.items.push(newItem);
+      return newItem;
     }
-
+  }
+
+  private async setupBm25(): Promise<any> {
+    this._bm25Engine = this._bm25Factory();
+    const nlp = winkNLP(model);
+    const its = nlp.its;
+    const prepTask = function (text: string) {
+      const tokens: any[] = [];
+      nlp.readDoc(text)
+        .tokens()
+        // Use only words ignoring punctuations etc and from them remove stop words
+        .filter((t: any) => (t.out(its.type) === 'word' && !t.out(its.stopWordFlag)))
+        // Handle negation and extract stem of the word
+        .each((t: any) => tokens.push((t.out(its.negationFlag)) ? '!' + t.out(its.stem) : t.out(its.stem)));
+      return tokens;
+    };
+    this._bm25Engine.defineConfig({ fldWeights: { body: 1 } });
+    // Step II: Define PrepTasks pipe.
+    this._bm25Engine.definePrepTasks([prepTask]);
+  }
+
+  private async bm25Search(searchQuery: string, _items: any, topK: number): Promise<any> {
+    const results = this._bm25Engine.search(searchQuery);
+    return results.slice(0, topK);
+  }
 }
 
 interface IndexData {
-    version: number;
-    metadata_config: {
-        indexed?: string[];
-    };
-    items: IndexItem[];
+  version: number;
+  metadata_config: {
+    indexed?: string[];
+  };
+  items: IndexItem[];
 }