verso-db 0.1.5 → 0.2.0

package/src/Collection.ts

@@ -1,1262 +0,0 @@
-import { HNSWIndex, DistanceMetric } from './HNSWIndex';
-import type { StorageBackend } from './storage/StorageBackend';
-import { DimensionMismatchError, DuplicateVectorError, VectorDBError } from './errors';
-
-/**
- * Configuration for adding vectors to a collection.
- */
-export interface AddConfig {
-  /** Unique string identifiers for each vector */
-  ids: string[];
-  /** Array of vectors to add (must match collection dimension) */
-  vectors: Array<number[] | Float32Array>;
-  /** Optional metadata for each vector (same length as ids/vectors, plain JSON objects only) */
-  metadata?: Array<Record<string, any>>;
-}
-
-/**
- * Configuration for querying a collection.
- *
- * @example
- * ```typescript
- * // Simple query
- * const results = await collection.query({
- *   queryVector: [0.1, 0.2, 0.3],
- *   k: 10
- * });
- *
- * // Query with metadata filter
- * const results = await collection.query({
- *   queryVector: [0.1, 0.2, 0.3],
- *   k: 10,
- *   filter: {
- *     category: 'science',           // Exact match
- *     year: { $gte: 2020 },          // Greater than or equal
- *     status: { $in: ['active', 'pending'] }  // In array
- *   }
- * });
- * ```
- */
-export interface QueryConfig {
-  /** The query vector (must match collection dimension) */
-  queryVector: number[] | Float32Array;
-  /** Number of nearest neighbors to return */
-  k: number;
-  /**
-   * Optional metadata filter. Supports MongoDB-style operators:
-   * - Simple equality: `{ field: value }`
-   * - `$gt`: Greater than `{ field: { $gt: 5 } }`
-   * - `$gte`: Greater than or equal `{ field: { $gte: 5 } }`
-   * - `$lt`: Less than `{ field: { $lt: 10 } }`
-   * - `$lte`: Less than or equal `{ field: { $lte: 10 } }`
-   * - `$ne`: Not equal `{ field: { $ne: 'excluded' } }`
-   * - `$in`: In array `{ field: { $in: ['a', 'b', 'c'] } }`
-   * - `$nin`: Not in array `{ field: { $nin: ['x', 'y'] } }`
-   */
-  filter?: Record<string, any>;
-  /** Search effort parameter (higher = better recall, slower). Default: max(k*2, 50) */
-  efSearch?: number;
-  /** Use int8 quantized search + float32 rescore. Default: auto (true when quantization enabled) */
-  useQuantizedSearch?: boolean;
-  /** Oversampling multiplier for quantized rescore (default: 3). Higher = better recall, slower. */
-  candidateMultiplier?: number;
-}
-
-export interface QueryResult {
-  ids: string[];
-  distances: number[];
-  metadata: Array<Record<string, any>>;
-}
-
-/** Supported metadata filter operators — hoisted to avoid per-call allocation */
-const SUPPORTED_OPERATORS = new Set(['$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin']);
-const COLLECTION_STATE_VERSION = 1;
-
-interface CollectionStorageState {
-  version: number;
-  indexKey: string;
-  metaKey: string;
-  deletedKey: string;
-}
-
-export class Collection {
-  private static readonly MAX_INTERNAL_ID = 0xFFFFFFFE;
-
-  private name: string;
-  private dimension: number;
-  private metric: DistanceMetric;
-  private M: number; // HNSW M parameter
-  private efConstruction: number; // HNSW efConstruction parameter
-  private storage: StorageBackend;
-  private readonly stateKey: string; // Storage key for atomic state pointer
-  private readonly defaultIndexKey: string;
-  private readonly defaultMetaKey: string;
-  private readonly defaultDeletedKey: string;
-  private indexKey: string;   // Storage key for HNSW index data
-  private metaKey: string;    // Storage key for metadata
-  private deletedKey: string; // Storage key for deleted IDs
-  private hnsw: HNSWIndex;
-  private idMap: Map<string, number>; // Maps string IDs to internal numeric IDs
-  private idReverseMap: Map<number, string>; // Maps internal numeric IDs to string IDs
-  private metadata: Map<number, Record<string, any>>; // Stores metadata for each vector
-  private deletedIds: Set<number>; // Tombstone markers for deleted vectors
-  private nextNumericId: number; // Monotonic counter for generating unique internal IDs
-  private activeCount: number; // O(1) counter for non-deleted vectors
-
-  constructor(name: string, config: { dimension: number; metric?: DistanceMetric; M?: number; efConstruction?: number }, storage: StorageBackend) {
-    this.name = name;
-    this.dimension = config.dimension;
-    this.metric = config.metric ?? 'cosine';
-    this.M = config.M ?? 16;
-    this.efConstruction = config.efConstruction ?? 200;
-    this.storage = storage;
-    this.defaultIndexKey = `${name}/${name}.hnsw`;
-    this.defaultMetaKey = `${name}/${name}.meta`;
-    this.defaultDeletedKey = `${name}/${name}.deleted`;
-    this.stateKey = `${name}/${name}.state`;
-    this.indexKey = this.defaultIndexKey;
-    this.metaKey = this.defaultMetaKey;
-    this.deletedKey = this.defaultDeletedKey;
-
-    // Initialize HNSW index with the specified parameters
-    this.hnsw = new HNSWIndex(
-      config.dimension,
-      this.metric,
-      this.M,
-      this.efConstruction
-    );
-
-    this.idMap = new Map();
-    this.idReverseMap = new Map();
-    this.metadata = new Map();
-    this.deletedIds = new Set();
-    this.nextNumericId = 0;
-    this.activeCount = 0;
-  }
-
-  async init(): Promise<void> {
-    // Load existing data if files exist
-    await this.loadFromDisk();
-  }
-
-  private reserveNumericIds(count: number): number {
-    if (!Number.isInteger(count) || count < 0) {
-      throw new VectorDBError(`Invalid ID reservation count: ${count}`, 'VALIDATION_ERROR');
-    }
-    if (count === 0) return this.nextNumericId;
-
-    const startId = this.nextNumericId;
-    const endId = startId + count - 1;
-
-    if (!Number.isSafeInteger(endId) || endId > Collection.MAX_INTERNAL_ID) {
-      throw new VectorDBError(
-        `Collection '${this.name}' cannot allocate ${count} internal IDs: exhausted ID space (max ${Collection.MAX_INTERNAL_ID})`,
-        'VALIDATION_ERROR'
-      );
-    }
-
-    this.nextNumericId = endId + 1;
-    return startId;
-  }
-
-  private setActiveStorageKeys(indexKey: string, metaKey: string, deletedKey: string): void {
-    this.indexKey = indexKey;
-    this.metaKey = metaKey;
-    this.deletedKey = deletedKey;
-  }
-
-  private isValidStorageState(value: unknown): value is CollectionStorageState {
-    if (!value || typeof value !== 'object') return false;
-    const state = value as Record<string, unknown>;
-    return (
-      state.version === COLLECTION_STATE_VERSION &&
-      typeof state.indexKey === 'string' &&
-      typeof state.metaKey === 'string' &&
-      typeof state.deletedKey === 'string'
-    );
-  }
-
-  private async loadStorageState(): Promise<CollectionStorageState | null> {
-    const stateBuffer = await this.storage.read(this.stateKey);
-    if (!stateBuffer) return null;
-
-    try {
-      const raw = JSON.parse(new TextDecoder().decode(stateBuffer));
-      if (this.isValidStorageState(raw)) {
-        return raw;
-      }
-    } catch {
-      // Ignore malformed state file and fall back to legacy keys.
-    }
-
-    return null;
-  }
-
-  private getVersionedDataKeys(saveId: string): { indexKey: string; metaKey: string; deletedKey: string } {
-    return {
-      indexKey: `${this.name}/${this.name}.${saveId}.hnsw`,
-      metaKey: `${this.name}/${this.name}.${saveId}.meta`,
-      deletedKey: `${this.name}/${this.name}.${saveId}.deleted`,
-    };
-  }
-
-  private isVersionedDataKey(key: string): boolean {
-    if (
-      key === this.defaultIndexKey ||
-      key === this.defaultMetaKey ||
-      key === this.defaultDeletedKey
-    ) {
-      return false;
-    }
-    const prefix = `${this.name}/${this.name}.`;
-    return key.startsWith(prefix) && (
-      key.endsWith('.hnsw') ||
-      key.endsWith('.meta') ||
-      key.endsWith('.deleted')
-    );
-  }
-
-  private validateLoadedIndexConfig(index: HNSWIndex): void {
-    if (
-      index.getDimension() !== this.dimension ||
-      index.getMetric() !== this.metric ||
-      index.getM() !== this.M ||
-      index.getEfConstruction() !== this.efConstruction
-    ) {
-      throw new VectorDBError(
-        `Collection '${this.name}' index parameters do not match manifest configuration`,
-        'CORRUPT_COLLECTION'
-      );
-    }
-  }
-
-  private async loadFromDisk(): Promise<void> {
-    // Reset in-memory state before loading.
-    this.idMap.clear();
-    this.idReverseMap.clear();
-    this.metadata.clear();
-    this.deletedIds.clear();
-    this.nextNumericId = 0;
-    this.activeCount = 0;
-
-    const persistedState = await this.loadStorageState();
-    if (persistedState) {
-      this.setActiveStorageKeys(persistedState.indexKey, persistedState.metaKey, persistedState.deletedKey);
-    } else {
-      this.setActiveStorageKeys(this.defaultIndexKey, this.defaultMetaKey, this.defaultDeletedKey);
-    }
-
-    // Load the binary HNSW index via StorageBackend
-    const indexData = await this.storage.read(this.indexKey);
-    if (indexData) {
-      try {
-        const loaded = HNSWIndex.deserialize(indexData);
-        this.validateLoadedIndexConfig(loaded);
-        this.hnsw = loaded;
-      } catch (e) {
-        if (e instanceof VectorDBError && e.code === 'CORRUPT_COLLECTION') {
-          throw e;
-        }
-        console.warn(`Failed to load HNSW index from ${this.indexKey}:`, e);
-        // Reset to an empty index to avoid stale in-memory graph state.
-        this.hnsw = new HNSWIndex(this.dimension, this.metric, this.M, this.efConstruction);
-      }
-    }
-
-    // Load metadata via StorageBackend
-    const metaData = await this.storage.read(this.metaKey);
-    if (metaData) {
-      try {
-        const metaContent = new TextDecoder().decode(metaData);
-        const lines = metaContent.split('\n');
-        for (let i = 0; i < lines.length; i++) {
-          const line = lines[i];
-          if (line.length === 0 || line.trim().length === 0) continue;
-
-          try {
-            const parsed = this.parseMetadataLine(line);
-            if (!parsed) continue;
-            const { id, internalId, metadata } = parsed;
-
-            this.idMap.set(id, internalId);
-            this.idReverseMap.set(internalId, id);
-            if (metadata !== undefined) {
-              this.metadata.set(internalId, metadata);
-            }
-          } catch {
-            // Skip malformed lines
-            console.warn(`Skipping malformed line in metadata file: ${line}`);
-          }
-        }
-      } catch {
-        // Ignore read errors
-      }
-    }
-
-    // Load deleted IDs via StorageBackend
-    const deletedData = await this.storage.read(this.deletedKey);
-    if (deletedData) {
-      try {
-        const deletedContent = new TextDecoder().decode(deletedData);
-        const deletedArray = JSON.parse(deletedContent);
-        if (Array.isArray(deletedArray)) {
-          this.deletedIds = new Set(deletedArray);
-        }
-      } catch {
-        // Ignore read errors, start with empty set
-      }
-    }
-
-    this.reconcileLoadedState();
-  }
-
-  async add(config: AddConfig): Promise<void> {
-    this.validateAddInput(config, false);
-
-    const points: Array<{ id: number; vector: Float32Array }> = new Array(config.ids.length);
-    const startId = this.reserveNumericIds(config.ids.length);
-
-    for (let i = 0; i < config.ids.length; i++) {
-      const vector = config.vectors[i];
-      const numericId = startId + i;
-      points[i] = { id: numericId, vector: new Float32Array(vector) };
-    }
-
-    await this.hnsw.addPointsBulk(points);
-
-    for (let i = 0; i < config.ids.length; i++) {
-      const id = config.ids[i];
-      const metadata = config.metadata ? config.metadata[i] : undefined;
-      const numericId = startId + i;
-
-      this.idMap.set(id, numericId);
-      this.idReverseMap.set(numericId, id);
-      if (metadata !== undefined) {
-        this.metadata.set(numericId, metadata);
-      }
-    }
-
-    this.activeCount += config.ids.length;
-  }
-
-  async query(config: QueryConfig): Promise<QueryResult> {
-    return this.executeQuery(config, 'Query vector');
-  }
-
-  /**
-   * Batch query for multiple vectors at once.
-   * Shares query semantics with query(), including adaptive candidate expansion
-   * for filters and tombstones.
-   *
-   * @param configs Array of query configurations
-   * @returns Array of query results, one per query
-   */
-  async queryBatch(configs: QueryConfig[]): Promise<QueryResult[]> {
-    if (!Array.isArray(configs)) {
-      throw new VectorDBError('Query batch must be an array of query configs', 'VALIDATION_ERROR');
-    }
-    if (configs.length === 0) return [];
-
-    for (let i = 0; i < configs.length; i++) {
-      if (!this.isPlainObject(configs[i])) {
-        throw new VectorDBError(`Query ${i} must be a plain object`, 'VALIDATION_ERROR');
-      }
-    }
-
-    const results: QueryResult[] = new Array(configs.length);
-    for (let i = 0; i < configs.length; i++) {
-      results[i] = this.executeQuery(configs[i], `Query ${i}`);
-    }
-    return results;
-  }
-
-  /**
-   * Brute-force KNN search for validation and correctness checking
-   * This checks all vectors and returns the true k nearest neighbors
-   */
-  async queryBruteForce(config: QueryConfig): Promise<QueryResult> {
-    this.validateQueryInput(config, 'Query vector');
-    const { queryVector, k, filter } = config;
-
-    // Compute distances to all vectors, applying filter during scan
-    const queryFloat = queryVector instanceof Float32Array ? queryVector : new Float32Array(queryVector);
-    const hasFilter = this.hasFilter(filter);
-    const distances: Array<{ id: number; distance: number }> = [];
-    for (const [numericId, vector] of this.hnsw.getAllVectors()) {
-      // Skip deleted vectors
-      if (this.deletedIds.has(numericId)) continue;
-
-      // Apply filter before distance computation when possible
-      if (hasFilter) {
-        const meta = this.metadata.get(numericId) || {};
-        if (!this.matchesFilter(meta, filter!)) continue;
-      }
-
-      const distance = this.hnsw.calculateDistance(queryFloat, vector);
-      distances.push({ id: numericId, distance });
-    }
-
-    // Sort by distance and take top k
-    distances.sort((a, b) => {
-      const diff = a.distance - b.distance;
-      return diff !== 0 ? diff : a.id - b.id;
-    });
-    const filteredResults = distances.slice(0, k);
-
-    // Prepare the output
-    const ids: string[] = [];
-    const distancesOut: number[] = [];
-    const metadata: Array<Record<string, any>> = [];
-
-    for (const result of filteredResults) {
-      const id = this.idReverseMap.get(result.id);
-      if (id) {
-        ids.push(id);
-        distancesOut.push(result.distance);
-        metadata.push(this.metadata.get(result.id) || {});
-      }
-    }
-
-    return {
-      ids,
-      distances: distancesOut,
-      metadata
-    };
-  }
-
-  /**
-   * Upsert vectors (insert or update).
-   *
-   * For existing IDs, the old vectors are tombstone-deleted and new vectors
-   * are inserted. Use compact() periodically to reclaim space from
-   * tombstoned vectors.
-   *
-   * For new IDs, behaves identically to add().
-   */
-  async upsert(config: AddConfig): Promise<void> {
-    this.validateAddInput(config, true);
-
-    const points: Array<{ id: number; vector: Float32Array }> = new Array(config.ids.length);
-    const startId = this.reserveNumericIds(config.ids.length);
-
-    for (let i = 0; i < config.ids.length; i++) {
-      points[i] = { id: startId + i, vector: new Float32Array(config.vectors[i]) };
-    }
-
-    await this.hnsw.addPointsBulk(points);
-
-    for (let i = 0; i < config.ids.length; i++) {
-      const id = config.ids[i];
-      const metadata = config.metadata ? config.metadata[i] : undefined;
-      const replacementId = this.idMap.get(id);
-      if (replacementId !== undefined) {
-        if (!this.deletedIds.has(replacementId)) {
-          this.activeCount--;
-        }
-        this.deletedIds.add(replacementId);
-        this.idReverseMap.delete(replacementId);
-        this.metadata.delete(replacementId);
-      }
-
-      const numericId = startId + i;
-      this.idMap.set(id, numericId);
-      this.idReverseMap.set(numericId, id);
-      if (metadata !== undefined) {
-        this.metadata.set(numericId, metadata);
-      }
-      this.activeCount++;
-    }
-  }
-
-  /** Returns the vector dimension for this collection. */
-  getDimension(): number { return this.dimension; }
-
-  /** Returns the distance metric for this collection. */
-  getMetric(): DistanceMetric { return this.metric; }
-
-  /** Returns the HNSW M parameter for this collection. */
-  getM(): number { return this.M; }
-
-  /** Returns the HNSW efConstruction parameter for this collection. */
-  getEfConstruction(): number { return this.efConstruction; }
-
-  /**
-   * Returns the number of active (non-deleted) vectors in the collection.
-   * O(1) via maintained counter.
-   */
-  count(): number {
-    return this.activeCount;
-  }
-
-  /**
-   * Returns the total number of tracked vectors including deleted (tombstoned) ones.
-   * Use this to determine when compaction might be beneficial.
-   */
-  countWithDeleted(): number {
-    return this.activeCount + this.deletedIds.size;
-  }
-
-  /**
-   * Returns the number of deleted (tombstoned) vectors awaiting compaction.
-   */
-  deletedCount(): number {
-    return this.deletedIds.size;
-  }
-
-  /**
-   * Mark a vector as deleted (tombstone deletion).
-   * The vector remains in the index but is excluded from search results.
-   * Use compact() to permanently remove deleted vectors and reclaim space.
-   *
-   * @param id The string ID of the vector to delete
-   * @returns true if the vector was deleted, false if it didn't exist or was already deleted
-   */
-  delete(id: string): boolean {
-    const numericId = this.idMap.get(id);
-    if (numericId === undefined) return false;
-    if (this.deletedIds.has(numericId)) return false;
-
-    this.deletedIds.add(numericId);
-    this.activeCount--;
-    return true;
-  }
-
-  /**
-   * Mark multiple vectors as deleted (tombstone deletion).
-   *
-   * @param ids Array of string IDs to delete
-   * @returns Number of vectors that were successfully deleted
-   */
-  deleteBatch(ids: string[]): number {
-    let deleted = 0;
-    for (const id of ids) {
-      if (this.delete(id)) deleted++;
-    }
-    return deleted;
-  }
-
-  /**
-   * Check if a vector exists and is not deleted.
-   */
-  has(id: string): boolean {
-    const numericId = this.idMap.get(id);
-    if (numericId === undefined) return false;
-    return !this.deletedIds.has(numericId);
-  }
-
-  /**
-   * Check if a vector was deleted (tombstoned).
-   */
-  isDeleted(id: string): boolean {
-    const numericId = this.idMap.get(id);
-    if (numericId === undefined) return false;
-    return this.deletedIds.has(numericId);
-  }
-
-  /**
-   * Reorder the internal HNSW index for BFS cache locality.
-   * Remaps all internal IDs so that graph neighbors are stored
-   * contiguously in memory, improving search cache hit rates.
-   */
-  reorderIndex(): void {
-    const oldToNew = this.hnsw.reorderForLocality();
-    if (oldToNew.size === 0) return;
-
-    // Remap idMap and idReverseMap
-    const newIdMap = new Map<string, number>();
-    const newIdReverseMap = new Map<number, string>();
-    for (const [strId, oldNum] of this.idMap) {
-      const newNum = oldToNew.get(oldNum);
-      if (newNum !== undefined) {
-        newIdMap.set(strId, newNum);
-        newIdReverseMap.set(newNum, strId);
-      }
-    }
-    this.idMap = newIdMap;
-    this.idReverseMap = newIdReverseMap;
-
-    // Remap metadata
-    const newMetadata = new Map<number, Record<string, any>>();
-    for (const [oldNum, meta] of this.metadata) {
-      const newNum = oldToNew.get(oldNum);
-      if (newNum !== undefined) {
-        newMetadata.set(newNum, meta);
-      }
-    }
-    this.metadata = newMetadata;
-
-    // Remap deletedIds
-    const newDeletedIds = new Set<number>();
-    for (const oldNum of this.deletedIds) {
-      const newNum = oldToNew.get(oldNum);
-      if (newNum !== undefined) {
-        newDeletedIds.add(newNum);
-      }
-    }
-    this.deletedIds = newDeletedIds;
-  }
-
-  async saveToDisk(): Promise<void> {
-    // Ensure the collection directory exists
-    await this.storage.mkdir(this.name);
-
-    const saveId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
-    const nextKeys = this.getVersionedDataKeys(saveId);
-    const previousKeys = {
-      indexKey: this.indexKey,
-      metaKey: this.metaKey,
-      deletedKey: this.deletedKey,
-    };
-
-    // Save the HNSW index as serialized binary via StorageBackend
-    const indexBuffer = this.hnsw.serialize();
-    const indexBytes = new Uint8Array(indexBuffer);
-    await this.storage.write(nextKeys.indexKey, indexBytes);
-
-    // Save metadata as JSON lines (safe for arbitrary string IDs).
-    const metaLines: string[] = [];
-    for (const [numericId, id] of this.idReverseMap) {
-      const meta = this.metadata.get(numericId);
-      metaLines.push(JSON.stringify({
-        id,
-        numericId,
-        metadata: meta ?? {},
-      }));
-    }
-    const metaBytes = new TextEncoder().encode(metaLines.join('\n'));
-    await this.storage.write(nextKeys.metaKey, metaBytes);
-
-    // Save deleted IDs
-    const deletedJson = JSON.stringify([...this.deletedIds]);
-    const deletedBytes = new TextEncoder().encode(deletedJson);
-    await this.storage.write(nextKeys.deletedKey, deletedBytes);
-
-    // Commit pointer last. Crash before this write keeps previous committed state.
-    const state: CollectionStorageState = {
-      version: COLLECTION_STATE_VERSION,
-      indexKey: nextKeys.indexKey,
-      metaKey: nextKeys.metaKey,
-      deletedKey: nextKeys.deletedKey,
-    };
-    await this.storage.write(this.stateKey, new TextEncoder().encode(JSON.stringify(state)));
-
-    this.setActiveStorageKeys(nextKeys.indexKey, nextKeys.metaKey, nextKeys.deletedKey);
-
-    // Backward-compatible shadow copies at legacy keys for tooling/tests that
-    // read fixed file names directly. Failures here are non-fatal because the
-    // committed state file is the source of truth for correctness.
-    await this.storage.write(this.defaultIndexKey, indexBytes).catch(() => {});
-    await this.storage.write(this.defaultMetaKey, metaBytes).catch(() => {});
-    await this.storage.write(this.defaultDeletedKey, deletedBytes).catch(() => {});
-
-    // Best-effort cleanup: only delete old versioned files. Legacy fixed keys are
-    // intentionally left untouched for backward compatibility.
-    const staleKeys = [previousKeys.indexKey, previousKeys.metaKey, previousKeys.deletedKey];
-    for (const key of staleKeys) {
-      if (!this.isVersionedDataKey(key)) continue;
-      if (key === this.indexKey || key === this.metaKey || key === this.deletedKey) continue;
-      await this.storage.delete(key).catch(() => {});
-    }
-  }
-
-  private executeQuery(config: QueryConfig, context: string): QueryResult {
-    this.validateQueryInput(config, context);
-
-    const queryVector = config.queryVector instanceof Float32Array
-      ? config.queryVector
-      : new Float32Array(config.queryVector);
-    const filter = this.hasFilter(config.filter) ? config.filter : undefined;
-    const efSearch = config.efSearch ?? Math.max(config.k * 2, 50);
-
-    // Determine whether to use quantized search:
-    // - Explicit true/false from config takes precedence
-    // - Otherwise auto-detect: use quantized when quantization is enabled
-    const useQuantized = config.useQuantizedSearch ?? this.hnsw.isQuantizationEnabled();
-    const candidateMultiplier = config.candidateMultiplier ?? 3;
-
-    const candidates = this.searchCandidates(queryVector, config.k, efSearch, filter, useQuantized, candidateMultiplier);
-    return this.materializeResults(candidates, config.k);
-  }
-
-  private searchCandidates(
-    queryVector: Float32Array,
-    k: number,
-    efSearch: number,
-    filter?: Record<string, any>,
-    useQuantized: boolean = false,
-    candidateMultiplier: number = 3
-  ): Array<{ id: number; distance: number }> {
-    const totalCandidates = this.countWithDeleted();
-    if (totalCandidates === 0) return [];
-
-    let requestK = Math.min(Math.max(k * 2, 50), totalCandidates);
-    let requestEf = Math.max(efSearch, requestK);
-    let filtered: Array<{ id: number; distance: number }> = [];
-
-    while (true) {
-      const rawResults = useQuantized
-        ? this.hnsw.searchKNNQuantized(queryVector, requestK, candidateMultiplier, requestEf)
-        : this.hnsw.searchKNN(queryVector, requestK, requestEf);
-      filtered = this.filterAndDeduplicateResults(rawResults, filter);
-
-      if (filtered.length >= k || requestK >= totalCandidates) {
-        return filtered;
-      }
-
-      const nextK = Math.min(totalCandidates, requestK * 2);
-      if (nextK === requestK) {
-        return filtered;
-      }
-      requestK = nextK;
-      requestEf = Math.max(requestEf, requestK);
-    }
-  }
-
-  private filterAndDeduplicateResults(
-    results: Array<{ id: number; distance: number }>,
-    filter?: Record<string, any>
-  ): Array<{ id: number; distance: number }> {
-    const filtered: Array<{ id: number; distance: number }> = [];
-    const hasFilter = !!filter;
-
-    for (const result of results) {
-      if (this.deletedIds.has(result.id)) {
-        continue;
-      }
-
-      if (hasFilter) {
-        const metadata = this.metadata.get(result.id) || {};
-        if (!this.matchesFilter(metadata, filter!)) {
-          continue;
-        }
-      }
-
-      filtered.push(result);
-    }
-
-    filtered.sort((a, b) => {
-      const diff = a.distance - b.distance;
-      return diff !== 0 ? diff : a.id - b.id;
-    });
-    return filtered;
-  }
-
-  private materializeResults(
-    candidates: Array<{ id: number; distance: number }>,
-    k: number
-  ): QueryResult {
-    const resultCount = Math.min(candidates.length, k);
-    const ids = new Array<string>(resultCount);
-    const distances = new Array<number>(resultCount);
-    const metadata = new Array<Record<string, any>>(resultCount);
-
-    let outIdx = 0;
-    for (let i = 0; i < resultCount; i++) {
-      const candidate = candidates[i];
-      const id = this.idReverseMap.get(candidate.id);
-      if (!id) continue;
-      ids[outIdx] = id;
-      distances[outIdx] = candidate.distance;
-      metadata[outIdx] = this.metadata.get(candidate.id) || {};
-      outIdx++;
-    }
-
-    if (outIdx < resultCount) {
-      ids.length = outIdx;
-      distances.length = outIdx;
-      metadata.length = outIdx;
-    }
-
-    return { ids, distances, metadata };
-  }
-
-  private validateQueryInput(config: QueryConfig, context: string): void {
-    if (!this.isPlainObject(config)) {
-      throw new VectorDBError(`${context} config must be a plain object`, 'VALIDATION_ERROR');
-    }
-
-    const queryVector = (config as QueryConfig).queryVector;
-    if (!(Array.isArray(queryVector) || queryVector instanceof Float32Array)) {
-      throw new VectorDBError(`${context} must include a queryVector (number[] or Float32Array)`, 'VALIDATION_ERROR');
-    }
-
-    if (!Number.isInteger(config.k) || config.k <= 0) {
-      throw new VectorDBError(
-        `k must be a positive integer, got ${config.k}`,
-        'VALIDATION_ERROR'
-      );
-    }
-
-    if (config.queryVector.length !== this.dimension) {
-      throw new DimensionMismatchError(this.dimension, config.queryVector.length, context);
-    }
-
-    for (let i = 0; i < config.queryVector.length; i++) {
-      if (!Number.isFinite(config.queryVector[i])) {
-        throw new VectorDBError(
-          `${context} contains non-finite value at dimension ${i}: ${config.queryVector[i]}`,
-          'VALIDATION_ERROR'
-        );
-      }
-    }
-
-    if (config.efSearch !== undefined) {
-      if (!Number.isInteger(config.efSearch) || config.efSearch <= 0) {
-        throw new VectorDBError(
-          `efSearch must be a positive integer, got ${config.efSearch}`,
-          'VALIDATION_ERROR'
-        );
-      }
-    }
-
-    if (config.filter !== undefined) {
-      if (!this.isPlainObject(config.filter)) {
-        throw new VectorDBError('Filter must be a plain object', 'VALIDATION_ERROR');
-      }
-      this.validateFilterOperators(config.filter);
-    }
-  }
-
-  private hasFilter(filter?: Record<string, any>): boolean {
-    return !!filter && Object.keys(filter).length > 0;
-  }
-
-  private reconcileLoadedState(): void {
-    const indexVectors = this.hnsw.getAllVectors();
-    const existingIds = new Set<number>();
-    let maxNumericId = -1;
-    for (const [id] of indexVectors) {
-      existingIds.add(id);
-      if (id > maxNumericId) maxNumericId = id;
-    }
-
-    for (const [id, numericId] of Array.from(this.idMap.entries())) {
-      if (!existingIds.has(numericId)) {
-        this.idMap.delete(id);
-      }
-    }
-
-    this.idReverseMap.clear();
-    for (const [id, numericId] of this.idMap.entries()) {
-      if (this.idReverseMap.has(numericId)) {
-        throw new VectorDBError(
-          `Collection '${this.name}' metadata is corrupt: duplicate numeric ID ${numericId}`,
-          'CORRUPT_COLLECTION'
-        );
-      }
-      this.idReverseMap.set(numericId, id);
-    }
-
-    for (const numericId of Array.from(this.deletedIds.values())) {
-      if (!existingIds.has(numericId)) {
-        this.deletedIds.delete(numericId);
-      }
-    }
-
-    for (const numericId of Array.from(this.metadata.keys())) {
-      if (!this.idReverseMap.has(numericId)) {
-        this.metadata.delete(numericId);
-      }
-    }
-
-    const missingMappings: number[] = [];
-    for (const numericId of existingIds) {
-      if (this.deletedIds.has(numericId)) continue;
-      if (!this.idReverseMap.has(numericId)) {
-        missingMappings.push(numericId);
-      }
-    }
-
-    if (missingMappings.length > 0) {
-      throw new VectorDBError(
-        `Collection '${this.name}' metadata is inconsistent with index: missing ${missingMappings.length} active ID mapping(s)`,
-        'CORRUPT_COLLECTION'
-      );
-    }
-
-    if (maxNumericId > Collection.MAX_INTERNAL_ID) {
-      throw new VectorDBError(
-        `Collection '${this.name}' contains unsupported internal ID ${maxNumericId} (max ${Collection.MAX_INTERNAL_ID})`,
-        'CORRUPT_COLLECTION'
-      );
-    }
-
-    this.nextNumericId = maxNumericId + 1;
-    this.activeCount = 0;
-    for (const numericId of this.idMap.values()) {
-      if (!this.deletedIds.has(numericId)) {
-        this.activeCount++;
-      }
-    }
-  }
-
-  private matchesFilter(metadata: Record<string, any>, filter: Record<string, any>): boolean {
-    const filterKeys = Object.keys(filter);
-    for (const key of filterKeys) {
-      const value = filter[key];
-      const metaValue = Object.prototype.hasOwnProperty.call(metadata, key)
-        ? metadata[key]
-        : undefined;
-
-      if (this.isOperatorObject(value)) {
-        for (const operator of Object.keys(value)) {
-          if (!SUPPORTED_OPERATORS.has(operator)) {
-            throw new VectorDBError(`Unsupported filter operator '${operator}'`, 'VALIDATION_ERROR');
-          }
-        }
-
-        if (metaValue === undefined && value.$ne === undefined && value.$nin === undefined) {
-          return false;
-        }
-
-        if (value.$eq !== undefined && !this.deepEqual(metaValue, value.$eq)) return false;
-        if (value.$ne !== undefined && this.deepEqual(metaValue, value.$ne)) return false;
-        // Comparison operators must reject undefined/missing fields explicitly,
-        // because JS comparisons like `undefined > 5` return false (not NaN/error),
-        // causing `!(undefined > 5)` to incorrectly pass.
-        if (value.$gt !== undefined && (metaValue === undefined || !(metaValue > value.$gt))) return false;
-        if (value.$lt !== undefined && (metaValue === undefined || !(metaValue < value.$lt))) return false;
-        if (value.$gte !== undefined && (metaValue === undefined || !(metaValue >= value.$gte))) return false;
-        if (value.$lte !== undefined && (metaValue === undefined || !(metaValue <= value.$lte))) return false;
-
-        if (value.$in !== undefined) {
-          if (!Array.isArray(value.$in) || !this.matchesInOperator(metaValue, value.$in)) return false;
-        }
-
-        if (value.$nin !== undefined) {
-          if (!Array.isArray(value.$nin)) return false;
-          if (this.matchesInOperator(metaValue, value.$nin)) return false;
-        }
-      } else {
-        if (!this.deepEqual(metaValue, value)) return false;
-      }
-    }
-    return true;
-  }
-
-  private validateFilterOperators(filter: Record<string, any>): void {
-    for (const key of Object.keys(filter)) {
-      const value = filter[key];
-      if (!this.isOperatorObject(value)) continue;
-      for (const operator of Object.keys(value)) {
-        if (!SUPPORTED_OPERATORS.has(operator)) {
-          throw new VectorDBError(`Unsupported filter operator '${operator}'`, 'VALIDATION_ERROR');
-        }
-      }
-    }
-  }
-
-  private validateAddInput(config: AddConfig, allowExistingIds: boolean): void {
-    if (!this.isPlainObject(config)) {
-      throw new VectorDBError('Add config must be a plain object', 'VALIDATION_ERROR');
-    }
-    if (!Array.isArray(config.ids)) {
-      throw new VectorDBError('ids must be an array of strings', 'VALIDATION_ERROR');
-    }
-    if (!Array.isArray(config.vectors)) {
-      throw new VectorDBError('vectors must be an array of number[] or Float32Array', 'VALIDATION_ERROR');
-    }
-
-    if (config.vectors.length !== config.ids.length) {
-      throw new VectorDBError('Number of vectors must match number of IDs', 'VALIDATION_ERROR');
-    }
-
-    if (config.metadata !== undefined && !Array.isArray(config.metadata)) {
-      throw new VectorDBError('metadata must be an array when provided', 'VALIDATION_ERROR');
-    }
-
-    if (config.metadata && config.metadata.length !== config.ids.length) {
-      throw new VectorDBError('Number of metadata entries must match number of IDs', 'VALIDATION_ERROR');
-    }
-
-    if (config.metadata) {
-      for (let i = 0; i < config.metadata.length; i++) {
-        const meta = config.metadata[i];
-        if (meta === undefined) continue;
-        if (!this.isPlainObject(meta)) {
-          throw new VectorDBError(
-            `Metadata at index ${i} must be a plain object`,
-            'VALIDATION_ERROR'
-          );
-        }
-        try {
-          JSON.stringify(meta);
-        } catch {
-          throw new VectorDBError(
-            `Metadata at index ${i} is not JSON-serializable`,
-            'VALIDATION_ERROR'
-          );
-        }
-      }
-    }
-
-    const seenIds = new Set<string>();
-    const duplicateIds = new Set<string>();
-    for (let i = 0; i < config.ids.length; i++) {
-      const id = config.ids[i];
-      if (typeof id !== 'string') {
-        throw new VectorDBError(
-          `ID at index ${i} must be a string`,
-          'VALIDATION_ERROR'
-        );
-      }
-      if (seenIds.has(id)) {
-        duplicateIds.add(id);
-      } else {
-        seenIds.add(id);
-      }
-      if (!allowExistingIds && this.idMap.has(id)) {
-        duplicateIds.add(id);
-      }
-    }
-
-    if (duplicateIds.size > 0) {
-      throw new DuplicateVectorError([...duplicateIds]);
-    }
-
-    for (let i = 0; i < config.vectors.length; i++) {
-      const vector = config.vectors[i];
-      if (!(Array.isArray(vector) || vector instanceof Float32Array)) {
-        throw new VectorDBError(
-          `Vector at index ${i} must be a number[] or Float32Array`,
-          'VALIDATION_ERROR'
-        );
-      }
-      if (vector.length !== this.dimension) {
-        throw new DimensionMismatchError(this.dimension, vector.length, `Vector at index ${i}`);
-      }
-      for (let d = 0; d < vector.length; d++) {
-        if (!Number.isFinite(vector[d])) {
-          throw new VectorDBError(
-            `Vector at index ${i} contains non-finite value at dimension ${d}: ${vector[d]}`,
-            'VALIDATION_ERROR'
-          );
-        }
-      }
-    }
-  }
-
-  private parseMetadataLine(line: string): { id: string; internalId: number; metadata?: Record<string, any> } | null {
-    const trimmed = line.trim();
-    if (trimmed.length === 0) return null;
-
-    // v2+ JSONL format
-    if (trimmed[0] === '{') {
-      const record = JSON.parse(trimmed) as { id?: unknown; numericId?: unknown; metadata?: unknown };
-      if (
-        typeof record.id !== 'string'
-        || !Number.isInteger(record.numericId)
-        || (record.numericId as number) < 0
-      ) {
-        return null;
-      }
-
-      if (record.metadata !== undefined && !this.isPlainObject(record.metadata)) {
-        return null;
-      }
-      return {
-        id: record.id,
-        internalId: record.numericId as number,
-        metadata: record.metadata === undefined ? undefined : (record.metadata as Record<string, any>),
-      };
-    }
-
-    // Legacy tab-separated format
-    const parts = line.split('\t');
-    if (parts.length < 2) return null;
-    const id = parts[0];
-    const internalId = parseInt(parts[1], 10);
-    if (!Number.isInteger(internalId) || internalId < 0) return null;
-
-    let metadata: Record<string, any> | undefined;
-    if (parts.length > 2) {
-      const parsedMetadata = JSON.parse(parts.slice(2).join('\t'));
-      if (!this.isPlainObject(parsedMetadata)) return null;
-      metadata = parsedMetadata as Record<string, any>;
-    }
-
-    return { id, internalId, metadata };
-  }
-
-  private isPlainObject(value: unknown): value is Record<string, any> {
-    return typeof value === 'object' && value !== null && !Array.isArray(value);
-  }
-
-  private isOperatorObject(value: unknown): value is Record<string, any> {
-    if (!this.isPlainObject(value)) return false;
-    const keys = Object.keys(value);
-    if (keys.length === 0) return false;
-    for (const key of keys) {
-      if (!key.startsWith('$')) return false;
-    }
-    return true;
-  }
-
-  private deepEqual(a: unknown, b: unknown): boolean {
-    if (Object.is(a, b)) return true;
-
-    if (Array.isArray(a) && Array.isArray(b)) {
-      if (a.length !== b.length) return false;
-      for (let i = 0; i < a.length; i++) {
-        if (!this.deepEqual(a[i], b[i])) return false;
-      }
-      return true;
-    }
-
-    if (this.isPlainObject(a) && this.isPlainObject(b)) {
-      const keysA = Object.keys(a);
-      const keysB = Object.keys(b);
-      if (keysA.length !== keysB.length) return false;
-      for (const key of keysA) {
-        if (!Object.prototype.hasOwnProperty.call(b, key)) return false;
-        if (!this.deepEqual(a[key], b[key])) return false;
-      }
-      return true;
-    }
-
-    return false;
-  }
-
-  private matchesInOperator(metaValue: unknown, candidates: unknown[]): boolean {
-    if (Array.isArray(metaValue)) {
-      for (const candidate of candidates) {
-        for (const item of metaValue) {
-          if (this.deepEqual(item, candidate)) return true;
-        }
-      }
-      return false;
-    }
-
-    for (const candidate of candidates) {
-      if (this.deepEqual(metaValue, candidate)) return true;
-    }
-    return false;
-  }
-
-  /**
-   * Compact the collection by rebuilding the index without deleted vectors.
-   * This permanently removes tombstoned vectors and reclaims space.
-   *
-   * @returns Number of vectors removed during compaction
-   */
-  async compact(): Promise<number> {
-    if (this.deletedIds.size === 0) return 0;
-
-    const removedCount = this.deletedIds.size;
-
-    // Collect all non-deleted vectors
-    const activeVectors: Array<{ id: string; numericId: number; vector: Float32Array; meta?: Record<string, any> }> = [];
-
-    for (const [numericId, vector] of this.hnsw.getAllVectors()) {
-      if (!this.deletedIds.has(numericId)) {
-        const stringId = this.idReverseMap.get(numericId);
-        if (stringId) {
-          activeVectors.push({
-            id: stringId,
-            numericId,
-            vector,
-            meta: this.metadata.get(numericId)
-          });
-        }
-      }
-    }
-
-    // Rebuild into temporary state first so we can roll back on failure.
-    const rebuiltHnsw = new HNSWIndex(
-      this.dimension,
-      this.metric,
-      this.M,
-      this.efConstruction
-    );
-    const rebuiltIdMap = new Map<string, number>();
-    const rebuiltIdReverseMap = new Map<number, string>();
-    const rebuiltMetadata = new Map<number, Record<string, any>>();
-    const rebuiltDeletedIds = new Set<number>();
-
-    // Prepare bulk insert data and re-populate mappings
-    const points: Array<{ id: number; vector: Float32Array }> = new Array(activeVectors.length);
-
-    for (let i = 0; i < activeVectors.length; i++) {
-      const { id, vector, meta } = activeVectors[i];
-      const newNumericId = i;
-
-      rebuiltIdMap.set(id, newNumericId);
-      rebuiltIdReverseMap.set(newNumericId, id);
-      if (meta) {
-        rebuiltMetadata.set(newNumericId, meta);
-      }
-
-      points[i] = { id: newNumericId, vector };
-    }
-
-    // Use bulk insert for better performance
-    try {
-      await rebuiltHnsw.addPointsBulk(points);
-    } catch (err) {
-      rebuiltHnsw.destroy();
-      throw err;
-    }
-
-    const previousState = {
-      hnsw: this.hnsw,
-      idMap: this.idMap,
-      idReverseMap: this.idReverseMap,
-      metadata: this.metadata,
-      deletedIds: this.deletedIds,
-      nextNumericId: this.nextNumericId,
-      activeCount: this.activeCount,
-      indexKey: this.indexKey,
-      metaKey: this.metaKey,
-      deletedKey: this.deletedKey,
-    };
-
-    this.hnsw = rebuiltHnsw;
-    this.idMap = rebuiltIdMap;
-    this.idReverseMap = rebuiltIdReverseMap;
-    this.metadata = rebuiltMetadata;
-    this.deletedIds = rebuiltDeletedIds;
-    this.nextNumericId = activeVectors.length;
-    this.activeCount = activeVectors.length;
-
-    try {
-      await this.saveToDisk();
-    } catch (err) {
-      this.hnsw.destroy();
-      this.hnsw = previousState.hnsw;
-      this.idMap = previousState.idMap;
-      this.idReverseMap = previousState.idReverseMap;
-      this.metadata = previousState.metadata;
-      this.deletedIds = previousState.deletedIds;
-      this.nextNumericId = previousState.nextNumericId;
-      this.activeCount = previousState.activeCount;
-      this.setActiveStorageKeys(previousState.indexKey, previousState.metaKey, previousState.deletedKey);
-      throw err;
-    }
-
-    previousState.hnsw.destroy();
-    return removedCount;
-  }
-
-  /**
-   * Destroy the collection, freeing all in-memory resources.
-   * @param save Whether to persist data to storage before destroying (default: true).
-   *             Pass false when the collection is being deleted entirely.
-   */
-  async destroy(save: boolean = true): Promise<void> {
-    if (save) {
-      await this.saveToDisk();
-    }
-
-    // Destroy the HNSW index to free memory
-    if (this.hnsw && typeof this.hnsw.destroy === 'function') {
-      this.hnsw.destroy();
-    }
-
-    // Clear all maps to free memory
-    this.idMap.clear();
-    this.idReverseMap.clear();
-    this.metadata.clear();
-    this.deletedIds.clear();
-    this.activeCount = 0;
-  }
-}