verso-db 0.1.5 → 0.2.0

package/src/VectorDB.ts

@@ -1,319 +0,0 @@
-import { Collection } from './Collection';
-import type { DistanceMetric } from './HNSWIndex';
-import type { StorageBackend } from './storage/StorageBackend';
-import { BunStorageBackend } from './storage/BunStorageBackend';
-import { MemoryBackend } from './storage/MemoryBackend';
-import { OPFSBackend } from './storage/OPFSBackend';
-import { CollectionExistsError, CollectionNotFoundError, VectorDBError } from './errors';
-
-export interface VectorDBConfig {
-  /**
-   * Path for storing collections on disk. Default: './vectordb_data'
-   * Ignored if a custom storageBackend is provided.
-   */
-  storagePath?: string;
-  /**
-   * Custom storage backend for platform-specific storage (e.g., OPFS for browsers).
-   * If not provided, backend is auto-selected for the current runtime.
-   */
-  storageBackend?: StorageBackend;
-}
-
-export interface CollectionConfig {
-  dimension: number;
-  metric?: DistanceMetric;
-  M?: number;
-  efConstruction?: number;
-}
-
-/** Manifest entry stored for each collection */
-interface ManifestEntry {
-  dimension: number;
-  metric: DistanceMetric;
-  M: number;
-  efConstruction: number;
-}
-
-const MANIFEST_KEY = 'manifest.json';
-
-export class VectorDB {
-  private collections: Map<string, Collection>;
-  private storageBackend: StorageBackend;
-  private initialized: boolean = false;
-  private initPromise: Promise<void> | null = null;
-  private createCollectionLocks: Map<string, Promise<void>> = new Map();
-
-  constructor(config?: VectorDBConfig) {
-    this.collections = new Map();
-    if (config?.storageBackend) {
-      this.storageBackend = config.storageBackend;
-    } else {
-      this.storageBackend = this.createDefaultStorageBackend(config?.storagePath);
-    }
-  }
-
-  private createDefaultStorageBackend(storagePath?: string): StorageBackend {
-    if (typeof Bun !== 'undefined') {
-      return new BunStorageBackend(storagePath || './vectordb_data');
-    }
-
-    if (typeof navigator !== 'undefined' && OPFSBackend.isAvailable()) {
-      return new OPFSBackend();
-    }
-
-    return new MemoryBackend();
-  }
-
-  /**
-   * Get the storage backend used by this VectorDB instance.
-   * Useful for custom storage operations or debugging.
-   */
-  getStorageBackend(): StorageBackend {
-    return this.storageBackend;
-  }
-
-  /**
-   * Initialize the VectorDB - creates storage directory and loads existing collections.
-   * Called automatically on first operation if not called explicitly.
-   * Safe to call concurrently — only runs initialization once.
-   */
-  async init(): Promise<void> {
-    if (this.initialized) return;
-    if (!this.initPromise) {
-      this.initPromise = this.doInit().catch((err) => {
-        // Allow retry after transient initialization failures.
-        this.initPromise = null;
-        this.initialized = false;
-        throw err;
-      });
-    }
-    return this.initPromise;
-  }
-
-  private async doInit(): Promise<void> {
-    // Use storage backend to create root directory
-    await this.storageBackend.mkdir('');
-
-    // Load existing collections from manifest
-    await this.loadManifest();
-
-    this.initialized = true;
-  }
-
-  /**
-   * Load collection metadata from the manifest file and restore collections.
-   */
-  private async loadManifest(): Promise<void> {
-    const data = await this.storageBackend.read(MANIFEST_KEY);
-    if (!data) return;
-
-    let manifest: Record<string, ManifestEntry>;
-    try {
-      const text = new TextDecoder().decode(data);
-      manifest = JSON.parse(text);
-    } catch {
-      // Corrupt manifest JSON — continue with empty collections
-      return;
-    }
-
-    const VALID_METRICS = new Set(['cosine', 'euclidean', 'dot_product']);
-
-    for (const [name, entry] of Object.entries(manifest)) {
-      // Skip collections already loaded
-      if (this.collections.has(name)) continue;
-
-      try {
-        this.validateCollectionName(name);
-      } catch {
-        console.warn(`Skipping collection '${name}' with invalid name in manifest`);
-        continue;
-      }
-
-      // Validate manifest entry types to catch corrupted manifests early
-      if (
-        typeof entry.dimension !== 'number' || entry.dimension <= 0 || !Number.isInteger(entry.dimension) ||
-        typeof entry.metric !== 'string' || !VALID_METRICS.has(entry.metric) ||
-        typeof entry.M !== 'number' || entry.M <= 0 || !Number.isInteger(entry.M) ||
-        typeof entry.efConstruction !== 'number' || entry.efConstruction <= 0 || !Number.isInteger(entry.efConstruction)
-      ) {
-        console.warn(`Skipping collection '${name}' with invalid manifest entry`);
-        continue;
-      }
-
-      const config: CollectionConfig = {
-        dimension: entry.dimension,
-        metric: entry.metric,
-        M: entry.M,
-        efConstruction: entry.efConstruction,
-      };
-
-      try {
-        const collection = new Collection(name, config, this.storageBackend);
-        await collection.init();
-        this.collections.set(name, collection);
-      } catch (e) {
-        // Per-collection init failure — skip this collection but continue loading others
-        console.warn(`Failed to load collection '${name}':`, e);
-      }
-    }
-  }
-
-  /**
-   * Save collection metadata to the manifest file.
-   */
-  private async saveManifest(): Promise<void> {
-    const manifest: Record<string, ManifestEntry> = {};
-
-    for (const [name, collection] of this.collections) {
-      manifest[name] = {
-        dimension: collection.getDimension(),
-        metric: collection.getMetric(),
-        M: collection.getM(),
-        efConstruction: collection.getEfConstruction(),
-      };
-    }
-
-    const data = new TextEncoder().encode(JSON.stringify(manifest));
-    await this.storageBackend.write(MANIFEST_KEY, data);
-  }
-
-  private async withCollectionCreateLock<T>(name: string, fn: () => Promise<T>): Promise<T> {
-    const previous = this.createCollectionLocks.get(name) ?? Promise.resolve();
-    let release!: () => void;
-    const lock = new Promise<void>((resolve) => {
-      release = resolve;
-    });
-    this.createCollectionLocks.set(name, lock);
-
-    await previous;
-    try {
-      return await fn();
-    } finally {
-      release();
-      if (this.createCollectionLocks.get(name) === lock) {
-        this.createCollectionLocks.delete(name);
-      }
-    }
-  }
-
-  async createCollection(name: string, config: CollectionConfig): Promise<Collection> {
-    this.validateCollectionName(name);
-
-    // Ensure VectorDB is initialized
-    await this.init();
-
-    return this.withCollectionCreateLock(name, async () => {
-      if (this.collections.has(name)) {
-        throw new CollectionExistsError(name);
-      }
-
-      // Create collection directory using storage backend
-      await this.storageBackend.mkdir(name);
-
-      const collection = new Collection(name, config, this.storageBackend);
-      await collection.init();
-      this.collections.set(name, collection);
-
-      // Persist manifest
-      await this.saveManifest();
-
-      return collection;
-    });
-  }
-
-  async getCollection(name: string): Promise<Collection | undefined> {
-    await this.init();
-    return this.collections.get(name);
-  }
-
-  async listCollections(): Promise<string[]> {
-    await this.init();
-    return [...this.collections.keys()];
-  }
-
-  async deleteCollection(name: string): Promise<void> {
-    await this.init();
-    const collection = this.collections.get(name);
-    if (!collection) {
-      throw new CollectionNotFoundError(name);
-    }
-
-    // Destroy without saving — we're deleting the collection
-    await collection.destroy(false);
-    this.collections.delete(name);
-
-    // Remove all files under the collection prefix, including versioned snapshots.
-    const keys = await this.storageBackend.list(name);
-    for (const key of keys) {
-      await this.storageBackend.delete(key).catch(() => {});
-    }
-
-    // Update manifest
-    await this.saveManifest();
-  }
-
-  private validateCollectionName(name: string): void {
-    if (!name || name.trim().length === 0) {
-      throw new VectorDBError('Collection name must be a non-empty string', 'VALIDATION_ERROR');
-    }
-    if (name.length > 255) {
-      throw new VectorDBError(
-        `Collection name must be 255 characters or fewer, got ${name.length}`,
-        'VALIDATION_ERROR'
-      );
-    }
-    if (/[\/\\.\x00]/.test(name)) {
-      throw new VectorDBError(
-        `Collection name '${name}' contains invalid characters (/, \\, ., or null)`,
-        'VALIDATION_ERROR'
-      );
-    }
-    // Reject control characters (U+0001–U+001F, U+007F)
-    // eslint-disable-next-line no-control-regex
-    if (/[\x01-\x1f\x7f]/.test(name)) {
-      throw new VectorDBError(
-        'Collection name contains control characters',
-        'VALIDATION_ERROR'
-      );
-    }
-    // Reject leading/trailing whitespace (checked after control chars since
-    // tab/etc. are control chars and should get the more specific message)
-    if (name !== name.trim()) {
-      throw new VectorDBError('Collection name must not have leading or trailing whitespace', 'VALIDATION_ERROR');
-    }
-  }
-
-  async close(): Promise<void> {
-    // Close all collections — continue on individual failures so remaining
-    // collections still get saved and cleanup always completes.
-    const errors: Error[] = [];
-    for (const collection of this.collections.values()) {
-      try {
-        await collection.destroy();
-      } catch (e) {
-        errors.push(e instanceof Error ? e : new Error(String(e)));
-      }
-    }
-
-    // Persist the manifest so collection metadata survives re-open
-    if (this.initialized) {
-      try {
-        await this.saveManifest();
-      } catch (e) {
-        errors.push(e instanceof Error ? e : new Error(String(e)));
-      }
-    }
-
-    this.collections.clear();
-    this.createCollectionLocks.clear();
-    this.initialized = false;
-    this.initPromise = null;
-
-    if (errors.length > 0) {
-      throw new VectorDBError(
-        `Failed to close ${errors.length} collection(s): ${errors.map(e => e.message).join('; ')}`,
-        'STORAGE_ERROR'
-      );
-    }
-  }
-}

package/src/WorkerPool.ts

@@ -1,222 +0,0 @@
-/**
- * Worker pool for parallel query processing.
- * Dispatches queries to workers via round-robin for concurrent execution.
- * Falls back to sequential processing when workers are unavailable.
- */
-
-import { HNSWIndex } from './HNSWIndex';
-
-interface WorkerHandle {
-  worker: Worker;
-  busy: boolean;
-  pendingResolve: Map<number, (results: Array<{ id: number; distance: number }>) => void>;
-  pendingReject: Map<number, (error: Error) => void>;
-}
-
-export class WorkerPool {
-  private workers: WorkerHandle[] = [];
-  private nextQueryId: number = 0;
-  private roundRobinIndex: number = 0;
-  private initialized: boolean = false;
-  private fallbackIndex: HNSWIndex | null = null;
-
-  /**
-   * Create a worker pool for parallel HNSW search.
-   *
-   * @param numWorkers Number of workers (default: available hardware concurrency or 4)
-   */
-  constructor(private numWorkers?: number) {
-    if (!numWorkers) {
-      this.numWorkers = typeof navigator !== 'undefined'
-        ? (navigator.hardwareConcurrency ?? 4)
-        : 4;
-    }
-  }
-
-  /**
-   * Initialize workers with shared index data.
-   * Each worker receives a copy of the flat vectors and graph structure
-   * and maintains its own independent search state.
-   *
-   * @param index The HNSW index to distribute to workers
-   */
-  async init(index: HNSWIndex): Promise<void> {
-    this.fallbackIndex = index;
-
-    // Check if Worker is available
-    if (typeof Worker === 'undefined') {
-      // Workers not available, will use fallback
-      this.initialized = true;
-      return;
-    }
-
-    const sharedData = index.getSharedSearchData();
-    if (!sharedData) {
-      // Index can't produce shared data, use fallback
-      this.initialized = true;
-      return;
-    }
-
-    const workerUrl = new URL('./SearchWorker.ts', import.meta.url).href;
-
-    const initPromises: Promise<void>[] = [];
-
-    for (let i = 0; i < this.numWorkers!; i++) {
-      try {
-        const worker = new Worker(workerUrl, { type: 'module' });
-        const handle: WorkerHandle = {
-          worker,
-          busy: false,
-          pendingResolve: new Map(),
-          pendingReject: new Map(),
-        };
-
-        worker.onmessage = (event: MessageEvent) => {
-          const msg = event.data;
-          if (msg.type === 'result') {
-            const resolve = handle.pendingResolve.get(msg.queryId);
-            if (resolve) {
-              handle.pendingResolve.delete(msg.queryId);
-              handle.pendingReject.delete(msg.queryId);
-              resolve(msg.results);
-            }
-          } else if (msg.type === 'error') {
-            const reject = handle.pendingReject.get(msg.queryId);
-            if (reject) {
-              handle.pendingResolve.delete(msg.queryId);
-              handle.pendingReject.delete(msg.queryId);
-              reject(new Error(msg.error));
-            }
-          }
-        };
-
-        worker.onerror = (event) => {
-          // Reject all pending queries for this worker
-          for (const [qid, reject] of handle.pendingReject) {
-            reject(new Error(`Worker error: ${event.message}`));
-          }
-          handle.pendingResolve.clear();
-          handle.pendingReject.clear();
-        };
-
-        const readyPromise = new Promise<void>((resolve) => {
-          const originalHandler = worker.onmessage!;
-          worker.onmessage = (event: MessageEvent) => {
-            if (event.data.type === 'ready') {
-              worker.onmessage = originalHandler;
-              resolve();
-            }
-          };
-        });
-
-        worker.postMessage({
-          type: 'init',
-          ...sharedData,
-        });
-
-        this.workers.push(handle);
-        initPromises.push(readyPromise);
-      } catch {
-        // Worker creation failed, will use fallback for remaining
-        break;
-      }
-    }
-
-    if (initPromises.length > 0) {
-      await Promise.all(initPromises);
-    }
-
-    this.initialized = true;
-  }
-
-  /**
-   * Search for k nearest neighbors using worker pool.
-   * Dispatches to next available worker via round-robin.
-   *
-   * @param query Query vector
-   * @param k Number of results
-   * @param efSearch Search effort parameter
-   * @returns Array of {id, distance} results
-   */
-  async search(
-    query: Float32Array,
-    k: number,
-    efSearch?: number
-  ): Promise<Array<{ id: number; distance: number }>> {
-    if (!this.initialized) {
-      throw new Error('WorkerPool not initialized. Call init() first.');
-    }
-
-    // Fallback to sequential if no workers available
-    if (this.workers.length === 0 && this.fallbackIndex) {
-      return this.fallbackIndex.searchKNN(query, k, efSearch);
-    }
-
-    const queryId = this.nextQueryId++;
-    const workerIdx = this.roundRobinIndex % this.workers.length;
-    this.roundRobinIndex++;
-    const handle = this.workers[workerIdx];
-
-    return new Promise<Array<{ id: number; distance: number }>>((resolve, reject) => {
-      handle.pendingResolve.set(queryId, resolve);
-      handle.pendingReject.set(queryId, reject);
-      handle.worker.postMessage({
-        type: 'search',
-        queryId,
-        query,
-        k,
-        efSearch: efSearch || Math.max(k * 2, 50),
-      });
-    });
-  }
-
-  /**
-   * Batch search: dispatch multiple queries in parallel across workers.
-   *
-   * @param queries Array of query vectors
-   * @param k Number of results per query
-   * @param efSearch Search effort parameter
-   * @returns Array of results, one per query
-   */
-  async searchBatch(
-    queries: Float32Array[],
-    k: number,
-    efSearch?: number
-  ): Promise<Array<Array<{ id: number; distance: number }>>> {
-    if (!this.initialized) {
-      throw new Error('WorkerPool not initialized. Call init() first.');
-    }
-
-    // Fallback to sequential
-    if (this.workers.length === 0 && this.fallbackIndex) {
-      return queries.map(q => this.fallbackIndex!.searchKNN(q, k, efSearch));
-    }
-
-    const promises = queries.map(q => this.search(q, k, efSearch));
-    return Promise.all(promises);
-  }
-
-  /**
-   * Terminate all workers and clean up.
-   */
-  destroy(): void {
-    for (const handle of this.workers) {
-      handle.worker.terminate();
-      for (const reject of handle.pendingReject.values()) {
-        reject(new Error('Worker pool destroyed'));
-      }
-      handle.pendingResolve.clear();
-      handle.pendingReject.clear();
-    }
-    this.workers = [];
-    this.initialized = false;
-    this.fallbackIndex = null;
-  }
-
-  /**
-   * Get the number of active workers.
-   */
-  get workerCount(): number {
-    return this.workers.length;
-  }
-}

package/src/backends/JsDistanceBackend.ts

@@ -1,171 +0,0 @@
-/**
- * Optimized JavaScript distance functions with SIMD-style unrolling.
- * These provide ~1.5-2x speedup over naive loops by reducing loop overhead
- * and enabling better CPU pipelining.
- */
-
-/**
- * Fast inline distance functions for single-vector comparisons.
- * Uses 8-wide unrolling with separate accumulators for instruction-level parallelism.
- */
-function assertMatchingDimensions(a: Float32Array, b: Float32Array, context: string): void {
-  if (a.length !== b.length) {
-    throw new Error(`${context}: vector length mismatch (${a.length} vs ${b.length})`);
-  }
-}
-
-export function dotProductFast(a: Float32Array, b: Float32Array): number {
-  assertMatchingDimensions(a, b, 'dotProductFast');
-  const len = a.length;
-  // Use 8 accumulators for better ILP (instruction-level parallelism)
-  let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
-  let sum4 = 0, sum5 = 0, sum6 = 0, sum7 = 0;
-  let i = 0;
-
-  // 8-wide unrolling for high-dimensional vectors (768D, 1536D)
-  const limit8 = len - 7;
-  for (; i < limit8; i += 8) {
-    sum0 += a[i] * b[i];
-    sum1 += a[i + 1] * b[i + 1];
-    sum2 += a[i + 2] * b[i + 2];
-    sum3 += a[i + 3] * b[i + 3];
-    sum4 += a[i + 4] * b[i + 4];
-    sum5 += a[i + 5] * b[i + 5];
-    sum6 += a[i + 6] * b[i + 6];
-    sum7 += a[i + 7] * b[i + 7];
-  }
-
-  // Handle remaining elements
-  for (; i < len; i++) {
-    sum0 += a[i] * b[i];
-  }
-
-  return sum0 + sum1 + sum2 + sum3 + sum4 + sum5 + sum6 + sum7;
-}
-
-export function l2SquaredFast(a: Float32Array, b: Float32Array): number {
-  assertMatchingDimensions(a, b, 'l2SquaredFast');
-  const len = a.length;
-  // Use 8 accumulators for better ILP
-  let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
-  let sum4 = 0, sum5 = 0, sum6 = 0, sum7 = 0;
-  let i = 0;
-
-  // 8-wide unrolling for high-dimensional vectors
-  const limit8 = len - 7;
-  for (; i < limit8; i += 8) {
-    const d0 = a[i] - b[i];
-    const d1 = a[i + 1] - b[i + 1];
-    const d2 = a[i + 2] - b[i + 2];
-    const d3 = a[i + 3] - b[i + 3];
-    const d4 = a[i + 4] - b[i + 4];
-    const d5 = a[i + 5] - b[i + 5];
-    const d6 = a[i + 6] - b[i + 6];
-    const d7 = a[i + 7] - b[i + 7];
-    sum0 += d0 * d0;
-    sum1 += d1 * d1;
-    sum2 += d2 * d2;
-    sum3 += d3 * d3;
-    sum4 += d4 * d4;
-    sum5 += d5 * d5;
-    sum6 += d6 * d6;
-    sum7 += d7 * d7;
-  }
-
-  // Handle remaining elements
-  for (; i < len; i++) {
-    const d = a[i] - b[i];
-    sum0 += d * d;
-  }
-
-  return sum0 + sum1 + sum2 + sum3 + sum4 + sum5 + sum6 + sum7;
-}
-
-/**
- * Normalize a vector in place.
- * Uses 8-wide unrolling for better ILP.
- * Returns the original norm of the vector.
- */
-export function normalizeInPlace(v: Float32Array): number {
-  const len = v.length;
-  let s0 = 0, s1 = 0, s2 = 0, s3 = 0;
-  let s4 = 0, s5 = 0, s6 = 0, s7 = 0;
-  let i = 0;
-  const limit8 = len - 7;
-
-  for (; i < limit8; i += 8) {
-    s0 += v[i] * v[i];
-    s1 += v[i + 1] * v[i + 1];
-    s2 += v[i + 2] * v[i + 2];
-    s3 += v[i + 3] * v[i + 3];
-    s4 += v[i + 4] * v[i + 4];
-    s5 += v[i + 5] * v[i + 5];
-    s6 += v[i + 6] * v[i + 6];
-    s7 += v[i + 7] * v[i + 7];
-  }
-  for (; i < len; i++) {
-    s0 += v[i] * v[i];
-  }
-
-  const norm = Math.sqrt(s0 + s1 + s2 + s3 + s4 + s5 + s6 + s7);
-  if (norm < 1e-10) {
-    // Zero or near-zero vector: leave as-is to avoid Infinity/NaN from division
-    return norm;
-  }
-  const invNorm = 1 / norm;
-  // 8-wide unrolling for the division loop matches the norm computation above
-  const divLimit8 = len - 7;
-  let j = 0;
-  for (; j < divLimit8; j += 8) {
-    v[j] *= invNorm; v[j + 1] *= invNorm; v[j + 2] *= invNorm; v[j + 3] *= invNorm;
-    v[j + 4] *= invNorm; v[j + 5] *= invNorm; v[j + 6] *= invNorm; v[j + 7] *= invNorm;
-  }
-  for (; j < len; j++) v[j] *= invNorm;
-  return norm;
-}
-
-/**
- * Compute cosine distance between two vectors.
- * For pre-normalized vectors, this is simply 1 - dot(a, b).
- * Uses 8-wide unrolling for norm computation.
- */
-export function cosineDistanceFast(a: Float32Array, b: Float32Array, aIsNormalized = false, bIsNormalized = false): number {
-  assertMatchingDimensions(a, b, 'cosineDistanceFast');
-  const dot = dotProductFast(a, b);
-
-  if (aIsNormalized && bIsNormalized) {
-    // Both vectors are normalized, cosine distance = 1 - dot
-    const dist = 1 - dot;
-    return dist < 1e-10 ? 0 : dist;
-  }
-
-  // Need to compute norms with 8-wide unrolling
-  const len = a.length;
-  let nA0 = 0, nA1 = 0, nA2 = 0, nA3 = 0;
-  let nB0 = 0, nB1 = 0, nB2 = 0, nB3 = 0;
-  let i = 0;
-  const limit8 = len - 7;
-
-  for (; i < limit8; i += 8) {
-    nA0 += a[i] * a[i] + a[i + 4] * a[i + 4];
-    nA1 += a[i + 1] * a[i + 1] + a[i + 5] * a[i + 5];
-    nA2 += a[i + 2] * a[i + 2] + a[i + 6] * a[i + 6];
-    nA3 += a[i + 3] * a[i + 3] + a[i + 7] * a[i + 7];
-    nB0 += b[i] * b[i] + b[i + 4] * b[i + 4];
-    nB1 += b[i + 1] * b[i + 1] + b[i + 5] * b[i + 5];
-    nB2 += b[i + 2] * b[i + 2] + b[i + 6] * b[i + 6];
-    nB3 += b[i + 3] * b[i + 3] + b[i + 7] * b[i + 7];
-  }
-  for (; i < len; i++) {
-    nA0 += a[i] * a[i];
-    nB0 += b[i] * b[i];
-  }
-
-  const normA = nA0 + nA1 + nA2 + nA3;
-  const normB = nB0 + nB1 + nB2 + nB3;
-  const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
-  if (magnitude === 0) return 1;
-
-  const dist = 1 - (dot / magnitude);
-  return dist < 1e-10 ? 0 : dist;
-}