verso-db 0.1.1

package/src/presets.ts

@@ -0,0 +1,229 @@
+/**
+ * HNSW Parameter Presets
+ *
+ * These presets are optimized based on extensive benchmarking to achieve
+ * the target recall@10 >= 95% for different dataset sizes and dimensions.
+ *
+ * Key parameters:
+ * - M: Maximum connections per node (higher = better recall, more memory)
+ * - efConstruction: Beam width during index building (higher = better quality, slower build)
+ * - efSearch: Beam width during search (higher = better recall, slower search)
+ */
+
+export interface HNSWPreset {
+  name: string;
+  description: string;
+  M: number;
+  efConstruction: number;
+  efSearch: number;
+  expectedRecall: number;
+  targetDimensions: string;
+  targetDatasetSize: string;
+}
+
+/**
+ * Preset for low-dimensional vectors (128D or less)
+ * Suitable for: Image features, word2vec, GloVe embeddings
+ */
+export const PRESET_LOW_DIM: HNSWPreset = {
+  name: 'low-dim',
+  description: 'Optimized for low-dimensional vectors (<=128D)',
+  M: 16,
+  efConstruction: 200,
+  efSearch: 100,
+  expectedRecall: 0.99,
+  targetDimensions: '<=128',
+  targetDatasetSize: '1K-100K',
+};
+
+/**
+ * Preset for medium-dimensional vectors (256-512D)
+ * Suitable for: Sentence embeddings, smaller transformer outputs
+ */
+export const PRESET_MEDIUM_DIM: HNSWPreset = {
+  name: 'medium-dim',
+  description: 'Optimized for medium-dimensional vectors (256-512D)',
+  M: 24,
+  efConstruction: 200,
+  efSearch: 150,
+  expectedRecall: 0.97,
+  targetDimensions: '256-512',
+  targetDatasetSize: '1K-100K',
+};
+
+/**
+ * Preset for high-dimensional vectors (768D+)
+ * Suitable for: BERT, GPT embeddings, Cohere, OpenAI embeddings
+ * This is the recommended preset for RAG applications
+ *
+ * Benchmarked on Cohere Wikipedia 1024D (495K vectors):
+ * - efSearch=128: 99.2% recall, 168 QPS, 10.72ms P99
+ */
+export const PRESET_HIGH_DIM: HNSWPreset = {
+  name: 'high-dim',
+  description: 'Optimized for high-dimensional vectors (768D+)',
+  M: 32,
+  efConstruction: 200,
+  efSearch: 128,
+  expectedRecall: 0.99,
+  targetDimensions: '>=768',
+  targetDatasetSize: '1K-500K',
+};
+
+/**
+ * Preset for very high-dimensional vectors (1536D+)
+ * Suitable for: OpenAI text-embedding-ada-002, text-embedding-3-large
+ *
+ * Scaled from PRESET_HIGH_DIM benchmarks (higher M for higher dimensions)
+ */
+export const PRESET_VERY_HIGH_DIM: HNSWPreset = {
+  name: 'very-high-dim',
+  description: 'Optimized for very high-dimensional vectors (1536D+)',
+  M: 48,
+  efConstruction: 300,
+  efSearch: 150,
+  expectedRecall: 0.99,
+  targetDimensions: '>=1536',
+  targetDatasetSize: '1K-500K',
+};
+
+/**
+ * Preset for small datasets (<10K vectors)
+ * Prioritizes recall over speed since brute-force is viable
+ */
+export const PRESET_SMALL_DATASET: HNSWPreset = {
+  name: 'small-dataset',
+  description: 'Optimized for small datasets (<10K vectors)',
+  M: 16,
+  efConstruction: 200,
+  efSearch: 200,
+  expectedRecall: 0.99,
+  targetDimensions: 'any',
+  targetDatasetSize: '<10K',
+};
+
+/**
+ * Preset for large datasets (100K-1M vectors)
+ * Balances recall with build time and memory
+ *
+ * Benchmarked on Cohere Wikipedia 1024D (495K vectors):
+ * - efSearch=128: 99.2% recall, 168 QPS
+ */
+export const PRESET_LARGE_DATASET: HNSWPreset = {
+  name: 'large-dataset',
+  description: 'Optimized for large datasets (100K-1M vectors)',
+  M: 32,
+  efConstruction: 200,
+  efSearch: 128,
+  expectedRecall: 0.99,
+  targetDimensions: 'any',
+  targetDatasetSize: '100K-1M',
+};
+
+/**
+ * Preset for maximum recall (prioritizes accuracy over speed)
+ * Use when recall is critical and latency is acceptable
+ */
+export const PRESET_MAX_RECALL: HNSWPreset = {
+  name: 'max-recall',
+  description: 'Maximum recall configuration',
+  M: 48,
+  efConstruction: 500,
+  efSearch: 400,
+  expectedRecall: 0.99,
+  targetDimensions: 'any',
+  targetDatasetSize: 'any',
+};
+
+/**
+ * Preset for minimum latency (prioritizes speed over recall)
+ * Use when latency is critical and 90% recall is acceptable
+ */
+export const PRESET_LOW_LATENCY: HNSWPreset = {
+  name: 'low-latency',
+  description: 'Minimum latency configuration (90% recall)',
+  M: 12,
+  efConstruction: 100,
+  efSearch: 50,
+  expectedRecall: 0.90,
+  targetDimensions: 'any',
+  targetDatasetSize: 'any',
+};
+
+/**
+ * All available presets
+ */
+export const PRESETS: Record<string, HNSWPreset> = {
+  'low-dim': PRESET_LOW_DIM,
+  'medium-dim': PRESET_MEDIUM_DIM,
+  'high-dim': PRESET_HIGH_DIM,
+  'very-high-dim': PRESET_VERY_HIGH_DIM,
+  'small-dataset': PRESET_SMALL_DATASET,
+  'large-dataset': PRESET_LARGE_DATASET,
+  'max-recall': PRESET_MAX_RECALL,
+  'low-latency': PRESET_LOW_LATENCY,
+};
+
+/**
+ * Get recommended preset based on dimension and dataset size
+ *
+ * For high-dimensional vectors (768D+), dimension takes priority over dataset size
+ * because recall degrades significantly without higher M values.
+ */
+export function getRecommendedPreset(dimension: number, datasetSize?: number): HNSWPreset {
+  // For high-dimensional vectors, always use dimension-based presets
+  // (dimension matters more than dataset size for recall)
+  // Note: Check 1536 BEFORE 768 since 1536 >= 768 would match first
+  if (dimension >= 1536) return PRESET_VERY_HIGH_DIM;
+  if (dimension >= 768) return PRESET_HIGH_DIM;
+
+  // For lower dimensions, consider dataset size
+  if (datasetSize !== undefined) {
+    if (datasetSize < 10000) return PRESET_SMALL_DATASET;
+    if (datasetSize > 100000) return PRESET_LARGE_DATASET;
+  }
+
+  // Dimension-based selection for medium dimensions
+  if (dimension <= 128) return PRESET_LOW_DIM;
+  if (dimension <= 512) return PRESET_MEDIUM_DIM;
+
+  return PRESET_HIGH_DIM;
+}
+
+/**
+ * Get preset by name
+ */
+export function getPreset(name: string): HNSWPreset | undefined {
+  return PRESETS[name];
+}
+
+/**
+ * RAG-specific preset recommendation
+ * For typical RAG applications using popular embedding models
+ */
+export function getRAGPreset(embeddingModel: string): HNSWPreset {
+  const model = embeddingModel.toLowerCase();
+
+  // OpenAI models
+  if (model.includes('ada-002') || model.includes('text-embedding-3')) {
+    return PRESET_VERY_HIGH_DIM;
+  }
+
+  // Cohere models
+  if (model.includes('cohere') || model.includes('embed-')) {
+    return PRESET_HIGH_DIM;
+  }
+
+  // BERT/Sentence Transformers
+  if (model.includes('bert') || model.includes('minilm') || model.includes('mpnet')) {
+    return PRESET_HIGH_DIM;
+  }
+
+  // E5 models
+  if (model.includes('e5-')) {
+    return model.includes('large') ? PRESET_HIGH_DIM : PRESET_MEDIUM_DIM;
+  }
+
+  // Default to high-dim for unknown models (most modern embeddings are 768D+)
+  return PRESET_HIGH_DIM;
+}

package/src/quantization/ScalarQuantizer.ts

@@ -0,0 +1,383 @@
+/**
+ * Scalar Quantizer for Int8 vector compression
+ *
+ * Provides 4x memory reduction by converting Float32 vectors to Int8.
+ * Expected performance:
+ * - Memory: 4x reduction (32 bits → 8 bits per dimension)
+ * - Speed: ~3.5x faster distance calculations (smaller data, better cache)
+ * - Recall: <2% loss compared to float32
+ *
+ * Uses range quantization: maps [min, max] to [-128, 127]
+ */
+
+export interface QuantizationParams {
+  min: Float32Array;  // Per-dimension minimum
+  max: Float32Array;  // Per-dimension maximum
+  scale: Float32Array; // Per-dimension scale factor
+  offset: Float32Array; // Per-dimension offset
+}
+
+export class ScalarQuantizer {
+  private dimension: number;
+  private params: QuantizationParams | null = null;
+  private trained: boolean = false;
+
+  constructor(dimension: number) {
+    this.dimension = dimension;
+  }
+
+  /**
+   * Train the quantizer on a set of vectors to determine optimal range
+   */
+  train(vectors: Float32Array[]): void {
+    if (vectors.length === 0) {
+      throw new Error('Cannot train quantizer with empty vector set');
+    }
+
+    const dim = this.dimension;
+    const min = new Float32Array(dim).fill(Infinity);
+    const max = new Float32Array(dim).fill(-Infinity);
+
+    // Find per-dimension min/max
+    for (const vector of vectors) {
+      for (let d = 0; d < dim; d++) {
+        if (vector[d] < min[d]) min[d] = vector[d];
+        if (vector[d] > max[d]) max[d] = vector[d];
+      }
+    }
+
+    // Compute scale and offset for mapping to [-128, 127]
+    const scale = new Float32Array(dim);
+    const offset = new Float32Array(dim);
+
+    for (let d = 0; d < dim; d++) {
+      const range = max[d] - min[d];
+      if (range === 0) {
+        // Handle constant dimensions
+        scale[d] = 1;
+        offset[d] = min[d];
+      } else {
+        // Map [min, max] to [-128, 127]
+        scale[d] = 255 / range;
+        offset[d] = min[d];
+      }
+    }
+
+    this.params = { min, max, scale, offset };
+    this.trained = true;
+  }
+
+  /**
+   * Get training status
+   */
+  isTrained(): boolean {
+    return this.trained;
+  }
+
+  /**
+   * Get quantization parameters
+   */
+  getParams(): QuantizationParams | null {
+    return this.params;
+  }
+
+  /**
+   * Set quantization parameters (for loading saved quantizer)
+   */
+  setParams(params: QuantizationParams): void {
+    this.params = params;
+    this.trained = true;
+  }
+
+  /**
+   * Quantize a single float32 vector to int8
+   */
+  quantize(vector: Float32Array): Int8Array {
+    if (!this.params) {
+      throw new Error('Quantizer not trained');
+    }
+
+    const dim = this.dimension;
+    const result = new Int8Array(dim);
+    const { scale, offset } = this.params;
+
+    for (let d = 0; d < dim; d++) {
+      // Map to [0, 255] then shift to [-128, 127]
+      const normalized = (vector[d] - offset[d]) * scale[d];
+      result[d] = Math.max(-128, Math.min(127, Math.round(normalized - 128)));
+    }
+
+    return result;
+  }
+
+  /**
+   * Quantize multiple vectors
+   */
+  quantizeBatch(vectors: Float32Array[]): Int8Array[] {
+    const result = new Array<Int8Array>(vectors.length);
+    for (let i = 0; i < vectors.length; i++) {
+      result[i] = this.quantize(vectors[i]);
+    }
+    return result;
+  }
+
+  /**
+   * Dequantize an int8 vector back to float32 (for rescoring)
+   */
+  dequantize(vector: Int8Array): Float32Array {
+    if (!this.params) {
+      throw new Error('Quantizer not trained');
+    }
+
+    const dim = this.dimension;
+    const result = new Float32Array(dim);
+    const { scale, offset } = this.params;
+
+    for (let d = 0; d < dim; d++) {
+      // Reverse the quantization
+      result[d] = ((vector[d] + 128) / scale[d]) + offset[d];
+    }
+
+    return result;
+  }
+
+  /**
+   * Serialize quantization parameters for saving
+   */
+  serialize(): ArrayBuffer {
+    if (!this.params) {
+      throw new Error('Quantizer not trained');
+    }
+
+    const dim = this.dimension;
+    // 4 bytes header (dimension) + 4 * dim * 4 bytes (4 float arrays)
+    const buffer = new ArrayBuffer(4 + 4 * dim * 4);
+    const view = new DataView(buffer);
+
+    view.setInt32(0, dim, true);
+
+    let offset = 4;
+    for (const arr of [this.params.min, this.params.max, this.params.scale, this.params.offset]) {
+      for (let d = 0; d < dim; d++) {
+        view.setFloat32(offset, arr[d], true);
+        offset += 4;
+      }
+    }
+
+    return buffer;
+  }
+
+  /**
+   * Load quantization parameters
+   */
+  static deserialize(buffer: ArrayBuffer): ScalarQuantizer {
+    const view = new DataView(buffer);
+    const dim = view.getInt32(0, true);
+
+    const quantizer = new ScalarQuantizer(dim);
+    const min = new Float32Array(dim);
+    const max = new Float32Array(dim);
+    const scale = new Float32Array(dim);
+    const offsetArr = new Float32Array(dim);
+
+    let offset = 4;
+    for (const arr of [min, max, scale, offsetArr]) {
+      for (let d = 0; d < dim; d++) {
+        arr[d] = view.getFloat32(offset, true);
+        offset += 4;
+      }
+    }
+
+    quantizer.setParams({ min, max, scale, offset: offsetArr });
+    return quantizer;
+  }
+}
+
+/**
+ * Fast Int8 distance calculations
+ * These are optimized for quantized vectors and provide significant speedup
+ */
+
+/**
+ * Compute dot product between two Int8 vectors
+ * Uses 8-wide unrolling for better ILP (instruction-level parallelism)
+ */
+export function dotProductInt8(a: Int8Array, b: Int8Array): number {
+  const len = a.length;
+  let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
+  let sum4 = 0, sum5 = 0, sum6 = 0, sum7 = 0;
+  let i = 0;
+
+  // 8-wide unrolling for high-dimensional vectors
+  const limit8 = len - 7;
+  for (; i < limit8; i += 8) {
+    sum0 += a[i] * b[i];
+    sum1 += a[i + 1] * b[i + 1];
+    sum2 += a[i + 2] * b[i + 2];
+    sum3 += a[i + 3] * b[i + 3];
+    sum4 += a[i + 4] * b[i + 4];
+    sum5 += a[i + 5] * b[i + 5];
+    sum6 += a[i + 6] * b[i + 6];
+    sum7 += a[i + 7] * b[i + 7];
+  }
+
+  // Handle remaining elements
+  for (; i < len; i++) {
+    sum0 += a[i] * b[i];
+  }
+
+  return sum0 + sum1 + sum2 + sum3 + sum4 + sum5 + sum6 + sum7;
+}
+
+/**
+ * Compute L2 squared distance between two Int8 vectors
+ * Uses 8-wide unrolling for better ILP
+ */
+export function l2SquaredInt8(a: Int8Array, b: Int8Array): number {
+  const len = a.length;
+  let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
+  let sum4 = 0, sum5 = 0, sum6 = 0, sum7 = 0;
+  let i = 0;
+
+  // 8-wide unrolling for high-dimensional vectors
+  const limit8 = len - 7;
+  for (; i < limit8; i += 8) {
+    const d0 = a[i] - b[i];
+    const d1 = a[i + 1] - b[i + 1];
+    const d2 = a[i + 2] - b[i + 2];
+    const d3 = a[i + 3] - b[i + 3];
+    const d4 = a[i + 4] - b[i + 4];
+    const d5 = a[i + 5] - b[i + 5];
+    const d6 = a[i + 6] - b[i + 6];
+    const d7 = a[i + 7] - b[i + 7];
+    sum0 += d0 * d0;
+    sum1 += d1 * d1;
+    sum2 += d2 * d2;
+    sum3 += d3 * d3;
+    sum4 += d4 * d4;
+    sum5 += d5 * d5;
+    sum6 += d6 * d6;
+    sum7 += d7 * d7;
+  }
+
+  // Handle remaining elements
+  for (; i < len; i++) {
+    const d = a[i] - b[i];
+    sum0 += d * d;
+  }
+
+  return sum0 + sum1 + sum2 + sum3 + sum4 + sum5 + sum6 + sum7;
+}
+
+/**
+ * Compute approximate cosine distance for Int8 vectors
+ * Note: This is approximate because quantization changes magnitude
+ * Uses 8-wide unrolling with separate accumulators for better ILP
+ */
+export function cosineDistanceInt8(a: Int8Array, b: Int8Array): number {
+  const len = a.length;
+  // Use separate accumulators for better ILP
+  let dot0 = 0, dot1 = 0, dot2 = 0, dot3 = 0;
+  let normA0 = 0, normA1 = 0, normA2 = 0, normA3 = 0;
+  let normB0 = 0, normB1 = 0, normB2 = 0, normB3 = 0;
+
+  let i = 0;
+  const limit8 = len - 7;
+  for (; i < limit8; i += 8) {
+    dot0 += a[i] * b[i] + a[i + 4] * b[i + 4];
+    dot1 += a[i + 1] * b[i + 1] + a[i + 5] * b[i + 5];
+    dot2 += a[i + 2] * b[i + 2] + a[i + 6] * b[i + 6];
+    dot3 += a[i + 3] * b[i + 3] + a[i + 7] * b[i + 7];
+    normA0 += a[i] * a[i] + a[i + 4] * a[i + 4];
+    normA1 += a[i + 1] * a[i + 1] + a[i + 5] * a[i + 5];
+    normA2 += a[i + 2] * a[i + 2] + a[i + 6] * a[i + 6];
+    normA3 += a[i + 3] * a[i + 3] + a[i + 7] * a[i + 7];
+    normB0 += b[i] * b[i] + b[i + 4] * b[i + 4];
+    normB1 += b[i + 1] * b[i + 1] + b[i + 5] * b[i + 5];
+    normB2 += b[i + 2] * b[i + 2] + b[i + 6] * b[i + 6];
+    normB3 += b[i + 3] * b[i + 3] + b[i + 7] * b[i + 7];
+  }
+
+  // Handle remaining elements
+  for (; i < len; i++) {
+    dot0 += a[i] * b[i];
+    normA0 += a[i] * a[i];
+    normB0 += b[i] * b[i];
+  }
+
+  const dot = dot0 + dot1 + dot2 + dot3;
+  const normA = normA0 + normA1 + normA2 + normA3;
+  const normB = normB0 + normB1 + normB2 + normB3;
+
+  const magnitude = Math.sqrt(normA * normB);
+  if (magnitude === 0) return 1;
+
+  const distance = 1 - (dot / magnitude);
+  return distance < 1e-10 ? 0 : distance;
+}
+
+/**
+ * QuantizedVectorStore - Efficient storage for quantized vectors
+ */
+export class QuantizedVectorStore {
+  private quantizer: ScalarQuantizer;
+  private vectors: Int8Array[];  // Quantized vectors
+  private originalVectors: Float32Array[] | null;  // Keep originals for rescoring
+  private keepOriginals: boolean;
+
+  constructor(dimension: number, keepOriginals = true) {
+    this.quantizer = new ScalarQuantizer(dimension);
+    this.vectors = [];
+    this.originalVectors = keepOriginals ? [] : null;
+    this.keepOriginals = keepOriginals;
+  }
+
+  /**
+   * Train the quantizer and add vectors
+   */
+  addVectors(vectors: Float32Array[]): void {
+    if (!this.quantizer.isTrained()) {
+      this.quantizer.train(vectors);
+    }
+
+    for (const v of vectors) {
+      this.vectors.push(this.quantizer.quantize(v));
+      if (this.keepOriginals && this.originalVectors) {
+        this.originalVectors.push(v);
+      }
+    }
+  }
+
+  /**
+   * Get quantized vector by index
+   */
+  getQuantized(index: number): Int8Array {
+    return this.vectors[index];
+  }
+
+  /**
+   * Get original float32 vector by index (for rescoring)
+   */
+  getOriginal(index: number): Float32Array | null {
+    if (!this.originalVectors) return null;
+    return this.originalVectors[index];
+  }
+
+  /**
+   * Get number of vectors
+   */
+  size(): number {
+    return this.vectors.length;
+  }
+
+  /**
+   * Calculate memory usage in bytes
+   */
+  memoryUsage(): { quantized: number; original: number; total: number } {
+    const quantized = this.vectors.reduce((sum, v) => sum + v.length, 0);
+    const original = this.originalVectors
+      ? this.originalVectors.reduce((sum, v) => sum + v.length * 4, 0)
+      : 0;
+    return { quantized, original, total: quantized + original };
+  }
+}