verso-db 0.1.5 → 0.2.0

package/src/encoding/DeltaEncoder.ts

@@ -1,236 +0,0 @@
-/**
- * Delta Encoding with Varint for Neighbor Lists
- *
- * Implements delta-encoded neighbor lists as used by Qdrant for ~38% storage reduction.
- * Neighbor IDs are sorted, then stored as deltas with variable-length encoding.
- *
- * Format:
- * - First ID stored as full uint32
- * - Subsequent IDs stored as varint deltas from previous ID
- *
- * Varint encoding (like Protocol Buffers):
- * - Values 0-127: 1 byte
- * - Values 128-16383: 2 bytes
- * - Values 16384-2097151: 3 bytes
- * - Values 2097152-268435455: 4 bytes
- * - Larger: 5 bytes
- */
-
-/**
- * Encode an unsigned integer as a varint
- * Returns the number of bytes written
- */
-export function encodeVarint(value: number, buffer: Uint8Array, offset: number): number {
-  let v = value >>> 0; // Ensure unsigned
-  let bytesWritten = 0;
-
-  while (v >= 0x80) {
-    buffer[offset + bytesWritten] = (v & 0x7f) | 0x80;
-    v >>>= 7;
-    bytesWritten++;
-  }
-  buffer[offset + bytesWritten] = v;
-  return bytesWritten + 1;
-}
-
-/**
- * Decode a varint from buffer
- * Returns [value, bytesRead]
- */
-export function decodeVarint(buffer: Uint8Array, offset: number): [number, number] {
-  let result = 0;
-  let shift = 0;
-  let bytesRead = 0;
-
-  while (offset + bytesRead < buffer.length) {
-    const byte = buffer[offset + bytesRead];
-    if (shift < 28) {
-      result |= (byte & 0x7f) << shift;
-    } else {
-      // At shift >= 28, use multiplication to avoid sign-bit corruption from << on int32
-      result = (result + (byte & 0x7f) * (2 ** shift)) >>> 0;
-    }
-    bytesRead++;
-
-    if ((byte & 0x80) === 0) {
-      return [result >>> 0, bytesRead];
-    }
-
-    shift += 7;
-    if (shift > 35) {
-      throw new Error('Varint too long');
-    }
-  }
-
-  throw new Error('Unexpected end of buffer');
-}
-
-/**
- * Calculate the number of bytes needed to encode a varint
- */
-export function varintSize(value: number): number {
-  let v = value >>> 0;
-  let size = 1;
-  while (v >= 0x80) {
-    v >>>= 7;
-    size++;
-  }
-  return size;
-}
-
-/**
- * Delta-encode a sorted array of neighbor IDs
- * Returns the encoded buffer
- */
-export function deltaEncodeNeighbors(neighbors: number[]): Uint8Array {
-  if (neighbors.length === 0) {
-    return new Uint8Array(0);
-  }
-
-  for (let i = 0; i < neighbors.length; i++) {
-    const id = neighbors[i];
-    if (!Number.isInteger(id) || id < 0 || id > 0xFFFFFFFF) {
-      throw new Error(`Invalid neighbor ID at index ${i}: ${id}`);
-    }
-  }
-
-  // Sort neighbors for optimal delta encoding
-  const sorted = neighbors.slice().sort((a, b) => a - b);
-
-  // Calculate required buffer size
-  let size = 4; // First ID as uint32
-  let prev = sorted[0];
-  for (let i = 1; i < sorted.length; i++) {
-    const delta = sorted[i] - prev;
-    size += varintSize(delta);
-    prev = sorted[i];
-  }
-
-  // Encode
-  const buffer = new Uint8Array(size);
-  const view = new DataView(buffer.buffer);
-
-  // First ID as full uint32 (little-endian)
-  view.setUint32(0, sorted[0], true);
-  let offset = 4;
-
-  // Remaining as deltas
-  prev = sorted[0];
-  for (let i = 1; i < sorted.length; i++) {
-    const delta = sorted[i] - prev;
-    offset += encodeVarint(delta, buffer, offset);
-    prev = sorted[i];
-  }
-
-  return buffer;
-}
-
-/**
- * Decode a delta-encoded neighbor list
- * Returns the original neighbor IDs (sorted)
- */
-export function deltaDecodeNeighbors(buffer: Uint8Array, count: number): number[] {
-  if (count === 0 || buffer.length === 0) {
-    return [];
-  }
-
-  const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
-  const neighbors = new Array<number>(count);
-
-  // First ID as full uint32
-  neighbors[0] = view.getUint32(0, true);
-  let offset = 4;
-
-  // Remaining as deltas
-  for (let i = 1; i < count; i++) {
-    const [delta, bytesRead] = decodeVarint(buffer, offset);
-    neighbors[i] = neighbors[i - 1] + delta;
-    offset += bytesRead;
-  }
-
-  return neighbors;
-}
-
-/**
- * Calculate the encoded size for a neighbor list without actually encoding
- * Useful for calculating total buffer size before serialization
- */
-export function deltaEncodedSize(neighbors: number[]): number {
-  if (neighbors.length === 0) {
-    return 0;
-  }
-
-  for (let i = 0; i < neighbors.length; i++) {
-    const id = neighbors[i];
-    if (!Number.isInteger(id) || id < 0 || id > 0xFFFFFFFF) {
-      throw new Error(`Invalid neighbor ID at index ${i}: ${id}`);
-    }
-  }
-
-  const sorted = neighbors.slice().sort((a, b) => a - b);
-
-  let size = 4; // First ID as uint32
-  let prev = sorted[0];
-  for (let i = 1; i < sorted.length; i++) {
-    const delta = sorted[i] - prev;
-    size += varintSize(delta);
-    prev = sorted[i];
-  }
-
-  return size;
-}
-
-/**
- * Batch encode multiple neighbor lists efficiently
- * Returns a single buffer with all encoded lists concatenated
- * Also returns offsets for each list
- */
-export function deltaEncodeBatch(neighborLists: number[][]): {
-  buffer: Uint8Array;
-  offsets: number[];
-  sizes: number[];
-} {
-  // Calculate total size and individual sizes
-  const sizes = neighborLists.map(list => deltaEncodedSize(list));
-  const totalSize = sizes.reduce((a, b) => a + b, 0);
-
-  // Allocate single buffer
-  const buffer = new Uint8Array(totalSize);
-  const offsets: number[] = [];
-
-  let currentOffset = 0;
-  for (let i = 0; i < neighborLists.length; i++) {
-    offsets.push(currentOffset);
-
-    if (neighborLists[i].length > 0) {
-      const encoded = deltaEncodeNeighbors(neighborLists[i]);
-      buffer.set(encoded, currentOffset);
-      currentOffset += encoded.length;
-    }
-  }
-
-  return { buffer, offsets, sizes };
-}
-
-/**
- * Decode a batch of neighbor lists from a single buffer
- */
-export function deltaDecodeBatch(
-  buffer: Uint8Array,
-  offsets: number[],
-  sizes: number[],
-  counts: number[]
-): number[][] {
-  const results: number[][] = [];
-
-  for (let i = 0; i < offsets.length; i++) {
-    if (counts[i] === 0 || sizes[i] === 0) {
-      results.push([]);
-    } else {
-      const slice = buffer.subarray(offsets[i], offsets[i] + sizes[i]);
-      results.push(deltaDecodeNeighbors(slice, counts[i]));
-    }
-  }
-
-  return results;
-}

package/src/errors.ts

@@ -1,110 +0,0 @@
-/**
- * Base error class for all VectorDB errors
- */
-export class VectorDBError extends Error {
-  readonly code: string;
-
-  constructor(message: string, code: string) {
-    super(message);
-    this.name = 'VectorDBError';
-    this.code = code;
-  }
-}
-
-/**
- * Thrown when vector dimensions don't match expected dimensions
- */
-export class DimensionMismatchError extends VectorDBError {
-  readonly expected: number;
-  readonly actual: number;
-
-  constructor(expected: number, actual: number, context?: string) {
-    const message = context
-      ? `${context}: expected dimension ${expected}, got ${actual}`
-      : `Dimension mismatch: expected ${expected}, got ${actual}`;
-    super(message, 'DIMENSION_MISMATCH');
-    this.name = 'DimensionMismatchError';
-    this.expected = expected;
-    this.actual = actual;
-  }
-}
-
-/**
- * Thrown when attempting to add a vector with an ID that already exists
- */
-export class DuplicateVectorError extends VectorDBError {
-  readonly ids: string[];
-
-  constructor(ids: string[]) {
-    const message = ids.length === 1
-      ? `Vector with ID '${ids[0]}' already exists`
-      : `Vectors with IDs already exist: ${ids.join(', ')}`;
-    super(message, 'DUPLICATE_VECTOR');
-    this.name = 'DuplicateVectorError';
-    this.ids = ids;
-  }
-}
-
-/**
- * Thrown when a requested collection does not exist
- */
-export class CollectionNotFoundError extends VectorDBError {
-  readonly collectionName: string;
-
-  constructor(collectionName: string) {
-    super(`Collection '${collectionName}' does not exist`, 'COLLECTION_NOT_FOUND');
-    this.name = 'CollectionNotFoundError';
-    this.collectionName = collectionName;
-  }
-}
-
-/**
- * Thrown when attempting to create a collection that already exists
- */
-export class CollectionExistsError extends VectorDBError {
-  readonly collectionName: string;
-
-  constructor(collectionName: string) {
-    super(`Collection '${collectionName}' already exists`, 'COLLECTION_EXISTS');
-    this.name = 'CollectionExistsError';
-    this.collectionName = collectionName;
-  }
-}
-
-/**
- * Thrown when a storage operation fails
- */
-export class StorageError extends VectorDBError {
-  readonly operation: string;
-  readonly path?: string;
-
-  constructor(operation: string, message: string, path?: string) {
-    super(`Storage ${operation} failed: ${message}`, 'STORAGE_ERROR');
-    this.name = 'StorageError';
-    this.operation = operation;
-    this.path = path;
-  }
-}
-
-/**
- * Thrown when quantization operations fail
- */
-export class QuantizationError extends VectorDBError {
-  constructor(message: string) {
-    super(message, 'QUANTIZATION_ERROR');
-    this.name = 'QuantizationError';
-  }
-}
-
-/**
- * Thrown when a vector is not found in the index
- */
-export class VectorNotFoundError extends VectorDBError {
-  readonly vectorId: string | number;
-
-  constructor(vectorId: string | number) {
-    super(`Vector '${vectorId}' not found`, 'VECTOR_NOT_FOUND');
-    this.name = 'VectorNotFoundError';
-    this.vectorId = vectorId;
-  }
-}

package/src/index.ts

@@ -1,106 +0,0 @@
-/**
- * verso-db - High-performance vector search with HNSW indexing
- *
- * Features:
- * - HNSW algorithm for approximate nearest neighbor search
- * - Multiple distance metrics: cosine, euclidean, dot product
- * - Int8 scalar quantization for 4x memory reduction
- * - Batch query support for improved throughput
- * - Parameter presets for different use cases
- * - Multi-platform: Bun (file system) and Browser (OPFS)
- *
- * @example
- * ```typescript
- * import { VectorDB, getRecommendedPreset } from 'verso-db';
- *
- * const db = new VectorDB({ storagePath: './vectors' });
- * const preset = getRecommendedPreset(768);
- *
- * const collection = await db.createCollection('my-vectors', {
- *   dimension: 768,
- *   metric: 'cosine',
- *   M: preset.M,
- *   efConstruction: preset.efConstruction
- * });
- *
- * await collection.add({
- *   ids: ['doc1', 'doc2'],
- *   vectors: [new Float32Array(768), new Float32Array(768)]
- * });
- *
- * const results = await collection.query({
- *   queryVector: new Float32Array(768),
- *   k: 10,
- *   efSearch: preset.efSearch
- * });
- * ```
- *
- * @packageDocumentation
- * @module verso-db
- */
-
-// ─── Public API ──────────────────────────────────────────────────────────────
-
-// Core
-export { VectorDB } from './VectorDB';
-export type { VectorDBConfig, CollectionConfig } from './VectorDB';
-export { Collection } from './Collection';
-export type { AddConfig, QueryConfig, QueryResult } from './Collection';
-export { HNSWIndex } from './HNSWIndex';
-export type { DistanceMetric } from './HNSWIndex';
-
-// Parameter presets
-export type { HNSWPreset } from './presets';
-export {
-  PRESET_LOW_DIM,
-  PRESET_MEDIUM_DIM,
-  PRESET_HIGH_DIM,
-  PRESET_VERY_HIGH_DIM,
-  PRESET_SMALL_DATASET,
-  PRESET_LARGE_DATASET,
-  PRESET_MAX_RECALL,
-  PRESET_LOW_LATENCY,
-  PRESETS,
-  getRecommendedPreset,
-  getPreset,
-  getRAGPreset
-} from './presets';
-
-// Quantization
-export { ScalarQuantizer, QuantizedVectorStore } from './quantization/ScalarQuantizer';
-export type { QuantizationParams } from './quantization/ScalarQuantizer';
-
-// Storage backends
-export type { StorageBackend, StorageOptions } from './storage/StorageBackend';
-export { BunStorageBackend } from './storage/BunStorageBackend';
-export { MemoryBackend } from './storage/MemoryBackend';
-export { OPFSBackend } from './storage/OPFSBackend';
-export {
-  createStorageBackend,
-  getRecommendedStorageType,
-  isStorageTypeAvailable,
-  type StorageType,
-  type CreateStorageOptions,
-} from './storage/createStorageBackend';
-
-// Write-ahead log and batch writer
-export { WriteAheadLog, WALOperationType } from './storage/WriteAheadLog';
-export type { WALEntry } from './storage/WriteAheadLog';
-export { BatchWriter, createBatchWriter } from './storage/BatchWriter';
-export type { BatchWriterOptions } from './storage/BatchWriter';
-
-// Parallel query processing
-export { WorkerPool } from './WorkerPool';
-export { WorkerSearchState } from './SearchWorker';
-
-// Error classes
-export {
-  VectorDBError,
-  DimensionMismatchError,
-  DuplicateVectorError,
-  CollectionNotFoundError,
-  CollectionExistsError,
-  StorageError,
-  QuantizationError,
-  VectorNotFoundError,
-} from './errors';

package/src/presets.ts

@@ -1,229 +0,0 @@
-/**
- * HNSW Parameter Presets
- *
- * These presets are optimized based on extensive benchmarking to achieve
- * the target recall@10 >= 95% for different dataset sizes and dimensions.
- *
- * Key parameters:
- * - M: Maximum connections per node (higher = better recall, more memory)
- * - efConstruction: Beam width during index building (higher = better quality, slower build)
- * - efSearch: Beam width during search (higher = better recall, slower search)
- */
-
-export interface HNSWPreset {
-  name: string;
-  description: string;
-  M: number;
-  efConstruction: number;
-  efSearch: number;
-  expectedRecall: number;
-  targetDimensions: string;
-  targetDatasetSize: string;
-}
-
-/**
- * Preset for low-dimensional vectors (128D or less)
- * Suitable for: Image features, word2vec, GloVe embeddings
- */
-export const PRESET_LOW_DIM: Readonly<HNSWPreset> = Object.freeze({
-  name: 'low-dim',
-  description: 'Optimized for low-dimensional vectors (<=128D)',
-  M: 16,
-  efConstruction: 200,
-  efSearch: 100,
-  expectedRecall: 0.99,
-  targetDimensions: '<=128',
-  targetDatasetSize: '1K-100K',
-});
-
-/**
- * Preset for medium-dimensional vectors (256-512D)
- * Suitable for: Sentence embeddings, smaller transformer outputs
- */
-export const PRESET_MEDIUM_DIM: Readonly<HNSWPreset> = Object.freeze({
-  name: 'medium-dim',
-  description: 'Optimized for medium-dimensional vectors (129-512D)',
-  M: 24,
-  efConstruction: 200,
-  efSearch: 150,
-  expectedRecall: 0.97,
-  targetDimensions: '129-512',
-  targetDatasetSize: '1K-100K',
-});
-
-/**
- * Preset for high-dimensional vectors (768D+)
- * Suitable for: BERT, GPT embeddings, Cohere, OpenAI embeddings
- * This is the recommended preset for RAG applications
- *
- * Benchmarked on Cohere Wikipedia 1024D (495K vectors):
- * - efSearch=128: 99.2% recall, 168 QPS, 10.72ms P99
- */
-export const PRESET_HIGH_DIM: Readonly<HNSWPreset> = Object.freeze({
-  name: 'high-dim',
-  description: 'Optimized for high-dimensional vectors (768D+)',
-  M: 32,
-  efConstruction: 200,
-  efSearch: 128,
-  expectedRecall: 0.99,
-  targetDimensions: '>=768',
-  targetDatasetSize: '1K-500K',
-});
-
-/**
- * Preset for very high-dimensional vectors (1536D+)
- * Suitable for: OpenAI text-embedding-ada-002, text-embedding-3-large
- *
- * Scaled from PRESET_HIGH_DIM benchmarks (higher M for higher dimensions)
- */
-export const PRESET_VERY_HIGH_DIM: Readonly<HNSWPreset> = Object.freeze({
-  name: 'very-high-dim',
-  description: 'Optimized for very high-dimensional vectors (1536D+)',
-  M: 48,
-  efConstruction: 300,
-  efSearch: 150,
-  expectedRecall: 0.99,
-  targetDimensions: '>=1536',
-  targetDatasetSize: '1K-500K',
-});
-
-/**
- * Preset for small datasets (<10K vectors)
- * Prioritizes recall over speed since brute-force is viable
- */
-export const PRESET_SMALL_DATASET: Readonly<HNSWPreset> = Object.freeze({
-  name: 'small-dataset',
-  description: 'Optimized for small datasets (<10K vectors)',
-  M: 16,
-  efConstruction: 200,
-  efSearch: 200,
-  expectedRecall: 0.99,
-  targetDimensions: 'any',
-  targetDatasetSize: '<10K',
-});
-
-/**
- * Preset for large datasets (100K-1M vectors)
- * Balances recall with build time and memory
- *
- * Benchmarked on Cohere Wikipedia 1024D (495K vectors):
- * - efSearch=128: 99.2% recall, 168 QPS
- */
-export const PRESET_LARGE_DATASET: Readonly<HNSWPreset> = Object.freeze({
-  name: 'large-dataset',
-  description: 'Optimized for large datasets (100K-1M vectors)',
-  M: 32,
-  efConstruction: 200,
-  efSearch: 128,
-  expectedRecall: 0.99,
-  targetDimensions: 'any',
-  targetDatasetSize: '100K-1M',
-});
-
-/**
- * Preset for maximum recall (prioritizes accuracy over speed)
- * Use when recall is critical and latency is acceptable
- */
-export const PRESET_MAX_RECALL: Readonly<HNSWPreset> = Object.freeze({
-  name: 'max-recall',
-  description: 'Maximum recall configuration',
-  M: 48,
-  efConstruction: 500,
-  efSearch: 400,
-  expectedRecall: 0.99,
-  targetDimensions: 'any',
-  targetDatasetSize: 'any',
-});
-
-/**
- * Preset for minimum latency (prioritizes speed over recall)
- * Use when latency is critical and 90% recall is acceptable
- */
-export const PRESET_LOW_LATENCY: Readonly<HNSWPreset> = Object.freeze({
-  name: 'low-latency',
-  description: 'Minimum latency configuration (90% recall)',
-  M: 12,
-  efConstruction: 100,
-  efSearch: 50,
-  expectedRecall: 0.90,
-  targetDimensions: 'any',
-  targetDatasetSize: 'any',
-});
-
-/**
- * All available presets
- */
-export const PRESETS: Readonly<Record<string, Readonly<HNSWPreset>>> = Object.freeze({
-  'low-dim': PRESET_LOW_DIM,
-  'medium-dim': PRESET_MEDIUM_DIM,
-  'high-dim': PRESET_HIGH_DIM,
-  'very-high-dim': PRESET_VERY_HIGH_DIM,
-  'small-dataset': PRESET_SMALL_DATASET,
-  'large-dataset': PRESET_LARGE_DATASET,
-  'max-recall': PRESET_MAX_RECALL,
-  'low-latency': PRESET_LOW_LATENCY,
-});
-
-/**
- * Get recommended preset based on dimension and dataset size
- *
- * For high-dimensional vectors (768D+), dimension takes priority over dataset size
- * because recall degrades significantly without higher M values.
- */
-export function getRecommendedPreset(dimension: number, datasetSize?: number): HNSWPreset {
-  // For high-dimensional vectors, always use dimension-based presets
-  // (dimension matters more than dataset size for recall)
-  // Note: Check 1536 BEFORE 768 since 1536 >= 768 would match first
-  if (dimension >= 1536) return PRESET_VERY_HIGH_DIM;
-  if (dimension >= 768) return PRESET_HIGH_DIM;
-
-  // For lower dimensions, consider dataset size
-  if (datasetSize !== undefined) {
-    if (datasetSize < 10000) return PRESET_SMALL_DATASET;
-    if (datasetSize > 100000) return PRESET_LARGE_DATASET;
-  }
-
-  // Dimension-based selection for medium dimensions
-  if (dimension <= 128) return PRESET_LOW_DIM;
-  if (dimension <= 512) return PRESET_MEDIUM_DIM;
-
-  return PRESET_HIGH_DIM;
-}
-
-/**
- * Get preset by name
- */
-export function getPreset(name: string): HNSWPreset | undefined {
-  return PRESETS[name];
-}
-
-/**
- * RAG-specific preset recommendation
- * For typical RAG applications using popular embedding models
- */
-export function getRAGPreset(embeddingModel: string): HNSWPreset {
-  const model = embeddingModel.toLowerCase();
-
-  // OpenAI models
-  if (model.includes('ada-002') || model.includes('text-embedding-3')) {
-    return PRESET_VERY_HIGH_DIM;
-  }
-
-  // Cohere models
-  if (model.includes('cohere') || model.includes('embed-')) {
-    return PRESET_HIGH_DIM;
-  }
-
-  // BERT/Sentence Transformers
-  if (model.includes('bert') || model.includes('minilm') || model.includes('mpnet')) {
-    return PRESET_HIGH_DIM;
-  }
-
-  // E5 models
-  if (model.includes('e5-')) {
-    return model.includes('large') ? PRESET_HIGH_DIM : PRESET_MEDIUM_DIM;
-  }
-
-  // Default to high-dim for unknown models (most modern embeddings are 768D+)
-  return PRESET_HIGH_DIM;
-}