verso-db 0.1.1

package/src/backends/DistanceBackend.ts

@@ -0,0 +1,17 @@
+export interface DistanceBackend {
+  batchL2(
+    base: Float32Array,
+    dim: number,
+    ids: Uint32Array,
+    query: Float32Array,
+    out: Float32Array
+  ): void;
+
+  batchDot(
+    base: Float32Array,
+    dim: number,
+    ids: Uint32Array,
+    query: Float32Array,
+    out: Float32Array
+  ): void;
+}

package/src/backends/JsDistanceBackend.ts

@@ -0,0 +1,227 @@
+import { DistanceBackend } from './DistanceBackend';
+
+/**
+ * Optimized JavaScript distance backend with SIMD-style 4-wide unrolling.
+ * This provides ~1.5-2x speedup over naive loops by reducing loop overhead
+ * and enabling better CPU pipelining.
+ */
+export class JsDistanceBackend implements DistanceBackend {
+  batchL2(base: Float32Array, dim: number, ids: Uint32Array, query: Float32Array, out: Float32Array): void {
+    const len = ids.length;
+    for (let i = 0; i < len; i++) {
+      const off = ids[i] * dim;
+      out[i] = this.l2SquaredUnrolled(base, off, query, dim);
+    }
+  }
+
+  batchDot(base: Float32Array, dim: number, ids: Uint32Array, query: Float32Array, out: Float32Array): void {
+    const len = ids.length;
+    for (let i = 0; i < len; i++) {
+      const off = ids[i] * dim;
+      out[i] = this.dotProductUnrolled(base, off, query, dim);
+    }
+  }
+
+  /**
+   * Compute dot product with 4-wide manual unrolling for better performance.
+   * Uses separate accumulators to enable instruction-level parallelism.
+   */
+  private dotProductUnrolled(base: Float32Array, offset: number, query: Float32Array, dim: number): number {
+    let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
+    let d = 0;
+
+    // Process 4 elements at a time
+    const limit = dim - 3;
+    for (; d < limit; d += 4) {
+      sum0 += base[offset + d] * query[d];
+      sum1 += base[offset + d + 1] * query[d + 1];
+      sum2 += base[offset + d + 2] * query[d + 2];
+      sum3 += base[offset + d + 3] * query[d + 3];
+    }
+
+    // Handle remaining elements
+    for (; d < dim; d++) {
+      sum0 += base[offset + d] * query[d];
+    }
+
+    return sum0 + sum1 + sum2 + sum3;
+  }
+
+  /**
+   * Compute L2 squared distance with 4-wide manual unrolling.
+   */
+  private l2SquaredUnrolled(base: Float32Array, offset: number, query: Float32Array, dim: number): number {
+    let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
+    let d = 0;
+
+    // Process 4 elements at a time
+    const limit = dim - 3;
+    for (; d < limit; d += 4) {
+      const diff0 = base[offset + d] - query[d];
+      const diff1 = base[offset + d + 1] - query[d + 1];
+      const diff2 = base[offset + d + 2] - query[d + 2];
+      const diff3 = base[offset + d + 3] - query[d + 3];
+      sum0 += diff0 * diff0;
+      sum1 += diff1 * diff1;
+      sum2 += diff2 * diff2;
+      sum3 += diff3 * diff3;
+    }
+
+    // Handle remaining elements
+    for (; d < dim; d++) {
+      const diff = base[offset + d] - query[d];
+      sum0 += diff * diff;
+    }
+
+    return sum0 + sum1 + sum2 + sum3;
+  }
+}
+
+/**
+ * Fast inline distance functions for single-vector comparisons.
+ * These avoid the batch interface overhead for point-to-point distance calculations.
+ */
+export function dotProductFast(a: Float32Array, b: Float32Array): number {
+  const len = a.length;
+  // Use 8 accumulators for better ILP (instruction-level parallelism)
+  let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
+  let sum4 = 0, sum5 = 0, sum6 = 0, sum7 = 0;
+  let i = 0;
+
+  // 8-wide unrolling for high-dimensional vectors (768D, 1536D)
+  const limit8 = len - 7;
+  for (; i < limit8; i += 8) {
+    sum0 += a[i] * b[i];
+    sum1 += a[i + 1] * b[i + 1];
+    sum2 += a[i + 2] * b[i + 2];
+    sum3 += a[i + 3] * b[i + 3];
+    sum4 += a[i + 4] * b[i + 4];
+    sum5 += a[i + 5] * b[i + 5];
+    sum6 += a[i + 6] * b[i + 6];
+    sum7 += a[i + 7] * b[i + 7];
+  }
+
+  // Handle remaining elements
+  for (; i < len; i++) {
+    sum0 += a[i] * b[i];
+  }
+
+  return sum0 + sum1 + sum2 + sum3 + sum4 + sum5 + sum6 + sum7;
+}
+
+export function l2SquaredFast(a: Float32Array, b: Float32Array): number {
+  const len = a.length;
+  // Use 8 accumulators for better ILP
+  let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
+  let sum4 = 0, sum5 = 0, sum6 = 0, sum7 = 0;
+  let i = 0;
+
+  // 8-wide unrolling for high-dimensional vectors
+  const limit8 = len - 7;
+  for (; i < limit8; i += 8) {
+    const d0 = a[i] - b[i];
+    const d1 = a[i + 1] - b[i + 1];
+    const d2 = a[i + 2] - b[i + 2];
+    const d3 = a[i + 3] - b[i + 3];
+    const d4 = a[i + 4] - b[i + 4];
+    const d5 = a[i + 5] - b[i + 5];
+    const d6 = a[i + 6] - b[i + 6];
+    const d7 = a[i + 7] - b[i + 7];
+    sum0 += d0 * d0;
+    sum1 += d1 * d1;
+    sum2 += d2 * d2;
+    sum3 += d3 * d3;
+    sum4 += d4 * d4;
+    sum5 += d5 * d5;
+    sum6 += d6 * d6;
+    sum7 += d7 * d7;
+  }
+
+  // Handle remaining elements
+  for (; i < len; i++) {
+    const d = a[i] - b[i];
+    sum0 += d * d;
+  }
+
+  return sum0 + sum1 + sum2 + sum3 + sum4 + sum5 + sum6 + sum7;
+}
+
+/**
+ * Normalize a vector in place.
+ * Uses 8-wide unrolling for better ILP.
+ */
+export function normalizeInPlace(v: Float32Array): number {
+  const len = v.length;
+  let s0 = 0, s1 = 0, s2 = 0, s3 = 0;
+  let s4 = 0, s5 = 0, s6 = 0, s7 = 0;
+  let i = 0;
+  const limit8 = len - 7;
+
+  for (; i < limit8; i += 8) {
+    s0 += v[i] * v[i];
+    s1 += v[i + 1] * v[i + 1];
+    s2 += v[i + 2] * v[i + 2];
+    s3 += v[i + 3] * v[i + 3];
+    s4 += v[i + 4] * v[i + 4];
+    s5 += v[i + 5] * v[i + 5];
+    s6 += v[i + 6] * v[i + 6];
+    s7 += v[i + 7] * v[i + 7];
+  }
+  for (; i < len; i++) {
+    s0 += v[i] * v[i];
+  }
+
+  const norm = Math.sqrt(s0 + s1 + s2 + s3 + s4 + s5 + s6 + s7);
+  if (norm > 0) {
+    const invNorm = 1 / norm;
+    for (let j = 0; j < len; j++) {
+      v[j] *= invNorm;
+    }
+  }
+  return norm;
+}
+
+/**
+ * Compute cosine distance between two vectors.
+ * For pre-normalized vectors, this is simply 1 - dot(a, b).
+ * Uses 8-wide unrolling for norm computation.
+ */
+export function cosineDistanceFast(a: Float32Array, b: Float32Array, aIsNormalized = false, bIsNormalized = false): number {
+  const dot = dotProductFast(a, b);
+
+  if (aIsNormalized && bIsNormalized) {
+    // Both vectors are normalized, cosine distance = 1 - dot
+    const dist = 1 - dot;
+    return dist < 1e-10 ? 0 : dist;
+  }
+
+  // Need to compute norms with 8-wide unrolling
+  const len = a.length;
+  let nA0 = 0, nA1 = 0, nA2 = 0, nA3 = 0;
+  let nB0 = 0, nB1 = 0, nB2 = 0, nB3 = 0;
+  let i = 0;
+  const limit8 = len - 7;
+
+  for (; i < limit8; i += 8) {
+    nA0 += a[i] * a[i] + a[i + 4] * a[i + 4];
+    nA1 += a[i + 1] * a[i + 1] + a[i + 5] * a[i + 5];
+    nA2 += a[i + 2] * a[i + 2] + a[i + 6] * a[i + 6];
+    nA3 += a[i + 3] * a[i + 3] + a[i + 7] * a[i + 7];
+    nB0 += b[i] * b[i] + b[i + 4] * b[i + 4];
+    nB1 += b[i + 1] * b[i + 1] + b[i + 5] * b[i + 5];
+    nB2 += b[i + 2] * b[i + 2] + b[i + 6] * b[i + 6];
+    nB3 += b[i + 3] * b[i + 3] + b[i + 7] * b[i + 7];
+  }
+  for (; i < len; i++) {
+    nA0 += a[i] * a[i];
+    nB0 += b[i] * b[i];
+  }
+
+  const normA = nA0 + nA1 + nA2 + nA3;
+  const normB = nB0 + nB1 + nB2 + nB3;
+  const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+  if (magnitude === 0) return 1;
+
+  const dist = 1 - (dot / magnitude);
+  return dist < 1e-10 ? 0 : dist;
+}

package/src/encoding/DeltaEncoder.ts

@@ -0,0 +1,217 @@
+/**
+ * Delta Encoding with Varint for Neighbor Lists
+ *
+ * Implements delta-encoded neighbor lists as used by Qdrant for ~38% storage reduction.
+ * Neighbor IDs are sorted, then stored as deltas with variable-length encoding.
+ *
+ * Format:
+ * - First ID stored as full uint32
+ * - Subsequent IDs stored as varint deltas from previous ID
+ *
+ * Varint encoding (like Protocol Buffers):
+ * - Values 0-127: 1 byte
+ * - Values 128-16383: 2 bytes
+ * - Values 16384-2097151: 3 bytes
+ * - Values 2097152-268435455: 4 bytes
+ * - Larger: 5 bytes
+ */
+
+/**
+ * Encode an unsigned integer as a varint
+ * Returns the number of bytes written
+ */
+export function encodeVarint(value: number, buffer: Uint8Array, offset: number): number {
+  let v = value >>> 0; // Ensure unsigned
+  let bytesWritten = 0;
+
+  while (v >= 0x80) {
+    buffer[offset + bytesWritten] = (v & 0x7f) | 0x80;
+    v >>>= 7;
+    bytesWritten++;
+  }
+  buffer[offset + bytesWritten] = v;
+  return bytesWritten + 1;
+}
+
+/**
+ * Decode a varint from buffer
+ * Returns [value, bytesRead]
+ */
+export function decodeVarint(buffer: Uint8Array, offset: number): [number, number] {
+  let result = 0;
+  let shift = 0;
+  let bytesRead = 0;
+
+  while (offset + bytesRead < buffer.length) {
+    const byte = buffer[offset + bytesRead];
+    result |= (byte & 0x7f) << shift;
+    bytesRead++;
+
+    if ((byte & 0x80) === 0) {
+      return [result >>> 0, bytesRead];
+    }
+
+    shift += 7;
+    if (shift > 35) {
+      throw new Error('Varint too long');
+    }
+  }
+
+  throw new Error('Unexpected end of buffer');
+}
+
+/**
+ * Calculate the number of bytes needed to encode a varint
+ */
+export function varintSize(value: number): number {
+  let v = value >>> 0;
+  let size = 1;
+  while (v >= 0x80) {
+    v >>>= 7;
+    size++;
+  }
+  return size;
+}
+
+/**
+ * Delta-encode a sorted array of neighbor IDs
+ * Returns the encoded buffer
+ */
+export function deltaEncodeNeighbors(neighbors: number[]): Uint8Array {
+  if (neighbors.length === 0) {
+    return new Uint8Array(0);
+  }
+
+  // Sort neighbors for optimal delta encoding
+  const sorted = neighbors.slice().sort((a, b) => a - b);
+
+  // Calculate required buffer size
+  let size = 4; // First ID as uint32
+  let prev = sorted[0];
+  for (let i = 1; i < sorted.length; i++) {
+    const delta = sorted[i] - prev;
+    size += varintSize(delta);
+    prev = sorted[i];
+  }
+
+  // Encode
+  const buffer = new Uint8Array(size);
+  const view = new DataView(buffer.buffer);
+
+  // First ID as full uint32 (little-endian)
+  view.setUint32(0, sorted[0], true);
+  let offset = 4;
+
+  // Remaining as deltas
+  prev = sorted[0];
+  for (let i = 1; i < sorted.length; i++) {
+    const delta = sorted[i] - prev;
+    offset += encodeVarint(delta, buffer, offset);
+    prev = sorted[i];
+  }
+
+  return buffer;
+}
+
+/**
+ * Decode a delta-encoded neighbor list
+ * Returns the original neighbor IDs (sorted)
+ */
+export function deltaDecodeNeighbors(buffer: Uint8Array, count: number): number[] {
+  if (count === 0 || buffer.length === 0) {
+    return [];
+  }
+
+  const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+  const neighbors = new Array<number>(count);
+
+  // First ID as full uint32
+  neighbors[0] = view.getUint32(0, true);
+  let offset = 4;
+
+  // Remaining as deltas
+  for (let i = 1; i < count; i++) {
+    const [delta, bytesRead] = decodeVarint(buffer, offset);
+    neighbors[i] = neighbors[i - 1] + delta;
+    offset += bytesRead;
+  }
+
+  return neighbors;
+}
+
+/**
+ * Calculate the encoded size for a neighbor list without actually encoding
+ * Useful for calculating total buffer size before serialization
+ */
+export function deltaEncodedSize(neighbors: number[]): number {
+  if (neighbors.length === 0) {
+    return 0;
+  }
+
+  const sorted = neighbors.slice().sort((a, b) => a - b);
+
+  let size = 4; // First ID as uint32
+  let prev = sorted[0];
+  for (let i = 1; i < sorted.length; i++) {
+    const delta = sorted[i] - prev;
+    size += varintSize(delta);
+    prev = sorted[i];
+  }
+
+  return size;
+}
+
+/**
+ * Batch encode multiple neighbor lists efficiently
+ * Returns a single buffer with all encoded lists concatenated
+ * Also returns offsets for each list
+ */
+export function deltaEncodeBatch(neighborLists: number[][]): {
+  buffer: Uint8Array;
+  offsets: number[];
+  sizes: number[];
+} {
+  // Calculate total size and individual sizes
+  const sizes = neighborLists.map(list => deltaEncodedSize(list));
+  const totalSize = sizes.reduce((a, b) => a + b, 0);
+
+  // Allocate single buffer
+  const buffer = new Uint8Array(totalSize);
+  const offsets: number[] = [];
+
+  let currentOffset = 0;
+  for (let i = 0; i < neighborLists.length; i++) {
+    offsets.push(currentOffset);
+
+    if (neighborLists[i].length > 0) {
+      const encoded = deltaEncodeNeighbors(neighborLists[i]);
+      buffer.set(encoded, currentOffset);
+      currentOffset += encoded.length;
+    }
+  }
+
+  return { buffer, offsets, sizes };
+}
+
+/**
+ * Decode a batch of neighbor lists from a single buffer
+ */
+export function deltaDecodeBatch(
+  buffer: Uint8Array,
+  offsets: number[],
+  sizes: number[],
+  counts: number[]
+): number[][] {
+  const results: number[][] = [];
+
+  for (let i = 0; i < offsets.length; i++) {
+    if (counts[i] === 0 || sizes[i] === 0) {
+      results.push([]);
+    } else {
+      const slice = buffer.subarray(offsets[i], offsets[i] + sizes[i]);
+      results.push(deltaDecodeNeighbors(slice, counts[i]));
+    }
+  }
+
+  return results;
+}

package/src/errors.ts

@@ -0,0 +1,110 @@
+/**
+ * Base error class for all VectorDB errors
+ */
+export class VectorDBError extends Error {
+  readonly code: string;
+
+  constructor(message: string, code: string) {
+    super(message);
+    this.name = 'VectorDBError';
+    this.code = code;
+  }
+}
+
+/**
+ * Thrown when vector dimensions don't match expected dimensions
+ */
+export class DimensionMismatchError extends VectorDBError {
+  readonly expected: number;
+  readonly actual: number;
+
+  constructor(expected: number, actual: number, context?: string) {
+    const message = context
+      ? `${context}: expected dimension ${expected}, got ${actual}`
+      : `Dimension mismatch: expected ${expected}, got ${actual}`;
+    super(message, 'DIMENSION_MISMATCH');
+    this.name = 'DimensionMismatchError';
+    this.expected = expected;
+    this.actual = actual;
+  }
+}
+
+/**
+ * Thrown when attempting to add a vector with an ID that already exists
+ */
+export class DuplicateVectorError extends VectorDBError {
+  readonly ids: string[];
+
+  constructor(ids: string[]) {
+    const message = ids.length === 1
+      ? `Vector with ID '${ids[0]}' already exists`
+      : `Vectors with IDs already exist: ${ids.join(', ')}`;
+    super(message, 'DUPLICATE_VECTOR');
+    this.name = 'DuplicateVectorError';
+    this.ids = ids;
+  }
+}
+
+/**
+ * Thrown when a requested collection does not exist
+ */
+export class CollectionNotFoundError extends VectorDBError {
+  readonly collectionName: string;
+
+  constructor(collectionName: string) {
+    super(`Collection '${collectionName}' does not exist`, 'COLLECTION_NOT_FOUND');
+    this.name = 'CollectionNotFoundError';
+    this.collectionName = collectionName;
+  }
+}
+
+/**
+ * Thrown when attempting to create a collection that already exists
+ */
+export class CollectionExistsError extends VectorDBError {
+  readonly collectionName: string;
+
+  constructor(collectionName: string) {
+    super(`Collection '${collectionName}' already exists`, 'COLLECTION_EXISTS');
+    this.name = 'CollectionExistsError';
+    this.collectionName = collectionName;
+  }
+}
+
+/**
+ * Thrown when a storage operation fails
+ */
+export class StorageError extends VectorDBError {
+  readonly operation: string;
+  readonly path?: string;
+
+  constructor(operation: string, message: string, path?: string) {
+    super(`Storage ${operation} failed: ${message}`, 'STORAGE_ERROR');
+    this.name = 'StorageError';
+    this.operation = operation;
+    this.path = path;
+  }
+}
+
+/**
+ * Thrown when quantization operations fail
+ */
+export class QuantizationError extends VectorDBError {
+  constructor(message: string) {
+    super(message, 'QUANTIZATION_ERROR');
+    this.name = 'QuantizationError';
+  }
+}
+
+/**
+ * Thrown when a vector is not found in the index
+ */
+export class VectorNotFoundError extends VectorDBError {
+  readonly vectorId: string | number;
+
+  constructor(vectorId: string | number) {
+    super(`Vector '${vectorId}' not found`, 'VECTOR_NOT_FOUND');
+    this.name = 'VectorNotFoundError';
+    this.vectorId = vectorId;
+  }
+}

package/src/index.ts

@@ -0,0 +1,138 @@
+/**
+ * verso - High-performance vector search with HNSW indexing
+ *
+ * Features:
+ * - HNSW algorithm for approximate nearest neighbor search
+ * - Multiple distance metrics: cosine, euclidean, dot product
+ * - Int8 scalar quantization for 4x memory reduction
+ * - Batch query support for improved throughput
+ * - Parameter presets for different use cases
+ * - Multi-platform: Bun (file system) and Browser (OPFS)
+ *
+ * @example
+ * ```typescript
+ * import { VectorDB, getRecommendedPreset } from 'verso';
+ *
+ * const db = new VectorDB();
+ * const preset = getRecommendedPreset(768);
+ *
+ * const collection = await db.createCollection('my-vectors', {
+ *   dimension: 768,
+ *   metric: 'cosine',
+ *   M: preset.M,
+ *   efConstruction: preset.efConstruction
+ * });
+ *
+ * await collection.add({
+ *   ids: ['doc1', 'doc2'],
+ *   vectors: [new Float32Array(768), new Float32Array(768)]
+ * });
+ *
+ * const results = await collection.query({
+ *   queryVector: new Float32Array(768),
+ *   k: 10,
+ *   efSearch: preset.efSearch
+ * });
+ * ```
+ *
+ * @packageDocumentation
+ * @module verso
+ */
+
+// Core components
+export { VectorDB } from './VectorDB';
+export type { VectorDBConfig } from './VectorDB';
+export { Collection } from './Collection';
+export type { AddConfig, QueryConfig, QueryResult } from './Collection';
+export { HNSWIndex } from './HNSWIndex';
+export type { DistanceMetric, Node } from './HNSWIndex';
+
+// Distance backends
+export type { DistanceBackend } from './backends/DistanceBackend';
+export {
+  JsDistanceBackend,
+  dotProductFast,
+  l2SquaredFast,
+  normalizeInPlace,
+  cosineDistanceFast
+} from './backends/JsDistanceBackend';
+
+// Data structures
+export { BinaryHeap } from './BinaryHeap';
+export { MaxBinaryHeap } from './MaxBinaryHeap';
+
+// Scalar Quantization (Int8 - 4x reduction)
+export {
+  ScalarQuantizer,
+  QuantizedVectorStore,
+  dotProductInt8,
+  l2SquaredInt8,
+  cosineDistanceInt8
+} from './quantization/ScalarQuantizer';
+export type { QuantizationParams } from './quantization/ScalarQuantizer';
+
+
+// Parameter presets
+export type { HNSWPreset } from './presets';
+export {
+  PRESET_LOW_DIM,
+  PRESET_MEDIUM_DIM,
+  PRESET_HIGH_DIM,
+  PRESET_VERY_HIGH_DIM,
+  PRESET_SMALL_DATASET,
+  PRESET_LARGE_DATASET,
+  PRESET_MAX_RECALL,
+  PRESET_LOW_LATENCY,
+  PRESETS,
+  getRecommendedPreset,
+  getPreset,
+  getRAGPreset
+} from './presets';
+
+// Storage backends (Bun, Browser, Memory)
+export type { StorageBackend, StorageOptions } from './storage/StorageBackend';
+export { BunStorageBackend } from './storage/BunStorageBackend';
+export { MemoryBackend } from './storage/MemoryBackend';
+export { OPFSBackend } from './storage/OPFSBackend';
+export {
+  createStorageBackend,
+  getRecommendedStorageType,
+  isStorageTypeAvailable,
+  type StorageType,
+  type CreateStorageOptions,
+} from './storage/createStorageBackend';
+
+// Write-ahead log for incremental updates
+export {
+  WriteAheadLog,
+  WALOperationType,
+  type WALEntry,
+} from './storage/WriteAheadLog';
+
+// Batch write coalescing for reduced I/O
+export {
+  BatchWriter,
+  createBatchWriter,
+  type BatchWriterOptions,
+} from './storage/BatchWriter';
+
+// Delta encoding for compact neighbor lists
+export {
+  deltaEncodeNeighbors,
+  deltaDecodeNeighbors,
+  deltaEncodedSize,
+  encodeVarint,
+  decodeVarint,
+} from './encoding/DeltaEncoder';
+
+// Error classes for better error handling
+export {
+  VectorDBError,
+  DimensionMismatchError,
+  DuplicateVectorError,
+  CollectionNotFoundError,
+  CollectionExistsError,
+  StorageError,
+  QuantizationError,
+  VectorNotFoundError,
+} from './errors';