verso-db 0.1.1

package/src/MaxBinaryHeap.ts

@@ -0,0 +1,175 @@
+/**
+ * MaxBinaryHeap - A max-heap implementation for HNSW result tracking.
+ *
+ * In HNSW search, we need to track the top-k closest results efficiently.
+ * A max-heap keeps the FURTHEST element at the top, allowing O(log n) eviction
+ * of the worst result when a better one is found.
+ *
+ * This complements BinaryHeap (min-heap) which is used for candidate exploration.
+ */
+export class MaxBinaryHeap {
+  private ids: Uint32Array;
+  private dists: Float32Array;
+  private _size: number;
+  private capacity: number;
+
+  constructor(capacity: number) {
+    this.capacity = capacity;
+    this.ids = new Uint32Array(capacity);
+    this.dists = new Float32Array(capacity);
+    this._size = 0;
+  }
+
+  /**
+   * Push an element onto the heap.
+   * When at capacity, replaces the max element if the new one is closer.
+   */
+  push(id: number, dist: number): void {
+    if (this._size >= this.capacity) {
+      // At capacity - replace max if new element is better (smaller distance)
+      if (dist < this.dists[0]) {
+        // Replace root with new element
+        this.ids[0] = id;
+        this.dists[0] = dist;
+        this.heapifyDown(0);
+      }
+      // Otherwise, new element is worse than max, don't add
+      return;
+    }
+
+    this.ids[this._size] = id;
+    this.dists[this._size] = dist;
+    this._size++;
+    this.heapifyUp(this._size - 1);
+  }
+
+  /**
+   * Remove and return the ID of the maximum element (furthest distance).
+   * Returns -1 if heap is empty.
+   */
+  pop(): number {
+    if (this._size === 0) return -1;
+
+    const result = this.ids[0];
+    this._size--;
+
+    if (this._size > 0) {
+      this.ids[0] = this.ids[this._size];
+      this.dists[0] = this.dists[this._size];
+      this.heapifyDown(0);
+    }
+
+    return result;
+  }
+
+  /**
+   * Peek at the ID of the maximum element without removing it.
+   * Returns -1 if heap is empty.
+   */
+  peek(): number {
+    return this._size > 0 ? this.ids[0] : -1;
+  }
+
+  /**
+   * Peek at the distance of the maximum element (furthest from query).
+   * Returns -Infinity if heap is empty.
+   */
+  peekValue(): number {
+    return this._size > 0 ? this.dists[0] : -Infinity;
+  }
+
+  /**
+   * Get the current number of elements in the heap.
+   */
+  size(): number {
+    return this._size;
+  }
+
+  /**
+   * Check if the heap is empty.
+   */
+  isEmpty(): boolean {
+    return this._size === 0;
+  }
+
+  /**
+   * Clear all elements from the heap.
+   */
+  clear(): void {
+    this._size = 0;
+  }
+
+  /**
+   * Get the capacity of the heap.
+   */
+  getCapacity(): number {
+    return this.capacity;
+  }
+
+  /**
+   * Heapify up: restore max-heap property after insertion.
+   * Parent should be LARGER than children in a max-heap.
+   */
+  private heapifyUp(index: number): void {
+    // Cache array references to avoid repeated property lookups
+    const ids = this.ids;
+    const dists = this.dists;
+
+    while (index > 0) {
+      // Use bitwise shift for faster integer division
+      const parentIndex = (index - 1) >> 1;
+      // MAX-HEAP: Parent should be >= child
+      if (dists[parentIndex] >= dists[index]) break;
+
+      // Inline swap for performance
+      const tmpId = ids[index];
+      ids[index] = ids[parentIndex];
+      ids[parentIndex] = tmpId;
+
+      const tmpDist = dists[index];
+      dists[index] = dists[parentIndex];
+      dists[parentIndex] = tmpDist;
+
+      index = parentIndex;
+    }
+  }
+
+  /**
+   * Heapify down: restore max-heap property after removal.
+   * Find largest among node and children, swap if needed.
+   */
+  private heapifyDown(index: number): void {
+    // Cache array references to avoid repeated property lookups
+    const ids = this.ids;
+    const dists = this.dists;
+    const size = this._size;
+
+    while (true) {
+      const leftChild = (index << 1) + 1; // 2 * index + 1
+      const rightChild = leftChild + 1;   // 2 * index + 2
+      let largest = index;
+
+      // MAX-HEAP: Find largest among node and children
+      if (leftChild < size && dists[leftChild] > dists[largest]) {
+        largest = leftChild;
+      }
+
+      if (rightChild < size && dists[rightChild] > dists[largest]) {
+        largest = rightChild;
+      }
+
+      if (largest === index) break;
+
+      // Inline swap for performance
+      const tmpId = ids[index];
+      ids[index] = ids[largest];
+      ids[largest] = tmpId;
+
+      const tmpDist = dists[index];
+      dists[index] = dists[largest];
+      dists[largest] = tmpDist;
+
+      index = largest;
+    }
+  }
+}

package/src/Storage.ts

@@ -0,0 +1,435 @@
+import * as path from 'path';
+
+// Binary format for metadata storage
+// Format: [count(4)] + repeated [id_len(4) + id + metadata_len(4) + metadata_json]
+
+export class MetadataStorage {
+  private metadataMap: Map<string, any> | null;
+  private idList: string[] | null;
+
+  constructor() {
+    this.metadataMap = new Map();
+    this.idList = [];
+  }
+
+  private ensureInitialized(): void {
+    if (!this.metadataMap || !this.idList) {
+      throw new Error('MetadataStorage has been destroyed and cannot be used');
+    }
+  }
+
+  set(id: string, metadata: any): void {
+    this.ensureInitialized();
+    // Check Map presence first (O(1)) instead of array.includes() (O(n))
+    const isNew = !this.metadataMap!.has(id);
+    this.metadataMap!.set(id, metadata);
+    if (isNew) {
+      this.idList!.push(id);
+    }
+  }
+
+  get(id: string): any | undefined {
+    this.ensureInitialized();
+    return this.metadataMap!.get(id);
+  }
+
+  has(id: string): boolean {
+    this.ensureInitialized();
+    return this.metadataMap!.has(id);
+  }
+
+  delete(id: string): boolean {
+    this.ensureInitialized();
+    // Use indexOf + splice for O(n) vs filter which creates new array
+    const idx = this.idList!.indexOf(id);
+    if (idx !== -1) {
+      this.idList!.splice(idx, 1);
+    }
+    return this.metadataMap!.delete(id);
+  }
+
+  getAllIds(): string[] {
+    this.ensureInitialized();
+    return [...this.idList!];
+  }
+
+  entries(): [string, any][] {
+    this.ensureInitialized();
+    return Array.from(this.metadataMap!.entries());
+  }
+
+  clear(): void {
+    this.ensureInitialized();
+    this.metadataMap!.clear();
+    this.idList = [];
+  }
+
+  destroy(): void {
+    if (this.metadataMap) this.metadataMap.clear();
+    this.idList = null;
+    this.metadataMap = null;
+  }
+
+  size(): number {
+    this.ensureInitialized();
+    return this.metadataMap!.size;
+  }
+
+  async saveToDisk(filePath: string): Promise<void> {
+    const buffer = this.createBinaryBuffer();
+    await Bun.write(filePath, buffer);
+  }
+
+  private createBinaryBuffer(): Buffer {
+    // Pre-serialize all metadata to avoid double JSON.stringify
+    const entries: Array<{ id: string; idLen: number; metaStr: string; metaLen: number }> = [];
+    let totalSize = 4; // count
+
+    for (const [id, metadata] of this.metadataMap!.entries()) {
+      const metaStr = JSON.stringify(metadata);
+      const idLen = Buffer.byteLength(id, 'utf8');
+      const metaLen = Buffer.byteLength(metaStr, 'utf8');
+      entries.push({ id, idLen, metaStr, metaLen });
+      totalSize += 4 + idLen + 4 + metaLen;
+    }
+
+    const buffer = Buffer.alloc(totalSize);
+    let offset = 0;
+
+    // Write count
+    buffer.writeUInt32LE(entries.length, offset);
+    offset += 4;
+
+    // Write each entry using cached values
+    for (const { id, idLen, metaStr, metaLen } of entries) {
+      buffer.writeUInt32LE(idLen, offset);
+      offset += 4;
+      buffer.write(id, offset, idLen, 'utf8');
+      offset += idLen;
+
+      buffer.writeUInt32LE(metaLen, offset);
+      offset += 4;
+      buffer.write(metaStr, offset, metaLen, 'utf8');
+      offset += metaLen;
+    }
+
+    return buffer;
+  }
+
+  async loadFromDisk(filePath: string): Promise<void> {
+    try {
+      const file = Bun.file(filePath);
+      const arrayBuffer = await file.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      this.loadFromBinaryBuffer(buffer);
+    } catch (error) {
+      // If file doesn't exist or is corrupted, start with empty storage
+      this.metadataMap!.clear();
+      this.idList = [];
+    }
+  }
+
+  private loadFromBinaryBuffer(buffer: Buffer): void {
+    let offset = 0;
+
+    // Read count
+    const count = buffer.readUInt32LE(offset);
+    offset += 4;
+
+    this.metadataMap!.clear();
+    this.idList = [];
+
+    // Read each entry
+    for (let i = 0; i < count; i++) {
+      // Read ID length and ID
+      const idLen = buffer.readUInt32LE(offset);
+      offset += 4;
+      const id = buffer.toString('utf8', offset, offset + idLen);
+      offset += idLen;
+
+      // Read metadata length and metadata
+      const metadataLen = buffer.readUInt32LE(offset);
+      offset += 4;
+      const metadataStr = buffer.toString('utf8', offset, offset + metadataLen);
+      offset += metadataLen;
+
+      const metadata = JSON.parse(metadataStr);
+      this.metadataMap!.set(id, metadata);
+      this.idList!.push(id);
+    }
+  }
+}
+
+// For efficient vector storage with memory mapping
+export class VectorStorage {
+  private dimension: number;
+  private vectors: Float32Array;
+  private vectorCount: number;
+  private capacity: number;
+  private filePath: string | null = null;
+  private fileHandle: any = null; // Bun's file handle for memory mapping
+
+  constructor(dimension: number, initialCapacity: number = 1000) {
+    this.dimension = dimension;
+    this.capacity = initialCapacity;
+    this.vectorCount = 0;
+    this.vectors = new Float32Array(this.capacity * dimension);
+  }
+
+  async initializeForMemoryMapping(filePath: string): Promise<void> {
+    this.filePath = filePath;
+
+    // Check if file exists and load it if it does
+    try {
+      const file = Bun.file(filePath);
+      if (await file.exists()) {
+        await this.loadFromDisk(filePath);
+      }
+    } catch (error) {
+      // File doesn't exist, create new one
+      // We'll create it when we save for the first time
+    }
+  }
+
+  addVector(vector: number[]): number {
+    const dim = this.dimension;
+    if (vector.length !== dim) {
+      throw new Error(`Vector dimension mismatch. Expected ${dim}, got ${vector.length}`);
+    }
+
+    // Ensure capacity
+    if (this.vectorCount >= this.capacity) {
+      this.expandCapacity();
+    }
+
+    const index = this.vectorCount;
+    const start = index * dim;
+
+    // Copy vector values (cache start offset)
+    for (let i = 0; i < dim; i++) {
+      this.vectors[start + i] = vector[i];
+    }
+
+    this.vectorCount++;
+    return index;
+  }
+
+  getVector(index: number): number[] {
+    if (index < 0 || index >= this.vectorCount) {
+      throw new Error(`Vector index out of bounds: ${index}`);
+    }
+
+    const dim = this.dimension;
+    const vector = new Array<number>(dim);
+    const start = index * dim;
+    for (let i = 0; i < dim; i++) {
+      vector[i] = this.vectors[start + i];
+    }
+    return vector;
+  }
+
+  getVectorAsTypedArray(index: number): Float32Array {
+    if (index < 0 || index >= this.vectorCount) {
+      throw new Error(`Vector index out of bounds: ${index}`);
+    }
+
+    const start = index * this.dimension;
+    return this.vectors.subarray(start, start + this.dimension);
+  }
+
+  getVectorCount(): number {
+    return this.vectorCount;
+  }
+
+  getDimension(): number {
+    return this.dimension;
+  }
+
+  private expandCapacity(): void {
+    const newCapacity = this.capacity * 2;
+    const newVectors = new Float32Array(newCapacity * this.dimension);
+    newVectors.set(this.vectors);
+    this.vectors = newVectors;
+    this.capacity = newCapacity;
+  }
+
+  async saveToDisk(filePath: string): Promise<void> {
+    const actualSize = this.vectorCount * this.dimension * 4;
+    const buffer = Buffer.from(this.vectors.buffer, 0, actualSize);
+
+    // Ensure directory exists using Bun's file operations
+    const dir = path.dirname(filePath);
+    try {
+      await Bun.write(dir + '/.tmp', new Uint8Array([0])); // Create a temp file to test/create directory
+      await Bun.write(dir + '/.tmp', new Uint8Array()); // Remove the temp file
+    } catch (e) {
+      // Directory might already exist, which is fine
+    }
+
+    await Bun.write(filePath, buffer);
+
+    // Update file path if needed
+    if (this.filePath !== filePath) {
+      this.filePath = filePath;
+    }
+  }
+
+  async loadFromDisk(filePath: string): Promise<void> {
+    try {
+      const file = Bun.file(filePath);
+      const arrayBuffer = await file.arrayBuffer();
+      const uint8Array = new Uint8Array(arrayBuffer);
+      const newVectors = new Float32Array(arrayBuffer);
+
+      // Update internal state
+      this.vectors = newVectors;
+      this.vectorCount = uint8Array.length / (4 * this.dimension);
+      this.capacity = this.vectors.length / this.dimension;
+
+      // Update file path
+      this.filePath = filePath;
+    } catch (error) {
+      // If file doesn't exist, start with empty storage
+      this.vectorCount = 0;
+    }
+  }
+
+  destroy(): void {
+    // Clear the vectors array to free memory
+    this.vectors = new Float32Array(0);
+    this.vectorCount = 0;
+    this.capacity = 0;
+    // Allow garbage collection
+    (this as any).vectors = null;
+  }
+}
+
+// Create a MemoryMappedVectorStorage that uses Bun's memory mapping when available
+export class MemoryMappedVectorStorage {
+  private dimension: number;
+  private vectorCount: number;
+  private filePath: string | null = null;
+  private mappedBuffer: ArrayBuffer | null = null;
+  private mappedView: Float32Array | null = null;
+
+  constructor(dimension: number) {
+    this.dimension = dimension;
+    this.vectorCount = 0;
+  }
+
+  async initializeMemoryMapping(filePath: string): Promise<void> {
+    this.filePath = filePath;
+
+    try {
+      // Check if file exists
+      const file = Bun.file(filePath);
+      const fileSize = (await file.stat()).size;
+
+      if (fileSize > 0) {
+        // Calculate expected vector count based on file size
+        this.vectorCount = fileSize / (4 * this.dimension);
+
+        // Create a file handle and memory map
+        this.mappedBuffer = await file.arrayBuffer();
+        this.mappedView = new Float32Array(this.mappedBuffer);
+      } else {
+        // Create empty file
+        await Bun.write(filePath, new ArrayBuffer(0));
+      }
+    } catch (error) {
+      // File doesn't exist, create new one
+      this.vectorCount = 0;
+      await Bun.write(filePath, new ArrayBuffer(0));
+    }
+  }
+
+  getVector(index: number): number[] {
+    if (!this.mappedView) {
+      throw new Error('Memory mapping not initialized');
+    }
+
+    if (index < 0 || index >= this.vectorCount) {
+      throw new Error(`Vector index out of bounds: ${index}`);
+    }
+
+    const dim = this.dimension;
+    const vector = new Array<number>(dim);
+    const start = index * dim;
+    for (let i = 0; i < dim; i++) {
+      vector[i] = this.mappedView[start + i];
+    }
+    return vector;
+  }
+
+  getVectorAsTypedArray(index: number): Float32Array {
+    if (!this.mappedView) {
+      throw new Error('Memory mapping not initialized');
+    }
+
+    if (index < 0 || index >= this.vectorCount) {
+      throw new Error(`Vector index out of bounds: ${index}`);
+    }
+
+    const start = index * this.dimension;
+    return this.mappedView.subarray(start, start + this.dimension);
+  }
+
+  getVectorCount(): number {
+    return this.vectorCount;
+  }
+
+  getDimension(): number {
+    return this.dimension;
+  }
+
+  async appendVector(vector: number[]): Promise<void> {
+    if (vector.length !== this.dimension) {
+      throw new Error(`Vector dimension mismatch. Expected ${this.dimension}, got ${vector.length}`);
+    }
+
+    if (!this.filePath) {
+      throw new Error('File path not set');
+    }
+
+    // For memory mapping, we'll append to the file and reload the mapping
+    // In a more optimized version, we could use a growable buffer
+    const newVectorCount = this.vectorCount + 1;
+    const newSize = newVectorCount * this.dimension * 4;
+
+    // Read the current file and append the new vector
+    let currentBuffer: ArrayBuffer;
+    try {
+      currentBuffer = await Bun.file(this.filePath).arrayBuffer();
+    } catch {
+      currentBuffer = new ArrayBuffer(0);
+    }
+
+    const newBuffer = new ArrayBuffer(newSize);
+    const newView = new Float32Array(newBuffer);
+
+    if (currentBuffer.byteLength > 0) {
+      const currentView = new Float32Array(currentBuffer);
+      newView.set(currentView);
+    }
+
+    // Add the new vector
+    for (let i = 0; i < this.dimension; i++) {
+      newView[this.vectorCount * this.dimension + i] = vector[i];
+    }
+
+    // Write the new buffer to file
+    await Bun.write(this.filePath, new Uint8Array(newBuffer));
+
+    // Update our mapping
+    this.vectorCount = newVectorCount;
+    this.mappedBuffer = newBuffer;
+    this.mappedView = newView;
+  }
+
+  destroy(): void {
+    // Clear the mapped buffer to free memory
+    this.mappedBuffer = null;
+    this.mappedView = null;
+    this.vectorCount = 0;
+  }
+}

package/src/VectorDB.ts

@@ -0,0 +1,109 @@
+import { Collection } from './Collection';
+import * as path from 'path';
+import type { StorageBackend } from './storage/StorageBackend';
+import { BunStorageBackend } from './storage/BunStorageBackend';
+
+export type DistanceMetric = 'cosine' | 'euclidean' | 'dot_product';
+
+export interface VectorDBConfig {
+  /**
+   * Path for storing collections on disk. Default: './vectordb_data'
+   * Ignored if a custom storageBackend is provided.
+   */
+  storagePath?: string;
+  /**
+   * Custom storage backend for platform-specific storage (e.g., OPFS for browsers).
+   * If not provided, defaults to BunStorageBackend with storagePath.
+   */
+  storageBackend?: StorageBackend;
+}
+
+export interface CollectionConfig {
+  dimension: number;
+  metric?: DistanceMetric;
+  M?: number;
+  efConstruction?: number;
+}
+
+export class VectorDB {
+  private collections: Map<string, Collection>;
+  private storagePath: string;
+  private storageBackend: StorageBackend;
+  private initialized: boolean = false;
+
+  constructor(config?: VectorDBConfig) {
+    this.collections = new Map();
+    this.storagePath = config?.storagePath || './vectordb_data';
+    // Use provided backend or create default BunStorageBackend
+    this.storageBackend = config?.storageBackend || new BunStorageBackend(this.storagePath);
+  }
+
+  /**
+   * Get the storage backend used by this VectorDB instance.
+   * Useful for custom storage operations or debugging.
+   */
+  getStorageBackend(): StorageBackend {
+    return this.storageBackend;
+  }
+
+  /**
+   * Initialize the VectorDB - creates storage directory if needed.
+   * Called automatically on first operation if not called explicitly.
+   */
+  async init(): Promise<void> {
+    if (this.initialized) return;
+
+    // Use storage backend to create root directory
+    await this.storageBackend.mkdir('');
+
+    this.initialized = true;
+  }
+
+  async createCollection(name: string, config: CollectionConfig): Promise<Collection> {
+    // Ensure VectorDB is initialized
+    await this.init();
+
+    if (this.collections.has(name)) {
+      throw new Error(`Collection ${name} already exists`);
+    }
+
+    const collectionPath = path.join(this.storagePath, name);
+
+    // Create collection directory using storage backend
+    await this.storageBackend.mkdir(name);
+
+    const collection = new Collection(name, config, collectionPath);
+    await collection.init();
+    this.collections.set(name, collection);
+    return collection;
+  }
+
+  getCollection(name: string): Collection | undefined {
+    return this.collections.get(name);
+  }
+
+  async listCollections(): Promise<string[]> {
+    return [...this.collections.keys()];
+  }
+
+  async deleteCollection(name: string): Promise<void> {
+    const collection = this.collections.get(name);
+    if (!collection) {
+      throw new Error(`Collection ${name} does not exist`);
+    }
+
+    await collection.destroy();
+    this.collections.delete(name);
+  }
+
+  async close(): Promise<void> {
+    // Close all collections
+    for (const collection of this.collections.values()) {
+      await collection.destroy();
+    }
+    this.collections.clear();
+
+    // Allow garbage collection by preventing access to collections
+    // but keep the map object for potential introspection
+  }
+}