verso-db 0.1.1

package/src/storage/OPFSBackend.ts

@@ -0,0 +1,250 @@
+/**
+ * Origin Private File System (OPFS) Storage Backend
+ *
+ * High-performance browser storage using the File System Access API.
+ * OPFS provides file-like semantics with significantly better performance
+ * than IndexedDB for large binary data (up to 10x faster).
+ *
+ * Features:
+ * - Auto-initializing (no manual init() required)
+ * - High performance for large binary data
+ *
+ * Browser Support:
+ * - Chrome 86+
+ * - Safari 15.2+
+ * - Firefox 111+
+ * - Edge 86+
+ *
+ * Note: This module is designed for browser environments only.
+ * It will not work in Node.js/Bun server environments.
+ */
+
+import type { StorageBackend } from './StorageBackend';
+
+export class OPFSBackend implements StorageBackend {
+  readonly type = 'opfs';
+  private root: FileSystemDirectoryHandle | null = null;
+  private initialized: boolean = false;
+
+  /**
+   * Initialize the OPFS backend (optional - auto-initializes on first operation)
+   * @deprecated No longer required - operations auto-initialize
+   */
+  async init(): Promise<void> {
+    if (this.initialized) return;
+
+    if (typeof navigator === 'undefined' || !navigator.storage?.getDirectory) {
+      throw new Error('OPFS not available in this environment. Use MemoryBackend or IndexedDBBackend instead.');
+    }
+
+    this.root = await navigator.storage.getDirectory();
+    this.initialized = true;
+  }
+
+  /**
+   * Ensure backend is initialized (lazy init on first use)
+   */
+  private async ensureInitialized(): Promise<void> {
+    if (!this.initialized) {
+      await this.init();
+    }
+  }
+
+  /**
+   * Get a file handle, creating parent directories as needed
+   */
+  private async getFileHandle(key: string, create: boolean = false): Promise<FileSystemFileHandle | null> {
+    await this.ensureInitialized();
+
+    const parts = key.split('/');
+    const fileName = parts.pop()!;
+
+    // Navigate/create directories
+    let currentDir = this.root!;
+    for (const part of parts) {
+      if (part === '') continue;
+      try {
+        currentDir = await currentDir.getDirectoryHandle(part, { create });
+      } catch {
+        if (!create) return null;
+        throw new Error(`Failed to create directory: ${part}`);
+      }
+    }
+
+    // Get file handle
+    try {
+      return await currentDir.getFileHandle(fileName, { create });
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Get a directory handle, creating if needed
+   */
+  private async getDirectoryHandle(path: string, create: boolean = false): Promise<FileSystemDirectoryHandle | null> {
+    await this.ensureInitialized();
+
+    const parts = path.split('/').filter(p => p !== '');
+
+    let currentDir = this.root!;
+    for (const part of parts) {
+      try {
+        currentDir = await currentDir.getDirectoryHandle(part, { create });
+      } catch {
+        if (!create) return null;
+        throw new Error(`Failed to get directory: ${part}`);
+      }
+    }
+
+    return currentDir;
+  }
+
+  async read(key: string): Promise<ArrayBuffer | null> {
+    await this.ensureInitialized();
+
+    const handle = await this.getFileHandle(key, false);
+    if (!handle) return null;
+
+    try {
+      const file = await handle.getFile();
+      return file.arrayBuffer();
+    } catch {
+      return null;
+    }
+  }
+
+  async write(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
+    await this.ensureInitialized();
+
+    const handle = await this.getFileHandle(key, true);
+    if (!handle) {
+      throw new Error(`Failed to create file: ${key}`);
+    }
+
+    const writable = await handle.createWritable();
+    try {
+      // Convert to ArrayBuffer for FileSystemWriteChunkType compatibility
+      const writeData = data instanceof ArrayBuffer ? data : data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
+      await writable.write(writeData);
+    } finally {
+      await writable.close();
+    }
+  }
+
+  async append(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
+    await this.ensureInitialized();
+
+    const handle = await this.getFileHandle(key, true);
+    if (!handle) {
+      throw new Error(`Failed to create file: ${key}`);
+    }
+
+    const appendData = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
+
+    // Get current file size to seek to end
+    let existingSize = 0;
+    try {
+      const file = await handle.getFile();
+      existingSize = file.size;
+    } catch {
+      existingSize = 0;
+    }
+
+    // Use keepExistingData: true for O(1) append (no full read required)
+    // Then seek to end and write only new data
+    const writable = await handle.createWritable({ keepExistingData: true });
+    try {
+      // Seek to end of file
+      await writable.seek(existingSize);
+      // Write only the new data - O(data.length) not O(existingSize + data.length)
+      // Convert to ArrayBuffer for FileSystemWriteChunkType compatibility
+      const writeData = appendData.buffer.slice(appendData.byteOffset, appendData.byteOffset + appendData.byteLength) as ArrayBuffer;
+      await writable.write(writeData);
+    } finally {
+      await writable.close();
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.ensureInitialized();
+
+    const parts = key.split('/');
+    const fileName = parts.pop()!;
+
+    // Navigate to parent directory
+    let currentDir = this.root!;
+    for (const part of parts) {
+      if (part === '') continue;
+      try {
+        currentDir = await currentDir.getDirectoryHandle(part, { create: false });
+      } catch {
+        return; // Parent doesn't exist, nothing to delete
+      }
+    }
+
+    // Remove the file
+    try {
+      await currentDir.removeEntry(fileName);
+    } catch {
+      // File may not exist
+    }
+  }
+
+  async exists(key: string): Promise<boolean> {
+    await this.ensureInitialized();
+
+    const handle = await this.getFileHandle(key, false);
+    return handle !== null;
+  }
+
+  async list(prefix?: string): Promise<string[]> {
+    await this.ensureInitialized();
+
+    const results: string[] = [];
+    const basePath = prefix || '';
+
+    const dir = basePath ? await this.getDirectoryHandle(basePath, false) : this.root!;
+    if (!dir) return results;
+
+    // Recursive list
+    const listDir = async (dirHandle: FileSystemDirectoryHandle, pathPrefix: string): Promise<void> => {
+      for await (const entry of (dirHandle as any).values()) {
+        const entryPath = pathPrefix ? `${pathPrefix}/${entry.name}` : entry.name;
+
+        if (entry.kind === 'file') {
+          results.push(entryPath);
+        } else if (entry.kind === 'directory') {
+          await listDir(entry, entryPath);
+        }
+      }
+    };
+
+    await listDir(dir, basePath);
+    return results;
+  }
+
+  async mkdir(path: string): Promise<void> {
+    await this.ensureInitialized();
+    await this.getDirectoryHandle(path, true);
+  }
+
+  /**
+   * Clear all data in OPFS
+   */
+  async clear(): Promise<void> {
+    await this.ensureInitialized();
+
+    // Delete all entries in root
+    for await (const entry of (this.root as any).values()) {
+      await this.root!.removeEntry(entry.name, { recursive: true });
+    }
+  }
+
+  /**
+   * Check if OPFS is available in the current environment
+   */
+  static isAvailable(): boolean {
+    return typeof navigator !== 'undefined' && !!navigator.storage?.getDirectory;
+  }
+}

package/src/storage/StorageBackend.ts

@@ -0,0 +1,74 @@
+/**
+ * Storage Backend Interface
+ *
+ * Provides a unified interface for persisting vector index data across different platforms:
+ * - Bun/Node.js: File system storage with Bun.file APIs
+ * - Browser: OPFS (Origin Private File System) or IndexedDB
+ * - Testing: In-memory storage
+ *
+ * All operations are async to support both sync and async underlying storage.
+ */
+
+export interface StorageBackend {
+  /**
+   * Read data from storage
+   * @param key Unique key/path for the data
+   * @returns ArrayBuffer of data or null if not found
+   */
+  read(key: string): Promise<ArrayBuffer | null>;
+
+  /**
+   * Write data to storage (overwrites existing)
+   * @param key Unique key/path for the data
+   * @param data Data to write
+   */
+  write(key: string, data: ArrayBuffer | Uint8Array): Promise<void>;
+
+  /**
+   * Append data to existing file (for WAL)
+   * @param key Unique key/path for the data
+   * @param data Data to append
+   */
+  append(key: string, data: ArrayBuffer | Uint8Array): Promise<void>;
+
+  /**
+   * Delete data from storage
+   * @param key Unique key/path to delete
+   */
+  delete(key: string): Promise<void>;
+
+  /**
+   * Check if key exists in storage
+   * @param key Unique key/path to check
+   */
+  exists(key: string): Promise<boolean>;
+
+  /**
+   * List all keys with given prefix
+   * @param prefix Optional prefix to filter keys
+   */
+  list(prefix?: string): Promise<string[]>;
+
+  /**
+   * Create a directory (for file-based backends)
+   * @param path Directory path to create
+   */
+  mkdir(path: string): Promise<void>;
+
+  /**
+   * Get storage type identifier
+   */
+  readonly type: string;
+}
+
+/**
+ * Options for creating a storage backend
+ */
+export interface StorageOptions {
+  /** Base path for file-based storage */
+  path?: string;
+  /** Database name for IndexedDB */
+  dbName?: string;
+  /** Store name for IndexedDB */
+  storeName?: string;
+}

package/src/storage/WriteAheadLog.ts

@@ -0,0 +1,326 @@
+/**
+ * Write-Ahead Log (WAL) for Incremental Index Updates
+ *
+ * Provides durability and efficient incremental writes for HNSW index operations.
+ * Instead of rewriting the entire index on each update, operations are appended
+ * to a log file. The log can be compacted into a full snapshot periodically.
+ *
+ * Benefits:
+ * - Fast appends (no full serialization)
+ * - Crash recovery (replay log after restart)
+ * - Reduced I/O for frequent updates
+ */
+
+import { appendFile, mkdir, unlink } from 'fs/promises';
+import * as path from 'path';
+
+export enum WALOperationType {
+  ADD_VECTOR = 1,
+  ADD_NEIGHBORS = 2,
+  UPDATE_ENTRY_POINT = 3,
+  CHECKPOINT = 4,
+}
+
+export interface WALEntry {
+  type: WALOperationType;
+  timestamp: number;
+  data: ArrayBuffer;
+}
+
+/**
+ * WriteAheadLog - Append-only log for incremental index updates
+ */
+export class WriteAheadLog {
+  private logPath: string;
+  private pendingEntries: WALEntry[] = [];
+  private flushThreshold: number;
+  private lastFlushTime: number = 0;
+  private entryCount: number = 0;
+
+  /**
+   * Create a new WAL
+   * @param basePath Base path for the index (WAL will be basePath.wal)
+   * @param flushThreshold Number of entries before auto-flush (default: 100)
+   */
+  constructor(basePath: string, flushThreshold: number = 100) {
+    this.logPath = `${basePath}.wal`;
+    this.flushThreshold = flushThreshold;
+  }
+
+  /**
+   * Get the WAL file path
+   */
+  getPath(): string {
+    return this.logPath;
+  }
+
+  /**
+   * Check if WAL file exists
+   */
+  async exists(): Promise<boolean> {
+    const file = Bun.file(this.logPath);
+    return file.exists();
+  }
+
+  /**
+   * Append a vector addition operation to the log
+   */
+  async appendVector(id: number, vector: Float32Array): Promise<void> {
+    // Format: [id (4 bytes)] [vector length (4 bytes)] [vector data]
+    const dataSize = 4 + 4 + vector.length * 4;
+    const buffer = new ArrayBuffer(dataSize);
+    const view = new DataView(buffer);
+
+    view.setUint32(0, id, true);
+    view.setUint32(4, vector.length, true);
+
+    const floatView = new Float32Array(buffer, 8);
+    floatView.set(vector);
+
+    const entry: WALEntry = {
+      type: WALOperationType.ADD_VECTOR,
+      timestamp: Date.now(),
+      data: buffer,
+    };
+
+    this.pendingEntries.push(entry);
+    this.entryCount++;
+
+    if (this.pendingEntries.length >= this.flushThreshold) {
+      await this.flush();
+    }
+  }
+
+  /**
+   * Append a neighbor update operation to the log
+   */
+  async appendNeighbors(nodeId: number, layer: number, neighbors: number[]): Promise<void> {
+    // Format: [nodeId (4)] [layer (4)] [neighborCount (4)] [neighbors...]
+    const dataSize = 4 + 4 + 4 + neighbors.length * 4;
+    const buffer = new ArrayBuffer(dataSize);
+    const view = new DataView(buffer);
+
+    view.setUint32(0, nodeId, true);
+    view.setUint32(4, layer, true);
+    view.setUint32(8, neighbors.length, true);
+
+    let offset = 12;
+    for (const neighbor of neighbors) {
+      view.setUint32(offset, neighbor, true);
+      offset += 4;
+    }
+
+    const entry: WALEntry = {
+      type: WALOperationType.ADD_NEIGHBORS,
+      timestamp: Date.now(),
+      data: buffer,
+    };
+
+    this.pendingEntries.push(entry);
+    this.entryCount++;
+
+    if (this.pendingEntries.length >= this.flushThreshold) {
+      await this.flush();
+    }
+  }
+
+  /**
+   * Append entry point update to the log
+   */
+  async appendEntryPointUpdate(entryPointId: number, maxLevel: number): Promise<void> {
+    const buffer = new ArrayBuffer(8);
+    const view = new DataView(buffer);
+
+    view.setInt32(0, entryPointId, true);
+    view.setInt32(4, maxLevel, true);
+
+    const entry: WALEntry = {
+      type: WALOperationType.UPDATE_ENTRY_POINT,
+      timestamp: Date.now(),
+      data: buffer,
+    };
+
+    this.pendingEntries.push(entry);
+    this.entryCount++;
+
+    if (this.pendingEntries.length >= this.flushThreshold) {
+      await this.flush();
+    }
+  }
+
+  /**
+   * Write a checkpoint marker to the log
+   */
+  async checkpoint(): Promise<void> {
+    const buffer = new ArrayBuffer(8);
+    const view = new DataView(buffer);
+    view.setFloat64(0, Date.now(), true);
+
+    const entry: WALEntry = {
+      type: WALOperationType.CHECKPOINT,
+      timestamp: Date.now(),
+      data: buffer,
+    };
+
+    this.pendingEntries.push(entry);
+    await this.flush();
+  }
+
+  /**
+   * Serialize a WAL entry to bytes
+   */
+  private serializeEntry(entry: WALEntry): Uint8Array {
+    // Format: [type (1)] [timestamp (8)] [dataLength (4)] [data...]
+    const headerSize = 1 + 8 + 4;
+    const totalSize = headerSize + entry.data.byteLength;
+    const buffer = new ArrayBuffer(totalSize);
+    const view = new DataView(buffer);
+
+    view.setUint8(0, entry.type);
+    view.setFloat64(1, entry.timestamp, true);
+    view.setUint32(9, entry.data.byteLength, true);
+
+    const dataView = new Uint8Array(buffer, headerSize);
+    dataView.set(new Uint8Array(entry.data));
+
+    return new Uint8Array(buffer);
+  }
+
+  /**
+   * Flush pending entries to disk
+   * Uses O(1) append instead of O(n) read-modify-write
+   */
+  async flush(): Promise<void> {
+    if (this.pendingEntries.length === 0) return;
+
+    // Serialize all pending entries
+    const serializedEntries = this.pendingEntries.map(e => this.serializeEntry(e));
+
+    // Calculate total size
+    let totalSize = 0;
+    for (const entry of serializedEntries) {
+      totalSize += entry.length;
+    }
+
+    // Combine into single buffer
+    const combined = new Uint8Array(totalSize);
+    let offset = 0;
+    for (const entry of serializedEntries) {
+      combined.set(entry, offset);
+      offset += entry.length;
+    }
+
+    // Use true append - O(1) instead of O(n) read-modify-write
+    // Ensure directory exists
+    const dir = path.dirname(this.logPath);
+    await mkdir(dir, { recursive: true }).catch(() => {});
+
+    // Append to file (creates if doesn't exist)
+    await appendFile(this.logPath, combined);
+
+    this.pendingEntries = [];
+    this.lastFlushTime = Date.now();
+  }
+
+  /**
+   * Read all entries from the WAL file
+   */
+  async readEntries(): Promise<WALEntry[]> {
+    const file = Bun.file(this.logPath);
+    if (!(await file.exists())) {
+      return [];
+    }
+
+    const buffer = await file.arrayBuffer();
+    const view = new DataView(buffer);
+    const entries: WALEntry[] = [];
+
+    let offset = 0;
+    while (offset < buffer.byteLength) {
+      // Read header
+      const type = view.getUint8(offset) as WALOperationType;
+      const timestamp = view.getFloat64(offset + 1, true);
+      const dataLength = view.getUint32(offset + 9, true);
+
+      // Read data
+      const data = buffer.slice(offset + 13, offset + 13 + dataLength);
+
+      entries.push({ type, timestamp, data });
+
+      offset += 13 + dataLength;
+    }
+
+    return entries;
+  }
+
+  /**
+   * Parse a vector addition entry
+   */
+  static parseVectorEntry(data: ArrayBuffer): { id: number; vector: Float32Array } {
+    const view = new DataView(data);
+    const id = view.getUint32(0, true);
+    const vectorLength = view.getUint32(4, true);
+    const vector = new Float32Array(data, 8, vectorLength);
+    return { id, vector };
+  }
+
+  /**
+   * Parse a neighbor update entry
+   */
+  static parseNeighborsEntry(data: ArrayBuffer): { nodeId: number; layer: number; neighbors: number[] } {
+    const view = new DataView(data);
+    const nodeId = view.getUint32(0, true);
+    const layer = view.getUint32(4, true);
+    const neighborCount = view.getUint32(8, true);
+
+    const neighbors: number[] = [];
+    for (let i = 0; i < neighborCount; i++) {
+      neighbors.push(view.getUint32(12 + i * 4, true));
+    }
+
+    return { nodeId, layer, neighbors };
+  }
+
+  /**
+   * Parse an entry point update entry
+   */
+  static parseEntryPointEntry(data: ArrayBuffer): { entryPointId: number; maxLevel: number } {
+    const view = new DataView(data);
+    return {
+      entryPointId: view.getInt32(0, true),
+      maxLevel: view.getInt32(4, true),
+    };
+  }
+
+  /**
+   * Get entry count since last compact
+   */
+  getEntryCount(): number {
+    return this.entryCount;
+  }
+
+  /**
+   * Clear the WAL (after successful compaction)
+   */
+  async clear(): Promise<void> {
+    const file = Bun.file(this.logPath);
+    if (await file.exists()) {
+      await Bun.write(this.logPath, new Uint8Array(0));
+    }
+    this.pendingEntries = [];
+    this.entryCount = 0;
+  }
+
+  /**
+   * Delete the WAL file
+   */
+  async delete(): Promise<void> {
+    try {
+      await unlink(this.logPath);
+    } catch {
+      // File may not exist
+    }
+    this.pendingEntries = [];
+    this.entryCount = 0;
+  }
+}