verso-db 0.1.5 → 0.2.0

package/src/storage/WriteAheadLog.ts

@@ -1,379 +0,0 @@
-/**
- * Write-Ahead Log (WAL) for Incremental Index Updates
- *
- * Provides durability and efficient incremental writes for HNSW index operations.
- * Instead of rewriting the entire index on each update, operations are appended
- * to a log file. The log can be compacted into a full snapshot periodically.
- *
- * Benefits:
- * - Fast appends (no full serialization)
- * - Crash recovery (replay log after restart)
- * - Reduced I/O for frequent updates
- */
-
-import type { StorageBackend } from './StorageBackend';
-
-export enum WALOperationType {
-  ADD_VECTOR = 1,
-  ADD_NEIGHBORS = 2,
-  UPDATE_ENTRY_POINT = 3,
-  CHECKPOINT = 4,
-}
-
-export interface WALEntry {
-  type: WALOperationType;
-  timestamp: number;
-  data: ArrayBuffer;
-}
-
-/**
- * WriteAheadLog - Append-only log for incremental index updates
- */
-export class WriteAheadLog {
-  private storage: StorageBackend;
-  private logKey: string;
-  private pendingEntries: WALEntry[] = [];
-  private flushThreshold: number;
-  private entryCount: number = 0;
-  private flushPromise: Promise<void> | null = null;
-
-  /**
-   * Create a new WAL
-   * @param storage StorageBackend for persistence
-   * @param logKey Storage key for the WAL data (e.g., "myindex.wal")
-   * @param flushThreshold Number of entries before auto-flush (default: 100)
-   */
-  constructor(storage: StorageBackend, logKey: string, flushThreshold: number = 100) {
-    this.storage = storage;
-    this.logKey = logKey;
-    this.flushThreshold = flushThreshold;
-  }
-
-  /**
-   * Get the WAL storage key
-   */
-  getKey(): string {
-    return this.logKey;
-  }
-
-  /**
-   * Check if WAL data exists
-   */
-  async exists(): Promise<boolean> {
-    return this.storage.exists(this.logKey);
-  }
-
-  /**
-   * Append a vector addition operation to the log
-   */
-  async appendVector(id: number, vector: Float32Array): Promise<void> {
-    // Format: [id (4 bytes)] [vector length (4 bytes)] [vector data]
-    const dataSize = 4 + 4 + vector.length * 4;
-    const buffer = new ArrayBuffer(dataSize);
-    const view = new DataView(buffer);
-
-    view.setUint32(0, id, true);
-    view.setUint32(4, vector.length, true);
-
-    const floatView = new Float32Array(buffer, 8);
-    floatView.set(vector);
-
-    const entry: WALEntry = {
-      type: WALOperationType.ADD_VECTOR,
-      timestamp: Date.now(),
-      data: buffer,
-    };
-
-    this.pendingEntries.push(entry);
-    this.entryCount++;
-
-    if (this.pendingEntries.length >= this.flushThreshold) {
-      await this.flush();
-    }
-  }
-
-  /**
-   * Append a neighbor update operation to the log
-   */
-  async appendNeighbors(nodeId: number, layer: number, neighbors: number[]): Promise<void> {
-    // Format: [nodeId (4)] [layer (4)] [neighborCount (4)] [neighbors...]
-    const dataSize = 4 + 4 + 4 + neighbors.length * 4;
-    const buffer = new ArrayBuffer(dataSize);
-    const view = new DataView(buffer);
-
-    view.setUint32(0, nodeId, true);
-    view.setUint32(4, layer, true);
-    view.setUint32(8, neighbors.length, true);
-
-    let offset = 12;
-    for (const neighbor of neighbors) {
-      view.setUint32(offset, neighbor, true);
-      offset += 4;
-    }
-
-    const entry: WALEntry = {
-      type: WALOperationType.ADD_NEIGHBORS,
-      timestamp: Date.now(),
-      data: buffer,
-    };
-
-    this.pendingEntries.push(entry);
-    this.entryCount++;
-
-    if (this.pendingEntries.length >= this.flushThreshold) {
-      await this.flush();
-    }
-  }
-
-  /**
-   * Append entry point update to the log
-   */
-  async appendEntryPointUpdate(entryPointId: number, maxLevel: number): Promise<void> {
-    const buffer = new ArrayBuffer(8);
-    const view = new DataView(buffer);
-
-    view.setInt32(0, entryPointId, true);
-    view.setInt32(4, maxLevel, true);
-
-    const entry: WALEntry = {
-      type: WALOperationType.UPDATE_ENTRY_POINT,
-      timestamp: Date.now(),
-      data: buffer,
-    };
-
-    this.pendingEntries.push(entry);
-    this.entryCount++;
-
-    if (this.pendingEntries.length >= this.flushThreshold) {
-      await this.flush();
-    }
-  }
-
-  /**
-   * Write a checkpoint marker to the log
-   */
-  async checkpoint(): Promise<void> {
-    const buffer = new ArrayBuffer(8);
-    const view = new DataView(buffer);
-    view.setFloat64(0, Date.now(), true);
-
-    const entry: WALEntry = {
-      type: WALOperationType.CHECKPOINT,
-      timestamp: Date.now(),
-      data: buffer,
-    };
-
-    this.pendingEntries.push(entry);
-    this.entryCount++;
-    await this.flush();
-  }
-
-  /**
-   * Serialize a WAL entry to bytes
-   */
-  private serializeEntry(entry: WALEntry): Uint8Array {
-    // Format: [type (1)] [timestamp (8)] [dataLength (4)] [data...]
-    const headerSize = 1 + 8 + 4;
-    const totalSize = headerSize + entry.data.byteLength;
-    const buffer = new ArrayBuffer(totalSize);
-    const view = new DataView(buffer);
-
-    view.setUint8(0, entry.type);
-    view.setFloat64(1, entry.timestamp, true);
-    view.setUint32(9, entry.data.byteLength, true);
-
-    const dataView = new Uint8Array(buffer, headerSize);
-    dataView.set(new Uint8Array(entry.data));
-
-    return new Uint8Array(buffer);
-  }
-
-  /**
-   * Flush pending entries to storage
-   * Uses append for efficient O(1) writes
-   */
-  async flush(): Promise<void> {
-    while (true) {
-      if (this.flushPromise) {
-        await this.flushPromise;
-      }
-
-      if (this.pendingEntries.length === 0) {
-        return;
-      }
-
-      // Capture and clear atomically before async work to avoid losing
-      // entries added by concurrent appendVector() calls during the await.
-      const toFlush = this.pendingEntries;
-      this.pendingEntries = [];
-      const currentFlush = this.flushEntries(toFlush);
-      this.flushPromise = currentFlush;
-
-      try {
-        await currentFlush;
-      } finally {
-        if (this.flushPromise === currentFlush) {
-          this.flushPromise = null;
-        }
-      }
-    }
-  }
-
-  private async flushEntries(toFlush: WALEntry[]): Promise<void> {
-    // Serialize captured entries
-    const serializedEntries = toFlush.map(e => this.serializeEntry(e));
-
-    // Calculate total size
-    let totalSize = 0;
-    for (const entry of serializedEntries) {
-      totalSize += entry.length;
-    }
-
-    // Combine into single buffer
-    const combined = new Uint8Array(totalSize);
-    let offset = 0;
-    for (const entry of serializedEntries) {
-      combined.set(entry, offset);
-      offset += entry.length;
-    }
-
-    // Append to storage
-    try {
-      await this.storage.append(this.logKey, combined);
-    } catch (err) {
-      // Restore entries on failure so they aren't lost
-      this.pendingEntries = [...toFlush, ...this.pendingEntries];
-      throw err;
-    }
-  }
-
-  /**
-   * Read all entries from the WAL
-   */
-  async readEntries(): Promise<WALEntry[]> {
-    const data = await this.storage.read(this.logKey);
-    if (!data || data.byteLength === 0) {
-      return [];
-    }
-
-    const buffer = data;
-    const view = new DataView(buffer);
-    const entries: WALEntry[] = [];
-
-    const headerSize = 13; // 1 (type) + 8 (timestamp) + 4 (dataLength)
-    let offset = 0;
-    while (offset < buffer.byteLength) {
-      // Check we can read the header
-      if (offset + headerSize > buffer.byteLength) {
-        console.warn(`WAL: truncated entry header at offset ${offset}, stopping replay`);
-        break;
-      }
-
-      // Read header
-      const type = view.getUint8(offset) as WALOperationType;
-      const timestamp = view.getFloat64(offset + 1, true);
-      const dataLength = view.getUint32(offset + 9, true);
-
-      // Validate data length
-      if (dataLength > buffer.byteLength - offset - headerSize) {
-        console.warn(`WAL: truncated entry data at offset ${offset} (expected ${dataLength} bytes, only ${buffer.byteLength - offset - headerSize} available), stopping replay`);
-        break;
-      }
-
-      // Validate operation type
-      if (type < 1 || type > 4) {
-        console.warn(`WAL: unknown operation type ${type} at offset ${offset}, stopping replay`);
-        break;
-      }
-
-      // Read data
-      const entryData = buffer.slice(offset + headerSize, offset + headerSize + dataLength);
-
-      entries.push({ type, timestamp, data: entryData });
-
-      offset += headerSize + dataLength;
-    }
-
-    this.entryCount = entries.length + this.pendingEntries.length;
-    return entries;
-  }
-
-  /**
-   * Parse a vector addition entry
-   */
-  static parseVectorEntry(data: ArrayBuffer): { id: number; vector: Float32Array } {
-    const view = new DataView(data);
-    const id = view.getUint32(0, true);
-    const vectorLength = view.getUint32(4, true);
-    const vector = new Float32Array(data, 8, vectorLength);
-    return { id, vector };
-  }
-
-  /**
-   * Parse a neighbor update entry
-   */
-  static parseNeighborsEntry(data: ArrayBuffer): { nodeId: number; layer: number; neighbors: number[] } {
-    const view = new DataView(data);
-    const nodeId = view.getUint32(0, true);
-    const layer = view.getUint32(4, true);
-    const neighborCount = view.getUint32(8, true);
-
-    const neighbors: number[] = [];
-    for (let i = 0; i < neighborCount; i++) {
-      neighbors.push(view.getUint32(12 + i * 4, true));
-    }
-
-    return { nodeId, layer, neighbors };
-  }
-
-  /**
-   * Parse an entry point update entry
-   */
-  static parseEntryPointEntry(data: ArrayBuffer): { entryPointId: number; maxLevel: number } {
-    const view = new DataView(data);
-    return {
-      entryPointId: view.getInt32(0, true),
-      maxLevel: view.getInt32(4, true),
-    };
-  }
-
-  /**
-   * Get entry count since last compact
-   */
-  getEntryCount(): number {
-    return this.entryCount;
-  }
-
-  /**
-   * Clear the WAL (after successful compaction)
-   */
-  async clear(): Promise<void> {
-    if (this.flushPromise) {
-      await this.flushPromise.catch(() => {});
-    }
-    // Clear in-memory state first to prevent flush() from writing entries
-    // after we truncate the on-disk WAL.
-    this.pendingEntries = [];
-    this.entryCount = 0;
-    if (await this.storage.exists(this.logKey)) {
-      await this.storage.write(this.logKey, new Uint8Array(0));
-    }
-  }
-
-  /**
-   * Delete the WAL data
-   */
-  async delete(): Promise<void> {
-    if (this.flushPromise) {
-      await this.flushPromise.catch(() => {});
-    }
-    // Clear in-memory state first to prevent race with concurrent appends
-    this.pendingEntries = [];
-    this.entryCount = 0;
-    try {
-      await this.storage.delete(this.logKey);
-    } catch {
-      // Key may not exist
-    }
-  }
-}

package/src/storage/createStorageBackend.ts

@@ -1,137 +0,0 @@
-/**
- * Storage Backend Factory
- *
- * Auto-detects the best storage backend for the current environment:
- * - Bun/Node.js: BunStorageBackend (file system)
- * - Modern browsers: OPFSBackend (Origin Private File System)
- * - Fallback: MemoryBackend (in-memory)
- */
-
-import type { StorageBackend, StorageOptions } from './StorageBackend';
-import { BunStorageBackend } from './BunStorageBackend';
-import { MemoryBackend } from './MemoryBackend';
-import { OPFSBackend } from './OPFSBackend';
-
-export type StorageType = 'auto' | 'bun' | 'opfs' | 'memory';
-
-export interface CreateStorageOptions extends StorageOptions {
-  /** Force a specific storage type */
-  type?: StorageType;
-}
-
-/**
- * Detect the current runtime environment
- */
-function detectEnvironment(): 'bun' | 'browser' | 'unknown' {
-  // Check for Bun
-  if (typeof Bun !== 'undefined') {
-    return 'bun';
-  }
-
-  // Check for browser
-  if (typeof window !== 'undefined' && typeof navigator !== 'undefined') {
-    return 'browser';
-  }
-
-  return 'unknown';
-}
-
-/**
- * Create the optimal storage backend for the current environment
- *
- * @param options Configuration options
- * @returns Initialized storage backend
- *
- * @example
- * ```typescript
- * // Auto-detect best backend
- * const storage = await createStorageBackend();
- *
- * // Force specific backend
- * const bunStorage = await createStorageBackend({ type: 'bun', path: './data' });
- * const memStorage = await createStorageBackend({ type: 'memory' });
- * ```
- */
-export async function createStorageBackend(options?: CreateStorageOptions): Promise<StorageBackend> {
-  const type = options?.type ?? 'auto';
-
-  // Force specific backend type
-  if (type === 'bun') {
-    const backend = new BunStorageBackend(options?.path ?? './vectordb_data');
-    await backend.init();
-    return backend;
-  }
-
-  if (type === 'opfs') {
-    if (!OPFSBackend.isAvailable()) {
-      throw new Error('OPFS not available in this environment');
-    }
-    const backend = new OPFSBackend();
-    await backend.init();
-    return backend;
-  }
-
-  if (type === 'memory') {
-    return new MemoryBackend();
-  }
-
-  // Auto-detect
-  const env = detectEnvironment();
-
-  if (env === 'bun') {
-    const backend = new BunStorageBackend(options?.path ?? './vectordb_data');
-    await backend.init();
-    return backend;
-  }
-
-  if (env === 'browser') {
-    // Try OPFS first (modern browsers)
-    if (OPFSBackend.isAvailable()) {
-      try {
-        const backend = new OPFSBackend();
-        await backend.init();
-        return backend;
-      } catch {
-        // Fall through to memory backend
-      }
-    }
-  }
-
-  // Fallback to memory backend
-  return new MemoryBackend();
-}
-
-/**
- * Get the recommended storage type for the current environment
- */
-export function getRecommendedStorageType(): StorageType {
-  const env = detectEnvironment();
-
-  if (env === 'bun') {
-    return 'bun';
-  }
-
-  if (env === 'browser' && OPFSBackend.isAvailable()) {
-    return 'opfs';
-  }
-
-  return 'memory';
-}
-
-/**
- * Check if a specific storage type is available
- */
-export function isStorageTypeAvailable(type: StorageType): boolean {
-  switch (type) {
-    case 'bun':
-      return typeof Bun !== 'undefined';
-    case 'opfs':
-      return OPFSBackend.isAvailable();
-    case 'memory':
-      return true;
-    case 'auto':
-      return true;
-    default:
-      return false;
-  }
-}