verso-db 0.1.1

package/src/storage/BatchWriter.ts

@@ -0,0 +1,336 @@
+/**
+ * Batch Write Coalescing for Storage Backends
+ *
+ * Buffers writes in memory and flushes them in batches to reduce I/O operations.
+ * This is especially beneficial for:
+ * - OPFS where each write has overhead
+ * - IndexedDB where transactions have cost
+ * - Network storage where round-trips are expensive
+ *
+ * Features:
+ * - Configurable flush thresholds (count and size)
+ * - Automatic flushing when thresholds are reached
+ * - Explicit flush for durability guarantees
+ * - Coalesces multiple writes to the same key
+ */
+
+import type { StorageBackend } from './StorageBackend';
+
+export interface BatchWriterOptions {
+  /** Maximum number of pending writes before auto-flush (default: 100) */
+  maxPendingWrites?: number;
+  /** Maximum total size of pending data in bytes before auto-flush (default: 1MB) */
+  maxPendingBytes?: number;
+  /** Auto-flush interval in milliseconds (0 = disabled, default: 0) */
+  autoFlushInterval?: number;
+}
+
+interface PendingWrite {
+  key: string;
+  data: Uint8Array;
+  append: boolean;
+}
+
+/**
+ * BatchWriter - Coalesces multiple writes into batched flushes
+ */
+export class BatchWriter {
+  private backend: StorageBackend;
+  private pendingWrites: Map<string, PendingWrite> = new Map();
+  private pendingAppends: Map<string, Uint8Array[]> = new Map();
+  private pendingBytes: number = 0;
+  private maxPendingWrites: number;
+  private maxPendingBytes: number;
+  private autoFlushInterval: number;
+  private flushTimer: ReturnType<typeof setTimeout> | null = null;
+  private isFlushing: boolean = false;
+  private flushPromise: Promise<void> | null = null;
+
+  constructor(backend: StorageBackend, options: BatchWriterOptions = {}) {
+    this.backend = backend;
+    this.maxPendingWrites = options.maxPendingWrites ?? 100;
+    this.maxPendingBytes = options.maxPendingBytes ?? 1024 * 1024; // 1MB
+    this.autoFlushInterval = options.autoFlushInterval ?? 0;
+
+    if (this.autoFlushInterval > 0) {
+      this.startAutoFlush();
+    }
+  }
+
+  /**
+   * Write data to a key (buffered)
+   * Multiple writes to the same key will coalesce to the last value
+   */
+  async write(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
+    const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
+
+    // If there's already a pending write, subtract its size
+    const existing = this.pendingWrites.get(key);
+    if (existing) {
+      this.pendingBytes -= existing.data.length;
+    }
+
+    // Clear any pending appends for this key (write overwrites)
+    const existingAppends = this.pendingAppends.get(key);
+    if (existingAppends) {
+      for (const append of existingAppends) {
+        this.pendingBytes -= append.length;
+      }
+      this.pendingAppends.delete(key);
+    }
+
+    this.pendingWrites.set(key, {
+      key,
+      data: bytes,
+      append: false
+    });
+    this.pendingBytes += bytes.length;
+
+    await this.checkThresholds();
+  }
+
+  /**
+   * Append data to a key (buffered)
+   * Multiple appends to the same key will be concatenated
+   */
+  async append(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
+    const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
+
+    // If there's a pending write, append to it instead
+    const existingWrite = this.pendingWrites.get(key);
+    if (existingWrite) {
+      // Concatenate with existing write
+      const newData = new Uint8Array(existingWrite.data.length + bytes.length);
+      newData.set(existingWrite.data);
+      newData.set(bytes, existingWrite.data.length);
+      this.pendingBytes -= existingWrite.data.length;
+      this.pendingBytes += newData.length;
+      existingWrite.data = newData;
+    } else {
+      // Add to pending appends
+      let appends = this.pendingAppends.get(key);
+      if (!appends) {
+        appends = [];
+        this.pendingAppends.set(key, appends);
+      }
+      appends.push(bytes);
+      this.pendingBytes += bytes.length;
+    }
+
+    await this.checkThresholds();
+  }
+
+  /**
+   * Delete a key (buffered)
+   * Clears any pending writes/appends for this key
+   */
+  async delete(key: string): Promise<void> {
+    // Clear pending operations
+    const existing = this.pendingWrites.get(key);
+    if (existing) {
+      this.pendingBytes -= existing.data.length;
+      this.pendingWrites.delete(key);
+    }
+
+    const existingAppends = this.pendingAppends.get(key);
+    if (existingAppends) {
+      for (const append of existingAppends) {
+        this.pendingBytes -= append.length;
+      }
+      this.pendingAppends.delete(key);
+    }
+
+    // Perform delete immediately (can't batch deletes effectively)
+    await this.backend.delete(key);
+  }
+
+  /**
+   * Read data from a key
+   * Returns pending data if available, otherwise reads from backend
+   */
+  async read(key: string): Promise<ArrayBuffer | null> {
+    // Check for pending write
+    const pending = this.pendingWrites.get(key);
+    if (pending) {
+      const buffer = pending.data.buffer.slice(
+        pending.data.byteOffset,
+        pending.data.byteOffset + pending.data.length
+      );
+      return buffer as ArrayBuffer;
+    }
+
+    // Check for pending appends - need to combine with existing data
+    const appends = this.pendingAppends.get(key);
+    if (appends && appends.length > 0) {
+      const existing = await this.backend.read(key);
+      const existingBytes = existing ? new Uint8Array(existing) : new Uint8Array(0);
+
+      // Calculate total size
+      let totalSize = existingBytes.length;
+      for (const append of appends) {
+        totalSize += append.length;
+      }
+
+      // Combine
+      const result = new Uint8Array(totalSize);
+      result.set(existingBytes);
+      let offset = existingBytes.length;
+      for (const append of appends) {
+        result.set(append, offset);
+        offset += append.length;
+      }
+
+      return result.buffer;
+    }
+
+    // No pending data, read from backend
+    return this.backend.read(key);
+  }
+
+  /**
+   * Check if thresholds are exceeded and flush if needed
+   */
+  private async checkThresholds(): Promise<void> {
+    const totalWrites = this.pendingWrites.size + this.pendingAppends.size;
+
+    if (totalWrites >= this.maxPendingWrites || this.pendingBytes >= this.maxPendingBytes) {
+      await this.flush();
+    }
+  }
+
+  /**
+   * Flush all pending writes to the backend
+   */
+  async flush(): Promise<void> {
+    // If already flushing, wait for it to complete
+    if (this.isFlushing && this.flushPromise) {
+      await this.flushPromise;
+      return;
+    }
+
+    if (this.pendingWrites.size === 0 && this.pendingAppends.size === 0) {
+      return;
+    }
+
+    this.isFlushing = true;
+    this.flushPromise = this.doFlush();
+
+    try {
+      await this.flushPromise;
+    } finally {
+      this.isFlushing = false;
+      this.flushPromise = null;
+    }
+  }
+
+  private async doFlush(): Promise<void> {
+    // Capture current pending writes and clear
+    const writes = Array.from(this.pendingWrites.values());
+    const appends = Array.from(this.pendingAppends.entries());
+
+    this.pendingWrites.clear();
+    this.pendingAppends.clear();
+    this.pendingBytes = 0;
+
+    // Execute all writes
+    const promises: Promise<void>[] = [];
+
+    for (const write of writes) {
+      promises.push(this.backend.write(write.key, write.data));
+    }
+
+    for (const [key, chunks] of appends) {
+      // Concatenate all appends for this key
+      let totalSize = 0;
+      for (const chunk of chunks) {
+        totalSize += chunk.length;
+      }
+
+      const combined = new Uint8Array(totalSize);
+      let offset = 0;
+      for (const chunk of chunks) {
+        combined.set(chunk, offset);
+        offset += chunk.length;
+      }
+
+      promises.push(this.backend.append(key, combined));
+    }
+
+    // Wait for all operations to complete
+    await Promise.all(promises);
+  }
+
+  /**
+   * Get statistics about pending writes
+   */
+  getStats(): {
+    pendingWrites: number;
+    pendingAppends: number;
+    pendingBytes: number;
+    maxPendingWrites: number;
+    maxPendingBytes: number;
+  } {
+    return {
+      pendingWrites: this.pendingWrites.size,
+      pendingAppends: this.pendingAppends.size,
+      pendingBytes: this.pendingBytes,
+      maxPendingWrites: this.maxPendingWrites,
+      maxPendingBytes: this.maxPendingBytes
+    };
+  }
+
+  /**
+   * Check if there are pending writes
+   */
+  hasPendingWrites(): boolean {
+    return this.pendingWrites.size > 0 || this.pendingAppends.size > 0;
+  }
+
+  /**
+   * Start auto-flush timer
+   */
+  private startAutoFlush(): void {
+    if (this.flushTimer) return;
+
+    this.flushTimer = setInterval(async () => {
+      if (this.hasPendingWrites()) {
+        await this.flush();
+      }
+    }, this.autoFlushInterval);
+  }
+
+  /**
+   * Stop auto-flush timer
+   */
+  stopAutoFlush(): void {
+    if (this.flushTimer) {
+      clearInterval(this.flushTimer);
+      this.flushTimer = null;
+    }
+  }
+
+  /**
+   * Close the batch writer, flushing any pending writes
+   */
+  async close(): Promise<void> {
+    this.stopAutoFlush();
+    await this.flush();
+  }
+
+  /**
+   * Get the underlying storage backend
+   */
+  getBackend(): StorageBackend {
+    return this.backend;
+  }
+}
+
+/**
+ * Create a batch writer that wraps an existing storage backend
+ */
+export function createBatchWriter(
+  backend: StorageBackend,
+  options?: BatchWriterOptions
+): BatchWriter {
+  return new BatchWriter(backend, options);
+}

package/src/storage/BunStorageBackend.ts

@@ -0,0 +1,161 @@
+/**
+ * Bun Storage Backend
+ *
+ * High-performance file system storage using Bun's native APIs.
+ * Features:
+ * - Auto-initializing (no manual init() required)
+ * - Automatic memory mapping for large files
+ * - Efficient async I/O with O(1) append
+ * - Native TypedArray support
+ */
+
+import { mkdir, readdir, unlink, rm, appendFile } from 'fs/promises';
+import * as path from 'path';
+import type { StorageBackend } from './StorageBackend';
+
+export class BunStorageBackend implements StorageBackend {
+  readonly type = 'bun';
+  private basePath: string;
+  // Cache of directories we've already ensured exist - avoids redundant mkdir calls
+  // Profiling showed 5-10% write overhead from mkdir on every write
+  private dirCache: Set<string> = new Set();
+
+  /**
+   * Create a new Bun storage backend
+   * @param basePath Base directory for all storage operations
+   */
+  constructor(basePath: string = './vectordb_data') {
+    this.basePath = basePath;
+  }
+
+  /**
+   * Ensure directory exists, using cache to avoid redundant mkdir calls
+   */
+  private async ensureDir(dir: string): Promise<void> {
+    if (this.dirCache.has(dir)) return;
+    await mkdir(dir, { recursive: true }).catch(() => {});
+    this.dirCache.add(dir);
+  }
+
+  /**
+   * Get the full path for a key
+   */
+  private getFullPath(key: string): string {
+    return path.join(this.basePath, key);
+  }
+
+  /**
+   * Ensure base directory exists (optional - operations auto-initialize)
+   * @deprecated No longer required - write/append create directories automatically
+   */
+  async init(): Promise<void> {
+    await mkdir(this.basePath, { recursive: true }).catch(() => {});
+  }
+
+  async read(key: string): Promise<ArrayBuffer | null> {
+    const fullPath = this.getFullPath(key);
+    const file = Bun.file(fullPath);
+
+    if (!(await file.exists())) {
+      return null;
+    }
+
+    return file.arrayBuffer();
+  }
+
+  async write(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
+    const fullPath = this.getFullPath(key);
+
+    // Ensure parent directory exists (cached to avoid redundant syscalls)
+    await this.ensureDir(path.dirname(fullPath));
+
+    await Bun.write(fullPath, data);
+  }
+
+  async append(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
+    const fullPath = this.getFullPath(key);
+
+    // Ensure parent directory exists (cached to avoid redundant syscalls)
+    await this.ensureDir(path.dirname(fullPath));
+
+    // Use true O(1) append instead of O(n) read-modify-write
+    const appendData = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
+    await appendFile(fullPath, appendData);
+  }
+
+  async delete(key: string): Promise<void> {
+    const fullPath = this.getFullPath(key);
+    try {
+      await unlink(fullPath);
+    } catch {
+      // File may not exist, which is fine
+    }
+  }
+
+  async exists(key: string): Promise<boolean> {
+    const fullPath = this.getFullPath(key);
+    const file = Bun.file(fullPath);
+    return file.exists();
+  }
+
+  async list(prefix?: string): Promise<string[]> {
+    const searchPath = prefix ? this.getFullPath(prefix) : this.basePath;
+
+    try {
+      const entries = await readdir(searchPath, { recursive: true });
+      return entries.map(entry => {
+        const fullPath = path.join(searchPath, entry);
+        // Return path relative to basePath
+        return path.relative(this.basePath, fullPath);
+      });
+    } catch {
+      return [];
+    }
+  }
+
+  async mkdir(dirPath: string): Promise<void> {
+    const fullPath = this.getFullPath(dirPath);
+    await mkdir(fullPath, { recursive: true }).catch(() => {});
+  }
+
+  /**
+   * Delete all data in the storage directory
+   */
+  async clear(): Promise<void> {
+    await rm(this.basePath, { recursive: true, force: true }).catch(() => {});
+    await mkdir(this.basePath, { recursive: true }).catch(() => {});
+    // Clear directory cache since directories were deleted
+    this.dirCache.clear();
+    this.dirCache.add(this.basePath);
+  }
+
+  /**
+   * Get file size without reading the entire file
+   */
+  async size(key: string): Promise<number> {
+    const fullPath = this.getFullPath(key);
+    const file = Bun.file(fullPath);
+
+    if (!(await file.exists())) {
+      return 0;
+    }
+
+    return file.size;
+  }
+
+  /**
+   * Read a file as a stream (for large files)
+   */
+  stream(key: string): ReadableStream<Uint8Array> | null {
+    const fullPath = this.getFullPath(key);
+    const file = Bun.file(fullPath);
+    return file.stream();
+  }
+
+  /**
+   * Get the base path
+   */
+  getBasePath(): string {
+    return this.basePath;
+  }
+}

package/src/storage/MemoryBackend.ts

@@ -0,0 +1,120 @@
+/**
+ * In-Memory Storage Backend
+ *
+ * Stores all data in memory using a Map. Useful for:
+ * - Testing without file system
+ * - Small datasets that fit in memory
+ * - Browser environments without OPFS/IndexedDB
+ * - Temporary caches
+ */
+
+import type { StorageBackend } from './StorageBackend';
+
+export class MemoryBackend implements StorageBackend {
+  readonly type = 'memory';
+  private storage: Map<string, ArrayBuffer> = new Map();
+  private directories: Set<string> = new Set();
+
+  constructor() {
+    this.directories.add(''); // Root directory always exists
+  }
+
+  async read(key: string): Promise<ArrayBuffer | null> {
+    const data = this.storage.get(key);
+    if (!data) return null;
+    // Return a copy to prevent mutation
+    return data.slice(0);
+  }
+
+  async write(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
+    // Store a copy to prevent external mutation
+    const buffer = data instanceof ArrayBuffer ? data : data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
+    this.storage.set(key, buffer.slice(0));
+  }
+
+  async append(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
+    const appendData = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
+
+    const existing = this.storage.get(key);
+    if (existing) {
+      const existingArray = new Uint8Array(existing);
+      const combined = new Uint8Array(existingArray.length + appendData.length);
+      combined.set(existingArray, 0);
+      combined.set(appendData, existingArray.length);
+      this.storage.set(key, combined.buffer.slice(0) as ArrayBuffer);
+    } else {
+      this.storage.set(key, appendData.buffer.slice(appendData.byteOffset, appendData.byteOffset + appendData.byteLength) as ArrayBuffer);
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    this.storage.delete(key);
+  }
+
+  async exists(key: string): Promise<boolean> {
+    return this.storage.has(key) || this.directories.has(key);
+  }
+
+  async list(prefix?: string): Promise<string[]> {
+    const keys: string[] = [];
+    for (const key of this.storage.keys()) {
+      if (!prefix || key.startsWith(prefix)) {
+        keys.push(key);
+      }
+    }
+    return keys;
+  }
+
+  async mkdir(path: string): Promise<void> {
+    this.directories.add(path);
+    // Also add parent directories
+    const parts = path.split('/');
+    for (let i = 1; i <= parts.length; i++) {
+      this.directories.add(parts.slice(0, i).join('/'));
+    }
+  }
+
+  /**
+   * Clear all data
+   */
+  clear(): void {
+    this.storage.clear();
+    this.directories.clear();
+    this.directories.add('');
+  }
+
+  /**
+   * Get memory usage statistics
+   */
+  getStats(): { keyCount: number; totalBytes: number } {
+    let totalBytes = 0;
+    for (const value of this.storage.values()) {
+      totalBytes += value.byteLength;
+    }
+    return {
+      keyCount: this.storage.size,
+      totalBytes,
+    };
+  }
+
+  /**
+   * Export all data as a serializable object
+   * Useful for debugging or saving state
+   */
+  export(): Record<string, ArrayBuffer> {
+    const result: Record<string, ArrayBuffer> = {};
+    for (const [key, value] of this.storage.entries()) {
+      result[key] = value.slice(0);
+    }
+    return result;
+  }
+
+  /**
+   * Import data from a serializable object
+   */
+  import(data: Record<string, ArrayBuffer>): void {
+    for (const [key, value] of Object.entries(data)) {
+      this.storage.set(key, value.slice(0));
+    }
+  }
+}