verso-db 0.1.5 → 0.2.0

package/src/storage/BatchWriter.ts

@@ -1,420 +0,0 @@
-/**
- * Batch Write Coalescing for Storage Backends
- *
- * Buffers writes in memory and flushes them in batches to reduce I/O operations.
- * This is especially beneficial for:
- * - OPFS where each write has overhead
- * - IndexedDB where transactions have cost
- * - Network storage where round-trips are expensive
- *
- * Features:
- * - Configurable flush thresholds (count and size)
- * - Automatic flushing when thresholds are reached
- * - Explicit flush for durability guarantees
- * - Coalesces multiple writes to the same key
- */
-
-import type { StorageBackend } from './StorageBackend';
-
-export interface BatchWriterOptions {
-  /** Maximum number of pending writes before auto-flush (default: 100) */
-  maxPendingWrites?: number;
-  /** Maximum total size of pending data in bytes before auto-flush (default: 1MB) */
-  maxPendingBytes?: number;
-  /** Auto-flush interval in milliseconds (0 = disabled, default: 0) */
-  autoFlushInterval?: number;
-}
-
-interface PendingWrite {
-  key: string;
-  data: Uint8Array;
-  append: boolean;
-}
-
-/**
- * BatchWriter - Coalesces multiple writes into batched flushes
- */
-export class BatchWriter {
-  private backend: StorageBackend;
-  private pendingWrites: Map<string, PendingWrite> = new Map();
-  private pendingAppends: Map<string, Uint8Array[]> = new Map();
-  private pendingBytes: number = 0;
-  private maxPendingWrites: number;
-  private maxPendingBytes: number;
-  private autoFlushInterval: number;
-  private flushTimer: ReturnType<typeof setTimeout> | null = null;
-  private isFlushing: boolean = false;
-  private flushPromise: Promise<void> | null = null;
-
-  constructor(backend: StorageBackend, options: BatchWriterOptions = {}) {
-    this.backend = backend;
-    this.maxPendingWrites = options.maxPendingWrites ?? 100;
-    this.maxPendingBytes = options.maxPendingBytes ?? 1024 * 1024; // 1MB
-    this.autoFlushInterval = options.autoFlushInterval ?? 0;
-
-    if (this.autoFlushInterval > 0) {
-      this.startAutoFlush();
-    }
-  }
-
-  /**
-   * Write data to a key (buffered)
-   * Multiple writes to the same key will coalesce to the last value
-   */
-  async write(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
-    const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
-    const bytesCopy = new Uint8Array(bytes.length);
-    bytesCopy.set(bytes);
-
-    // If there's already a pending write, subtract its size
-    const existing = this.pendingWrites.get(key);
-    if (existing) {
-      this.pendingBytes -= existing.data.length;
-    }
-
-    // Clear any pending appends for this key (write overwrites)
-    const existingAppends = this.pendingAppends.get(key);
-    if (existingAppends) {
-      for (const append of existingAppends) {
-        this.pendingBytes -= append.length;
-      }
-      this.pendingAppends.delete(key);
-    }
-
-    this.pendingWrites.set(key, {
-      key,
-      data: bytesCopy,
-      append: false
-    });
-    this.pendingBytes += bytesCopy.length;
-
-    await this.checkThresholds();
-  }
-
-  /**
-   * Append data to a key (buffered)
-   * Multiple appends to the same key will be concatenated
-   */
-  async append(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
-    const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
-    const bytesCopy = new Uint8Array(bytes.length);
-    bytesCopy.set(bytes);
-
-    // If there's a pending write, append to it instead
-    const existingWrite = this.pendingWrites.get(key);
-    if (existingWrite) {
-      // Concatenate with existing write
-      const newData = new Uint8Array(existingWrite.data.length + bytesCopy.length);
-      newData.set(existingWrite.data);
-      newData.set(bytesCopy, existingWrite.data.length);
-      this.pendingBytes -= existingWrite.data.length;
-      this.pendingBytes += newData.length;
-      existingWrite.data = newData;
-    } else {
-      // Add to pending appends
-      let appends = this.pendingAppends.get(key);
-      if (!appends) {
-        appends = [];
-        this.pendingAppends.set(key, appends);
-      }
-      appends.push(bytesCopy);
-      this.pendingBytes += bytesCopy.length;
-    }
-
-    await this.checkThresholds();
-  }
-
-  /**
-   * Delete a key (buffered)
-   * Clears any pending writes/appends for this key
-   */
-  async delete(key: string): Promise<void> {
-    // Clear pending operations
-    const existing = this.pendingWrites.get(key);
-    if (existing) {
-      this.pendingBytes -= existing.data.length;
-      this.pendingWrites.delete(key);
-    }
-
-    const existingAppends = this.pendingAppends.get(key);
-    if (existingAppends) {
-      for (const append of existingAppends) {
-        this.pendingBytes -= append.length;
-      }
-      this.pendingAppends.delete(key);
-    }
-
-    // Perform delete immediately (can't batch deletes effectively)
-    await this.backend.delete(key);
-  }
-
-  /**
-   * Read data from a key
-   * Returns pending data if available, otherwise reads from backend
-   */
-  async read(key: string): Promise<ArrayBuffer | null> {
-    // Check for pending write
-    const pending = this.pendingWrites.get(key);
-    if (pending) {
-      const buffer = pending.data.buffer.slice(
-        pending.data.byteOffset,
-        pending.data.byteOffset + pending.data.length
-      );
-      return buffer as ArrayBuffer;
-    }
-
-    // Check for pending appends - need to combine with existing data.
-    // Re-validate pending state after backend read to avoid returning stale
-    // appends that may have already been flushed.
-    while (true) {
-      const appends = this.pendingAppends.get(key);
-      if (!appends || appends.length === 0) {
-        // No pending data, read from backend
-        return this.backend.read(key);
-      }
-
-      const appendCount = appends.length;
-      const existing = await this.backend.read(key);
-
-      // A write supersedes appends and should take precedence if it appeared
-      // while we were waiting on backend I/O.
-      const latestWrite = this.pendingWrites.get(key);
-      if (latestWrite) {
-        const buffer = latestWrite.data.buffer.slice(
-          latestWrite.data.byteOffset,
-          latestWrite.data.byteOffset + latestWrite.data.length
-        );
-        return buffer as ArrayBuffer;
-      }
-
-      const latestAppends = this.pendingAppends.get(key);
-      if (!latestAppends || latestAppends !== appends || latestAppends.length !== appendCount) {
-        // Pending append state changed (flushed/new appends/etc.), retry with
-        // a fresh snapshot to avoid duplicate or stale data.
-        continue;
-      }
-
-      const existingBytes = existing ? new Uint8Array(existing) : new Uint8Array(0);
-
-      // Calculate total size
-      let totalSize = existingBytes.length;
-      for (let i = 0; i < appendCount; i++) {
-        totalSize += latestAppends[i].length;
-      }
-
-      // Combine
-      const result = new Uint8Array(totalSize);
-      result.set(existingBytes);
-      let offset = existingBytes.length;
-      for (let i = 0; i < appendCount; i++) {
-        const append = latestAppends[i];
-        result.set(append, offset);
-        offset += append.length;
-      }
-
-      return result.buffer;
-    }
-  }
-
-  /**
-   * Check if thresholds are exceeded and flush if needed
-   */
-  private async checkThresholds(): Promise<void> {
-    const totalWrites = this.pendingWrites.size + this.pendingAppends.size;
-
-    if (totalWrites >= this.maxPendingWrites || this.pendingBytes >= this.maxPendingBytes) {
-      await this.flush();
-    }
-  }
-
-  /**
-   * Flush all pending writes to the backend
-   */
-  async flush(): Promise<void> {
-    // If already flushing, wait for it to complete then re-check
-    if (this.isFlushing && this.flushPromise) {
-      await this.flushPromise;
-      // Re-check: writes may have arrived during the previous flush
-      if (this.pendingWrites.size === 0 && this.pendingAppends.size === 0) {
-        return;
-      }
-    }
-
-    if (this.pendingWrites.size === 0 && this.pendingAppends.size === 0) {
-      return;
-    }
-
-    this.isFlushing = true;
-    this.flushPromise = this.doFlush();
-
-    try {
-      await this.flushPromise;
-    } finally {
-      this.isFlushing = false;
-      this.flushPromise = null;
-    }
-  }
-
-  private async doFlush(): Promise<void> {
-    // Capture current pending writes and clear
-    const writes = Array.from(this.pendingWrites.entries());
-    const appendEntries = Array.from(this.pendingAppends.entries());
-
-    this.pendingWrites.clear();
-    this.pendingAppends.clear();
-    this.pendingBytes = 0;
-
-    // Execute writes and appends sequentially per-key to avoid partial-failure
-    // ambiguity with Promise.all (where we can't tell which appends succeeded,
-    // risking duplicate data on retry since append is not idempotent).
-    const failedWrites: Array<[string, PendingWrite]> = [];
-    const failedAppends: Array<[string, Uint8Array[]]> = [];
-    let firstError: unknown = null;
-
-    for (const [key, write] of writes) {
-      try {
-        await this.backend.write(write.key, write.data);
-      } catch (err) {
-        if (!firstError) firstError = err;
-        failedWrites.push([key, write]);
-      }
-    }
-
-    for (const [key, chunks] of appendEntries) {
-      // Concatenate all appends for this key
-      let totalSize = 0;
-      for (const chunk of chunks) {
-        totalSize += chunk.length;
-      }
-
-      const combined = new Uint8Array(totalSize);
-      let offset = 0;
-      for (const chunk of chunks) {
-        combined.set(chunk, offset);
-        offset += chunk.length;
-      }
-
-      try {
-        await this.backend.append(key, combined);
-      } catch (err) {
-        if (!firstError) firstError = err;
-        failedAppends.push([key, chunks]);
-      }
-    }
-
-    if (firstError) {
-      // Only restore entries that actually failed — successful ones are persisted.
-      let restoredBytes = 0;
-      for (const [key, write] of failedWrites) {
-        if (!this.pendingWrites.has(key)) {
-          this.pendingWrites.set(key, write);
-          restoredBytes += write.data.length;
-        }
-      }
-      for (const [key, chunks] of failedAppends) {
-        // If a newer pending write exists for this key, it supersedes old appends.
-        // Restoring those stale appends would violate write ordering and corrupt data.
-        if (this.pendingWrites.has(key)) {
-          continue;
-        }
-
-        const existing = this.pendingAppends.get(key);
-        if (existing) {
-          // Prepend old chunks before any new ones
-          this.pendingAppends.set(key, [...chunks, ...existing]);
-        } else {
-          this.pendingAppends.set(key, chunks);
-        }
-        for (const chunk of chunks) {
-          restoredBytes += chunk.length;
-        }
-      }
-      this.pendingBytes += restoredBytes;
-      throw firstError;
-    }
-  }
-
-  /**
-   * Get statistics about pending writes
-   */
-  getStats(): {
-    pendingWrites: number;
-    pendingAppends: number;
-    pendingBytes: number;
-    maxPendingWrites: number;
-    maxPendingBytes: number;
-  } {
-    return {
-      pendingWrites: this.pendingWrites.size,
-      pendingAppends: this.pendingAppends.size,
-      pendingBytes: this.pendingBytes,
-      maxPendingWrites: this.maxPendingWrites,
-      maxPendingBytes: this.maxPendingBytes
-    };
-  }
-
-  /**
-   * Check if there are pending writes
-   */
-  hasPendingWrites(): boolean {
-    return this.pendingWrites.size > 0 || this.pendingAppends.size > 0;
-  }
-
-  /**
-   * Start auto-flush timer
-   */
-  private startAutoFlush(): void {
-    if (this.flushTimer) return;
-
-    this.flushTimer = setInterval(async () => {
-      if (this.hasPendingWrites()) {
-        try {
-          await this.flush();
-        } catch {
-          // Preserve pending data and retry on next interval/explicit flush.
-        }
-      }
-    }, this.autoFlushInterval);
-
-    // Allow the process to exit even if the timer is still running.
-    // Without this, the timer keeps the event loop alive indefinitely.
-    if (typeof this.flushTimer === 'object' && 'unref' in this.flushTimer) {
-      (this.flushTimer as any).unref();
-    }
-  }
-
-  /**
-   * Stop auto-flush timer
-   */
-  stopAutoFlush(): void {
-    if (this.flushTimer) {
-      clearInterval(this.flushTimer);
-      this.flushTimer = null;
-    }
-  }
-
-  /**
-   * Close the batch writer, flushing any pending writes
-   */
-  async close(): Promise<void> {
-    this.stopAutoFlush();
-    await this.flush();
-  }
-
-  /**
-   * Get the underlying storage backend
-   */
-  getBackend(): StorageBackend {
-    return this.backend;
-  }
-}
-
-/**
- * Create a batch writer that wraps an existing storage backend
- */
-export function createBatchWriter(
-  backend: StorageBackend,
-  options?: BatchWriterOptions
-): BatchWriter {
-  return new BatchWriter(backend, options);
-}

package/src/storage/BunStorageBackend.ts

@@ -1,199 +0,0 @@
-/**
- * Bun Storage Backend
- *
- * High-performance file system storage using Bun's native APIs.
- * Features:
- * - Auto-initializing (no manual init() required)
- * - Automatic memory mapping for large files
- * - Efficient async I/O with O(1) append
- * - Native TypedArray support
- */
-
-import { mkdir, readdir, unlink, rm, appendFile, rename } from 'fs/promises';
-import * as path from 'path';
-import type { StorageBackend } from './StorageBackend';
-
-export class BunStorageBackend implements StorageBackend {
-  readonly type = 'bun';
-  private basePath: string;
-  // Cache of directories we've already ensured exist - avoids redundant mkdir calls
-  // Profiling showed 5-10% write overhead from mkdir on every write
-  private dirCache: Set<string> = new Set();
-
-  /**
-   * Create a new Bun storage backend
-   * @param basePath Base directory for all storage operations
-   */
-  constructor(basePath: string = './vectordb_data') {
-    this.basePath = basePath;
-  }
-
-  /**
-   * Ensure directory exists, using cache to avoid redundant mkdir calls
-   */
-  private async ensureDir(dir: string): Promise<void> {
-    if (this.dirCache.has(dir)) return;
-    try {
-      await mkdir(dir, { recursive: true });
-      this.dirCache.add(dir);
-    } catch (err: unknown) {
-      // EEXIST is fine - directory already exists
-      if (err instanceof Error && (err as NodeJS.ErrnoException).code === 'EEXIST') {
-        this.dirCache.add(dir);
-        return;
-      }
-      // Re-throw real errors (permissions, disk full, etc.) so callers get
-      // a clear failure instead of a confusing "file not found" on the
-      // subsequent write attempt.
-      throw err;
-    }
-  }
-
-  /**
-   * Get the full path for a key.
-   * Validates that the resolved path stays within basePath to prevent path traversal.
-   */
-  private getFullPath(key: string): string {
-    const fullPath = path.resolve(this.basePath, key);
-    const resolvedBase = path.resolve(this.basePath);
-    if (!fullPath.startsWith(resolvedBase + path.sep) && fullPath !== resolvedBase) {
-      throw new Error(`Path traversal detected: key '${key}' resolves outside storage directory`);
-    }
-    return fullPath;
-  }
-
-  /**
-   * Ensure base directory exists (optional - operations auto-initialize)
-   * @deprecated No longer required - write/append create directories automatically
-   */
-  async init(): Promise<void> {
-    await mkdir(this.basePath, { recursive: true }).catch(() => {});
-  }
-
-  async read(key: string): Promise<ArrayBuffer | null> {
-    const fullPath = this.getFullPath(key);
-    const file = Bun.file(fullPath);
-
-    if (!(await file.exists())) {
-      return null;
-    }
-
-    return file.arrayBuffer();
-  }
-
-  async write(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
-    const fullPath = this.getFullPath(key);
-
-    // Ensure parent directory exists (cached to avoid redundant syscalls)
-    await this.ensureDir(path.dirname(fullPath));
-
-    // Atomic write: write to temp file then rename.
-    // If the process crashes mid-write, the original file is untouched.
-    const tmpPath = fullPath + '.tmp';
-    await Bun.write(tmpPath, data);
-    await rename(tmpPath, fullPath);
-  }
-
-  async append(key: string, data: ArrayBuffer | Uint8Array): Promise<void> {
-    const fullPath = this.getFullPath(key);
-
-    // Ensure parent directory exists (cached to avoid redundant syscalls)
-    await this.ensureDir(path.dirname(fullPath));
-
-    // Use true O(1) append instead of O(n) read-modify-write
-    const appendData = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
-    await appendFile(fullPath, appendData);
-  }
-
-  async delete(key: string): Promise<void> {
-    const fullPath = this.getFullPath(key);
-    try {
-      await unlink(fullPath);
-    } catch (err: any) {
-      // File not existing is fine — all other errors should propagate
-      if (err?.code !== 'ENOENT') {
-        throw err;
-      }
-    }
-  }
-
-  async exists(key: string): Promise<boolean> {
-    const fullPath = this.getFullPath(key);
-    const file = Bun.file(fullPath);
-    return file.exists();
-  }
-
-  async list(prefix?: string): Promise<string[]> {
-    const searchPath = prefix ? this.getFullPath(prefix) : this.basePath;
-
-    try {
-      const entries = await readdir(searchPath, { recursive: true, withFileTypes: true });
-      const results: string[] = [];
-      for (const entry of entries) {
-        if (entry.isFile()) {
-          // parentPath is available on Dirent when using recursive readdir
-          const entryDir = (entry as any).parentPath ?? (entry as any).path ?? searchPath;
-          const fullPath = path.join(entryDir, entry.name);
-          results.push(path.relative(this.basePath, fullPath));
-        }
-      }
-      return results;
-    } catch {
-      return [];
-    }
-  }
-
-  async mkdir(dirPath: string): Promise<void> {
-    const fullPath = this.getFullPath(dirPath);
-    try {
-      await mkdir(fullPath, { recursive: true });
-    } catch (err: unknown) {
-      if (err instanceof Error && (err as NodeJS.ErrnoException).code === 'EEXIST') {
-        return;
-      }
-      throw err;
-    }
-  }
-
-  /**
-   * Delete all data in the storage directory
-   */
-  async clear(): Promise<void> {
-    await rm(this.basePath, { recursive: true, force: true }).catch(() => {});
-    await mkdir(this.basePath, { recursive: true }).catch(() => {});
-    // Clear directory cache since directories were deleted
-    this.dirCache.clear();
-    this.dirCache.add(path.resolve(this.basePath));
-  }
-
-  /**
-   * Get file size without reading the entire file
-   */
-  async size(key: string): Promise<number> {
-    const fullPath = this.getFullPath(key);
-    const file = Bun.file(fullPath);
-
-    if (!(await file.exists())) {
-      return 0;
-    }
-
-    return file.size;
-  }
-
-  /**
-   * Read a file as a stream (for large files).
-   * Note: Callers should check exists() before calling stream(),
-   * as Bun.file().stream() on a non-existent file will error at read time.
-   */
-  stream(key: string): ReadableStream<Uint8Array> | null {
-    const fullPath = this.getFullPath(key);
-    return Bun.file(fullPath).stream();
-  }
-
-  /**
-   * Get the base path
-   */
-  getBasePath(): string {
-    return this.basePath;
-  }
-}