sehawq.db 3.0.0 → 4.0.0

package/src/performance/Cache.js

@@ -0,0 +1,339 @@
+/**
+ * Smart Cache System - Makes everything faster with magic 🪄
+ * 
+ * Implements LRU cache with TTL and memory management
+ * Because waiting is not an option in 2024 ⚡
+ */
+
+class Cache {
+  constructor(options = {}) {
+    this.options = {
+      maxSize: 1000,
+      ttl: 5 * 60 * 1000, // 5 minutes default
+      cleanupInterval: 60 * 1000, // Clean every minute
+      ...options
+    };
+
+    // Double-linked list for LRU + HashMap for O(1) access
+    this.cache = new Map();
+    this.head = { key: null, value: null, next: null, prev: null, expires: 0 };
+    this.tail = { key: null, value: null, next: null, prev: this.head, expires: 0 };
+    this.head.next = this.tail;
+    
+    this.stats = {
+      hits: 0,
+      misses: 0,
+      evictions: 0,
+      sets: 0,
+      gets: 0,
+      memoryUsage: 0
+    };
+
+    this._startCleanupInterval();
+  }
+
+  /**
+   * Get value from cache - moves item to front (most recently used)
+   */
+  get(key) {
+    this.stats.gets++;
+
+    const node = this.cache.get(key);
+    
+    // Check if exists and not expired
+    if (!node || this._isExpired(node)) {
+      this.stats.misses++;
+      
+      if (node) {
+        // Remove expired node
+        this._removeNode(node);
+        this.cache.delete(key);
+        this.stats.evictions++;
+      }
+      
+      return undefined;
+    }
+
+    // Move to front (most recently used)
+    this._moveToFront(node);
+    this.stats.hits++;
+
+    return node.value;
+  }
+
+  /**
+   * Set value in cache - handles LRU eviction if needed
+   */
+  set(key, value, ttl = this.options.ttl) {
+    this.stats.sets++;
+
+    let node = this.cache.get(key);
+    const expires = Date.now() + ttl;
+
+    if (node) {
+      // Update existing node
+      node.value = value;
+      node.expires = expires;
+      this._moveToFront(node);
+    } else {
+      // Create new node
+      node = {
+        key,
+        value,
+        expires,
+        prev: this.head,
+        next: this.head.next
+      };
+
+      // Add to cache and linked list
+      this.cache.set(key, node);
+      this.head.next.prev = node;
+      this.head.next = node;
+
+      // Evict if over capacity
+      if (this.cache.size > this.options.maxSize) {
+        this._evictLRU();
+      }
+    }
+
+    // Update memory usage stats
+    this._updateMemoryStats();
+
+    return true;
+  }
+
+  /**
+   * Check if key exists in cache (without updating LRU)
+   */
+  has(key) {
+    const node = this.cache.get(key);
+    return !!(node && !this._isExpired(node));
+  }
+
+  /**
+   * Delete key from cache
+   */
+  delete(key) {
+    const node = this.cache.get(key);
+    if (node) {
+      this._removeNode(node);
+      this.cache.delete(key);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Clear entire cache
+   */
+  clear() {
+    this.cache.clear();
+    this.head.next = this.tail;
+    this.tail.prev = this.head;
+    this.stats.memoryUsage = 0;
+  }
+
+  /**
+   * Get cache size (number of items)
+   */
+  size() {
+    return this.cache.size;
+  }
+
+  /**
+   * Get all keys in cache (for debugging)
+   */
+  keys() {
+    const keys = [];
+    let node = this.head.next;
+    
+    while (node !== this.tail) {
+      if (!this._isExpired(node)) {
+        keys.push(node.key);
+      }
+      node = node.next;
+    }
+    
+    return keys;
+  }
+
+  /**
+   * Get all values in cache (for debugging)
+   */
+  values() {
+    const values = [];
+    let node = this.head.next;
+    
+    while (node !== this.tail) {
+      if (!this._isExpired(node)) {
+        values.push(node.value);
+      }
+      node = node.next;
+    }
+    
+    return values;
+  }
+
+  /**
+   * Get cache statistics
+   */
+  getStats() {
+    const hitRate = this.stats.hits + this.stats.misses > 0 
+      ? (this.stats.hits / (this.stats.hits + this.stats.misses) * 100).toFixed(2)
+      : 0;
+
+    return {
+      ...this.stats,
+      hitRate: `${hitRate}%`,
+      size: this.cache.size,
+      maxSize: this.options.maxSize,
+      utilization: `${((this.cache.size / this.options.maxSize) * 100).toFixed(1)}%`
+    };
+  }
+
+  /**
+   * Move node to front of LRU list
+   */
+  _moveToFront(node) {
+    // Remove from current position
+    this._removeNode(node);
+    
+    // Insert after head
+    node.prev = this.head;
+    node.next = this.head.next;
+    this.head.next.prev = node;
+    this.head.next = node;
+  }
+
+  /**
+   * Remove node from linked list
+   */
+  _removeNode(node) {
+    node.prev.next = node.next;
+    node.next.prev = node.prev;
+  }
+
+  /**
+   * Evict least recently used item
+   */
+  _evictLRU() {
+    const lruNode = this.tail.prev;
+    
+    if (lruNode !== this.head) {
+      this._removeNode(lruNode);
+      this.cache.delete(lruNode.key);
+      this.stats.evictions++;
+      this._updateMemoryStats();
+    }
+  }
+
+  /**
+   * Check if node has expired
+   */
+  _isExpired(node) {
+    return Date.now() > node.expires;
+  }
+
+  /**
+   * Start periodic cleanup of expired items
+   */
+  _startCleanupInterval() {
+    setInterval(() => {
+      this._cleanupExpired();
+    }, this.options.cleanupInterval);
+  }
+
+  /**
+   * Remove all expired items from cache
+   */
+  _cleanupExpired() {
+    const now = Date.now();
+    let node = this.head.next;
+    let expiredCount = 0;
+
+    while (node !== this.tail) {
+      const nextNode = node.next;
+      
+      if (now > node.expires) {
+        this._removeNode(node);
+        this.cache.delete(node.key);
+        expiredCount++;
+      }
+      
+      node = nextNode;
+    }
+
+    if (expiredCount > 0 && this.options.debug) {
+      console.log(`🧹 Cache cleanup: removed ${expiredCount} expired items`);
+    }
+
+    this._updateMemoryStats();
+  }
+
+  /**
+   * Estimate memory usage (rough calculation)
+   */
+  _updateMemoryStats() {
+    let totalSize = 0;
+    
+    for (const [key, node] of this.cache) {
+      // Rough estimation: key size + value size (stringify for simplicity)
+      totalSize += Buffer.byteLength(key, 'utf8');
+      totalSize += Buffer.byteLength(JSON.stringify(node.value), 'utf8');
+    }
+    
+    this.stats.memoryUsage = totalSize;
+  }
+
+  /**
+   * Pre-warm cache with data
+   */
+  async warmup(dataMap, ttl = this.options.ttl) {
+    for (const [key, value] of Object.entries(dataMap)) {
+      this.set(key, value, ttl);
+    }
+    
+    if (this.options.debug) {
+      console.log(`🔥 Cache warmup complete: ${Object.keys(dataMap).length} items`);
+    }
+  }
+
+  /**
+   * Get cache snapshot for debugging
+   */
+  getSnapshot() {
+    const snapshot = {};
+    let node = this.head.next;
+    
+    while (node !== this.tail) {
+      if (!this._isExpired(node)) {
+        snapshot[node.key] = {
+          value: node.value,
+          expiresIn: node.expires - Date.now(),
+          ttl: this.options.ttl
+        };
+      }
+      node = node.next;
+    }
+    
+    return snapshot;
+  }
+
+  /**
+   * Resize cache (useful for dynamic memory management)
+   */
+  resize(newSize) {
+    this.options.maxSize = newSize;
+    
+    // Evict excess items if needed
+    while (this.cache.size > newSize) {
+      this._evictLRU();
+    }
+    
+    if (this.options.debug) {
+      console.log(`📏 Cache resized: ${newSize} items`);
+    }
+  }
+}
+
+module.exports = Cache;

package/src/performance/LazyLoader.js

@@ -0,0 +1,355 @@
+/**
+ * LazyLoader - Loads data only when needed, saves memory like a boss 💾
+ * 
+ * Why load everything when you only need some things?
+ * This made our memory usage drop faster than my grades in college 😅
+ */
+
+class LazyLoader {
+  constructor(storage, options = {}) {
+    this.storage = storage;
+    this.options = {
+      chunkSize: 100, // Items per chunk
+      prefetch: true, // Load next chunk in background
+      maxLoadedChunks: 5, // Keep this many chunks in memory
+      autoUnload: true, // Unload old chunks automatically
+      ...options
+    };
+
+    // Chunk management
+    this.chunks = new Map(); // chunkIndex -> data
+    this.chunkIndex = new Map(); // key -> chunkIndex
+    this.accessHistory = []; // LRU for chunks
+    this.loadedChunksCount = 0;
+
+    // Performance tracking
+    this.stats = {
+      chunksLoaded: 0,
+      chunksUnloaded: 0,
+      keysLoaded: 0,
+      memorySaved: 0,
+      cacheHits: 0,
+      cacheMisses: 0,
+      prefetchHits: 0
+    };
+
+    this._initialized = false;
+  }
+
+  /**
+   * Initialize the lazy loader - build chunk index
+   */
+  async initialize(allKeys) {
+    if (this._initialized) return;
+
+    // Build chunk index from all keys
+    let chunkIndex = 0;
+    let currentChunkSize = 0;
+
+    for (const key of allKeys) {
+      this.chunkIndex.set(key, chunkIndex);
+      currentChunkSize++;
+
+      if (currentChunkSize >= this.options.chunkSize) {
+        chunkIndex++;
+        currentChunkSize = 0;
+      }
+    }
+
+    this.totalChunks = chunkIndex + 1;
+    this._initialized = true;
+
+    if (this.options.debug) {
+      console.log(`📊 LazyLoader: ${allKeys.length} keys in ${this.totalChunks} chunks`);
+    }
+  }
+
+  /**
+   * Get value by key - loads chunk if needed
+   */
+  async get(key) {
+    if (!this._initialized) {
+      throw new Error('LazyLoader not initialized. Call initialize() first.');
+    }
+
+    const chunkIndex = this.chunkIndex.get(key);
+    
+    if (chunkIndex === undefined) {
+      this.stats.cacheMisses++;
+      return undefined; // Key not found
+    }
+
+    // Check if chunk is already loaded
+    if (this.chunks.has(chunkIndex)) {
+      this.stats.cacheHits++;
+      this._updateAccessHistory(chunkIndex);
+      return this.chunks.get(chunkIndex).get(key);
+    }
+
+    this.stats.cacheMisses++;
+
+    // Load the chunk
+    await this._loadChunk(chunkIndex);
+    
+    // Prefetch adjacent chunks in background
+    if (this.options.prefetch) {
+      this._prefetchAdjacentChunks(chunkIndex);
+    }
+
+    const chunk = this.chunks.get(chunkIndex);
+    return chunk ? chunk.get(key) : undefined;
+  }
+
+  /**
+   * Set value - updates chunk if loaded, or defers to storage
+   */
+  async set(key, value) {
+    const chunkIndex = this.chunkIndex.get(key);
+    
+    if (chunkIndex !== undefined && this.chunks.has(chunkIndex)) {
+      // Update in loaded chunk
+      this.chunks.get(chunkIndex).set(key, value);
+      this._updateAccessHistory(chunkIndex);
+    }
+    
+    // Always update in storage
+    await this.storage.set(key, value);
+  }
+
+  /**
+   * Check if key exists (without loading chunk)
+   */
+  has(key) {
+    return this.chunkIndex.has(key);
+  }
+
+  /**
+   * Get all keys (without loading data)
+   */
+  getAllKeys() {
+    return Array.from(this.chunkIndex.keys());
+  }
+
+  /**
+   * Load a specific chunk into memory
+   */
+  async _loadChunk(chunkIndex) {
+    // Unload least recently used chunks if we're at the limit
+    if (this.loadedChunksCount >= this.options.maxLoadedChunks) {
+      await this._unloadLRUChunk();
+    }
+
+    // Get all keys in this chunk
+    const chunkKeys = [];
+    for (const [key, index] of this.chunkIndex) {
+      if (index === chunkIndex) {
+        chunkKeys.push(key);
+      }
+    }
+
+    // Load data for these keys
+    const chunkData = new Map();
+    for (const key of chunkKeys) {
+      const value = await this.storage.get(key);
+      if (value !== undefined) {
+        chunkData.set(key, value);
+      }
+    }
+
+    // Store the chunk
+    this.chunks.set(chunkIndex, chunkData);
+    this._updateAccessHistory(chunkIndex);
+    this.loadedChunksCount++;
+    this.stats.chunksLoaded++;
+    this.stats.keysLoaded += chunkData.size;
+
+    if (this.options.debug) {
+      console.log(`📁 Loaded chunk ${chunkIndex} with ${chunkData.size} items`);
+    }
+
+    return chunkData;
+  }
+
+  /**
+   * Unload least recently used chunk
+   */
+  async _unloadLRUChunk() {
+    if (this.accessHistory.length === 0) return;
+
+    const lruChunkIndex = this.accessHistory[0];
+    await this._unloadChunk(lruChunkIndex);
+  }
+
+  /**
+   * Unload specific chunk
+   */
+  async _unloadChunk(chunkIndex) {
+    const chunk = this.chunks.get(chunkIndex);
+    if (!chunk) return;
+
+    // Calculate memory saved (rough estimate)
+    let chunkSize = 0;
+    for (const [key, value] of chunk) {
+      chunkSize += this._estimateSize(key) + this._estimateSize(value);
+    }
+
+    this.chunks.delete(chunkIndex);
+    this.accessHistory = this.accessHistory.filter(idx => idx !== chunkIndex);
+    this.loadedChunksCount--;
+    this.stats.chunksUnloaded++;
+    this.stats.memorySaved += chunkSize;
+
+    if (this.options.debug) {
+      console.log(`🗑️  Unloaded chunk ${chunkIndex} (saved ~${chunkSize} bytes)`);
+    }
+  }
+
+  /**
+   * Prefetch chunks around the currently loaded one
+   */
+  _prefetchAdjacentChunks(currentChunkIndex) {
+    const prefetchIndices = [
+      currentChunkIndex + 1, // Next chunk
+      currentChunkIndex - 1  // Previous chunk
+    ].filter(index => index >= 0 && index < this.totalChunks && !this.chunks.has(index));
+
+    // Prefetch in background (don't await)
+    for (const index of prefetchIndices) {
+      this._loadChunk(index).then(() => {
+        this.stats.prefetchHits++;
+      }).catch(error => {
+        console.error(`Prefetch failed for chunk ${index}:`, error);
+      });
+    }
+  }
+
+  /**
+   * Update access history for LRU tracking
+   */
+  _updateAccessHistory(chunkIndex) {
+    // Remove existing entry
+    this.accessHistory = this.accessHistory.filter(idx => idx !== chunkIndex);
+    // Add to end (most recently used)
+    this.accessHistory.push(chunkIndex);
+  }
+
+  /**
+   * Estimate size of an object in bytes
+   */
+  _estimateSize(obj) {
+    if (obj === null || obj === undefined) return 0;
+    
+    switch (typeof obj) {
+      case 'string':
+        return obj.length * 2; // 2 bytes per character
+      case 'number':
+        return 8; // 8 bytes for number
+      case 'boolean':
+        return 4; // 4 bytes for boolean
+      case 'object':
+        if (Array.isArray(obj)) {
+          return obj.reduce((size, item) => size + this._estimateSize(item), 0);
+        } else {
+          let size = 0;
+          for (const key in obj) {
+            if (obj.hasOwnProperty(key)) {
+              size += this._estimateSize(key) + this._estimateSize(obj[key]);
+            }
+          }
+          return size;
+        }
+      default:
+        return 0;
+    }
+  }
+
+  /**
+   * Manually load a chunk (for eager loading)
+   */
+  async loadChunk(chunkIndex) {
+    return await this._loadChunk(chunkIndex);
+  }
+
+  /**
+   * Manually unload a chunk
+   */
+  async unloadChunk(chunkIndex) {
+    return await this._unloadChunk(chunkIndex);
+  }
+
+  /**
+   * Get currently loaded chunks
+   */
+  getLoadedChunks() {
+    return Array.from(this.chunks.keys());
+  }
+
+  /**
+   * Get chunk information for a key
+   */
+  getChunkInfo(key) {
+    const chunkIndex = this.chunkIndex.get(key);
+    if (chunkIndex === undefined) return null;
+
+    const isLoaded = this.chunks.has(chunkIndex);
+    const keysInChunk = Array.from(this.chunkIndex.entries())
+      .filter(([k, idx]) => idx === chunkIndex)
+      .map(([k]) => k);
+
+    return {
+      chunkIndex,
+      isLoaded,
+      keysInChunk,
+      loadedChunks: this.loadedChunksCount,
+      totalChunks: this.totalChunks
+    };
+  }
+
+  /**
+   * Get performance statistics
+   */
+  getStats() {
+    const totalAccesses = this.stats.cacheHits + this.stats.cacheMisses;
+    const hitRate = totalAccesses > 0 
+      ? (this.stats.cacheHits / totalAccesses * 100).toFixed(2)
+      : 0;
+
+    return {
+      ...this.stats,
+      hitRate: `${hitRate}%`,
+      loadedChunks: this.loadedChunksCount,
+      totalChunks: this.totalChunks,
+      memorySaved: `${(this.stats.memorySaved / 1024 / 1024).toFixed(2)} MB`,
+      prefetchEffectiveness: this.stats.prefetchHits > 0 
+        ? `${((this.stats.prefetchHits / this.stats.chunksLoaded) * 100).toFixed(1)}%`
+        : '0%'
+    };
+  }
+
+  /**
+   * Clear all loaded chunks
+   */
+  clear() {
+    this.chunks.clear();
+    this.accessHistory = [];
+    this.loadedChunksCount = 0;
+    
+    if (this.options.debug) {
+      console.log('🧹 Cleared all loaded chunks');
+    }
+  }
+
+  /**
+   * Preload specific chunks (for startup optimization)
+   */
+  async preloadChunks(chunkIndices) {
+    const loadPromises = chunkIndices.map(index => this._loadChunk(index));
+    await Promise.all(loadPromises);
+    
+    if (this.options.debug) {
+      console.log(`🔥 Preloaded ${chunkIndices.length} chunks`);
+    }
+  }
+}
+
+module.exports = LazyLoader;