s3db.js 13.6.0 → 14.0.2

package/src/plugins/cache/multi-tier-cache.class.js

@@ -0,0 +1,371 @@
+/**
+ * Multi-Tier Cache
+ *
+ * Cascading cache implementation that chains multiple cache drivers (L1 → L2 → L3).
+ * Provides automatic promotion of hot data to faster layers and fallback on errors.
+ *
+ * @example
+ * // Memory → Redis → S3 cascade
+ * const cache = new MultiTierCache({
+ *   drivers: [
+ *     { driver: memoryInstance, name: 'L1-Memory' },
+ *     { driver: redisInstance, name: 'L2-Redis' },
+ *     { driver: s3Instance, name: 'L3-S3' }
+ *   ],
+ *   promoteOnHit: true,
+ *   strategy: 'write-through'
+ * });
+ */
+import { Cache } from "./cache.class.js";
+import { CacheError } from "../cache.errors.js";
+
+export class MultiTierCache extends Cache {
+  constructor({
+    drivers = [],
+    promoteOnHit = true,
+    strategy = 'write-through', // 'write-through' | 'lazy-promotion'
+    fallbackOnError = true,
+    verbose = false
+  }) {
+    super();
+
+    if (!Array.isArray(drivers) || drivers.length === 0) {
+      throw new CacheError('MultiTierCache requires at least one driver', {
+        operation: 'constructor',
+        driver: 'MultiTierCache',
+        provided: drivers,
+        suggestion: 'Pass drivers array with at least one cache driver instance'
+      });
+    }
+
+    this.drivers = drivers.map((d, index) => ({
+      instance: d.driver,
+      name: d.name || `L${index + 1}`,
+      tier: index + 1
+    }));
+
+    this.config = {
+      promoteOnHit,
+      strategy,
+      fallbackOnError,
+      verbose
+    };
+
+    // Statistics per tier
+    this.stats = {
+      enabled: true,
+      tiers: this.drivers.map(d => ({
+        name: d.name,
+        hits: 0,
+        misses: 0,
+        promotions: 0,
+        errors: 0,
+        sets: 0
+      }))
+    };
+  }
+
+  /**
+   * Log message if verbose enabled
+   * @private
+   */
+  _log(...args) {
+    if (this.config.verbose) {
+      console.log('[MultiTierCache]', ...args);
+    }
+  }
+
+  /**
+   * Get value from cache tiers (cascade L1 → L2 → L3)
+   * @private
+   */
+  async _get(key) {
+    for (let i = 0; i < this.drivers.length; i++) {
+      const tier = this.drivers[i];
+      const tierStats = this.stats.tiers[i];
+
+      try {
+        const value = await tier.instance.get(key);
+
+        if (value !== null && value !== undefined) {
+          // Cache hit!
+          tierStats.hits++;
+          this._log(`✓ Cache HIT on ${tier.name} for key: ${key}`);
+
+          // Promote to faster tiers if enabled and not already in L1
+          if (this.config.promoteOnHit && i > 0) {
+            this._promoteToFasterTiers(key, value, i);
+          }
+
+          return value;
+        } else {
+          // Cache miss on this tier, try next
+          tierStats.misses++;
+          this._log(`✗ Cache MISS on ${tier.name} for key: ${key}`);
+        }
+      } catch (error) {
+        tierStats.errors++;
+        this._log(`⚠ Error on ${tier.name} for key: ${key}`, error.message);
+
+        if (!this.config.fallbackOnError) {
+          throw new CacheError(`Cache get failed on ${tier.name}`, {
+            operation: 'get',
+            driver: 'MultiTierCache',
+            tier: tier.name,
+            key,
+            cause: error,
+            suggestion: 'Enable fallbackOnError to skip failed tiers'
+          });
+        }
+
+        // Continue to next tier on error (fallback)
+        continue;
+      }
+    }
+
+    // Miss on all tiers
+    this._log(`✗ Cache MISS on ALL tiers for key: ${key}`);
+    return null;
+  }
+
+  /**
+   * Promote value to faster tiers (L2 hit → write to L1, L3 hit → write to L1+L2)
+   * @private
+   */
+  async _promoteToFasterTiers(key, value, hitTierIndex) {
+    // Write to all tiers faster than the one where we found the value
+    for (let i = 0; i < hitTierIndex; i++) {
+      const tier = this.drivers[i];
+      const tierStats = this.stats.tiers[i];
+
+      try {
+        await tier.instance.set(key, value);
+        tierStats.promotions++;
+        this._log(`↑ Promoted key "${key}" to ${tier.name}`);
+      } catch (error) {
+        tierStats.errors++;
+        this._log(`⚠ Failed to promote key "${key}" to ${tier.name}:`, error.message);
+        // Continue promoting to other tiers even if one fails
+      }
+    }
+  }
+
+  /**
+   * Set value in cache tiers
+   * @private
+   */
+  async _set(key, data) {
+    if (this.config.strategy === 'write-through') {
+      // Write to ALL tiers immediately
+      return this._writeToAllTiers(key, data);
+    } else if (this.config.strategy === 'lazy-promotion') {
+      // Write only to L1, let promotions handle the rest
+      return this._writeToL1Only(key, data);
+    }
+  }
+
+  /**
+   * Write-through strategy: write to all tiers immediately
+   * @private
+   */
+  async _writeToAllTiers(key, data) {
+    const results = await Promise.allSettled(
+      this.drivers.map(async (tier, index) => {
+        try {
+          await tier.instance.set(key, data);
+          this.stats.tiers[index].sets++;
+          this._log(`✓ Wrote key "${key}" to ${tier.name}`);
+          return { success: true, tier: tier.name };
+        } catch (error) {
+          this.stats.tiers[index].errors++;
+          this._log(`⚠ Failed to write key "${key}" to ${tier.name}:`, error.message);
+          return { success: false, tier: tier.name, error };
+        }
+      })
+    );
+
+    // Check if at least L1 succeeded
+    const l1Success = results[0]?.status === 'fulfilled' && results[0].value.success;
+
+    if (!l1Success && !this.config.fallbackOnError) {
+      throw new CacheError('Failed to write to L1 cache', {
+        operation: 'set',
+        driver: 'MultiTierCache',
+        key,
+        results,
+        suggestion: 'Enable fallbackOnError or check L1 cache health'
+      });
+    }
+
+    return true;
+  }
+
+  /**
+   * Lazy-promotion strategy: write only to L1
+   * @private
+   */
+  async _writeToL1Only(key, data) {
+    const tier = this.drivers[0];
+    const tierStats = this.stats.tiers[0];
+
+    try {
+      await tier.instance.set(key, data);
+      tierStats.sets++;
+      this._log(`✓ Wrote key "${key}" to ${tier.name} (lazy-promotion)`);
+      return true;
+    } catch (error) {
+      tierStats.errors++;
+      throw new CacheError(`Failed to write to ${tier.name}`, {
+        operation: 'set',
+        driver: 'MultiTierCache',
+        tier: tier.name,
+        key,
+        cause: error,
+        suggestion: 'Check L1 cache health'
+      });
+    }
+  }
+
+  /**
+   * Delete key from all tiers
+   * @private
+   */
+  async _del(key) {
+    const results = await Promise.allSettled(
+      this.drivers.map(async (tier) => {
+        try {
+          await tier.instance.del(key);
+          this._log(`✓ Deleted key "${key}" from ${tier.name}`);
+          return { success: true, tier: tier.name };
+        } catch (error) {
+          this._log(`⚠ Failed to delete key "${key}" from ${tier.name}:`, error.message);
+          return { success: false, tier: tier.name, error };
+        }
+      })
+    );
+
+    // Consider successful if at least one tier succeeded
+    const anySuccess = results.some(r => r.status === 'fulfilled' && r.value.success);
+
+    if (!anySuccess && !this.config.fallbackOnError) {
+      throw new CacheError('Failed to delete from all cache tiers', {
+        operation: 'delete',
+        driver: 'MultiTierCache',
+        key,
+        results,
+        suggestion: 'Enable fallbackOnError or check cache health'
+      });
+    }
+
+    return true;
+  }
+
+  /**
+   * Clear all keys from all tiers
+   * @private
+   */
+  async _clear(prefix) {
+    const results = await Promise.allSettled(
+      this.drivers.map(async (tier) => {
+        try {
+          await tier.instance.clear(prefix);
+          this._log(`✓ Cleared ${prefix ? `prefix "${prefix}"` : 'all keys'} from ${tier.name}`);
+          return { success: true, tier: tier.name };
+        } catch (error) {
+          this._log(`⚠ Failed to clear ${tier.name}:`, error.message);
+          return { success: false, tier: tier.name, error };
+        }
+      })
+    );
+
+    return true;
+  }
+
+  /**
+   * Get total size across all tiers (may have duplicates)
+   */
+  async size() {
+    let totalSize = 0;
+
+    for (const tier of this.drivers) {
+      try {
+        if (typeof tier.instance.size === 'function') {
+          const size = await tier.instance.size();
+          totalSize += size;
+        }
+      } catch (error) {
+        this._log(`⚠ Failed to get size from ${tier.name}:`, error.message);
+      }
+    }
+
+    return totalSize;
+  }
+
+  /**
+   * Get all keys from all tiers (deduplicated)
+   */
+  async keys() {
+    const allKeys = new Set();
+
+    for (const tier of this.drivers) {
+      try {
+        if (typeof tier.instance.keys === 'function') {
+          const keys = await tier.instance.keys();
+          keys.forEach(k => allKeys.add(k));
+        }
+      } catch (error) {
+        this._log(`⚠ Failed to get keys from ${tier.name}:`, error.message);
+      }
+    }
+
+    return Array.from(allKeys);
+  }
+
+  /**
+   * Get comprehensive statistics
+   */
+  getStats() {
+    // Calculate totals
+    const totals = {
+      hits: 0,
+      misses: 0,
+      promotions: 0,
+      errors: 0,
+      sets: 0
+    };
+
+    for (const tierStats of this.stats.tiers) {
+      totals.hits += tierStats.hits;
+      totals.misses += tierStats.misses;
+      totals.promotions += tierStats.promotions;
+      totals.errors += tierStats.errors;
+      totals.sets += tierStats.sets;
+    }
+
+    const total = totals.hits + totals.misses;
+    const hitRate = total > 0 ? totals.hits / total : 0;
+
+    return {
+      enabled: true,
+      strategy: this.config.strategy,
+      promoteOnHit: this.config.promoteOnHit,
+      tiers: this.stats.tiers.map(t => {
+        const tierTotal = t.hits + t.misses;
+        const tierHitRate = tierTotal > 0 ? t.hits / tierTotal : 0;
+        return {
+          ...t,
+          hitRate: tierHitRate,
+          hitRatePercent: (tierHitRate * 100).toFixed(2) + '%'
+        };
+      }),
+      totals: {
+        ...totals,
+        total,
+        hitRate,
+        hitRatePercent: (hitRate * 100).toFixed(2) + '%'
+      }
+    };
+  }
+}
+
+export default MultiTierCache;

package/src/plugins/cache/partition-aware-filesystem-cache.class.js

@@ -27,6 +27,7 @@ import fs from 'fs';
 import { mkdir, rm as rmdir, readdir, stat, writeFile, readFile } from 'fs/promises';
 import { FilesystemCache } from './filesystem-cache.class.js';
 import tryFn from '../../concerns/try-fn.js';
+import { CacheError } from '../cache.errors.js';
 
 export class PartitionAwareFilesystemCache extends FilesystemCache {
   constructor({
@@ -56,55 +57,68 @@ export class PartitionAwareFilesystemCache extends FilesystemCache {
    * Generate partition-aware cache key
    */
   _getPartitionCacheKey(resource, action, partition, partitionValues = {}, params = {}) {
-    const keyParts = [`resource=${resource}`, `action=${action}`];
+    const segments = [];
+
+    if (resource) {
+      segments.push(`resource=${this._sanitizePathValue(resource)}`);
+    }
+
+    const hasPartitionValues = partitionValues && Object.values(partitionValues).some(value => value !== null && value !== undefined && value !== '');
+
+    if (partition && hasPartitionValues) {
+      segments.push(`partition=${this._sanitizePathValue(partition)}`);
+
+      const sortedFields = Object.entries(partitionValues)
+        .filter(([, value]) => value !== null && value !== undefined)
+        .sort(([a], [b]) => a.localeCompare(b));
 
-    if (partition && Object.keys(partitionValues).length > 0) {
-      keyParts.push(`partition=${partition}`);
-      
-      // Sort fields for consistent keys
-      const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
       for (const [field, value] of sortedFields) {
-        if (value !== null && value !== undefined) {
-          keyParts.push(`${field}=${value}`);
-        }
+        segments.push(`${field}=${this._sanitizePathValue(value)}`);
       }
     }
 
-    // Add params hash if exists
-    if (Object.keys(params).length > 0) {
+    if (action) {
+      segments.push(`action=${this._sanitizePathValue(action)}`);
+    }
+
+    if (params && Object.keys(params).length > 0) {
       const paramsStr = Object.entries(params)
         .sort(([a], [b]) => a.localeCompare(b))
         .map(([k, v]) => `${k}=${v}`)
         .join('|');
-      keyParts.push(`params=${Buffer.from(paramsStr).toString('base64')}`);
+      segments.push(`params=${this._sanitizePathValue(Buffer.from(paramsStr).toString('base64url'))}`);
     }
 
-    return keyParts.join('/') + this.fileExtension;
+    return segments.join('/');
   }
 
   /**
    * Get directory path for partition cache
    */
   _getPartitionDirectory(resource, partition, partitionValues = {}) {
-    const basePath = path.join(this.directory, `resource=${resource}`);
+    const baseSegments = [];
+    if (resource) {
+      baseSegments.push(`resource=${this._sanitizePathValue(resource)}`);
+    }
 
     if (!partition) {
-      return basePath;
+      return path.join(this.directory, ...baseSegments);
     }
 
     if (this.partitionStrategy === 'flat') {
-      // Flat structure: all partitions in same level
-      return path.join(basePath, 'partitions');
+      return path.join(this.directory, ...baseSegments, 'partitions');
     }
 
     if (this.partitionStrategy === 'temporal' && this._isTemporalPartition(partition, partitionValues)) {
-      // Temporal structure: organize by time hierarchy
-      return this._getTemporalDirectory(basePath, partition, partitionValues);
+      return this._getTemporalDirectory(path.join(this.directory, ...baseSegments), partition, partitionValues);
     }
 
-    // Hierarchical structure (default)
-    const pathParts = [basePath, `partition=${partition}`];
-    
+    const pathParts = [
+      this.directory,
+      ...baseSegments,
+      `partition=${this._sanitizePathValue(partition)}`
+    ];
+
     const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
     for (const [field, value] of sortedFields) {
       if (value !== null && value !== undefined) {
@@ -122,21 +136,15 @@ export class PartitionAwareFilesystemCache extends FilesystemCache {
     const { resource, action, partition, partitionValues, params } = options;
 
     if (resource && partition) {
-      // Use partition-aware storage
       const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
-      const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
-      
-      await this._ensureDirectory(partitionDir);
-      
-      const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
-      
-      // Track usage if enabled
+
+      await this._ensurePartitionDirectoryForKey(partitionKey);
+
       if (this.trackUsage) {
         await this._trackPartitionUsage(resource, partition, partitionValues);
       }
-      
-      // Store with partition metadata
-      const partitionData = {
+
+      const payload = {
         data,
         metadata: {
           resource,
@@ -146,8 +154,9 @@ export class PartitionAwareFilesystemCache extends FilesystemCache {
           ttl: this.ttl
         }
       };
-      
-      return this._writeFileWithMetadata(filePath, partitionData);
+
+      await super._set(partitionKey, payload);
+      return data;
     }
 
     // Fallback to standard set
@@ -190,24 +199,20 @@ export class PartitionAwareFilesystemCache extends FilesystemCache {
 
     if (resource && partition) {
       const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
-      const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
-      const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
+      const payload = await super._get(partitionKey);
 
-      if (!await this._fileExists(filePath)) {
-        // Try preloading related partitions
+      if (!payload) {
         if (this.preloadRelated) {
           await this._preloadRelatedPartitions(resource, partition, partitionValues);
         }
         return null;
       }
 
-      const result = await this._readFileWithMetadata(filePath);
-      
-      if (result && this.trackUsage) {
+      if (this.trackUsage) {
         await this._trackPartitionUsage(resource, partition, partitionValues);
       }
 
-      return result?.data || null;
+      return payload?.data ?? null;
     }
 
     // Fallback to standard get
@@ -261,6 +266,48 @@ export class PartitionAwareFilesystemCache extends FilesystemCache {
     return ok;
   }
 
+  async _clear(prefix) {
+    await super._clear(prefix);
+
+    if (!prefix) {
+      const [entriesOk, , entries] = await tryFn(() => readdir(this.directory));
+      if (entriesOk && entries) {
+        for (const entry of entries) {
+          const entryPath = path.join(this.directory, entry);
+          const [statOk, , entryStat] = await tryFn(() => stat(entryPath));
+          if (statOk && entryStat.isDirectory() && entry.startsWith('resource=')) {
+            await rmdir(entryPath, { recursive: true }).catch(() => {});
+          }
+        }
+      }
+      this.partitionUsage.clear();
+      await this._saveUsageStats();
+      return true;
+    }
+
+    const segments = this._splitKeySegments(prefix).map(segment => this._sanitizeFileName(segment));
+    if (segments.length > 0) {
+      const dirPath = path.join(this.directory, ...segments);
+      if (await this._fileExists(dirPath)) {
+        await rmdir(dirPath, { recursive: true }).catch(() => {});
+      }
+
+      const resourceSeg = segments.find(seg => seg.startsWith('resource='));
+      const partitionSeg = segments.find(seg => seg.startsWith('partition='));
+      const resourceVal = resourceSeg ? resourceSeg.split('=').slice(1).join('=') : '';
+      const partitionVal = partitionSeg ? partitionSeg.split('=').slice(1).join('=') : '';
+      const usagePrefix = `${resourceVal}/${partitionVal}`;
+      for (const key of Array.from(this.partitionUsage.keys())) {
+        if (key.startsWith(usagePrefix)) {
+          this.partitionUsage.delete(key);
+        }
+      }
+      await this._saveUsageStats();
+    }
+
+    return true;
+  }
+
   /**
    * Get partition cache statistics
    */
@@ -378,12 +425,14 @@ export class PartitionAwareFilesystemCache extends FilesystemCache {
   }
 
   _getUsageKey(resource, partition, partitionValues) {
+    const sanitizedResource = this._sanitizePathValue(resource || '');
+    const sanitizedPartition = this._sanitizePathValue(partition || '');
     const valuePart = Object.entries(partitionValues)
       .sort(([a], [b]) => a.localeCompare(b))
-      .map(([k, v]) => `${k}=${v}`)
+      .map(([k, v]) => `${k}=${this._sanitizePathValue(v)}`)
       .join('|');
     
-    return `${resource}/${partition}/${valuePart}`;
+    return `${sanitizedResource}/${sanitizedPartition}/${valuePart}`;
   }
 
   async _preloadRelatedPartitions(resource, partition, partitionValues) {
@@ -426,6 +475,34 @@ export class PartitionAwareFilesystemCache extends FilesystemCache {
     return filename.replace(/[<>:"/\\|?*]/g, '_');
   }
 
+  _splitKeySegments(key) {
+    return key.split('/').filter(Boolean);
+  }
+
+  async _ensurePartitionDirectoryForKey(key) {
+    const segments = this._splitKeySegments(key);
+    if (segments.length <= 1) {
+      return;
+    }
+
+    const dirPath = path.join(
+      this.directory,
+      ...segments.slice(0, -1).map(segment => this._sanitizeFileName(segment))
+    );
+
+    await this._ensureDirectory(dirPath);
+  }
+
+  _getFilePath(key) {
+    const segments = this._splitKeySegments(key).map(segment => this._sanitizeFileName(segment));
+    const fileName = segments.pop() || this._sanitizeFileName(key);
+    const dirPath = segments.length > 0
+      ? path.join(this.directory, ...segments)
+      : this.directory;
+
+    return path.join(dirPath, `${this.prefix}_${fileName}${this.fileExtension}`);
+  }
+
   async _calculateDirectoryStats(dir, stats) {
     const [ok, err, files] = await tryFn(() => readdir(dir));
     if (!ok) return;
@@ -479,7 +556,15 @@ export class PartitionAwareFilesystemCache extends FilesystemCache {
     });
 
     if (!ok) {
-      throw new Error(`Failed to write cache file: ${err.message}`);
+      throw new CacheError(`Failed to write cache file: ${err.message}`, {
+        driver: 'filesystem-partitioned',
+        operation: 'writeFileWithMetadata',
+        statusCode: 500,
+        retriable: false,
+        suggestion: 'Check filesystem permissions and disk space for the partition-aware cache directory.',
+        filePath,
+        original: err
+      });
     }
 
     return true;
@@ -498,4 +583,40 @@ export class PartitionAwareFilesystemCache extends FilesystemCache {
       return { data: content }; // Fallback for non-JSON data
     }
   }
-} 
+
+  async size() {
+    const keys = await this.keys();
+    return keys.length;
+  }
+
+  async keys() {
+    const keys = [];
+    await this._collectKeysRecursive(this.directory, [], keys);
+    return keys;
+  }
+
+  async _collectKeysRecursive(currentDir, segments, result) {
+    const [ok, err, entries] = await tryFn(() => readdir(currentDir, { withFileTypes: true }));
+    if (!ok || !entries) {
+      return;
+    }
+
+    for (const entry of entries) {
+      const entryPath = path.join(currentDir, entry.name);
+      if (entry.isDirectory()) {
+        await this._collectKeysRecursive(entryPath, [...segments, entry.name], result);
+        continue;
+      }
+
+      if (!entry.isFile()) continue;
+
+      if (!entry.name.startsWith(`${this.prefix}_`) || !entry.name.endsWith(this.fileExtension)) {
+        continue;
+      }
+
+      const keyPart = entry.name.slice(this.prefix.length + 1, -this.fileExtension.length);
+      const fullSegments = segments.length > 0 ? [...segments, keyPart] : [keyPart];
+      result.push(fullSegments.join('/'));
+    }
+  }
+}