s3db.js 7.2.1 → 7.3.1

package/src/plugins/cache/partition-aware-filesystem-cache.class.js

@@ -0,0 +1,480 @@
+/**
+ * Partition-Aware Filesystem Cache Implementation
+ * 
+ * Extends FilesystemCache to provide intelligent caching for s3db.js partitions.
+ * Creates hierarchical directory structures that mirror partition organization.
+ * 
+ * @example
+ * // Basic partition-aware caching
+ * const cache = new PartitionAwareFilesystemCache({
+ *   directory: './cache',
+ *   partitionStrategy: 'hierarchical',
+ *   preloadRelated: true
+ * });
+ * 
+ * @example
+ * // Advanced configuration with analytics
+ * const cache = new PartitionAwareFilesystemCache({
+ *   directory: './data/cache',
+ *   partitionStrategy: 'incremental',
+ *   trackUsage: true,
+ *   preloadThreshold: 10,
+ *   maxCacheSize: '1GB'
+ * });
+ */
+import path from 'path';
+import fs from 'fs';
+import { promisify } from 'util';
+import { FilesystemCache } from './filesystem-cache.class.js';
+import tryFn from '../../concerns/try-fn.js';
+
+const mkdir = promisify(fs.mkdir);
+const rmdir = promisify(fs.rmdir);
+const readdir = promisify(fs.readdir);
+const stat = promisify(fs.stat);
+const writeFile = promisify(fs.writeFile);
+const readFile = promisify(fs.readFile);
+
+export class PartitionAwareFilesystemCache extends FilesystemCache {
+  constructor({
+    partitionStrategy = 'hierarchical', // 'hierarchical', 'flat', 'temporal'
+    trackUsage = true,
+    preloadRelated = false,
+    preloadThreshold = 10,
+    maxCacheSize = null,
+    usageStatsFile = 'partition-usage.json',
+    ...config
+  }) {
+    super(config);
+    
+    this.partitionStrategy = partitionStrategy;
+    this.trackUsage = trackUsage;
+    this.preloadRelated = preloadRelated;
+    this.preloadThreshold = preloadThreshold;
+    this.maxCacheSize = maxCacheSize;
+    this.usageStatsFile = path.join(this.directory, usageStatsFile);
+    
+    // Partition usage statistics
+    this.partitionUsage = new Map();
+    this.loadUsageStats();
+  }
+
+  /**
+   * Generate partition-aware cache key
+   */
+  _getPartitionCacheKey(resource, action, partition, partitionValues = {}, params = {}) {
+    const keyParts = [`resource=${resource}`, `action=${action}`];
+
+    if (partition && Object.keys(partitionValues).length > 0) {
+      keyParts.push(`partition=${partition}`);
+      
+      // Sort fields for consistent keys
+      const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
+      for (const [field, value] of sortedFields) {
+        if (value !== null && value !== undefined) {
+          keyParts.push(`${field}=${value}`);
+        }
+      }
+    }
+
+    // Add params hash if exists
+    if (Object.keys(params).length > 0) {
+      const paramsStr = Object.entries(params)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([k, v]) => `${k}=${v}`)
+        .join('|');
+      keyParts.push(`params=${Buffer.from(paramsStr).toString('base64')}`);
+    }
+
+    return keyParts.join('/') + this.fileExtension;
+  }
+
+  /**
+   * Get directory path for partition cache
+   */
+  _getPartitionDirectory(resource, partition, partitionValues = {}) {
+    const basePath = path.join(this.directory, `resource=${resource}`);
+
+    if (!partition) {
+      return basePath;
+    }
+
+    if (this.partitionStrategy === 'flat') {
+      // Flat structure: all partitions in same level
+      return path.join(basePath, 'partitions');
+    }
+
+    if (this.partitionStrategy === 'temporal' && this._isTemporalPartition(partition, partitionValues)) {
+      // Temporal structure: organize by time hierarchy
+      return this._getTemporalDirectory(basePath, partition, partitionValues);
+    }
+
+    // Hierarchical structure (default)
+    const pathParts = [basePath, `partition=${partition}`];
+    
+    const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
+    for (const [field, value] of sortedFields) {
+      if (value !== null && value !== undefined) {
+        pathParts.push(`${field}=${this._sanitizePathValue(value)}`);
+      }
+    }
+
+    return path.join(...pathParts);
+  }
+
+  /**
+   * Enhanced set method with partition awareness
+   */
+  async _set(key, data, options = {}) {
+    const { resource, action, partition, partitionValues, params } = options;
+
+    if (resource && partition) {
+      // Use partition-aware storage
+      const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
+      const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
+      
+      await this._ensureDirectory(partitionDir);
+      
+      const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
+      
+      // Track usage if enabled
+      if (this.trackUsage) {
+        await this._trackPartitionUsage(resource, partition, partitionValues);
+      }
+      
+      // Store with partition metadata
+      const partitionData = {
+        data,
+        metadata: {
+          resource,
+          partition,
+          partitionValues,
+          timestamp: Date.now(),
+          ttl: this.ttl
+        }
+      };
+      
+      return this._writeFileWithMetadata(filePath, partitionData);
+    }
+
+    // Fallback to standard set
+    return super._set(key, data);
+  }
+
+  /**
+   * Enhanced get method with partition awareness
+   */
+  async _get(key, options = {}) {
+    const { resource, action, partition, partitionValues, params } = options;
+
+    if (resource && partition) {
+      const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
+      const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
+      const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
+
+      if (!await this._fileExists(filePath)) {
+        // Try preloading related partitions
+        if (this.preloadRelated) {
+          await this._preloadRelatedPartitions(resource, partition, partitionValues);
+        }
+        return null;
+      }
+
+      const result = await this._readFileWithMetadata(filePath);
+      
+      if (result && this.trackUsage) {
+        await this._trackPartitionUsage(resource, partition, partitionValues);
+      }
+
+      return result?.data || null;
+    }
+
+    // Fallback to standard get
+    return super._get(key);
+  }
+
+  /**
+   * Clear cache for specific partition
+   */
+  async clearPartition(resource, partition, partitionValues = {}) {
+    const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
+    
+    const [ok, err] = await tryFn(async () => {
+      if (await this._fileExists(partitionDir)) {
+        await rmdir(partitionDir, { recursive: true });
+      }
+    });
+
+    if (!ok) {
+      console.warn(`Failed to clear partition cache: ${err.message}`);
+    }
+
+    // Clear from usage stats
+    const usageKey = this._getUsageKey(resource, partition, partitionValues);
+    this.partitionUsage.delete(usageKey);
+    await this._saveUsageStats();
+
+    return ok;
+  }
+
+  /**
+   * Clear all partitions for a resource
+   */
+  async clearResourcePartitions(resource) {
+    const resourceDir = path.join(this.directory, `resource=${resource}`);
+    
+    const [ok, err] = await tryFn(async () => {
+      if (await this._fileExists(resourceDir)) {
+        await rmdir(resourceDir, { recursive: true });
+      }
+    });
+
+    // Clear usage stats for resource
+    for (const [key] of this.partitionUsage.entries()) {
+      if (key.startsWith(`${resource}/`)) {
+        this.partitionUsage.delete(key);
+      }
+    }
+    await this._saveUsageStats();
+
+    return ok;
+  }
+
+  /**
+   * Get partition cache statistics
+   */
+  async getPartitionStats(resource, partition = null) {
+    const stats = {
+      totalFiles: 0,
+      totalSize: 0,
+      partitions: {},
+      usage: {}
+    };
+
+    const resourceDir = path.join(this.directory, `resource=${resource}`);
+    
+    if (!await this._fileExists(resourceDir)) {
+      return stats;
+    }
+
+    await this._calculateDirectoryStats(resourceDir, stats);
+
+    // Add usage statistics
+    for (const [key, usage] of this.partitionUsage.entries()) {
+      if (key.startsWith(`${resource}/`)) {
+        const partitionName = key.split('/')[1];
+        if (!partition || partitionName === partition) {
+          stats.usage[partitionName] = usage;
+        }
+      }
+    }
+
+    return stats;
+  }
+
+  /**
+   * Get cache recommendations based on usage patterns
+   */
+  async getCacheRecommendations(resource) {
+    const recommendations = [];
+    const now = Date.now();
+    const dayMs = 24 * 60 * 60 * 1000;
+
+    for (const [key, usage] of this.partitionUsage.entries()) {
+      if (key.startsWith(`${resource}/`)) {
+        const [, partition] = key.split('/');
+        const daysSinceLastAccess = (now - usage.lastAccess) / dayMs;
+        const accessesPerDay = usage.count / Math.max(1, daysSinceLastAccess);
+
+        let recommendation = 'keep';
+        let priority = usage.count;
+
+        if (daysSinceLastAccess > 30) {
+          recommendation = 'archive';
+          priority = 0;
+        } else if (accessesPerDay < 0.1) {
+          recommendation = 'reduce_ttl';
+          priority = 1;
+        } else if (accessesPerDay > 10) {
+          recommendation = 'preload';
+          priority = 100;
+        }
+
+        recommendations.push({
+          partition,
+          recommendation,
+          priority,
+          usage: accessesPerDay,
+          lastAccess: new Date(usage.lastAccess).toISOString()
+        });
+      }
+    }
+
+    return recommendations.sort((a, b) => b.priority - a.priority);
+  }
+
+  /**
+   * Preload frequently accessed partitions
+   */
+  async warmPartitionCache(resource, options = {}) {
+    const { partitions = [], maxFiles = 1000 } = options;
+    let warmedCount = 0;
+
+    for (const partition of partitions) {
+      const usageKey = `${resource}/${partition}`;
+      const usage = this.partitionUsage.get(usageKey);
+
+      if (usage && usage.count >= this.preloadThreshold) {
+        // This would integrate with the actual resource to preload data
+        console.log(`🔥 Warming cache for ${resource}/${partition} (${usage.count} accesses)`);
+        warmedCount++;
+      }
+
+      if (warmedCount >= maxFiles) break;
+    }
+
+    return warmedCount;
+  }
+
+  // Private helper methods
+
+  async _trackPartitionUsage(resource, partition, partitionValues) {
+    const usageKey = this._getUsageKey(resource, partition, partitionValues);
+    const current = this.partitionUsage.get(usageKey) || {
+      count: 0,
+      firstAccess: Date.now(),
+      lastAccess: Date.now()
+    };
+
+    current.count++;
+    current.lastAccess = Date.now();
+    this.partitionUsage.set(usageKey, current);
+
+    // Periodically save stats
+    if (current.count % 10 === 0) {
+      await this._saveUsageStats();
+    }
+  }
+
+  _getUsageKey(resource, partition, partitionValues) {
+    const valuePart = Object.entries(partitionValues)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([k, v]) => `${k}=${v}`)
+      .join('|');
+    
+    return `${resource}/${partition}/${valuePart}`;
+  }
+
+  async _preloadRelatedPartitions(resource, partition, partitionValues) {
+    // This would implement intelligent preloading based on:
+    // - Temporal patterns (load next/previous time periods)
+    // - Geographic patterns (load adjacent regions)
+    // - Categorical patterns (load related categories)
+    
+    console.log(`🎯 Preloading related partitions for ${resource}/${partition}`);
+    
+    // Example: for date partitions, preload next day
+    if (partitionValues.timestamp || partitionValues.date) {
+      // Implementation would go here
+    }
+  }
+
+  _isTemporalPartition(partition, partitionValues) {
+    const temporalFields = ['date', 'timestamp', 'createdAt', 'updatedAt'];
+    return Object.keys(partitionValues).some(field => 
+      temporalFields.some(tf => field.toLowerCase().includes(tf))
+    );
+  }
+
+  _getTemporalDirectory(basePath, partition, partitionValues) {
+    // Create year/month/day hierarchy for temporal data
+    const dateValue = Object.values(partitionValues)[0];
+    if (typeof dateValue === 'string' && dateValue.match(/^\d{4}-\d{2}-\d{2}/)) {
+      const [year, month, day] = dateValue.split('-');
+      return path.join(basePath, 'temporal', year, month, day);
+    }
+    
+    return path.join(basePath, `partition=${partition}`);
+  }
+
+  _sanitizePathValue(value) {
+    return String(value).replace(/[<>:"/\\|?*]/g, '_');
+  }
+
+  _sanitizeFileName(filename) {
+    return filename.replace(/[<>:"/\\|?*]/g, '_');
+  }
+
+  async _calculateDirectoryStats(dir, stats) {
+    const [ok, err, files] = await tryFn(() => readdir(dir));
+    if (!ok) return;
+
+    for (const file of files) {
+      const filePath = path.join(dir, file);
+      const [statOk, statErr, fileStat] = await tryFn(() => stat(filePath));
+      
+      if (statOk) {
+        if (fileStat.isDirectory()) {
+          await this._calculateDirectoryStats(filePath, stats);
+        } else {
+          stats.totalFiles++;
+          stats.totalSize += fileStat.size;
+        }
+      }
+    }
+  }
+
+  async loadUsageStats() {
+    const [ok, err, content] = await tryFn(async () => {
+      const data = await readFile(this.usageStatsFile, 'utf8');
+      return JSON.parse(data);
+    });
+
+    if (ok && content) {
+      this.partitionUsage = new Map(Object.entries(content));
+    }
+  }
+
+  async _saveUsageStats() {
+    const statsObject = Object.fromEntries(this.partitionUsage);
+    
+    await tryFn(async () => {
+      await writeFile(
+        this.usageStatsFile, 
+        JSON.stringify(statsObject, null, 2),
+        'utf8'
+      );
+    });
+  }
+
+  async _writeFileWithMetadata(filePath, data) {
+    const content = JSON.stringify(data);
+    
+    const [ok, err] = await tryFn(async () => {
+      await writeFile(filePath, content, {
+        encoding: this.encoding,
+        mode: this.fileMode
+      });
+    });
+
+    if (!ok) {
+      throw new Error(`Failed to write cache file: ${err.message}`);
+    }
+
+    return true;
+  }
+
+  async _readFileWithMetadata(filePath) {
+    const [ok, err, content] = await tryFn(async () => {
+      return await readFile(filePath, this.encoding);
+    });
+
+    if (!ok || !content) return null;
+    
+    try {
+      return JSON.parse(content);
+    } catch (error) {
+      return { data: content }; // Fallback for non-JSON data
+    }
+  }
+} 

package/src/plugins/cache.plugin.js

@@ -4,6 +4,8 @@ import { sha256 } from "../concerns/crypto.js";
 import Plugin from "./plugin.class.js";
 import S3Cache from "./cache/s3-cache.class.js";
 import MemoryCache from "./cache/memory-cache.class.js";
+import { FilesystemCache } from "./cache/filesystem-cache.class.js";
+import { PartitionAwareFilesystemCache } from "./cache/partition-aware-filesystem-cache.class.js";
 import tryFn from "../concerns/try-fn.js";
 
 export class CachePlugin extends Plugin {
@@ -12,6 +14,10 @@ export class CachePlugin extends Plugin {
     this.driver = options.driver;
     this.config = {
       includePartitions: options.includePartitions !== false,
+      partitionStrategy: options.partitionStrategy || 'hierarchical',
+      partitionAware: options.partitionAware !== false,
+      trackUsage: options.trackUsage !== false,
+      preloadRelated: options.preloadRelated !== false,
       ...options
     };
   }
@@ -27,6 +33,18 @@ export class CachePlugin extends Plugin {
       this.driver = this.config.driver;
     } else if (this.config.driverType === 'memory') {
       this.driver = new MemoryCache(this.config.memoryOptions || {});
+    } else if (this.config.driverType === 'filesystem') {
+      // Use partition-aware filesystem cache if enabled
+      if (this.config.partitionAware) {
+        this.driver = new PartitionAwareFilesystemCache({
+          partitionStrategy: this.config.partitionStrategy,
+          trackUsage: this.config.trackUsage,
+          preloadRelated: this.config.preloadRelated,
+          ...this.config.filesystemOptions
+        });
+      } else {
+        this.driver = new FilesystemCache(this.config.filesystemOptions || {});
+      }
     } else {
       // Default to S3Cache, sempre passa o client do database
       this.driver = new S3Cache({ client: this.database.client, ...(this.config.s3Options || {}) });
@@ -89,6 +107,25 @@ export class CachePlugin extends Plugin {
       return this.generateCacheKey(resource, action, params, partition, partitionValues);
     };
 
+    // Add partition-aware methods if using PartitionAwareFilesystemCache
+    if (this.driver instanceof PartitionAwareFilesystemCache) {
+      resource.clearPartitionCache = async (partition, partitionValues = {}) => {
+        return await this.driver.clearPartition(resource.name, partition, partitionValues);
+      };
+      
+      resource.getPartitionCacheStats = async (partition = null) => {
+        return await this.driver.getPartitionStats(resource.name, partition);
+      };
+      
+      resource.getCacheRecommendations = async () => {
+        return await this.driver.getCacheRecommendations(resource.name);
+      };
+      
+      resource.warmPartitionCache = async (partitions = [], options = {}) => {
+        return await this.driver.warmPartitionCache(resource.name, { partitions, ...options });
+      };
+    }
+
     // List of methods to cache
     const cacheMethods = [
       'count', 'listIds', 'getMany', 'getAll', 'page', 'list', 'get'
@@ -110,14 +147,50 @@ export class CachePlugin extends Plugin {
         } else if (method === 'get') {
           key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
         }
-        // Try cache
-        const [ok, err, cached] = await tryFn(() => resource.cache.get(key));
-        if (ok && cached !== null && cached !== undefined) return cached;
-        if (!ok && err.name !== 'NoSuchKey') throw err;
-        // Not cached, call next
-        const result = await next();
-        await resource.cache.set(key, result);
-        return result;
+        // Try cache with partition awareness
+        let cached;
+        if (this.driver instanceof PartitionAwareFilesystemCache) {
+          // Extract partition info for partition-aware cache
+          let partition, partitionValues;
+          if (method === 'list' || method === 'listIds' || method === 'count' || method === 'page') {
+            const args = ctx.args[0] || {};
+            partition = args.partition;
+            partitionValues = args.partitionValues;
+          }
+          
+          const [ok, err, result] = await tryFn(() => resource.cache._get(key, {
+            resource: resource.name,
+            action: method,
+            partition,
+            partitionValues
+          }));
+          
+          if (ok && result !== null && result !== undefined) return result;
+          if (!ok && err.name !== 'NoSuchKey') throw err;
+          
+          // Not cached, call next
+          const freshResult = await next();
+          
+          // Store with partition context
+          await resource.cache._set(key, freshResult, {
+            resource: resource.name,
+            action: method,
+            partition,
+            partitionValues
+          });
+          
+          return freshResult;
+        } else {
+          // Standard cache behavior
+          const [ok, err, result] = await tryFn(() => resource.cache.get(key));
+          if (ok && result !== null && result !== undefined) return result;
+          if (!ok && err.name !== 'NoSuchKey') throw err;
+          
+          // Not cached, call next
+          const freshResult = await next();
+          await resource.cache.set(key, freshResult);
+          return freshResult;
+        }
       });
     }
 
@@ -240,7 +313,13 @@ export class CachePlugin extends Plugin {
 
     const { includePartitions = true } = options;
 
-    // Warm main cache using the wrapped method (which will call the original)
+    // Use partition-aware warming if available
+    if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
+      const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
+      return await resource.warmPartitionCache(partitionNames, options);
+    }
+
+    // Fallback to standard warming
     await resource.getAll();
 
     // Warm partition caches if enabled
@@ -270,6 +349,77 @@ export class CachePlugin extends Plugin {
       }
     }
   }
+
+  // Partition-specific methods
+  async getPartitionCacheStats(resourceName, partition = null) {
+    if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
+      throw new Error('Partition cache statistics are only available with PartitionAwareFilesystemCache');
+    }
+    
+    return await this.driver.getPartitionStats(resourceName, partition);
+  }
+
+  async getCacheRecommendations(resourceName) {
+    if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
+      throw new Error('Cache recommendations are only available with PartitionAwareFilesystemCache');
+    }
+    
+    return await this.driver.getCacheRecommendations(resourceName);
+  }
+
+  async clearPartitionCache(resourceName, partition, partitionValues = {}) {
+    if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
+      throw new Error('Partition cache clearing is only available with PartitionAwareFilesystemCache');
+    }
+    
+    return await this.driver.clearPartition(resourceName, partition, partitionValues);
+  }
+
+  async analyzeCacheUsage() {
+    if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
+      return { message: 'Cache usage analysis is only available with PartitionAwareFilesystemCache' };
+    }
+
+    const analysis = {
+      totalResources: Object.keys(this.database.resources).length,
+      resourceStats: {},
+      recommendations: {},
+      summary: {
+        mostUsedPartitions: [],
+        leastUsedPartitions: [],
+        suggestedOptimizations: []
+      }
+    };
+
+    // Analyze each resource
+    for (const [resourceName, resource] of Object.entries(this.database.resources)) {
+      try {
+        analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
+        analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
+      } catch (error) {
+        analysis.resourceStats[resourceName] = { error: error.message };
+      }
+    }
+
+    // Generate summary
+    const allRecommendations = Object.values(analysis.recommendations).flat();
+    analysis.summary.mostUsedPartitions = allRecommendations
+      .filter(r => r.recommendation === 'preload')
+      .sort((a, b) => b.priority - a.priority)
+      .slice(0, 5);
+
+    analysis.summary.leastUsedPartitions = allRecommendations
+      .filter(r => r.recommendation === 'archive')
+      .slice(0, 5);
+
+    analysis.summary.suggestedOptimizations = [
+      `Consider preloading ${analysis.summary.mostUsedPartitions.length} high-usage partitions`,
+      `Archive ${analysis.summary.leastUsedPartitions.length} unused partitions`,
+      `Monitor cache hit rates for partition efficiency`
+    ];
+
+    return analysis;
+  }
 }
 
 export default CachePlugin;

package/src/plugins/consumers/index.js

@@ -10,9 +10,9 @@ export const CONSUMER_DRIVERS = {
 };
 
 /**
- * Cria uma instância de consumer baseado no driver
- * @param {string} driver - Tipo do driver (sqs, rabbitmq, kafka...)
- * @param {Object} config - Configuração do consumer
+ * Creates a consumer instance based on the driver
+ * @param {string} driver - Driver type (sqs, rabbitmq, kafka...)
+ * @param {Object} config - Consumer configuration
  * @returns {SqsConsumer|RabbitMqConsumer|KafkaConsumer}
  */
 export function createConsumer(driver, config) {