s3db.js 11.3.2 → 12.0.1

package/src/plugins/cache/s3-cache.class.js

@@ -105,72 +105,117 @@
  * - TTL is enforced by checking object creation time
  */
 import zlib from "node:zlib";
-import { join } from "path";
-
-import { Cache } from "./cache.class.js"
-import { streamToString } from "../../stream/index.js";
-import tryFn from "../../concerns/try-fn.js";
+import { PluginStorage } from "../../concerns/plugin-storage.js";
+import { Cache } from "./cache.class.js";
 
 export class S3Cache extends Cache {
-  constructor({ 
-    client, 
+  constructor({
+    client,
     keyPrefix = 'cache',
     ttl = 0,
-    prefix = undefined
+    prefix = undefined,
+    enableCompression = true,
+    compressionThreshold = 1024
   }) {
     super();
-    this.client = client
+    this.client = client;
     this.keyPrefix = keyPrefix;
     this.config.ttl = ttl;
     this.config.client = client;
     this.config.prefix = prefix !== undefined ? prefix : keyPrefix + (keyPrefix.endsWith('/') ? '' : '/');
+    this.config.enableCompression = enableCompression;
+    this.config.compressionThreshold = compressionThreshold;
+
+    // Create PluginStorage instance for consistent storage operations with TTL support
+    this.storage = new PluginStorage(client, 'cache');
+  }
+
+  /**
+   * Compress data if enabled and above threshold
+   * @private
+   */
+  _compressData(data) {
+    const jsonString = JSON.stringify(data);
+
+    // Don't compress if disabled or below threshold
+    if (!this.config.enableCompression || jsonString.length < this.config.compressionThreshold) {
+      return {
+        data: jsonString,
+        compressed: false,
+        originalSize: jsonString.length
+      };
+    }
+
+    // Compress with gzip
+    const compressed = zlib.gzipSync(jsonString).toString('base64');
+    return {
+      data: compressed,
+      compressed: true,
+      originalSize: jsonString.length,
+      compressedSize: compressed.length,
+      compressionRatio: (compressed.length / jsonString.length).toFixed(2)
+    };
+  }
+
+  /**
+   * Decompress data if needed
+   * @private
+   */
+  _decompressData(storedData) {
+    if (!storedData || !storedData.compressed) {
+      // Not compressed - parse JSON directly
+      return storedData && storedData.data ? JSON.parse(storedData.data) : null;
+    }
+
+    // Decompress gzip data
+    const buffer = Buffer.from(storedData.data, 'base64');
+    const decompressed = zlib.unzipSync(buffer).toString();
+    return JSON.parse(decompressed);
   }
 
   async _set(key, data) {
-    let body = JSON.stringify(data);
-    const lengthSerialized = body.length;
-    body = zlib.gzipSync(body).toString('base64');
-
-    return this.client.putObject({
-      key: join(this.keyPrefix, key),
-      body,
-      contentEncoding: "gzip",
-      contentType: "application/gzip",
-      metadata: {
-        compressor: "zlib",
-        compressed: 'true',
-        "client-id": this.client.id,
-        "length-serialized": String(lengthSerialized),
-        "length-compressed": String(body.length),
-        "compression-gain": (body.length/lengthSerialized).toFixed(2),
-      },
-    });
+    const compressed = this._compressData(data);
+
+    // Use PluginStorage with body-only behavior (compressed data doesn't benefit from metadata encoding)
+    // TTL is handled automatically by PluginStorage
+    return this.storage.set(
+      this.storage.getPluginKey(null, this.keyPrefix, key),
+      compressed,
+      {
+        ttl: this.config.ttl,
+        behavior: 'body-only', // Compressed data is already optimized, skip metadata encoding
+        contentType: compressed.compressed ? 'application/gzip' : 'application/json'
+      }
+    );
   }
 
   async _get(key) {
-    const [ok, err, result] = await tryFn(async () => {
-      const { Body } = await this.client.getObject(join(this.keyPrefix, key));
-      let content = await streamToString(Body);
-      content = Buffer.from(content, 'base64');
-      content = zlib.unzipSync(content).toString();
-      return JSON.parse(content);
-    });
-    if (ok) return result;
-    if (err.name === 'NoSuchKey' || err.name === 'NotFound') return null;
-    throw err;
+    // PluginStorage automatically checks TTL and deletes expired items
+    const storedData = await this.storage.get(
+      this.storage.getPluginKey(null, this.keyPrefix, key)
+    );
+
+    if (!storedData) return null;
+
+    return this._decompressData(storedData);
   }
 
   async _del(key) {
-    await this.client.deleteObject(join(this.keyPrefix, key));
-    return true
+    await this.storage.delete(
+      this.storage.getPluginKey(null, this.keyPrefix, key)
+    );
+    return true;
   }
 
   async _clear() {
-    const keys = await this.client.getAllKeys({ 
-      prefix: this.keyPrefix,
-    });
+    // Get all keys with the cache plugin prefix
+    const pluginPrefix = `plugin=cache/${this.keyPrefix}`;
+    const allKeys = await this.client.getAllKeys({ prefix: pluginPrefix });
 
-    await this.client.deleteObjects(keys);
+    // Delete all cache keys
+    for (const key of allKeys) {
+      await this.storage.delete(key);
+    }
   }
 
   async size() {
@@ -179,10 +224,13 @@ export class S3Cache extends Cache {
   }
 
   async keys() {
-    // Busca todas as chaves com o prefixo do cache e remove o prefixo
-    const allKeys = await this.client.getAllKeys({ prefix: this.keyPrefix });
-    const prefix = this.keyPrefix.endsWith('/') ? this.keyPrefix : this.keyPrefix + '/';
-    return allKeys.map(k => k.startsWith(prefix) ? k.slice(prefix.length) : k);
+    // Get all keys with the cache plugin prefix
+    const pluginPrefix = `plugin=cache/${this.keyPrefix}`;
+    const allKeys = await this.client.getAllKeys({ prefix: pluginPrefix });
+
+    // Remove the plugin prefix to return just the cache keys
+    const prefixToRemove = `plugin=cache/${this.keyPrefix}/`;
+    return allKeys.map(k => k.startsWith(prefixToRemove) ? k.slice(prefixToRemove.length) : k);
   }
 }
 

package/src/plugins/cache.plugin.js

@@ -126,6 +126,16 @@ export class CachePlugin extends Plugin {
       // Logging
       verbose: options.verbose || false
     };
+
+    // Initialize stats tracking
+    this.stats = {
+      hits: 0,
+      misses: 0,
+      writes: 0,
+      deletes: 0,
+      errors: 0,
+      startTime: Date.now()
+    };
   }
 
   async onInstall() {
@@ -332,30 +342,46 @@ export class CachePlugin extends Plugin {
             partition,
             partitionValues
           }));
-          
-          if (ok && result !== null && result !== undefined) return result;
-          if (!ok && err.name !== 'NoSuchKey') throw err;
-          
+
+          if (ok && result !== null && result !== undefined) {
+            this.stats.hits++;
+            return result;
+          }
+          if (!ok && err.name !== 'NoSuchKey') {
+            this.stats.errors++;
+            throw err;
+          }
+
           // Not cached, call next
+          this.stats.misses++;
           const freshResult = await next();
-          
+
           // Store with partition context
+          this.stats.writes++;
           await resource.cache._set(key, freshResult, {
             resource: resource.name,
             action: method,
             partition,
             partitionValues
           });
-          
+
           return freshResult;
         } else {
           // Standard cache behavior
           const [ok, err, result] = await tryFn(() => resource.cache.get(key));
-          if (ok && result !== null && result !== undefined) return result;
-          if (!ok && err.name !== 'NoSuchKey') throw err;
-          
+          if (ok && result !== null && result !== undefined) {
+            this.stats.hits++;
+            return result;
+          }
+          if (!ok && err.name !== 'NoSuchKey') {
+            this.stats.errors++;
+            throw err;
+          }
+
           // Not cached, call next
+          this.stats.misses++;
           const freshResult = await next();
+          this.stats.writes++;
           await resource.cache.set(key, freshResult);
           return freshResult;
         }
@@ -476,6 +502,7 @@ export class CachePlugin extends Plugin {
       const [ok, err] = await tryFn(() => cache.clear(key));
 
       if (ok) {
+        this.stats.deletes++;
         return [true, null];
       }
 
@@ -682,6 +709,77 @@ export class CachePlugin extends Plugin {
 
     return analysis;
   }
+
+  /**
+   * Get cache statistics including hit/miss rates
+   * @returns {Object} Stats object with hits, misses, writes, deletes, errors, and calculated metrics
+   */
+  getStats() {
+    const total = this.stats.hits + this.stats.misses;
+    const hitRate = total > 0 ? (this.stats.hits / total) * 100 : 0;
+    const missRate = total > 0 ? (this.stats.misses / total) * 100 : 0;
+    const uptime = Date.now() - this.stats.startTime;
+    const uptimeSeconds = Math.floor(uptime / 1000);
+
+    return {
+      // Raw counters
+      hits: this.stats.hits,
+      misses: this.stats.misses,
+      writes: this.stats.writes,
+      deletes: this.stats.deletes,
+      errors: this.stats.errors,
+
+      // Calculated metrics
+      total,
+      hitRate: hitRate.toFixed(2) + '%',
+      missRate: missRate.toFixed(2) + '%',
+      hitRateDecimal: hitRate / 100,
+      missRateDecimal: missRate / 100,
+
+      // Uptime
+      uptime: uptimeSeconds,
+      uptimeFormatted: this._formatUptime(uptimeSeconds),
+      startTime: new Date(this.stats.startTime).toISOString(),
+
+      // Rates per second
+      hitsPerSecond: uptimeSeconds > 0 ? (this.stats.hits / uptimeSeconds).toFixed(2) : 0,
+      missesPerSecond: uptimeSeconds > 0 ? (this.stats.misses / uptimeSeconds).toFixed(2) : 0,
+      writesPerSecond: uptimeSeconds > 0 ? (this.stats.writes / uptimeSeconds).toFixed(2) : 0
+    };
+  }
+
+  /**
+   * Reset cache statistics
+   */
+  resetStats() {
+    this.stats = {
+      hits: 0,
+      misses: 0,
+      writes: 0,
+      deletes: 0,
+      errors: 0,
+      startTime: Date.now()
+    };
+  }
+
+  /**
+   * Format uptime in human-readable format
+   * @private
+   */
+  _formatUptime(seconds) {
+    const days = Math.floor(seconds / 86400);
+    const hours = Math.floor((seconds % 86400) / 3600);
+    const minutes = Math.floor((seconds % 3600) / 60);
+    const secs = seconds % 60;
+
+    const parts = [];
+    if (days > 0) parts.push(`${days}d`);
+    if (hours > 0) parts.push(`${hours}h`);
+    if (minutes > 0) parts.push(`${minutes}m`);
+    if (secs > 0 || parts.length === 0) parts.push(`${secs}s`);
+
+    return parts.join(' ');
+  }
 }
 
 export default CachePlugin;

package/src/plugins/concerns/plugin-dependencies.js

@@ -0,0 +1,313 @@
+/**
+ * Plugin Dependency Validation System
+ *
+ * Validates that optional plugin dependencies are installed and meet version requirements.
+ * This keeps the core s3db.js package lightweight while ensuring plugins work correctly.
+ *
+ * @example
+ * // In a plugin constructor:
+ * await requirePluginDependency('postgresql-replicator');
+ */
+
+/**
+ * Plugin dependency registry
+ * Maps plugin identifiers to their required dependencies
+ */
+export const PLUGIN_DEPENDENCIES = {
+  'postgresql-replicator': {
+    name: 'PostgreSQL Replicator',
+    dependencies: {
+      'pg': {
+        version: '^8.0.0',
+        description: 'PostgreSQL client for Node.js',
+        installCommand: 'pnpm add pg'
+      }
+    }
+  },
+  'bigquery-replicator': {
+    name: 'BigQuery Replicator',
+    dependencies: {
+      '@google-cloud/bigquery': {
+        version: '^7.0.0',
+        description: 'Google Cloud BigQuery SDK',
+        installCommand: 'pnpm add @google-cloud/bigquery'
+      }
+    }
+  },
+  'sqs-replicator': {
+    name: 'SQS Replicator',
+    dependencies: {
+      '@aws-sdk/client-sqs': {
+        version: '^3.0.0',
+        description: 'AWS SDK for SQS',
+        installCommand: 'pnpm add @aws-sdk/client-sqs'
+      }
+    }
+  },
+  'sqs-consumer': {
+    name: 'SQS Queue Consumer',
+    dependencies: {
+      '@aws-sdk/client-sqs': {
+        version: '^3.0.0',
+        description: 'AWS SDK for SQS',
+        installCommand: 'pnpm add @aws-sdk/client-sqs'
+      }
+    }
+  },
+  'rabbitmq-consumer': {
+    name: 'RabbitMQ Queue Consumer',
+    dependencies: {
+      'amqplib': {
+        version: '^0.10.0',
+        description: 'AMQP 0-9-1 library for RabbitMQ',
+        installCommand: 'pnpm add amqplib'
+      }
+    }
+  },
+  'tfstate-plugin': {
+    name: 'Terraform State Plugin',
+    dependencies: {
+      'node-cron': {
+        version: '^4.0.0',
+        description: 'Cron job scheduler for auto-sync functionality',
+        installCommand: 'pnpm add node-cron'
+      }
+    }
+  },
+  'api-plugin': {
+    name: 'API Plugin',
+    dependencies: {
+      'hono': {
+        version: '^4.0.0',
+        description: 'Ultra-light HTTP server framework',
+        installCommand: 'pnpm add hono'
+      },
+      '@hono/node-server': {
+        version: '^1.0.0',
+        description: 'Node.js adapter for Hono',
+        installCommand: 'pnpm add @hono/node-server'
+      },
+      '@hono/swagger-ui': {
+        version: '^0.4.0',
+        description: 'Swagger UI integration for Hono',
+        installCommand: 'pnpm add @hono/swagger-ui'
+      }
+    }
+  }
+};
+
+/**
+ * Simple semver comparison for major version checking
+ * @param {string} actual - Actual version (e.g., "8.11.3")
+ * @param {string} required - Required version range (e.g., "^8.0.0")
+ * @returns {boolean} True if version is compatible
+ */
+function isVersionCompatible(actual, required) {
+  if (!actual || !required) return false;
+
+  // Remove ^ and ~ prefixes
+  const cleanRequired = required.replace(/^[\^~]/, '');
+
+  // Extract major versions
+  const actualMajor = parseInt(actual.split('.')[0], 10);
+  const requiredMajor = parseInt(cleanRequired.split('.')[0], 10);
+
+  // For ^X.Y.Z, accept any version >= X.Y.Z with same major
+  if (required.startsWith('^')) {
+    return actualMajor === requiredMajor;
+  }
+
+  // For ~X.Y.Z, accept any version >= X.Y.Z with same major.minor
+  if (required.startsWith('~')) {
+    const actualMinor = parseInt(actual.split('.')[1] || '0', 10);
+    const requiredMinor = parseInt(cleanRequired.split('.')[1] || '0', 10);
+    return actualMajor === requiredMajor && actualMinor >= requiredMinor;
+  }
+
+  // Exact match for unspecified ranges
+  return actualMajor >= requiredMajor;
+}
+
+/**
+ * Try to load a package and get its version
+ * @param {string} packageName - NPM package name
+ * @returns {Promise<{installed: boolean, version: string|null, error: Error|null}>}
+ */
+async function tryLoadPackage(packageName) {
+  try {
+    // Try to import the package
+    const pkg = await import(packageName);
+
+    // Try to get version from package.json
+    let version = null;
+    try {
+      const pkgJson = await import(`${packageName}/package.json`, {
+        assert: { type: 'json' }
+      });
+      version = pkgJson.default?.version || pkgJson.version || null;
+    } catch (e) {
+      // Package.json not accessible, version unknown but package exists
+      version = 'unknown';
+    }
+
+    return { installed: true, version, error: null };
+  } catch (error) {
+    return { installed: false, version: null, error };
+  }
+}
+
+/**
+ * Validate that a plugin's dependencies are installed and meet version requirements
+ * @param {string} pluginId - Plugin identifier from PLUGIN_DEPENDENCIES
+ * @param {Object} options - Validation options
+ * @param {boolean} options.throwOnError - Throw error if validation fails (default: true)
+ * @param {boolean} options.checkVersions - Check version compatibility (default: true)
+ * @returns {Promise<{valid: boolean, missing: string[], incompatible: string[], messages: string[]}>}
+ * @throws {Error} If throwOnError=true and validation fails
+ */
+export async function requirePluginDependency(pluginId, options = {}) {
+  const {
+    throwOnError = true,
+    checkVersions = true
+  } = options;
+
+  const pluginDef = PLUGIN_DEPENDENCIES[pluginId];
+
+  if (!pluginDef) {
+    const error = new Error(
+      `Unknown plugin identifier: ${pluginId}. ` +
+      `Available plugins: ${Object.keys(PLUGIN_DEPENDENCIES).join(', ')}`
+    );
+    if (throwOnError) throw error;
+    return { valid: false, missing: [], incompatible: [], messages: [error.message] };
+  }
+
+  const missing = [];
+  const incompatible = [];
+  const messages = [];
+
+  // Check each dependency
+  for (const [pkgName, pkgInfo] of Object.entries(pluginDef.dependencies)) {
+    const { installed, version, error } = await tryLoadPackage(pkgName);
+
+    if (!installed) {
+      missing.push(pkgName);
+      messages.push(
+        `❌ Missing dependency: ${pkgName}\n` +
+        `   Description: ${pkgInfo.description}\n` +
+        `   Required: ${pkgInfo.version}\n` +
+        `   Install: ${pkgInfo.installCommand}`
+      );
+      continue;
+    }
+
+    // Check version compatibility if requested
+    if (checkVersions && version && version !== 'unknown') {
+      const compatible = isVersionCompatible(version, pkgInfo.version);
+
+      if (!compatible) {
+        incompatible.push(pkgName);
+        messages.push(
+          `⚠️  Incompatible version: ${pkgName}\n` +
+          `   Installed: ${version}\n` +
+          `   Required: ${pkgInfo.version}\n` +
+          `   Update: ${pkgInfo.installCommand}`
+        );
+      } else {
+        messages.push(
+          `✅ ${pkgName}@${version} (compatible with ${pkgInfo.version})`
+        );
+      }
+    } else {
+      messages.push(
+        `✅ ${pkgName}@${version || 'unknown'} (installed)`
+      );
+    }
+  }
+
+  const valid = missing.length === 0 && incompatible.length === 0;
+
+  // Throw comprehensive error if validation failed
+  if (!valid && throwOnError) {
+    const errorMsg = [
+      `\n${pluginDef.name} - Missing dependencies detected!\n`,
+      `Plugin ID: ${pluginId}`,
+      '',
+      ...messages,
+      '',
+      'Quick fix - Run all install commands:',
+      Object.values(pluginDef.dependencies)
+        .map(dep => `  ${dep.installCommand}`)
+        .join('\n'),
+      '',
+      'Or install all peer dependencies at once:',
+      `  pnpm add ${Object.keys(pluginDef.dependencies).join(' ')}`
+    ].join('\n');
+
+    throw new Error(errorMsg);
+  }
+
+  return { valid, missing, incompatible, messages };
+}
+
+/**
+ * Check multiple plugin dependencies at once
+ * @param {string[]} pluginIds - Array of plugin identifiers
+ * @param {Object} options - Validation options
+ * @returns {Promise<Map<string, {valid: boolean, missing: string[], incompatible: string[], messages: string[]}>>}
+ */
+export async function checkPluginDependencies(pluginIds, options = {}) {
+  const results = new Map();
+
+  for (const pluginId of pluginIds) {
+    const result = await requirePluginDependency(pluginId, {
+      ...options,
+      throwOnError: false
+    });
+    results.set(pluginId, result);
+  }
+
+  return results;
+}
+
+/**
+ * Get a report of all plugin dependencies and their status
+ * @returns {Promise<string>} Formatted report
+ */
+export async function getPluginDependencyReport() {
+  const pluginIds = Object.keys(PLUGIN_DEPENDENCIES);
+  const results = await checkPluginDependencies(pluginIds);
+
+  const lines = [
+    '╔═══════════════════════════════════════════════════════════════╗',
+    '║           S3DB.JS - Plugin Dependency Status Report          ║',
+    '╚═══════════════════════════════════════════════════════════════╝',
+    ''
+  ];
+
+  for (const [pluginId, result] of results.entries()) {
+    const pluginDef = PLUGIN_DEPENDENCIES[pluginId];
+    const status = result.valid ? '✅ READY' : '❌ MISSING';
+
+    lines.push(`${status} - ${pluginDef.name}`);
+
+    if (result.messages.length > 0) {
+      result.messages.forEach(msg => {
+        lines.push(`      ${msg.replace(/\n/g, '\n      ')}`);
+      });
+    }
+
+    lines.push('');
+  }
+
+  const totalPlugins = pluginIds.length;
+  const readyPlugins = Array.from(results.values()).filter(r => r.valid).length;
+
+  lines.push('─────────────────────────────────────────────────────────────────');
+  lines.push(`Summary: ${readyPlugins}/${totalPlugins} plugins ready to use`);
+  lines.push('─────────────────────────────────────────────────────────────────');
+
+  return lines.join('\n');
+}
+
+export default requirePluginDependency;