s3db.js 6.2.0 → 7.0.0

package/src/plugins/cache/s3-cache.class.js

@@ -0,0 +1,189 @@
+/**
+ * S3 Cache Configuration Documentation
+ * 
+ * This cache implementation stores data in Amazon S3, providing persistent storage
+ * that survives process restarts and can be shared across multiple instances.
+ * It's suitable for large datasets and distributed caching scenarios.
+ * 
+ * @typedef {Object} S3CacheConfig
+ * @property {string} bucket - The name of the S3 bucket to use for cache storage
+ * @property {string} [region='us-east-1'] - AWS region where the S3 bucket is located
+ * @property {string} [accessKeyId] - AWS access key ID (if not using IAM roles)
+ * @property {string} [secretAccessKey] - AWS secret access key (if not using IAM roles)
+ * @property {string} [sessionToken] - AWS session token for temporary credentials
+ * @property {string} [prefix='cache/'] - S3 key prefix for all cache objects
+ * @property {number} [ttl=3600000] - Time to live in milliseconds (1 hour default)
+ * @property {boolean} [enableCompression=true] - Whether to compress cache values using gzip
+ * @property {number} [compressionThreshold=1024] - Minimum size in bytes to trigger compression
+ * @property {string} [storageClass='STANDARD'] - S3 storage class: 'STANDARD', 'STANDARD_IA', 'ONEZONE_IA', 'GLACIER', 'DEEP_ARCHIVE'
+ * @property {boolean} [enableEncryption=true] - Whether to use S3 server-side encryption (AES256)
+ * @property {string} [encryptionAlgorithm='AES256'] - Encryption algorithm: 'AES256' or 'aws:kms'
+ * @property {string} [kmsKeyId] - KMS key ID for encryption (if using aws:kms)
+ * @property {number} [maxConcurrency=10] - Maximum number of concurrent S3 operations
+ * @property {number} [retryAttempts=3] - Number of retry attempts for failed S3 operations
+ * @property {number} [retryDelay=1000] - Delay in milliseconds between retry attempts
+ * @property {boolean} [logOperations=false] - Whether to log S3 operations to console for debugging
+ * @property {Object} [metadata] - Additional metadata to include with all cache objects
+ *   - Key: metadata name (e.g., 'environment', 'version')
+ *   - Value: metadata value (e.g., 'production', '1.0.0')
+ * @property {string} [contentType='application/json'] - Content type for cache objects
+ * @property {boolean} [enableVersioning=false] - Whether to enable S3 object versioning for cache objects
+ * @property {number} [maxKeys=1000] - Maximum number of keys to retrieve in list operations
+ * @property {boolean} [enableCacheControl=false] - Whether to set Cache-Control headers on S3 objects
+ * @property {string} [cacheControl='max-age=3600'] - Cache-Control header value for S3 objects
+ * @property {Object} [s3ClientOptions] - Additional options to pass to the S3 client constructor
+ * @property {boolean} [enableLocalCache=false] - Whether to use local memory cache as a layer on top of S3
+ * @property {number} [localCacheSize=100] - Size of local memory cache when enabled
+ * @property {number} [localCacheTtl=300000] - TTL for local memory cache in milliseconds (5 minutes default)
+ * 
+ * @example
+ * // Basic configuration with compression and encryption
+ * {
+ *   bucket: 'my-cache-bucket',
+ *   region: 'us-west-2',
+ *   accessKeyId: 'AKIAIOSFODNN7EXAMPLE',
+ *   secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+ *   prefix: 'app-cache/',
+ *   ttl: 7200000, // 2 hours
+ *   enableCompression: true,
+ *   enableEncryption: true,
+ *   storageClass: 'STANDARD_IA'
+ * }
+ * 
+ * @example
+ * // Configuration with KMS encryption and local caching
+ * {
+ *   bucket: 'secure-cache-bucket',
+ *   region: 'eu-west-1',
+ *   prefix: 'encrypted-cache/',
+ *   enableEncryption: true,
+ *   encryptionAlgorithm: 'aws:kms',
+ *   kmsKeyId: 'arn:aws:kms:eu-west-1:123456789012:key/abcd1234-5678-90ef-ghij-klmnopqrstuv',
+ *   enableLocalCache: true,
+ *   localCacheSize: 500,
+ *   localCacheTtl: 600000, // 10 minutes
+ *   metadata: {
+ *     'environment': 'production',
+ *     'cache_type': 's3'
+ *   }
+ * }
+ * 
+ * @example
+ * // Configuration with cost optimization
+ * {
+ *   bucket: 'cost-optimized-cache',
+ *   region: 'us-east-1',
+ *   prefix: 'cache/',
+ *   storageClass: 'STANDARD_IA',
+ *   ttl: 86400000, // 24 hours
+ *   enableCompression: true,
+ *   compressionThreshold: 512,
+ *   maxConcurrency: 5,
+ *   enableCacheControl: true,
+ *   cacheControl: 'max-age=86400, public'
+ * }
+ * 
+ * @example
+ * // Minimal configuration using IAM roles
+ * {
+ *   bucket: 'my-cache-bucket',
+ *   region: 'us-east-1'
+ * }
+ * 
+ * @notes
+ * - Requires AWS credentials with S3 read/write permissions
+ * - S3 storage costs depend on storage class and data transfer
+ * - Compression reduces storage costs but increases CPU usage
+ * - Encryption provides security but may impact performance
+ * - Local cache layer improves performance for frequently accessed data
+ * - Storage class affects cost, availability, and retrieval time
+ * - Versioning allows recovery of deleted cache objects
+ * - Cache-Control headers help with CDN integration
+ * - Retry mechanism handles temporary S3 service issues
+ * - Concurrent operations improve performance but may hit rate limits
+ * - Metadata is useful for cache management and monitoring
+ * - TTL is enforced by checking object creation time
+ */
+import zlib from "zlib";
+import { join } from "path";
+
+import { Cache } from "./cache.class.js"
+import { streamToString } from "#src/stream/index.js";
+import tryFn from "../../concerns/try-fn.js";
+
+export class S3Cache extends Cache {
+  constructor({ 
+    client, 
+    keyPrefix = 'cache',
+    ttl = 0,
+    prefix = undefined
+  }) {
+    super({ client, keyPrefix, ttl, prefix });
+    this.client = client
+    this.keyPrefix = keyPrefix;
+    this.config.ttl = ttl;
+    this.config.client = client;
+    this.config.prefix = prefix !== undefined ? prefix : keyPrefix + (keyPrefix.endsWith('/') ? '' : '/');
+  }
+
+  async _set(key, data) {
+    let body = JSON.stringify(data);
+    const lengthSerialized = body.length;
+    body = zlib.gzipSync(body).toString('base64');
+
+    return this.client.putObject({
+      key: join(this.keyPrefix, key),
+      body,
+      contentEncoding: "gzip",
+      contentType: "application/gzip",
+      metadata: {
+        compressor: "zlib",
+        compressed: 'true',
+        "client-id": this.client.id,
+        "length-serialized": String(lengthSerialized),
+        "length-compressed": String(body.length),
+        "compression-gain": (body.length/lengthSerialized).toFixed(2),
+      },
+    });
+  }
+
+  async _get(key) {
+    const [ok, err, result] = await tryFn(async () => {
+      const { Body } = await this.client.getObject(join(this.keyPrefix, key));
+      let content = await streamToString(Body);
+      content = Buffer.from(content, 'base64');
+      content = zlib.unzipSync(content).toString();
+      return JSON.parse(content);
+    });
+    if (ok) return result;
+    if (err.name === 'NoSuchKey' || err.name === 'NotFound') return null;
+    throw err;
+  }
+
+  async _del(key) {
+    await this.client.deleteObject(join(this.keyPrefix, key));
+    return true
+  }
+
+  async _clear() {
+    const keys = await this.client.getAllKeys({ 
+      prefix: this.keyPrefix,
+    });
+
+    await this.client.deleteObjects(keys);
+  }
+
+  async size() {
+    const keys = await this.keys();
+    return keys.length;
+  }
+
+  async keys() {
+    // Busca todas as chaves com o prefixo do cache e remove o prefixo
+    const allKeys = await this.client.getAllKeys({ prefix: this.keyPrefix });
+    const prefix = this.keyPrefix.endsWith('/') ? this.keyPrefix : this.keyPrefix + '/';
+    return allKeys.map(k => k.startsWith(prefix) ? k.slice(prefix.length) : k);
+  }
+}
+
+export default S3Cache

package/src/plugins/cache.plugin.js

@@ -0,0 +1,275 @@
+import { join } from "path";
+
+import { sha256 } from "../concerns/crypto.js";
+import Plugin from "./plugin.class.js";
+import S3Cache from "./cache/s3-cache.class.js";
+import MemoryCache from "./cache/memory-cache.class.js";
+import tryFn from "../concerns/try-fn.js";
+
+export class CachePlugin extends Plugin {
+  constructor(options = {}) {
+    super(options);
+    this.driver = options.driver;
+    this.config = {
+      includePartitions: options.includePartitions !== false,
+      ...options
+    };
+  }
+
+  async setup(database) {
+    await super.setup(database);
+  }
+
+  async onSetup() {
+    // Initialize cache driver
+    if (this.config.driver) {
+      // Use custom driver if provided
+      this.driver = this.config.driver;
+    } else if (this.config.driverType === 'memory') {
+      this.driver = new MemoryCache(this.config.memoryOptions || {});
+    } else {
+      // Default to S3Cache, sempre passa o client do database
+      this.driver = new S3Cache({ client: this.database.client, ...(this.config.s3Options || {}) });
+    }
+
+    // Install database proxy for new resources
+    this.installDatabaseProxy();
+    
+    // Install hooks for existing resources
+    this.installResourceHooks();
+  }
+
+  async onStart() {
+    // Plugin is ready
+  }
+
+  async onStop() {
+    // Cleanup if needed
+  }
+
+  installDatabaseProxy() {
+    if (this.database._cacheProxyInstalled) {
+      return; // Already installed
+    }
+    
+    const installResourceHooks = this.installResourceHooks.bind(this);
+    
+    // Store original method
+    this.database._originalCreateResourceForCache = this.database.createResource;
+    
+    // Create new method that doesn't call itself
+    this.database.createResource = async function (...args) {
+      const resource = await this._originalCreateResourceForCache(...args);
+      installResourceHooks(resource);
+      return resource;
+    };
+    
+    // Mark as installed
+    this.database._cacheProxyInstalled = true;
+  }
+
+  installResourceHooks() {
+    for (const resource of Object.values(this.database.resources)) {
+      this.installResourceHooksForResource(resource);
+    }
+  }
+
+  installResourceHooksForResource(resource) {
+    if (!this.driver) return;
+
+    // Add cache methods to resource
+    Object.defineProperty(resource, 'cache', {
+      value: this.driver,
+      writable: true,
+      configurable: true,
+      enumerable: false
+    });
+    resource.cacheKeyFor = async (options = {}) => {
+      const { action, params = {}, partition, partitionValues } = options;
+      return this.generateCacheKey(resource, action, params, partition, partitionValues);
+    };
+
+    // List of methods to cache
+    const cacheMethods = [
+      'count', 'listIds', 'getMany', 'getAll', 'page', 'list', 'get'
+    ];
+    for (const method of cacheMethods) {
+      resource.useMiddleware(method, async (ctx, next) => {
+        // Build cache key
+        let key;
+        if (method === 'getMany') {
+          key = await resource.cacheKeyFor({ action: method, params: { ids: ctx.args[0] } });
+        } else if (method === 'page') {
+          const { offset, size, partition, partitionValues } = ctx.args[0] || {};
+          key = await resource.cacheKeyFor({ action: method, params: { offset, size }, partition, partitionValues });
+        } else if (method === 'list' || method === 'listIds' || method === 'count') {
+          const { partition, partitionValues } = ctx.args[0] || {};
+          key = await resource.cacheKeyFor({ action: method, partition, partitionValues });
+        } else if (method === 'getAll') {
+          key = await resource.cacheKeyFor({ action: method });
+        } else if (method === 'get') {
+          key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
+        }
+        // Try cache
+        const [ok, err, cached] = await tryFn(() => resource.cache.get(key));
+        if (ok && cached !== null && cached !== undefined) return cached;
+        if (!ok && err.name !== 'NoSuchKey') throw err;
+        // Not cached, call next
+        const result = await next();
+        await resource.cache.set(key, result);
+        return result;
+      });
+    }
+
+    // List of methods to clear cache on write
+    const writeMethods = ['insert', 'update', 'delete', 'deleteMany'];
+    for (const method of writeMethods) {
+      resource.useMiddleware(method, async (ctx, next) => {
+        const result = await next();
+        // Determine which records to clear
+        if (method === 'insert') {
+          await this.clearCacheForResource(resource, ctx.args[0]);
+        } else if (method === 'update') {
+          await this.clearCacheForResource(resource, { id: ctx.args[0], ...ctx.args[1] });
+        } else if (method === 'delete') {
+          let data = { id: ctx.args[0] };
+          if (typeof resource.get === 'function') {
+            const [ok, err, full] = await tryFn(() => resource.get(ctx.args[0]));
+            if (ok && full) data = full;
+          }
+          await this.clearCacheForResource(resource, data);
+        } else if (method === 'deleteMany') {
+          // After all deletions, clear all aggregate and partition caches
+          await this.clearCacheForResource(resource);
+        }
+        return result;
+      });
+    }
+  }
+
+  async clearCacheForResource(resource, data) {
+    if (!resource.cache) return; // Skip if no cache is available
+    
+    const keyPrefix = `resource=${resource.name}`;
+    
+    // Always clear main cache for this resource
+    await resource.cache.clear(keyPrefix);
+    
+    // Only clear partition cache if partitions are enabled AND resource has partitions AND includePartitions is true
+    if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
+      if (!data) {
+        // If no data, clear all partition caches
+        for (const partitionName of Object.keys(resource.config.partitions)) {
+          const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
+          await resource.cache.clear(partitionKeyPrefix);
+        }
+      } else {
+        const partitionValues = this.getPartitionValues(data, resource);
+        for (const [partitionName, values] of Object.entries(partitionValues)) {
+          // Only clear partition cache if there are actual values
+          if (values && Object.keys(values).length > 0 && Object.values(values).some(v => v !== null && v !== undefined)) {
+            const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
+            await resource.cache.clear(partitionKeyPrefix);
+          }
+        }
+      }
+    }
+  }
+
+  async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
+    const keyParts = [
+      `resource=${resource.name}`,
+      `action=${action}`
+    ];
+
+    // Add partition information if available
+    if (partition && partitionValues && Object.keys(partitionValues).length > 0) {
+      keyParts.push(`partition:${partition}`);
+      for (const [field, value] of Object.entries(partitionValues)) {
+        if (value !== null && value !== undefined) {
+          keyParts.push(`${field}:${value}`);
+        }
+      }
+    }
+
+    // Add params if they exist
+    if (Object.keys(params).length > 0) {
+      const paramsHash = await this.hashParams(params);
+      keyParts.push(paramsHash);
+    }
+
+    return join(...keyParts) + '.json.gz';
+  }
+
+  async hashParams(params) {
+    const sortedParams = Object.keys(params)
+      .sort()
+      .map(key => `${key}:${params[key]}`)
+      .join('|') || 'empty';
+    
+    return await sha256(sortedParams);
+  }
+
+  // Utility methods
+  async getCacheStats() {
+    if (!this.driver) return null;
+    
+    return {
+      size: await this.driver.size(),
+      keys: await this.driver.keys(),
+      driver: this.driver.constructor.name
+    };
+  }
+
+  async clearAllCache() {
+    if (!this.driver) return;
+    
+    for (const resource of Object.values(this.database.resources)) {
+      if (resource.cache) {
+        const keyPrefix = `resource=${resource.name}`;
+        await resource.cache.clear(keyPrefix);
+      }
+    }
+  }
+
+  async warmCache(resourceName, options = {}) {
+    const resource = this.database.resources[resourceName];
+    if (!resource) {
+      throw new Error(`Resource '${resourceName}' not found`);
+    }
+
+    const { includePartitions = true } = options;
+
+    // Warm main cache using the wrapped method (which will call the original)
+    await resource.getAll();
+
+    // Warm partition caches if enabled
+    if (includePartitions && resource.config.partitions) {
+      for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
+        if (partitionDef.fields) {
+          // Get some sample partition values and warm those caches
+          const allRecords = await resource.getAll();
+          
+          // Ensure allRecords is an array
+          const recordsArray = Array.isArray(allRecords) ? allRecords : [];
+          const partitionValues = new Set();
+          
+          for (const record of recordsArray.slice(0, 10)) { // Sample first 10 records
+            const values = this.getPartitionValues(record, resource);
+            if (values[partitionName]) {
+              partitionValues.add(JSON.stringify(values[partitionName]));
+            }
+          }
+          
+          // Warm cache for each partition value
+          for (const partitionValueStr of partitionValues) {
+            const partitionValues = JSON.parse(partitionValueStr);
+            await resource.list({ partition: partitionName, partitionValues });
+          }
+        }
+      }
+    }
+  }
+}
+
+export default CachePlugin;

package/src/plugins/consumers/index.js

@@ -0,0 +1,24 @@
+import { SqsConsumer } from './sqs-consumer.js';
+import { RabbitMqConsumer } from './rabbitmq-consumer.js';
+
+export { SqsConsumer, RabbitMqConsumer };
+
+export const CONSUMER_DRIVERS = {
+  sqs: SqsConsumer,
+  rabbitmq: RabbitMqConsumer,
+  // kafka: KafkaConsumer, // futuro
+};
+
+/**
+ * Cria uma instância de consumer baseado no driver
+ * @param {string} driver - Tipo do driver (sqs, rabbitmq, kafka...)
+ * @param {Object} config - Configuração do consumer
+ * @returns {SqsConsumer|RabbitMqConsumer|KafkaConsumer}
+ */
+export function createConsumer(driver, config) {
+  const ConsumerClass = CONSUMER_DRIVERS[driver];
+  if (!ConsumerClass) {
+    throw new Error(`Unknown consumer driver: ${driver}. Available: ${Object.keys(CONSUMER_DRIVERS).join(', ')}`);
+  }
+  return new ConsumerClass(config);
+} 

package/src/plugins/consumers/rabbitmq-consumer.js

@@ -0,0 +1,56 @@
+import amqp from 'amqplib';
+import tryFn from "../../concerns/try-fn.js";
+
+export class RabbitMqConsumer {
+  constructor({ amqpUrl, queue, prefetch = 10, reconnectInterval = 2000, onMessage, onError, driver = 'rabbitmq' }) {
+    this.amqpUrl = amqpUrl;
+    this.queue = queue;
+    this.prefetch = prefetch;
+    this.reconnectInterval = reconnectInterval;
+    this.onMessage = onMessage;
+    this.onError = onError;
+    this.driver = driver;
+    this.connection = null;
+    this.channel = null;
+    this._stopped = false;
+  }
+
+  async start() {
+    this._stopped = false;
+    await this._connect();
+  }
+
+  async stop() {
+    this._stopped = true;https://github.com/browserbase/stagehand
+    if (this.channel) await this.channel.close();
+    if (this.connection) await this.connection.close();
+  }
+
+  async _connect() {
+    const [ok, err] = await tryFn(async () => {
+      this.connection = await amqp.connect(this.amqpUrl);
+      this.channel = await this.connection.createChannel();
+      await this.channel.assertQueue(this.queue, { durable: true });
+      this.channel.prefetch(this.prefetch);
+      this.channel.consume(this.queue, async (msg) => {
+        if (msg !== null) {
+          const [okMsg, errMsg] = await tryFn(async () => {
+            const content = JSON.parse(msg.content.toString());
+            await this.onMessage({ $body: content, $raw: msg });
+            this.channel.ack(msg);
+          });
+          if (!okMsg) {
+            if (this.onError) this.onError(errMsg, msg);
+            this.channel.nack(msg, false, false);
+          }
+        }
+      });
+    });
+    if (!ok) {
+      if (this.onError) this.onError(err);
+      if (!this._stopped) {
+        setTimeout(() => this._connect(), this.reconnectInterval);
+      }
+    }
+  }
+} 

package/src/plugins/consumers/sqs-consumer.js

@@ -0,0 +1,102 @@
+import tryFn from "../../concerns/try-fn.js";
+// Remover o import estático do SDK
+// import { SQSClient, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';
+
+export class SqsConsumer {
+  constructor({ queueUrl, onMessage, onError, poolingInterval = 5000, maxMessages = 10, region = 'us-east-1', credentials, endpoint, driver = 'sqs' }) {
+    this.driver = driver;
+    this.queueUrl = queueUrl;
+    this.onMessage = onMessage;
+    this.onError = onError;
+    this.poolingInterval = poolingInterval;
+    this.maxMessages = maxMessages;
+    this.region = region;
+    this.credentials = credentials;
+    this.endpoint = endpoint;
+    this.sqs = null; // será inicializado dinamicamente
+    this._stopped = false;
+    this._timer = null;
+    this._pollPromise = null;
+    this._pollResolve = null;
+    // SDK classes
+    this._SQSClient = null;
+    this._ReceiveMessageCommand = null;
+    this._DeleteMessageCommand = null;
+  }
+
+  async start() {
+    // Carregar SDK dinamicamente
+    const [ok, err, sdk] = await tryFn(() => import('@aws-sdk/client-sqs'));
+    if (!ok) throw new Error('SqsConsumer: @aws-sdk/client-sqs is not installed. Please install it to use the SQS consumer.');
+    const { SQSClient, ReceiveMessageCommand, DeleteMessageCommand } = sdk;
+    this._SQSClient = SQSClient;
+    this._ReceiveMessageCommand = ReceiveMessageCommand;
+    this._DeleteMessageCommand = DeleteMessageCommand;
+    this.sqs = new SQSClient({ region: this.region, credentials: this.credentials, endpoint: this.endpoint });
+    this._stopped = false;
+    this._pollPromise = new Promise((resolve) => { this._pollResolve = resolve; });
+    this._poll();
+  }
+
+  async stop() {
+    this._stopped = true;
+    if (this._timer) {
+      clearTimeout(this._timer);
+      this._timer = null;
+    }
+    // Don't wait for poll promise as it might hang for up to 10 seconds
+    // The _poll method checks _stopped and will resolve the promise
+    if (this._pollResolve) {
+      this._pollResolve();
+    }
+  }
+
+  async _poll() {
+    if (this._stopped) {
+      if (this._pollResolve) this._pollResolve();
+      return;
+    }
+    const [ok, err, result] = await tryFn(async () => {
+      const cmd = new this._ReceiveMessageCommand({
+        QueueUrl: this.queueUrl,
+        MaxNumberOfMessages: this.maxMessages,
+        WaitTimeSeconds: 10,
+        MessageAttributeNames: ['All'],
+      });
+      const { Messages } = await this.sqs.send(cmd);
+      if (Messages && Messages.length > 0) {
+        for (const msg of Messages) {
+          const [okMsg, errMsg] = await tryFn(async () => {
+            const parsedMsg = this._parseMessage(msg);
+            await this.onMessage(parsedMsg, msg);
+            // Delete after successful processing
+            await this.sqs.send(new this._DeleteMessageCommand({
+              QueueUrl: this.queueUrl,
+              ReceiptHandle: msg.ReceiptHandle
+            }));
+          });
+          if (!okMsg && this.onError) {
+            this.onError(errMsg, msg);
+          }
+        }
+      }
+    });
+    if (!ok && this.onError) {
+      this.onError(err);
+    }
+    this._timer = setTimeout(() => this._poll(), this.poolingInterval);
+  }
+
+  _parseMessage(msg) {
+    let body;
+    const [ok, err, parsed] = tryFn(() => JSON.parse(msg.Body));
+    body = ok ? parsed : msg.Body;
+    const attributes = {};
+    if (msg.MessageAttributes) {
+      for (const [k, v] of Object.entries(msg.MessageAttributes)) {
+        attributes[k] = v.StringValue;
+      }
+    }
+    return { $body: body, $attributes: attributes, $raw: msg };
+  }
+}