s3db.js 11.3.2 → 12.0.1

package/src/partition-drivers/index.js

@@ -1,66 +0,0 @@
-import { SyncPartitionDriver } from './sync-partition-driver.js';
-import { MemoryPartitionDriver } from './memory-partition-driver.js';
-import { SQSPartitionDriver } from './sqs-partition-driver.js';
-import { PartitionDriverError } from '../errors.js';
-
-/**
- * Partition driver factory
- */
-export class PartitionDriverFactory {
-  static drivers = {
-    sync: SyncPartitionDriver,
-    memory: MemoryPartitionDriver,
-    sqs: SQSPartitionDriver
-  };
-
-  /**
-   * Create a partition driver instance
-   * @param {string|Object} config - Driver name or configuration object
-   * @returns {BasePartitionDriver} Driver instance
-   */
-  static create(config) {
-    // Handle string shorthand
-    if (typeof config === 'string') {
-      config = { driver: config };
-    }
-    
-    // Default to memory driver
-    const driverName = config.driver || 'memory';
-    
-    // Get driver class
-    const DriverClass = this.drivers[driverName];
-    if (!DriverClass) {
-      throw new PartitionDriverError(`Unknown partition driver: ${driverName}`, {
-        driver: driverName,
-        operation: 'create',
-        availableDrivers: Object.keys(this.drivers),
-        suggestion: `Use one of the available drivers: ${Object.keys(this.drivers).join(', ')}, or register a custom driver`
-      });
-    }
-    
-    // Create and initialize driver
-    const driver = new DriverClass(config);
-    
-    return driver;
-  }
-
-  /**
-   * Register a custom driver
-   */
-  static register(name, DriverClass) {
-    this.drivers[name] = DriverClass;
-  }
-
-  /**
-   * Get available driver names
-   */
-  static getAvailableDrivers() {
-    return Object.keys(this.drivers);
-  }
-}
-
-// Export individual drivers
-export { BasePartitionDriver } from './base-partition-driver.js';
-export { SyncPartitionDriver } from './sync-partition-driver.js';
-export { MemoryPartitionDriver } from './memory-partition-driver.js';
-export { SQSPartitionDriver } from './sqs-partition-driver.js';

package/src/partition-drivers/memory-partition-driver.js

@@ -1,289 +0,0 @@
-import { BasePartitionDriver } from './base-partition-driver.js';
-import { PromisePool } from '@supercharge/promise-pool';
-import { PartitionDriverError } from '../errors.js';
-
-/**
- * In-memory partition driver with background processing
- * Queues operations in memory and processes them asynchronously
- * Fast and efficient for single-instance applications
- */
-export class MemoryPartitionDriver extends BasePartitionDriver {
-  constructor(options = {}) {
-    super(options);
-    this.name = 'memory';
-    
-    // Configuration
-    this.batchSize = options.batchSize || 100;
-    this.concurrency = options.concurrency || 10;
-    this.flushInterval = options.flushInterval || 1000;
-    this.maxQueueSize = options.maxQueueSize || 10000;
-    this.maxRetries = options.maxRetries || 3;
-    
-    // State
-    this.queue = [];
-    this.isProcessing = false;
-    this.flushTimer = null;
-    this.retryQueue = [];
-  }
-
-  async initialize() {
-    // Start background processor
-    this.startProcessor();
-  }
-
-  /**
-   * Add operation to in-memory queue
-   */
-  async queue(operation) {
-    // Check queue size limit
-    if (this.queue.length >= this.maxQueueSize) {
-      const error = new PartitionDriverError('Memory queue full - backpressure detected', {
-        driver: 'memory',
-        operation: 'queue',
-        queueSize: this.queue.length,
-        maxQueueSize: this.maxQueueSize,
-        suggestion: 'Increase maxQueueSize, enable rejectOnFull, or reduce operation rate'
-      });
-      this.emit('queueFull', { operation, queueSize: this.queue.length });
-
-      if (this.options.rejectOnFull) {
-        throw error;
-      }
-
-      // Wait for some space
-      await this.waitForSpace();
-    }
-    
-    // Add to queue with metadata
-    const queueItem = {
-      ...operation,
-      id: `${Date.now()}-${Math.random()}`,
-      queuedAt: new Date(),
-      attempts: 0
-    };
-    
-    this.queue.push(queueItem);
-    this.stats.queued++;
-    
-    // Auto-flush when batch size reached
-    if (this.queue.length >= this.batchSize) {
-      this.triggerFlush();
-    }
-    
-    return { 
-      success: true, 
-      driver: 'memory',
-      queuePosition: this.queue.length,
-      queueId: queueItem.id
-    };
-  }
-
-  /**
-   * Start the background processor
-   */
-  startProcessor() {
-    // Set up periodic flush
-    if (this.flushInterval > 0) {
-      this.flushTimer = setInterval(() => {
-        if (this.queue.length > 0 && !this.isProcessing) {
-          this.processQueue();
-        }
-      }, this.flushInterval);
-    }
-  }
-
-  /**
-   * Trigger immediate flush
-   */
-  triggerFlush() {
-    if (!this.isProcessing) {
-      setImmediate(() => this.processQueue());
-    }
-  }
-
-  /**
-   * Process queued operations in batches
-   */
-  async processQueue() {
-    if (this.isProcessing || this.queue.length === 0) return;
-    
-    this.isProcessing = true;
-    
-    try {
-      // Take a batch from the queue
-      const batch = this.queue.splice(0, this.batchSize);
-      
-      // Process in parallel with concurrency control
-      const { results, errors } = await PromisePool
-        .for(batch)
-        .withConcurrency(this.concurrency)
-        .process(async (item) => {
-          try {
-            await this.processOperation(item);
-            return { success: true, item };
-          } catch (error) {
-            return this.handleError(item, error);
-          }
-        });
-      
-      // Handle successful results
-      const successful = results.filter(r => r.success);
-      this.emit('batchProcessed', {
-        processed: successful.length,
-        failed: errors.length,
-        retried: results.filter(r => r.retried).length
-      });
-      
-    } finally {
-      this.isProcessing = false;
-      
-      // Continue processing if more items
-      if (this.queue.length > 0) {
-        setImmediate(() => this.processQueue());
-      }
-      
-      // Process retry queue if needed
-      if (this.retryQueue.length > 0) {
-        this.processRetryQueue();
-      }
-    }
-  }
-
-  /**
-   * Handle processing errors with retry logic
-   */
-  handleError(item, error) {
-    item.attempts++;
-    item.lastError = error;
-    
-    if (item.attempts < this.maxRetries) {
-      // Add to retry queue with exponential backoff
-      const delay = Math.min(1000 * Math.pow(2, item.attempts - 1), 30000);
-      
-      setTimeout(() => {
-        this.retryQueue.push(item);
-        if (!this.isProcessing) {
-          this.processRetryQueue();
-        }
-      }, delay);
-      
-      this.emit('retry', { item, error, attempt: item.attempts, delay });
-      return { success: false, retried: true, item };
-    } else {
-      // Max retries exceeded
-      this.emit('failed', { item, error, attempts: item.attempts });
-      return { success: false, retried: false, item };
-    }
-  }
-
-  /**
-   * Process retry queue
-   */
-  async processRetryQueue() {
-    if (this.retryQueue.length === 0) return;
-    
-    // Move retry items back to main queue
-    const retryItems = this.retryQueue.splice(0, this.batchSize);
-    this.queue.unshift(...retryItems);
-    
-    // Trigger processing
-    this.triggerFlush();
-  }
-
-  /**
-   * Wait for queue space
-   */
-  async waitForSpace() {
-    const checkInterval = 100;
-    const maxWait = 30000;
-    const startTime = Date.now();
-    
-    while (this.queue.length >= this.maxQueueSize) {
-      if (Date.now() - startTime > maxWait) {
-        throw new PartitionDriverError('Timeout waiting for queue space', {
-          driver: 'memory',
-          operation: 'waitForSpace',
-          queueSize: this.queue.length,
-          maxQueueSize: this.maxQueueSize,
-          waitedMs: Date.now() - startTime,
-          maxWaitMs: maxWait,
-          suggestion: 'Queue is full and not draining fast enough. Increase maxQueueSize or concurrency'
-        });
-      }
-
-      await new Promise(resolve => setTimeout(resolve, checkInterval));
-    }
-  }
-
-  /**
-   * Force flush all pending operations
-   */
-  async flush() {
-    // Process all remaining items
-    while (this.queue.length > 0 || this.retryQueue.length > 0 || this.isProcessing) {
-      await this.processQueue();
-      await new Promise(resolve => setTimeout(resolve, 10));
-    }
-  }
-
-  /**
-   * Get detailed statistics
-   */
-  getStats() {
-    return {
-      ...super.getStats(),
-      queueLength: this.queue.length,
-      retryQueueLength: this.retryQueue.length,
-      isProcessing: this.isProcessing,
-      memoryUsage: this.estimateMemoryUsage()
-    };
-  }
-
-  /**
-   * Estimate memory usage of the queue
-   */
-  estimateMemoryUsage() {
-    // Rough estimate: 1KB per queue item
-    const bytes = (this.queue.length + this.retryQueue.length) * 1024;
-    return {
-      bytes,
-      mb: (bytes / 1024 / 1024).toFixed(2)
-    };
-  }
-
-  /**
-   * Shutdown the driver
-   */
-  async shutdown() {
-    // Stop the flush timer
-    if (this.flushTimer) {
-      clearInterval(this.flushTimer);
-      this.flushTimer = null;
-    }
-    
-    // Flush remaining items
-    await this.flush();
-    
-    // Clear queues
-    this.queue = [];
-    this.retryQueue = [];
-    
-    await super.shutdown();
-  }
-
-  getInfo() {
-    return {
-      name: this.name,
-      mode: 'asynchronous',
-      description: 'In-memory queue with background processing',
-      config: {
-        batchSize: this.batchSize,
-        concurrency: this.concurrency,
-        flushInterval: this.flushInterval,
-        maxQueueSize: this.maxQueueSize,
-        maxRetries: this.maxRetries
-      },
-      stats: this.getStats()
-    };
-  }
-}

package/src/partition-drivers/sqs-partition-driver.js

@@ -1,337 +0,0 @@
-import { BasePartitionDriver } from './base-partition-driver.js';
-import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';
-import { PartitionDriverError } from '../errors.js';
-
-/**
- * SQS-based partition driver for distributed processing
- * Sends partition operations to SQS for processing by workers
- * Ideal for high-volume, distributed systems
- */
-export class SQSPartitionDriver extends BasePartitionDriver {
-  constructor(options = {}) {
-    super(options);
-    this.name = 'sqs';
-    
-    // SQS Configuration
-    this.queueUrl = options.queueUrl;
-    if (!this.queueUrl) {
-      throw new PartitionDriverError('SQS queue URL is required', {
-        driver: 'sqs',
-        operation: 'constructor',
-        suggestion: 'Provide queueUrl in options: new SQSPartitionDriver({ queueUrl: "https://sqs.region.amazonaws.com/account/queue" })'
-      });
-    }
-    
-    this.region = options.region || 'us-east-1';
-    this.credentials = options.credentials;
-    this.dlqUrl = options.dlqUrl; // Dead Letter Queue
-    this.messageGroupId = options.messageGroupId || 's3db-partitions';
-    this.visibilityTimeout = options.visibilityTimeout || 300; // 5 minutes
-    this.batchSize = options.batchSize || 10; // SQS max batch size
-    
-    // Worker configuration
-    this.isWorker = options.isWorker || false;
-    this.workerConcurrency = options.workerConcurrency || 5;
-    this.pollInterval = options.pollInterval || 1000;
-    
-    // Initialize SQS client
-    this.sqsClient = new SQSClient({
-      region: this.region,
-      credentials: this.credentials
-    });
-    
-    this.workerRunning = false;
-    this.messageBuffer = [];
-  }
-
-  async initialize() {
-    // Start worker if configured
-    if (this.isWorker) {
-      await this.startWorker();
-    }
-  }
-
-  /**
-   * Send partition operation to SQS
-   */
-  async queue(operation) {
-    try {
-      // Prepare message
-      const message = {
-        id: `${Date.now()}-${Math.random()}`,
-        timestamp: new Date().toISOString(),
-        operation: {
-          type: operation.type,
-          resourceName: operation.resource.name,
-          data: this.serializeData(operation.data)
-        }
-      };
-      
-      // Buffer messages for batch sending
-      this.messageBuffer.push(message);
-      this.stats.queued++;
-      
-      // Send batch when buffer is full
-      if (this.messageBuffer.length >= this.batchSize) {
-        await this.flushMessages();
-      } else {
-        // Schedule flush if not already scheduled
-        if (!this.flushTimeout) {
-          this.flushTimeout = setTimeout(() => this.flushMessages(), 100);
-        }
-      }
-      
-      return {
-        success: true,
-        driver: 'sqs',
-        messageId: message.id,
-        queueUrl: this.queueUrl
-      };
-      
-    } catch (error) {
-      this.emit('error', { operation, error });
-      throw error;
-    }
-  }
-
-  /**
-   * Flush buffered messages to SQS
-   */
-  async flushMessages() {
-    if (this.messageBuffer.length === 0) return;
-    
-    clearTimeout(this.flushTimeout);
-    this.flushTimeout = null;
-    
-    const messages = this.messageBuffer.splice(0, this.batchSize);
-    
-    try {
-      // For FIFO queues, add deduplication ID
-      const isFifo = this.queueUrl.includes('.fifo');
-      
-      for (const message of messages) {
-        const params = {
-          QueueUrl: this.queueUrl,
-          MessageBody: JSON.stringify(message),
-          MessageAttributes: {
-            Type: {
-              DataType: 'String',
-              StringValue: message.operation.type
-            },
-            Resource: {
-              DataType: 'String',
-              StringValue: message.operation.resourceName
-            }
-          }
-        };
-        
-        if (isFifo) {
-          params.MessageGroupId = this.messageGroupId;
-          params.MessageDeduplicationId = message.id;
-        }
-        
-        await this.sqsClient.send(new SendMessageCommand(params));
-      }
-      
-      this.emit('messagesSent', { count: messages.length });
-      
-    } catch (error) {
-      // Return messages to buffer for retry
-      this.messageBuffer.unshift(...messages);
-      this.emit('sendError', { error, messages: messages.length });
-      throw error;
-    }
-  }
-
-  /**
-   * Start SQS worker to process messages
-   */
-  async startWorker() {
-    if (this.workerRunning) return;
-    
-    this.workerRunning = true;
-    this.emit('workerStarted', { concurrency: this.workerConcurrency });
-    
-    // Start multiple concurrent workers
-    for (let i = 0; i < this.workerConcurrency; i++) {
-      this.pollMessages(i);
-    }
-  }
-
-  /**
-   * Poll SQS for messages
-   */
-  async pollMessages(workerId) {
-    while (this.workerRunning) {
-      try {
-        // Receive messages from SQS
-        const params = {
-          QueueUrl: this.queueUrl,
-          MaxNumberOfMessages: 10,
-          WaitTimeSeconds: 20, // Long polling
-          VisibilityTimeout: this.visibilityTimeout,
-          MessageAttributeNames: ['All']
-        };
-        
-        const response = await this.sqsClient.send(new ReceiveMessageCommand(params));
-        
-        if (response.Messages && response.Messages.length > 0) {
-          // Process messages
-          for (const message of response.Messages) {
-            await this.processMessage(message, workerId);
-          }
-        }
-        
-      } catch (error) {
-        this.emit('pollError', { workerId, error });
-        // Wait before retrying
-        await new Promise(resolve => setTimeout(resolve, this.pollInterval));
-      }
-    }
-  }
-
-  /**
-   * Process a single SQS message
-   */
-  async processMessage(message, workerId) {
-    try {
-      // Parse message body
-      const data = JSON.parse(message.Body);
-      const operation = {
-        type: data.operation.type,
-        data: this.deserializeData(data.operation.data)
-      };
-      
-      // Process the partition operation
-      // Note: We need the actual resource instance to process
-      // This would typically be handled by a separate worker service
-      this.emit('processingMessage', { workerId, messageId: message.MessageId });
-      
-      // In a real implementation, you'd look up the resource and process:
-      // await this.processOperation(operation);
-      
-      // Delete message from queue after successful processing
-      await this.sqsClient.send(new DeleteMessageCommand({
-        QueueUrl: this.queueUrl,
-        ReceiptHandle: message.ReceiptHandle
-      }));
-      
-      this.stats.processed++;
-      this.emit('messageProcessed', { workerId, messageId: message.MessageId });
-      
-    } catch (error) {
-      this.stats.failed++;
-      this.emit('processError', { workerId, error, messageId: message.MessageId });
-      
-      // Message will become visible again after VisibilityTimeout
-      // and eventually move to DLQ if configured
-    }
-  }
-
-  /**
-   * Serialize data for SQS transport
-   */
-  serializeData(data) {
-    // Remove circular references and functions
-    return JSON.parse(JSON.stringify(data, (key, value) => {
-      if (typeof value === 'function') return undefined;
-      if (value instanceof Buffer) return value.toString('base64');
-      return value;
-    }));
-  }
-
-  /**
-   * Deserialize data from SQS
-   */
-  deserializeData(data) {
-    return data;
-  }
-
-  /**
-   * Stop the worker
-   */
-  async stopWorker() {
-    this.workerRunning = false;
-    this.emit('workerStopped');
-  }
-
-  /**
-   * Force flush all pending messages
-   */
-  async flush() {
-    await this.flushMessages();
-  }
-
-  /**
-   * Get queue metrics from SQS
-   */
-  async getQueueMetrics() {
-    try {
-      const { Attributes } = await this.sqsClient.send(new GetQueueAttributesCommand({
-        QueueUrl: this.queueUrl,
-        AttributeNames: [
-          'ApproximateNumberOfMessages',
-          'ApproximateNumberOfMessagesNotVisible',
-          'ApproximateNumberOfMessagesDelayed'
-        ]
-      }));
-      
-      return {
-        messagesAvailable: parseInt(Attributes.ApproximateNumberOfMessages || 0),
-        messagesInFlight: parseInt(Attributes.ApproximateNumberOfMessagesNotVisible || 0),
-        messagesDelayed: parseInt(Attributes.ApproximateNumberOfMessagesDelayed || 0)
-      };
-    } catch (error) {
-      return null;
-    }
-  }
-
-  /**
-   * Get detailed statistics
-   */
-  async getStats() {
-    const baseStats = super.getStats();
-    const queueMetrics = await this.getQueueMetrics();
-    
-    return {
-      ...baseStats,
-      bufferLength: this.messageBuffer.length,
-      isWorker: this.isWorker,
-      workerRunning: this.workerRunning,
-      queue: queueMetrics
-    };
-  }
-
-  /**
-   * Shutdown the driver
-   */
-  async shutdown() {
-    // Stop worker if running
-    await this.stopWorker();
-    
-    // Flush remaining messages
-    await this.flush();
-    
-    // Clear buffer
-    this.messageBuffer = [];
-    
-    await super.shutdown();
-  }
-
-  getInfo() {
-    return {
-      name: this.name,
-      mode: 'distributed',
-      description: 'SQS-based queue for distributed partition processing',
-      config: {
-        queueUrl: this.queueUrl,
-        region: this.region,
-        dlqUrl: this.dlqUrl,
-        isWorker: this.isWorker,
-        workerConcurrency: this.workerConcurrency,
-        visibilityTimeout: this.visibilityTimeout
-      },
-      stats: this.getStats()
-    };
-  }
-}