s3db.js 9.2.2 → 10.0.0

package/src/partition-drivers/index.js

@@ -0,0 +1,60 @@
+import { SyncPartitionDriver } from './sync-partition-driver.js';
+import { MemoryPartitionDriver } from './memory-partition-driver.js';
+import { SQSPartitionDriver } from './sqs-partition-driver.js';
+
+/**
+ * Partition driver factory
+ */
+export class PartitionDriverFactory {
+  static drivers = {
+    sync: SyncPartitionDriver,
+    memory: MemoryPartitionDriver,
+    sqs: SQSPartitionDriver
+  };
+
+  /**
+   * Create a partition driver instance
+   * @param {string|Object} config - Driver name or configuration object
+   * @returns {BasePartitionDriver} Driver instance
+   */
+  static create(config) {
+    // Handle string shorthand
+    if (typeof config === 'string') {
+      config = { driver: config };
+    }
+    
+    // Default to memory driver
+    const driverName = config.driver || 'memory';
+    
+    // Get driver class
+    const DriverClass = this.drivers[driverName];
+    if (!DriverClass) {
+      throw new Error(`Unknown partition driver: ${driverName}. Available: ${Object.keys(this.drivers).join(', ')}`);
+    }
+    
+    // Create and initialize driver
+    const driver = new DriverClass(config);
+    
+    return driver;
+  }
+
+  /**
+   * Register a custom driver
+   */
+  static register(name, DriverClass) {
+    this.drivers[name] = DriverClass;
+  }
+
+  /**
+   * Get available driver names
+   */
+  static getAvailableDrivers() {
+    return Object.keys(this.drivers);
+  }
+}
+
+// Export individual drivers
+export { BasePartitionDriver } from './base-partition-driver.js';
+export { SyncPartitionDriver } from './sync-partition-driver.js';
+export { MemoryPartitionDriver } from './memory-partition-driver.js';
+export { SQSPartitionDriver } from './sqs-partition-driver.js';

package/src/partition-drivers/memory-partition-driver.js

@@ -0,0 +1,274 @@
+import { BasePartitionDriver } from './base-partition-driver.js';
+import { PromisePool } from '@supercharge/promise-pool';
+
+/**
+ * In-memory partition driver with background processing
+ * Queues operations in memory and processes them asynchronously
+ * Fast and efficient for single-instance applications
+ */
+export class MemoryPartitionDriver extends BasePartitionDriver {
+  constructor(options = {}) {
+    super(options);
+    this.name = 'memory';
+    
+    // Configuration
+    this.batchSize = options.batchSize || 100;
+    this.concurrency = options.concurrency || 10;
+    this.flushInterval = options.flushInterval || 1000;
+    this.maxQueueSize = options.maxQueueSize || 10000;
+    this.maxRetries = options.maxRetries || 3;
+    
+    // State
+    this.queue = [];
+    this.isProcessing = false;
+    this.flushTimer = null;
+    this.retryQueue = [];
+  }
+
+  async initialize() {
+    // Start background processor
+    this.startProcessor();
+  }
+
+  /**
+   * Add operation to in-memory queue
+   */
+  async queue(operation) {
+    // Check queue size limit
+    if (this.queue.length >= this.maxQueueSize) {
+      const error = new Error(`Memory queue full (${this.maxQueueSize} items)`);
+      this.emit('queueFull', { operation, queueSize: this.queue.length });
+      
+      if (this.options.rejectOnFull) {
+        throw error;
+      }
+      
+      // Wait for some space
+      await this.waitForSpace();
+    }
+    
+    // Add to queue with metadata
+    const queueItem = {
+      ...operation,
+      id: `${Date.now()}-${Math.random()}`,
+      queuedAt: new Date(),
+      attempts: 0
+    };
+    
+    this.queue.push(queueItem);
+    this.stats.queued++;
+    
+    // Auto-flush when batch size reached
+    if (this.queue.length >= this.batchSize) {
+      this.triggerFlush();
+    }
+    
+    return { 
+      success: true, 
+      driver: 'memory',
+      queuePosition: this.queue.length,
+      queueId: queueItem.id
+    };
+  }
+
+  /**
+   * Start the background processor
+   */
+  startProcessor() {
+    // Set up periodic flush
+    if (this.flushInterval > 0) {
+      this.flushTimer = setInterval(() => {
+        if (this.queue.length > 0 && !this.isProcessing) {
+          this.processQueue();
+        }
+      }, this.flushInterval);
+    }
+  }
+
+  /**
+   * Trigger immediate flush
+   */
+  triggerFlush() {
+    if (!this.isProcessing) {
+      setImmediate(() => this.processQueue());
+    }
+  }
+
+  /**
+   * Process queued operations in batches
+   */
+  async processQueue() {
+    if (this.isProcessing || this.queue.length === 0) return;
+    
+    this.isProcessing = true;
+    
+    try {
+      // Take a batch from the queue
+      const batch = this.queue.splice(0, this.batchSize);
+      
+      // Process in parallel with concurrency control
+      const { results, errors } = await PromisePool
+        .for(batch)
+        .withConcurrency(this.concurrency)
+        .process(async (item) => {
+          try {
+            await this.processOperation(item);
+            return { success: true, item };
+          } catch (error) {
+            return this.handleError(item, error);
+          }
+        });
+      
+      // Handle successful results
+      const successful = results.filter(r => r.success);
+      this.emit('batchProcessed', {
+        processed: successful.length,
+        failed: errors.length,
+        retried: results.filter(r => r.retried).length
+      });
+      
+    } finally {
+      this.isProcessing = false;
+      
+      // Continue processing if more items
+      if (this.queue.length > 0) {
+        setImmediate(() => this.processQueue());
+      }
+      
+      // Process retry queue if needed
+      if (this.retryQueue.length > 0) {
+        this.processRetryQueue();
+      }
+    }
+  }
+
+  /**
+   * Handle processing errors with retry logic
+   */
+  handleError(item, error) {
+    item.attempts++;
+    item.lastError = error;
+    
+    if (item.attempts < this.maxRetries) {
+      // Add to retry queue with exponential backoff
+      const delay = Math.min(1000 * Math.pow(2, item.attempts - 1), 30000);
+      
+      setTimeout(() => {
+        this.retryQueue.push(item);
+        if (!this.isProcessing) {
+          this.processRetryQueue();
+        }
+      }, delay);
+      
+      this.emit('retry', { item, error, attempt: item.attempts, delay });
+      return { success: false, retried: true, item };
+    } else {
+      // Max retries exceeded
+      this.emit('failed', { item, error, attempts: item.attempts });
+      return { success: false, retried: false, item };
+    }
+  }
+
+  /**
+   * Process retry queue
+   */
+  async processRetryQueue() {
+    if (this.retryQueue.length === 0) return;
+    
+    // Move retry items back to main queue
+    const retryItems = this.retryQueue.splice(0, this.batchSize);
+    this.queue.unshift(...retryItems);
+    
+    // Trigger processing
+    this.triggerFlush();
+  }
+
+  /**
+   * Wait for queue space
+   */
+  async waitForSpace() {
+    const checkInterval = 100;
+    const maxWait = 30000;
+    const startTime = Date.now();
+    
+    while (this.queue.length >= this.maxQueueSize) {
+      if (Date.now() - startTime > maxWait) {
+        throw new Error('Timeout waiting for queue space');
+      }
+      
+      await new Promise(resolve => setTimeout(resolve, checkInterval));
+    }
+  }
+
+  /**
+   * Force flush all pending operations
+   */
+  async flush() {
+    // Process all remaining items
+    while (this.queue.length > 0 || this.retryQueue.length > 0 || this.isProcessing) {
+      await this.processQueue();
+      await new Promise(resolve => setTimeout(resolve, 10));
+    }
+  }
+
+  /**
+   * Get detailed statistics
+   */
+  getStats() {
+    return {
+      ...super.getStats(),
+      queueLength: this.queue.length,
+      retryQueueLength: this.retryQueue.length,
+      isProcessing: this.isProcessing,
+      memoryUsage: this.estimateMemoryUsage()
+    };
+  }
+
+  /**
+   * Estimate memory usage of the queue
+   */
+  estimateMemoryUsage() {
+    // Rough estimate: 1KB per queue item
+    const bytes = (this.queue.length + this.retryQueue.length) * 1024;
+    return {
+      bytes,
+      mb: (bytes / 1024 / 1024).toFixed(2)
+    };
+  }
+
+  /**
+   * Shutdown the driver
+   */
+  async shutdown() {
+    // Stop the flush timer
+    if (this.flushTimer) {
+      clearInterval(this.flushTimer);
+      this.flushTimer = null;
+    }
+    
+    // Flush remaining items
+    await this.flush();
+    
+    // Clear queues
+    this.queue = [];
+    this.retryQueue = [];
+    
+    await super.shutdown();
+  }
+
+  getInfo() {
+    return {
+      name: this.name,
+      mode: 'asynchronous',
+      description: 'In-memory queue with background processing',
+      config: {
+        batchSize: this.batchSize,
+        concurrency: this.concurrency,
+        flushInterval: this.flushInterval,
+        maxQueueSize: this.maxQueueSize,
+        maxRetries: this.maxRetries
+      },
+      stats: this.getStats()
+    };
+  }
+}

package/src/partition-drivers/sqs-partition-driver.js

@@ -0,0 +1,332 @@
+import { BasePartitionDriver } from './base-partition-driver.js';
+import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';
+
+/**
+ * SQS-based partition driver for distributed processing
+ * Sends partition operations to SQS for processing by workers
+ * Ideal for high-volume, distributed systems
+ */
+export class SQSPartitionDriver extends BasePartitionDriver {
+  constructor(options = {}) {
+    super(options);
+    this.name = 'sqs';
+    
+    // SQS Configuration
+    this.queueUrl = options.queueUrl;
+    if (!this.queueUrl) {
+      throw new Error('SQS queue URL is required for SQSPartitionDriver');
+    }
+    
+    this.region = options.region || 'us-east-1';
+    this.credentials = options.credentials;
+    this.dlqUrl = options.dlqUrl; // Dead Letter Queue
+    this.messageGroupId = options.messageGroupId || 's3db-partitions';
+    this.visibilityTimeout = options.visibilityTimeout || 300; // 5 minutes
+    this.batchSize = options.batchSize || 10; // SQS max batch size
+    
+    // Worker configuration
+    this.isWorker = options.isWorker || false;
+    this.workerConcurrency = options.workerConcurrency || 5;
+    this.pollInterval = options.pollInterval || 1000;
+    
+    // Initialize SQS client
+    this.sqsClient = new SQSClient({
+      region: this.region,
+      credentials: this.credentials
+    });
+    
+    this.workerRunning = false;
+    this.messageBuffer = [];
+  }
+
+  async initialize() {
+    // Start worker if configured
+    if (this.isWorker) {
+      await this.startWorker();
+    }
+  }
+
+  /**
+   * Send partition operation to SQS
+   */
+  async queue(operation) {
+    try {
+      // Prepare message
+      const message = {
+        id: `${Date.now()}-${Math.random()}`,
+        timestamp: new Date().toISOString(),
+        operation: {
+          type: operation.type,
+          resourceName: operation.resource.name,
+          data: this.serializeData(operation.data)
+        }
+      };
+      
+      // Buffer messages for batch sending
+      this.messageBuffer.push(message);
+      this.stats.queued++;
+      
+      // Send batch when buffer is full
+      if (this.messageBuffer.length >= this.batchSize) {
+        await this.flushMessages();
+      } else {
+        // Schedule flush if not already scheduled
+        if (!this.flushTimeout) {
+          this.flushTimeout = setTimeout(() => this.flushMessages(), 100);
+        }
+      }
+      
+      return {
+        success: true,
+        driver: 'sqs',
+        messageId: message.id,
+        queueUrl: this.queueUrl
+      };
+      
+    } catch (error) {
+      this.emit('error', { operation, error });
+      throw error;
+    }
+  }
+
+  /**
+   * Flush buffered messages to SQS
+   */
+  async flushMessages() {
+    if (this.messageBuffer.length === 0) return;
+    
+    clearTimeout(this.flushTimeout);
+    this.flushTimeout = null;
+    
+    const messages = this.messageBuffer.splice(0, this.batchSize);
+    
+    try {
+      // For FIFO queues, add deduplication ID
+      const isFifo = this.queueUrl.includes('.fifo');
+      
+      for (const message of messages) {
+        const params = {
+          QueueUrl: this.queueUrl,
+          MessageBody: JSON.stringify(message),
+          MessageAttributes: {
+            Type: {
+              DataType: 'String',
+              StringValue: message.operation.type
+            },
+            Resource: {
+              DataType: 'String',
+              StringValue: message.operation.resourceName
+            }
+          }
+        };
+        
+        if (isFifo) {
+          params.MessageGroupId = this.messageGroupId;
+          params.MessageDeduplicationId = message.id;
+        }
+        
+        await this.sqsClient.send(new SendMessageCommand(params));
+      }
+      
+      this.emit('messagesSent', { count: messages.length });
+      
+    } catch (error) {
+      // Return messages to buffer for retry
+      this.messageBuffer.unshift(...messages);
+      this.emit('sendError', { error, messages: messages.length });
+      throw error;
+    }
+  }
+
+  /**
+   * Start SQS worker to process messages
+   */
+  async startWorker() {
+    if (this.workerRunning) return;
+    
+    this.workerRunning = true;
+    this.emit('workerStarted', { concurrency: this.workerConcurrency });
+    
+    // Start multiple concurrent workers
+    for (let i = 0; i < this.workerConcurrency; i++) {
+      this.pollMessages(i);
+    }
+  }
+
+  /**
+   * Poll SQS for messages
+   */
+  async pollMessages(workerId) {
+    while (this.workerRunning) {
+      try {
+        // Receive messages from SQS
+        const params = {
+          QueueUrl: this.queueUrl,
+          MaxNumberOfMessages: 10,
+          WaitTimeSeconds: 20, // Long polling
+          VisibilityTimeout: this.visibilityTimeout,
+          MessageAttributeNames: ['All']
+        };
+        
+        const response = await this.sqsClient.send(new ReceiveMessageCommand(params));
+        
+        if (response.Messages && response.Messages.length > 0) {
+          // Process messages
+          for (const message of response.Messages) {
+            await this.processMessage(message, workerId);
+          }
+        }
+        
+      } catch (error) {
+        this.emit('pollError', { workerId, error });
+        // Wait before retrying
+        await new Promise(resolve => setTimeout(resolve, this.pollInterval));
+      }
+    }
+  }
+
+  /**
+   * Process a single SQS message
+   */
+  async processMessage(message, workerId) {
+    try {
+      // Parse message body
+      const data = JSON.parse(message.Body);
+      const operation = {
+        type: data.operation.type,
+        data: this.deserializeData(data.operation.data)
+      };
+      
+      // Process the partition operation
+      // Note: We need the actual resource instance to process
+      // This would typically be handled by a separate worker service
+      this.emit('processingMessage', { workerId, messageId: message.MessageId });
+      
+      // In a real implementation, you'd look up the resource and process:
+      // await this.processOperation(operation);
+      
+      // Delete message from queue after successful processing
+      await this.sqsClient.send(new DeleteMessageCommand({
+        QueueUrl: this.queueUrl,
+        ReceiptHandle: message.ReceiptHandle
+      }));
+      
+      this.stats.processed++;
+      this.emit('messageProcessed', { workerId, messageId: message.MessageId });
+      
+    } catch (error) {
+      this.stats.failed++;
+      this.emit('processError', { workerId, error, messageId: message.MessageId });
+      
+      // Message will become visible again after VisibilityTimeout
+      // and eventually move to DLQ if configured
+    }
+  }
+
+  /**
+   * Serialize data for SQS transport
+   */
+  serializeData(data) {
+    // Remove circular references and functions
+    return JSON.parse(JSON.stringify(data, (key, value) => {
+      if (typeof value === 'function') return undefined;
+      if (value instanceof Buffer) return value.toString('base64');
+      return value;
+    }));
+  }
+
+  /**
+   * Deserialize data from SQS
+   */
+  deserializeData(data) {
+    return data;
+  }
+
+  /**
+   * Stop the worker
+   */
+  async stopWorker() {
+    this.workerRunning = false;
+    this.emit('workerStopped');
+  }
+
+  /**
+   * Force flush all pending messages
+   */
+  async flush() {
+    await this.flushMessages();
+  }
+
+  /**
+   * Get queue metrics from SQS
+   */
+  async getQueueMetrics() {
+    try {
+      const { Attributes } = await this.sqsClient.send(new GetQueueAttributesCommand({
+        QueueUrl: this.queueUrl,
+        AttributeNames: [
+          'ApproximateNumberOfMessages',
+          'ApproximateNumberOfMessagesNotVisible',
+          'ApproximateNumberOfMessagesDelayed'
+        ]
+      }));
+      
+      return {
+        messagesAvailable: parseInt(Attributes.ApproximateNumberOfMessages || 0),
+        messagesInFlight: parseInt(Attributes.ApproximateNumberOfMessagesNotVisible || 0),
+        messagesDelayed: parseInt(Attributes.ApproximateNumberOfMessagesDelayed || 0)
+      };
+    } catch (error) {
+      return null;
+    }
+  }
+
+  /**
+   * Get detailed statistics
+   */
+  async getStats() {
+    const baseStats = super.getStats();
+    const queueMetrics = await this.getQueueMetrics();
+    
+    return {
+      ...baseStats,
+      bufferLength: this.messageBuffer.length,
+      isWorker: this.isWorker,
+      workerRunning: this.workerRunning,
+      queue: queueMetrics
+    };
+  }
+
+  /**
+   * Shutdown the driver
+   */
+  async shutdown() {
+    // Stop worker if running
+    await this.stopWorker();
+    
+    // Flush remaining messages
+    await this.flush();
+    
+    // Clear buffer
+    this.messageBuffer = [];
+    
+    await super.shutdown();
+  }
+
+  getInfo() {
+    return {
+      name: this.name,
+      mode: 'distributed',
+      description: 'SQS-based queue for distributed partition processing',
+      config: {
+        queueUrl: this.queueUrl,
+        region: this.region,
+        dlqUrl: this.dlqUrl,
+        isWorker: this.isWorker,
+        workerConcurrency: this.workerConcurrency,
+        visibilityTimeout: this.visibilityTimeout
+      },
+      stats: this.getStats()
+    };
+  }
+}

package/src/partition-drivers/sync-partition-driver.js

@@ -0,0 +1,38 @@
+import { BasePartitionDriver } from './base-partition-driver.js';
+
+/**
+ * Synchronous partition driver
+ * Creates partitions immediately during insert/update/delete
+ * Use this when data consistency is critical
+ */
+export class SyncPartitionDriver extends BasePartitionDriver {
+  constructor(options = {}) {
+    super(options);
+    this.name = 'sync';
+  }
+
+  /**
+   * Process partition operations synchronously
+   */
+  async queue(operation) {
+    this.stats.queued++;
+    
+    try {
+      // Process immediately and wait for completion
+      await this.processOperation(operation);
+      return { success: true, driver: 'sync' };
+    } catch (error) {
+      // Re-throw to make the main operation fail
+      throw error;
+    }
+  }
+
+  getInfo() {
+    return {
+      name: this.name,
+      mode: 'synchronous',
+      description: 'Processes partitions immediately, blocking the main operation',
+      stats: this.getStats()
+    };
+  }
+}

package/src/plugins/backup.plugin.js

@@ -101,7 +101,7 @@ export class BackupPlugin extends Plugin {
       include: options.include || null,
       exclude: options.exclude || [],
       backupMetadataResource: options.backupMetadataResource || 'backup_metadata',
-      tempDir: options.tempDir || './tmp/backups',
+      tempDir: options.tempDir || '/tmp/s3db/backups',
       verbose: options.verbose || false,
       
       // Hooks