s3db.js 11.3.2 → 12.0.0

package/SECURITY.md

@@ -1,76 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-| Version | Supported          |
-| ------- | ------------------ |
-| 11.x.x  | :white_check_mark: |
-| < 11.0  | :x:                |
-
-## Known Security Advisories
-
-### Development Dependencies
-
-The following vulnerabilities exist in **development-only** dependencies and **do not affect** the published npm package or runtime security:
-
-#### pkg (GHSA-22r3-9w55-cj54) - MODERATE
-- **Status**: Acknowledged, monitored
-- **Impact**: Local privilege escalation
-- **Scope**: Only affects developers running `pnpm run build:binaries`
-- **Mitigation**: pkg is deprecated and archived. No patched version available (`<0.0.0`).
-- **Risk Assessment**: LOW - Only used for creating standalone binaries during release process
-- **Future Plans**: Migrate to Node.js Single Executable Applications (SEA) when stable
-
-#### tar-fs - HIGH
-- **Status**: RESOLVED in v11.1.1+
-- **Fix**: Updated to patched version 2.1.4+
-
-## Reporting a Vulnerability
-
-If you discover a security vulnerability in the **runtime code** (not dev dependencies), please report it by:
-
-1. **DO NOT** open a public issue
-2. Email: [security contact - update this]
-3. Include:
-   - Description of the vulnerability
-   - Steps to reproduce
-   - Potential impact
-   - Suggested fix (if any)
-
-### Response Timeline
-
-- **Initial Response**: Within 48 hours
-- **Status Update**: Within 7 days
-- **Fix Timeline**: Depends on severity
-  - Critical: 7 days
-  - High: 14 days
-  - Medium: 30 days
-  - Low: 60 days
-
-## Security Best Practices
-
-### For Users
-
-1. **Always encrypt sensitive data**: Use `secret` field type for passwords, tokens, etc.
-2. **Validate credentials**: Never commit AWS credentials to version control
-3. **Use IAM policies**: Implement least-privilege access for S3 buckets
-4. **Enable paranoid mode**: For production, use `paranoid: true` for soft deletes
-5. **Audit hooks**: Review serialized functions before deploying to production
-
-### For Contributors
-
-1. **No secrets in tests**: Use environment variables or LocalStack
-2. **Validate input**: All user input should be validated before S3 operations
-3. **Handle errors safely**: Never expose AWS error details to end users
-4. **Review dependencies**: Run `pnpm audit` before submitting PRs
-5. **Test encryption**: Verify `secret` fields are actually encrypted in S3
-
-## Audit Configuration
-
-This project uses `audit-level=high` in `.npmrc` to focus on critical vulnerabilities affecting production. Moderate/low severity issues in dev-only dependencies are monitored but may not block releases if:
-
-- They only affect development tools
-- No patch is available
-- The risk is assessed as acceptable
-
-Current audit threshold: **HIGH** (ignores moderate/low in dev dependencies)

package/src/partition-drivers/base-partition-driver.js

@@ -1,106 +0,0 @@
-import { EventEmitter } from 'events';
-import { PartitionDriverError } from '../errors.js';
-
-/**
- * Base class for all partition drivers
- * Defines the interface that all drivers must implement
- */
-export class BasePartitionDriver extends EventEmitter {
-  constructor(options = {}) {
-    super();
-    this.options = options;
-    this.stats = {
-      queued: 0,
-      processed: 0,
-      failed: 0,
-      processing: 0
-    };
-  }
-
-  /**
-   * Initialize the driver
-   */
-  async initialize() {
-    // Override in subclasses if needed
-  }
-
-  /**
-   * Queue partition operations for processing
-   * @param {Object} operation - The partition operation to queue
-   * @param {string} operation.type - 'create', 'update', or 'delete'
-   * @param {Object} operation.resource - The resource instance
-   * @param {Object} operation.data - The data for the operation
-   */
-  async queue(operation) {
-    throw new PartitionDriverError('queue() must be implemented by subclass', {
-      driver: this.name || 'BasePartitionDriver',
-      operation: 'queue',
-      suggestion: 'Extend BasePartitionDriver and implement the queue() method'
-    });
-  }
-
-  /**
-   * Process a single partition operation
-   */
-  async processOperation(operation) {
-    const { type, resource, data } = operation;
-    
-    try {
-      this.stats.processing++;
-      
-      switch (type) {
-        case 'create':
-          await resource.createPartitionReferences(data.object);
-          break;
-          
-        case 'update':
-          await resource.handlePartitionReferenceUpdates(data.original, data.updated);
-          break;
-          
-        case 'delete':
-          await resource.deletePartitionReferences(data.object);
-          break;
-          
-        default:
-          throw new PartitionDriverError(`Unknown partition operation type: ${type}`, {
-            driver: this.name || 'BasePartitionDriver',
-            operation: type,
-            availableOperations: ['create', 'update', 'delete'],
-            suggestion: 'Use one of the supported partition operations: create, update, or delete'
-          });
-      }
-      
-      this.stats.processed++;
-      this.emit('processed', operation);
-      
-    } catch (error) {
-      this.stats.failed++;
-      this.emit('error', { operation, error });
-      throw error;
-    } finally {
-      this.stats.processing--;
-    }
-  }
-
-  /**
-   * Flush any pending operations
-   */
-  async flush() {
-    // Override in subclasses if needed
-  }
-
-  /**
-   * Get driver statistics
-   */
-  getStats() {
-    return { ...this.stats };
-  }
-
-  /**
-   * Shutdown the driver
-   */
-  async shutdown() {
-    await this.flush();
-    this.removeAllListeners();
-  }
-}

package/src/partition-drivers/index.js

@@ -1,66 +0,0 @@
-import { SyncPartitionDriver } from './sync-partition-driver.js';
-import { MemoryPartitionDriver } from './memory-partition-driver.js';
-import { SQSPartitionDriver } from './sqs-partition-driver.js';
-import { PartitionDriverError } from '../errors.js';
-
-/**
- * Partition driver factory
- */
-export class PartitionDriverFactory {
-  static drivers = {
-    sync: SyncPartitionDriver,
-    memory: MemoryPartitionDriver,
-    sqs: SQSPartitionDriver
-  };
-
-  /**
-   * Create a partition driver instance
-   * @param {string|Object} config - Driver name or configuration object
-   * @returns {BasePartitionDriver} Driver instance
-   */
-  static create(config) {
-    // Handle string shorthand
-    if (typeof config === 'string') {
-      config = { driver: config };
-    }
-    
-    // Default to memory driver
-    const driverName = config.driver || 'memory';
-    
-    // Get driver class
-    const DriverClass = this.drivers[driverName];
-    if (!DriverClass) {
-      throw new PartitionDriverError(`Unknown partition driver: ${driverName}`, {
-        driver: driverName,
-        operation: 'create',
-        availableDrivers: Object.keys(this.drivers),
-        suggestion: `Use one of the available drivers: ${Object.keys(this.drivers).join(', ')}, or register a custom driver`
-      });
-    }
-    
-    // Create and initialize driver
-    const driver = new DriverClass(config);
-    
-    return driver;
-  }
-
-  /**
-   * Register a custom driver
-   */
-  static register(name, DriverClass) {
-    this.drivers[name] = DriverClass;
-  }
-
-  /**
-   * Get available driver names
-   */
-  static getAvailableDrivers() {
-    return Object.keys(this.drivers);
-  }
-}
-
-// Export individual drivers
-export { BasePartitionDriver } from './base-partition-driver.js';
-export { SyncPartitionDriver } from './sync-partition-driver.js';
-export { MemoryPartitionDriver } from './memory-partition-driver.js';
-export { SQSPartitionDriver } from './sqs-partition-driver.js';

package/src/partition-drivers/memory-partition-driver.js

@@ -1,289 +0,0 @@
-import { BasePartitionDriver } from './base-partition-driver.js';
-import { PromisePool } from '@supercharge/promise-pool';
-import { PartitionDriverError } from '../errors.js';
-
-/**
- * In-memory partition driver with background processing
- * Queues operations in memory and processes them asynchronously
- * Fast and efficient for single-instance applications
- */
-export class MemoryPartitionDriver extends BasePartitionDriver {
-  constructor(options = {}) {
-    super(options);
-    this.name = 'memory';
-    
-    // Configuration
-    this.batchSize = options.batchSize || 100;
-    this.concurrency = options.concurrency || 10;
-    this.flushInterval = options.flushInterval || 1000;
-    this.maxQueueSize = options.maxQueueSize || 10000;
-    this.maxRetries = options.maxRetries || 3;
-    
-    // State
-    this.queue = [];
-    this.isProcessing = false;
-    this.flushTimer = null;
-    this.retryQueue = [];
-  }
-
-  async initialize() {
-    // Start background processor
-    this.startProcessor();
-  }
-
-  /**
-   * Add operation to in-memory queue
-   */
-  async queue(operation) {
-    // Check queue size limit
-    if (this.queue.length >= this.maxQueueSize) {
-      const error = new PartitionDriverError('Memory queue full - backpressure detected', {
-        driver: 'memory',
-        operation: 'queue',
-        queueSize: this.queue.length,
-        maxQueueSize: this.maxQueueSize,
-        suggestion: 'Increase maxQueueSize, enable rejectOnFull, or reduce operation rate'
-      });
-      this.emit('queueFull', { operation, queueSize: this.queue.length });
-
-      if (this.options.rejectOnFull) {
-        throw error;
-      }
-
-      // Wait for some space
-      await this.waitForSpace();
-    }
-    
-    // Add to queue with metadata
-    const queueItem = {
-      ...operation,
-      id: `${Date.now()}-${Math.random()}`,
-      queuedAt: new Date(),
-      attempts: 0
-    };
-    
-    this.queue.push(queueItem);
-    this.stats.queued++;
-    
-    // Auto-flush when batch size reached
-    if (this.queue.length >= this.batchSize) {
-      this.triggerFlush();
-    }
-    
-    return { 
-      success: true, 
-      driver: 'memory',
-      queuePosition: this.queue.length,
-      queueId: queueItem.id
-    };
-  }
-
-  /**
-   * Start the background processor
-   */
-  startProcessor() {
-    // Set up periodic flush
-    if (this.flushInterval > 0) {
-      this.flushTimer = setInterval(() => {
-        if (this.queue.length > 0 && !this.isProcessing) {
-          this.processQueue();
-        }
-      }, this.flushInterval);
-    }
-  }
-
-  /**
-   * Trigger immediate flush
-   */
-  triggerFlush() {
-    if (!this.isProcessing) {
-      setImmediate(() => this.processQueue());
-    }
-  }
-
-  /**
-   * Process queued operations in batches
-   */
-  async processQueue() {
-    if (this.isProcessing || this.queue.length === 0) return;
-    
-    this.isProcessing = true;
-    
-    try {
-      // Take a batch from the queue
-      const batch = this.queue.splice(0, this.batchSize);
-      
-      // Process in parallel with concurrency control
-      const { results, errors } = await PromisePool
-        .for(batch)
-        .withConcurrency(this.concurrency)
-        .process(async (item) => {
-          try {
-            await this.processOperation(item);
-            return { success: true, item };
-          } catch (error) {
-            return this.handleError(item, error);
-          }
-        });
-      
-      // Handle successful results
-      const successful = results.filter(r => r.success);
-      this.emit('batchProcessed', {
-        processed: successful.length,
-        failed: errors.length,
-        retried: results.filter(r => r.retried).length
-      });
-      
-    } finally {
-      this.isProcessing = false;
-      
-      // Continue processing if more items
-      if (this.queue.length > 0) {
-        setImmediate(() => this.processQueue());
-      }
-      
-      // Process retry queue if needed
-      if (this.retryQueue.length > 0) {
-        this.processRetryQueue();
-      }
-    }
-  }
-
-  /**
-   * Handle processing errors with retry logic
-   */
-  handleError(item, error) {
-    item.attempts++;
-    item.lastError = error;
-    
-    if (item.attempts < this.maxRetries) {
-      // Add to retry queue with exponential backoff
-      const delay = Math.min(1000 * Math.pow(2, item.attempts - 1), 30000);
-      
-      setTimeout(() => {
-        this.retryQueue.push(item);
-        if (!this.isProcessing) {
-          this.processRetryQueue();
-        }
-      }, delay);
-      
-      this.emit('retry', { item, error, attempt: item.attempts, delay });
-      return { success: false, retried: true, item };
-    } else {
-      // Max retries exceeded
-      this.emit('failed', { item, error, attempts: item.attempts });
-      return { success: false, retried: false, item };
-    }
-  }
-
-  /**
-   * Process retry queue
-   */
-  async processRetryQueue() {
-    if (this.retryQueue.length === 0) return;
-    
-    // Move retry items back to main queue
-    const retryItems = this.retryQueue.splice(0, this.batchSize);
-    this.queue.unshift(...retryItems);
-    
-    // Trigger processing
-    this.triggerFlush();
-  }
-
-  /**
-   * Wait for queue space
-   */
-  async waitForSpace() {
-    const checkInterval = 100;
-    const maxWait = 30000;
-    const startTime = Date.now();
-    
-    while (this.queue.length >= this.maxQueueSize) {
-      if (Date.now() - startTime > maxWait) {
-        throw new PartitionDriverError('Timeout waiting for queue space', {
-          driver: 'memory',
-          operation: 'waitForSpace',
-          queueSize: this.queue.length,
-          maxQueueSize: this.maxQueueSize,
-          waitedMs: Date.now() - startTime,
-          maxWaitMs: maxWait,
-          suggestion: 'Queue is full and not draining fast enough. Increase maxQueueSize or concurrency'
-        });
-      }
-
-      await new Promise(resolve => setTimeout(resolve, checkInterval));
-    }
-  }
-
-  /**
-   * Force flush all pending operations
-   */
-  async flush() {
-    // Process all remaining items
-    while (this.queue.length > 0 || this.retryQueue.length > 0 || this.isProcessing) {
-      await this.processQueue();
-      await new Promise(resolve => setTimeout(resolve, 10));
-    }
-  }
-
-  /**
-   * Get detailed statistics
-   */
-  getStats() {
-    return {
-      ...super.getStats(),
-      queueLength: this.queue.length,
-      retryQueueLength: this.retryQueue.length,
-      isProcessing: this.isProcessing,
-      memoryUsage: this.estimateMemoryUsage()
-    };
-  }
-
-  /**
-   * Estimate memory usage of the queue
-   */
-  estimateMemoryUsage() {
-    // Rough estimate: 1KB per queue item
-    const bytes = (this.queue.length + this.retryQueue.length) * 1024;
-    return {
-      bytes,
-      mb: (bytes / 1024 / 1024).toFixed(2)
-    };
-  }
-
-  /**
-   * Shutdown the driver
-   */
-  async shutdown() {
-    // Stop the flush timer
-    if (this.flushTimer) {
-      clearInterval(this.flushTimer);
-      this.flushTimer = null;
-    }
-    
-    // Flush remaining items
-    await this.flush();
-    
-    // Clear queues
-    this.queue = [];
-    this.retryQueue = [];
-    
-    await super.shutdown();
-  }
-
-  getInfo() {
-    return {
-      name: this.name,
-      mode: 'asynchronous',
-      description: 'In-memory queue with background processing',
-      config: {
-        batchSize: this.batchSize,
-        concurrency: this.concurrency,
-        flushInterval: this.flushInterval,
-        maxQueueSize: this.maxQueueSize,
-        maxRetries: this.maxRetries
-      },
-      stats: this.getStats()
-    };
-  }
-}