s3db.js 9.3.0 → 10.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "9.3.0",
+  "version": "10.0.0",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",

package/src/client.class.js

@@ -41,8 +41,8 @@ export class Client extends EventEmitter {
     this.httpClientOptions = {
       keepAlive: true, // Enabled for better performance
       keepAliveMsecs: 1000, // 1 second keep-alive
-      maxSockets: 50, // Balanced for most applications
-      maxFreeSockets: 10, // Good connection reuse
+      maxSockets: httpClientOptions.maxSockets || 500, // High concurrency support
+      maxFreeSockets: httpClientOptions.maxFreeSockets || 100, // Better connection reuse
       timeout: 60000, // 60 second timeout
       ...httpClientOptions,
     };

package/src/concerns/high-performance-inserter.js

@@ -0,0 +1,285 @@
+import { PromisePool } from '@supercharge/promise-pool';
+import { tryFn } from './try-fn.js';
+
+/**
+ * High-performance bulk inserter for S3DB
+ * Optimized for continuous high-volume inserts with partitions
+ */
+export class HighPerformanceInserter {
+  constructor(resource, options = {}) {
+    this.resource = resource;
+    
+    // Performance tuning
+    this.batchSize = options.batchSize || 100;
+    this.concurrency = options.concurrency || 50; // Parallel S3 operations
+    this.flushInterval = options.flushInterval || 1000; // ms
+    this.disablePartitions = options.disablePartitions || false;
+    this.useStreamMode = options.useStreamMode || false;
+    
+    // Buffers
+    this.insertBuffer = [];
+    this.partitionBuffer = new Map(); // Deferred partition operations
+    this.stats = {
+      inserted: 0,
+      failed: 0,
+      partitionsPending: 0,
+      avgInsertTime: 0
+    };
+    
+    // Auto-flush timer
+    this.flushTimer = null;
+    this.isProcessing = false;
+    
+    // Partition processing queue
+    this.partitionQueue = [];
+    this.partitionProcessor = null;
+  }
+
+  /**
+   * Add item to insert buffer (non-blocking)
+   */
+  async add(data) {
+    this.insertBuffer.push({
+      data,
+      timestamp: Date.now(),
+      promise: null
+    });
+    
+    // Auto-flush when buffer is full
+    if (this.insertBuffer.length >= this.batchSize) {
+      setImmediate(() => this.flush());
+    } else if (!this.flushTimer) {
+      // Set flush timer if not already set
+      this.flushTimer = setTimeout(() => this.flush(), this.flushInterval);
+    }
+    
+    return { queued: true, position: this.insertBuffer.length };
+  }
+
+  /**
+   * Bulk add items
+   */
+  async bulkAdd(items) {
+    for (const item of items) {
+      await this.add(item);
+    }
+    return { queued: items.length };
+  }
+
+  /**
+   * Process buffered inserts in parallel
+   */
+  async flush() {
+    if (this.isProcessing || this.insertBuffer.length === 0) return;
+    
+    this.isProcessing = true;
+    clearTimeout(this.flushTimer);
+    this.flushTimer = null;
+    
+    // Take current buffer and reset
+    const batch = this.insertBuffer.splice(0, this.batchSize);
+    const startTime = Date.now();
+    
+    try {
+      // Process inserts in parallel with connection pooling
+      const { results, errors } = await PromisePool
+        .for(batch)
+        .withConcurrency(this.concurrency)
+        .process(async (item) => {
+          return await this.performInsert(item);
+        });
+      
+      // Update stats
+      const duration = Date.now() - startTime;
+      this.stats.inserted += results.filter(r => r.success).length;
+      this.stats.failed += errors.length;
+      this.stats.avgInsertTime = duration / batch.length;
+      
+      // Process partition queue separately (non-blocking)
+      if (!this.disablePartitions && this.partitionQueue.length > 0) {
+        this.processPartitionsAsync();
+      }
+      
+    } finally {
+      this.isProcessing = false;
+      
+      // Continue processing if more items
+      if (this.insertBuffer.length > 0) {
+        setImmediate(() => this.flush());
+      }
+    }
+  }
+
+  /**
+   * Perform single insert with optimizations
+   */
+  async performInsert(item) {
+    const { data } = item;
+    
+    try {
+      // Temporarily disable partitions for the insert
+      const originalAsyncPartitions = this.resource.config.asyncPartitions;
+      const originalPartitions = this.resource.config.partitions;
+      
+      if (this.disablePartitions) {
+        // Completely bypass partitions during insert
+        this.resource.config.partitions = {};
+      }
+      
+      // Perform insert
+      const [ok, err, result] = await tryFn(() => this.resource.insert(data));
+      
+      if (!ok) {
+        return { success: false, error: err };
+      }
+      
+      // Queue partition creation for later (if not disabled)
+      if (!this.disablePartitions && originalPartitions && Object.keys(originalPartitions).length > 0) {
+        this.partitionQueue.push({
+          operation: 'create',
+          data: result,
+          partitions: originalPartitions
+        });
+        this.stats.partitionsPending++;
+      }
+      
+      // Restore original config
+      this.resource.config.partitions = originalPartitions;
+      this.resource.config.asyncPartitions = originalAsyncPartitions;
+      
+      return { success: true, data: result };
+      
+    } catch (error) {
+      return { success: false, error };
+    }
+  }
+
+  /**
+   * Process partitions asynchronously in background
+   */
+  async processPartitionsAsync() {
+    if (this.partitionProcessor) return; // Already processing
+    
+    this.partitionProcessor = setImmediate(async () => {
+      const batch = this.partitionQueue.splice(0, 100); // Process 100 at a time
+      
+      if (batch.length === 0) {
+        this.partitionProcessor = null;
+        return;
+      }
+      
+      // Create partitions in parallel with lower priority
+      await PromisePool
+        .for(batch)
+        .withConcurrency(10) // Lower concurrency for partitions
+        .process(async (item) => {
+          try {
+            await this.resource.createPartitionReferences(item.data);
+            this.stats.partitionsPending--;
+          } catch (err) {
+            // Silently handle partition errors
+            this.resource.emit('partitionIndexError', {
+              operation: 'bulk-insert',
+              error: err
+            });
+          }
+        });
+      
+      // Continue processing if more partitions
+      if (this.partitionQueue.length > 0) {
+        this.processPartitionsAsync();
+      } else {
+        this.partitionProcessor = null;
+      }
+    });
+  }
+
+  /**
+   * Force flush all pending operations
+   */
+  async forceFlush() {
+    while (this.insertBuffer.length > 0 || this.isProcessing) {
+      await this.flush();
+      await new Promise(resolve => setTimeout(resolve, 10));
+    }
+  }
+
+  /**
+   * Get current statistics
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      bufferSize: this.insertBuffer.length,
+      isProcessing: this.isProcessing,
+      throughput: this.stats.avgInsertTime > 0 
+        ? Math.round(1000 / this.stats.avgInsertTime) 
+        : 0 // inserts per second
+    };
+  }
+
+  /**
+   * Destroy and cleanup
+   */
+  destroy() {
+    clearTimeout(this.flushTimer);
+    this.insertBuffer = [];
+    this.partitionQueue = [];
+  }
+}
+
+/**
+ * Stream-based inserter for maximum performance
+ */
+export class StreamInserter {
+  constructor(resource, options = {}) {
+    this.resource = resource;
+    this.concurrency = options.concurrency || 100;
+    this.skipPartitions = options.skipPartitions !== false;
+    this.skipHooks = options.skipHooks || false;
+    this.skipValidation = options.skipValidation || false;
+  }
+
+  /**
+   * Direct S3 write bypassing most S3DB overhead
+   */
+  async fastInsert(data) {
+    const id = data.id || this.resource.generateId();
+    const key = this.resource.getResourceKey(id);
+    
+    // Minimal processing
+    const metadata = this.skipValidation 
+      ? { id, ...data }
+      : await this.resource.schema.mapper({ id, ...data });
+    
+    // Direct S3 put
+    const command = {
+      Bucket: this.resource.client.config.bucket,
+      Key: key,
+      Metadata: metadata,
+      Body: '' // Empty body for speed
+    };
+    
+    await this.resource.client.client.send(new PutObjectCommand(command));
+    
+    return { id, inserted: true };
+  }
+
+  /**
+   * Bulk insert with maximum parallelism
+   */
+  async bulkInsert(items) {
+    const { results, errors } = await PromisePool
+      .for(items)
+      .withConcurrency(this.concurrency)
+      .process(async (item) => {
+        return await this.fastInsert(item);
+      });
+    
+    return {
+      success: results.length,
+      failed: errors.length,
+      errors: errors.slice(0, 10) // First 10 errors
+    };
+  }
+}

package/src/concerns/partition-queue.js

@@ -0,0 +1,171 @@
+import { EventEmitter } from 'events';
+
+/**
+ * Robust partition operation queue with retry and persistence
+ */
+export class PartitionQueue extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.maxRetries = options.maxRetries || 3;
+    this.retryDelay = options.retryDelay || 1000;
+    this.persistence = options.persistence || null; // Could be filesystem/redis/etc
+    this.queue = [];
+    this.processing = false;
+    this.failures = [];
+  }
+
+  /**
+   * Add partition operation to queue
+   */
+  async enqueue(operation) {
+    const item = {
+      id: `${Date.now()}-${Math.random()}`,
+      operation,
+      retries: 0,
+      createdAt: new Date(),
+      status: 'pending'
+    };
+    
+    this.queue.push(item);
+    
+    // Persist if configured
+    if (this.persistence) {
+      await this.persistence.save(item);
+    }
+    
+    // Start processing if not already
+    if (!this.processing) {
+      setImmediate(() => this.process());
+    }
+    
+    return item.id;
+  }
+
+  /**
+   * Process queue items
+   */
+  async process() {
+    if (this.processing || this.queue.length === 0) return;
+    
+    this.processing = true;
+    
+    while (this.queue.length > 0) {
+      const item = this.queue.shift();
+      
+      try {
+        await this.executeOperation(item);
+        item.status = 'completed';
+        this.emit('success', item);
+        
+        // Remove from persistence
+        if (this.persistence) {
+          await this.persistence.remove(item.id);
+        }
+      } catch (error) {
+        item.retries++;
+        item.lastError = error;
+        
+        if (item.retries < this.maxRetries) {
+          // Retry with exponential backoff
+          const delay = this.retryDelay * Math.pow(2, item.retries - 1);
+          item.status = 'retrying';
+          
+          setTimeout(() => {
+            this.queue.push(item);
+            if (!this.processing) this.process();
+          }, delay);
+          
+          this.emit('retry', { item, error, delay });
+        } else {
+          // Max retries reached
+          item.status = 'failed';
+          this.failures.push(item);
+          this.emit('failure', { item, error });
+          
+          // Move to DLQ in persistence
+          if (this.persistence) {
+            await this.persistence.moveToDLQ(item);
+          }
+        }
+      }
+    }
+    
+    this.processing = false;
+  }
+
+  /**
+   * Execute the actual partition operation
+   */
+  async executeOperation(item) {
+    const { type, resource, data } = item.operation;
+    
+    switch (type) {
+      case 'create':
+        return await resource.createPartitionReferences(data);
+      case 'update':
+        return await resource.handlePartitionReferenceUpdates(data.original, data.updated);
+      case 'delete':
+        return await resource.deletePartitionReferences(data);
+      default:
+        throw new Error(`Unknown operation type: ${type}`);
+    }
+  }
+
+  /**
+   * Recover from persistence on startup
+   */
+  async recover() {
+    if (!this.persistence) return;
+    
+    const items = await this.persistence.getPending();
+    this.queue.push(...items);
+    
+    if (this.queue.length > 0) {
+      this.emit('recovered', { count: this.queue.length });
+      setImmediate(() => this.process());
+    }
+  }
+
+  /**
+   * Get queue statistics
+   */
+  getStats() {
+    return {
+      pending: this.queue.length,
+      failures: this.failures.length,
+      processing: this.processing,
+      failureRate: this.failures.length / (this.queue.length + this.failures.length) || 0
+    };
+  }
+}
+
+/**
+ * Simple in-memory persistence (can be replaced with Redis, filesystem, etc)
+ */
+export class InMemoryPersistence {
+  constructor() {
+    this.items = new Map();
+    this.dlq = new Map();
+  }
+
+  async save(item) {
+    this.items.set(item.id, item);
+  }
+
+  async remove(id) {
+    this.items.delete(id);
+  }
+
+  async moveToDLQ(item) {
+    this.items.delete(item.id);
+    this.dlq.set(item.id, item);
+  }
+
+  async getPending() {
+    return Array.from(this.items.values());
+  }
+
+  async getDLQ() {
+    return Array.from(this.dlq.values());
+  }
+}

package/src/errors.js

@@ -225,8 +225,16 @@ export function mapAwsError(err, context = {}) {
     return new MissingMetadata({ ...context, original: err, metadata, commandName, commandInput, suggestion });
   }
   // Outros mapeamentos podem ser adicionados aqui
-  suggestion = 'Check the error details and AWS documentation.';
-  return new UnknownError('Unknown error', { ...context, original: err, metadata, commandName, commandInput, suggestion });
+  // Incluir detalhes do erro original para facilitar debug
+  const errorDetails = [
+    `Unknown error: ${err.message || err.toString()}`,
+    err.code && `Code: ${err.code}`,
+    err.statusCode && `Status: ${err.statusCode}`,
+    err.stack && `Stack: ${err.stack.split('\n')[0]}`,
+  ].filter(Boolean).join(' | ');
+  
+  suggestion = `Check the error details and AWS documentation. Original error: ${err.message || err.toString()}`;
+  return new UnknownError(errorDetails, { ...context, original: err, metadata, commandName, commandInput, suggestion });
 }
 
 export class ConnectionStringError extends S3dbError {

package/src/partition-drivers/base-partition-driver.js

@@ -0,0 +1,96 @@
+import { EventEmitter } from 'events';
+
+/**
+ * Base class for all partition drivers
+ * Defines the interface that all drivers must implement
+ */
+export class BasePartitionDriver extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.options = options;
+    this.stats = {
+      queued: 0,
+      processed: 0,
+      failed: 0,
+      processing: 0
+    };
+  }
+
+  /**
+   * Initialize the driver
+   */
+  async initialize() {
+    // Override in subclasses if needed
+  }
+
+  /**
+   * Queue partition operations for processing
+   * @param {Object} operation - The partition operation to queue
+   * @param {string} operation.type - 'create', 'update', or 'delete'
+   * @param {Object} operation.resource - The resource instance
+   * @param {Object} operation.data - The data for the operation
+   */
+  async queue(operation) {
+    throw new Error('queue() must be implemented by subclass');
+  }
+
+  /**
+   * Process a single partition operation
+   */
+  async processOperation(operation) {
+    const { type, resource, data } = operation;
+    
+    try {
+      this.stats.processing++;
+      
+      switch (type) {
+        case 'create':
+          await resource.createPartitionReferences(data.object);
+          break;
+          
+        case 'update':
+          await resource.handlePartitionReferenceUpdates(data.original, data.updated);
+          break;
+          
+        case 'delete':
+          await resource.deletePartitionReferences(data.object);
+          break;
+          
+        default:
+          throw new Error(`Unknown partition operation type: ${type}`);
+      }
+      
+      this.stats.processed++;
+      this.emit('processed', operation);
+      
+    } catch (error) {
+      this.stats.failed++;
+      this.emit('error', { operation, error });
+      throw error;
+    } finally {
+      this.stats.processing--;
+    }
+  }
+
+  /**
+   * Flush any pending operations
+   */
+  async flush() {
+    // Override in subclasses if needed
+  }
+
+  /**
+   * Get driver statistics
+   */
+  getStats() {
+    return { ...this.stats };
+  }
+
+  /**
+   * Shutdown the driver
+   */
+  async shutdown() {
+    await this.flush();
+    this.removeAllListeners();
+  }
+}

package/src/partition-drivers/index.js

@@ -0,0 +1,60 @@
+import { SyncPartitionDriver } from './sync-partition-driver.js';
+import { MemoryPartitionDriver } from './memory-partition-driver.js';
+import { SQSPartitionDriver } from './sqs-partition-driver.js';
+
+/**
+ * Partition driver factory
+ */
+export class PartitionDriverFactory {
+  static drivers = {
+    sync: SyncPartitionDriver,
+    memory: MemoryPartitionDriver,
+    sqs: SQSPartitionDriver
+  };
+
+  /**
+   * Create a partition driver instance
+   * @param {string|Object} config - Driver name or configuration object
+   * @returns {BasePartitionDriver} Driver instance
+   */
+  static create(config) {
+    // Handle string shorthand
+    if (typeof config === 'string') {
+      config = { driver: config };
+    }
+    
+    // Default to memory driver
+    const driverName = config.driver || 'memory';
+    
+    // Get driver class
+    const DriverClass = this.drivers[driverName];
+    if (!DriverClass) {
+      throw new Error(`Unknown partition driver: ${driverName}. Available: ${Object.keys(this.drivers).join(', ')}`);
+    }
+    
+    // Create and initialize driver
+    const driver = new DriverClass(config);
+    
+    return driver;
+  }
+
+  /**
+   * Register a custom driver
+   */
+  static register(name, DriverClass) {
+    this.drivers[name] = DriverClass;
+  }
+
+  /**
+   * Get available driver names
+   */
+  static getAvailableDrivers() {
+    return Object.keys(this.drivers);
+  }
+}
+
+// Export individual drivers
+export { BasePartitionDriver } from './base-partition-driver.js';
+export { SyncPartitionDriver } from './sync-partition-driver.js';
+export { MemoryPartitionDriver } from './memory-partition-driver.js';
+export { SQSPartitionDriver } from './sqs-partition-driver.js';