procbay-schema 1.0.0

package/prisma/seeders/utils/performance-config.js

@@ -0,0 +1,341 @@
+import os from 'os';
+
+/**
+ * Performance configuration based on system resources and data characteristics
+ */
+export class PerformanceConfig {
+  constructor(options = {}) {
+    this.cpuCount = os.cpus().length;
+    this.totalMemory = os.totalmem();
+    this.availableMemory = os.freemem();
+    this.customOptions = options;
+    
+    // Calculate optimal configurations
+    this.config = this.calculateOptimalConfig();
+  }
+
+  calculateOptimalConfig() {
+    // Base configuration
+    const baseConfig = {
+      // Geographic data - high volume
+      geographic: {
+        cities: {
+          batchSize: 15000,
+          maxConcurrentBatches: Math.min(8, this.cpuCount),
+          memoryThreshold: 0.7, // Use 70% of available memory
+          connectionPoolSize: 15,
+          queryTimeout: 120000, // 2 minutes
+        },
+        countries: {
+          batchSize: 50,
+          maxConcurrentBatches: Math.min(4, this.cpuCount),
+          memoryThreshold: 0.5,
+          connectionPoolSize: 10,
+          queryTimeout: 60000,
+          retryAttempts: 3,
+          retryDelay: 1000,
+          exponentialBackoff: true,
+        },
+        states: {
+          batchSize: 500,
+          maxConcurrentBatches: Math.min(6, this.cpuCount),
+          memoryThreshold: 0.6,
+          connectionPoolSize: 12,
+          queryTimeout: 90000,
+          retryAttempts: 3,
+          retryDelay: 1000,
+          exponentialBackoff: true,
+        }
+      },
+      
+      // Core data - low volume, high precision
+      core: {
+        users: {
+          batchSize: 50,
+          maxConcurrentBatches: 2,
+          memoryThreshold: 0.3,
+          connectionPoolSize: 5,
+          queryTimeout: 30000,
+          retryAttempts: 3,
+          retryDelay: 1000,
+          exponentialBackoff: true,
+          criticalEntity: true,
+        },
+        roles: {
+          batchSize: 25,
+          maxConcurrentBatches: 1,
+          memoryThreshold: 0.2,
+          connectionPoolSize: 3,
+          queryTimeout: 15000,
+          retryAttempts: 5,
+          retryDelay: 500,
+          exponentialBackoff: true,
+          criticalEntity: true,
+        },
+        permissions: {
+          batchSize: 100,
+          maxConcurrentBatches: 2,
+          memoryThreshold: 0.4,
+          connectionPoolSize: 6,
+          queryTimeout: 45000,
+          retryAttempts: 3,
+          retryDelay: 750,
+          exponentialBackoff: true,
+          criticalEntity: true,
+        },
+        departments: {
+          batchSize: 50,
+          maxConcurrentBatches: 2,
+          memoryThreshold: 0.3,
+          connectionPoolSize: 4,
+          queryTimeout: 20000,
+          retryAttempts: 3,
+          retryDelay: 800,
+          exponentialBackoff: true,
+          criticalEntity: true,
+        },
+        currencies: {
+          batchSize: 75,
+          maxConcurrentBatches: 2,
+          memoryThreshold: 0.3,
+          connectionPoolSize: 4,
+          queryTimeout: 25000,
+          retryAttempts: 3,
+          retryDelay: 600,
+          exponentialBackoff: true,
+          criticalEntity: true,
+        }
+      },
+      
+      // Content data - medium volume with enhanced configurations
+      content: {
+        categories: {
+          batchSize: 750, // Optimized medium batch size (500-1000)
+          maxConcurrentBatches: Math.min(4, this.cpuCount),
+          memoryThreshold: 0.5,
+          connectionPoolSize: 8,
+          queryTimeout: 60000,
+          retryAttempts: 3,
+          retryDelay: 750,
+          exponentialBackoff: true,
+          enableUniqueConstraintMapping: true,
+          enableMemoryMonitoring: true,
+          memoryCheckInterval: 100, // Check memory every 100 records
+          useConnectionPooling: true,
+          batchUpsertMode: true,
+          uniqueFields: ['name', 'slug'], // Fields for constraint mapping
+          enableBatchOptimization: true
+        },
+        items: {
+          batchSize: 2000,
+          maxConcurrentBatches: Math.min(6, this.cpuCount),
+          memoryThreshold: 0.6,
+          connectionPoolSize: 10,
+          queryTimeout: 90000,
+          retryAttempts: 3,
+          retryDelay: 1000,
+          exponentialBackoff: true,
+          enableUniqueConstraintMapping: true,
+          enableMemoryMonitoring: true,
+          memoryCheckInterval: 200,
+          useConnectionPooling: true,
+          batchUpsertMode: true,
+          uniqueFields: ['name', 'sku'],
+          enableBatchOptimization: true
+        },
+        attributes: {
+          batchSize: 500,
+          maxConcurrentBatches: Math.min(3, this.cpuCount),
+          memoryThreshold: 0.4,
+          connectionPoolSize: 6,
+          queryTimeout: 45000,
+          retryAttempts: 3,
+          retryDelay: 500,
+          exponentialBackoff: true,
+          enableUniqueConstraintMapping: true,
+          enableMemoryMonitoring: true,
+          memoryCheckInterval: 50,
+          useConnectionPooling: true,
+          batchUpsertMode: true,
+          uniqueFields: ['name'],
+          enableBatchOptimization: true
+        },
+        templates: {
+          batchSize: 1000,
+          maxConcurrentBatches: Math.min(3, this.cpuCount),
+          memoryThreshold: 0.4,
+          connectionPoolSize: 6,
+          queryTimeout: 75000,
+          retryAttempts: 3,
+          retryDelay: 800,
+          exponentialBackoff: true,
+          enableUniqueConstraintMapping: true,
+          enableMemoryMonitoring: true,
+          memoryCheckInterval: 100,
+          useConnectionPooling: true,
+          batchUpsertMode: true,
+          uniqueFields: ['name', 'type'],
+          enableBatchOptimization: true
+        }
+      }
+    };
+
+    // Adjust based on available memory
+    const memoryGb = this.totalMemory / (1024 * 1024 * 1024);
+    
+    if (memoryGb < 4) {
+      // Low memory system - reduce batch sizes and concurrency
+      this.adjustForLowMemory(baseConfig);
+    } else if (memoryGb > 16) {
+      // High memory system - increase batch sizes
+      this.adjustForHighMemory(baseConfig);
+    }
+
+    // Apply custom overrides
+    return this.applyCustomOverrides(baseConfig);
+  }
+
+  adjustForLowMemory(config) {
+    const reductionFactor = 0.5;
+    
+    Object.values(config).forEach(category => {
+      Object.values(category).forEach(entityConfig => {
+        entityConfig.batchSize = Math.max(100, Math.floor(entityConfig.batchSize * reductionFactor));
+        entityConfig.maxConcurrentBatches = Math.max(1, Math.floor(entityConfig.maxConcurrentBatches * 0.5));
+        entityConfig.connectionPoolSize = Math.max(3, Math.floor(entityConfig.connectionPoolSize * 0.6));
+      });
+    });
+  }
+
+  adjustForHighMemory(config) {
+    const increaseFactor = 1.5;
+    
+    Object.values(config).forEach(category => {
+      Object.values(category).forEach(entityConfig => {
+        entityConfig.batchSize = Math.floor(entityConfig.batchSize * increaseFactor);
+        entityConfig.maxConcurrentBatches = Math.min(this.cpuCount * 2, Math.floor(entityConfig.maxConcurrentBatches * increaseFactor));
+        entityConfig.connectionPoolSize = Math.min(20, Math.floor(entityConfig.connectionPoolSize * 1.2));
+      });
+    });
+  }
+
+  applyCustomOverrides(config) {
+    if (this.customOptions.profile) {
+      switch (this.customOptions.profile) {
+        case 'speed':
+          this.optimizeForSpeed(config);
+          break;
+        case 'memory':
+          this.optimizeForMemory(config);
+          break;
+        case 'stability':
+          this.optimizeForStability(config);
+          break;
+      }
+    }
+
+    // Apply specific overrides
+    if (this.customOptions.overrides) {
+      this.deepMerge(config, this.customOptions.overrides);
+    }
+
+    return config;
+  }
+
+  optimizeForSpeed(config) {
+    Object.values(config).forEach(category => {
+      Object.values(category).forEach(entityConfig => {
+        entityConfig.batchSize *= 2;
+        entityConfig.maxConcurrentBatches = Math.min(this.cpuCount * 2, entityConfig.maxConcurrentBatches * 2);
+        entityConfig.queryTimeout *= 0.75; // Shorter timeouts for faster failure detection
+      });
+    });
+  }
+
+  optimizeForMemory(config) {
+    Object.values(config).forEach(category => {
+      Object.values(category).forEach(entityConfig => {
+        entityConfig.batchSize = Math.max(50, Math.floor(entityConfig.batchSize * 0.4));
+        entityConfig.maxConcurrentBatches = Math.max(1, Math.floor(entityConfig.maxConcurrentBatches * 0.5));
+        entityConfig.memoryThreshold *= 0.7;
+      });
+    });
+  }
+
+  optimizeForStability(config) {
+    Object.values(config).forEach(category => {
+      Object.values(category).forEach(entityConfig => {
+        entityConfig.maxConcurrentBatches = Math.max(1, Math.floor(entityConfig.maxConcurrentBatches * 0.6));
+        entityConfig.queryTimeout *= 2; // Longer timeouts for stability
+        entityConfig.retryAttempts = 5;
+        entityConfig.retryDelay = 2000;
+      });
+    });
+  }
+
+  deepMerge(target, source) {
+    for (const key in source) {
+      if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
+        if (!target[key]) target[key] = {};
+        this.deepMerge(target[key], source[key]);
+      } else {
+        target[key] = source[key];
+      }
+    }
+  }
+
+  getConfig(category, entity) {
+    return this.config[category]?.[entity] || this.config.core.users; // fallback
+  }
+
+  logConfiguration() {
+    console.log('\n=== Performance Configuration ===');
+    console.log(`System: ${this.cpuCount} CPUs, ${(this.totalMemory / (1024**3)).toFixed(1)}GB RAM`);
+    console.log(`Available Memory: ${(this.availableMemory / (1024**3)).toFixed(1)}GB`);
+    console.log('\nOptimized Configurations:');
+    
+    Object.entries(this.config).forEach(([category, entities]) => {
+      console.log(`\n${category.toUpperCase()}:`);
+      Object.entries(entities).forEach(([entity, config]) => {
+        console.log(`  ${entity}: batch=${config.batchSize}, concurrent=${config.maxConcurrentBatches}, pool=${config.connectionPoolSize}`);
+      });
+    });
+    console.log('\n');
+  }
+}
+
+// Factory function
+export function createPerformanceConfig(options = {}) {
+  return new PerformanceConfig(options);
+}
+
+// Preset configurations
+export const PERFORMANCE_PRESETS = {
+  DEVELOPMENT: {
+    profile: 'stability',
+    overrides: {
+      geographic: {
+        cities: { batchSize: 5000, maxConcurrentBatches: 2 }
+      }
+    }
+  },
+  
+  PRODUCTION: {
+    profile: 'speed',
+    overrides: {
+      geographic: {
+        cities: { batchSize: 20000, maxConcurrentBatches: 12 }
+      }
+    }
+  },
+  
+  CI_CD: {
+    profile: 'memory',
+    overrides: {
+      geographic: {
+        cities: { batchSize: 2000, maxConcurrentBatches: 1 }
+      }
+    }
+  }
+};
+

package/prisma/seeders/utils/performance-helpers.js

@@ -0,0 +1,206 @@
+import colors from "colors";
+
+/**
+ * Enhanced retry logic with exponential backoff for geographic seeders
+ */
+export class RetryHandler {
+  constructor(maxAttempts = 3, baseDelay = 1000) {
+    this.maxAttempts = maxAttempts;
+    this.baseDelay = baseDelay;
+  }
+
+  async execute(operation, context = 'operation') {
+    let lastError;
+    
+    for (let attempt = 1; attempt <= this.maxAttempts; attempt++) {
+      try {
+        return await operation();
+      } catch (error) {
+        lastError = error;
+        
+        if (attempt < this.maxAttempts) {
+          const delay = this.baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
+          console.warn(colors.yellow(`${context} failed (attempt ${attempt}/${this.maxAttempts}), retrying in ${delay}ms...`));
+          await this.sleep(delay);
+        }
+      }
+    }
+    
+    throw new Error(`${context} failed after ${this.maxAttempts} attempts: ${lastError.message}`);
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+/**
+ * Memory management utilities for large batch operations
+ */
+export class MemoryManager {
+  static triggerGarbageCollection() {
+    if (global.gc) {
+      global.gc();
+    }
+  }
+
+  static async forceGarbageCollectionWithDelay(delayMs = 10) {
+    if (global.gc) {
+      global.gc();
+      // Small delay to allow GC to complete
+      await new Promise(resolve => setTimeout(resolve, delayMs));
+    }
+  }
+
+  static logMemoryUsage(context = 'Memory') {
+    const used = process.memoryUsage();
+    const formatBytes = (bytes) => (bytes / 1024 / 1024).toFixed(2) + ' MB';
+    
+    console.log(colors.gray(`${context} - RSS: ${formatBytes(used.rss)} | Heap Used: ${formatBytes(used.heapUsed)} | Heap Total: ${formatBytes(used.heapTotal)} | External: ${formatBytes(used.external)}`));
+  }
+
+  static getMemoryUsageThreshold(availableMemory, threshold = 0.7) {
+    return availableMemory * threshold;
+  }
+
+  static isMemoryThresholdExceeded(threshold) {
+    const used = process.memoryUsage();
+    return used.heapUsed > threshold;
+  }
+}
+
+/**
+ * Performance utilities for geographic data operations
+ */
+export class GeographicPerformanceUtils {
+  static createOptimizedBatchProcessor(performanceConfig, retryHandler) {
+    return {
+      batchSize: performanceConfig.batchSize,
+      maxConcurrentBatches: performanceConfig.maxConcurrentBatches,
+      connectionPoolSize: performanceConfig.connectionPoolSize,
+      queryTimeout: performanceConfig.queryTimeout,
+      retryHandler,
+      
+      async processBatches(data, processor, entityName) {
+        const chunks = this.createChunks(data, this.batchSize);
+        let totalProcessed = 0;
+        let totalCreated = 0;
+        const errors = [];
+
+        console.log(colors.cyan(`Processing ${data.length.toLocaleString()} ${entityName} in ${chunks.length} chunks...`));
+
+        for (let i = 0; i < chunks.length; i += this.maxConcurrentBatches) {
+          const currentBatch = chunks.slice(i, i + this.maxConcurrentBatches);
+          
+          const batchPromises = currentBatch.map(async (chunk, index) => {
+            const chunkIndex = i + index;
+            
+            try {
+              const result = await processor(chunk, chunkIndex);
+              return { success: true, ...result };
+            } catch (error) {
+              errors.push(`Chunk ${chunkIndex} failed: ${error.message}`);
+              return { success: false, error: error.message };
+            }
+          });
+
+          const batchResults = await Promise.allSettled(batchPromises);
+          
+          batchResults.forEach((result) => {
+            if (result.status === 'fulfilled' && result.value.success) {
+              totalProcessed += result.value.processed || 0;
+              totalCreated += result.value.created || 0;
+            }
+          });
+
+          // Memory management after each concurrent batch
+          MemoryManager.triggerGarbageCollection();
+          
+          // Progress reporting
+          const progress = Math.min(((i + this.maxConcurrentBatches) / chunks.length) * 100, 100);
+          console.log(colors.green(`Progress: ${progress.toFixed(1)}% | Created: ${totalCreated.toLocaleString()}`));
+        }
+
+        return {
+          totalProcessed,
+          totalCreated,
+          errors,
+          success: errors.length === 0
+        };
+      },
+
+      createChunks(data, chunkSize) {
+        const chunks = [];
+        for (let i = 0; i < data.length; i += chunkSize) {
+          chunks.push(data.slice(i, i + chunkSize));
+        }
+        return chunks;
+      }
+    };
+  }
+
+  /**
+   * Creates an optimized ID mapping function for geographic entities
+   */
+  static createIdMapper(prisma, model, keyField, valueField) {
+    return async (whereClause = {}) => {
+      const records = await prisma[model].findMany({
+        where: whereClause,
+        select: {
+          [keyField]: true,
+          [valueField]: true
+        }
+      });
+
+      const idMap = new Map();
+      records.forEach(record => {
+        idMap.set(record[keyField], record[valueField]);
+      });
+
+      return idMap;
+    };
+  }
+}
+
+/**
+ * Factory function to create optimized retry handler with geographic defaults
+ */
+export function createGeographicRetryHandler(category = 'geographic') {
+  const retryConfig = {
+    geographic: { maxAttempts: 3, baseDelay: 1000 },
+    core: { maxAttempts: 2, baseDelay: 500 },
+    content: { maxAttempts: 3, baseDelay: 750 }
+  };
+
+  const config = retryConfig[category] || retryConfig.geographic;
+  return new RetryHandler(config.maxAttempts, config.baseDelay);
+}
+
+/**
+ * Convenience function for batch processing with performance optimizations
+ */
+export async function processWithPerformanceConfig(
+  category, 
+  entity, 
+  data, 
+  processor, 
+  options = {}
+) {
+  const { createPerformanceConfig } = await import('./performance-config.js');
+  const performanceConfig = createPerformanceConfig().getConfig(category, entity);
+  const retryHandler = createGeographicRetryHandler(category);
+  
+  const utils = GeographicPerformanceUtils.createOptimizedBatchProcessor(
+    performanceConfig, 
+    retryHandler
+  );
+
+  MemoryManager.logMemoryUsage(`Before ${entity} processing`);
+  
+  const result = await utils.processBatches(data, processor, entity);
+  
+  MemoryManager.logMemoryUsage(`After ${entity} processing`);
+  MemoryManager.forceGarbageCollectionWithDelay();
+  
+  return result;
+}