procbay-schema 1.0.0

package/prisma/seeders/utils/progress-tracker.js

@@ -0,0 +1,364 @@
+import colors from "colors";
+
+/**
+ * Creates a progress tracker for seeding operations
+ * @param {number} total - Total number of items to process
+ * @param {string} entityName - Name of the entity being processed (e.g., "users", "roles")
+ * @returns {Object} - Progress tracker object with update and complete methods
+ */
+export function createProgressTracker(total, entityName) {
+  let processed = 0;
+  let lastPercent = -1; // Track last percentage to reduce unnecessary updates
+
+  // Create a normalized entity name for display
+  const displayName = entityName.charAt(0).toUpperCase() + entityName.slice(1);
+
+  /**
+   * Updates the progress bar
+   * @param {number} increment - Amount to increment progress by (default: 1)
+   */
+  const update = (increment = 1) => {
+    processed += increment;
+    const percent = Math.floor((processed / total) * 100);
+
+    // For large datasets, update less frequently to improve performance
+    const updateThreshold = total > 100000 ? Math.max(Math.floor(total / 50), 1000) : Math.max(Math.floor(total / 100), 10);
+    
+    // Only update display when percentage changes or at regular intervals for large data
+    if (
+      percent !== lastPercent ||
+      processed % updateThreshold === 0 ||
+      processed === total
+    ) {
+      lastPercent = percent;
+      const filled = Math.floor((40 * processed) / total);
+      const bar =
+        colors.magenta("█".repeat(filled)) +
+        colors.gray("░".repeat(40 - filled));
+
+      // Add processing rate for large datasets
+      const progressText = total > 10000 
+        ? `\rProcessing ${entityName}: ${bar} ${colors.bold(`${percent}%`)} (${processed.toLocaleString()}/${total.toLocaleString()})`
+        : `\rProcessing ${entityName}: ${bar} ${colors.bold(`${percent}%`)} (${processed}/${total})`;
+      
+      process.stdout.write(progressText);
+
+      if (processed === total) {
+        process.stdout.write("\n");
+      }
+    }
+  };
+
+  /**
+   * Forces completion of the progress bar
+   */
+  const complete = () => {
+    if (processed < total) {
+      processed = total;
+      const percent = 100;
+      const bar = colors.magenta("█".repeat(40));
+
+      process.stdout.write(
+        `\rProcessing ${entityName}: ${bar} ${colors.bold(
+          `${percent}%`
+        )} (${processed.toLocaleString()}/${total.toLocaleString()})`
+      );
+      process.stdout.write("\n");
+    }
+  };
+
+  /**
+   * Logs a message with the current progress context
+   * @param {string} message - Message to log
+   * @param {string} type - Type of message (info, success, warning, error)
+   */
+  const log = (message, type = "info") => {
+    // First clear the current line if we're showing a progress bar
+    if (processed > 0) {
+      process.stdout.write("\r" + " ".repeat(100) + "\r");
+    }
+
+    let formattedMessage;
+
+    switch (type) {
+      case "success":
+        formattedMessage = colors.green(message);
+        break;
+      case "warning":
+        formattedMessage = colors.yellow(message);
+        break;
+      case "error":
+        formattedMessage = colors.red(message);
+        break;
+      case "info":
+      default:
+        formattedMessage = colors.gray(message);
+        break;
+    }
+
+    console.log(formattedMessage);
+
+    // Redraw the progress bar if we're in the middle of displaying it
+    if (processed > 0 && processed < total) {
+      const percent = Math.floor((processed / total) * 100);
+      const filled = Math.floor((40 * processed) / total);
+      const bar =
+        colors.magenta("█".repeat(filled)) +
+        colors.gray("░".repeat(40 - filled));
+
+      process.stdout.write(
+        `\rProcessing ${entityName}: ${bar} ${colors.bold(
+          `${percent}%`
+        )} (${processed.toLocaleString()}/${total.toLocaleString()})`
+      );
+    }
+  };
+
+  return {
+    update,
+    complete,
+    log,
+    get progress() {
+      return processed;
+    },
+    get total() {
+      return total;
+    },
+    get percentage() {
+      return Math.floor((processed / total) * 100);
+    },
+  };
+}
+
+/**
+ * Logs a seeder start message
+ * @param {string} entityName - Name of the entity being seeded
+ */
+export function logSeederStart(entityName) {
+  console.log(colors.cyan(`Seeding ${entityName} table...`));
+}
+
+/**
+ * Logs a seeder completion message
+ * @param {string} entityName - Name of the entity that was seeded
+ */
+export function logSeederComplete(entityName) {
+  console.log(colors.green(`${entityName} table seeded successfully`));
+}
+
+/**
+ * Logs a seeder error message
+ * @param {string} entityName - Name of the entity that was being seeded
+ * @param {Error} error - Error object
+ */
+export function logSeederError(entityName, error) {
+  console.error(colors.red(`Error seeding ${entityName}: ${error.message}`));
+}
+
+// Global error tracking and skipped records system
+const globalErrorTracker = {
+  totalErrors: 0,
+  errorsByEntity: new Map(),
+  skippedRecords: new Map(),
+  processingSummary: new Map()
+};
+
+/**
+ * Enhanced entity error logging with detailed tracking
+ * @param {string} entityType - Type of entity (e.g., "user", "role")
+ * @param {string} entityName - Name of the specific entity instance
+ * @param {string} errorMessage - Error message
+ * @param {Object} errorDetails - Additional error context
+ * @param {Object} options - Logging options
+ */
+export function logEntityError(entityType, entityName, errorMessage, errorDetails = {}, options = {}) {
+  const timestamp = new Date().toISOString();
+  const errorId = `${entityType}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  
+  // Track the error globally
+  globalErrorTracker.totalErrors++;
+  
+  if (!globalErrorTracker.errorsByEntity.has(entityType)) {
+    globalErrorTracker.errorsByEntity.set(entityType, []);
+  }
+  
+  const errorRecord = {
+    id: errorId,
+    timestamp,
+    entityType,
+    entityName,
+    errorMessage,
+    errorDetails,
+    severity: errorDetails.severity || 'error',
+    recoverable: errorDetails.recoverable !== false,
+    batchIndex: errorDetails.batchIndex,
+    attemptNumber: errorDetails.attemptNumber || 1
+  };
+  
+  globalErrorTracker.errorsByEntity.get(entityType).push(errorRecord);
+  
+  // Log based on severity and options
+  const logMessage = options.includeTimestamp 
+    ? `[${timestamp}] Error creating ${entityType} ${colors.bold(entityName)}: ${errorMessage}`
+    : `Error creating ${entityType} ${colors.bold(entityName)}: ${errorMessage}`;
+    
+  if (errorDetails.severity === 'critical') {
+    console.error(colors.red(logMessage));
+    if (errorDetails.stackTrace) {
+      console.error(colors.gray(`Stack: ${errorDetails.stackTrace}`));
+    }
+  } else if (errorDetails.severity === 'warning') {
+    console.warn(colors.yellow(logMessage));
+  } else {
+    console.error(colors.yellow(logMessage));
+  }
+  
+  // Log additional context if provided
+  if (errorDetails.batchIndex !== undefined) {
+    console.error(colors.gray(`  Batch: ${errorDetails.batchIndex}, Attempt: ${errorDetails.attemptNumber || 1}`));
+  }
+  
+  if (errorDetails.context) {
+    console.error(colors.gray(`  Context: ${JSON.stringify(errorDetails.context)}`));
+  }
+  
+  return errorId;
+}
+
+/**
+ * Track skipped records with reason
+ * @param {string} entityType - Type of entity
+ * @param {string} entityName - Name of the entity
+ * @param {string} reason - Reason for skipping
+ * @param {Object} record - The actual record that was skipped
+ */
+export function trackSkippedRecord(entityType, entityName, reason, record = {}) {
+  if (!globalErrorTracker.skippedRecords.has(entityType)) {
+    globalErrorTracker.skippedRecords.set(entityType, []);
+  }
+  
+  const skippedRecord = {
+    timestamp: new Date().toISOString(),
+    entityType,
+    entityName,
+    reason,
+    record: record || {},
+    id: `${entityType}_skip_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`
+  };
+  
+  globalErrorTracker.skippedRecords.get(entityType).push(skippedRecord);
+  
+  console.warn(colors.cyan(`Skipped ${entityType} ${colors.bold(entityName)}: ${reason}`));
+  
+  return skippedRecord.id;
+}
+
+/**
+ * Get error summary for an entity type
+ * @param {string} entityType - Type of entity
+ * @returns {Object} Error summary
+ */
+export function getErrorSummary(entityType = null) {
+  if (entityType) {
+    return {
+      errors: globalErrorTracker.errorsByEntity.get(entityType) || [],
+      skipped: globalErrorTracker.skippedRecords.get(entityType) || [],
+      totalErrors: (globalErrorTracker.errorsByEntity.get(entityType) || []).length,
+      totalSkipped: (globalErrorTracker.skippedRecords.get(entityType) || []).length
+    };
+  }
+  
+  return {
+    totalErrors: globalErrorTracker.totalErrors,
+    errorsByEntity: Object.fromEntries(globalErrorTracker.errorsByEntity),
+    skippedRecords: Object.fromEntries(globalErrorTracker.skippedRecords),
+    summary: generateGlobalErrorSummary()
+  };
+}
+
+/**
+ * Generate a comprehensive error summary
+ */
+function generateGlobalErrorSummary() {
+  const summary = {
+    totalEntitiesWithErrors: globalErrorTracker.errorsByEntity.size,
+    totalEntitiesWithSkipped: globalErrorTracker.skippedRecords.size,
+    totalErrors: globalErrorTracker.totalErrors,
+    totalSkipped: 0,
+    errorsByEntityType: {},
+    skippedByEntityType: {}
+  };
+  
+  // Count errors by entity type
+  for (const [entityType, errors] of globalErrorTracker.errorsByEntity) {
+    summary.errorsByEntityType[entityType] = errors.length;
+  }
+  
+  // Count skipped by entity type
+  for (const [entityType, skipped] of globalErrorTracker.skippedRecords) {
+    summary.skippedByEntityType[entityType] = skipped.length;
+    summary.totalSkipped += skipped.length;
+  }
+  
+  return summary;
+}
+
+/**
+ * Reset error tracking (useful for testing or multiple runs)
+ */
+export function resetErrorTracking() {
+  globalErrorTracker.totalErrors = 0;
+  globalErrorTracker.errorsByEntity.clear();
+  globalErrorTracker.skippedRecords.clear();
+  globalErrorTracker.processingSummary.clear();
+}
+
+/**
+ * Log final validation summary comparing source vs processed counts
+ * @param {Object} validationData - Validation data
+ */
+export function logValidationSummary(validationData) {
+  console.log(colors.cyan("\n📊 FINAL VALIDATION SUMMARY"));
+  console.log(colors.cyan("================================"));
+  
+  const summary = getErrorSummary();
+  
+  // Source vs processed counts
+  if (validationData.sourceData) {
+    for (const [entityType, sourceCount] of Object.entries(validationData.sourceData)) {
+      const processed = validationData.processedData?.[entityType] || 0;
+      const created = validationData.createdData?.[entityType] || 0;
+      const errors = summary.errorsByEntityType[entityType] || 0;
+      const skipped = summary.skippedByEntityType[entityType] || 0;
+      
+      console.log(colors.white(`\n${entityType.toUpperCase()}:`));
+      console.log(colors.gray(`  Source records: ${sourceCount.toLocaleString()}`));
+      console.log(colors.gray(`  Processed: ${processed.toLocaleString()}`));
+      console.log(colors.green(`  Created: ${created.toLocaleString()}`));
+      
+      if (errors > 0) {
+        console.log(colors.red(`  Errors: ${errors.toLocaleString()}`));
+      }
+      
+      if (skipped > 0) {
+        console.log(colors.yellow(`  Skipped: ${skipped.toLocaleString()}`));
+      }
+      
+      const successRate = sourceCount > 0 ? ((created / sourceCount) * 100).toFixed(2) : '0.00';
+      console.log(colors.cyan(`  Success rate: ${successRate}%`));
+    }
+  }
+  
+  console.log(colors.cyan("\n📈 OVERALL SUMMARY:"));
+  console.log(colors.gray(`Total errors: ${summary.totalErrors.toLocaleString()}`));
+  console.log(colors.gray(`Total skipped: ${summary.totalSkipped.toLocaleString()}`));
+  console.log(colors.gray(`Entities with issues: ${summary.totalEntitiesWithErrors}`));
+  
+  if (summary.totalErrors > 0 || summary.totalSkipped > 0) {
+    console.log(colors.yellow("\n⚠️  Some records had issues. Check logs above for details."));
+  } else {
+    console.log(colors.green("\n✅ All records processed successfully!"));
+  }
+  
+  console.log(colors.cyan("================================\n"));
+}

package/prisma/seeders/utils/seeder-config-mapper.js

@@ -0,0 +1,316 @@
+/**
+ * Seeder Configuration Mapping System
+ * 
+ * This module provides:
+ * 1. Mapping of seeders to categories (Geographic, Core, Content)
+ * 2. Auto-config selection based on data volume
+ * 3. Fallback config for unmapped seeders
+ */
+
+import { createPerformanceConfig, PERFORMANCE_PRESETS } from './performance-config.js';
+import { getCurrentEnvironmentConfig } from './config.js';
+
+/**
+ * Seeder category mappings
+ */
+export const SEEDER_CATEGORIES = {
+  GEOGRAPHIC: 'geographic',
+  CORE: 'core',
+  CONTENT: 'content'
+};
+
+/**
+ * Map each seeder to its category and configuration key
+ */
+export const SEEDER_MAPPING = {
+  // Geographic seeders
+  'country': { category: SEEDER_CATEGORIES.GEOGRAPHIC, configKey: 'countries' },
+  'state': { category: SEEDER_CATEGORIES.GEOGRAPHIC, configKey: 'states' },
+  'city': { category: SEEDER_CATEGORIES.GEOGRAPHIC, configKey: 'cities' },
+  'region': { category: SEEDER_CATEGORIES.GEOGRAPHIC, configKey: 'regions' },
+  'subregion': { category: SEEDER_CATEGORIES.GEOGRAPHIC, configKey: 'subregions' },
+  
+  // Core seeders  
+  'user': { category: SEEDER_CATEGORIES.CORE, configKey: 'users' },
+  'role': { category: SEEDER_CATEGORIES.CORE, configKey: 'roles' },
+  'permission': { category: SEEDER_CATEGORIES.CORE, configKey: 'permissions' },
+  'department': { category: SEEDER_CATEGORIES.CORE, configKey: 'departments' },
+  'currency': { category: SEEDER_CATEGORIES.CORE, configKey: 'currencies' },
+  'role-permission': { category: SEEDER_CATEGORIES.CORE, configKey: 'permissions' },
+  'user-role': { category: SEEDER_CATEGORIES.CORE, configKey: 'users' },
+  'user-department': { category: SEEDER_CATEGORIES.CORE, configKey: 'users' },
+  'serial-number-config': { category: SEEDER_CATEGORIES.CORE, configKey: 'departments' },
+  
+  // Content seeders
+  'category': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'categories' },
+  'item': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'items' },
+  'attribute': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'attributes' },
+  'attribute-value': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'attributes' },
+  'template': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'templates' },
+  'email-template': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'templates' },
+  'dynamic-form': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'templates' },
+  'form-master': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'templates' },
+  'faq': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'categories' },
+  'faq-category': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'categories' },
+  'support-category': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'categories' },
+  'approval-hierarchy': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'categories' },
+  'customer-notification': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'templates' },
+  'smtp-config': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'templates' },
+  'strategies': { category: SEEDER_CATEGORIES.CONTENT, configKey: 'categories' }
+};
+
+/**
+ * Data volume thresholds for auto-config selection
+ */
+const DATA_VOLUME_THRESHOLDS = {
+  // Number of records thresholds
+  HIGH_VOLUME: 50000,
+  MEDIUM_VOLUME: 5000,
+  LOW_VOLUME: 500,
+  
+  // Memory usage thresholds (in MB)
+  HIGH_MEMORY: 1024,
+  MEDIUM_MEMORY: 512,
+  LOW_MEMORY: 128
+};
+
+/**
+ * Fallback configuration for unmapped seeders
+ */
+const FALLBACK_CONFIG = {
+  batchSize: 100,
+  maxConcurrentBatches: 2,
+  memoryThreshold: 0.4,
+  connectionPoolSize: 5,
+  queryTimeout: 30000,
+  retryAttempts: 3,
+  retryDelay: 1000,
+  exponentialBackoff: true
+};
+
+/**
+ * Auto-determine data volume based on various factors
+ * 
+ * @param {Object} options - Options for volume determination
+ * @param {number} options.expectedRecords - Expected number of records
+ * @param {number} options.availableMemory - Available memory in bytes
+ * @param {string[]} options.countryFilter - Country filter array
+ * @returns {string} 'High', 'Medium', or 'Low'
+ */
+export function determineDataVolume(options = {}) {
+  const {
+    expectedRecords = 0,
+    availableMemory = 0,
+    countryFilter = [],
+    seederType = 'unknown'
+  } = options;
+
+  // Convert memory to MB
+  const availableMemoryMB = availableMemory / (1024 * 1024);
+  
+  // Geographic data volume is heavily influenced by country filtering
+  if (seederType === 'geographic') {
+    if (countryFilter.length === 0) {
+      // All countries - high volume
+      return 'High';
+    } else if (countryFilter.length <= 3) {
+      // Few countries - medium volume
+      return 'Medium';
+    } else {
+      // Many countries but filtered - medium to high
+      return countryFilter.length > 10 ? 'High' : 'Medium';
+    }
+  }
+  
+  // For other types, use record count and memory
+  if (expectedRecords > DATA_VOLUME_THRESHOLDS.HIGH_VOLUME || 
+      availableMemoryMB > DATA_VOLUME_THRESHOLDS.HIGH_MEMORY) {
+    return 'High';
+  } else if (expectedRecords > DATA_VOLUME_THRESHOLDS.MEDIUM_VOLUME || 
+             availableMemoryMB > DATA_VOLUME_THRESHOLDS.MEDIUM_MEMORY) {
+    return 'Medium';
+  } else {
+    return 'Low';
+  }
+}
+
+/**
+ * Select appropriate configuration preset based on data volume and environment
+ * 
+ * @param {string} dataVolume - 'High', 'Medium', or 'Low'
+ * @param {string} environment - Environment name
+ * @returns {Object} Performance preset configuration
+ */
+export function selectConfigPreset(dataVolume, environment = 'development') {
+  // Environment takes precedence
+  if (environment === 'production') {
+    return PERFORMANCE_PRESETS.PRODUCTION;
+  } else if (environment === 'test') {
+    return PERFORMANCE_PRESETS.CI_CD;
+  }
+  
+  // For development, adjust based on data volume
+  switch (dataVolume) {
+    case 'High':
+      return {
+        ...PERFORMANCE_PRESETS.PRODUCTION,
+        profile: 'speed',
+        overrides: {
+          ...PERFORMANCE_PRESETS.PRODUCTION.overrides,
+          // Reduce batch sizes slightly for development stability
+          geographic: {
+            cities: { batchSize: 15000, maxConcurrentBatches: 8 }
+          }
+        }
+      };
+    case 'Medium':
+      return PERFORMANCE_PRESETS.DEVELOPMENT;
+    case 'Low':
+    default:
+      return {
+        ...PERFORMANCE_PRESETS.CI_CD,
+        profile: 'stability'
+      };
+  }
+}
+
+/**
+ * Get configuration for a specific seeder
+ * 
+ * @param {string} seederName - Name of the seeder
+ * @param {Object} performanceConfig - Performance configuration instance
+ * @returns {Object} Configuration for the seeder
+ */
+export function getSeederConfig(seederName, performanceConfig) {
+  // Clean seeder name (remove .seed.js suffix if present)
+  const cleanName = seederName.replace(/\.seed\.js$/, '');
+  
+  // Check if seeder is mapped
+  const mapping = SEEDER_MAPPING[cleanName];
+  
+  if (mapping) {
+    // Get config from performance config
+    const config = performanceConfig.getConfig(mapping.category, mapping.configKey);
+    if (config) {
+      return {
+        ...config,
+        category: mapping.category,
+        seederName: cleanName
+      };
+    }
+  }
+  
+  // Return fallback config for unmapped seeders
+  return {
+    ...FALLBACK_CONFIG,
+    category: 'unknown',
+    seederName: cleanName,
+    fallback: true
+  };
+}
+
+/**
+ * Get all seeders by category
+ * 
+ * @param {string} category - Category name
+ * @returns {Array} Array of seeder names in the category
+ */
+export function getSeedersByCategory(category) {
+  return Object.entries(SEEDER_MAPPING)
+    .filter(([_, mapping]) => mapping.category === category)
+    .map(([seederName]) => seederName);
+}
+
+/**
+ * Create a configured seeder manager
+ * 
+ * @param {Object} options - Configuration options
+ * @returns {Object} Seeder manager with configuration methods
+ */
+export function createSeederManager(options = {}) {
+  const {
+    environment = process.env.NODE_ENV || 'development',
+    countryFilter = [],
+    expectedRecords = 0,
+    availableMemory = process.memoryUsage().free
+  } = options;
+  
+  // Determine data volume
+  const dataVolume = determineDataVolume({
+    expectedRecords,
+    availableMemory,
+    countryFilter,
+    seederType: 'mixed'
+  });
+  
+  // Select appropriate preset
+  const preset = selectConfigPreset(dataVolume, environment);
+  
+  // Create performance configuration
+  const performanceConfig = createPerformanceConfig(preset);
+  
+  return {
+    dataVolume,
+    preset,
+    performanceConfig,
+    
+    /**
+     * Get configuration for a specific seeder
+     */
+    getConfig(seederName) {
+      return getSeederConfig(seederName, performanceConfig);
+    },
+    
+    /**
+     * Get all seeders by category
+     */
+    getSeedersByCategory(category) {
+      return getSeedersByCategory(category);
+    },
+    
+    /**
+     * Get category for a seeder
+     */
+    getSeederCategory(seederName) {
+      const cleanName = seederName.replace(/\.seed\.js$/, '');
+      const mapping = SEEDER_MAPPING[cleanName];
+      return mapping ? mapping.category : 'unknown';
+    },
+    
+    /**
+     * Check if seeder is mapped
+     */
+    isSeederMapped(seederName) {
+      const cleanName = seederName.replace(/\.seed\.js$/, '');
+      return SEEDER_MAPPING.hasOwnProperty(cleanName);
+    },
+    
+    /**
+     * Log configuration summary
+     */
+    logConfigSummary() {
+      console.log('\n=== Seeder Configuration Summary ===');
+      console.log(`Environment: ${environment}`);
+      console.log(`Data Volume: ${dataVolume}`);
+      console.log(`Preset: ${preset.profile || 'custom'}`);
+      console.log(`Country Filter: ${countryFilter.length > 0 ? countryFilter.join(', ') : 'None'}`);
+      
+      console.log('\nCategory Mappings:');
+      Object.values(SEEDER_CATEGORIES).forEach(category => {
+        const seeders = getSeedersByCategory(category);
+        console.log(`  ${category.toUpperCase()}: ${seeders.length} seeders`);
+        seeders.forEach(seeder => {
+          const config = this.getConfig(seeder);
+          console.log(`    - ${seeder}: batch=${config.batchSize}, concurrent=${config.maxConcurrentBatches}`);
+        });
+      });
+      
+      console.log('\n====================================\n');
+    }
+  };
+}
+
+/**
+ * Export category constants for easy access
+ */
+export { SEEDER_CATEGORIES as CATEGORIES };