procbay-schema 1.0.0

package/prisma/seeders/utils/bulk-seeder.js

@@ -0,0 +1,300 @@
+import colors from "colors";
+import { logSeederStart, logSeederComplete, logSeederError } from "./progress-tracker.js";
+
+/**
+ * High-performance bulk seeder for processing millions of records
+ * Uses chunked parallel processing and optimized database operations
+ */
+export class BulkSeeder {
+  constructor(prismaClient, options = {}) {
+    this.prisma = prismaClient;
+    this.options = {
+      batchSize: options.batchSize || 10000,
+      maxConcurrentBatches: options.maxConcurrentBatches || 5,
+      retryAttempts: options.retryAttempts || 3,
+      retryDelay: options.retryDelay || 1000,
+      logProgress: options.logProgress !== false,
+      ...options
+    };
+  }
+
+  /**
+   * Processes data in optimized chunks with parallel execution
+   * @param {Array} data - Array of data to process
+   * @param {Function} processor - Function to process each chunk
+   * @param {string} entityName - Name for logging
+   * @returns {Promise<Object>} Results summary
+   */
+  async processInChunks(data, processor, entityName) {
+    if (!data || data.length === 0) {
+      return { totalProcessed: 0, totalCreated: 0, errors: [] };
+    }
+
+    logSeederStart(entityName);
+    const startTime = Date.now();
+    
+    let totalProcessed = 0;
+    let totalCreated = 0;
+    const errors = [];
+
+    // Create chunks
+    const chunks = this.createChunks(data, this.options.batchSize);
+    
+    if (this.options.logProgress) {
+      console.log(colors.cyan(`Processing ${data.length.toLocaleString()} ${entityName} in ${chunks.length} chunks of ${this.options.batchSize}...`));
+    }
+
+    // Process chunks in parallel batches
+    for (let i = 0; i < chunks.length; i += this.options.maxConcurrentBatches) {
+      const currentBatch = chunks.slice(i, i + this.options.maxConcurrentBatches);
+      
+      const batchPromises = currentBatch.map(async (chunk, index) => {
+        const chunkIndex = i + index;
+        return this.processChunkWithRetry(chunk, processor, chunkIndex, entityName);
+      });
+
+      const batchResults = await Promise.allSettled(batchPromises);
+
+      // Aggregate results
+      batchResults.forEach((result, index) => {
+        if (result.status === 'fulfilled') {
+          totalProcessed += result.value.processed;
+          totalCreated += result.value.created;
+        } else {
+          const chunkIndex = i + index;
+          const error = `Chunk ${chunkIndex} failed: ${result.reason.message}`;
+          errors.push(error);
+          if (this.options.logProgress) {
+            console.error(colors.red(error));
+          }
+        }
+      });
+
+      // Progress update
+      if (this.options.logProgress) {
+        const progress = Math.min(((i + this.options.maxConcurrentBatches) / chunks.length) * 100, 100);
+        const elapsed = (Date.now() - startTime) / 1000;
+        const rate = totalProcessed / elapsed;
+        
+        let rateText;
+        if (rate > 100000) {
+          rateText = `${(rate / 1000).toFixed(0)}k/s`;
+        } else if (rate > 1000) {
+          rateText = `${(rate / 1000).toFixed(1)}k/s`;
+        } else {
+          rateText = `${rate.toFixed(0)}/s`;
+        }
+
+        process.stdout.write(`\r${colors.green('▸')} Progress: ${progress.toFixed(1)}% | Processed: ${totalProcessed.toLocaleString()}/${data.length.toLocaleString()} | Rate: ${rateText}`);
+      }
+    }
+
+    if (this.options.logProgress) {
+      process.stdout.write('\n');
+    }
+
+    const totalTime = (Date.now() - startTime) / 1000;
+    const summary = {
+      totalProcessed,
+      totalCreated,
+      errors,
+      duration: totalTime,
+      rate: totalProcessed / totalTime
+    };
+
+    if (errors.length === 0) {
+      logSeederComplete(`${entityName} (${totalCreated.toLocaleString()} created in ${totalTime.toFixed(2)}s)`);
+    } else {
+      console.log(colors.yellow(`${entityName} completed with ${errors.length} errors. ${totalCreated.toLocaleString()} created in ${totalTime.toFixed(2)}s`));
+    }
+
+    return summary;
+  }
+
+  /**
+   * Process a single chunk with retry logic
+   */
+  async processChunkWithRetry(chunk, processor, chunkIndex, entityName) {
+    let lastError;
+    
+    for (let attempt = 1; attempt <= this.options.retryAttempts; attempt++) {
+      try {
+        const result = await processor(chunk, chunkIndex);
+        return {
+          processed: chunk.length,
+          created: result.created || 0
+        };
+      } catch (error) {
+        lastError = error;
+        if (attempt < this.options.retryAttempts) {
+          const delay = this.options.retryDelay * attempt;
+          if (this.options.logProgress) {
+            console.log(colors.yellow(`Chunk ${chunkIndex} attempt ${attempt} failed, retrying in ${delay}ms...`));
+          }
+          await this.sleep(delay);
+        }
+      }
+    }
+    
+    throw new Error(`Chunk ${chunkIndex} failed after ${this.options.retryAttempts} attempts: ${lastError.message}`);
+  }
+
+  /**
+   * Create chunks from data array
+   */
+  createChunks(data, chunkSize) {
+    const chunks = [];
+    for (let i = 0; i < data.length; i += chunkSize) {
+      chunks.push(data.slice(i, i + chunkSize));
+    }
+    return chunks;
+  }
+
+  /**
+   * Sleep utility
+   */
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Optimized bulk insert with conflict resolution
+   */
+  async bulkInsert(model, data, options = {}) {
+    if (!data || data.length === 0) {
+      return { count: 0 };
+    }
+
+    const insertOptions = {
+      skipDuplicates: true,
+      ...options
+    };
+
+    try {
+      return await this.prisma[model].createMany({
+        data,
+        ...insertOptions
+      });
+    } catch (error) {
+      // Log the error with sample data for debugging
+      console.error(colors.red(`Bulk insert failed for ${model}:`), error.message);
+      if (data.length > 0) {
+        console.error(colors.gray('Sample data:'), JSON.stringify(data[0], null, 2));
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Efficient ID mapping after bulk operations
+   */
+  async buildIdMap(model, keyField, valueField, whereClause = {}) {
+    const records = await this.prisma[model].findMany({
+      where: whereClause,
+      select: {
+        [keyField]: true,
+        [valueField]: true
+      }
+    });
+
+    const idMap = new Map();
+    records.forEach(record => {
+      idMap.set(record[keyField], record[valueField]);
+    });
+
+    return idMap;
+  }
+}
+
+/**
+ * Factory function to create a BulkSeeder instance
+ */
+export function createBulkSeeder(prismaClient, options = {}) {
+  return new BulkSeeder(prismaClient, options);
+}
+
+/**
+ * Optimized geographic data seeder
+ */
+export async function seedGeographicDataBulk(prisma, data, options = {}) {
+  const bulkSeeder = createBulkSeeder(prisma, {
+    batchSize: 15000,
+    maxConcurrentBatches: 8,
+    logProgress: true,
+    ...options
+  });
+
+  const results = {};
+
+  // Seed regions
+  if (data.regions) {
+    const regionProcessor = async (chunk) => {
+      return await bulkSeeder.bulkInsert('region', chunk);
+    };
+    results.regions = await bulkSeeder.processInChunks(data.regions, regionProcessor, 'regions');
+  }
+
+  // Seed subregions
+  if (data.subregions && data.regionIdMap) {
+    const subregionProcessor = async (chunk) => {
+      const subregionsToCreate = chunk
+        .map(sub => ({
+          ...sub,
+          region_id: data.regionIdMap.get(sub.region_id)
+        }))
+        .filter(sub => sub.region_id);
+      
+      return await bulkSeeder.bulkInsert('subregion', subregionsToCreate);
+    };
+    results.subregions = await bulkSeeder.processInChunks(data.subregions, subregionProcessor, 'subregions');
+  }
+
+  // Seed countries
+  if (data.countries && data.regionIdMap && data.subregionIdMap) {
+    const countryProcessor = async (chunk) => {
+      const countriesToCreate = chunk
+        .map(country => ({
+          ...country,
+          region_id: data.regionIdMap.get(country.region_id),
+          subregion_id: data.subregionIdMap.get(country.subregion_id)
+        }))
+        .filter(country => country.region_id);
+      
+      return await bulkSeeder.bulkInsert('country', countriesToCreate);
+    };
+    results.countries = await bulkSeeder.processInChunks(data.countries, countryProcessor, 'countries');
+  }
+
+  // Seed states
+  if (data.states && data.countryIdMap) {
+    const stateProcessor = async (chunk) => {
+      const statesToCreate = chunk
+        .map(state => ({
+          ...state,
+          country_id: data.countryIdMap.get(state.country_id)
+        }))
+        .filter(state => state.country_id);
+      
+      return await bulkSeeder.bulkInsert('state', statesToCreate);
+    };
+    results.states = await bulkSeeder.processInChunks(data.states, stateProcessor, 'states');
+  }
+
+  // Seed cities
+  if (data.cities && data.countryIdMap && data.stateIdMap) {
+    const cityProcessor = async (chunk) => {
+      const citiesToCreate = chunk
+        .map(city => ({
+          ...city,
+          country_id: data.countryIdMap.get(city.country_id),
+          state_id: data.stateIdMap.get(city.state_id)
+        }))
+        .filter(city => city.country_id); // State is optional, country is required
+      
+      return await bulkSeeder.bulkInsert('city', citiesToCreate);
+    };
+    results.cities = await bulkSeeder.processInChunks(data.cities, cityProcessor, 'cities');
+  }
+
+  return results;
+}

package/prisma/seeders/utils/config.js

@@ -0,0 +1,103 @@
+/**
+ * Configuration values for seeders
+ */
+
+// The email of the current admin user
+export const CURRENT_ADMIN_USER_EMAIL = 'admin@gmail.com';
+
+export const TENANT_EMAILS = [
+  {
+    default: "admin@gmail.com",
+  },
+  {
+    harsh: "hsinghi@katalysttech.com",
+  },
+  {
+    azhar: "apathak@katalysttech.com",
+  },
+  {
+    arvind: "pipl.arvind@gmail.com",
+  },
+  {
+    vaibhav: "vkanthak@katalysttech.com",
+  },
+];
+
+// Core seeder configuration
+export const CORE_SEEDER_CONFIG = {
+  // Default batch sizes for core entities (optimized for smaller batches)
+  BATCH_SIZES: {
+    users: 50,
+    roles: 25,
+    permissions: 100,
+    departments: 50,
+    currencies: 75
+  },
+  
+  // Retry configuration for critical entities
+  RETRY_CONFIG: {
+    enabled: true,
+    maxAttempts: {
+      users: 3,
+      roles: 5, // Roles are most critical
+      permissions: 3,
+      departments: 3,
+      currencies: 3
+    },
+    baseDelay: {
+      users: 1000,
+      roles: 500,
+      permissions: 750,
+      departments: 800,
+      currencies: 600
+    },
+    exponentialBackoff: true
+  },
+  
+  // Progress tracking configuration
+  PROGRESS_CONFIG: {
+    enabled: true,
+    logFrequency: 'batch', // 'batch' or 'item'
+    showMemoryUsage: false,
+    showErrorDetails: true
+  },
+  
+  // Performance optimization flags
+  OPTIMIZATION_FLAGS: {
+    enableParallelBatches: true,
+    enableMemoryManagement: true,
+    enableProgressiveRetry: true,
+    individualUpserts: true, // Maintain individual upserts for better error isolation
+    batchErrorResilience: true
+  }
+};
+
+// Environment-specific configurations
+export const ENVIRONMENT_CONFIGS = {
+  development: {
+    profile: 'stability',
+    logLevel: 'verbose',
+    enableRetries: true,
+    batchSizeMultiplier: 0.8 // Reduce batch sizes for dev
+  },
+  test: {
+    profile: 'memory',
+    logLevel: 'minimal',
+    enableRetries: false,
+    batchSizeMultiplier: 0.5
+  },
+  production: {
+    profile: 'speed',
+    logLevel: 'standard',
+    enableRetries: true,
+    batchSizeMultiplier: 1.2
+  }
+};
+
+// Get current environment configuration
+export function getCurrentEnvironmentConfig() {
+  const env = process.env.NODE_ENV || 'development';
+  return ENVIRONMENT_CONFIGS[env] || ENVIRONMENT_CONFIGS.development;
+}
+
+// Add other constants as needed