procbay-schema 1.0.0

package/prisma/seeders/utils/content-seeder.js

@@ -0,0 +1,472 @@
+import colors from "colors";
+import { connectionManager, withContentPool } from "./connection-manager.js";
+import { MemoryManager } from "./performance-helpers.js";
+import { logSeederStart, logSeederComplete, logSeederError } from "./progress-tracker.js";
+
+/**
+ * Enhanced content seeder for medium-volume content data (500-1000 records)
+ * Features: batch upserts, memory monitoring, unique constraint mapping, connection pooling
+ */
+export class ContentSeeder {
+  constructor(options = {}) {
+    this.options = {
+      batchSize: options.batchSize || 750,
+      maxConcurrentBatches: options.maxConcurrentBatches || 4,
+      retryAttempts: options.retryAttempts || 3,
+      retryDelay: options.retryDelay || 750,
+      exponentialBackoff: options.exponentialBackoff !== false,
+      enableMemoryMonitoring: options.enableMemoryMonitoring !== false,
+      memoryCheckInterval: options.memoryCheckInterval || 100,
+      memoryThreshold: options.memoryThreshold || 0.5,
+      useConnectionPooling: options.useConnectionPooling !== false,
+      enableUniqueConstraintMapping: options.enableUniqueConstraintMapping !== false,
+      batchUpsertMode: options.batchUpsertMode !== false,
+      uniqueFields: options.uniqueFields || ['name'],
+      dbName: options.dbName || 'default',
+      dbUrl: options.dbUrl || process.env.DATABASE_URL,
+      logProgress: options.logProgress !== false,
+      ...options
+    };
+
+    this.uniqueConstraintMaps = new Map();
+    this.memoryStats = {
+      maxHeapUsed: 0,
+      startHeapUsed: 0,
+      currentHeapUsed: 0,
+      gcTriggered: 0
+    };
+  }
+
+  /**
+   * Main seeding method with enhanced batching and monitoring
+   */
+  async seedContent(entityName, data, processor, prismaClient = null) {
+    if (!data || data.length === 0) {
+      console.log(colors.yellow(`No data to seed for ${entityName}`));
+      return { totalProcessed: 0, totalCreated: 0, errors: [] };
+    }
+
+    logSeederStart(entityName);
+    this.initializeMemoryTracking();
+
+    let totalProcessed = 0;
+    let totalCreated = 0;
+    const errors = [];
+    const startTime = Date.now();
+
+    try {
+      // Use connection pooling if enabled
+      if (this.options.useConnectionPooling && !prismaClient) {
+        return await connectionManager.executeWithConnection(
+          this.options.dbName,
+          this.options.dbUrl,
+          async (pooledClient) => {
+            return await this.performSeeding(entityName, data, processor, pooledClient);
+          }
+        );
+      } else {
+        return await this.performSeeding(entityName, data, processor, prismaClient);
+      }
+    } catch (error) {
+      logSeederError(entityName, error.message);
+      throw error;
+    }
+  }
+
+  /**
+   * Core seeding logic with batch processing
+   */
+  async performSeeding(entityName, data, processor, prismaClient) {
+    const chunks = this.createChunks(data, this.options.batchSize);
+    let totalProcessed = 0;
+    let totalCreated = 0;
+    const errors = [];
+
+    if (this.options.logProgress) {
+      console.log(colors.cyan(
+        `Processing ${data.length.toLocaleString()} ${entityName} in ${chunks.length} batches of ${this.options.batchSize}...`
+      ));
+    }
+
+    // Build unique constraint map if enabled
+    if (this.options.enableUniqueConstraintMapping) {
+      await this.buildUniqueConstraintMap(entityName, prismaClient);
+    }
+
+    // Process chunks in parallel batches
+    for (let i = 0; i < chunks.length; i += this.options.maxConcurrentBatches) {
+      const currentBatch = chunks.slice(i, i + this.options.maxConcurrentBatches);
+      
+      const batchPromises = currentBatch.map(async (chunk, index) => {
+        const chunkIndex = i + index;
+        return await this.processChunkWithRetry(
+          chunk, 
+          processor, 
+          chunkIndex, 
+          entityName,
+          prismaClient
+        );
+      });
+
+      const batchResults = await Promise.allSettled(batchPromises);
+
+      // Aggregate results
+      batchResults.forEach((result, index) => {
+        if (result.status === 'fulfilled') {
+          totalProcessed += result.value.processed;
+          totalCreated += result.value.created;
+        } else {
+          const chunkIndex = i + index;
+          const error = `Chunk ${chunkIndex} failed: ${result.reason.message}`;
+          errors.push(error);
+          if (this.options.logProgress) {
+            console.error(colors.red(error));
+          }
+        }
+      });
+
+      // Memory monitoring
+      if (this.options.enableMemoryMonitoring) {
+        this.checkMemoryUsage(totalProcessed, entityName);
+      }
+
+      // Progress reporting
+      if (this.options.logProgress) {
+        this.reportProgress(i + this.options.maxConcurrentBatches, chunks.length, totalProcessed, totalCreated, data.length);
+      }
+    }
+
+    if (this.options.logProgress) {
+      process.stdout.write('\n');
+    }
+
+    const duration = (Date.now() - Date.now()) / 1000;
+    const summary = {
+      totalProcessed,
+      totalCreated,
+      errors,
+      duration,
+      rate: totalProcessed / duration,
+      memoryStats: this.memoryStats
+    };
+
+    if (errors.length === 0) {
+      logSeederComplete(`${entityName} (${totalCreated.toLocaleString()} created)`);
+    } else {
+      console.log(colors.yellow(
+        `${entityName} completed with ${errors.length} errors. ${totalCreated.toLocaleString()} created.`
+      ));
+    }
+
+    // Final memory cleanup
+    this.finalMemoryCleanup();
+
+    return summary;
+  }
+
+  /**
+   * Process chunk with retry logic and exponential backoff
+   */
+  async processChunkWithRetry(chunk, processor, chunkIndex, entityName, prismaClient) {
+    let lastError;
+    
+    for (let attempt = 1; attempt <= this.options.retryAttempts; attempt++) {
+      try {
+        let result;
+
+        if (this.options.batchUpsertMode) {
+          // Use batch upsert if enabled
+          result = await this.performBatchUpsert(chunk, entityName, prismaClient);
+        } else {
+          // Use custom processor
+          result = await processor(chunk, chunkIndex, prismaClient);
+        }
+
+        return {
+          processed: chunk.length,
+          created: result.created || result.count || chunk.length
+        };
+      } catch (error) {
+        lastError = error;
+        if (attempt < this.options.retryAttempts) {
+          const delay = this.options.exponentialBackoff 
+            ? this.options.retryDelay * Math.pow(2, attempt - 1)
+            : this.options.retryDelay;
+          
+          if (this.options.logProgress) {
+            console.log(colors.yellow(
+              `Chunk ${chunkIndex} attempt ${attempt} failed, retrying in ${delay}ms...`
+            ));
+          }
+          await this.sleep(delay);
+        }
+      }
+    }
+    
+    throw new Error(
+      `Chunk ${chunkIndex} failed after ${this.options.retryAttempts} attempts: ${lastError.message}`
+    );
+  }
+
+  /**
+   * Perform batch upsert with unique constraint handling
+   */
+  async performBatchUpsert(chunk, entityName, prismaClient) {
+    const model = entityName.toLowerCase().replace(/s$/, ''); // Simple pluralization handling
+    
+    if (this.options.enableUniqueConstraintMapping) {
+      // Filter out records that would violate unique constraints
+      const validRecords = this.filterUniqueConstraints(chunk, entityName);
+      
+      if (validRecords.length === 0) {
+        return { created: 0 };
+      }
+
+      chunk = validRecords;
+    }
+
+    try {
+      // Use createMany for batch insert with skip duplicates
+      const result = await prismaClient[model].createMany({
+        data: chunk,
+        skipDuplicates: true
+      });
+
+      // Update unique constraint map
+      if (this.options.enableUniqueConstraintMapping) {
+        this.updateUniqueConstraintMap(chunk, entityName);
+      }
+
+      return result;
+    } catch (error) {
+      // Fallback to individual upserts if batch fails
+      console.warn(colors.yellow(`Batch upsert failed for ${entityName}, falling back to individual upserts`));
+      return await this.fallbackToIndividualUpserts(chunk, model, prismaClient);
+    }
+  }
+
+  /**
+   * Fallback to individual upserts if batch operation fails
+   */
+  async fallbackToIndividualUpserts(chunk, model, prismaClient) {
+    let created = 0;
+    
+    for (const record of chunk) {
+      try {
+        await prismaClient[model].upsert({
+          where: this.buildWhereClause(record),
+          update: record,
+          create: record
+        });
+        created++;
+      } catch (error) {
+        // Log individual errors but continue processing
+        console.warn(colors.yellow(`Individual upsert failed: ${error.message}`));
+      }
+    }
+
+    return { created };
+  }
+
+  /**
+   * Build unique constraint map for efficient duplicate detection
+   */
+  async buildUniqueConstraintMap(entityName, prismaClient) {
+    const model = entityName.toLowerCase().replace(/s$/, '');
+    
+    try {
+      const existingRecords = await prismaClient[model].findMany({
+        select: this.buildSelectClause()
+      });
+
+      const constraintMap = new Map();
+      
+      existingRecords.forEach(record => {
+        this.options.uniqueFields.forEach(field => {
+          if (record[field]) {
+            constraintMap.set(`${field}:${record[field]}`, true);
+          }
+        });
+      });
+
+      this.uniqueConstraintMaps.set(entityName, constraintMap);
+      
+      if (this.options.logProgress) {
+        console.log(colors.debug(
+          `Built unique constraint map for ${entityName}: ${constraintMap.size} existing records`
+        ));
+      }
+    } catch (error) {
+      console.warn(colors.yellow(`Failed to build unique constraint map for ${entityName}: ${error.message}`));
+    }
+  }
+
+  /**
+   * Filter records to avoid unique constraint violations
+   */
+  filterUniqueConstraints(chunk, entityName) {
+    const constraintMap = this.uniqueConstraintMaps.get(entityName);
+    if (!constraintMap) return chunk;
+
+    return chunk.filter(record => {
+      for (const field of this.options.uniqueFields) {
+        if (record[field] && constraintMap.has(`${field}:${record[field]}`)) {
+          return false; // Skip this record as it would violate unique constraint
+        }
+      }
+      return true;
+    });
+  }
+
+  /**
+   * Update unique constraint map with new records
+   */
+  updateUniqueConstraintMap(chunk, entityName) {
+    const constraintMap = this.uniqueConstraintMaps.get(entityName);
+    if (!constraintMap) return;
+
+    chunk.forEach(record => {
+      this.options.uniqueFields.forEach(field => {
+        if (record[field]) {
+          constraintMap.set(`${field}:${record[field]}`, true);
+        }
+      });
+    });
+  }
+
+  /**
+   * Build WHERE clause for upsert operations
+   */
+  buildWhereClause(record) {
+    const whereClause = {};
+    
+    // Use the first unique field as the primary identifier
+    const primaryField = this.options.uniqueFields[0];
+    if (record[primaryField]) {
+      whereClause[primaryField] = record[primaryField];
+    }
+
+    return whereClause;
+  }
+
+  /**
+   * Build SELECT clause for unique constraint mapping
+   */
+  buildSelectClause() {
+    const selectClause = { id: true };
+    
+    this.options.uniqueFields.forEach(field => {
+      selectClause[field] = true;
+    });
+
+    return selectClause;
+  }
+
+  /**
+   * Initialize memory tracking
+   */
+  initializeMemoryTracking() {
+    const memUsage = process.memoryUsage();
+    this.memoryStats.startHeapUsed = memUsage.heapUsed;
+    this.memoryStats.currentHeapUsed = memUsage.heapUsed;
+    this.memoryStats.maxHeapUsed = memUsage.heapUsed;
+    
+    if (this.options.logProgress) {
+      MemoryManager.logMemoryUsage(`${this.options.dbName} seeding start`);
+    }
+  }
+
+  /**
+   * Check memory usage and trigger GC if needed
+   */
+  checkMemoryUsage(processedCount, entityName) {
+    const memUsage = process.memoryUsage();
+    this.memoryStats.currentHeapUsed = memUsage.heapUsed;
+    this.memoryStats.maxHeapUsed = Math.max(this.memoryStats.maxHeapUsed, memUsage.heapUsed);
+
+    // Check every N records based on interval
+    if (processedCount % this.options.memoryCheckInterval === 0) {
+      const memoryThresholdBytes = this.options.memoryThreshold * this.memoryStats.startHeapUsed * 10; // Allow 10x growth
+      
+      if (memUsage.heapUsed > memoryThresholdBytes) {
+        if (this.options.logProgress) {
+          console.log(colors.yellow(`Memory threshold exceeded, triggering GC...`));
+        }
+        
+        MemoryManager.forceGarbageCollectionWithDelay();
+        this.memoryStats.gcTriggered++;
+        
+        if (this.options.logProgress) {
+          MemoryManager.logMemoryUsage(`After GC (${entityName})`);
+        }
+      }
+    }
+  }
+
+  /**
+   * Final memory cleanup
+   */
+  finalMemoryCleanup() {
+    MemoryManager.forceGarbageCollectionWithDelay();
+    
+    if (this.options.logProgress) {
+      MemoryManager.logMemoryUsage('Seeding complete');
+      
+      const heapGrowth = ((this.memoryStats.maxHeapUsed - this.memoryStats.startHeapUsed) / 1024 / 1024).toFixed(2);
+      console.log(colors.gray(`Memory growth: ${heapGrowth}MB, GC triggered: ${this.memoryStats.gcTriggered} times`));
+    }
+  }
+
+  /**
+   * Report processing progress
+   */
+  reportProgress(currentIndex, totalChunks, processed, created, total) {
+    const progress = Math.min((currentIndex / totalChunks) * 100, 100);
+    
+    process.stdout.write(
+      `\r${colors.green('▸')} Progress: ${progress.toFixed(1)}% | ` +
+      `Processed: ${processed.toLocaleString()}/${total.toLocaleString()} | ` +
+      `Created: ${created.toLocaleString()}`
+    );
+  }
+
+  /**
+   * Create chunks from data array
+   */
+  createChunks(data, chunkSize) {
+    const chunks = [];
+    for (let i = 0; i < data.length; i += chunkSize) {
+      chunks.push(data.slice(i, i + chunkSize));
+    }
+    return chunks;
+  }
+
+  /**
+   * Sleep utility
+   */
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+/**
+ * Factory function to create ContentSeeder with performance config
+ */
+export async function createContentSeeder(entityName, options = {}) {
+  const { createPerformanceConfig } = await import('./performance-config.js');
+  const performanceConfig = createPerformanceConfig();
+  const config = performanceConfig.getConfig('content', entityName);
+  
+  return new ContentSeeder({
+    ...config,
+    ...options
+  });
+}
+
+/**
+ * Convenience function for content seeding with optimized defaults
+ */
+export async function seedContentData(entityName, data, processor = null, options = {}) {
+  const seeder = await createContentSeeder(entityName, options);
+  return await seeder.seedContent(entityName, data, processor);
+}