procbay-schema 1.0.0

package/prisma/seeders/utils/core-seeder.js

@@ -0,0 +1,330 @@
+import colors from "colors";
+import { createProgressTracker, logSeederStart, logSeederComplete, logSeederError, logEntityError } from "./progress-tracker.js";
+import { createPerformanceConfig } from "./performance-config.js";
+
+/**
+ * Enhanced core data seeder with optimized batch processing and retry logic
+ * Specifically designed for critical entities like users, roles, permissions, departments, and currencies
+ */
+export class CoreSeeder {
+  constructor(prismaClient, options = {}) {
+    this.prisma = prismaClient;
+    this.performanceConfig = createPerformanceConfig(options.performanceConfig || {});
+    this.options = {
+      logProgress: options.logProgress !== false,
+      enableRetries: options.enableRetries !== false,
+      ...options
+    };
+  }
+
+  /**
+   * Process core data with optimized batching and retry logic
+   * @param {Array} data - Array of data to process
+   * @param {Function} processor - Function to process each item (upsert operation)
+   * @param {string} entityType - Type of entity (users, roles, permissions, etc.)
+   * @param {Object} processorOptions - Additional options for the processor function
+   * @returns {Promise<Object>} Results summary
+   */
+  async processCore(data, processor, entityType, processorOptions = {}) {
+    if (!data || data.length === 0) {
+      return { totalProcessed: 0, totalCreated: 0, totalUpdated: 0, errors: [] };
+    }
+
+    logSeederStart(entityType);
+    const startTime = Date.now();
+    
+    // Get optimized configuration for this entity type
+    const config = this.performanceConfig.getConfig('core', entityType);
+    
+    if (this.options.logProgress) {
+      console.log(colors.cyan(`Processing ${data.length.toLocaleString()} ${entityType} with optimized core configuration:`));
+      console.log(colors.gray(`  Batch size: ${config.batchSize}, Max concurrent: ${config.maxConcurrentBatches}, Retries: ${config.retryAttempts || 'disabled'}`));
+    }
+
+    let totalProcessed = 0;
+    let totalCreated = 0;
+    let totalUpdated = 0;
+    const errors = [];
+
+    // Create progress tracker
+    const progressTracker = createProgressTracker(data.length, entityType);
+
+    // Create batches
+    const batches = this.createBatches(data, config.batchSize);
+    
+    // Process batches with controlled concurrency
+    for (let i = 0; i < batches.length; i += config.maxConcurrentBatches) {
+      const currentBatchGroup = batches.slice(i, i + config.maxConcurrentBatches);
+      
+      const batchPromises = currentBatchGroup.map(async (batch, index) => {
+        const batchIndex = i + index;
+        return this.processBatchWithRetry(
+          batch, 
+          processor, 
+          batchIndex, 
+          entityType, 
+          config,
+          processorOptions
+        );
+      });
+
+      const batchResults = await Promise.allSettled(batchPromises);
+
+      // Aggregate results and update progress
+      batchResults.forEach((result, index) => {
+        const batch = currentBatchGroup[index];
+        const batchIndex = i + index;
+        
+        if (result.status === 'fulfilled') {
+          totalProcessed += result.value.processed;
+          totalCreated += result.value.created;
+          totalUpdated += result.value.updated;
+          
+          // Update progress tracker for each item in the batch
+          for (let j = 0; j < batch.length; j++) {
+            progressTracker.update();
+          }
+        } else {
+          const error = `Batch ${batchIndex} failed: ${result.reason.message}`;
+          errors.push(error);
+          
+          if (this.options.logProgress) {
+            progressTracker.log(error, 'error');
+          }
+          
+          // Still update progress for failed items
+          for (let j = 0; j < batch.length; j++) {
+            progressTracker.update();
+          }
+        }
+      });
+
+      // Memory management after each batch group
+      if (global.gc && (i + config.maxConcurrentBatches) % (config.maxConcurrentBatches * 5) === 0) {
+        global.gc();
+      }
+    }
+
+    progressTracker.complete();
+
+    const totalTime = (Date.now() - startTime) / 1000;
+    const summary = {
+      totalProcessed,
+      totalCreated,
+      totalUpdated,
+      errors,
+      duration: totalTime,
+      rate: totalProcessed / totalTime,
+      entityType,
+      config: {
+        batchSize: config.batchSize,
+        maxConcurrentBatches: config.maxConcurrentBatches,
+        retryAttempts: config.retryAttempts || 0
+      }
+    };
+
+    if (errors.length === 0) {
+      logSeederComplete(`${entityType} (${totalCreated} created, ${totalUpdated} updated in ${totalTime.toFixed(2)}s)`);
+    } else {
+      const message = `${entityType} completed with ${errors.length} errors. ${totalCreated} created, ${totalUpdated} updated in ${totalTime.toFixed(2)}s`;
+      console.log(colors.yellow(message));
+      
+      if (config.criticalEntity && errors.length > 0) {
+        console.log(colors.red(`WARNING: Critical entity ${entityType} had ${errors.length} errors!`));
+      }
+    }
+
+    return summary;
+  }
+
+  /**
+   * Process a single batch with retry logic for critical entities
+   */
+  async processBatchWithRetry(batch, processor, batchIndex, entityType, config, processorOptions) {
+    const retryAttempts = this.options.enableRetries && config.criticalEntity ? 
+      (config.retryAttempts || 3) : 1;
+    
+    let lastError;
+    
+    for (let attempt = 1; attempt <= retryAttempts; attempt++) {
+      try {
+        const result = await this.processBatch(batch, processor, batchIndex, entityType, processorOptions);
+        
+        if (attempt > 1 && this.options.logProgress) {
+          console.log(colors.green(`Batch ${batchIndex} succeeded on attempt ${attempt}`));
+        }
+        
+        return result;
+      } catch (error) {
+        lastError = error;
+        
+        if (attempt < retryAttempts) {
+          const delay = config.exponentialBackoff ? 
+            config.retryDelay * Math.pow(2, attempt - 1) : 
+            config.retryDelay;
+          
+          if (this.options.logProgress) {
+            console.log(colors.yellow(
+              `Batch ${batchIndex} attempt ${attempt}/${retryAttempts} failed, retrying in ${delay}ms...`
+            ));
+          }
+          
+          await this.sleep(delay);
+        }
+      }
+    }
+    
+    throw new Error(`Batch ${batchIndex} failed after ${retryAttempts} attempts: ${lastError.message}`);
+  }
+
+  /**
+   * Process a single batch of data
+   */
+  async processBatch(batch, processor, batchIndex, entityType, processorOptions) {
+    let processed = 0;
+    let created = 0;
+    let updated = 0;
+    
+    // Process each item in the batch individually for better error isolation
+    const results = await Promise.allSettled(
+      batch.map(item => processor(item, processorOptions))
+    );
+    
+    results.forEach((result, index) => {
+      processed++;
+      
+      if (result.status === 'fulfilled') {
+        if (result.value?.created) {
+          created++;
+        } else if (result.value?.updated) {
+          updated++;
+        }
+      } else {
+        const item = batch[index];
+        const itemName = this.getItemIdentifier(item, entityType);
+        logEntityError(entityType, itemName, result.reason.message);
+      }
+    });
+    
+    return { processed, created, updated };
+  }
+
+  /**
+   * Get a meaningful identifier for an item for error logging
+   */
+  getItemIdentifier(item, entityType) {
+    switch (entityType) {
+      case 'users':
+        return item.name || item.email_id || item.unique_id;
+      case 'roles':
+        return item.role_name || item.slug;
+      case 'permissions':
+        return item.permissions_name || item.slug;
+      case 'departments':
+        return item.department_name || item.department_slug;
+      case 'currencies':
+        return item.name || item.code;
+      default:
+        return item.name || item.slug || item.id || 'unknown';
+    }
+  }
+
+  /**
+   * Create batches from data array
+   */
+  createBatches(data, batchSize) {
+    const batches = [];
+    for (let i = 0; i < data.length; i += batchSize) {
+      batches.push(data.slice(i, i + batchSize));
+    }
+    return batches;
+  }
+
+  /**
+   * Sleep utility for retry delays
+   */
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Create an optimized upsert processor for standard core entities
+   */
+  static createUpsertProcessor(prisma, model, whereField, updateFields, createFields) {
+    return async (item, options = {}) => {
+      try {
+        const result = await prisma[model].upsert({
+          where: { [whereField]: item[whereField] },
+          update: updateFields.reduce((acc, field) => {
+            if (item[field] !== undefined) {
+              acc[field] = item[field];
+            }
+            return acc;
+          }, {}),
+          create: createFields.reduce((acc, field) => {
+            if (item[field] !== undefined) {
+              acc[field] = item[field];
+            }
+            return acc;
+          }, {}),
+          ...options.prismaOptions
+        });
+
+        // Determine if this was a create or update operation
+        // This is a simple heuristic - could be enhanced with more sophisticated detection
+        return { 
+          created: result.created_at && new Date(result.created_at).getTime() > Date.now() - 1000,
+          updated: !result.created_at || new Date(result.created_at).getTime() <= Date.now() - 1000
+        };
+      } catch (error) {
+        throw error;
+      }
+    };
+  }
+
+  /**
+   * Utility method to create a core seeder instance with standard configuration
+   */
+  static create(prismaClient, options = {}) {
+    return new CoreSeeder(prismaClient, {
+      performanceConfig: {
+        profile: options.profile || 'stability'
+      },
+      logProgress: options.logProgress !== false,
+      enableRetries: options.enableRetries !== false,
+      ...options
+    });
+  }
+}
+
+/**
+ * Factory function for creating core seeders
+ */
+export function createCoreSeeder(prismaClient, options = {}) {
+  return CoreSeeder.create(prismaClient, options);
+}
+
+/**
+ * Convenience function for processing core entity data with standard patterns
+ */
+export async function seedCoreEntity(
+  prismaClient, 
+  data, 
+  entityType, 
+  config = {},
+  options = {}
+) {
+  const seeder = createCoreSeeder(prismaClient, options);
+  
+  // Create processor based on entity type and config
+  const processor = config.customProcessor || 
+    CoreSeeder.createUpsertProcessor(
+      prismaClient,
+      config.model || entityType.slice(0, -1), // Remove 's' from end
+      config.whereField,
+      config.updateFields,
+      config.createFields
+    );
+  
+  return seeder.processCore(data, processor, entityType, config.processorOptions);
+}

package/prisma/seeders/utils/json-loader.js

@@ -0,0 +1,69 @@
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const BASE_JSON_DIR = path.resolve(__dirname, "../../json");
+
+/**
+ * Load JSON data from the appropriate directory
+ *
+ * @param {string} fileName - The name of the JSON file without extension
+ * @param {string} category - The category ('core-data', 'content-data', or 'geographic-data')
+ * @param {string} subdir - Optional subdirectory within the category
+ * @returns {Array|Object} The parsed JSON data
+ */
+export function loadJsonData(fileName, category, subdir = "") {
+  let filePath;
+
+  if (subdir) {
+    filePath = path.resolve(
+      BASE_JSON_DIR,
+      category,
+      subdir,
+      `${fileName}.json`
+    );
+  } else {
+    filePath = path.resolve(BASE_JSON_DIR, category, `${fileName}.json`);
+  }
+
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch (error) {
+    console.error(`Error loading JSON file ${filePath}: ${error.message}`);
+    return [];
+  }
+}
+
+/**
+ * Load core data JSON
+ *
+ * @param {string} fileName - The name of the JSON file without extension
+ * @param {string} subdir - Optional subdirectory within core-data
+ * @returns {Array|Object} The parsed JSON data
+ */
+export function loadCoreData(fileName, subdir = "") {
+  return loadJsonData(fileName, "core-data", subdir);
+}
+
+/**
+ * Load content data JSON
+ *
+ * @param {string} fileName - The name of the JSON file without extension
+ * @param {string} subdir - Optional subdirectory within content-data
+ * @returns {Array|Object} The parsed JSON data
+ */
+export function loadContentData(fileName, subdir = "") {
+  return loadJsonData(fileName, "content-data", subdir);
+}
+
+/**
+ * Load geographic data JSON
+ *
+ * @param {string} fileName - The name of the JSON file without extension
+ * @param {string} subdir - Optional subdirectory within geographic-data
+ * @returns {Array|Object} The parsed JSON data
+ */
+export function loadGeoData(fileName, subdir = "") {
+  return loadJsonData(fileName, "geographic-data", subdir);
+}