procbay-schema 1.0.58 → 1.0.60

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "procbay-schema",
-  "version": "1.0.58",
+  "version": "1.0.60",
   "description": "A set of utilities for managing Prisma database schemas, seeding, and maintenance operations for the Procure-to-Pay system",
   "main": "src/prisma/index.js",
   "type": "module",

package/prisma/migrations/20251030104502_init/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "export_logs" ALTER COLUMN "expires_at" SET DEFAULT NOW() + INTERVAL '7 days';

package/prisma/seeders/industry.seed.js

@@ -23,7 +23,7 @@ export async function seedIndustries(prismaClient) {
   const prisma = prismaClient || new PrismaClient();
 
   try {
-    // Load categories from JSON
+    // Load industries from JSON
     const industryData = loadContentData("industries");
 
     if (!industryData || industryData.length === 0) {
@@ -40,7 +40,7 @@ export async function seedIndustries(prismaClient) {
       enableMemoryMonitoring: true,
       batchUpsertMode: true,
       // Industry-specific unique fields for constraint mapping
-      uniqueFields: ["name", "slug"],
+      uniqueFields: ["name"],
       // Medium batch size as specified (500-1000)
       batchSize: 750,
       // Memory monitoring configuration
@@ -60,29 +60,24 @@ export async function seedIndustries(prismaClient) {
 
       for (const industry of chunk) {
         try {
-          // Ensure parent_id defaults to 0 if not provided
-          const industryRecord = {
-            ...industry,
-          };
-
           await client.industry.upsert({
             where: {
-              name: industryRecord.name,
+              name: industry.name,
             },
             update: {
-              slug: industryRecord.slug,
-              description: industryRecord.description,
-              is_active: industryRecord.is_active,
-              created_by: industryRecord.created_by,
-              updated_by: industryRecord.updated_by,
+              slug: industry.slug,
+              description: industry.description,
+              is_active: industry.is_active,
+              created_by: industry.created_by,
+              updated_by: industry.updated_by,
             },
             create: {
-              name: industryRecord.name,
-              slug: industryRecord.slug,
-              description: industryRecord.description,
-              is_active: industryRecord.is_active,
-              created_by: industryRecord.created_by,
-              updated_by: industryRecord.updated_by,
+              name: industry.name,
+              slug: industry.slug,
+              description: industry.description,
+              is_active: industry.is_active,
+              created_by: industry.created_by,
+              updated_by: industry.updated_by,
             },
           });
           created++;

package/prisma/seeders/utils/content-seeder.js

@@ -1,7 +1,11 @@
 import colors from "colors";
 import { connectionManager, withContentPool } from "./connection-manager.js";
 import { MemoryManager } from "./performance-helpers.js";
-import { logSeederStart, logSeederComplete, logSeederError } from "./progress-tracker.js";
+import {
+  logSeederStart,
+  logSeederComplete,
+  logSeederError,
+} from "./progress-tracker.js";
 
 /**
  * Enhanced content seeder for medium-volume content data (500-1000 records)
@@ -19,13 +23,14 @@ export class ContentSeeder {
       memoryCheckInterval: options.memoryCheckInterval || 100,
       memoryThreshold: options.memoryThreshold || 0.5,
       useConnectionPooling: options.useConnectionPooling !== false,
-      enableUniqueConstraintMapping: options.enableUniqueConstraintMapping !== false,
+      enableUniqueConstraintMapping:
+        options.enableUniqueConstraintMapping !== false,
       batchUpsertMode: options.batchUpsertMode !== false,
-      uniqueFields: options.uniqueFields || ['name'],
-      dbName: options.dbName || 'default',
+      uniqueFields: options.uniqueFields || ["name"],
+      dbName: options.dbName || "default",
       dbUrl: options.dbUrl || process.env.DATABASE_URL,
       logProgress: options.logProgress !== false,
-      ...options
+      ...options,
     };
 
     this.uniqueConstraintMaps = new Map();
@@ -33,7 +38,7 @@ export class ContentSeeder {
       maxHeapUsed: 0,
       startHeapUsed: 0,
       currentHeapUsed: 0,
-      gcTriggered: 0
+      gcTriggered: 0,
     };
   }
 
@@ -61,11 +66,21 @@ export class ContentSeeder {
           this.options.dbName,
           this.options.dbUrl,
           async (pooledClient) => {
-            return await this.performSeeding(entityName, data, processor, pooledClient);
+            return await this.performSeeding(
+              entityName,
+              data,
+              processor,
+              pooledClient
+            );
           }
         );
       } else {
-        return await this.performSeeding(entityName, data, processor, prismaClient);
+        return await this.performSeeding(
+          entityName,
+          data,
+          processor,
+          prismaClient
+        );
       }
     } catch (error) {
       logSeederError(entityName, error.message);
@@ -83,9 +98,11 @@ export class ContentSeeder {
     const errors = [];
 
     if (this.options.logProgress) {
-      console.log(colors.cyan(
-        `Processing ${data.length.toLocaleString()} ${entityName} in ${chunks.length} batches of ${this.options.batchSize}...`
-      ));
+      console.log(
+        colors.cyan(
+          `Processing ${data.length.toLocaleString()} ${entityName} in ${chunks.length} batches of ${this.options.batchSize}...`
+        )
+      );
     }
 
     // Build unique constraint map if enabled
@@ -95,14 +112,17 @@ export class ContentSeeder {
 
     // Process chunks in parallel batches
     for (let i = 0; i < chunks.length; i += this.options.maxConcurrentBatches) {
-      const currentBatch = chunks.slice(i, i + this.options.maxConcurrentBatches);
-      
+      const currentBatch = chunks.slice(
+        i,
+        i + this.options.maxConcurrentBatches
+      );
+
       const batchPromises = currentBatch.map(async (chunk, index) => {
         const chunkIndex = i + index;
         return await this.processChunkWithRetry(
-          chunk, 
-          processor, 
-          chunkIndex, 
+          chunk,
+          processor,
+          chunkIndex,
           entityName,
           prismaClient
         );
@@ -112,7 +132,7 @@ export class ContentSeeder {
 
       // Aggregate results
       batchResults.forEach((result, index) => {
-        if (result.status === 'fulfilled') {
+        if (result.status === "fulfilled") {
           totalProcessed += result.value.processed;
           totalCreated += result.value.created;
         } else {
@@ -132,12 +152,18 @@ export class ContentSeeder {
 
       // Progress reporting
       if (this.options.logProgress) {
-        this.reportProgress(i + this.options.maxConcurrentBatches, chunks.length, totalProcessed, totalCreated, data.length);
+        this.reportProgress(
+          i + this.options.maxConcurrentBatches,
+          chunks.length,
+          totalProcessed,
+          totalCreated,
+          data.length
+        );
       }
     }
 
     if (this.options.logProgress) {
-      process.stdout.write('\n');
+      process.stdout.write("\n");
     }
 
     const duration = (Date.now() - Date.now()) / 1000;
@@ -147,15 +173,19 @@ export class ContentSeeder {
       errors,
       duration,
       rate: totalProcessed / duration,
-      memoryStats: this.memoryStats
+      memoryStats: this.memoryStats,
     };
 
     if (errors.length === 0) {
-      logSeederComplete(`${entityName} (${totalCreated.toLocaleString()} created)`);
+      logSeederComplete(
+        `${entityName} (${totalCreated.toLocaleString()} created)`
+      );
     } else {
-      console.log(colors.yellow(
-        `${entityName} completed with ${errors.length} errors. ${totalCreated.toLocaleString()} created.`
-      ));
+      console.log(
+        colors.yellow(
+          `${entityName} completed with ${errors.length} errors. ${totalCreated.toLocaleString()} created.`
+        )
+      );
     }
 
     // Final memory cleanup
@@ -167,16 +197,26 @@ export class ContentSeeder {
   /**
    * Process chunk with retry logic and exponential backoff
    */
-  async processChunkWithRetry(chunk, processor, chunkIndex, entityName, prismaClient) {
+  async processChunkWithRetry(
+    chunk,
+    processor,
+    chunkIndex,
+    entityName,
+    prismaClient
+  ) {
     let lastError;
-    
+
     for (let attempt = 1; attempt <= this.options.retryAttempts; attempt++) {
       try {
         let result;
 
         if (this.options.batchUpsertMode) {
           // Use batch upsert if enabled
-          result = await this.performBatchUpsert(chunk, entityName, prismaClient);
+          result = await this.performBatchUpsert(
+            chunk,
+            entityName,
+            prismaClient
+          );
         } else {
           // Use custom processor
           result = await processor(chunk, chunkIndex, prismaClient);
@@ -184,25 +224,27 @@ export class ContentSeeder {
 
         return {
           processed: chunk.length,
-          created: result.created || result.count || chunk.length
+          created: result.created || result.count || chunk.length,
         };
       } catch (error) {
         lastError = error;
         if (attempt < this.options.retryAttempts) {
-          const delay = this.options.exponentialBackoff 
+          const delay = this.options.exponentialBackoff
             ? this.options.retryDelay * Math.pow(2, attempt - 1)
             : this.options.retryDelay;
-          
+
           if (this.options.logProgress) {
-            console.log(colors.yellow(
-              `Chunk ${chunkIndex} attempt ${attempt} failed, retrying in ${delay}ms...`
-            ));
+            console.log(
+              colors.yellow(
+                `Chunk ${chunkIndex} attempt ${attempt} failed, retrying in ${delay}ms...`
+              )
+            );
           }
           await this.sleep(delay);
         }
       }
     }
-    
+
     throw new Error(
       `Chunk ${chunkIndex} failed after ${this.options.retryAttempts} attempts: ${lastError.message}`
     );
@@ -212,12 +254,12 @@ export class ContentSeeder {
    * Perform batch upsert with unique constraint handling
    */
   async performBatchUpsert(chunk, entityName, prismaClient) {
-    const model = entityName.toLowerCase().replace(/s$/, ''); // Simple pluralization handling
-    
+    const model = this.getPrismaModelName(entityName);
+
     if (this.options.enableUniqueConstraintMapping) {
       // Filter out records that would violate unique constraints
       const validRecords = this.filterUniqueConstraints(chunk, entityName);
-      
+
       if (validRecords.length === 0) {
         return { created: 0 };
       }
@@ -229,7 +271,7 @@ export class ContentSeeder {
       // Use createMany for batch insert with skip duplicates
       const result = await prismaClient[model].createMany({
         data: chunk,
-        skipDuplicates: true
+        skipDuplicates: true,
       });
 
       // Update unique constraint map
@@ -240,7 +282,11 @@ export class ContentSeeder {
       return result;
     } catch (error) {
       // Fallback to individual upserts if batch fails
-      console.warn(colors.yellow(`Batch upsert failed for ${entityName}, falling back to individual upserts`));
+      console.warn(
+        colors.yellow(
+          `Batch upsert failed for ${entityName}, falling back to individual upserts`
+        )
+      );
       return await this.fallbackToIndividualUpserts(chunk, model, prismaClient);
     }
   }
@@ -250,18 +296,20 @@ export class ContentSeeder {
    */
   async fallbackToIndividualUpserts(chunk, model, prismaClient) {
     let created = 0;
-    
+
     for (const record of chunk) {
       try {
         await prismaClient[model].upsert({
           where: this.buildWhereClause(record),
           update: record,
-          create: record
+          create: record,
         });
         created++;
       } catch (error) {
         // Log individual errors but continue processing
-        console.warn(colors.yellow(`Individual upsert failed: ${error.message}`));
+        console.warn(
+          colors.yellow(`Individual upsert failed: ${error.message}`)
+        );
       }
     }
 
@@ -272,17 +320,17 @@ export class ContentSeeder {
    * Build unique constraint map for efficient duplicate detection
    */
   async buildUniqueConstraintMap(entityName, prismaClient) {
-    const model = entityName.toLowerCase().replace(/s$/, '');
-    
+    const model = this.getPrismaModelName(entityName);
+
     try {
       const existingRecords = await prismaClient[model].findMany({
-        select: this.buildSelectClause()
+        select: this.buildSelectClause(),
       });
 
       const constraintMap = new Map();
-      
-      existingRecords.forEach(record => {
-        this.options.uniqueFields.forEach(field => {
+
+      existingRecords.forEach((record) => {
+        this.options.uniqueFields.forEach((field) => {
           if (record[field]) {
             constraintMap.set(`${field}:${record[field]}`, true);
           }
@@ -290,14 +338,20 @@ export class ContentSeeder {
       });
 
       this.uniqueConstraintMaps.set(entityName, constraintMap);
-      
+
       if (this.options.logProgress) {
-        console.log(colors.debug(
-          `Built unique constraint map for ${entityName}: ${constraintMap.size} existing records`
-        ));
+        console.log(
+          colors.debug(
+            `Built unique constraint map for ${entityName}: ${constraintMap.size} existing records`
+          )
+        );
       }
     } catch (error) {
-      console.warn(colors.yellow(`Failed to build unique constraint map for ${entityName}: ${error.message}`));
+      console.warn(
+        colors.yellow(
+          `Failed to build unique constraint map for ${entityName}: ${error.message}`
+        )
+      );
     }
   }
 
@@ -308,7 +362,7 @@ export class ContentSeeder {
     const constraintMap = this.uniqueConstraintMaps.get(entityName);
     if (!constraintMap) return chunk;
 
-    return chunk.filter(record => {
+    return chunk.filter((record) => {
       for (const field of this.options.uniqueFields) {
         if (record[field] && constraintMap.has(`${field}:${record[field]}`)) {
           return false; // Skip this record as it would violate unique constraint
@@ -325,8 +379,8 @@ export class ContentSeeder {
     const constraintMap = this.uniqueConstraintMaps.get(entityName);
     if (!constraintMap) return;
 
-    chunk.forEach(record => {
-      this.options.uniqueFields.forEach(field => {
+    chunk.forEach((record) => {
+      this.options.uniqueFields.forEach((field) => {
         if (record[field]) {
           constraintMap.set(`${field}:${record[field]}`, true);
         }
@@ -334,12 +388,73 @@ export class ContentSeeder {
     });
   }
 
+  /**
+   * Convert entity name (plural) to Prisma model name (singular)
+   * Handles common English pluralization patterns
+   */
+  getPrismaModelName(entityName) {
+    const lower = entityName.toLowerCase();
+
+    // Handle special cases
+    const specialCases = {
+      industries: "industry",
+      categories: "category",
+      countries: "country",
+      cities: "city",
+      states: "state",
+      currencies: "currency",
+      companies: "company",
+      activities: "activity",
+      properties: "property",
+      facilities: "facility",
+      authorities: "authority",
+      deliveries: "delivery",
+      templates: "template",
+      forms: "form",
+      attributes: "attribute",
+      roles: "role",
+      permissions: "permission",
+      users: "user",
+      departments: "department",
+      vendors: "vendor",
+    };
+
+    // Return special case if exists
+    if (specialCases[lower]) {
+      return specialCases[lower];
+    }
+
+    // Handle -ies ending (e.g., "stories" -> "story")
+    if (lower.endsWith("ies")) {
+      return lower.slice(0, -3) + "y";
+    }
+
+    // Handle -es ending after s, x, z, ch, sh (e.g., "boxes" -> "box")
+    if (
+      lower.endsWith("ses") ||
+      lower.endsWith("xes") ||
+      lower.endsWith("zes") ||
+      lower.endsWith("ches") ||
+      lower.endsWith("shes")
+    ) {
+      return lower.slice(0, -2);
+    }
+
+    // Handle regular -s ending
+    if (lower.endsWith("s")) {
+      return lower.slice(0, -1);
+    }
+
+    // Return as-is if no pluralization detected
+    return lower;
+  }
+
   /**
    * Build WHERE clause for upsert operations
    */
   buildWhereClause(record) {
     const whereClause = {};
-    
+
     // Use the first unique field as the primary identifier
     const primaryField = this.options.uniqueFields[0];
     if (record[primaryField]) {
@@ -354,8 +469,8 @@ export class ContentSeeder {
    */
   buildSelectClause() {
     const selectClause = { id: true };
-    
-    this.options.uniqueFields.forEach(field => {
+
+    this.options.uniqueFields.forEach((field) => {
       selectClause[field] = true;
     });
 
@@ -370,7 +485,7 @@ export class ContentSeeder {
     this.memoryStats.startHeapUsed = memUsage.heapUsed;
     this.memoryStats.currentHeapUsed = memUsage.heapUsed;
     this.memoryStats.maxHeapUsed = memUsage.heapUsed;
-    
+
     if (this.options.logProgress) {
       MemoryManager.logMemoryUsage(`${this.options.dbName} seeding start`);
     }
@@ -382,20 +497,26 @@ export class ContentSeeder {
   checkMemoryUsage(processedCount, entityName) {
     const memUsage = process.memoryUsage();
     this.memoryStats.currentHeapUsed = memUsage.heapUsed;
-    this.memoryStats.maxHeapUsed = Math.max(this.memoryStats.maxHeapUsed, memUsage.heapUsed);
+    this.memoryStats.maxHeapUsed = Math.max(
+      this.memoryStats.maxHeapUsed,
+      memUsage.heapUsed
+    );
 
     // Check every N records based on interval
     if (processedCount % this.options.memoryCheckInterval === 0) {
-      const memoryThresholdBytes = this.options.memoryThreshold * this.memoryStats.startHeapUsed * 10; // Allow 10x growth
-      
+      const memoryThresholdBytes =
+        this.options.memoryThreshold * this.memoryStats.startHeapUsed * 10; // Allow 10x growth
+
       if (memUsage.heapUsed > memoryThresholdBytes) {
         if (this.options.logProgress) {
-          console.log(colors.yellow(`Memory threshold exceeded, triggering GC...`));
+          console.log(
+            colors.yellow(`Memory threshold exceeded, triggering GC...`)
+          );
         }
-        
+
         MemoryManager.forceGarbageCollectionWithDelay();
         this.memoryStats.gcTriggered++;
-        
+
         if (this.options.logProgress) {
           MemoryManager.logMemoryUsage(`After GC (${entityName})`);
         }
@@ -408,12 +529,20 @@ export class ContentSeeder {
    */
   finalMemoryCleanup() {
     MemoryManager.forceGarbageCollectionWithDelay();
-    
+
     if (this.options.logProgress) {
-      MemoryManager.logMemoryUsage('Seeding complete');
-      
-      const heapGrowth = ((this.memoryStats.maxHeapUsed - this.memoryStats.startHeapUsed) / 1024 / 1024).toFixed(2);
-      console.log(colors.gray(`Memory growth: ${heapGrowth}MB, GC triggered: ${this.memoryStats.gcTriggered} times`));
+      MemoryManager.logMemoryUsage("Seeding complete");
+
+      const heapGrowth = (
+        (this.memoryStats.maxHeapUsed - this.memoryStats.startHeapUsed) /
+        1024 /
+        1024
+      ).toFixed(2);
+      console.log(
+        colors.gray(
+          `Memory growth: ${heapGrowth}MB, GC triggered: ${this.memoryStats.gcTriggered} times`
+        )
+      );
     }
   }
 
@@ -422,11 +551,11 @@ export class ContentSeeder {
    */
   reportProgress(currentIndex, totalChunks, processed, created, total) {
     const progress = Math.min((currentIndex / totalChunks) * 100, 100);
-    
+
     process.stdout.write(
-      `\r${colors.green('▸')} Progress: ${progress.toFixed(1)}% | ` +
-      `Processed: ${processed.toLocaleString()}/${total.toLocaleString()} | ` +
-      `Created: ${created.toLocaleString()}`
+      `\r${colors.green("▸")} Progress: ${progress.toFixed(1)}% | ` +
+        `Processed: ${processed.toLocaleString()}/${total.toLocaleString()} | ` +
+        `Created: ${created.toLocaleString()}`
     );
   }
 
@@ -445,7 +574,7 @@ export class ContentSeeder {
    * Sleep utility
    */
   sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
+    return new Promise((resolve) => setTimeout(resolve, ms));
   }
 }
 
@@ -453,20 +582,25 @@ export class ContentSeeder {
  * Factory function to create ContentSeeder with performance config
  */
 export async function createContentSeeder(entityName, options = {}) {
-  const { createPerformanceConfig } = await import('./performance-config.js');
+  const { createPerformanceConfig } = await import("./performance-config.js");
   const performanceConfig = createPerformanceConfig();
-  const config = performanceConfig.getConfig('content', entityName);
-  
+  const config = performanceConfig.getConfig("content", entityName);
+
   return new ContentSeeder({
     ...config,
-    ...options
+    ...options,
   });
 }
 
 /**
  * Convenience function for content seeding with optimized defaults
  */
-export async function seedContentData(entityName, data, processor = null, options = {}) {
+export async function seedContentData(
+  entityName,
+  data,
+  processor = null,
+  options = {}
+) {
   const seeder = await createContentSeeder(entityName, options);
   return await seeder.seedContent(entityName, data, processor);
 }