twenty-migrate-espocrm 1.0.0

package/src/load.ts

@@ -0,0 +1,280 @@
+import axios, { AxiosInstance } from 'axios';
+import { TransformedData, TwentyPerson, TwentyCompany, TwentyOpportunity, TwentyNote } from './transform';
+import { SingleBar } from 'cli-progress';
+import * as fs from 'fs';
+
+export interface LoadResult {
+  success: number;
+  errors: number;
+  errorLog: string[];
+  objectResults: {
+    people: { success: number; errors: number };
+    companies: { success: number; errors: number };
+    opportunities: { success: number; errors: number };
+    notes: { success: number; errors: number };
+  };
+}
+
+export interface BatchResult {
+  success: boolean;
+  count: number;
+  error?: string;
+}
+
+export class TwentyAPIClient {
+  private client: AxiosInstance;
+  private baseUrl: string;
+  private apiKey: string;
+  private existingEmails: Set<string> = new Set();
+  private existingCompanies: Set<string> = new Set();
+
+  constructor(baseUrl: string, apiKey: string) {
+    this.baseUrl = baseUrl.replace(/\/$/, '');
+    this.apiKey = apiKey;
+    
+    this.client = axios.create({
+      baseURL: `${this.baseUrl}`,
+      headers: {
+        'Authorization': `Bearer ${this.apiKey}`,
+        'Content-Type': 'application/json'
+      },
+      timeout: 30000
+    });
+  }
+
+  async testConnection(): Promise<boolean> {
+    try {
+      const response = await this.client.get('/api/ping');
+      return response.status === 200;
+    } catch (error: any) {
+      console.error('❌ Connection test failed:', error.response?.data || error.message);
+      return false;
+    }
+  }
+
+  async loadExistingData(): Promise<void> {
+    console.log('🔍 Loading existing data to prevent duplicates...');
+    
+    try {
+      // Load existing emails
+      const peopleResponse = await this.client.get('/api/people');
+      if (peopleResponse.data && Array.isArray(peopleResponse.data)) {
+        peopleResponse.data.forEach((person: any) => {
+          if (person.email) {
+            this.existingEmails.add(person.email.toLowerCase());
+          }
+        });
+      }
+
+      // Load existing companies
+      const companiesResponse = await this.client.get('/api/companies');
+      if (companiesResponse.data && Array.isArray(companiesResponse.data)) {
+        companiesResponse.data.forEach((company: any) => {
+          if (company.name) {
+            this.existingCompanies.add(company.name.toLowerCase());
+          }
+        });
+      }
+
+      console.log(`✅ Loaded ${this.existingEmails.size} existing emails and ${this.existingCompanies.size} existing companies`);
+    } catch (error: any) {
+      console.warn('⚠️ Could not load existing data, proceeding without duplicate checking:', error.message);
+    }
+  }
+
+  async createRecords(objectType: string, records: any[]): Promise<BatchResult> {
+    try {
+      // Filter out duplicates
+      const filteredRecords = await this.filterDuplicates(objectType, records);
+      
+      if (filteredRecords.length === 0) {
+        return { success: true, count: 0, error: 'All records were duplicates' };
+      }
+
+      // Transform records for Twenty CRM API
+      const transformedRecords = records.map(record => this.transformRecordForAPI(objectType, record));
+
+      const response = await this.client.post(`/api/${objectType}`, transformedRecords);
+      
+      return {
+        success: true,
+        count: filteredRecords.length
+      };
+    } catch (error: any) {
+      return {
+        success: false,
+        count: 0,
+        error: error.response?.data?.message || error.message
+      };
+    }
+  }
+
+  private async filterDuplicates(objectType: string, records: any[]): Promise<any[]> {
+    return records.filter(record => {
+      switch (objectType) {
+        case 'people':
+          return record.email && !this.existingEmails.has(record.email.toLowerCase());
+        case 'companies':
+          return record.name && !this.existingCompanies.has(record.name.toLowerCase());
+        default:
+          return true; // No duplicate checking for other objects
+      }
+    });
+  }
+
+  private transformRecordForAPI(objectType: string, record: any): any {
+    switch (objectType) {
+      case 'people':
+        return {
+          name: record.name,
+          email: record.email,
+          phone: record.phone,
+          jobTitle: record.jobTitle,
+          company: record.company,
+          city: record.city,
+          country: record.country,
+          source: record.source,
+          espocrmId: record.espocrmId
+        };
+      case 'companies':
+        return {
+          name: record.name,
+          domainName: record.domainName,
+          industry: record.industry,
+          description: record.description,
+          phone: record.phone,
+          website: record.website,
+          source: record.source,
+          espocrmId: record.espocrmId
+        };
+      case 'opportunities':
+        return {
+          name: record.name,
+          amount: record.amount,
+          closeDate: record.closeDate,
+          pipeline: record.pipeline,
+          stage: record.stage,
+          source: record.source,
+          espocrmId: record.espocrmId,
+          personId: record.personId,
+          companyId: record.companyId
+        };
+      case 'notes':
+        return {
+          body: record.body,
+          source: record.source,
+          espocrmId: record.espocrmId,
+          personId: record.personId,
+          companyId: record.companyId,
+          opportunityId: record.opportunityId
+        };
+      default:
+        return record;
+    }
+  }
+}
+
+export async function loadToTwenty(
+  data: TransformedData,
+  twentyUrl: string,
+  twentyKey: string,
+  batchSize: number,
+  progressBar: SingleBar
+): Promise<LoadResult> {
+  const client = new TwentyAPIClient(twentyUrl, twentyKey);
+  const errorLog: string[] = [];
+  const objectResults = {
+    people: { success: 0, errors: 0 },
+    companies: { success: 0, errors: 0 },
+    opportunities: { success: 0, errors: 0 },
+    notes: { success: 0, errors: 0 }
+  };
+
+  // Test connection first
+  console.log('🔗 Testing Twenty CRM connection...');
+  const isConnected = await client.testConnection();
+  if (!isConnected) {
+    throw new Error('Failed to connect to Twenty CRM. Check URL and API key.');
+  }
+  console.log('✅ Connection successful');
+
+  // Load existing data for duplicate checking
+  await client.loadExistingData();
+
+  console.log(`📤 Loading data to Twenty CRM...`);
+
+  // Process each object type
+  const objectTypes = ['people', 'companies', 'opportunities', 'notes'] as const;
+  
+  for (const objectType of objectTypes) {
+    const records = data[objectType];
+    if (!records || records.length === 0) {
+      console.log(`⏭️ Skipping ${objectType} - no records`);
+      continue;
+    }
+
+    console.log(`\n📤 Processing ${objectType} (${records.length} records)...`);
+
+    // Process in batches
+    for (let i = 0; i < records.length; i += batchSize) {
+      const batch = records.slice(i, i + batchSize);
+      const batchNumber = Math.floor(i / batchSize) + 1;
+      const totalBatches = Math.ceil(records.length / batchSize);
+
+      try {
+        progressBar.update((i / records.length) * 100, {
+          batch: `${batchNumber}/${totalBatches}`,
+          records: `${i + 1}-${Math.min(i + batchSize, records.length)}`,
+          type: objectType
+        });
+
+        const batchResult = await client.createRecords(objectType, batch);
+        
+        if (batchResult.success) {
+          objectResults[objectType].success += batchResult.count;
+          console.log(`✅ Batch ${batchNumber} (${objectType}): ${batchResult.count} records imported`);
+        } else {
+          objectResults[objectType].errors += batch.length;
+          const errorMsg = `Batch ${batchNumber} (${objectType}): ${batchResult.error}`;
+          errorLog.push(errorMsg);
+          console.error(`❌ ${errorMsg}`);
+        }
+      } catch (error: any) {
+        objectResults[objectType].errors += batch.length;
+        const errorMsg = `Batch ${batchNumber} (${objectType}): ${error.message}`;
+        errorLog.push(errorMsg);
+        console.error(`❌ ${errorMsg}`);
+      }
+
+      // Rate limiting - 100 requests per minute = 600ms between requests
+      await new Promise(resolve => setTimeout(resolve, 600));
+    }
+  }
+
+  const totalSuccess = Object.values(objectResults).reduce((sum, result) => sum + result.success, 0);
+  const totalErrors = Object.values(objectResults).reduce((sum, result) => sum + result.errors, 0);
+
+  return {
+    success: totalSuccess,
+    errors: totalErrors,
+    errorLog,
+    objectResults
+  };
+}
+
+export function saveMigrationReport(result: LoadResult, filename: string): void {
+  const report = {
+    timestamp: new Date().toISOString(),
+    summary: {
+      total: result.success + result.errors,
+      success: result.success,
+      errors: result.errors,
+      successRate: result.success + result.errors > 0 ? ((result.success / (result.success + result.errors)) * 100).toFixed(2) + '%' : '0%'
+    },
+    objectResults: result.objectResults,
+    errorLog: result.errorLog
+  };
+
+  fs.writeFileSync(filename, JSON.stringify(report, null, 2), 'utf8');
+  console.log(`📄 Migration report saved to ${filename}`);
+}

package/src/reporter.ts

@@ -0,0 +1,196 @@
+import { SingleBar, Presets } from 'cli-progress';
+import { LoadResult } from './load';
+import * as fs from 'fs';
+
+export function showProgress(total: number): SingleBar {
+  const progressBar = new SingleBar({
+    format: '📤 Migration |{bar}| {percentage}% | {value}/{total} | {type} | Batch: {batch}',
+    barCompleteChar: '\u2588',
+    barIncompleteChar: '\u2591',
+    hideCursor: true
+  }, Presets.shades_classic);
+
+  progressBar.start(total, 0, {
+    batch: '0/0',
+    records: '0-0',
+    type: 'initializing'
+  });
+
+  return progressBar;
+}
+
+export async function generateMigrationReport(result: LoadResult, objects: string): Promise<void> {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const reportFilename = `migration-report-${timestamp}.json`;
+  const errorLogFilename = `migration-errors-${timestamp}.log`;
+
+  // Generate detailed report
+  const report = {
+    timestamp: new Date().toISOString(),
+    migration: {
+      tool: 'twenty-migrate-espocrm',
+      version: '1.0.0',
+      objects: objects.split(',').map(obj => obj.trim()),
+      dryRun: false
+    },
+    summary: {
+      total: result.success + result.errors,
+      success: result.success,
+      errors: result.errors,
+      successRate: result.success + result.errors > 0 ? ((result.success / (result.success + result.errors)) * 100).toFixed(2) + '%' : '0%'
+    },
+    objectResults: {
+      people: {
+        attempted: result.objectResults.people.success + result.objectResults.people.errors,
+        success: result.objectResults.people.success,
+        errors: result.objectResults.people.errors,
+        successRate: result.objectResults.people.success + result.objectResults.people.errors > 0 ? 
+          ((result.objectResults.people.success / (result.objectResults.people.success + result.objectResults.people.errors)) * 100).toFixed(2) + '%' : '0%'
+      },
+      companies: {
+        attempted: result.objectResults.companies.success + result.objectResults.companies.errors,
+        success: result.objectResults.companies.success,
+        errors: result.objectResults.companies.errors,
+        successRate: result.objectResults.companies.success + result.objectResults.companies.errors > 0 ? 
+          ((result.objectResults.companies.success / (result.objectResults.companies.success + result.objectResults.companies.errors)) * 100).toFixed(2) + '%' : '0%'
+      },
+      opportunities: {
+        attempted: result.objectResults.opportunities.success + result.objectResults.opportunities.errors,
+        success: result.objectResults.opportunities.success,
+        errors: result.objectResults.opportunities.errors,
+        successRate: result.objectResults.opportunities.success + result.objectResults.opportunities.errors > 0 ? 
+          ((result.objectResults.opportunities.success / (result.objectResults.opportunities.success + result.objectResults.opportunities.errors)) * 100).toFixed(2) + '%' : '0%'
+      },
+      notes: {
+        attempted: result.objectResults.notes.success + result.objectResults.notes.errors,
+        success: result.objectResults.notes.success,
+        errors: result.objectResults.notes.errors,
+        successRate: result.objectResults.notes.success + result.objectResults.notes.errors > 0 ? 
+          ((result.objectResults.notes.success / (result.objectResults.notes.success + result.objectResults.notes.errors)) * 100).toFixed(2) + '%' : '0%'
+      }
+    },
+    errors: result.errorLog,
+    recommendations: generateRecommendations(result)
+  };
+
+  // Save JSON report
+  fs.writeFileSync(reportFilename, JSON.stringify(report, null, 2), 'utf8');
+  console.log(`📄 Migration report saved to ${reportFilename}`);
+
+  // Save error log if there are errors
+  if (result.errorLog.length > 0) {
+    fs.writeFileSync(errorLogFilename, result.errorLog.join('\n'), 'utf8');
+    console.log(`📄 Error log saved to ${errorLogFilename}`);
+  }
+
+  // Display summary
+  console.log('\n📊 Migration Summary:');
+  console.log(`📈 Total Records: ${report.summary.total}`);
+  console.log(`✅ Successful: ${report.summary.success}`);
+  console.log(`❌ Errors: ${report.summary.errors}`);
+  console.log(`📊 Success Rate: ${report.summary.successRate}`);
+
+  console.log('\n📋 Object Breakdown:');
+  Object.entries(report.objectResults).forEach(([objectType, results]) => {
+    if (results.attempted > 0) {
+      console.log(`👤 ${objectType}: ${results.success}/${results.attempted} (${results.successRate})`);
+    }
+  });
+
+  if (result.errorLog.length > 0) {
+    console.log(`\n❌ Errors encountered: ${result.errorLog.length}`);
+    console.log(`📄 See ${errorLogFilename} for details`);
+  }
+
+  // Display recommendations
+  if (report.recommendations.length > 0) {
+    console.log('\n💡 Recommendations:');
+    report.recommendations.forEach((rec, index) => {
+      console.log(`${index + 1}. ${rec}`);
+    });
+  }
+}
+
+function generateRecommendations(result: LoadResult): string[] {
+  const recommendations: string[] = [];
+
+  // Overall success rate recommendations
+  const totalSuccessRate = result.success + result.errors > 0 ? 
+    (result.success / (result.success + result.errors)) * 100 : 0;
+
+  if (totalSuccessRate < 90) {
+    recommendations.push('Consider reviewing data quality and formatting before migration');
+  }
+
+  if (totalSuccessRate < 75) {
+    recommendations.push('Run migration in dry-run mode first to identify potential issues');
+  }
+
+  // Object-specific recommendations
+  Object.entries(result.objectResults).forEach(([objectType, results]) => {
+    const successRate = results.success + results.errors > 0 ? 
+      (results.success / (results.success + results.errors)) * 100 : 0;
+
+    if (successRate < 80) {
+      recommendations.push(`Review ${objectType} data mapping - ${results.errors} records failed`);
+    }
+
+    if (results.errors > 0 && successRate < 50) {
+      recommendations.push(`Consider manual review of ${objectType} data before retrying`);
+    }
+  });
+
+  // Error pattern recommendations
+  const errorPatterns = analyzeErrorPatterns(result.errorLog);
+  if (errorPatterns.duplicateErrors > 0) {
+    recommendations.push('Consider running with duplicate detection disabled if duplicates are expected');
+  }
+
+  if (errorPatterns.rateLimitErrors > 0) {
+    recommendations.push('Increase rate limit delay in configuration');
+  }
+
+  if (errorPatterns.validationErrors > 0) {
+    recommendations.push('Review data validation rules and field requirements');
+  }
+
+  return recommendations;
+}
+
+function analyzeErrorPatterns(errorLog: string[]): {
+  duplicateErrors: number;
+  rateLimitErrors: number;
+  validationErrors: number;
+  otherErrors: number;
+} {
+  const patterns = {
+    duplicateErrors: 0,
+    rateLimitErrors: 0,
+    validationErrors: 0,
+    otherErrors: 0
+  };
+
+  errorLog.forEach(error => {
+    const errorLower = error.toLowerCase();
+    
+    if (errorLower.includes('duplicate') || errorLower.includes('already exists')) {
+      patterns.duplicateErrors++;
+    } else if (errorLower.includes('rate limit') || errorLower.includes('too many requests')) {
+      patterns.rateLimitErrors++;
+    } else if (errorLower.includes('validation') || errorLower.includes('required') || errorLower.includes('invalid')) {
+      patterns.validationErrors++;
+    } else {
+      patterns.otherErrors++;
+    }
+  });
+
+  return patterns;
+}
+
+export function saveErrorLog(errorLog: string[], filename: string): void {
+  if (errorLog.length === 0) return;
+  
+  const logContent = errorLog.join('\n');
+  fs.writeFileSync(filename, logContent, 'utf8');
+  console.log(`📄 Error log saved to ${filename}`);
+}