twenty-import-csv 1.0.0

package/src/browser-automation.ts

@@ -0,0 +1,350 @@
+import { chromium, Browser, Page, BrowserContext } from 'playwright';
+import { ParsedRecord } from './parser';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export interface BrowserImportResult {
+  success: number;
+  errors: number;
+  errorLog: string[];
+}
+
+export class TwentyBrowserImporter {
+  private browser: Browser | null = null;
+  private context: BrowserContext | null = null;
+  private page: Page | null = null;
+  private twentyUrl: string;
+
+  constructor(twentyUrl: string) {
+    this.twentyUrl = twentyUrl.replace(/\/$/, '');
+  }
+
+  async initialize(): Promise<void> {
+    console.log('🌐 Инициализация браузера...');
+    
+    this.browser = await chromium.launch({
+      headless: false, // Показываем браузер для отладки
+      slowMo: 100 // Замедляем действия для надежности
+    });
+
+    this.context = await this.browser.newContext({
+      viewport: { width: 1280, height: 720 }
+    });
+
+    this.page = await this.context.newPage();
+    
+    console.log('✅ Браузер инициализирован');
+  }
+
+  async login(apiKey: string): Promise<void> {
+    if (!this.page) throw new Error('Браузер не инициализирован');
+
+    console.log('🔐 Вход в Twenty CRM...');
+    
+    await this.page.goto(`${this.twentyUrl}/auth/sign-in`);
+    await this.page.waitForLoadState('networkidle');
+
+    // Ищем форму входа
+    await this.page.waitForSelector('input[type="email"], input[name="email"]', { timeout: 10000 });
+    
+    // Если есть API ключ, пробуем использовать его
+    if (apiKey) {
+      // Ищем поле для API ключа или специальный вход
+      const apiKeySelector = await this.page.$('input[placeholder*="API"], input[name*="api"], input[type="password"]');
+      if (apiKeySelector) {
+        await apiKeySelector.fill(apiKey);
+        await this.page.click('button[type="submit"]');
+        await this.page.waitForLoadState('networkidle');
+        console.log('✅ Вход выполнен через API ключ');
+        return;
+      }
+    }
+
+    console.log('⚠️ Требуется ручной вход. Пожалуйста, войдите в систему...');
+    await this.page.waitForURL('**/dashboard', { timeout: 60000 });
+    console.log('✅ Вход выполнен');
+  }
+
+  async navigateToPeople(): Promise<void> {
+    if (!this.page) throw new Error('Браузер не инициализирован');
+
+    console.log('👥 Переход к разделу People...');
+    
+    await this.page.goto(`${this.twentyUrl}/people`);
+    await this.page.waitForLoadState('networkidle');
+    
+    // Ждем загрузки таблицы или списка
+    await this.page.waitForSelector('table, [data-testid="people-list"], .people-container', { timeout: 10000 });
+    
+    console.log('✅ Раздел People загружен');
+  }
+
+  async findImportButton(): Promise<boolean> {
+    if (!this.page) throw new Error('Браузер не инициализирован');
+
+    console.log('🔍 Поиск кнопки импорта...');
+    
+    const importSelectors = [
+      'button[data-testid="import-button"]',
+      'button[title*="Import"]',
+      'button:has-text("Import")',
+      'a[data-testid="import-link"]',
+      'a[title*="Import"]',
+      'a:has-text("Import")',
+      '[data-testid="add-button"]', // Кнопка добавления
+      'button:has-text("Add")'
+    ];
+
+    for (const selector of importSelectors) {
+      try {
+        const element = await this.page.$(selector);
+        if (element) {
+          console.log(`✅ Найдена кнопка импорта: ${selector}`);
+          return true;
+        }
+      } catch (error) {
+        // Продолжаем поиск
+      }
+    }
+
+    console.log('❌ Кнопка импорта не найдена');
+    return false;
+  }
+
+  async importCSV(csvFilePath: string): Promise<void> {
+    if (!this.page) throw new Error('Браузер не инициализирован');
+
+    console.log(`📁 Импорт CSV файла: ${csvFilePath}`);
+
+    // Пробуем найти и нажать кнопку импорта
+    const hasImportButton = await this.findImportButton();
+    
+    if (hasImportButton) {
+      // Нажимаем кнопку импорта
+      await this.page.click('button[data-testid="import-button"], button:has-text("Import"), [data-testid="add-button"]');
+      await this.page.waitForTimeout(1000);
+
+      // Ищем поле для загрузки файла
+      const fileInputSelectors = [
+        'input[type="file"]',
+        'input[accept*=".csv"]',
+        '[data-testid="file-input"]'
+      ];
+
+      for (const selector of fileInputSelectors) {
+        try {
+          const fileInput = await this.page.$(selector);
+          if (fileInput) {
+            await fileInput.setInputFiles(csvFilePath);
+            console.log('✅ Файл загружен');
+            break;
+          }
+        } catch (error) {
+          // Продолжаем поиск
+        }
+      }
+
+      // Ищем кнопку подтверждения импорта
+      const submitSelectors = [
+        'button[data-testid="import-submit"]',
+        'button:has-text("Import")',
+        'button:has-text("Upload")',
+        'button[type="submit"]'
+      ];
+
+      for (const selector of submitSelectors) {
+        try {
+          const submitButton = await this.page.$(selector);
+          if (submitButton) {
+            await submitButton.click();
+            console.log('✅ Импорт запущен');
+            break;
+          }
+        } catch (error) {
+          // Продолжаем поиск
+        }
+      }
+
+      // Ждем завершения импорта
+      await this.page.waitForTimeout(5000);
+    } else {
+      console.log('❌ Автоматический импорт невозможен');
+      console.log('💡 Рекомендации:');
+      console.log('   1. Импортируйте данные вручную через UI');
+      console.log('   2. Используйте формат CSV, поддерживаемый Twenty CRM');
+      console.log('   3. Проверьте права доступа для импорта');
+    }
+  }
+
+  async importRecords(records: ParsedRecord[]): Promise<BrowserImportResult> {
+    const result: BrowserImportResult = {
+      success: 0,
+      errors: 0,
+      errorLog: []
+    };
+
+    if (!this.page) throw new Error('Браузер не инициализирован');
+
+    console.log(`👥 Импорт ${records.length} записей...`);
+
+    for (let i = 0; i < records.length; i++) {
+      const record = records[i];
+      
+      try {
+        console.log(`📝 Импорт записи ${i + 1}/${records.length}`);
+        
+        // Переходим к созданию новой записи
+        await this.navigateToPeople();
+        
+        // Находим кнопку добавления
+        const addButton = await this.page.$('button[data-testid="add-button"], button:has-text("Add"), button:has-text("New")');
+        if (addButton) {
+          await addButton.click();
+          await this.page.waitForTimeout(1000);
+        }
+
+        // Заполняем форму
+        await this.fillPersonForm(record);
+        
+        // Сохраняем запись
+        await this.savePersonForm();
+        
+        result.success++;
+        console.log(`✅ Запись ${i + 1} успешно импортирована`);
+        
+      } catch (error) {
+        result.errors++;
+        const errorMsg = `Запись ${i + 1}: ${error}`;
+        result.errorLog.push(errorMsg);
+        console.error(`❌ ${errorMsg}`);
+      }
+    }
+
+    return result;
+  }
+
+  private async fillPersonForm(record: ParsedRecord): Promise<void> {
+    if (!this.page) throw new Error('Браузер не инициализирован');
+
+    // Заполняем имя
+    if (record['name.firstName']) {
+      const firstNameField = await this.page.$('input[name*="firstName"], input[placeholder*="First"], input[data-testid*="first"]');
+      if (firstNameField) {
+        await firstNameField.fill(record['name.firstName']);
+      }
+    }
+
+    if (record['name.lastName']) {
+      const lastNameField = await this.page.$('input[name*="lastName"], input[placeholder*="Last"], input[data-testid*="last"]');
+      if (lastNameField) {
+        await lastNameField.fill(record['name.lastName']);
+      }
+    }
+
+    // Заполняем email
+    if (record.email) {
+      const emailField = await this.page.$('input[name*="email"], input[type="email"], input[placeholder*="Email"]');
+      if (emailField) {
+        await emailField.fill(record.email);
+      }
+    }
+
+    // Заполняем телефон
+    if (record.phone) {
+      const phoneField = await this.page.$('input[name*="phone"], input[type="tel"], input[placeholder*="Phone"]');
+      if (phoneField) {
+        await phoneField.fill(record.phone);
+      }
+    }
+
+    // Заполняем должность
+    if (record.jobTitle) {
+      const jobTitleField = await this.page.$('input[name*="jobTitle"], input[placeholder*="Job"], input[placeholder*="Position"]');
+      if (jobTitleField) {
+        await jobTitleField.fill(record.jobTitle);
+      }
+    }
+  }
+
+  private async savePersonForm(): Promise<void> {
+    if (!this.page) throw new Error('Браузер не инициализирован');
+
+    // Ищем кнопку сохранения
+    const saveSelectors = [
+      'button[data-testid="save-button"]',
+      'button:has-text("Save")',
+      'button:has-text("Create")',
+      'button[type="submit"]'
+    ];
+
+    for (const selector of saveSelectors) {
+      try {
+        const saveButton = await this.page.$(selector);
+        if (saveButton) {
+          await saveButton.click();
+          await this.page.waitForTimeout(2000);
+          break;
+        }
+      } catch (error) {
+        // Продолжаем поиск
+      }
+    }
+  }
+
+  async cleanup(): Promise<void> {
+    console.log('🧹 Очистка ресурсов...');
+    
+    if (this.context) {
+      await this.context.close();
+    }
+    
+    if (this.browser) {
+      await this.browser.close();
+    }
+    
+    console.log('✅ Ресурсы очищены');
+  }
+}
+
+export async function importViaBrowser(
+  records: ParsedRecord[],
+  objectType: string,
+  twentyUrl: string,
+  apiKey: string,
+  csvFilePath?: string
+): Promise<BrowserImportResult> {
+  const importer = new TwentyBrowserImporter(twentyUrl);
+  
+  try {
+    await importer.initialize();
+    await importer.login(apiKey);
+    
+    if (csvFilePath) {
+      // Пробуем импорт через CSV файл
+      await importer.navigateToPeople();
+      await importer.importCSV(csvFilePath);
+      
+      return {
+        success: records.length,
+        errors: 0,
+        errorLog: []
+      };
+    } else {
+      // Импорт через заполнение форм
+      if (objectType === 'people') {
+        return await importer.importRecords(records);
+      } else {
+        throw new Error(`Тип объекта ${objectType} пока не поддерживается`);
+      }
+    }
+    
+  } catch (error) {
+    return {
+      success: 0,
+      errors: records.length,
+      errorLog: [`Общая ошибка: ${error}`]
+    };
+  } finally {
+    await importer.cleanup();
+  }
+}

package/src/cli-browser.ts

@@ -0,0 +1,134 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander';
+import { parseCSV } from './parser';
+import { importViaBrowser } from './browser-automation';
+import { showProgress, generateReport } from './reporter';
+import * as fs from 'fs';
+import * as path from 'path';
+
+interface Options {
+  file: string;
+  object: string;
+  twentyUrl: string;
+  twentyKey: string;
+  dryRun: boolean;
+  browser: boolean;
+  batch?: number;
+}
+
+const program = new Command();
+
+program
+  .name('twenty-import-csv')
+  .description('Universal CSV import tool for Twenty CRM (Browser Automation)')
+  .version('1.0.0');
+
+program
+  .requiredOption('-f, --file <path>', 'CSV file to import')
+  .requiredOption('-o, --object <type>', 'Twenty object type (people, companies, opportunities, etc.)')
+  .requiredOption('-u, --twenty-url <url>', 'Twenty CRM URL')
+  .requiredOption('-k, --twenty-key <key>', 'Twenty CRM API key')
+  .option('-d, --dry-run', 'Preview import without writing data', false)
+  .option('-b, --browser', 'Use browser automation (recommended)', true)
+  .option('--batch <number>', 'Batch size for API calls', '60')
+  .parse();
+
+const options = program.opts() as Options;
+
+async function main() {
+  try {
+    console.log('🚀 Twenty CSV Import Tool (Browser Automation)');
+    console.log(`📁 File: ${options.file}`);
+    console.log(`🎯 Object: ${options.object}`);
+    console.log(`🔗 Twenty URL: ${options.twentyUrl}`);
+    console.log(`🌐 Browser Mode: ${options.browser ? 'YES' : 'NO'}`);
+    console.log(`🧪 Dry Run: ${options.dryRun ? 'YES' : 'NO'}`);
+    
+    // Check if file exists
+    if (!fs.existsSync(options.file)) {
+      console.error(`❌ Error: File ${options.file} not found`);
+      process.exit(1);
+    }
+
+    // Parse CSV
+    console.log('\n📖 Parsing CSV file...');
+    const data = await parseCSV(options.file);
+    console.log(`✅ Parsed ${data.length} records`);
+
+    // Simple field mapping for people
+    const mappedData = data.map((record: any) => {
+      const mapped: any = {};
+      
+      // Auto-map common fields for people
+      if (options.object === 'people') {
+        mapped['name.firstName'] = record.first_name || record.firstName || '';
+        mapped['name.lastName'] = record.last_name || record.lastName || '';
+        mapped['email'] = record.email || '';
+        mapped['phone'] = record.phone || '';
+        mapped['jobTitle'] = record.job_title || record.jobTitle || '';
+        mapped['city'] = record.city || '';
+        mapped['country'] = record.country || '';
+      } else {
+        // For other objects, just pass through
+        Object.assign(mapped, record);
+      }
+      
+      return mapped;
+    });
+
+    // Show preview for dry run
+    if (options.dryRun) {
+      console.log('\n👀 DRY RUN - Preview (first 5 records):');
+      console.log(JSON.stringify(mappedData.slice(0, 5), null, 2));
+      console.log(`\n📊 Summary: ${mappedData.length} records ready for import`);
+      console.log('\n🌐 Browser automation will be used for import.');
+      return;
+    }
+
+    if (!options.browser) {
+      console.log('\n❌ Browser automation is required for Twenty CRM');
+      console.log('💡 Twenty CRM does not provide a proper REST API');
+      console.log('💡 Use --browser flag to enable browser automation');
+      process.exit(1);
+    }
+
+    // Import data using browser automation
+    console.log('\n🌐 Starting browser automation...');
+    const progressBar = showProgress(mappedData.length);
+    
+    const result = await importViaBrowser(
+      mappedData,
+      options.object,
+      options.twentyUrl,
+      options.twentyKey,
+      options.file // Pass CSV file for direct import
+    );
+
+    progressBar.stop();
+    
+    // Generate report
+    await generateReport(result, options.object);
+
+    console.log('\n✅ Import completed!');
+    console.log(`📊 Success: ${result.success}`);
+    console.log(`❌ Errors: ${result.errors}`);
+    
+    if (result.errors > 0) {
+      console.log(`📄 Error log: import-errors-${Date.now()}.log`);
+      console.log('\n🔍 Error details:');
+      result.errorLog.forEach(error => {
+        console.log(`  ❌ ${error}`);
+      });
+    }
+
+    console.log('\n💡 Browser automation completed!');
+    console.log('💡 Please check Twenty CRM to verify the imported data.');
+
+  } catch (error: any) {
+    console.error('❌ Import failed:', error.message);
+    process.exit(1);
+  }
+}
+
+main();

package/src/cli-simple.ts

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as Papa from 'papaparse';
+import axios from 'axios';
+
+interface Options {
+  file: string;
+  object: string;
+  twentyUrl: string;
+  twentyKey: string;
+  dryRun: boolean;
+  batch?: number;
+}
+
+const program = new Command();
+
+program
+  .name('twenty-import-csv')
+  .description('Universal CSV import tool for Twenty CRM')
+  .version('1.0.0');
+
+program
+  .requiredOption('-f, --file <path>', 'CSV file to import')
+  .requiredOption('-o, --object <type>', 'Twenty object type (people, companies, opportunities, etc.)')
+  .requiredOption('-u, --twenty-url <url>', 'Twenty CRM URL')
+  .requiredOption('-k, --twenty-key <key>', 'Twenty CRM API key')
+  .option('-d, --dry-run', 'Preview import without writing data', false)
+  .option('-b, --batch <number>', 'Batch size for API calls', '60')
+  .parse();
+
+const options = program.opts() as Options;
+
+async function parseCSV(filePath: string): Promise<any[]> {
+  return new Promise((resolve, reject) => {
+    const fileContent = fs.readFileSync(filePath, 'utf8');
+    
+    Papa.parse(fileContent, {
+      header: true,
+      skipEmptyLines: true,
+      complete: (results) => {
+        if (results.errors.length > 0) {
+          console.warn(`⚠️ CSV parsing warnings: ${results.errors.length}`);
+        }
+        resolve(results.data);
+      },
+      error: (error: any) => {
+        reject(new Error(`CSV parsing failed: ${error.message}`));
+      }
+    });
+  });
+}
+
+async function main() {
+  try {
+    console.log('🚀 Twenty CSV Import Tool');
+    console.log(`📁 File: ${options.file}`);
+    console.log(`🎯 Object: ${options.object}`);
+    console.log(`🔗 Twenty URL: ${options.twentyUrl}`);
+    console.log(`🧪 Dry Run: ${options.dryRun ? 'YES' : 'NO'}`);
+    
+    // Check if file exists
+    if (!fs.existsSync(options.file)) {
+      console.error(`❌ Error: File ${options.file} not found`);
+      process.exit(1);
+    }
+
+    // Parse CSV
+    console.log('\n📖 Parsing CSV file...');
+    const data = await parseCSV(options.file);
+    console.log(`✅ Parsed ${data.length} records`);
+
+    // Simple field mapping for people
+    const mappedData = data.map((record: any) => {
+      const mapped: any = {};
+      
+      // Auto-map common fields for people
+      if (options.object === 'people') {
+        mapped['name.firstName'] = record.first_name || record.firstName || '';
+        mapped['name.lastName'] = record.last_name || record.lastName || '';
+        mapped['email'] = record.email || '';
+        mapped['phone'] = record.phone || '';
+        mapped['jobTitle'] = record.job_title || record.jobTitle || '';
+        mapped['city'] = record.city || '';
+        mapped['country'] = record.country || '';
+      } else {
+        // For other objects, just pass through
+        Object.assign(mapped, record);
+      }
+      
+      return mapped;
+    });
+
+    // Show preview for dry run
+    if (options.dryRun) {
+      console.log('\n👀 DRY RUN - Preview (first 5 records):');
+      console.log(JSON.stringify(mappedData.slice(0, 5), null, 2));
+      console.log(`\n📊 Summary: ${mappedData.length} records ready for import`);
+      return;
+    }
+
+    // Load data to Twenty
+    console.log('\n📤 Loading data to Twenty CRM...');
+    const batchSize = parseInt(String(options.batch || '60'));
+    const client = axios.create({
+      baseURL: `${options.twentyUrl.replace(/\/$/, '')}/rest`,
+      headers: {
+        'Authorization': `Bearer ${options.twentyKey}`,
+        'Content-Type': 'application/json'
+      },
+      timeout: 30000
+    });
+
+    let successCount = 0;
+    let errorCount = 0;
+
+    // Process in batches
+    for (let i = 0; i < mappedData.length; i += batchSize) {
+      const batch = mappedData.slice(i, i + batchSize);
+      const batchNumber = Math.floor(i / batchSize) + 1;
+      const totalBatches = Math.ceil(mappedData.length / batchSize);
+
+      console.log(`📤 Batch ${batchNumber}/${totalBatches}: ${batch.length} records`);
+
+      try {
+        const response = await client.post(`/${options.object}`, {
+          data: batch
+        });
+        
+        if (response.status === 200) {
+          successCount += batch.length;
+          console.log(`✅ Batch ${batchNumber} imported successfully`);
+        } else {
+          throw new Error(`HTTP ${response.status}`);
+        }
+      } catch (error: any) {
+        errorCount += batch.length;
+        console.error(`❌ Batch ${batchNumber} failed:`, error.response?.data || error.message);
+      }
+
+      // Small delay to avoid rate limiting
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+
+    console.log('\n✅ Import completed!');
+    console.log(`📊 Success: ${successCount}`);
+    console.log(`❌ Errors: ${errorCount}`);
+    console.log(`📈 Success rate: ${((successCount / (successCount + errorCount)) * 100).toFixed(2)}%`);
+
+  } catch (error) {
+    console.error('❌ Import failed:', (error as Error).message);
+    process.exit(1);
+  }
+}
+
+main();