sf-data-extractor 1.0.0

package/index.js

@@ -0,0 +1,682 @@
+#!/usr/bin/env node
+
+const { Command } = require('commander');
+const fs = require('fs').promises;
+const { execSync } = require('child_process');
+const path = require('path');
+const archiver = require('archiver');
+const {openResource} = require('open-resource');
+
+
+// Package info
+const PACKAGE_VERSION = '1.0.0';
+const PACKAGE_NAME = 'sf-data-extractor';
+
+const program = new Command();
+
+program
+  .name('sf-data-extractor')
+  .description('Extract Salesforce data based on load plan and generate HTML report')
+  .version('1.0.0')
+  .requiredOption('-o, --org <username>', 'Salesforce org username or alias')
+  .requiredOption('-l, --load-plan <file>', 'Path to load-plan.json file')
+  .option('-d, --output-dir <directory>', 'Output directory for results', './sf-extract-output')
+  .option('-z, --zip', 'Create a zip file containing all CSV files')
+  .parse(process.argv);
+
+const options = program.opts();
+
+async function main() {
+  try {
+    console.log('🚀 Starting Salesforce Data Extraction...\n');
+    
+    // Read load plan
+    console.log(`📄 Reading load plan from: ${options.loadPlan}`);
+    const loadPlanContent = await fs.readFile(options.loadPlan, 'utf8');
+    const loadPlan = JSON.parse(loadPlanContent);
+    
+    // Create output directory
+    await fs.mkdir(options.outputDir, { recursive: true });
+    
+    // Extract data for each object
+    const results = [];
+    
+    for (let i = 0; i < loadPlan.length; i++) {
+      const obj = loadPlan[i];
+      console.log(`\n[${i + 1}/${loadPlan.length}] Processing ${obj.object}...`);
+      
+      try {
+        const csvPath = path.join(options.outputDir, `${obj.object}.csv`);
+        const command = `sf data query -q "${obj.query}" -o ${options.org} -r csv > "${csvPath}"`;
+        
+        console.log(`  ⏳ Executing query...`);
+        execSync(command, { stdio: 'inherit' });
+        
+        // Read CSV data
+        const csvData = await fs.readFile(csvPath, 'utf8');
+        const rows = csvData.trim().split('\n');
+        const recordCount = Math.max(0, rows.length - 1);
+        
+        results.push({
+          object: obj.object,
+          query: obj.query,
+          recordCount: recordCount,
+          csvPath: csvPath,
+          csvData: csvData,
+          status: 'success'
+        });
+        
+        console.log(`  ✅ Extracted ${recordCount} records`);
+      } catch (error) {
+        console.error(`  ❌ Error extracting ${obj.object}:`, error.message);
+        results.push({
+          object: obj.object,
+          query: obj.query,
+          recordCount: 0,
+          error: error.message,
+          status: 'error'
+        });
+      }
+    }
+    
+    // Generate HTML report
+    console.log('\n📊 Generating HTML report...');
+    const htmlPath = path.join(options.outputDir, 'report.html');
+    const html = generateHTML(results, options.org);
+    await fs.writeFile(htmlPath, html);
+    
+    // Create ZIP file if requested
+    if (options.zip) {
+      console.log('\n📦 Creating ZIP archive...');
+      const zipPath = path.join(options.outputDir, 'data-export.zip');
+      await createZipArchive(results, options.outputDir, zipPath);
+      console.log(`✅ ZIP file created: ${zipPath}`);
+    }
+    
+    console.log(`\n✨ Extraction complete!`);
+    console.log(`📁 Output directory: ${options.outputDir}`);
+    console.log(`🌐 Open report: ${htmlPath}`);
+    openResource(htmlPath);
+    if (options.zip) {
+      console.log(`📦 ZIP archive: ${path.join(options.outputDir, 'data-export.zip')}`);
+    }
+    
+    // Summary
+    const successCount = results.filter(r => r.status === 'success').length;
+    const totalRecords = results.reduce((sum, r) => sum + r.recordCount, 0);
+    console.log(`📈 Summary: ${successCount}/${results.length} objects extracted, ${totalRecords} total records`);
+    
+  } catch (error) {
+    console.error('❌ Fatal error:', error.message);
+    process.exit(1);
+  }
+}
+
+async function createZipArchive(results, outputDir, zipPath) {
+  return new Promise((resolve, reject) => {
+    const output = require('fs').createWriteStream(zipPath);
+    const archive = archiver('zip', {
+      zlib: { level: 9 } // Maximum compression
+    });
+
+    output.on('close', () => {
+      const sizeMB = (archive.pointer() / 1024 / 1024).toFixed(2);
+      console.log(`  📊 Total size: ${sizeMB} MB`);
+      resolve();
+    });
+
+    archive.on('error', (err) => {
+      reject(err);
+    });
+
+    archive.on('warning', (err) => {
+      if (err.code === 'ENOENT') {
+        console.warn('  ⚠️  Warning:', err.message);
+      } else {
+        reject(err);
+      }
+    });
+
+    archive.pipe(output);
+
+    // Add each CSV file to the archive
+    let addedCount = 0;
+    results.forEach(result => {
+      if (result.status === 'success' && result.csvPath) {
+        const fileName = path.basename(result.csvPath);
+        archive.file(result.csvPath, { name: `csv/${fileName}` });
+        addedCount++;
+      }
+    });
+
+    console.log(`  📄 Adding ${addedCount} CSV files to archive...`);
+    
+    archive.finalize();
+  });
+}
+
+function generateHTML(results, orgUsername) {
+  const dataJson = JSON.stringify(results.map(r => ({
+    object: r.object,
+    query: r.query,
+    recordCount: r.recordCount,
+    status: r.status,
+    error: r.error,
+    data: parseCSV(r.csvData || '')
+  })));
+
+  return `<!DOCTYPE html>
+<html lang="en" class="dark">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Salesforce Data Extract Report</title>
+  <script src="https://cdn.tailwindcss.com"></script>
+   <link rel="icon" type="image/x-icon" href="https://mohan-chinnappan-n5.github.io/dfv/img/mc_favIcon.ico" />
+  <script>
+    tailwind.config = {
+      darkMode: 'class',
+      theme: {
+        extend: {
+          colors: {
+            dark: {
+              bg: '#0f172a',
+              surface: '#1e293b',
+              border: '#334155'
+            }
+          }
+        }
+      }
+    }
+  </script>
+  <style>
+    body { font-family: 'Inter', system-ui, sans-serif; }
+    .scrollbar-thin::-webkit-scrollbar { width: 8px; height: 8px; }
+    .scrollbar-thin::-webkit-scrollbar-track { background: #1e293b; }
+    .scrollbar-thin::-webkit-scrollbar-thumb { background: #475569; border-radius: 4px; }
+    .scrollbar-thin::-webkit-scrollbar-thumb:hover { background: #64748b; }
+  </style>
+</head>
+<body class="bg-dark-bg text-gray-100">
+  <div class="flex h-screen overflow-hidden">
+    <!-- Left Sidebar -->
+    <div class="w-80 bg-dark-surface border-r border-dark-border flex flex-col">
+      <div class="p-6 border-b border-dark-border">
+        <h1 class="text-2xl font-bold text-white mb-2">📊 Data Extract</h1>
+        <p class="text-sm text-gray-400">Org: <span class="text-blue-400">${orgUsername}</span></p>
+        <p class="text-sm text-gray-400">Date: ${new Date().toLocaleString()}</p>
+      </div>
+      
+      <div class="p-4 border-b border-dark-border">
+        <input 
+          type="text" 
+          id="objectSearch" 
+          placeholder="Search objects..." 
+          class="w-full px-3 py-2 bg-dark-bg border border-dark-border rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-blue-500"
+        />
+      </div>
+      
+      <div class="flex-1 overflow-y-auto scrollbar-thin p-4" id="objectList">
+        <!-- Objects will be populated here -->
+      </div>
+      
+      <div class="p-4 border-t border-dark-border text-xs text-gray-400">
+        <div class="flex justify-between mb-1">
+          <span>Total Objects:</span>
+          <span id="totalObjects" class="text-white font-semibold">0</span>
+        </div>
+        <div class="flex justify-between">
+          <span>Total Records:</span>
+          <span id="totalRecords" class="text-white font-semibold">0</span>
+        </div>
+      </div>
+    </div>
+    
+    <!-- Right Content Area -->
+    <div class="flex-1 flex flex-col overflow-hidden">
+      <div class="p-6 border-b border-dark-border bg-dark-surface">
+        <div id="objectHeader">
+          <h2 class="text-xl font-semibold text-gray-400">Select an object to view details</h2>
+        </div>
+      </div>
+      
+      <div class="flex-1 overflow-auto scrollbar-thin p-6" id="contentArea">
+        <div class="text-center text-gray-500 mt-20">
+          <svg class="w-16 h-16 mx-auto mb-4 opacity-50" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"></path>
+          </svg>
+          <p>Select an object from the left panel to view its data</p>
+        </div>
+      </div>
+    </div>
+  </div>
+
+  <script>
+    const data = ${dataJson};
+    let currentObject = null;
+    let currentSort = { column: null, direction: 'asc' };
+    let currentPage = 1;
+    let rowsPerPage = 50;
+    let filteredData = [];
+
+    function init() {
+      renderObjectList(data);
+      updateSummary();
+      setupSearch();
+    }
+
+    function renderObjectList(objects) {
+      const list = document.getElementById('objectList');
+      list.innerHTML = objects.map((obj, idx) => {
+        const statusColor = obj.status === 'success' ? 'bg-green-500' : 'bg-red-500';
+        return \`
+          <div 
+            class="p-3 mb-2 bg-dark-bg hover:bg-slate-700 rounded-lg cursor-pointer transition-colors border border-dark-border hover:border-blue-500"
+            onclick="selectObject(\${idx})"
+            id="obj-\${idx}"
+          >
+            <div class="flex items-start justify-between mb-1">
+              <span class="font-medium text-sm text-white">\${obj.object}</span>
+              <span class="w-2 h-2 rounded-full \${statusColor} mt-1"></span>
+            </div>
+            <div class="text-xs text-gray-400">
+              \${obj.status === 'success' ? \`\${obj.recordCount} records\` : 'Error'}
+            </div>
+          </div>
+        \`;
+      }).join('');
+    }
+
+    function selectObject(idx) {
+      currentObject = data[idx];
+      
+      // Update active state
+      document.querySelectorAll('[id^="obj-"]').forEach(el => {
+        el.classList.remove('border-blue-500', 'bg-slate-700');
+      });
+      document.getElementById(\`obj-\${idx}\`).classList.add('border-blue-500', 'bg-slate-700');
+      
+      renderObjectDetails(currentObject);
+    }
+
+    function renderObjectDetails(obj) {
+      const header = document.getElementById('objectHeader');
+      const content = document.getElementById('contentArea');
+      
+      // Reset pagination
+      currentPage = 1;
+      filteredData = obj.data || [];
+      
+      header.innerHTML = \`
+        <div class="flex items-center justify-between">
+          <div>
+            <h2 class="text-2xl font-bold text-white">\${obj.object}</h2>
+            <p class="text-sm text-gray-400 mt-1">\${obj.recordCount} records extracted</p>
+          </div>
+          <button 
+            onclick="exportCSV()"
+            class="px-4 py-2 bg-blue-600 hover:bg-blue-700 rounded-lg text-sm font-medium transition-colors"
+          >
+            📥 Export CSV
+          </button>
+        </div>
+      \`;
+      
+      if (obj.status === 'error') {
+        content.innerHTML = \`
+          <div class="bg-red-900/20 border border-red-500 rounded-lg p-4 mb-6">
+            <h3 class="font-semibold text-red-400 mb-2">❌ Error</h3>
+            <p class="text-sm text-gray-300">\${obj.error}</p>
+          </div>
+          <div class="bg-dark-surface rounded-lg p-4">
+            <h3 class="font-semibold mb-2">Query</h3>
+            <pre class="text-xs bg-dark-bg p-3 rounded overflow-x-auto text-gray-300">\${obj.query}</pre>
+          </div>
+        \`;
+        return;
+      }
+      
+      content.innerHTML = \`
+        <div class="space-y-4">
+          <div class="bg-dark-surface rounded-lg p-4">
+            <h3 class="font-semibold mb-2 text-white">SOQL Query</h3>
+            <pre class="text-xs bg-dark-bg p-3 rounded overflow-x-auto text-gray-300 border border-dark-border">\${obj.query}</pre>
+          </div>
+          
+          <div class="bg-dark-surface rounded-lg p-4">
+            <div class="flex items-center justify-between mb-4 flex-wrap gap-3">
+              <h3 class="font-semibold text-white">Data Table</h3>
+              <div class="flex items-center gap-3 flex-wrap">
+                <select 
+                  id="rowsPerPageSelect" 
+                  onchange="changeRowsPerPage()"
+                  class="px-3 py-2 bg-dark-bg border border-dark-border rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-blue-500"
+                >
+                  <option value="25">25 rows</option>
+                  <option value="50" selected>50 rows</option>
+                  <option value="100">100 rows</option>
+                  <option value="250">250 rows</option>
+                  <option value="500">500 rows</option>
+                  <option value="-1">All rows</option>
+                </select>
+                <input 
+                  type="text" 
+                  id="tableSearch" 
+                  placeholder="Search in table..." 
+                  onkeyup="filterTable()"
+                  class="px-3 py-2 bg-dark-bg border border-dark-border rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 w-64"
+                />
+              </div>
+            </div>
+            <div class="overflow-x-auto">
+              \${renderDataTable(obj.data)}
+            </div>
+            <div id="paginationContainer"></div>
+          </div>
+        </div>
+      \`;
+    }
+
+    function renderDataTable(data) {
+      if (!data || data.length === 0) {
+        return '<p class="text-gray-500 text-center py-8">No data available</p>';
+      }
+      
+      filteredData = data;
+      const headers = Object.keys(data[0]);
+      const totalPages = rowsPerPage === -1 ? 1 : Math.ceil(filteredData.length / rowsPerPage);
+      const startIdx = rowsPerPage === -1 ? 0 : (currentPage - 1) * rowsPerPage;
+      const endIdx = rowsPerPage === -1 ? filteredData.length : startIdx + rowsPerPage;
+      const pageData = filteredData.slice(startIdx, endIdx);
+      
+      const tableHTML = \`
+        <table class="w-full text-sm" id="dataTable">
+          <thead class="bg-dark-bg sticky top-0">
+            <tr>
+              \${headers.map(h => \`
+                <th class="px-4 py-3 text-left font-semibold text-gray-300 cursor-pointer hover:text-white border-b border-dark-border whitespace-nowrap" onclick="sortTable('\${h}')">
+                  <div class="flex items-center gap-2">
+                    \${h}
+                    <span class="sort-indicator text-gray-600">⇅</span>
+                  </div>
+                </th>
+              \`).join('')}
+            </tr>
+          </thead>
+          <tbody id="tableBody">
+            \${pageData.map((row, idx) => \`
+              <tr class="border-b border-dark-border hover:bg-slate-700/50 transition-colors">
+                \${headers.map(h => \`
+                  <td class="px-4 py-3 text-gray-300 whitespace-nowrap">\${escapeHtml(row[h] || '')}</td>
+                \`).join('')}
+              </tr>
+            \`).join('')}
+          </tbody>
+        </table>
+      \`;
+      
+      // Render pagination after table
+      setTimeout(() => renderPagination(totalPages), 0);
+      
+      return tableHTML;
+    }
+
+    function renderPagination(totalPages) {
+      const container = document.getElementById('paginationContainer');
+      if (!container || totalPages <= 1) {
+        if (container) container.innerHTML = '';
+        return;
+      }
+      
+      const startRecord = (currentPage - 1) * rowsPerPage + 1;
+      const endRecord = Math.min(currentPage * rowsPerPage, filteredData.length);
+      
+      let pages = [];
+      
+      // Always show first page
+      pages.push(1);
+      
+      // Show pages around current page
+      for (let i = Math.max(2, currentPage - 1); i <= Math.min(totalPages - 1, currentPage + 1); i++) {
+        pages.push(i);
+      }
+      
+      // Always show last page
+      if (totalPages > 1) pages.push(totalPages);
+      
+      // Remove duplicates and sort
+      pages = [...new Set(pages)].sort((a, b) => a - b);
+      
+      container.innerHTML = \`
+        <div class="flex items-center justify-between mt-4 pt-4 border-t border-dark-border">
+          <div class="text-sm text-gray-400">
+            Showing <span class="text-white font-semibold">\${startRecord}</span> to 
+            <span class="text-white font-semibold">\${endRecord}</span> of 
+            <span class="text-white font-semibold">\${filteredData.length}</span> records
+          </div>
+          
+          <div class="flex items-center gap-2">
+            <button 
+              onclick="goToPage(\${currentPage - 1})"
+              \${currentPage === 1 ? 'disabled' : ''}
+              class="px-3 py-1 bg-dark-bg border border-dark-border rounded hover:bg-slate-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors text-sm"
+            >
+              ← Previous
+            </button>
+            
+            \${pages.map((page, idx) => {
+              const prevPage = idx > 0 ? pages[idx - 1] : 0;
+              const showEllipsis = page - prevPage > 1;
+              
+              return \`
+                \${showEllipsis ? '<span class="px-2 text-gray-600">...</span>' : ''}
+                <button 
+                  onclick="goToPage(\${page})"
+                  class="px-3 py-1 rounded transition-colors text-sm \${page === currentPage ? 'bg-blue-600 text-white font-semibold' : 'bg-dark-bg border border-dark-border hover:bg-slate-700'}"
+                >
+                  \${page}
+                </button>
+              \`;
+            }).join('')}
+            
+            <button 
+              onclick="goToPage(\${currentPage + 1})"
+              \${currentPage === totalPages ? 'disabled' : ''}
+              class="px-3 py-1 bg-dark-bg border border-dark-border rounded hover:bg-slate-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors text-sm"
+            >
+              Next →
+            </button>
+          </div>
+        </div>
+      \`;
+    }
+
+    function goToPage(page) {
+      const totalPages = Math.ceil(filteredData.length / rowsPerPage);
+      if (page < 1 || page > totalPages) return;
+      
+      currentPage = page;
+      renderObjectDetails(currentObject);
+    }
+
+    function changeRowsPerPage() {
+      const select = document.getElementById('rowsPerPageSelect');
+      rowsPerPage = parseInt(select.value);
+      currentPage = 1;
+      renderObjectDetails(currentObject);
+    }
+
+    function sortTable(column) {
+      if (!currentObject || !currentObject.data) return;
+      
+      if (currentSort.column === column) {
+        currentSort.direction = currentSort.direction === 'asc' ? 'desc' : 'asc';
+      } else {
+        currentSort.column = column;
+        currentSort.direction = 'asc';
+      }
+      
+      currentObject.data.sort((a, b) => {
+        const aVal = a[column] || '';
+        const bVal = b[column] || '';
+        
+        const compare = aVal < bVal ? -1 : aVal > bVal ? 1 : 0;
+        return currentSort.direction === 'asc' ? compare : -compare;
+      });
+      
+      renderObjectDetails(currentObject);
+    }
+
+    function filterTable() {
+      const search = document.getElementById('tableSearch').value.toLowerCase();
+      
+      if (!search) {
+        filteredData = currentObject.data || [];
+      } else {
+        filteredData = (currentObject.data || []).filter(row => {
+          return Object.values(row).some(val => 
+            String(val).toLowerCase().includes(search)
+          );
+        });
+      }
+      
+      currentPage = 1;
+      renderObjectDetails(currentObject);
+    }
+
+    function exportCSV() {
+      if (!currentObject || !currentObject.data) return;
+      
+      const headers = Object.keys(currentObject.data[0]);
+      const csv = [
+        headers.join(','),
+        ...currentObject.data.map(row => 
+          headers.map(h => {
+            const val = row[h] || '';
+            return \`"\${String(val).replace(/"/g, '""')}"\`;
+          }).join(',')
+        )
+      ].join('\\n');
+      
+      const blob = new Blob([csv], { type: 'text/csv' });
+      const url = URL.createObjectURL(blob);
+      const a = document.createElement('a');
+      a.href = url;
+      a.download = \`\${currentObject.object}.csv\`;
+      a.click();
+      URL.revokeObjectURL(url);
+    }
+
+    function setupSearch() {
+      document.getElementById('objectSearch').addEventListener('keyup', (e) => {
+        const search = e.target.value.toLowerCase();
+        const filtered = data.filter(obj => 
+          obj.object.toLowerCase().includes(search)
+        );
+        renderObjectList(filtered);
+      });
+    }
+
+    function updateSummary() {
+      document.getElementById('totalObjects').textContent = data.length;
+      const total = data.reduce((sum, obj) => sum + obj.recordCount, 0);
+      document.getElementById('totalRecords').textContent = total.toLocaleString();
+    }
+
+    function parseCSV(csv) {
+      if (!csv) return [];
+      const lines = csv.trim().split('\\n');
+      if (lines.length < 2) return [];
+      
+      const headers = lines[0].split(',').map(h => h.trim().replace(/^"|"$/g, ''));
+      return lines.slice(1).map(line => {
+        const values = parseCSVLine(line);
+        const obj = {};
+        headers.forEach((h, i) => {
+          obj[h] = values[i] || '';
+        });
+        return obj;
+      });
+    }
+
+    function parseCSVLine(line) {
+      const result = [];
+      let current = '';
+      let inQuotes = false;
+      
+      for (let i = 0; i < line.length; i++) {
+        const char = line[i];
+        const next = line[i + 1];
+        
+        if (char === '"' && inQuotes && next === '"') {
+          current += '"';
+          i++;
+        } else if (char === '"') {
+          inQuotes = !inQuotes;
+        } else if (char === ',' && !inQuotes) {
+          result.push(current.trim());
+          current = '';
+        } else {
+          current += char;
+        }
+      }
+      result.push(current.trim());
+      return result;
+    }
+
+    function escapeHtml(text) {
+      const div = document.createElement('div');
+      div.textContent = text;
+      return div.innerHTML;
+    }
+
+    init();
+  </script>
+</body>
+</html>`;
+}
+
+function parseCSV(csvData) {
+  if (!csvData) return [];
+  
+  const lines = csvData.trim().split('\n');
+  if (lines.length < 2) return [];
+  
+  const headers = lines[0].split(',').map(h => h.trim().replace(/^"|"$/g, ''));
+  
+  return lines.slice(1).map(line => {
+    const values = parseCSVLine(line);
+    const obj = {};
+    headers.forEach((h, idx) => {
+      obj[h] = values[idx] ? values[idx].replace(/^"|"$/g, '') : '';
+    });
+    return obj;
+  });
+}
+
+function parseCSVLine(line) {
+  const result = [];
+  let current = '';
+  let inQuotes = false;
+  
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+    const next = line[i + 1];
+    
+    if (char === '"' && inQuotes && next === '"') {
+      current += '"';
+      i++;
+    } else if (char === '"') {
+      inQuotes = !inQuotes;
+    } else if (char === ',' && !inQuotes) {
+      result.push(current);
+      current = '';
+    } else {
+      current += char;
+    }
+  }
+  result.push(current);
+  return result;
+}
+
+main();