salesforce-to-sqlite 1.0.0

package/README.md

@@ -0,0 +1,164 @@
+# Salesforce to SQLite CLI Tool
+
+> A Node.js CLI tool that extracts data from Salesforce and loads it into a SQLite database based on a load plan configuration.
+
+## Features
+
+- Extracts data from Salesforce using SF CLI
+- Creates SQLite tables automatically based on query results
+- Handles Salesforce relationship fields (e.g., `Parent.Name` becomes `Parent_Name` in SQLite)
+- Automatic data type inference
+- Progress logging with colored output
+- Verbose mode for debugging
+
+## Prerequisites
+
+- Node.js (v14 or higher)
+- Salesforce CLI (`sf`) installed and authenticated
+- Active Salesforce org connection
+
+## Installation
+
+```bash
+npm install -g salesforce-to-sqlite
+```
+
+## Usage
+
+### Basic Usage
+
+```bash
+sf2sqlite  -o <salesforce-username> -l load-plan.json
+```
+
+### With Custom Database Name
+
+```bash
+sf2sqlite  -o myorg@example.com -l load-plan.json -d my-data.db
+```
+
+### Verbose Mode
+
+```bash
+sf2sqlite  -o myorg@example.com -l load-plan.json -v
+```
+
+
+## Options
+
+- `-o, --org <username>` - Salesforce org username or alias (required)
+- `-l, --load-plan <file>` - Path to load plan JSON file (required)
+- `-d, --database <file>` - SQLite database file path (default: `salesforce.db`)
+- `-v, --verbose` - Enable verbose logging
+- `-h, --help` - Display help information
+- `-V, --version` - Display version number
+
+## Load Plan Format
+
+The load plan is a JSON array of object configurations. Each configuration includes:
+
+```json
+{
+  "object": "ObjectName",
+  "compositeKeys": ["Field1", "Field2"],
+  "query": "SELECT Field1, Field2, Related.Field FROM ObjectName",
+  "fieldMappings": {
+    "Field1": "Field1",
+    "Field2": {
+      "lookup": {
+        "object": "RelatedObject",
+        "key": "KeyField",
+        "field": "Related.Field"
+      }
+    }
+  }
+}
+```
+
+## How It Works
+
+1. **Read Load Plan**: Parses the JSON configuration file
+2. **Query Salesforce**: Executes each SOQL query using `sf data query`
+3. **Field Name Sanitization**: Converts relationship fields (e.g., `Account.Name`) to SQLite-compatible names (`Account_Name`)
+4. **Table Creation**: Creates SQLite tables with inferred data types
+5. **Data Insertion**: Inserts queried data into SQLite tables
+6. **Type Conversion**: Automatically converts Salesforce data types to SQLite equivalents
+
+## Field Name Handling
+
+Salesforce relationship fields like `Parent.Name` or `Account.Owner.Email` are automatically converted to SQLite-compatible field names:
+
+- `Parent.Name` → `Parent_Name`
+- `Account.Owner.Email` → `Account_Owner_Email`
+- `Product2.StockKeepingUnit` → `Product2_StockKeepingUnit`
+
+## Data Type Mapping
+
+| Salesforce Type | SQLite Type | Notes |
+|----------------|-------------|-------|
+| String/Text | TEXT | Default for most fields |
+| Boolean | INTEGER | true=1, false=0 |
+| Number (Integer) | INTEGER | Whole numbers |
+| Number (Decimal) | REAL | Decimal numbers |
+| Date/DateTime | TEXT | ISO format strings |
+| ID | TEXT | Salesforce IDs |
+
+## Examples
+
+### Extract Product Catalog Data
+
+```bash
+sf2sqlite  -o production-org -l load-plan.json -d catalog.db
+```
+
+### Debug Query Issues
+
+```bash
+sf2sqlite  -o sandbox -l load-plan.json -v
+```
+
+
+
+## Output
+
+The tool provides colored console output:
+- 🔵 Info messages
+- ✅ Success messages
+- ⚠️  Warning messages
+- ❌ Error messages
+
+At the end, a summary shows:
+- Number of successfully processed objects
+- Number of failed objects
+- Location of the SQLite database file
+
+## Error Handling
+
+- Invalid load plan JSON: Exits with error message
+- Failed Salesforce queries: Logs warning and continues with next object
+- Table creation errors: Logs error and skips object
+- Record insertion errors: Logs error but continues with other records
+
+## Performance Notes
+
+- Large queries use a 50MB buffer for CSV data
+- SQLite WAL mode is enabled for better performance
+- Batch inserts using transactions for speed
+- Progress is logged for each object
+
+## Troubleshooting
+
+### "Command not found: sf"
+Install Salesforce CLI: https://developer.salesforce.com/tools/salesforcecli
+
+### "No auth found for org"
+Authenticate with: `sf org login web -a <alias>`
+
+### "Query timeout"
+Try splitting large objects into smaller queries or increase the buffer size in the code
+
+### "Failed to parse CSV"
+Check query syntax and ensure fields exist in your org
+
+## License
+MIT (c) Mohan Chinnappan

package/index.js

@@ -0,0 +1,349 @@
+#!/usr/bin/env node
+
+const { Command } = require('commander');
+const fs = require('fs');
+const path = require('path');
+const { execSync } = require('child_process');
+const Database = require('better-sqlite3');
+const { parse } = require('csv-parse/sync');
+const chalk = require('chalk');
+
+const program = new Command();
+
+program
+  .name('sf-to-sqlite')
+  .description('Extract Salesforce data and load into SQLite database')
+  .version('1.0.0')
+  .requiredOption('-o, --org <username>', 'Salesforce org username or alias')
+  .requiredOption('-l, --load-plan <file>', 'Load plan JSON file')
+  .option('-d, --database <file>', 'SQLite database file', 'salesforce.db')
+  .option('-v, --verbose', 'Verbose output')
+  .parse(process.argv);
+
+const options = program.opts();
+
+// Utility functions
+function log(message, type = 'info') {
+  if (type === 'info') {
+    console.log(chalk.blue('ℹ'), message);
+  } else if (type === 'success') {
+    console.log(chalk.green('✓'), message);
+  } else if (type === 'error') {
+    console.log(chalk.red('✗'), message);
+  } else if (type === 'warn') {
+    console.log(chalk.yellow('⚠'), message);
+  }
+}
+
+function logVerbose(message) {
+  if (options.verbose) {
+    console.log(chalk.gray(message));
+  }
+}
+
+// Read and parse load plan
+function readLoadPlan(filePath) {
+  try {
+    const content = fs.readFileSync(filePath, 'utf-8');
+    return JSON.parse(content);
+  } catch (error) {
+    log(`Failed to read load plan: ${error.message}`, 'error');
+    process.exit(1);
+  }
+}
+
+// Sanitize field names for SQLite (replace dots with underscores)
+function sanitizeFieldName(fieldName) {
+  return fieldName.replace(/\./g, '_');
+}
+
+// Extract field names from query
+function extractFieldsFromQuery(query) {
+  const selectMatch = query.match(/SELECT\s+(.+?)\s+FROM/i);
+  if (!selectMatch) {
+    return [];
+  }
+  
+  const fieldsString = selectMatch[1];
+  const fields = fieldsString.split(',').map(f => f.trim());
+  
+  return fields.map(field => {
+    // Handle field with alias or relationship
+    const parts = field.split('.');
+    return {
+      original: field,
+      sanitized: sanitizeFieldName(field)
+    };
+  });
+}
+
+// Get SQLite data type based on Salesforce field type
+function getSQLiteType(fieldName, fieldValue) {
+  // Check common patterns
+  if (fieldName.toLowerCase().includes('date')) {
+    return 'TEXT'; // Store dates as ISO strings
+  }
+  if (fieldName.toLowerCase().includes('is') || 
+      fieldName.toLowerCase().includes('active') ||
+      fieldName.toLowerCase().includes('required') ||
+      fieldName.toLowerCase().includes('hidden')) {
+    return 'INTEGER'; // Boolean as 0/1
+  }
+  if (fieldName.toLowerCase().includes('price') || 
+      fieldName.toLowerCase().includes('quantity') ||
+      fieldName.toLowerCase().includes('amount')) {
+    return 'REAL';
+  }
+  
+  // Infer from value
+  if (fieldValue === null || fieldValue === undefined || fieldValue === '') {
+    return 'TEXT';
+  }
+  
+  if (fieldValue === 'true' || fieldValue === 'false') {
+    return 'INTEGER';
+  }
+  
+  if (!isNaN(fieldValue) && fieldValue !== '') {
+    if (fieldValue.includes('.')) {
+      return 'REAL';
+    }
+    return 'INTEGER';
+  }
+  
+  return 'TEXT';
+}
+
+// Create table in SQLite
+function createTable(db, objectConfig, sampleData) {
+  const tableName = objectConfig.object;
+  
+  logVerbose(`Creating table: ${tableName}`);
+  
+  // Drop table if exists
+  db.exec(`DROP TABLE IF EXISTS ${tableName}`);
+  
+  // Get fields from query
+  const fields = extractFieldsFromQuery(objectConfig.query);
+  
+  if (fields.length === 0) {
+    log(`No fields found in query for ${tableName}`, 'warn');
+    return;
+  }
+  
+  // Build CREATE TABLE statement
+  let createTableSQL = `CREATE TABLE ${tableName} (\n`;
+  createTableSQL += `  Id INTEGER PRIMARY KEY AUTOINCREMENT,\n`;
+  
+  const columnDefs = fields.map(field => {
+    let dataType = 'TEXT';
+    
+    // Try to infer type from sample data
+    if (sampleData && sampleData.length > 0) {
+      const sampleValue = sampleData[0][field.original];
+      dataType = getSQLiteType(field.sanitized, sampleValue);
+    }
+    
+    return `  ${field.sanitized} ${dataType}`;
+  });
+  
+  createTableSQL += columnDefs.join(',\n');
+  createTableSQL += '\n)';
+  
+  logVerbose(`SQL: ${createTableSQL}`);
+  
+  try {
+    db.exec(createTableSQL);
+    log(`Created table: ${tableName}`, 'success');
+  } catch (error) {
+    log(`Failed to create table ${tableName}: ${error.message}`, 'error');
+    throw error;
+  }
+  
+  return fields;
+}
+
+// Query Salesforce using SF CLI
+function querySalesforce(query, orgUsername) {
+  log(`Executing query...`);
+  logVerbose(`Query: ${query}`);
+  
+  try {
+    const command = `sf data query -q "${query}" -r csv -o ${orgUsername}`;
+    logVerbose(`Command: ${command}`);
+    
+    const result = execSync(command, {
+      encoding: 'utf-8',
+      maxBuffer: 50 * 1024 * 1024 // 50MB buffer
+    });
+    
+    return result;
+  } catch (error) {
+    log(`Query failed: ${error.message}`, 'error');
+    if (error.stderr) {
+      logVerbose(`Error details: ${error.stderr}`);
+    }
+    return null;
+  }
+}
+
+// Parse CSV data
+function parseCSV(csvData) {
+  if (!csvData || csvData.trim() === '') {
+    return [];
+  }
+  
+  try {
+    const records = parse(csvData, {
+      columns: true,
+      skip_empty_lines: true,
+      trim: true,
+      relax_column_count: true
+    });
+    
+    return records;
+  } catch (error) {
+    log(`Failed to parse CSV: ${error.message}`, 'error');
+    return [];
+  }
+}
+
+// Insert data into SQLite
+function insertData(db, tableName, fields, records) {
+  if (!records || records.length === 0) {
+    log(`No records to insert for ${tableName}`, 'warn');
+    return 0;
+  }
+  
+  logVerbose(`Inserting ${records.length} records into ${tableName}`);
+  
+  // Build INSERT statement
+  const sanitizedFields = fields.map(f => f.sanitized);
+  const placeholders = sanitizedFields.map(() => '?').join(', ');
+  const insertSQL = `INSERT INTO ${tableName} (${sanitizedFields.join(', ')}) VALUES (${placeholders})`;
+  
+  logVerbose(`Insert SQL: ${insertSQL}`);
+  
+  const insert = db.prepare(insertSQL);
+  
+  let inserted = 0;
+  const insertMany = db.transaction((records) => {
+    for (const record of records) {
+      try {
+        const values = fields.map(field => {
+          let value = record[field.original];
+          
+          // Convert boolean strings to integers
+          if (value === 'true') return 1;
+          if (value === 'false') return 0;
+          
+          // Handle null/empty values
+          if (value === '' || value === null || value === undefined) {
+            return null;
+          }
+          
+          return value;
+        });
+        
+        insert.run(values);
+        inserted++;
+      } catch (error) {
+        logVerbose(`Failed to insert record: ${error.message}`);
+        if (options.verbose) {
+          console.log('Record:', record);
+        }
+      }
+    }
+  });
+  
+  insertMany(records);
+  
+  log(`Inserted ${inserted} records into ${tableName}`, 'success');
+  return inserted;
+}
+
+// Process single object
+function processObject(db, objectConfig, orgUsername) {
+  log(`\n${'='.repeat(60)}`);
+  log(`Processing: ${objectConfig.object}`);
+  log(`${'='.repeat(60)}`);
+  
+  // Query Salesforce
+  const csvData = querySalesforce(objectConfig.query, orgUsername);
+  
+  if (!csvData) {
+    log(`Skipping ${objectConfig.object} due to query failure`, 'warn');
+    return;
+  }
+  
+  // Parse CSV
+  const records = parseCSV(csvData);
+  
+  if (records.length === 0) {
+    log(`No records returned for ${objectConfig.object}`, 'warn');
+  }
+  
+  // Create table with sample data for type inference
+  const fields = createTable(db, objectConfig, records);
+  
+  if (!fields || fields.length === 0) {
+    return;
+  }
+  
+  // Insert data
+  insertData(db, objectConfig.object, fields, records);
+}
+
+// Main function
+function main() {
+  log(`Salesforce to SQLite Extractor`, 'info');
+  log(`Org: ${options.org}`);
+  log(`Load Plan: ${options.loadPlan}`);
+  log(`Database: ${options.database}`);
+  
+  // Read load plan
+  const loadPlan = readLoadPlan(options.loadPlan);
+  log(`Loaded ${loadPlan.length} object configurations`, 'success');
+  
+  // Initialize SQLite database
+  const db = new Database(options.database);
+  db.pragma('journal_mode = WAL');
+  
+  log(`Database initialized: ${options.database}`, 'success');
+  
+  // Process each object
+  let successCount = 0;
+  let failCount = 0;
+  
+  for (const objectConfig of loadPlan) {
+    try {
+      processObject(db, objectConfig, options.org);
+      successCount++;
+    } catch (error) {
+      log(`Failed to process ${objectConfig.object}: ${error.message}`, 'error');
+      failCount++;
+    }
+  }
+  
+  // Close database
+  db.close();
+  
+  // Summary
+  log(`\n${'='.repeat(60)}`);
+  log(`Summary`, 'info');
+  log(`${'='.repeat(60)}`);
+  log(`Successfully processed: ${successCount} objects`, 'success');
+  if (failCount > 0) {
+    log(`Failed to process: ${failCount} objects`, 'error');
+  }
+  log(`Database saved: ${options.database}`, 'success');
+}
+
+// Run main function
+try {
+  main();
+} catch (error) {
+  log(`Fatal error: ${error.message}`, 'error');
+  console.error(error);
+  process.exit(1);
+}

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "salesforce-to-sqlite",
+  "version": "1.0.0",
+  "description": "CLI tool to extract Salesforce data and load into SQLite",
+  "main": "index.js",
+  "bin": {
+    "sf-to-sqlite": "./index.js",
+    "sf2sqlite": "./index.js"
+
+  },
+  "scripts": {
+    "start": "node index.js"
+  },
+  "keywords": [
+    "salesforce",
+    "sqlite",
+    "cli"
+  ],
+  "author": "Mohan Chinnappan",
+  "license": "MIT",
+  "dependencies": {
+    "commander": "^11.1.0",
+    "express": "^4.18.2",
+    "better-sqlite3": "^9.2.2",
+    "csv-parse": "^5.5.3",
+    "chalk": "^4.1.2"
+  }
+}