wanas-zcrm-extractor 1.1.0 → 1.2.0

package/README.md

@@ -10,13 +10,16 @@ A robust, production-quality CLI utility and local web dashboard designed to aut
 
 ---
 
-> ### ⚠️ Important: metadata only — no access to your CRM data
+> ### ⚠️ Important: read-only — it never modifies your CRM
 >
-> `zcrm` reads **metadata/schema only** — module and field definitions, layouts, picklists,
-> related lists, profiles, roles, webhooks, and custom function source code. It has **no access
-> to your CRM records** and **cannot create, read, update, or delete** any data in Zoho CRM.
-> It is a one-way, read-only extractor: it pulls the *shape* of your org so you (or an AI
-> assistant) can write correct Deluge and API code — it never modifies anything in your CRM.
+> `zcrm` is strictly **read-only**: it **never creates, updates, or deletes** anything in Zoho
+> CRM. By **default** it pulls **schema/metadata only** — module and field definitions, layouts,
+> picklists, related lists, tags, profiles, roles, webhooks, org-wide settings, and custom
+> function source code — with **no access to your CRM records**.
+>
+> Reading actual record data is always **opt-in and explicit**: per-module record *counts* only
+> with `zcrm pull --with-counts` (~50 API credits/module), and audit-log data only via the
+> `zcrm audit` command. Even then, the tool only ever *reads* — it cannot change anything in your CRM.
 
 ---
 
@@ -123,6 +126,7 @@ zcrm pull [options]
 **Options:**
 * `-o, --output <dir>` - Destination folder path (defaults to `./metadata` in the current working directory).
 * `-c, --concurrency <n>` - Maximum number of simultaneous Zoho API calls, `1`–`25` (defaults to `5`).
+* `--with-counts` - Also fetch each module's **record count**. This reads record data (not just schema) and costs **~50 API credits per module**, so it is **off by default**. Omit it to keep the pull strictly metadata-only.
 
 **What happens during a pull:**
 1. Verifies authentication and refreshes active OAuth access tokens automatically.
@@ -229,6 +233,20 @@ zcrm skill --ide claude --metadata-dir ./crm-meta
 
 ---
 
+### `zcrm audit`
+Triggers an asynchronous export of the Zoho CRM audit log. By default, it exports all audit logs for the last 3 years. It automatically polls the API until the job is complete and downloads the resulting CSV or ZIP file to `storage/audit_logs/`.
+
+```bash
+# Export all audit logs
+zcrm audit
+
+# Export with specific criteria using a JSON file
+zcrm audit --filter input.json
+```
+*(See Zoho CRM documentation for the format of the filter JSON).*
+
+---
+
 ### 7. `zcrm logout`
 Clears stored configuration credentials, OAuth tokens, and active sessions from the workspace.
 
@@ -257,31 +275,52 @@ metadata/
 │       └── functions.json      # Extracted functions catalog
 │
 ├── crm/                        # Standard Zoho CRM metadata layout
-│   └── meta/
-│       ├── modules/            # Individual Module schemas
-│       │   └── <Module_API_Name>/
-│       │       ├── <Module_API_Name>.modules-meta.json
-│       │       ├── summary.json # Custom module summary (counts, fields, picklists, formulas)
-│       │       ├── fields/     # Field configurations
-│       │       │   └── <Field_API_Name>.fields-meta.json
-│       │       ├── layouts/    # Page Layout configurations
-│       │       │   └── <Layout_API_Name>.layouts-meta.json
-│       │       ├── custom_views/ # Custom View configurations
-│       │       │   └── <View_API_Name>.custom_views-meta.json
-│       │       └── related_lists/ # Related List configurations
-│       │           └── <List_API_Name>.related_lists-meta.json
-│       ├── profiles/           # Profile Permissions mapping
-│       │   └── <Profile_Name>.profiles-meta.json
-│       ├── roles/              # Role Hierarchy definitions
-│       │   └── <Role_Name>.roles-meta.json
-│       └── functions/          # Custom Deluge function files (.ds)
-│           ├── standalone/     # Deluge scripts sorted by namespace
-│           │   └── standalone.<Function_API_Name>.ds
-│           ├── automation/
-│           ├── button/
-│           ├── related_list/
-│           ├── schedule/
-│           └── salessignals/
+│   ├── meta/
+│   │   ├── custom_links/       # Global Custom Links
+│   │   ├── data_sharing/       # Data Sharing Settings
+│   │   ├── data_sharing_actions_summary/ # Data Sharing Actions Summary
+│   │   ├── data_sharing_rules/ # Data Sharing Rules
+│   │   ├── email_templates/    # Email Templates
+│   │   ├── global_picklists/   # Global Picklists
+│   │   ├── inventory_templates/# Inventory Templates
+│   │   ├── territories/        # Territories
+│   │   ├── variables/          # Global Variables
+│   │   ├── wizards/            # CRM Wizards
+│   │   ├── workflow_rules/     # Global Workflow Rules
+│   │   ├── modules/            # Individual Module schemas
+│   │   │   └── <Module_API_Name>/
+│   │   │       ├── <Module_API_Name>.modules-meta.json
+│   │   │       ├── summary.json # Custom module summary (counts, fields, picklists, formulas, records_count)
+│   │   │       ├── duplicate_check_preference.json # Duplicate Check Preferences
+│   │   │       ├── fields/     # Field configurations
+│   │   │       │   └── <Field_API_Name>.fields-meta.json
+│   │   │       ├── layouts/    # Page Layout configurations
+│   │   │       │   ├── <Layout_API_Name>.layouts-meta.json
+│   │   │       │   └── map_dependencies/
+│   │   │       │       └── <Layout_ID>.json
+│   │   │       ├── custom_views/ # Custom View configurations
+│   │   │       │   └── <View_API_Name>.custom_views-meta.json
+│   │   │       ├── record_locking_configurations/ # Record Locking Rules
+│   │   │       │   └── <Config_ID>.json
+│   │   │       ├── related_lists/ # Related List configurations
+│   │   │       │   └── <List_API_Name>.related_lists-meta.json
+│   │   │       ├── tags/       # Module Tags
+│   │   │       │   └── <Tag_ID>.json
+│   │   │       └── workflow_configurations/ # Module Workflow Configurations
+│   │   │           └── <Config_ID>.json
+│   │   ├── profiles/           # Profile Permissions mapping
+│   │   │   └── <Profile_Name>.profiles-meta.json
+│   │   └── roles/              # Role Hierarchy definitions
+│   │       └── <Role_Name>.roles-meta.json
+│   └── functions/              # Custom Deluge function files (.ds)
+│       ├── functions.json      # Complete list of extracted functions
+│       ├── standalone/         # Deluge scripts sorted by namespace
+│       │   └── standalone.<Function_API_Name>.ds
+│       ├── automation/
+│       ├── button/
+│       ├── related_list/
+│       ├── schedule/
+│       └── salessignals/
 │
 ├── zcrm-project.json           # ZCRM CLI project settings config
 ├── meta.json                   # ZCRM resource configuration
@@ -292,14 +331,20 @@ metadata/
 
 ## 🏢 About Wanas Apps
 
-**[Wanas Apps](https://www.wanasapps.com)** is an elite **Zoho Premium Partner** and consulting firm in the Middle East and North Africa (MENA) region. We specialize in digital transformation, Zoho implementation, custom CRM customization, full-stack application development on Zoho Catalyst, and building certified business compliance tools (such as Egyptian E-Invoice compliance integrations). 
+**[Wanas Apps](https://www.wanasapps.com)** is an elite **Zoho Premium Partner** and advanced technical consulting firm serving the Middle East and North Africa (MENA) region. We bridge the gap between standard business processes and advanced technical architecture, specializing in end-to-end digital transformation.
+
+### 🌟 Our Core Expertise
+- **Zoho Ecosystem Implementation:** Tailored deployments of Zoho CRM, Zoho Books, Zoho Creator, and the wider Zoho suite.
+- **Custom Application Development:** Full-stack, scalable, and serverless solutions built on **Zoho Catalyst**, Node.js, and modern frontends like Angular.
+- **Advanced Integrations & Automations:** We start where off-the-shelf software ends—building robust API gateways, custom webhooks, and complex CRM workflow automations.
+- **Compliance & Localization:** Certified business compliance tools, including seamless native integrations for the Egyptian E-Invoice and E-Receipt systems (ETA).
 
-We start where off-the-shelf software ends, building custom features that fit around your business blueprint.
+*We don't just implement software; we build custom features that fit seamlessly around your unique business blueprint.*
 
 ---
 
 ## ✉️ Support & Feedback
 
-If you encounter any issues, have feature requests, or need custom Zoho integrations, please reach out to our team at **support@wanasapps.com**.
+If you encounter any issues, have feature requests, or need custom Zoho integrations, please reach out to our team at **support@wanasapps.com** or visit **[wanasapps.com](https://www.wanasapps.com)**.
 
 This project is maintained by Wanas Apps and licensed under the **ISC License**.

package/banner.txt

@@ -1,7 +1,8 @@
-      ___  
-      \  \       __        __                         _                    
-   ___ \  \      \ \      / /_ _ _ __   __ _ ___     / \   _ __  _ __  ___ 
-  /  / /  /       \ \ /\ / / _` | '_ \ / _` / __|   / _ \ | '_ \| '_ \/ __|
- /  / /__/         \ V  V / (_| | | | | (_| \__ \  / ___ \| |_) | |_) \__ \
- \  \               \_/\_/ \__,_|_| |_|\__,_|___/ /_/   \_\ .__/| .__/|___/
-  \__\                                                    |_|   |_|        
+ __        __                         _                    
+ \ \      / /_ _ _ __   __ _ ___     / \   _ __  _ __  ___ 
+  \ \ /\ / / _` | '_ \ / _` / __|   / _ \ | '_ \| '_ \/ __|
+   \ V  V / (_| | | | | (_| \__ \  / ___ \| |_) | |_) \__ \
+    \_/\_/ \__,_|_| |_|\__,_|___/ /_/   \_\ .__/| .__/|___/
+                                          |_|   |_|        
+      For Support Contact us on support@wanasapps.com
+                     www.wanasapps.com

package/bin/cli.js

@@ -12,6 +12,7 @@ const { exec, spawn } = require('child_process');
 const configStore = require('../src/utils/configStore');
 const authService = require('../src/services/authService');
 const crmMetadataService = require('../src/services/metadata/crmMetadataService');
+const auditLogService = require('../src/services/auditLogService');
 const skillGenerator = require('../src/utils/skillGenerator');
 
 /**
@@ -38,10 +39,13 @@ function openBrowser(targetUrl) {
   }
 }
 
-// A short, prominent reminder that this tool only reads metadata schema.
+// A short, prominent reminder of the core guarantee: the tool is strictly
+// read-only and never modifies Zoho CRM. By default it pulls schema/metadata
+// only; reading record data is opt-in (pull --with-counts) or explicit (audit).
 const READ_ONLY_NOTICE =
-  '\x1b[1;33mℹ Note:\x1b[0m This tool extracts \x1b[1mmetadata/schema only\x1b[0m. It has \x1b[1mno access to your CRM\n' +
-  '  records\x1b[0m and \x1b[1mcannot create, read, update, or delete\x1b[0m any data in Zoho CRM.';
+  '\x1b[1;33mℹ Note:\x1b[0m This tool is \x1b[1mread-only\x1b[0m — it \x1b[1mnever creates, updates, or deletes\x1b[0m anything\n' +
+  '  in Zoho CRM. By default it extracts \x1b[1mschema/metadata only\x1b[0m (record counts and audit\n' +
+  '  logs are read only when you explicitly ask via --with-counts or the audit command).';
 
 // Default client credentials for generating Zoho login links
 const DEFAULT_CLIENT_ID = '1000.M1XE1M7CA4VFDDIRANB8E5AKIBQ72U';
@@ -412,6 +416,7 @@ program
   .description('Pull Zoho CRM metadata (read-only) and sync it into a local JSON directory structure')
   .option('-o, --output <dir>', 'Output folder path', './metadata')
   .option('-c, --concurrency <n>', 'Max concurrent Zoho API calls (1-25; default 5 — safe for all editions)', '5')
+  .option('--with-counts', 'Also fetch per-module record counts (reads record data; ~50 API credits per module)')
   .action(async (options) => {
     console.log(getBanner());
     // Reload tokens
@@ -429,7 +434,11 @@ program
 
     console.log(`\x1b[36mInitializing CRM Metadata Pulling...\x1b[0m`);
     console.log(`Destination Folder: ${outputDir}`);
-    console.log(`Concurrency: ${concurrency} simultaneous API call(s)\n`);
+    console.log(`Concurrency: ${concurrency} simultaneous API call(s)`);
+    if (options.withCounts) {
+      console.log('\x1b[33mRecord counts: ON\x1b[0m (reads record data; ~50 API credits per module)');
+    }
+    console.log('');
 
     let inModulesPhase = false;
     let inFunctionsPhase = false;
@@ -524,7 +533,7 @@ program
             }
             break;
         }
-      }, outputDir, { concurrency });
+      }, outputDir, { concurrency, withCounts: !!options.withCounts });
 
       process.exit(0);
     } catch (error) {
@@ -533,6 +542,40 @@ program
     }
   });
 
+// REGISTER COMMAND: audit
+program
+  .command('audit')
+  .description('Export the Zoho CRM audit log (asynchronous job)')
+  .option('-f, --filter <path>', 'Path to a JSON file containing the audit_log_export criteria')
+  .action(async (options) => {
+    console.log(getBanner());
+    // This command DOES read/export audit-log data, so it gets its own honest
+    // notice rather than the generic "metadata only / no data access" one.
+    console.log(
+      '\x1b[1;33mℹ Note:\x1b[0m This exports your Zoho CRM \x1b[1maudit-log data\x1b[0m (read-only).\n' +
+      '  It never creates, updates, or deletes anything in Zoho CRM.\n'
+    );
+
+    // Ensure user is authenticated
+    authService.loadTokens();
+    if (!authService.isAuthenticated()) {
+      console.error('\n❌ You are not logged in. Please run `zcrm login` first.');
+      process.exit(1);
+    }
+
+    try {
+      await authService.getAccessToken(); // Refresh if necessary
+      await auditLogService.exportAuditLog(options.filter);
+      console.log('\n\x1b[1;32m====================================================================\x1b[0m');
+      console.log('🎉 \x1b[1;32mAudit Log Export Completed Successfully!\x1b[0m');
+      console.log('\x1b[1;32m====================================================================\x1b[0m\n');
+      process.exit(0);
+    } catch (error) {
+      console.error(`\n❌ Audit log export failed: ${error.message}`);
+      process.exit(1);
+    }
+  });
+
 // REGISTER COMMAND: llm
 program
   .command('llm')
@@ -653,6 +696,7 @@ program
   .option('-p, --port <port>', 'Port to run the dashboard on', '3000')
   .action((options) => {
     process.env.PORT = options.port;
+    process.env.ZCRM_VERBOSE = 'true';
     // Keep ZCRM_CLI_MODE='true' (set at startup) so the dashboard shares the
     // same global credential/token store as `zcrm login` and `zcrm pull`.
     // Flipping it to 'false' here would desync the already-initialized

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wanas-zcrm-extractor",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Local Zoho CRM V8 Metadata Extractor CLI and Web Dashboard Utility",
   "main": "server.js",
   "bin": {

package/src/services/auditLogService.js

@@ -0,0 +1,139 @@
+const fs = require('fs-extra');
+const path = require('path');
+const axios = require('axios');
+const apiClient = require('../utils/apiClient');
+const authService = require('./authService');
+const { logInfo, logError } = require('../utils/logger');
+
+async function delay(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function exportAuditLog(filterFile) {
+  try {
+    let requestBody = { audit_log_export: [{}] }; // default: all logs up to 3 years
+
+    if (filterFile) {
+      const filterPath = path.resolve(process.cwd(), filterFile);
+      if (await fs.pathExists(filterPath)) {
+        requestBody = await fs.readJson(filterPath);
+        logInfo(`Loaded audit log filter criteria from ${filterFile}`);
+      } else {
+        throw new Error(`Filter file not found: ${filterFile}`);
+      }
+    } else {
+      logInfo('No filter specified. Requesting full audit log export (up to 3 years).');
+    }
+
+    logInfo('Starting Audit Log Export job...');
+    const triggerRes = await apiClient.post('/crm/v8/settings/audit_log_export', requestBody);
+    
+    if (!triggerRes.audit_log_export || triggerRes.audit_log_export.length === 0) {
+      throw new Error('Unexpected response format when starting export job.');
+    }
+
+    const jobDetails = triggerRes.audit_log_export[0];
+    if (jobDetails.status !== 'success' || !jobDetails.details || !jobDetails.details.id) {
+      throw new Error(`Failed to schedule job: ${JSON.stringify(jobDetails)}`);
+    }
+
+    const jobId = jobDetails.details.id;
+    logInfo(`Export job scheduled successfully. Job ID: ${jobId}`);
+    logInfo('Polling for job completion... This may take several minutes depending on the data size.');
+
+    let downloadUrl = null;
+    let attempts = 0;
+    const POLL_INTERVAL_MS = 15000;
+    const MAX_ATTEMPTS = 120; // 120 × 15s = 30 minutes, then give up rather than hang forever
+
+    // Poll until completion (bounded so a stuck job can never hang the CLI).
+    while (attempts < MAX_ATTEMPTS) {
+      attempts++;
+      // Wait between polls
+      await delay(POLL_INTERVAL_MS);
+
+      logInfo(`Checking status... (Attempt ${attempts}/${MAX_ATTEMPTS})`);
+      const statusRes = await apiClient.get(`/crm/v8/settings/audit_log_export/${jobId}`);
+
+      if (!statusRes.audit_log_export || statusRes.audit_log_export.length === 0) {
+        throw new Error('Invalid status response from server.');
+      }
+
+      const statusDetails = statusRes.audit_log_export[0];
+      const jobStatus = (statusDetails.status || '').toLowerCase();
+
+      if (jobStatus === 'finished') {
+        if (!statusDetails.download_links || statusDetails.download_links.length === 0) {
+          throw new Error('Job finished but no download links were provided.');
+        }
+        downloadUrl = statusDetails.download_links[0];
+        logInfo(`Job finished! Download URL obtained.`);
+        break;
+      } else if (jobStatus === 'failed') {
+        throw new Error('Export job failed on the server side.');
+      } else {
+        // Status is 'progress' or 'scheduled'
+        logInfo(`Job status: ${jobStatus}. Waiting...`);
+      }
+    }
+
+    if (!downloadUrl) {
+      throw new Error(`Audit log export did not finish within ${Math.round(MAX_ATTEMPTS * POLL_INTERVAL_MS / 60000)} minutes (job ${jobId}). Try a narrower filter or re-run later.`);
+    }
+
+    // Download the file
+    const outputDir = path.join(process.cwd(), 'storage', 'audit_logs');
+    await fs.ensureDir(outputDir);
+
+    // Determine the extension from the URL *path* (signed URLs carry a query
+    // string, so endsWith('.zip') on the whole URL would always miss). Default
+    // to .zip, the usual audit-export format.
+    let ext = '.zip';
+    try {
+      const urlPath = new URL(downloadUrl).pathname.toLowerCase();
+      const parsedExt = path.extname(urlPath);
+      if (parsedExt === '.csv' || parsedExt === '.zip') ext = parsedExt;
+    } catch (urlErr) {
+      // Keep the .zip default if the URL can't be parsed.
+    }
+    const outputPath = path.join(outputDir, `AuditLog_${jobId}${ext}`);
+
+    logInfo(`Downloading result to ${outputPath}...`);
+
+    const accessToken = await authService.getAccessToken();
+    const writer = fs.createWriteStream(outputPath);
+
+    const downloadResponse = await axios({
+      method: 'GET',
+      url: downloadUrl,
+      responseType: 'stream',
+      headers: {
+        'Authorization': `Zoho-oauthtoken ${accessToken}`
+      }
+    });
+
+    try {
+      await new Promise((resolve, reject) => {
+        downloadResponse.data.on('error', reject); // surface read-stream errors
+        writer.on('error', reject);
+        writer.on('finish', resolve);
+        downloadResponse.data.pipe(writer);
+      });
+    } catch (streamErr) {
+      // Don't leave a half-written file behind on failure.
+      writer.destroy();
+      await fs.remove(outputPath).catch(() => {});
+      throw streamErr;
+    }
+
+    logInfo(`Audit log successfully downloaded and saved to: ${outputPath}`);
+    
+  } catch (error) {
+    logError(error, 'auditLogService.exportAuditLog');
+    throw error;
+  }
+}
+
+module.exports = {
+  exportAuditLog
+};

package/src/services/metadata/crmMetadataService.js

@@ -5,6 +5,7 @@ const apiClient = require('../../utils/apiClient');
 const { mapWithConcurrency } = require('../../utils/concurrency');
 const { sweepDir, sweepDeletedDirs, normalizePath } = require('../../utils/fsSync');
 const { logError, logInfo } = require('../../utils/logger');
+const { formatDeluge } = require('../../utils/delugeFormatter');
 
 const STORAGE_DIR = path.join(process.cwd(), 'storage/metadata');
 
@@ -73,6 +74,11 @@ class CrmMetadataService extends BaseMetadataService {
     const concurrency = Math.max(1, Math.min(25, parseInt(options.concurrency, 10) || DEFAULT_CONCURRENCY));
     apiClient.setConcurrency(concurrency);
 
+    // Per-module record counts read RECORD DATA (not schema) and cost ~50 API
+    // credits per module, so they are strictly opt-in. Default pulls stay
+    // metadata-only with no record-data access.
+    const withCounts = options.withCounts === true;
+
     // Track every file written this run so stale files can be swept afterward.
     const writtenFiles = new Set();
     const recordWrite = (p) => writtenFiles.add(normalizePath(p));
@@ -242,15 +248,26 @@ class CrmMetadataService extends BaseMetadataService {
 
           fieldMap[apiName] = {};
 
-          // The five module sub-resources are independent — fetch concurrently.
+          // The module sub-resources are independent — fetch concurrently.
           // (Overall in-flight calls are still capped by the apiClient gate.)
-          const [detailR, fieldsR, layoutsR, relatedR, viewsR] = await Promise.allSettled([
+          const subResourceCalls = [
             apiClient.get(`/crm/v8/settings/modules/${apiName}`),
             apiClient.get('/crm/v8/settings/fields', { module: apiName }),
             apiClient.get('/crm/v8/settings/layouts', { module: apiName }),
             apiClient.get('/crm/v8/settings/related_lists', { module: apiName }),
-            apiClient.get('/crm/v8/settings/custom_views', { module: apiName })
-          ]);
+            apiClient.get('/crm/v8/settings/custom_views', { module: apiName }),
+            apiClient.get('/crm/v8/settings/duplicate_check_preference', { module: apiName }),
+            apiClient.get('/crm/v8/settings/record_locking_configurations', { module: apiName }),
+            apiClient.get('/crm/v8/settings/tags', { module: apiName }),
+            apiClient.get('/crm/v8/workflow_configurations', { module: apiName })
+          ];
+          // Record count is opt-in (reads record data; ~50 credits/module).
+          if (withCounts) {
+            subResourceCalls.push(
+              apiClient.get(`/crm/v8/${apiName}/actions/count`).catch(() => ({ count: 'unauthorized_or_unsupported' }))
+            );
+          }
+          const [detailR, fieldsR, layoutsR, relatedR, viewsR, dupCheckR, lockConfigR, tagsR, workflowConfigR, countR] = await Promise.allSettled(subResourceCalls);
 
           // 3a. Module Details
           if (detailR.status === 'fulfilled') {
@@ -308,12 +325,25 @@ class CrmMetadataService extends BaseMetadataService {
 
             if (hasLayouts && layouts.length > 0) {
               const layoutsDir = path.join(moduleSubDir, 'layouts');
+              const mapDepDir = path.join(layoutsDir, 'map_dependencies');
               await fs.ensureDir(layoutsDir);
               for (const layout of layouts) {
-                const layoutApiName = layout.api_name;
+                const layoutApiName = layout.api_name || layout.name;
+                const layoutId = layout.id;
                 if (layoutApiName) {
                   await outJson(path.join(layoutsDir, `${layoutApiName}.layouts-meta.json`), layout, { spaces: 2 });
                 }
+                if (layoutId) {
+                  try {
+                    const depR = await apiClient.get(`/crm/v8/settings/layouts/${layoutId}/map_dependency`, { module: apiName });
+                    if (depR && Array.isArray(depR.map_dependency) && depR.map_dependency.length > 0) {
+                      await fs.ensureDir(mapDepDir);
+                      await outJson(path.join(mapDepDir, `${layoutId}.json`), depR, { spaces: 2 });
+                    }
+                  } catch (depErr) {
+                    handleResourceError(depErr, `/crm/v8/settings/layouts/${layoutId}/map_dependency?module=${apiName}`, apiName, 'Layout Map Dependency');
+                  }
+                }
               }
             }
 
@@ -372,6 +402,82 @@ class CrmMetadataService extends BaseMetadataService {
             handleResourceError(viewsR.reason, `/crm/v8/settings/custom_views?module=${apiName}`, apiName, 'Custom Views');
           }
 
+          // 3e-1. Duplicate Check Preference
+          if (dupCheckR.status === 'fulfilled') {
+             const dupResp = dupCheckR.value;
+             if (dupResp && dupResp.duplicate_check_preference) {
+                 await outJson(path.join(moduleSubDir, 'duplicate_check_preference.json'), dupResp, { spaces: 2 });
+             }
+          } else {
+            handleResourceError(dupCheckR.reason, `/crm/v8/settings/duplicate_check_preference?module=${apiName}`, apiName, 'Duplicate Check Preference');
+          }
+
+          // 3e-2. Record Locking Configurations
+          let lockHasItems = false;
+          if (lockConfigR.status === 'fulfilled') {
+             const lockResp = lockConfigR.value;
+             if (lockResp && Array.isArray(lockResp.record_locking_configurations) && lockResp.record_locking_configurations.length > 0) {
+                 lockHasItems = true;
+                 const lockDir = path.join(moduleSubDir, 'record_locking_configurations');
+                 await fs.ensureDir(lockDir);
+                 for (const config of lockResp.record_locking_configurations) {
+                     const configId = config.id || config.name;
+                     if (configId) {
+                         await outJson(path.join(lockDir, `${configId}.json`), config, { spaces: 2 });
+                     }
+                 }
+             }
+          } else {
+            handleResourceError(lockConfigR.reason, `/crm/v8/settings/record_locking_configurations?module=${apiName}`, apiName, 'Record Locking Configurations');
+          }
+
+          // 3e-3. Tags
+          let tagsHasItems = false;
+          if (tagsR.status === 'fulfilled') {
+             const tagsResp = tagsR.value;
+             if (tagsResp && Array.isArray(tagsResp.tags) && tagsResp.tags.length > 0) {
+                 tagsHasItems = true;
+                 const tagsDir = path.join(moduleSubDir, 'tags');
+                 await fs.ensureDir(tagsDir);
+                 for (const tag of tagsResp.tags) {
+                     const tagId = tag.id || tag.name;
+                     if (tagId) {
+                         await outJson(path.join(tagsDir, `${tagId}.json`), tag, { spaces: 2 });
+                     }
+                 }
+             }
+          } else {
+            handleResourceError(tagsR.reason, `/crm/v8/settings/tags?module=${apiName}`, apiName, 'Tags');
+          }
+
+          // 3e-4. Workflow Configurations
+          let wfHasItems = false;
+          if (workflowConfigR.status === 'fulfilled') {
+             const wfResp = workflowConfigR.value;
+             if (wfResp && Array.isArray(wfResp.workflow_configurations) && wfResp.workflow_configurations.length > 0) {
+                 wfHasItems = true;
+                 const wfDir = path.join(moduleSubDir, 'workflow_configurations');
+                 await fs.ensureDir(wfDir);
+                 for (const config of wfResp.workflow_configurations) {
+                     const configId = config.id || config.name;
+                     if (configId) {
+                         await outJson(path.join(wfDir, `${configId}.json`), config, { spaces: 2 });
+                     }
+                 }
+             }
+          } else {
+            handleResourceError(workflowConfigR.reason, `/crm/v8/workflow_configurations?module=${apiName}`, apiName, 'Workflow Configurations');
+          }
+
+          // 3e-5. Record Count (actions/count) — only when --with-counts is set.
+          if (withCounts) {
+            if (countR && countR.status === 'fulfilled' && countR.value && countR.value.count !== undefined) {
+               summary.records_count = countR.value.count;
+            } else {
+               summary.records_count = 'unknown';
+            }
+          }
+
           // 3f. Save summary.json
           await outJson(path.join(moduleSubDir, 'summary.json'), summary, { spaces: 2 });
 
@@ -386,9 +492,19 @@ class CrmMetadataService extends BaseMetadataService {
           // shared field would lose updates).
           let sweptInModule = 0;
           if (summary.fields_count > 0) sweptInModule += await sweepDir(path.join(moduleSubDir, 'fields'), writtenFiles);
-          if (summary.layouts_count > 0) sweptInModule += await sweepDir(path.join(moduleSubDir, 'layouts'), writtenFiles);
+          if (summary.layouts_count > 0) {
+            sweptInModule += await sweepDir(path.join(moduleSubDir, 'layouts'), writtenFiles);
+            if (await fs.pathExists(path.join(moduleSubDir, 'layouts', 'map_dependencies'))) {
+              sweptInModule += await sweepDir(path.join(moduleSubDir, 'layouts', 'map_dependencies'), writtenFiles);
+            }
+          }
           if (summary.related_lists_count > 0) sweptInModule += await sweepDir(path.join(moduleSubDir, 'related_lists'), writtenFiles);
           if (viewsCount > 0) sweptInModule += await sweepDir(path.join(moduleSubDir, 'custom_views'), writtenFiles);
+          // Reconcile the new per-module resources only when this run fetched them
+          // successfully and non-empty — a transient failure must not delete them.
+          if (lockHasItems) sweptInModule += await sweepDir(path.join(moduleSubDir, 'record_locking_configurations'), writtenFiles);
+          if (tagsHasItems) sweptInModule += await sweepDir(path.join(moduleSubDir, 'tags'), writtenFiles);
+          if (wfHasItems) sweptInModule += await sweepDir(path.join(moduleSubDir, 'workflow_configurations'), writtenFiles);
           stats.removedStale += sweptInModule;
 
           stats.modulesProcessed++;
@@ -476,6 +592,67 @@ class CrmMetadataService extends BaseMetadataService {
         onProgress('FETCH_ROLES_WARNING', 'Could not fetch roles.');
       }
 
+      // 4d. Fetch Additional Global Settings
+      onProgress('FETCH_GLOBAL_SETTINGS', 'Fetching additional global settings...');
+      const globalEndpoints = [
+        { key: 'custom_links', endpoint: '/crm/v8/settings/custom_links', label: 'Custom Links', detailPath: (item) => `/crm/v8/settings/custom_links/${item.id}` },
+        { key: 'inventory_templates', endpoint: '/crm/v8/settings/inventory_templates', label: 'Inventory Templates', detailPath: (item) => `/crm/v8/settings/inventory_templates/${item.id}` },
+        { key: 'email_templates', endpoint: '/crm/v8/settings/email_templates', label: 'Email Templates', detailPath: (item) => `/crm/v8/settings/email_templates/${item.id}` },
+        { key: 'wizards', endpoint: '/crm/v8/settings/wizards', label: 'Wizards', detailPath: (item) => `/crm/v8/settings/wizards/${item.id}${item.layout?.id ? '?layout_id=' + item.layout.id : ''}` },
+        { key: 'global_picklists', endpoint: '/crm/v8/settings/global_picklists', label: 'Global Picklists', detailPath: (item) => `/crm/v8/settings/global_picklists/${item.id}` },
+        { key: 'rules', metaKey: 'data_sharing_rules', endpoint: '/crm/v8/settings/data_sharing/rules', label: 'Data Sharing Rules', detailPath: (item) => `/crm/v8/settings/data_sharing/rules/${item.id}` },
+        { key: 'data_sharing', endpoint: '/crm/v8/settings/data_sharing', label: 'Data Sharing Settings' },
+        { key: 'summary', metaKey: 'data_sharing_actions_summary', endpoint: '/crm/v8/settings/data_sharing/rules/actions/summary', label: 'Data Sharing Actions Summary' },
+        { key: 'variables', endpoint: '/crm/v8/settings/variables', label: 'Variables', detailPath: (item) => `/crm/v8/settings/variables/${item.id}${item.variable_group?.id ? '?group=' + item.variable_group.id : ''}` },
+        { key: 'workflow_rules', endpoint: '/crm/v8/settings/automation/workflow_rules', label: 'Workflow Rules', detailPath: (item) => `/crm/v8/settings/automation/workflow_rules/${item.id}` },
+        { key: 'territories', endpoint: '/crm/v8/settings/territories', label: 'Territories', detailPath: (item) => `/crm/v8/settings/territories/${item.id}` }
+      ];
+
+      for (const reqConfig of globalEndpoints) {
+        try {
+          const response = await apiClient.get(reqConfig.endpoint);
+          const items = response[reqConfig.key] || [];
+          const outputKey = reqConfig.metaKey || reqConfig.key;
+          completeMetadata[outputKey] = items;
+          const itemsDir = path.join(zcrmMetaDir, outputKey);
+          await fs.ensureDir(itemsDir);
+          
+          if (items.length > 0 && reqConfig.detailPath) {
+            await mapWithConcurrency(items, concurrency, async (item) => {
+              try {
+                if (item.id) {
+                  const detailResponse = await apiClient.get(reqConfig.detailPath(item));
+                  const detailedItem = (detailResponse[reqConfig.key] && detailResponse[reqConfig.key][0]) ? detailResponse[reqConfig.key][0] : item;
+                  const itemApiName = detailedItem.api_name || detailedItem.id || detailedItem.name;
+                  if (itemApiName) await outJson(path.join(itemsDir, `${itemApiName}.meta.json`), detailedItem, { spaces: 2 });
+                } else {
+                  const itemApiName = item.api_name || item.name;
+                  if (itemApiName) await outJson(path.join(itemsDir, `${itemApiName}.meta.json`), item, { spaces: 2 });
+                }
+              } catch (detailErr) {
+                const itemApiName = item.api_name || item.id || item.name;
+                if (itemApiName) await outJson(path.join(itemsDir, `${itemApiName}.meta.json`), item, { spaces: 2 });
+              }
+            });
+          } else {
+            for (const item of items) {
+              const itemApiName = item.api_name || item.id || item.name;
+              if (itemApiName) {
+                await outJson(path.join(itemsDir, `${itemApiName}.meta.json`), item, { spaces: 2 });
+              }
+            }
+          }
+
+          if (items.length > 0) {
+            const removedItems = await sweepDir(itemsDir, writtenFiles);
+            stats.removedStale += removedItems;
+          }
+        } catch (err) {
+          handleResourceError(err, reqConfig.endpoint, 'Global', reqConfig.label);
+          onProgress(`FETCH_${reqConfig.key.toUpperCase()}_WARNING`, `Could not fetch ${reqConfig.label}.`);
+        }
+      }
+
       // 5. Retrieve Functions and their Deluge scripts (bounded concurrency)
       onProgress('FETCH_FUNCTIONS', 'Fetching CRM functions...');
       try {
@@ -485,6 +662,7 @@ class CrmMetadataService extends BaseMetadataService {
 
         const functionsList = functionsResponse.functions || [];
         await fs.ensureDir(functionsDir);
+        await outJson(path.join(functionsDir, 'functions.json'), functionsResponse, { spaces: 2 });
 
         onProgress('FUNCTIONS_START', `Found ${functionsList.length} functions. Extracting deluge scripts (concurrency: ${concurrency})...`, {
           total: functionsList.length,
@@ -507,7 +685,8 @@ class CrmMetadataService extends BaseMetadataService {
             try {
               const scriptContent = await apiClient.get(`/crm/v8/settings/functions/${fnId}/code`);
               if (scriptContent && typeof scriptContent === 'string') {
-                await outFile(filePath, scriptContent, 'utf8');
+                const formattedScript = formatDeluge(scriptContent);
+                await outFile(filePath, formattedScript, 'utf8');
               } else {
                 throw new Error('Empty script code returned');
               }

package/src/utils/apiClient.js

@@ -65,16 +65,19 @@ function setConcurrency(n) {
  * @param {object} params - Optional query parameters
  * @param {number} retryCount - Tracking retries internally
  */
-async function request(endpoint, params = {}, retryCount = 0) {
+async function request(method, endpoint, params = {}, data = null, retryCount = 0) {
   try {
     const accessToken = await authService.getAccessToken();
     const baseUrl = authService.getApiBaseUrl();
     const url = `${baseUrl}${endpoint}`;
 
-    logInfo(`GET request to ${url} (Retry: ${retryCount})`, 'apiClient.get');
+    logInfo(`${method.toUpperCase()} request to ${url} (Retry: ${retryCount})`, 'apiClient.request');
 
-    const response = await apiClient.get(url, {
+    const response = await apiClient({
+      method,
+      url,
       params,
+      data,
       headers: {
         'Authorization': `Zoho-oauthtoken ${accessToken}`
       }
@@ -94,10 +97,10 @@ async function request(endpoint, params = {}, retryCount = 0) {
 
     // Handle 401 Unauthorized (Expired token or session)
     if (status === 401 && retryCount < 2) {
-      logInfo('Received 401 Unauthorized. Forcing token refresh and retrying...', 'apiClient.get');
+      logInfo('Received 401 Unauthorized. Forcing token refresh and retrying...', 'apiClient.request');
       try {
         await authService.refreshAccessToken();
-        return await request(endpoint, params, retryCount + 1);
+        return await request(method, endpoint, params, data, retryCount + 1);
       } catch (refreshErr) {
         logError(refreshErr, 'apiClient.get - Token refresh retry failed');
         throw error;
@@ -107,17 +110,17 @@ async function request(endpoint, params = {}, retryCount = 0) {
     // Handle 429 Too Many Requests (Rate limit / concurrency limit hit)
     if ((status === 429 || errorData?.code === 'TOO_MANY_REQUESTS') && retryCount < 5) {
       const waitTime = Math.pow(2, retryCount) * 1000; // 1s, 2s, 4s, 8s, 16s backoff
-      logInfo(`Rate/concurrency limit hit (429). Retrying in ${waitTime}ms...`, 'apiClient.get');
+      logInfo(`Rate/concurrency limit hit (429). Retrying in ${waitTime}ms...`, 'apiClient.request');
       await delay(waitTime);
-      return await request(endpoint, params, retryCount + 1);
+      return await request(method, endpoint, params, data, retryCount + 1);
     }
 
     // Handle temporary network or server failures (5xx)
     if (status >= 500 && status < 600 && retryCount < 2) {
       const waitTime = 1000;
-      logInfo(`Server error ${status}. Retrying in ${waitTime}ms...`, 'apiClient.get');
+      logInfo(`Server error ${status}. Retrying in ${waitTime}ms...`, 'apiClient.request');
       await delay(waitTime);
-      return await request(endpoint, params, retryCount + 1);
+      return await request(method, endpoint, params, data, retryCount + 1);
     }
 
     throw error;
@@ -137,7 +140,16 @@ async function request(endpoint, params = {}, retryCount = 0) {
 async function get(endpoint, params = {}) {
   await acquireSlot();
   try {
-    return await request(endpoint, params, 0);
+    return await request('get', endpoint, params, null, 0);
+  } finally {
+    releaseSlot();
+  }
+}
+
+async function post(endpoint, data = {}, params = {}) {
+  await acquireSlot();
+  try {
+    return await request('post', endpoint, params, data, 0);
   } finally {
     releaseSlot();
   }
@@ -145,5 +157,6 @@ async function get(endpoint, params = {}) {
 
 module.exports = {
   get,
+  post,
   setConcurrency
 };

package/src/utils/delugeFormatter.js

@@ -0,0 +1,45 @@
+/**
+ * Basic formatter for Zoho Deluge scripts.
+ * Adjusts indentation based on block scopes '{' and '}'.
+ * Note: This is a simple line-based formatter and does not fully parse the AST.
+ */
+function formatDeluge(code) {
+  if (!code || typeof code !== 'string') return code;
+
+  let formatted = '';
+  let indentLevel = 0;
+  const indentString = '    '; // 4 spaces
+
+  // Replace standard block formatting to ensure newlines before/after braces if it's on one line
+  // (We skip deep parsing, assuming the code from Zoho generally has basic newlines)
+  const lines = code.split('\n');
+
+  for (let line of lines) {
+    line = line.trim();
+    
+    // Preserve empty lines
+    if (!line) {
+      formatted += '\n';
+      continue;
+    }
+
+    // Determine if this line closes a block
+    if (line.startsWith('}') || line.startsWith(']')) {
+      indentLevel = Math.max(0, indentLevel - 1);
+    }
+
+    // Apply indentation
+    formatted += indentString.repeat(indentLevel) + line + '\n';
+
+    // Determine if this line opens a new block
+    if (line.endsWith('{') || line.endsWith('[')) {
+      indentLevel++;
+    }
+  }
+
+  return formatted.trim() + '\n';
+}
+
+module.exports = {
+  formatDeluge
+};