wanas-zcrm-extractor 1.2.0 → 1.4.0

package/README.md

@@ -276,17 +276,24 @@ metadata/
 │
 ├── crm/                        # Standard Zoho CRM metadata layout
 │   ├── meta/
-│   │   ├── custom_links/       # Global Custom Links
 │   │   ├── data_sharing/       # Data Sharing Settings
 │   │   ├── data_sharing_actions_summary/ # Data Sharing Actions Summary
 │   │   ├── data_sharing_rules/ # Data Sharing Rules
-│   │   ├── email_templates/    # Email Templates
+│   │   ├── email_templates/    # Email Templates (grouped by module)
+│   │   │   └── <Module_API_Name>/
+│   │   │       ├── <Template_Name>.meta.json
+│   │   │       └── <Template_Name>.html  # rendered from mail_content
 │   │   ├── global_picklists/   # Global Picklists
-│   │   ├── inventory_templates/# Inventory Templates
+│   │   ├── inventory_templates/# Inventory Templates (grouped by module)
+│   │   │   └── <Module_API_Name>/
+│   │   │       ├── <Template_Name>.meta.json
+│   │   │       └── <Template_Name>.html  # template HTML body
 │   │   ├── territories/        # Territories
 │   │   ├── variables/          # Global Variables
 │   │   ├── wizards/            # CRM Wizards
-│   │   ├── workflow_rules/     # Global Workflow Rules
+│   │   ├── workflow_rules/     # Workflow Rules (grouped by trigger module)
+│   │   │   └── <Trigger_Module_API_Name>/
+│   │   │       └── <Rule_Name>.json
 │   │   ├── modules/            # Individual Module schemas
 │   │   │   └── <Module_API_Name>/
 │   │   │       ├── <Module_API_Name>.modules-meta.json
@@ -300,6 +307,8 @@ metadata/
 │   │   │       │       └── <Layout_ID>.json
 │   │   │       ├── custom_views/ # Custom View configurations
 │   │   │       │   └── <View_API_Name>.custom_views-meta.json
+│   │   │       ├── custom_links/ # Module custom links/buttons
+│   │   │       │   └── <Link_ID>.json
 │   │   │       ├── record_locking_configurations/ # Record Locking Rules
 │   │   │       │   └── <Config_ID>.json
 │   │   │       ├── related_lists/ # Related List configurations

package/bin/cli.js

@@ -522,8 +522,11 @@ program
             console.log(` \x1b[1mStale Files Removed:\x1b[0m \x1b[33m${details.removedStale || 0}\x1b[0m (deleted on Zoho CRM since last pull)`);
             console.log(` \x1b[1mOutput Directory:\x1b[0m  \x1b[34m${outputDir}\x1b[0m`);
 
+            if (details.skipped > 0) {
+              console.log(` \x1b[1mSkipped (optional):\x1b[0m \x1b[90m${details.skipped}\x1b[0m settings not available for some modules/org (normal).`);
+            }
             if (details.errors && details.errors.length > 0) {
-              console.log(`\n \x1b[1;33m⚠️  Warnings:\x1b[0m \x1b[33m${details.errors.length} API request(s) failed or were skipped due to permissions.\x1b[0m`);
+              console.log(`\n \x1b[1;33m⚠️  Errors:\x1b[0m \x1b[33m${details.errors.length} unexpected API failure(s).\x1b[0m Details in the log file.`);
             }
             console.log('\x1b[1;32m====================================================================\x1b[0m\n');
             break;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wanas-zcrm-extractor",
-  "version": "1.2.0",
+  "version": "1.4.0",
   "description": "Local Zoho CRM V8 Metadata Extractor CLI and Web Dashboard Utility",
   "main": "server.js",
   "bin": {

package/src/services/metadata/crmMetadataService.js

@@ -40,6 +40,38 @@ function getNamespaceForCategory(category) {
   }
 }
 
+/**
+ * Makes an arbitrary string safe to use as a file/directory name across
+ * platforms (strips illegal characters, collapses whitespace, trims trailing
+ * dots/spaces which Windows forbids, and caps the length).
+ * @param {*} name
+ * @returns {string}
+ */
+function sanitizeFileName(name) {
+  const cleaned = String(name == null ? '' : name)
+    .replace(/[\\/:*?"<>|]/g, '_')
+    .replace(/\s+/g, '_')
+    .replace(/_+/g, '_');
+  const truncated = cleaned.slice(0, 120).replace(/^[._]+|[._]+$/g, '');
+  return truncated || 'unnamed';
+}
+
+/**
+ * Returns the first non-empty string value among the candidate fields (used to
+ * locate a template's HTML body, whose field name varies by template type).
+ * @param {object} item
+ * @param {string[]} fields
+ * @returns {string|null}
+ */
+function pickHtmlContent(item, fields) {
+  if (!item) return null;
+  for (const field of fields) {
+    const value = item[field];
+    if (typeof value === 'string' && value.trim().length > 0) return value;
+  }
+  return null;
+}
+
 /**
  * Metadata extractor service implementation for Zoho CRM V8.
  */
@@ -105,6 +137,7 @@ class CrmMetadataService extends BaseMetadataService {
       totalCustomViews: 0,
       removedStale: 0,
       concurrency,
+      skipped: 0,
       errors: []
     };
 
@@ -125,14 +158,30 @@ class CrmMetadataService extends BaseMetadataService {
 
     const fieldMap = {};
 
-    // Records a failed sub-resource fetch: noisy errors are logged loudly,
-    // expected permission/support errors are logged quietly.
-    const handleResourceError = (reason, endpoint, apiName, label) => {
-      stats.errors.push({ endpoint, error: reason?.message || String(reason) });
+    // Records a failed sub-resource fetch. Many sub-resources simply don't apply
+    // to every module/org — Zoho returns 400 ("module not supported", "layout
+    // deactivated", "feature not enabled", etc.). Those are EXPECTED: they are
+    // counted as `skipped` and logged quietly (file only) so they neither spam
+    // the console nor corrupt the progress bar. Only genuinely unexpected
+    // failures are recorded as `errors` and logged loudly. Pass optional=true
+    // for supplementary resources whose absence is always normal.
+    const EXPECTED_ERROR_CODES = new Set([
+      'NO_PERMISSION', 'NOT_SUPPORTED', 'INVALID_MODULE', 'INVALID_URL_PATTERN',
+      'FEATURE_NOT_ENABLED', 'REQUIRED_PARAM_MISSING', 'MANDATORY_NOT_FOUND', 'OAUTH_SCOPE_MISMATCH'
+    ]);
+    const EXPECTED_ERROR_RE = /not supported|not enabled|seems to be invalid|invalid module|deactivated|parameter is missing|expected parameter|no permission|not allowed|not available/i;
+
+    const handleResourceError = (reason, endpoint, apiName, label, optional = false) => {
+      const status = reason?.response?.status;
       const errCode = reason?.response?.data?.code;
-      if (errCode === 'NO_PERMISSION' || errCode === 'NOT_SUPPORTED') {
-        logInfo(`${label} not accessible/supported for module ${apiName} (Code: ${errCode})`, 'CrmMetadataService.extract');
+      const errMsg = reason?.response?.data?.message || reason?.message || '';
+      const expected = optional || EXPECTED_ERROR_CODES.has(errCode) || EXPECTED_ERROR_RE.test(errMsg);
+
+      if (expected) {
+        stats.skipped++;
+        logInfo(`${label} not available for ${apiName} (${errCode || status || 'n/a'}): ${errMsg}`, 'CrmMetadataService.extract');
       } else {
+        stats.errors.push({ endpoint, error: errMsg || String(reason) });
         logError(reason, `CrmMetadataService.extract - ${label}: ${apiName}`);
       }
     };
@@ -259,7 +308,9 @@ class CrmMetadataService extends BaseMetadataService {
             apiClient.get('/crm/v8/settings/duplicate_check_preference', { module: apiName }),
             apiClient.get('/crm/v8/settings/record_locking_configurations', { module: apiName }),
             apiClient.get('/crm/v8/settings/tags', { module: apiName }),
-            apiClient.get('/crm/v8/workflow_configurations', { module: apiName })
+            apiClient.get('/crm/v8/workflow_configurations', { module: apiName }),
+            // Custom links are module-scoped; the endpoint requires the module param.
+            apiClient.get('/crm/v8/settings/custom_links', { module: apiName })
           ];
           // Record count is opt-in (reads record data; ~50 credits/module).
           if (withCounts) {
@@ -267,7 +318,7 @@ class CrmMetadataService extends BaseMetadataService {
               apiClient.get(`/crm/v8/${apiName}/actions/count`).catch(() => ({ count: 'unauthorized_or_unsupported' }))
             );
           }
-          const [detailR, fieldsR, layoutsR, relatedR, viewsR, dupCheckR, lockConfigR, tagsR, workflowConfigR, countR] = await Promise.allSettled(subResourceCalls);
+          const [detailR, fieldsR, layoutsR, relatedR, viewsR, dupCheckR, lockConfigR, tagsR, workflowConfigR, customLinksR, countR] = await Promise.allSettled(subResourceCalls);
 
           // 3a. Module Details
           if (detailR.status === 'fulfilled') {
@@ -341,7 +392,7 @@ class CrmMetadataService extends BaseMetadataService {
                       await outJson(path.join(mapDepDir, `${layoutId}.json`), depR, { spaces: 2 });
                     }
                   } catch (depErr) {
-                    handleResourceError(depErr, `/crm/v8/settings/layouts/${layoutId}/map_dependency?module=${apiName}`, apiName, 'Layout Map Dependency');
+                    handleResourceError(depErr, `/crm/v8/settings/layouts/${layoutId}/map_dependency?module=${apiName}`, apiName, 'Layout Map Dependency', true);
                   }
                 }
               }
@@ -409,7 +460,7 @@ class CrmMetadataService extends BaseMetadataService {
                  await outJson(path.join(moduleSubDir, 'duplicate_check_preference.json'), dupResp, { spaces: 2 });
              }
           } else {
-            handleResourceError(dupCheckR.reason, `/crm/v8/settings/duplicate_check_preference?module=${apiName}`, apiName, 'Duplicate Check Preference');
+            handleResourceError(dupCheckR.reason, `/crm/v8/settings/duplicate_check_preference?module=${apiName}`, apiName, 'Duplicate Check Preference', true);
           }
 
           // 3e-2. Record Locking Configurations
@@ -428,7 +479,7 @@ class CrmMetadataService extends BaseMetadataService {
                  }
              }
           } else {
-            handleResourceError(lockConfigR.reason, `/crm/v8/settings/record_locking_configurations?module=${apiName}`, apiName, 'Record Locking Configurations');
+            handleResourceError(lockConfigR.reason, `/crm/v8/settings/record_locking_configurations?module=${apiName}`, apiName, 'Record Locking Configurations', true);
           }
 
           // 3e-3. Tags
@@ -447,7 +498,7 @@ class CrmMetadataService extends BaseMetadataService {
                  }
              }
           } else {
-            handleResourceError(tagsR.reason, `/crm/v8/settings/tags?module=${apiName}`, apiName, 'Tags');
+            handleResourceError(tagsR.reason, `/crm/v8/settings/tags?module=${apiName}`, apiName, 'Tags', true);
           }
 
           // 3e-4. Workflow Configurations
@@ -466,7 +517,26 @@ class CrmMetadataService extends BaseMetadataService {
                  }
              }
           } else {
-            handleResourceError(workflowConfigR.reason, `/crm/v8/workflow_configurations?module=${apiName}`, apiName, 'Workflow Configurations');
+            handleResourceError(workflowConfigR.reason, `/crm/v8/workflow_configurations?module=${apiName}`, apiName, 'Workflow Configurations', true);
+          }
+
+          // 3e-4b. Custom Links (module-scoped)
+          let customLinksHasItems = false;
+          if (customLinksR.status === 'fulfilled') {
+             const clResp = customLinksR.value;
+             if (clResp && Array.isArray(clResp.custom_links) && clResp.custom_links.length > 0) {
+                 customLinksHasItems = true;
+                 const clDir = path.join(moduleSubDir, 'custom_links');
+                 await fs.ensureDir(clDir);
+                 for (const link of clResp.custom_links) {
+                     const linkId = link.id || link.name;
+                     if (linkId) {
+                         await outJson(path.join(clDir, `${linkId}.json`), link, { spaces: 2 });
+                     }
+                 }
+             }
+          } else {
+            handleResourceError(customLinksR.reason, `/crm/v8/settings/custom_links?module=${apiName}`, apiName, 'Custom Links', true);
           }
 
           // 3e-5. Record Count (actions/count) — only when --with-counts is set.
@@ -505,6 +575,7 @@ class CrmMetadataService extends BaseMetadataService {
           if (lockHasItems) sweptInModule += await sweepDir(path.join(moduleSubDir, 'record_locking_configurations'), writtenFiles);
           if (tagsHasItems) sweptInModule += await sweepDir(path.join(moduleSubDir, 'tags'), writtenFiles);
           if (wfHasItems) sweptInModule += await sweepDir(path.join(moduleSubDir, 'workflow_configurations'), writtenFiles);
+          if (customLinksHasItems) sweptInModule += await sweepDir(path.join(moduleSubDir, 'custom_links'), writtenFiles);
           stats.removedStale += sweptInModule;
 
           stats.modulesProcessed++;
@@ -595,19 +666,46 @@ class CrmMetadataService extends BaseMetadataService {
       // 4d. Fetch Additional Global Settings
       onProgress('FETCH_GLOBAL_SETTINGS', 'Fetching additional global settings...');
       const globalEndpoints = [
-        { key: 'custom_links', endpoint: '/crm/v8/settings/custom_links', label: 'Custom Links', detailPath: (item) => `/crm/v8/settings/custom_links/${item.id}` },
-        { key: 'inventory_templates', endpoint: '/crm/v8/settings/inventory_templates', label: 'Inventory Templates', detailPath: (item) => `/crm/v8/settings/inventory_templates/${item.id}` },
-        { key: 'email_templates', endpoint: '/crm/v8/settings/email_templates', label: 'Email Templates', detailPath: (item) => `/crm/v8/settings/email_templates/${item.id}` },
+        { key: 'inventory_templates', endpoint: '/crm/v8/settings/inventory_templates', label: 'Inventory Templates', detailPath: (item) => `/crm/v8/settings/inventory_templates/${item.id}`, groupBy: (it) => it.module && it.module.api_name, htmlFields: ['content', 'html_content', 'mail_content', 'body'], nameFirst: true },
+        { key: 'email_templates', endpoint: '/crm/v8/settings/email_templates', label: 'Email Templates', detailPath: (item) => `/crm/v8/settings/email_templates/${item.id}`, groupBy: (it) => it.module && it.module.api_name, htmlFields: ['mail_content', 'content'], nameFirst: true },
         { key: 'wizards', endpoint: '/crm/v8/settings/wizards', label: 'Wizards', detailPath: (item) => `/crm/v8/settings/wizards/${item.id}${item.layout?.id ? '?layout_id=' + item.layout.id : ''}` },
         { key: 'global_picklists', endpoint: '/crm/v8/settings/global_picklists', label: 'Global Picklists', detailPath: (item) => `/crm/v8/settings/global_picklists/${item.id}` },
         { key: 'rules', metaKey: 'data_sharing_rules', endpoint: '/crm/v8/settings/data_sharing/rules', label: 'Data Sharing Rules', detailPath: (item) => `/crm/v8/settings/data_sharing/rules/${item.id}` },
         { key: 'data_sharing', endpoint: '/crm/v8/settings/data_sharing', label: 'Data Sharing Settings' },
         { key: 'summary', metaKey: 'data_sharing_actions_summary', endpoint: '/crm/v8/settings/data_sharing/rules/actions/summary', label: 'Data Sharing Actions Summary' },
         { key: 'variables', endpoint: '/crm/v8/settings/variables', label: 'Variables', detailPath: (item) => `/crm/v8/settings/variables/${item.id}${item.variable_group?.id ? '?group=' + item.variable_group.id : ''}` },
-        { key: 'workflow_rules', endpoint: '/crm/v8/settings/automation/workflow_rules', label: 'Workflow Rules', detailPath: (item) => `/crm/v8/settings/automation/workflow_rules/${item.id}` },
+        { key: 'workflow_rules', endpoint: '/crm/v8/settings/automation/workflow_rules', label: 'Workflow Rules', detailPath: (item) => `/crm/v8/settings/automation/workflow_rules/${item.id}`, groupBy: (it) => (it.trigger_module && (it.trigger_module.api_name || it.trigger_module)) || (it.module && it.module.api_name), jsonExt: '.json', nameFirst: true },
         { key: 'territories', endpoint: '/crm/v8/settings/territories', label: 'Territories', detailPath: (item) => `/crm/v8/settings/territories/${item.id}` }
       ];
 
+      // Writes one global-settings item: places it (optionally) in a subfolder
+      // grouped by module, names it, and emits an .html sidecar for templates.
+      const writeGlobalItem = async (detailedItem, reqConfig, itemsDir) => {
+        if (!detailedItem) return;
+
+        let targetDir = itemsDir;
+        if (reqConfig.groupBy) {
+          const groupName = sanitizeFileName(reqConfig.groupBy(detailedItem) || '_ungrouped');
+          targetDir = path.join(itemsDir, groupName);
+          await fs.ensureDir(targetDir);
+        }
+
+        const rawBase = reqConfig.nameFirst
+          ? (detailedItem.name || detailedItem.api_name || detailedItem.id)
+          : (detailedItem.api_name || detailedItem.id || detailedItem.name);
+        if (!rawBase) return;
+        const base = sanitizeFileName(rawBase);
+
+        const ext = reqConfig.jsonExt || '.meta.json';
+        await outJson(path.join(targetDir, `${base}${ext}`), detailedItem, { spaces: 2 });
+
+        // For templates, write the HTML body as a standalone .html file.
+        if (reqConfig.htmlFields) {
+          const html = pickHtmlContent(detailedItem, reqConfig.htmlFields);
+          if (html) await outFile(path.join(targetDir, `${base}.html`), html, 'utf8');
+        }
+      };
+
       for (const reqConfig of globalEndpoints) {
         try {
           const response = await apiClient.get(reqConfig.endpoint);
@@ -616,39 +714,33 @@ class CrmMetadataService extends BaseMetadataService {
           completeMetadata[outputKey] = items;
           const itemsDir = path.join(zcrmMetaDir, outputKey);
           await fs.ensureDir(itemsDir);
-          
+
           if (items.length > 0 && reqConfig.detailPath) {
             await mapWithConcurrency(items, concurrency, async (item) => {
-              try {
-                if (item.id) {
+              let detailedItem = item;
+              if (item.id) {
+                try {
                   const detailResponse = await apiClient.get(reqConfig.detailPath(item));
-                  const detailedItem = (detailResponse[reqConfig.key] && detailResponse[reqConfig.key][0]) ? detailResponse[reqConfig.key][0] : item;
-                  const itemApiName = detailedItem.api_name || detailedItem.id || detailedItem.name;
-                  if (itemApiName) await outJson(path.join(itemsDir, `${itemApiName}.meta.json`), detailedItem, { spaces: 2 });
-                } else {
-                  const itemApiName = item.api_name || item.name;
-                  if (itemApiName) await outJson(path.join(itemsDir, `${itemApiName}.meta.json`), item, { spaces: 2 });
+                  detailedItem = (detailResponse[reqConfig.key] && detailResponse[reqConfig.key][0]) ? detailResponse[reqConfig.key][0] : item;
+                } catch (detailErr) {
+                  detailedItem = item; // fall back to the list item on detail-fetch failure
                 }
-              } catch (detailErr) {
-                const itemApiName = item.api_name || item.id || item.name;
-                if (itemApiName) await outJson(path.join(itemsDir, `${itemApiName}.meta.json`), item, { spaces: 2 });
               }
+              await writeGlobalItem(detailedItem, reqConfig, itemsDir);
             });
           } else {
             for (const item of items) {
-              const itemApiName = item.api_name || item.id || item.name;
-              if (itemApiName) {
-                await outJson(path.join(itemsDir, `${itemApiName}.meta.json`), item, { spaces: 2 });
-              }
+              await writeGlobalItem(item, reqConfig, itemsDir);
             }
           }
 
+          // Recursive sweep reconciles grouped subfolders too (and prunes empties).
           if (items.length > 0) {
             const removedItems = await sweepDir(itemsDir, writtenFiles);
             stats.removedStale += removedItems;
           }
         } catch (err) {
-          handleResourceError(err, reqConfig.endpoint, 'Global', reqConfig.label);
+          handleResourceError(err, reqConfig.endpoint, 'Global', reqConfig.label, true);
           onProgress(`FETCH_${reqConfig.key.toUpperCase()}_WARNING`, `Could not fetch ${reqConfig.label}.`);
         }
       }