wanas-zcrm-extractor 1.3.0 → 1.5.0

package/README.md

@@ -1,6 +1,6 @@
 <p align="center">
   <a href="https://www.wanasapps.com">
-    <img src="https://www.wanasapps.com/WanasApps-logo.60b1ba4ed8605755.svg" alt="Wanas Apps Logo" width="320">
+    <img src="./WanasApps-logo.svg" alt="Wanas Apps Logo" width="320">
   </a>
 </p>
 
@@ -135,7 +135,9 @@ zcrm pull [options]
 4. Fetches Profiles, Roles, and Webhooks settings.
 5. Downloads all custom Deluge function scripts.
 6. Compiles a consolidated database of schemas and summaries.
-7. Prints a rich-text CLI statistics summary showing modules, layouts, fields, and stale files removed.
+7. Generates a **`README.md`** in the output folder describing the snapshot (org, counts, a per-module table, and the directory layout) — regenerated on every pull from the real data.
+8. Writes a **debug log** for the run to `<output>/.zcrm/.logs/ZCRM-CLI-<date>.log` (includes the quiet "skipped" entries that don't print to the console).
+9. Prints a rich-text CLI statistics summary showing modules, layouts, fields, stale files removed, and any skipped/errored requests.
 
 #### ⚡ Concurrency (faster pulls, safely)
 A pull makes many independent metadata requests, so they are run **in parallel** with a global cap on simultaneous calls. Zoho enforces a per-org **concurrency limit** based on your edition — **5** (Free), **10** (Standard/Starter), **15** (Professional), **20** (Enterprise/Zoho One), **25** (Ultimate/CRM Plus). The metadata endpoints used here are 1 credit each and are *not* subject to Zoho's stricter sub-concurrency limits.
@@ -234,7 +236,9 @@ zcrm skill --ide claude --metadata-dir ./crm-meta
 ---
 
 ### `zcrm audit`
-Triggers an asynchronous export of the Zoho CRM audit log. By default, it exports all audit logs for the last 3 years. It automatically polls the API until the job is complete and downloads the resulting CSV or ZIP file to `storage/audit_logs/`.
+Triggers an asynchronous export of the Zoho CRM audit log. By default, it exports all audit logs for the last 3 years. It automatically polls the API (with a bounded timeout) until the job is complete and downloads the resulting CSV or ZIP file to `storage/audit_logs/`.
+
+> **Requires the `ZohoCRM.settings.audit_logs.ALL` OAuth scope.** It is included in the default scopes, but if you authenticated *before* this scope was added you'll get a `401 invalid oauth scope` — just re-authenticate to grant it: `zcrm logout` then `zcrm login`.
 
 ```bash
 # Export all audit logs
@@ -276,7 +280,6 @@ metadata/
 │
 ├── crm/                        # Standard Zoho CRM metadata layout
 │   ├── meta/
-│   │   ├── custom_links/       # Global Custom Links
 │   │   ├── data_sharing/       # Data Sharing Settings
 │   │   ├── data_sharing_actions_summary/ # Data Sharing Actions Summary
 │   │   ├── data_sharing_rules/ # Data Sharing Rules
@@ -308,6 +311,8 @@ metadata/
 │   │   │       │       └── <Layout_ID>.json
 │   │   │       ├── custom_views/ # Custom View configurations
 │   │   │       │   └── <View_API_Name>.custom_views-meta.json
+│   │   │       ├── custom_links/ # Module custom links/buttons
+│   │   │       │   └── <Link_ID>.json
 │   │   │       ├── record_locking_configurations/ # Record Locking Rules
 │   │   │       │   └── <Config_ID>.json
 │   │   │       ├── related_lists/ # Related List configurations

package/bin/cli.js

@@ -521,9 +521,15 @@ program
             console.log(` \x1b[1mCustom Views:\x1b[0m      \x1b[35m${details.totalCustomViews || 0}\x1b[0m (saved inside crm/meta/modules/)`);
             console.log(` \x1b[1mStale Files Removed:\x1b[0m \x1b[33m${details.removedStale || 0}\x1b[0m (deleted on Zoho CRM since last pull)`);
             console.log(` \x1b[1mOutput Directory:\x1b[0m  \x1b[34m${outputDir}\x1b[0m`);
+            if (details.logFile) {
+              console.log(` \x1b[1mDebug Log:\x1b[0m         \x1b[34m${details.logFile}\x1b[0m`);
+            }
 
+            if (details.skipped > 0) {
+              console.log(` \x1b[1mSkipped (optional):\x1b[0m \x1b[90m${details.skipped}\x1b[0m settings not available for some modules/org (normal).`);
+            }
             if (details.errors && details.errors.length > 0) {
-              console.log(`\n \x1b[1;33m⚠️  Warnings:\x1b[0m \x1b[33m${details.errors.length} API request(s) failed or were skipped due to permissions.\x1b[0m`);
+              console.log(`\n \x1b[1;33m⚠️  Errors:\x1b[0m \x1b[33m${details.errors.length} unexpected API failure(s).\x1b[0m Details in the log file.`);
             }
             console.log('\x1b[1;32m====================================================================\x1b[0m\n');
             break;
@@ -571,7 +577,14 @@ program
       console.log('\x1b[1;32m====================================================================\x1b[0m\n');
       process.exit(0);
     } catch (error) {
-      console.error(`\n❌ Audit log export failed: ${error.message}`);
+      const apiMsg = error.response?.data?.message || error.message;
+      const isScope = error.response?.data?.code === 'OAUTH_SCOPE_MISMATCH' || /scope/i.test(apiMsg || '');
+      console.error(`\n❌ Audit log export failed: ${apiMsg}`);
+      if (isScope) {
+        console.error('\n\x1b[33mThis needs the \x1b[1mZohoCRM.settings.audit_logs.ALL\x1b[0m\x1b[33m OAuth scope, which your current');
+        console.error('session was not granted. Re-authenticate to add it:\x1b[0m');
+        console.error('   \x1b[36mzcrm logout\x1b[0m  then  \x1b[36mzcrm login\x1b[0m');
+      }
       process.exit(1);
     }
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wanas-zcrm-extractor",
-  "version": "1.3.0",
+  "version": "1.5.0",
   "description": "Local Zoho CRM V8 Metadata Extractor CLI and Web Dashboard Utility",
   "main": "server.js",
   "bin": {

package/src/services/authService.js

@@ -54,7 +54,7 @@ class AuthService {
    * @returns {string} Commas-separated list of scopes.
    */
   getScopes() {
-    const defaultScopes = 'ZohoCRM.modules.ALL,ZohoCRM.settings.ALL,ZohoCRM.users.ALL,ZohoCRM.org.ALL,ZohoCRM.bulk.ALL,ZohoCRM.notifications.ALL,ZohoCRM.coql.READ,ZohoCRM.settings.functions.ALL';
+    const defaultScopes = 'ZohoCRM.modules.ALL,ZohoCRM.settings.ALL,ZohoCRM.users.ALL,ZohoCRM.org.ALL,ZohoCRM.bulk.ALL,ZohoCRM.notifications.ALL,ZohoCRM.coql.READ,ZohoCRM.settings.functions.ALL,ZohoCRM.settings.audit_logs.ALL';
     if (process.env.ZCRM_CLI_MODE === 'true') {
       return configStore.get('scopes') || process.env.ZOHO_SCOPES || defaultScopes;
     }

package/src/utils/apiClient.js

@@ -95,8 +95,13 @@ async function request(method, endpoint, params = {}, data = null, retryCount =
 
     logInfo(`Request to ${endpoint} failed. Status: ${status}, Message: ${JSON.stringify(errorData || error.message)}`, 'apiClient.get');
 
+    // A 401 caused by a missing OAuth scope cannot be fixed by refreshing the
+    // token (the new token has the same scopes), so fail fast instead of
+    // refreshing and retrying twice (which just produced duplicate errors).
+    const isScopeError = errorData?.code === 'OAUTH_SCOPE_MISMATCH' || /scope/i.test(errorData?.message || '');
+
     // Handle 401 Unauthorized (Expired token or session)
-    if (status === 401 && retryCount < 2) {
+    if (status === 401 && !isScopeError && retryCount < 2) {
       logInfo('Received 401 Unauthorized. Forcing token refresh and retrying...', 'apiClient.request');
       try {
         await authService.refreshAccessToken();

package/src/utils/logger.js

@@ -13,6 +13,44 @@ const getLogFile = () => {
   return path.join(process.cwd(), 'storage/logs/errors.log');
 };
 
+// Optional per-run "session" log file. When set (e.g. during `zcrm pull`), every
+// log entry is ALSO appended here so the result folder gets a self-contained
+// debug log — including the quiet info/skip entries suppressed from the console.
+let sessionLogPath = null;
+
+/**
+ * Sets an additional log file that receives a copy of every log entry until
+ * cleared. Used to drop a debug log into the pull's output folder.
+ * @param {string|null} filePath
+ */
+function setSessionLog(filePath) {
+  sessionLogPath = filePath || null;
+}
+
+/** Stops mirroring log entries to the session log file. */
+function clearSessionLog() {
+  sessionLogPath = null;
+}
+
+/**
+ * Appends an entry to the default log file and (if set) the session log file.
+ * @param {string} entry
+ */
+function writeEntry(entry) {
+  try {
+    fs.outputFileSync(getLogFile(), entry, { flag: 'a' });
+  } catch (ioErr) {
+    console.error('Failed to write to log file:', ioErr);
+  }
+  if (sessionLogPath) {
+    try {
+      fs.outputFileSync(sessionLogPath, entry, { flag: 'a' });
+    } catch (ioErr) {
+      // Best-effort; never let session logging break the run.
+    }
+  }
+}
+
 /**
  * Log error message to console and output log file.
  * @param {Error|object|string} error - The error to log.
@@ -47,11 +85,7 @@ function logError(error, context = '') {
     console.error(`[ERROR] ${context ? `(${context}) ` : ''}`, error);
   }
 
-  try {
-    fs.outputFileSync(getLogFile(), logEntry, { flag: 'a' });
-  } catch (ioErr) {
-    console.error('Failed to write to log file:', ioErr);
-  }
+  writeEntry(logEntry);
 }
 
 /**
@@ -62,20 +96,18 @@ function logError(error, context = '') {
 function logInfo(message, context = '') {
   const timestamp = new Date().toISOString();
   const logEntry = `[${timestamp}] [INFO] ${context ? `(${context}) ` : ''}${message}\n`;
-  
+
   // Suppress info console logging when running CLI normally (unless verbose mode is enabled)
   if (process.env.ZCRM_CLI_MODE !== 'true' || process.env.ZCRM_VERBOSE === 'true') {
     console.log(`[INFO] ${context ? `(${context}) ` : ''}${message}`);
   }
-  
-  try {
-    fs.outputFileSync(getLogFile(), logEntry, { flag: 'a' });
-  } catch (ioErr) {
-    console.error('Failed to write to log file:', ioErr);
-  }
+
+  writeEntry(logEntry);
 }
 
 module.exports = {
   logError,
-  logInfo
+  logInfo,
+  setSessionLog,
+  clearSessionLog
 };

package/src/utils/readmeTemplate.js

@@ -0,0 +1,166 @@
+/**
+ * Builds the README.md that documents a metadata extraction's output folder.
+ * The content is generated dynamically from the data actually pulled, so it
+ * always reflects the real modules, counts, and settings in the snapshot.
+ */
+
+/** Formats an ISO timestamp into a readable UTC string. */
+function fmtDate(iso) {
+  try {
+    return new Date(iso).toUTCString();
+  } catch (e) {
+    return iso || 'unknown';
+  }
+}
+
+/**
+ * @param {object} info
+ * @param {string} info.orgName
+ * @param {string} info.generatedAt - ISO timestamp.
+ * @param {string} info.version - CLI version.
+ * @param {string} info.dc - Zoho data center.
+ * @param {object} info.stats - Extraction stats (totals, skipped, errors[]).
+ * @param {Array<{api_name:string,label:string,custom:boolean,fields:number,layouts:number,related_lists:number}>} info.modules
+ * @param {number} info.functionsCount
+ * @param {number} info.profilesCount
+ * @param {number} info.rolesCount
+ * @param {Array<{key:string,label:string,count:number}>} info.globalSettings
+ * @returns {string} Markdown.
+ */
+function buildResultReadme(info) {
+  const {
+    orgName = 'Unknown Org',
+    generatedAt = new Date(0).toISOString(),
+    version = '',
+    dc = 'com',
+    stats = {},
+    modules = [],
+    functionsCount = 0,
+    profilesCount = 0,
+    rolesCount = 0,
+    globalSettings = []
+  } = info || {};
+
+  const errorsCount = Array.isArray(stats.errors) ? stats.errors.length : 0;
+  const sortedModules = [...modules].sort((a, b) => (b.fields || 0) - (a.fields || 0));
+  const customCount = modules.filter(m => m.custom).length;
+  const standardCount = modules.length - customCount;
+
+  const lines = [];
+  lines.push(`# Zoho CRM Metadata Snapshot — ${orgName}`);
+  lines.push('');
+  lines.push(`> Generated by the \`zcrm\` CLI (Wanas Apps — Zoho CRM V8 Metadata Extractor${version ? ` v${version}` : ''}) on **${fmtDate(generatedAt)}**.`);
+  lines.push('');
+  lines.push('> ⚠️ **Read-only schema snapshot.** This folder describes the *structure* of the CRM');
+  lines.push('> (modules, fields, layouts, picklists, functions, settings). It contains **no CRM record');
+  lines.push('> data** and was produced by a tool that **never creates, updates, or deletes** anything in');
+  lines.push('> Zoho CRM. Use it to look up exact API names and constraints when writing Deluge or REST code.');
+  lines.push('');
+  lines.push('---');
+  lines.push('');
+
+  // ---- Summary ----
+  lines.push('## 📊 Snapshot summary');
+  lines.push('');
+  lines.push('| Resource | Count |');
+  lines.push('| :--- | ---: |');
+  lines.push(`| Organization | ${orgName} |`);
+  lines.push(`| Data center | \`${dc}\` |`);
+  lines.push(`| Modules (standard / custom) | **${modules.length}** (${standardCount} / ${customCount}) |`);
+  lines.push(`| Fields | ${stats.totalFields || 0} |`);
+  lines.push(`| Layouts | ${stats.totalLayouts || 0} |`);
+  lines.push(`| Related lists | ${stats.totalRelatedLists || 0} |`);
+  lines.push(`| Custom views | ${stats.totalCustomViews || 0} |`);
+  lines.push(`| Profiles | ${profilesCount} |`);
+  lines.push(`| Roles | ${rolesCount} |`);
+  lines.push(`| Deluge functions | ${functionsCount} |`);
+  lines.push(`| Stale files removed this pull | ${stats.removedStale || 0} |`);
+  lines.push(`| Skipped (optional, not available) | ${stats.skipped || 0} |`);
+  lines.push(`| Errors (unexpected) | ${errorsCount} |`);
+  lines.push('');
+  if (stats.skipped) {
+    lines.push(`> ℹ️ "Skipped" are optional settings that some modules/this org simply don't have (e.g. tags or duplicate-check on system modules). That is normal and not an error.`);
+    lines.push('');
+  }
+
+  // ---- Directory layout ----
+  lines.push('## 📂 What\'s in this folder');
+  lines.push('');
+  lines.push('```text');
+  lines.push('.');
+  lines.push('├── crm/');
+  lines.push('│   ├── meta/');
+  lines.push('│   │   ├── modules/<Module_API_Name>/   # per-module schema (see table below)');
+  lines.push('│   │   │   ├── <Module>.modules-meta.json   # module definition');
+  lines.push('│   │   │   ├── summary.json                 # quick counts + field classifications');
+  lines.push('│   │   │   ├── fields/                      # every field (.fields-meta.json)');
+  lines.push('│   │   │   ├── layouts/                     # page layouts (+ map_dependencies/)');
+  lines.push('│   │   │   ├── custom_views/                # custom views');
+  lines.push('│   │   │   ├── custom_links/                # module links/buttons');
+  lines.push('│   │   │   ├── related_lists/               # related lists');
+  lines.push('│   │   │   ├── tags/                        # module tags');
+  lines.push('│   │   │   ├── record_locking_configurations/');
+  lines.push('│   │   │   └── workflow_configurations/');
+  lines.push('│   │   ├── profiles/      # permission profiles');
+  lines.push('│   │   ├── roles/         # role hierarchy');
+  lines.push('│   │   ├── email_templates/<Module>/      # .meta.json + .html (body)');
+  lines.push('│   │   ├── inventory_templates/<Module>/  # .meta.json + .html (body)');
+  lines.push('│   │   ├── workflow_rules/<Trigger_Module>/<Rule>.json');
+  lines.push('│   │   └── ...               # other org-wide settings (see below)');
+  lines.push('│   └── functions/        # custom Deluge scripts (.ds), by namespace');
+  lines.push('├── .zcrm/');
+  lines.push('│   ├── .store/           # consolidated indexes (index.json, field_map.json, ...)');
+  lines.push('│   ├── .org/org.json     # organization metadata');
+  lines.push('│   └── .logs/            # debug log for this pull');
+  lines.push('├── zcrm-project.json     # ZCRM project config');
+  lines.push('└── README.md             # this file');
+  lines.push('```');
+  lines.push('');
+  lines.push('Handy starting points for an AI assistant or a developer:');
+  lines.push('');
+  lines.push('- **`.zcrm/.store/index.json`** — master list of module API names.');
+  lines.push('- **`.zcrm/.store/field_map.json`** — `{ Module: { Field: data_type } }` lookup.');
+  lines.push('- **`crm/meta/modules/<Module>/summary.json`** — required fields, picklists, lookups, subforms.');
+  lines.push('');
+
+  // ---- Modules table ----
+  if (sortedModules.length > 0) {
+    lines.push('## 🧩 Modules');
+    lines.push('');
+    lines.push('| Module (API name) | Label | Type | Fields | Layouts | Related lists |');
+    lines.push('| :--- | :--- | :--- | ---: | ---: | ---: |');
+    for (const m of sortedModules) {
+      const type = m.custom ? 'Custom' : 'Standard';
+      lines.push(`| \`${m.api_name}\` | ${m.label || ''} | ${type} | ${m.fields || 0} | ${m.layouts || 0} | ${m.related_lists || 0} |`);
+    }
+    lines.push('');
+  }
+
+  // ---- Global settings ----
+  const presentSettings = globalSettings.filter(g => g && g.count > 0);
+  if (presentSettings.length > 0) {
+    lines.push('## ⚙️ Org-wide settings extracted');
+    lines.push('');
+    lines.push('| Setting | Items |');
+    lines.push('| :--- | ---: |');
+    for (const g of presentSettings) {
+      lines.push(`| ${g.label || g.key} | ${g.count} |`);
+    }
+    lines.push('');
+  }
+
+  // ---- Usage ----
+  lines.push('## 🚀 Using this snapshot');
+  lines.push('');
+  lines.push('- Generate an AI-assistant context file from this snapshot: **`zcrm skill`**.');
+  lines.push('- Read the AI ingestion guide: **`zcrm llm`**.');
+  lines.push('- Browse it in a local web UI: **`zcrm dashboard`**.');
+  lines.push('- Refresh it any time: **`zcrm pull`** (overwrites changed files, removes data deleted on Zoho).');
+  lines.push('');
+  lines.push('_This file is regenerated on every `zcrm pull`._');
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+module.exports = { buildResultReadme };