wanas-zcrm-extractor 1.5.2 → 1.6.0

package/README.md

@@ -329,7 +329,7 @@ zcrm audit                      # last 3 years by default
 zcrm audit --filter input.json  # custom export criteria
 ```
 
-> **Requires the `ZohoCRM.settings.audit_logs.ALL` OAuth scope.** It's in the default scopes, but if you authenticated *before* it was added you'll see `401 invalid oauth scope` — just re-authenticate to grant it: `zcrm logout` then `zcrm login`.
+> **Requires the audit-log OAuth scopes** — `ZohoCRM.settings.audit_logs.CREATE` and `ZohoCRM.settings.audit_logs.READ` (both in the default scopes). If you authenticated *before* they were added you'll see `401 invalid oauth scope` — just re-authenticate: `zcrm logout` then `zcrm login`. *(If the download step later reports a file-scope error, add `ZohoFiles.files.READ` to `ZOHO_SCOPES` and log in again.)*
 </details>
 
 <br>
@@ -418,6 +418,12 @@ metadata/
 
 ---
 
+## 🩺 Troubleshooting
+
+Every command writes a timestamped **debug log** to `~/.zcrm/logs/` (e.g. `pull-<timestamp>.log`, `audit-<timestamp>.log`) capturing all activity for that run — including the quiet "skipped" lines and the **full API error bodies** that aren't printed to the console. When a command fails, its log path is shown in the output. `zcrm pull` additionally writes a copy into its output folder at `<output>/.zcrm/.logs/`.
+
+---
+
 ## ✉️ Support & Feedback
 
 Found a bug, have a feature request, or need custom Zoho integrations? Reach out:

package/bin/cli.js

@@ -9,11 +9,43 @@ const http = require('http');
 const url = require('url');
 const path = require('path');
 const { exec, spawn } = require('child_process');
+const os = require('os');
 const configStore = require('../src/utils/configStore');
 const authService = require('../src/services/authService');
 const crmMetadataService = require('../src/services/metadata/crmMetadataService');
 const auditLogService = require('../src/services/auditLogService');
 const skillGenerator = require('../src/utils/skillGenerator');
+const { addSessionLog, logInfo, shortErrorMessage } = require('../src/utils/logger');
+
+// ---------------------------------------------------------------------------
+// Per-command debug log
+// ---------------------------------------------------------------------------
+// Every real command writes a timestamped debug log to ~/.zcrm/logs/ that
+// captures ALL log entries for the run — including the quiet info/skip lines
+// and full API error bodies that are not printed to the console. The path is
+// shown if the command fails. (Command args are intentionally NOT logged, to
+// avoid writing secrets like --client-secret / --refresh-token to disk.)
+const LOGGED_COMMANDS = ['login', 'logout', 'whoami', 'status', 'pull', 'audit', 'llm', 'skill'];
+let debugLogPath = null;
+(function setupDebugLog() {
+  try {
+    const cmd = process.argv[2];
+    if (!LOGGED_COMMANDS.includes(cmd)) return;
+    const ts = new Date().toISOString().replace(/[:.]/g, '-');
+    debugLogPath = path.join(os.homedir(), '.zcrm', 'logs', `${cmd}-${ts}.log`);
+    addSessionLog(debugLogPath);
+    logInfo(`zcrm ${cmd} started.`, 'cli');
+  } catch (err) {
+    // Debug-logging setup must never break the command.
+  }
+})();
+
+// Point the user at the debug log when a command exits with an error.
+process.on('exit', (code) => {
+  if (code && debugLogPath) {
+    try { process.stderr.write(`\x1b[90mℹ Debug log: ${debugLogPath}\x1b[0m\n`); } catch (e) { /* ignore */ }
+  }
+});
 
 /**
  * Opens a URL in the user's default browser in a cross-platform-safe way.
@@ -577,13 +609,15 @@ program
       console.log('\x1b[1;32m====================================================================\x1b[0m\n');
       process.exit(0);
     } catch (error) {
-      const apiMsg = error.response?.data?.message || error.message;
+      const apiMsg = shortErrorMessage(error);
       const isScope = error.response?.data?.code === 'OAUTH_SCOPE_MISMATCH' || /scope/i.test(apiMsg || '');
       console.error(`\n❌ Audit log export failed: ${apiMsg}`);
       if (isScope) {
-        console.error('\n\x1b[33mThis needs the \x1b[1mZohoCRM.settings.audit_logs.ALL\x1b[0m\x1b[33m OAuth scope, which your current');
-        console.error('session was not granted. Re-authenticate to add it:\x1b[0m');
-        console.error('   \x1b[36mzcrm logout\x1b[0m  then  \x1b[36mzcrm login\x1b[0m');
+        console.error('\n\x1b[33mThis needs the audit-log scopes \x1b[1mZohoCRM.settings.audit_logs.CREATE\x1b[0m\x1b[33m and');
+        console.error('\x1b[1mZohoCRM.settings.audit_logs.READ\x1b[0m\x1b[33m, which your current session was not granted.');
+        console.error('Re-authenticate to add them:\x1b[0m  \x1b[36mzcrm logout\x1b[0m  then  \x1b[36mzcrm login\x1b[0m');
+        console.error('\x1b[90m(If the download step then reports a file-scope error, add ZohoFiles.files.READ');
+        console.error('to ZOHO_SCOPES and log in again.)\x1b[0m');
       }
       process.exit(1);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wanas-zcrm-extractor",
-  "version": "1.5.2",
+  "version": "1.6.0",
   "description": "Local Zoho CRM V8 Metadata Extractor CLI and Web Dashboard Utility",
   "main": "server.js",
   "bin": {

package/src/services/auditLogService.js

@@ -11,7 +11,11 @@ async function delay(ms) {
 
 async function exportAuditLog(filterFile) {
   try {
-    let requestBody = { audit_log_export: [{}] }; // default: all logs up to 3 years
+    // For a full export the request body is OPTIONAL — and sending an object
+    // without a non-empty `criteria` is rejected by Zoho (400). So the default
+    // (no filter) sends NO body, and a filter file (which must contain a valid
+    // `audit_log_export` criteria object) overrides it when supplied.
+    let requestBody = null;
 
     if (filterFile) {
       const filterPath = path.resolve(process.cwd(), filterFile);

package/src/services/authService.js

@@ -54,7 +54,13 @@ class AuthService {
    * @returns {string} Commas-separated list of scopes.
    */
   getScopes() {
-    const defaultScopes = 'ZohoCRM.modules.ALL,ZohoCRM.settings.ALL,ZohoCRM.users.ALL,ZohoCRM.org.ALL,ZohoCRM.bulk.ALL,ZohoCRM.notifications.ALL,ZohoCRM.coql.READ,ZohoCRM.settings.functions.ALL,ZohoCRM.settings.audit_logs.ALL';
+    // Every scope below is a documented ZohoCRM V8 scope. Audit logs have NO
+    // `.ALL` variant — the export flow uses the specific operation scopes
+    // (CREATE to initiate, READ to poll). The download step's cross-service
+    // ZohoFiles scope is intentionally NOT requested here so a single login
+    // stays entirely within ZohoCRM scopes (it can be added via ZOHO_SCOPES if
+    // a particular account needs it for the audit download).
+    const defaultScopes = 'ZohoCRM.modules.ALL,ZohoCRM.settings.ALL,ZohoCRM.users.ALL,ZohoCRM.org.ALL,ZohoCRM.bulk.ALL,ZohoCRM.notifications.ALL,ZohoCRM.coql.READ,ZohoCRM.settings.functions.ALL,ZohoCRM.settings.audit_logs.CREATE,ZohoCRM.settings.audit_logs.READ';
     if (process.env.ZCRM_CLI_MODE === 'true') {
       return configStore.get('scopes') || process.env.ZOHO_SCOPES || defaultScopes;
     }

package/src/services/metadata/crmMetadataService.js

@@ -4,7 +4,7 @@ const BaseMetadataService = require('./baseMetadataService');
 const apiClient = require('../../utils/apiClient');
 const { mapWithConcurrency } = require('../../utils/concurrency');
 const { sweepDir, sweepDeletedDirs, normalizePath } = require('../../utils/fsSync');
-const { logError, logInfo, setSessionLog, clearSessionLog } = require('../../utils/logger');
+const { logError, logInfo, addSessionLog, removeSessionLog } = require('../../utils/logger');
 const { formatDeluge } = require('../../utils/delugeFormatter');
 const { buildResultReadme } = require('../../utils/readmeTemplate');
 const pkg = require('../../../package.json');
@@ -113,6 +113,10 @@ class CrmMetadataService extends BaseMetadataService {
     // metadata-only with no record-data access.
     const withCounts = options.withCounts === true;
 
+    // Path of the debug log written into the output folder (set once the run
+    // starts; declared here so the finally block can stop mirroring to it).
+    let outputLogPath = null;
+
     // Track every file written this run so stale files can be swept afterward.
     const writtenFiles = new Set();
     const recordWrite = (p) => writtenFiles.add(normalizePath(p));
@@ -207,13 +211,13 @@ class CrmMetadataService extends BaseMetadataService {
       const month = String(today.getMonth() + 1).padStart(2, '0');
       const date = String(today.getDate()).padStart(2, '0');
       const logFileName = `ZCRM-CLI-${year}-${month}-${date}.log`;
-      const logFilePath = path.join(finalOutputDir, '.zcrm', '.logs', logFileName);
+      outputLogPath = path.join(finalOutputDir, '.zcrm', '.logs', logFileName);
       await fs.outputJson(path.join(finalOutputDir, '.zcrm', '.logs', '.audit.json'), {}, { spaces: 2 });
-      await fs.outputFile(logFilePath, `[${today.toISOString()}] zcrm pull started (concurrency ${concurrency}${withCounts ? ', record counts ON' : ''}).\n`, 'utf8');
+      await fs.outputFile(outputLogPath, `[${today.toISOString()}] zcrm pull started (concurrency ${concurrency}${withCounts ? ', record counts ON' : ''}).\n`, 'utf8');
       // Mirror every subsequent log entry (including the quiet info/skip lines
       // suppressed from the console) into a debug log in the result folder.
-      setSessionLog(logFilePath);
-      stats.logFile = logFilePath;
+      addSessionLog(outputLogPath);
+      stats.logFile = outputLogPath;
 
       // 1. Fetch Org Information to get Org ID/Name
       onProgress('FETCH_ORG', 'Fetching CRM Organization details...');
@@ -900,8 +904,9 @@ class CrmMetadataService extends BaseMetadataService {
       onProgress('FAILURE', `Extraction failed: ${error.message}`);
       throw error;
     } finally {
-      // Stop mirroring logs to the result folder once the pull is done.
-      clearSessionLog();
+      // Stop mirroring logs to the result folder once the pull is done (the
+      // CLI's own per-command debug log stays active).
+      if (outputLogPath) removeSessionLog(outputLogPath);
     }
   }
 }

package/src/utils/logger.js

@@ -3,7 +3,7 @@ const path = require('path');
 const os = require('os');
 
 /**
- * Dynamically resolves the log file path based on running mode (CLI vs Web).
+ * Dynamically resolves the default log file path based on running mode (CLI vs Web).
  * @returns {string} The resolved absolute file path for logging.
  */
 const getLogFile = () => {
@@ -13,71 +13,110 @@ const getLogFile = () => {
   return path.join(process.cwd(), 'storage/logs/errors.log');
 };
 
-// Optional per-run "session" log file. When set (e.g. during `zcrm pull`), every
-// log entry is ALSO appended here so the result folder gets a self-contained
-// debug log — including the quiet info/skip entries suppressed from the console.
-let sessionLogPath = null;
+// Additional "session" log files that receive a copy of every log entry. The
+// CLI adds one per command run (for debugging), and `zcrm pull` adds another in
+// its output folder. Multiple targets can be active at once.
+let sessionLogPaths = [];
 
-/**
- * Sets an additional log file that receives a copy of every log entry until
- * cleared. Used to drop a debug log into the pull's output folder.
- * @param {string|null} filePath
- */
-function setSessionLog(filePath) {
-  sessionLogPath = filePath || null;
+/** Adds an extra log file that mirrors every subsequent log entry. */
+function addSessionLog(filePath) {
+  if (filePath && !sessionLogPaths.includes(filePath)) sessionLogPaths.push(filePath);
+}
+
+/** Removes a previously-added session log file. */
+function removeSessionLog(filePath) {
+  sessionLogPaths = sessionLogPaths.filter((p) => p !== filePath);
 }
 
-/** Stops mirroring log entries to the session log file. */
-function clearSessionLog() {
-  sessionLogPath = null;
+/** Removes all session log files. */
+function clearSessionLogs() {
+  sessionLogPaths = [];
 }
 
 /**
- * Appends an entry to the default log file and (if set) the session log file.
+ * Appends an entry to the default log file and every active session log file.
  * @param {string} entry
  */
 function writeEntry(entry) {
   try {
     fs.outputFileSync(getLogFile(), entry, { flag: 'a' });
   } catch (ioErr) {
-    console.error('Failed to write to log file:', ioErr);
+    // Last resort — don't crash the command over a logging failure.
   }
-  if (sessionLogPath) {
+  for (const p of sessionLogPaths) {
     try {
-      fs.outputFileSync(sessionLogPath, entry, { flag: 'a' });
+      fs.outputFileSync(p, entry, { flag: 'a' });
     } catch (ioErr) {
-      // Best-effort; never let session logging break the run.
+      // best-effort
     }
   }
 }
 
 /**
- * Log error message to console and output log file.
+ * Builds a detailed, file-friendly description of an error — including the full
+ * HTTP response body for Axios errors, which is essential for debugging API
+ * failures (e.g. a Zoho 400 whose real reason lives in the response body).
+ * @param {Error|object|string} error
+ * @returns {string}
+ */
+function describeError(error) {
+  if (error && error.isAxiosError) {
+    const method = error.config?.method?.toUpperCase() || 'GET';
+    const url = error.config?.url || '';
+    const status = error.response?.status || 'no-status';
+    const body = error.response?.data;
+    const bodyStr = body == null
+      ? '(no response body)'
+      : (typeof body === 'string' ? body : JSON.stringify(body, null, 2));
+    return [
+      `${error.message}`,
+      `Request: ${method} ${url}`,
+      `Status: ${status}`,
+      `Response body: ${bodyStr}`,
+      `Stack: ${error.stack}`
+    ].join('\n');
+  }
+  if (error instanceof Error) {
+    return `${error.message}\nStack: ${error.stack}`;
+  }
+  if (typeof error === 'object') {
+    return JSON.stringify(error, null, 2);
+  }
+  return String(error);
+}
+
+/**
+ * Returns a concise one-line message for console output from any error,
+ * preferring the Zoho response message/code when present.
+ * @param {Error|object|string} error
+ * @returns {string}
+ */
+function shortErrorMessage(error) {
+  const body = error?.response?.data;
+  if (body) {
+    if (typeof body === 'string') return body;
+    if (body.message) return `${body.message}${body.code ? ` [${body.code}]` : ''}`;
+    return JSON.stringify(body);
+  }
+  return error?.message || String(error);
+}
+
+/**
+ * Log error message to console and log file(s).
  * @param {Error|object|string} error - The error to log.
  * @param {string} [context] - Context name where error occurred.
  */
 function logError(error, context = '') {
   const timestamp = new Date().toISOString();
-  let errorMessage = '';
-
-  if (error instanceof Error) {
-    errorMessage = `${error.message}\nStack: ${error.stack}`;
-  } else if (typeof error === 'object') {
-    errorMessage = JSON.stringify(error, null, 2);
-  } else {
-    errorMessage = String(error);
-  }
-
-  const logEntry = `[${timestamp}] [ERROR] ${context ? `(${context}) ` : ''}${errorMessage}\n--------------------------------------------------\n`;
+  const logEntry = `[${timestamp}] [ERROR] ${context ? `(${context}) ` : ''}${describeError(error)}\n--------------------------------------------------\n`;
 
-  // Output cleaner message to CLI instead of dumping the huge Axios error object
+  // Output a clean one-liner to the CLI instead of dumping the huge Axios object.
   if (process.env.ZCRM_CLI_MODE === 'true') {
     if (error && error.isAxiosError) {
       const method = error.config?.method?.toUpperCase() || 'GET';
       const url = error.config?.url || '';
       const status = error.response?.status || 'Unknown Status';
-      const msg = error.response?.data?.message || error.message;
-      console.error(`[ERROR] ${context ? `(${context}) ` : ''}API Request Failed: ${method} ${url} -> Status: ${status} (${msg})`);
+      console.error(`[ERROR] ${context ? `(${context}) ` : ''}API Request Failed: ${method} ${url} -> Status: ${status} (${shortErrorMessage(error)})`);
     } else {
       console.error(`[ERROR] ${context ? `(${context}) ` : ''}${error.message || error}`);
     }
@@ -89,7 +128,7 @@ function logError(error, context = '') {
 }
 
 /**
- * Log informational message to console and output log file.
+ * Log informational message to console and log file(s).
  * @param {string} message - The message.
  * @param {string} [context] - Context name.
  */
@@ -97,7 +136,7 @@ function logInfo(message, context = '') {
   const timestamp = new Date().toISOString();
   const logEntry = `[${timestamp}] [INFO] ${context ? `(${context}) ` : ''}${message}\n`;
 
-  // Suppress info console logging when running CLI normally (unless verbose mode is enabled)
+  // Suppress info console logging when running CLI normally (unless verbose).
   if (process.env.ZCRM_CLI_MODE !== 'true' || process.env.ZCRM_VERBOSE === 'true') {
     console.log(`[INFO] ${context ? `(${context}) ` : ''}${message}`);
   }
@@ -108,6 +147,8 @@ function logInfo(message, context = '') {
 module.exports = {
   logError,
   logInfo,
-  setSessionLog,
-  clearSessionLog
+  shortErrorMessage,
+  addSessionLog,
+  removeSessionLog,
+  clearSessionLogs
 };