wanas-zcrm-extractor 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -34
- package/banner.txt +8 -7
- package/bin/cli.js +49 -5
- package/package.json +1 -1
- package/src/services/auditLogService.js +139 -0
- package/src/services/metadata/crmMetadataService.js +0 -0
- package/src/utils/apiClient.js +23 -10
- package/src/utils/delugeFormatter.js +45 -0
package/README.md
CHANGED
|
@@ -10,13 +10,16 @@ A robust, production-quality CLI utility and local web dashboard designed to aut
|
|
|
10
10
|
|
|
11
11
|
---
|
|
12
12
|
|
|
13
|
-
> ### ⚠️ Important:
|
|
13
|
+
> ### ⚠️ Important: read-only — it never modifies your CRM
|
|
14
14
|
>
|
|
15
|
-
> `zcrm`
|
|
16
|
-
>
|
|
17
|
-
>
|
|
18
|
-
>
|
|
19
|
-
>
|
|
15
|
+
> `zcrm` is strictly **read-only**: it **never creates, updates, or deletes** anything in Zoho
|
|
16
|
+
> CRM. By **default** it pulls **schema/metadata only** — module and field definitions, layouts,
|
|
17
|
+
> picklists, related lists, tags, profiles, roles, webhooks, org-wide settings, and custom
|
|
18
|
+
> function source code — with **no access to your CRM records**.
|
|
19
|
+
>
|
|
20
|
+
> Reading actual record data is always **opt-in and explicit**: per-module record *counts* only
|
|
21
|
+
> with `zcrm pull --with-counts` (~50 API credits/module), and audit-log data only via the
|
|
22
|
+
> `zcrm audit` command. Even then, the tool only ever *reads* — it cannot change anything in your CRM.
|
|
20
23
|
|
|
21
24
|
---
|
|
22
25
|
|
|
@@ -123,6 +126,7 @@ zcrm pull [options]
|
|
|
123
126
|
**Options:**
|
|
124
127
|
* `-o, --output <dir>` - Destination folder path (defaults to `./metadata` in the current working directory).
|
|
125
128
|
* `-c, --concurrency <n>` - Maximum number of simultaneous Zoho API calls, `1`–`25` (defaults to `5`).
|
|
129
|
+
* `--with-counts` - Also fetch each module's **record count**. This reads record data (not just schema) and costs **~50 API credits per module**, so it is **off by default**. Omit it to keep the pull strictly metadata-only.
|
|
126
130
|
|
|
127
131
|
**What happens during a pull:**
|
|
128
132
|
1. Verifies authentication and refreshes active OAuth access tokens automatically.
|
|
@@ -229,6 +233,20 @@ zcrm skill --ide claude --metadata-dir ./crm-meta
|
|
|
229
233
|
|
|
230
234
|
---
|
|
231
235
|
|
|
236
|
+
### `zcrm audit`
|
|
237
|
+
Triggers an asynchronous export of the Zoho CRM audit log. By default, it exports all audit logs for the last 3 years. It automatically polls the API until the job is complete and downloads the resulting CSV or ZIP file to `storage/audit_logs/`.
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
# Export all audit logs
|
|
241
|
+
zcrm audit
|
|
242
|
+
|
|
243
|
+
# Export with specific criteria using a JSON file
|
|
244
|
+
zcrm audit --filter input.json
|
|
245
|
+
```
|
|
246
|
+
*(See Zoho CRM documentation for the format of the filter JSON).*
|
|
247
|
+
|
|
248
|
+
---
|
|
249
|
+
|
|
232
250
|
### 7. `zcrm logout`
|
|
233
251
|
Clears stored configuration credentials, OAuth tokens, and active sessions from the workspace.
|
|
234
252
|
|
|
@@ -257,31 +275,60 @@ metadata/
|
|
|
257
275
|
│ └── functions.json # Extracted functions catalog
|
|
258
276
|
│
|
|
259
277
|
├── crm/ # Standard Zoho CRM metadata layout
|
|
260
|
-
│
|
|
261
|
-
│
|
|
262
|
-
│
|
|
263
|
-
│
|
|
264
|
-
│
|
|
265
|
-
│
|
|
266
|
-
│
|
|
267
|
-
│
|
|
268
|
-
│
|
|
269
|
-
│
|
|
270
|
-
│
|
|
271
|
-
│
|
|
272
|
-
│
|
|
273
|
-
│
|
|
274
|
-
│
|
|
275
|
-
│
|
|
276
|
-
│
|
|
277
|
-
│
|
|
278
|
-
│
|
|
279
|
-
│
|
|
280
|
-
│
|
|
281
|
-
│
|
|
282
|
-
│
|
|
283
|
-
│
|
|
284
|
-
│
|
|
278
|
+
│ ├── meta/
|
|
279
|
+
│ │ ├── custom_links/ # Global Custom Links
|
|
280
|
+
│ │ ├── data_sharing/ # Data Sharing Settings
|
|
281
|
+
│ │ ├── data_sharing_actions_summary/ # Data Sharing Actions Summary
|
|
282
|
+
│ │ ├── data_sharing_rules/ # Data Sharing Rules
|
|
283
|
+
│ │ ├── email_templates/ # Email Templates (grouped by module)
|
|
284
|
+
│ │ │ └── <Module_API_Name>/
|
|
285
|
+
│ │ │ ├── <Template_Name>.meta.json
|
|
286
|
+
│ │ │ └── <Template_Name>.html # rendered from mail_content
|
|
287
|
+
│ │ ├── global_picklists/ # Global Picklists
|
|
288
|
+
│ │ ├── inventory_templates/# Inventory Templates (grouped by module)
|
|
289
|
+
│ │ │ └── <Module_API_Name>/
|
|
290
|
+
│ │ │ ├── <Template_Name>.meta.json
|
|
291
|
+
│ │ │ └── <Template_Name>.html # template HTML body
|
|
292
|
+
│ │ ├── territories/ # Territories
|
|
293
|
+
│ │ ├── variables/ # Global Variables
|
|
294
|
+
│ │ ├── wizards/ # CRM Wizards
|
|
295
|
+
│ │ ├── workflow_rules/ # Workflow Rules (grouped by trigger module)
|
|
296
|
+
│ │ │ └── <Trigger_Module_API_Name>/
|
|
297
|
+
│ │ │ └── <Rule_Name>.json
|
|
298
|
+
│ │ ├── modules/ # Individual Module schemas
|
|
299
|
+
│ │ │ └── <Module_API_Name>/
|
|
300
|
+
│ │ │ ├── <Module_API_Name>.modules-meta.json
|
|
301
|
+
│ │ │ ├── summary.json # Custom module summary (counts, fields, picklists, formulas, records_count)
|
|
302
|
+
│ │ │ ├── duplicate_check_preference.json # Duplicate Check Preferences
|
|
303
|
+
│ │ │ ├── fields/ # Field configurations
|
|
304
|
+
│ │ │ │ └── <Field_API_Name>.fields-meta.json
|
|
305
|
+
│ │ │ ├── layouts/ # Page Layout configurations
|
|
306
|
+
│ │ │ │ ├── <Layout_API_Name>.layouts-meta.json
|
|
307
|
+
│ │ │ │ └── map_dependencies/
|
|
308
|
+
│ │ │ │ └── <Layout_ID>.json
|
|
309
|
+
│ │ │ ├── custom_views/ # Custom View configurations
|
|
310
|
+
│ │ │ │ └── <View_API_Name>.custom_views-meta.json
|
|
311
|
+
│ │ │ ├── record_locking_configurations/ # Record Locking Rules
|
|
312
|
+
│ │ │ │ └── <Config_ID>.json
|
|
313
|
+
│ │ │ ├── related_lists/ # Related List configurations
|
|
314
|
+
│ │ │ │ └── <List_API_Name>.related_lists-meta.json
|
|
315
|
+
│ │ │ ├── tags/ # Module Tags
|
|
316
|
+
│ │ │ │ └── <Tag_ID>.json
|
|
317
|
+
│ │ │ └── workflow_configurations/ # Module Workflow Configurations
|
|
318
|
+
│ │ │ └── <Config_ID>.json
|
|
319
|
+
│ │ ├── profiles/ # Profile Permissions mapping
|
|
320
|
+
│ │ │ └── <Profile_Name>.profiles-meta.json
|
|
321
|
+
│ │ └── roles/ # Role Hierarchy definitions
|
|
322
|
+
│ │ └── <Role_Name>.roles-meta.json
|
|
323
|
+
│ └── functions/ # Custom Deluge function files (.ds)
|
|
324
|
+
│ ├── functions.json # Complete list of extracted functions
|
|
325
|
+
│ ├── standalone/ # Deluge scripts sorted by namespace
|
|
326
|
+
│ │ └── standalone.<Function_API_Name>.ds
|
|
327
|
+
│ ├── automation/
|
|
328
|
+
│ ├── button/
|
|
329
|
+
│ ├── related_list/
|
|
330
|
+
│ ├── schedule/
|
|
331
|
+
│ └── salessignals/
|
|
285
332
|
│
|
|
286
333
|
├── zcrm-project.json # ZCRM CLI project settings config
|
|
287
334
|
├── meta.json # ZCRM resource configuration
|
|
@@ -292,14 +339,20 @@ metadata/
|
|
|
292
339
|
|
|
293
340
|
## 🏢 About Wanas Apps
|
|
294
341
|
|
|
295
|
-
**[Wanas Apps](https://www.wanasapps.com)** is an elite **Zoho Premium Partner** and consulting firm
|
|
342
|
+
**[Wanas Apps](https://www.wanasapps.com)** is an elite **Zoho Premium Partner** and advanced technical consulting firm serving the Middle East and North Africa (MENA) region. We bridge the gap between standard business processes and advanced technical architecture, specializing in end-to-end digital transformation.
|
|
343
|
+
|
|
344
|
+
### 🌟 Our Core Expertise
|
|
345
|
+
- **Zoho Ecosystem Implementation:** Tailored deployments of Zoho CRM, Zoho Books, Zoho Creator, and the wider Zoho suite.
|
|
346
|
+
- **Custom Application Development:** Full-stack, scalable, and serverless solutions built on **Zoho Catalyst**, Node.js, and modern frontends like Angular.
|
|
347
|
+
- **Advanced Integrations & Automations:** We start where off-the-shelf software ends—building robust API gateways, custom webhooks, and complex CRM workflow automations.
|
|
348
|
+
- **Compliance & Localization:** Certified business compliance tools, including seamless native integrations for the Egyptian E-Invoice and E-Receipt systems (ETA).
|
|
296
349
|
|
|
297
|
-
We
|
|
350
|
+
*We don't just implement software; we build custom features that fit seamlessly around your unique business blueprint.*
|
|
298
351
|
|
|
299
352
|
---
|
|
300
353
|
|
|
301
354
|
## ✉️ Support & Feedback
|
|
302
355
|
|
|
303
|
-
If you encounter any issues, have feature requests, or need custom Zoho integrations, please reach out to our team at **support@wanasapps.com**.
|
|
356
|
+
If you encounter any issues, have feature requests, or need custom Zoho integrations, please reach out to our team at **support@wanasapps.com** or visit **[wanasapps.com](https://www.wanasapps.com)**.
|
|
304
357
|
|
|
305
358
|
This project is maintained by Wanas Apps and licensed under the **ISC License**.
|
package/banner.txt
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
__ __ _
|
|
2
|
+
\ \ / /_ _ _ __ __ _ ___ / \ _ __ _ __ ___
|
|
3
|
+
\ \ /\ / / _` | '_ \ / _` / __| / _ \ | '_ \| '_ \/ __|
|
|
4
|
+
\ V V / (_| | | | | (_| \__ \ / ___ \| |_) | |_) \__ \
|
|
5
|
+
\_/\_/ \__,_|_| |_|\__,_|___/ /_/ \_\ .__/| .__/|___/
|
|
6
|
+
|_| |_|
|
|
7
|
+
For Support Contact us on support@wanasapps.com
|
|
8
|
+
www.wanasapps.com
|
package/bin/cli.js
CHANGED
|
@@ -12,6 +12,7 @@ const { exec, spawn } = require('child_process');
|
|
|
12
12
|
const configStore = require('../src/utils/configStore');
|
|
13
13
|
const authService = require('../src/services/authService');
|
|
14
14
|
const crmMetadataService = require('../src/services/metadata/crmMetadataService');
|
|
15
|
+
const auditLogService = require('../src/services/auditLogService');
|
|
15
16
|
const skillGenerator = require('../src/utils/skillGenerator');
|
|
16
17
|
|
|
17
18
|
/**
|
|
@@ -38,10 +39,13 @@ function openBrowser(targetUrl) {
|
|
|
38
39
|
}
|
|
39
40
|
}
|
|
40
41
|
|
|
41
|
-
// A short, prominent reminder
|
|
42
|
+
// A short, prominent reminder of the core guarantee: the tool is strictly
|
|
43
|
+
// read-only and never modifies Zoho CRM. By default it pulls schema/metadata
|
|
44
|
+
// only; reading record data is opt-in (pull --with-counts) or explicit (audit).
|
|
42
45
|
const READ_ONLY_NOTICE =
|
|
43
|
-
'\x1b[1;33mℹ Note:\x1b[0m This tool
|
|
44
|
-
'
|
|
46
|
+
'\x1b[1;33mℹ Note:\x1b[0m This tool is \x1b[1mread-only\x1b[0m — it \x1b[1mnever creates, updates, or deletes\x1b[0m anything\n' +
|
|
47
|
+
' in Zoho CRM. By default it extracts \x1b[1mschema/metadata only\x1b[0m (record counts and audit\n' +
|
|
48
|
+
' logs are read only when you explicitly ask via --with-counts or the audit command).';
|
|
45
49
|
|
|
46
50
|
// Default client credentials for generating Zoho login links
|
|
47
51
|
const DEFAULT_CLIENT_ID = '1000.M1XE1M7CA4VFDDIRANB8E5AKIBQ72U';
|
|
@@ -412,6 +416,7 @@ program
|
|
|
412
416
|
.description('Pull Zoho CRM metadata (read-only) and sync it into a local JSON directory structure')
|
|
413
417
|
.option('-o, --output <dir>', 'Output folder path', './metadata')
|
|
414
418
|
.option('-c, --concurrency <n>', 'Max concurrent Zoho API calls (1-25; default 5 — safe for all editions)', '5')
|
|
419
|
+
.option('--with-counts', 'Also fetch per-module record counts (reads record data; ~50 API credits per module)')
|
|
415
420
|
.action(async (options) => {
|
|
416
421
|
console.log(getBanner());
|
|
417
422
|
// Reload tokens
|
|
@@ -429,7 +434,11 @@ program
|
|
|
429
434
|
|
|
430
435
|
console.log(`\x1b[36mInitializing CRM Metadata Pulling...\x1b[0m`);
|
|
431
436
|
console.log(`Destination Folder: ${outputDir}`);
|
|
432
|
-
console.log(`Concurrency: ${concurrency} simultaneous API call(s)
|
|
437
|
+
console.log(`Concurrency: ${concurrency} simultaneous API call(s)`);
|
|
438
|
+
if (options.withCounts) {
|
|
439
|
+
console.log('\x1b[33mRecord counts: ON\x1b[0m (reads record data; ~50 API credits per module)');
|
|
440
|
+
}
|
|
441
|
+
console.log('');
|
|
433
442
|
|
|
434
443
|
let inModulesPhase = false;
|
|
435
444
|
let inFunctionsPhase = false;
|
|
@@ -524,7 +533,7 @@ program
|
|
|
524
533
|
}
|
|
525
534
|
break;
|
|
526
535
|
}
|
|
527
|
-
}, outputDir, { concurrency });
|
|
536
|
+
}, outputDir, { concurrency, withCounts: !!options.withCounts });
|
|
528
537
|
|
|
529
538
|
process.exit(0);
|
|
530
539
|
} catch (error) {
|
|
@@ -533,6 +542,40 @@ program
|
|
|
533
542
|
}
|
|
534
543
|
});
|
|
535
544
|
|
|
545
|
+
// REGISTER COMMAND: audit
|
|
546
|
+
program
|
|
547
|
+
.command('audit')
|
|
548
|
+
.description('Export the Zoho CRM audit log (asynchronous job)')
|
|
549
|
+
.option('-f, --filter <path>', 'Path to a JSON file containing the audit_log_export criteria')
|
|
550
|
+
.action(async (options) => {
|
|
551
|
+
console.log(getBanner());
|
|
552
|
+
// This command DOES read/export audit-log data, so it gets its own honest
|
|
553
|
+
// notice rather than the generic "metadata only / no data access" one.
|
|
554
|
+
console.log(
|
|
555
|
+
'\x1b[1;33mℹ Note:\x1b[0m This exports your Zoho CRM \x1b[1maudit-log data\x1b[0m (read-only).\n' +
|
|
556
|
+
' It never creates, updates, or deletes anything in Zoho CRM.\n'
|
|
557
|
+
);
|
|
558
|
+
|
|
559
|
+
// Ensure user is authenticated
|
|
560
|
+
authService.loadTokens();
|
|
561
|
+
if (!authService.isAuthenticated()) {
|
|
562
|
+
console.error('\n❌ You are not logged in. Please run `zcrm login` first.');
|
|
563
|
+
process.exit(1);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
try {
|
|
567
|
+
await authService.getAccessToken(); // Refresh if necessary
|
|
568
|
+
await auditLogService.exportAuditLog(options.filter);
|
|
569
|
+
console.log('\n\x1b[1;32m====================================================================\x1b[0m');
|
|
570
|
+
console.log('🎉 \x1b[1;32mAudit Log Export Completed Successfully!\x1b[0m');
|
|
571
|
+
console.log('\x1b[1;32m====================================================================\x1b[0m\n');
|
|
572
|
+
process.exit(0);
|
|
573
|
+
} catch (error) {
|
|
574
|
+
console.error(`\n❌ Audit log export failed: ${error.message}`);
|
|
575
|
+
process.exit(1);
|
|
576
|
+
}
|
|
577
|
+
});
|
|
578
|
+
|
|
536
579
|
// REGISTER COMMAND: llm
|
|
537
580
|
program
|
|
538
581
|
.command('llm')
|
|
@@ -653,6 +696,7 @@ program
|
|
|
653
696
|
.option('-p, --port <port>', 'Port to run the dashboard on', '3000')
|
|
654
697
|
.action((options) => {
|
|
655
698
|
process.env.PORT = options.port;
|
|
699
|
+
process.env.ZCRM_VERBOSE = 'true';
|
|
656
700
|
// Keep ZCRM_CLI_MODE='true' (set at startup) so the dashboard shares the
|
|
657
701
|
// same global credential/token store as `zcrm login` and `zcrm pull`.
|
|
658
702
|
// Flipping it to 'false' here would desync the already-initialized
|
package/package.json
CHANGED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
const fs = require('fs-extra');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const axios = require('axios');
|
|
4
|
+
const apiClient = require('../utils/apiClient');
|
|
5
|
+
const authService = require('./authService');
|
|
6
|
+
const { logInfo, logError } = require('../utils/logger');
|
|
7
|
+
|
|
8
|
+
async function delay(ms) {
|
|
9
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
async function exportAuditLog(filterFile) {
|
|
13
|
+
try {
|
|
14
|
+
let requestBody = { audit_log_export: [{}] }; // default: all logs up to 3 years
|
|
15
|
+
|
|
16
|
+
if (filterFile) {
|
|
17
|
+
const filterPath = path.resolve(process.cwd(), filterFile);
|
|
18
|
+
if (await fs.pathExists(filterPath)) {
|
|
19
|
+
requestBody = await fs.readJson(filterPath);
|
|
20
|
+
logInfo(`Loaded audit log filter criteria from ${filterFile}`);
|
|
21
|
+
} else {
|
|
22
|
+
throw new Error(`Filter file not found: ${filterFile}`);
|
|
23
|
+
}
|
|
24
|
+
} else {
|
|
25
|
+
logInfo('No filter specified. Requesting full audit log export (up to 3 years).');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
logInfo('Starting Audit Log Export job...');
|
|
29
|
+
const triggerRes = await apiClient.post('/crm/v8/settings/audit_log_export', requestBody);
|
|
30
|
+
|
|
31
|
+
if (!triggerRes.audit_log_export || triggerRes.audit_log_export.length === 0) {
|
|
32
|
+
throw new Error('Unexpected response format when starting export job.');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const jobDetails = triggerRes.audit_log_export[0];
|
|
36
|
+
if (jobDetails.status !== 'success' || !jobDetails.details || !jobDetails.details.id) {
|
|
37
|
+
throw new Error(`Failed to schedule job: ${JSON.stringify(jobDetails)}`);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const jobId = jobDetails.details.id;
|
|
41
|
+
logInfo(`Export job scheduled successfully. Job ID: ${jobId}`);
|
|
42
|
+
logInfo('Polling for job completion... This may take several minutes depending on the data size.');
|
|
43
|
+
|
|
44
|
+
let downloadUrl = null;
|
|
45
|
+
let attempts = 0;
|
|
46
|
+
const POLL_INTERVAL_MS = 15000;
|
|
47
|
+
const MAX_ATTEMPTS = 120; // 120 × 15s = 30 minutes, then give up rather than hang forever
|
|
48
|
+
|
|
49
|
+
// Poll until completion (bounded so a stuck job can never hang the CLI).
|
|
50
|
+
while (attempts < MAX_ATTEMPTS) {
|
|
51
|
+
attempts++;
|
|
52
|
+
// Wait between polls
|
|
53
|
+
await delay(POLL_INTERVAL_MS);
|
|
54
|
+
|
|
55
|
+
logInfo(`Checking status... (Attempt ${attempts}/${MAX_ATTEMPTS})`);
|
|
56
|
+
const statusRes = await apiClient.get(`/crm/v8/settings/audit_log_export/${jobId}`);
|
|
57
|
+
|
|
58
|
+
if (!statusRes.audit_log_export || statusRes.audit_log_export.length === 0) {
|
|
59
|
+
throw new Error('Invalid status response from server.');
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const statusDetails = statusRes.audit_log_export[0];
|
|
63
|
+
const jobStatus = (statusDetails.status || '').toLowerCase();
|
|
64
|
+
|
|
65
|
+
if (jobStatus === 'finished') {
|
|
66
|
+
if (!statusDetails.download_links || statusDetails.download_links.length === 0) {
|
|
67
|
+
throw new Error('Job finished but no download links were provided.');
|
|
68
|
+
}
|
|
69
|
+
downloadUrl = statusDetails.download_links[0];
|
|
70
|
+
logInfo(`Job finished! Download URL obtained.`);
|
|
71
|
+
break;
|
|
72
|
+
} else if (jobStatus === 'failed') {
|
|
73
|
+
throw new Error('Export job failed on the server side.');
|
|
74
|
+
} else {
|
|
75
|
+
// Status is 'progress' or 'scheduled'
|
|
76
|
+
logInfo(`Job status: ${jobStatus}. Waiting...`);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if (!downloadUrl) {
|
|
81
|
+
throw new Error(`Audit log export did not finish within ${Math.round(MAX_ATTEMPTS * POLL_INTERVAL_MS / 60000)} minutes (job ${jobId}). Try a narrower filter or re-run later.`);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Download the file
|
|
85
|
+
const outputDir = path.join(process.cwd(), 'storage', 'audit_logs');
|
|
86
|
+
await fs.ensureDir(outputDir);
|
|
87
|
+
|
|
88
|
+
// Determine the extension from the URL *path* (signed URLs carry a query
|
|
89
|
+
// string, so endsWith('.zip') on the whole URL would always miss). Default
|
|
90
|
+
// to .zip, the usual audit-export format.
|
|
91
|
+
let ext = '.zip';
|
|
92
|
+
try {
|
|
93
|
+
const urlPath = new URL(downloadUrl).pathname.toLowerCase();
|
|
94
|
+
const parsedExt = path.extname(urlPath);
|
|
95
|
+
if (parsedExt === '.csv' || parsedExt === '.zip') ext = parsedExt;
|
|
96
|
+
} catch (urlErr) {
|
|
97
|
+
// Keep the .zip default if the URL can't be parsed.
|
|
98
|
+
}
|
|
99
|
+
const outputPath = path.join(outputDir, `AuditLog_${jobId}${ext}`);
|
|
100
|
+
|
|
101
|
+
logInfo(`Downloading result to ${outputPath}...`);
|
|
102
|
+
|
|
103
|
+
const accessToken = await authService.getAccessToken();
|
|
104
|
+
const writer = fs.createWriteStream(outputPath);
|
|
105
|
+
|
|
106
|
+
const downloadResponse = await axios({
|
|
107
|
+
method: 'GET',
|
|
108
|
+
url: downloadUrl,
|
|
109
|
+
responseType: 'stream',
|
|
110
|
+
headers: {
|
|
111
|
+
'Authorization': `Zoho-oauthtoken ${accessToken}`
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
try {
|
|
116
|
+
await new Promise((resolve, reject) => {
|
|
117
|
+
downloadResponse.data.on('error', reject); // surface read-stream errors
|
|
118
|
+
writer.on('error', reject);
|
|
119
|
+
writer.on('finish', resolve);
|
|
120
|
+
downloadResponse.data.pipe(writer);
|
|
121
|
+
});
|
|
122
|
+
} catch (streamErr) {
|
|
123
|
+
// Don't leave a half-written file behind on failure.
|
|
124
|
+
writer.destroy();
|
|
125
|
+
await fs.remove(outputPath).catch(() => {});
|
|
126
|
+
throw streamErr;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
logInfo(`Audit log successfully downloaded and saved to: ${outputPath}`);
|
|
130
|
+
|
|
131
|
+
} catch (error) {
|
|
132
|
+
logError(error, 'auditLogService.exportAuditLog');
|
|
133
|
+
throw error;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
module.exports = {
|
|
138
|
+
exportAuditLog
|
|
139
|
+
};
|
|
Binary file
|
package/src/utils/apiClient.js
CHANGED
|
@@ -65,16 +65,19 @@ function setConcurrency(n) {
|
|
|
65
65
|
* @param {object} params - Optional query parameters
|
|
66
66
|
* @param {number} retryCount - Tracking retries internally
|
|
67
67
|
*/
|
|
68
|
-
async function request(endpoint, params = {}, retryCount = 0) {
|
|
68
|
+
async function request(method, endpoint, params = {}, data = null, retryCount = 0) {
|
|
69
69
|
try {
|
|
70
70
|
const accessToken = await authService.getAccessToken();
|
|
71
71
|
const baseUrl = authService.getApiBaseUrl();
|
|
72
72
|
const url = `${baseUrl}${endpoint}`;
|
|
73
73
|
|
|
74
|
-
logInfo(
|
|
74
|
+
logInfo(`${method.toUpperCase()} request to ${url} (Retry: ${retryCount})`, 'apiClient.request');
|
|
75
75
|
|
|
76
|
-
const response = await apiClient
|
|
76
|
+
const response = await apiClient({
|
|
77
|
+
method,
|
|
78
|
+
url,
|
|
77
79
|
params,
|
|
80
|
+
data,
|
|
78
81
|
headers: {
|
|
79
82
|
'Authorization': `Zoho-oauthtoken ${accessToken}`
|
|
80
83
|
}
|
|
@@ -94,10 +97,10 @@ async function request(endpoint, params = {}, retryCount = 0) {
|
|
|
94
97
|
|
|
95
98
|
// Handle 401 Unauthorized (Expired token or session)
|
|
96
99
|
if (status === 401 && retryCount < 2) {
|
|
97
|
-
logInfo('Received 401 Unauthorized. Forcing token refresh and retrying...', 'apiClient.
|
|
100
|
+
logInfo('Received 401 Unauthorized. Forcing token refresh and retrying...', 'apiClient.request');
|
|
98
101
|
try {
|
|
99
102
|
await authService.refreshAccessToken();
|
|
100
|
-
return await request(endpoint, params, retryCount + 1);
|
|
103
|
+
return await request(method, endpoint, params, data, retryCount + 1);
|
|
101
104
|
} catch (refreshErr) {
|
|
102
105
|
logError(refreshErr, 'apiClient.get - Token refresh retry failed');
|
|
103
106
|
throw error;
|
|
@@ -107,17 +110,17 @@ async function request(endpoint, params = {}, retryCount = 0) {
|
|
|
107
110
|
// Handle 429 Too Many Requests (Rate limit / concurrency limit hit)
|
|
108
111
|
if ((status === 429 || errorData?.code === 'TOO_MANY_REQUESTS') && retryCount < 5) {
|
|
109
112
|
const waitTime = Math.pow(2, retryCount) * 1000; // 1s, 2s, 4s, 8s, 16s backoff
|
|
110
|
-
logInfo(`Rate/concurrency limit hit (429). Retrying in ${waitTime}ms...`, 'apiClient.
|
|
113
|
+
logInfo(`Rate/concurrency limit hit (429). Retrying in ${waitTime}ms...`, 'apiClient.request');
|
|
111
114
|
await delay(waitTime);
|
|
112
|
-
return await request(endpoint, params, retryCount + 1);
|
|
115
|
+
return await request(method, endpoint, params, data, retryCount + 1);
|
|
113
116
|
}
|
|
114
117
|
|
|
115
118
|
// Handle temporary network or server failures (5xx)
|
|
116
119
|
if (status >= 500 && status < 600 && retryCount < 2) {
|
|
117
120
|
const waitTime = 1000;
|
|
118
|
-
logInfo(`Server error ${status}. Retrying in ${waitTime}ms...`, 'apiClient.
|
|
121
|
+
logInfo(`Server error ${status}. Retrying in ${waitTime}ms...`, 'apiClient.request');
|
|
119
122
|
await delay(waitTime);
|
|
120
|
-
return await request(endpoint, params, retryCount + 1);
|
|
123
|
+
return await request(method, endpoint, params, data, retryCount + 1);
|
|
121
124
|
}
|
|
122
125
|
|
|
123
126
|
throw error;
|
|
@@ -137,7 +140,16 @@ async function request(endpoint, params = {}, retryCount = 0) {
|
|
|
137
140
|
async function get(endpoint, params = {}) {
|
|
138
141
|
await acquireSlot();
|
|
139
142
|
try {
|
|
140
|
-
return await request(endpoint, params, 0);
|
|
143
|
+
return await request('get', endpoint, params, null, 0);
|
|
144
|
+
} finally {
|
|
145
|
+
releaseSlot();
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function post(endpoint, data = {}, params = {}) {
|
|
150
|
+
await acquireSlot();
|
|
151
|
+
try {
|
|
152
|
+
return await request('post', endpoint, params, data, 0);
|
|
141
153
|
} finally {
|
|
142
154
|
releaseSlot();
|
|
143
155
|
}
|
|
@@ -145,5 +157,6 @@ async function get(endpoint, params = {}) {
|
|
|
145
157
|
|
|
146
158
|
module.exports = {
|
|
147
159
|
get,
|
|
160
|
+
post,
|
|
148
161
|
setConcurrency
|
|
149
162
|
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Basic formatter for Zoho Deluge scripts.
|
|
3
|
+
* Adjusts indentation based on block scopes '{' and '}'.
|
|
4
|
+
* Note: This is a simple line-based formatter and does not fully parse the AST.
|
|
5
|
+
*/
|
|
6
|
+
function formatDeluge(code) {
|
|
7
|
+
if (!code || typeof code !== 'string') return code;
|
|
8
|
+
|
|
9
|
+
let formatted = '';
|
|
10
|
+
let indentLevel = 0;
|
|
11
|
+
const indentString = ' '; // 4 spaces
|
|
12
|
+
|
|
13
|
+
// Replace standard block formatting to ensure newlines before/after braces if it's on one line
|
|
14
|
+
// (We skip deep parsing, assuming the code from Zoho generally has basic newlines)
|
|
15
|
+
const lines = code.split('\n');
|
|
16
|
+
|
|
17
|
+
for (let line of lines) {
|
|
18
|
+
line = line.trim();
|
|
19
|
+
|
|
20
|
+
// Preserve empty lines
|
|
21
|
+
if (!line) {
|
|
22
|
+
formatted += '\n';
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Determine if this line closes a block
|
|
27
|
+
if (line.startsWith('}') || line.startsWith(']')) {
|
|
28
|
+
indentLevel = Math.max(0, indentLevel - 1);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Apply indentation
|
|
32
|
+
formatted += indentString.repeat(indentLevel) + line + '\n';
|
|
33
|
+
|
|
34
|
+
// Determine if this line opens a new block
|
|
35
|
+
if (line.endsWith('{') || line.endsWith('[')) {
|
|
36
|
+
indentLevel++;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return formatted.trim() + '\n';
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
module.exports = {
|
|
44
|
+
formatDeluge
|
|
45
|
+
};
|