sfmc-dataloader 2.0.2 → 2.1.0

package/README.md

@@ -101,16 +101,16 @@ Interactive: type `YES` when prompted. In CI, add `--i-accept-clear-data-risk` a
 
 ## Options
 
-| Option | Description |
-|--------|-------------|
-| `-p, --project` | Project root (default: cwd) |
-| `--format` | `csv` (default), `tsv`, or `json` |
-| `--git` | Stable export filenames: `<key>.mcdata.<ext>` (no timestamp segment) |
-| `--mode` | `upsert` (default) or `insert` — async bulk REST API only |
-| `--from <cred>/<bu>` | Export: source BU (repeatable). Import API mode: single source BU (use with `--to` and `--de`) |
-| `--to <cred>/<bu>` | Import: target BU (repeatable). API mode: use with `--from`/`--de`. File mode: use with `--file` (no `--from` needed) |
-| `--clear-before-import` | SOAP `ClearData` before REST import |
-| `--i-accept-clear-data-risk` | Non-interactive consent for clear |
+| Option                       | Description                                                                                                           |
+| ---------------------------- | --------------------------------------------------------------------------------------------------------------------- |
+| `-p, --project`              | Project root (default: cwd)                                                                                           |
+| `--format`                   | `csv` (default), `tsv`, or `json`                                                                                     |
+| `--git`                      | Stable export filenames: `<key>.mcdata.<ext>` (no timestamp segment)                                                  |
+| `--mode`                     | `upsert` (default) or `insert` — async bulk REST API only                                                             |
+| `--from <cred>/<bu>`         | Export: source BU (repeatable). Import API mode: single source BU (use with `--to` and `--de`)                        |
+| `--to <cred>/<bu>`           | Import: target BU (repeatable). API mode: use with `--from`/`--de`. File mode: use with `--file` (no `--from` needed) |
+| `--clear-before-import`      | SOAP `ClearData` before REST import                                                                                   |
+| `--i-accept-clear-data-risk` | Non-interactive consent for clear                                                                                     |
 
 Log lines use paths **relative** to the project root (POSIX-style, `./…`) and include **row counts** where applicable.
 

package/bin/mcdata.mjs

@@ -3,7 +3,7 @@ import { main } from '../lib/cli.mjs';
 
 main(process.argv)
     .then((code) => process.exit(typeof code === 'number' ? code : 0))
-    .catch((err) => {
-        console.error(err);
+    .catch((ex) => {
+        console.error(ex);
         process.exit(1);
     });

package/lib/clear-de.mjs

@@ -4,7 +4,7 @@
  *
  * @param {*} soap - SDK soap instance
  * @param {string} customerKey - DE external key
- * @returns {Promise<any>}
+ * @returns {Promise.<any>}
  */
 export async function clearDataExtensionRows(soap, customerKey) {
     return soap.perform('DataExtension', 'ClearData', {

package/lib/cli.mjs

@@ -9,6 +9,7 @@ import {
     parseCredBu,
     resolveCredentialAndMid,
     buildSdkAuthObject,
+    buildSdkOptions,
 } from './config.mjs';
 import { dataDirectoryForBu, projectRelativePosix } from './paths.mjs';
 import { exportDataExtensionToFile } from './export-de.mjs';
@@ -19,6 +20,7 @@ import { clearDataExtensionRows } from './clear-de.mjs';
 import { confirmClearBeforeImport } from './confirm-clear.mjs';
 import { multiBuExport } from './multi-bu-export.mjs';
 import { crossBuImport } from './cross-bu-import.mjs';
+import { initDebugLogger } from './debug-logger.mjs';
 
 /** @returns {string} semver from this package's package.json */
 function readCliPackageVersion() {
@@ -44,9 +46,10 @@ Usage:
 Options:
   --version               Print version and exit
   -p, --project <dir>     mcdev project root (default: cwd)
-  --format <csv|tsv|json> File format (default: csv)
+  --format <csv|tsv|json> Export file format (default: csv); ignored for imports
   --json-pretty           Pretty-print JSON on export
   --git                   Stable filenames: <key>.mcdata.<ext> (no timestamp)
+  --debug                 Write API requests/responses to ./logs/data/*.log
 
 Import options:
   --mode <upsert|insert>  Row write mode (default: upsert; async REST bulk API)
@@ -61,8 +64,9 @@ Multi-BU options:
 
 Notes:
   Exports are written under ./data/<credential>/<bu>/ using ".mcdata." in the filename.
-  Import with --de resolves the latest matching file in that folder (by mtime).
+  Import with --de resolves the latest matching file (csv/tsv/json) in that folder (by mtime).
   Import with --file parses the DE key from the basename (.mcdata. format).
+  Import format is auto-detected from file extension (.csv, .tsv, .json).
   Cross-BU import stores a download file in each target BU's data directory.
 
 Clear data warning:
@@ -73,7 +77,7 @@ Clear data warning:
 
 /**
  * @param {string[]} argv
- * @returns {Promise<number>} exit code
+ * @returns {Promise.<number>} exit code
  */
 export async function main(argv) {
     let values;
@@ -95,14 +99,15 @@ export async function main(argv) {
                 'clear-before-import': { type: 'boolean', default: false },
                 'i-accept-clear-data-risk': { type: 'boolean', default: false },
                 'json-pretty': { type: 'boolean', default: false },
+                debug: { type: 'boolean', default: false },
                 help: { type: 'boolean', short: 'h', default: false },
                 version: { type: 'boolean', default: false },
             },
         });
         values = parsed.values;
         positionals = parsed.positionals;
-    } catch (e) {
-        console.error(e.message);
+    } catch (ex) {
+        console.error(ex.message);
         printHelp();
         return 1;
     }
@@ -137,6 +142,15 @@ export async function main(argv) {
     const hasTo = toFlags.length > 0;
     const hasPositional = !!credBuRaw;
 
+    // Initialize debug logger if --debug flag is set
+    /** @type {import('./debug-logger.mjs').DebugLogger|null} */
+    const logger = values.debug
+        ? initDebugLogger(projectRoot, readCliPackageVersion(), argv)
+        : null;
+    if (logger) {
+        console.error(`Debug log: ${projectRelativePosix(projectRoot, logger.logPath)}`);
+    }
+
     // ── export ──────────────────────────────────────────────────────────────
     if (sub === 'export') {
         if (hasTo) {
@@ -144,19 +158,23 @@ export async function main(argv) {
             return 1;
         }
 
-        const des = [].concat(values.de ?? []);
+        const des = [values.de ?? []].flat();
         if (des.length === 0) {
             console.error('export requires at least one --de <customerKey>');
             return 1;
         }
 
         if (hasFrom && hasPositional) {
-            console.error('Cannot mix a positional <credential>/<bu> with --from. Use one or the other.');
+            console.error(
+                'Cannot mix a positional <credential>/<bu> with --from. Use one or the other.',
+            );
             return 1;
         }
 
         if (!hasFrom && !hasPositional) {
-            console.error('export requires either a positional <credential>/<bu> or at least one --from <cred>/<bu>.');
+            console.error(
+                'export requires either a positional <credential>/<bu> or at least one --from <cred>/<bu>.',
+            );
             printHelp();
             return 1;
         }
@@ -165,8 +183,8 @@ export async function main(argv) {
             let sources;
             try {
                 sources = fromFlags.map(parseCredBu);
-            } catch (e) {
-                console.error(e.message);
+            } catch (ex) {
+                console.error(ex.message);
                 return 1;
             }
             const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
@@ -179,6 +197,7 @@ export async function main(argv) {
                 format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
                 jsonPretty: values['json-pretty'],
                 useGit,
+                logger,
             });
             return 0;
         }
@@ -186,7 +205,7 @@ export async function main(argv) {
         const { credential, bu } = parseCredBu(credBuRaw);
         const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
         const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
-        const sdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
+        const sdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
         for (const deKey of des) {
             const { path: out, rowCount } = await exportDataExtensionToFile(sdk, {
                 projectRoot,
@@ -217,19 +236,23 @@ export async function main(argv) {
         // ── File-to-multi-BU import: --to + --file (no --from) ─────────────
         if (hasTo && !hasFrom && values.file?.length > 0) {
             if (hasPositional) {
-                console.error('Cannot mix a positional <credential>/<bu> with --to/--file. Use one or the other.');
+                console.error(
+                    'Cannot mix a positional <credential>/<bu> with --to/--file. Use one or the other.',
+                );
                 return 1;
             }
             if (values.de?.length > 0) {
-                console.error('Cannot mix --de with --file in multi-target import. Use --file only.');
+                console.error(
+                    'Cannot mix --de with --file in multi-target import. Use --file only.',
+                );
                 return 1;
             }
             const filePaths = values.file;
             let targets;
             try {
                 targets = toFlags.map(parseCredBu);
-            } catch (e) {
-                console.error(e.message);
+            } catch (ex) {
+                console.error(ex.message);
                 return 1;
             }
             const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
@@ -245,6 +268,7 @@ export async function main(argv) {
                 acceptRiskFlag: acceptRisk,
                 isTTY: process.stdin.isTTY === true,
                 useGit,
+                logger,
             });
             return 0;
         }
@@ -252,26 +276,36 @@ export async function main(argv) {
         // ── Cross-BU import (API mode): --from + --to + --de ────────────────
         if (hasFrom || hasTo) {
             if (hasPositional) {
-                console.error('Cannot mix a positional <credential>/<bu> with --from/--to. Use one or the other.');
+                console.error(
+                    'Cannot mix a positional <credential>/<bu> with --from/--to. Use one or the other.',
+                );
                 return 1;
             }
             if (!hasFrom) {
-                console.error('--to requires --from <cred>/<bu> to specify the source Business Unit.');
+                console.error(
+                    '--to requires --from <cred>/<bu> to specify the source Business Unit.',
+                );
                 return 1;
             }
             if (!hasTo) {
-                console.error('--from requires at least one --to <cred>/<bu> to specify target Business Unit(s).');
+                console.error(
+                    '--from requires at least one --to <cred>/<bu> to specify target Business Unit(s).',
+                );
                 return 1;
             }
             if (fromFlags.length > 1) {
-                console.error('import accepts exactly one --from <cred>/<bu> (use multiple --to for multiple targets).');
+                console.error(
+                    'import accepts exactly one --from <cred>/<bu> (use multiple --to for multiple targets).',
+                );
                 return 1;
             }
             if (values.file?.length > 0) {
-                console.error('--file cannot be combined with --from/--to/--de. For file-based multi-target import use --to + --file (without --from).');
+                console.error(
+                    '--file cannot be combined with --from/--to/--de. For file-based multi-target import use --to + --file (without --from).',
+                );
                 return 1;
             }
-            const deKeys = [].concat(values.de ?? []);
+            const deKeys = [values.de ?? []].flat();
             if (deKeys.length === 0) {
                 console.error('Cross-BU import requires at least one --de <customerKey>.');
                 return 1;
@@ -281,8 +315,8 @@ export async function main(argv) {
             try {
                 sourceParsed = parseCredBu(fromFlags[0]);
                 targets = toFlags.map(parseCredBu);
-            } catch (e) {
-                console.error(e.message);
+            } catch (ex) {
+                console.error(ex.message);
                 return 1;
             }
             const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
@@ -300,13 +334,16 @@ export async function main(argv) {
                 acceptRiskFlag: acceptRisk,
                 isTTY: process.stdin.isTTY === true,
                 useGit,
+                logger,
             });
             return 0;
         }
 
         // ── Single-BU import (original behavior) ────────────────────────────
         if (!hasPositional) {
-            console.error('import requires either a positional <credential>/<bu> or --from/--to flags.');
+            console.error(
+                'import requires either a positional <credential>/<bu> or --from/--to flags.',
+            );
             printHelp();
             return 1;
         }
@@ -314,17 +351,19 @@ export async function main(argv) {
         const hasDe = values.de?.length > 0;
         const hasFile = values.file?.length > 0;
         if (hasDe === hasFile) {
-            console.error('import requires exactly one of: repeated --de <key> OR repeated --file <path>');
+            console.error(
+                'import requires exactly one of: repeated --de <key> OR repeated --file <path>',
+            );
             return 1;
         }
 
         const { credential, bu } = parseCredBu(credBuRaw);
         const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
         const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
-        const sdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
+        const sdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
 
         if (hasDe) {
-            const deKeys = [].concat(values.de ?? []);
+            const deKeys = [values.de ?? []].flat();
             const dataDir = dataDirectoryForBu(projectRoot, credential, bu);
             if (clear) {
                 await confirmClearBeforeImport({
@@ -337,9 +376,11 @@ export async function main(argv) {
                 }
             }
             for (const deKey of deKeys) {
-                const candidates = await findImportCandidates(dataDir, deKey, fmt);
+                const candidates = await findImportCandidates(dataDir, deKey);
                 if (candidates.length === 0) {
-                    console.error(`No ${fmt} file found for DE "${deKey}" under ${projectRelativePosix(projectRoot, dataDir)}`);
+                    console.error(
+                        `No import file (csv/tsv/json) found for DE "${deKey}" under ${projectRelativePosix(projectRoot, dataDir)}`,
+                    );
                     return 1;
                 }
                 const filePath =
@@ -347,7 +388,6 @@ export async function main(argv) {
                 const n = await importFromFile(sdk, {
                     filePath,
                     deKey,
-                    format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
                     mode: /** @type {'upsert'|'insert'} */ (mode),
                 });
                 const rel = projectRelativePosix(projectRoot, filePath);
@@ -357,7 +397,9 @@ export async function main(argv) {
         }
 
         const fileList = values.file ?? [];
-        const keysFromFiles = fileList.map((fp) => parseExportBasename(path.basename(fp)).customerKey);
+        const keysFromFiles = fileList.map(
+            (fp) => parseExportBasename(path.basename(fp)).customerKey,
+        );
         if (clear) {
             await confirmClearBeforeImport({
                 deKeys: keysFromFiles,
@@ -374,7 +416,6 @@ export async function main(argv) {
             const n = await importFromFile(sdk, {
                 filePath,
                 deKey: customerKey,
-                format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
                 mode: /** @type {'upsert'|'insert'} */ (mode),
             });
             const rel = projectRelativePosix(projectRoot, filePath);

package/lib/config.mjs

@@ -67,20 +67,22 @@ export function resolveCredentialAndMid(mcdevrc, mcdevAuth, credentialName, buNa
     if (midRaw === undefined || midRaw === null) {
         throw new Error(`Unknown business unit "${buName}" under credential "${credentialName}"`);
     }
-    const mid =
-        typeof midRaw === 'number' ? midRaw : Number.parseInt(String(midRaw), 10);
+    const mid = typeof midRaw === 'number' ? midRaw : Number.parseInt(String(midRaw), 10);
     if (!Number.isInteger(mid)) {
-        throw new Error(`Invalid MID for ${credentialName}/${buName}: ${midRaw}`);
+        throw new TypeError(`Invalid MID for ${credentialName}/${buName}: ${midRaw}`);
     }
     const authCred = mcdevAuth[credentialName];
     if (!authCred?.client_id || !authCred?.client_secret || !authCred?.auth_url) {
-        throw new Error(`Missing auth fields for credential "${credentialName}" in .mcdev-auth.json`);
+        throw new Error(
+            `Missing auth fields for credential "${credentialName}" in .mcdev-auth.json`,
+        );
     }
     return { mid, authCred };
 }
 
 /**
  * Auth object for sfmc-sdk `Auth` / `SDK` constructor.
+ *
  * @param {AuthCredential} authCred
  * @param {number} mid
  * @returns {import('sfmc-sdk').AuthObject}
@@ -93,3 +95,48 @@ export function buildSdkAuthObject(authCred, mid) {
         account_id: mid,
     };
 }
+
+/**
+ * @typedef {object} DebugLogger
+ * @property {string} logPath - Absolute path to the log file
+ * @property {(text: string) => void} write - Append a line to the log file
+ */
+
+/**
+ * Options object for sfmc-sdk `SDK` constructor.
+ * When a logger is provided, includes event handlers to log API requests/responses to file.
+ *
+ * @param {DebugLogger|null} [logger] - Debug logger object, or null/undefined to disable logging
+ * @returns {import('sfmc-sdk').SdkOptions}
+ */
+export function buildSdkOptions(logger = null) {
+    /** @type {import('sfmc-sdk').SdkOptions} */
+    const options = { requestAttempts: 3 };
+    if (logger) {
+        options.eventHandlers = {
+            logRequest: (req) => {
+                const msg = structuredClone(req);
+                if (msg.headers?.Authorization) {
+                    msg.headers.Authorization = 'Bearer *** TOKEN REMOVED ***';
+                }
+                logger.write(`API REQUEST >> ${msg.method?.toUpperCase() || 'GET'} ${msg.url}`);
+                if (msg.data) {
+                    const body =
+                        typeof msg.data === 'string' ? msg.data : JSON.stringify(msg.data, null, 2);
+                    logger.write(`REQUEST BODY >> ${body}`);
+                }
+            },
+            logResponse: (res) => {
+                logger.write(`API RESPONSE << ${res.status || res.statusCode || '(no status)'}`);
+                const body =
+                    typeof res.data === 'string' ? res.data : JSON.stringify(res.data, null, 2);
+                const indentedBody = body
+                    .split('\n')
+                    .map((line) => '  ' + line)
+                    .join('\n');
+                logger.write(`RESPONSE BODY <<\n${indentedBody}`);
+            },
+        };
+    }
+    return options;
+}

package/lib/confirm-clear.mjs

@@ -10,10 +10,10 @@ import { stdin as input, stdout as output } from 'node:process';
  * @param {string[]} opts.deKeys
  * @param {boolean} opts.acceptRiskFlag
  * @param {boolean} opts.isTTY
- * @param {CredBuTarget[]} [opts.targets]  When present, renders a per-BU breakdown.
+ * @param {CredBuTarget[]} [opts.targets] When present, renders a per-BU breakdown.
  * @param {NodeJS.ReadableStream} [opts.stdin]
  * @param {NodeJS.WritableStream} [opts.stdout]
- * @returns {Promise<void>}
+ * @returns {Promise.<void>}
  */
 export async function confirmClearBeforeImport(opts) {
     const { deKeys, targets, acceptRiskFlag, isTTY } = opts;
@@ -25,30 +25,26 @@ export async function confirmClearBeforeImport(opts) {
     if (!isTTY) {
         throw new Error(
             'Refusing to clear data in non-interactive mode without --i-accept-clear-data-risk. ' +
-                'All rows in the target Data Extension(s) would be permanently deleted.'
+                'All rows in the target Data Extension(s) would be permanently deleted.',
         );
     }
-    let msg;
-    if (targets && targets.length > 0) {
-        msg =
-            '\n*** DANGER: CLEAR DATA ***\n' +
-            `This will permanently DELETE ALL ROWS across ${targets.length} Business Unit(s):\n\n` +
-            targets
-                .map(
-                    ({ credential, bu }) =>
-                        `  ${credential}/${bu}:\n` + deKeys.map((k) => `    - ${k}\n`).join('')
-                )
-                .join('') +
-            '\nThis cannot be undone. Enterprise 2.0 / admin / shared-DE rules may apply.\n' +
-            'Type YES to continue, anything else to abort: ';
-    } else {
-        msg =
-            '\n*** DANGER: CLEAR DATA ***\n' +
-            'This will permanently DELETE ALL ROWS in:\n' +
-            deKeys.map((k) => `  - ${k}\n`).join('') +
-            'This cannot be undone. Enterprise 2.0 / admin / shared-DE rules may apply.\n' +
-            'Type YES to continue, anything else to abort: ';
-    }
+    const msg =
+        targets && targets.length > 0
+            ? '\n*** DANGER: CLEAR DATA ***\n' +
+              `This will permanently DELETE ALL ROWS across ${targets.length} Business Unit(s):\n\n` +
+              targets
+                  .map(
+                      ({ credential, bu }) =>
+                          `  ${credential}/${bu}:\n` + deKeys.map((k) => `    - ${k}\n`).join(''),
+                  )
+                  .join('') +
+              '\nThis cannot be undone. Enterprise 2.0 / admin / shared-DE rules may apply.\n' +
+              'Type YES to continue, anything else to abort: '
+            : '\n*** DANGER: CLEAR DATA ***\n' +
+              'This will permanently DELETE ALL ROWS in:\n' +
+              deKeys.map((k) => `  - ${k}\n`).join('') +
+              'This cannot be undone. Enterprise 2.0 / admin / shared-DE rules may apply.\n' +
+              'Type YES to continue, anything else to abort: ';
     stdout.write(msg);
     const rl = readline.createInterface({ input: stdin, output: stdout });
     try {

package/lib/cross-bu-import.mjs

@@ -3,8 +3,9 @@ import path from 'node:path';
 import readline from 'node:readline/promises';
 import { stdin as input, stdout as output } from 'node:process';
 import SDK from 'sfmc-sdk';
-import { resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
+import { resolveCredentialAndMid, buildSdkAuthObject, buildSdkOptions } from './config.mjs';
 import { fetchAllRowObjects, serializeRows, exportDataExtensionToFile } from './export-de.mjs';
+import { formatFromExtension } from './file-resolve.mjs';
 import { importRowsForDe } from './import-de.mjs';
 import { readRowsFromFile } from './read-rows.mjs';
 import { clearDataExtensionRows } from './clear-de.mjs';
@@ -25,7 +26,7 @@ import { buildExportBasename, filesystemSafeTimestamp, parseExportBasename } fro
  * @param {string[]} opts.deKeys
  * @param {NodeJS.ReadableStream} [opts.stdin]
  * @param {NodeJS.WritableStream} [opts.stdout]
- * @returns {Promise<boolean>}
+ * @returns {Promise.<boolean>}
  */
 async function offerPreExportBackup({ targets, deKeys, stdin: stdinStream, stdout: stdoutStream }) {
     const stdinSrc = stdinStream ?? input;
@@ -66,8 +67,8 @@ async function offerPreExportBackup({ targets, deKeys, stdin: stdinStream, stdou
  * @param {import('./config.mjs').Mcdevrc} params.mcdevrc
  * @param {Record<string, import('./config.mjs').AuthCredential>} params.mcdevAuth
  * @param {string} [params.sourceCred] - API mode only
- * @param {string} [params.sourceBu]   - API mode only
- * @param {string[]} [params.deKeys]   - API mode only
+ * @param {string} [params.sourceBu] - API mode only
+ * @param {string[]} [params.deKeys] - API mode only
  * @param {string[]} [params.filePaths] - File mode only; mutually exclusive with sourceCred/sourceBu/deKeys
  * @param {CredBuTarget[]} params.targets
  * @param {'csv'|'tsv'|'json'} params.format
@@ -76,9 +77,10 @@ async function offerPreExportBackup({ targets, deKeys, stdin: stdinStream, stdou
  * @param {boolean} params.acceptRiskFlag
  * @param {boolean} params.isTTY
  * @param {boolean} [params.useGit] - stable snapshot basename (no timestamp)
- * @param {NodeJS.ReadableStream} [params.stdin]  Override for testing
+ * @param {import('./config.mjs').DebugLogger|null} [params.logger] - debug logger for API requests/responses
+ * @param {NodeJS.ReadableStream} [params.stdin] Override for testing
  * @param {NodeJS.WritableStream} [params.stdout] Override for testing
- * @returns {Promise<void>}
+ * @returns {Promise.<void>}
  */
 export async function crossBuImport(params) {
     const {
@@ -92,6 +94,7 @@ export async function crossBuImport(params) {
         acceptRiskFlag,
         isTTY,
         useGit = false,
+        logger = null,
     } = params;
     const stdin = params.stdin;
     const stdout = params.stdout;
@@ -123,9 +126,12 @@ export async function crossBuImport(params) {
     let srcSdk = null;
     if (!isFileBased) {
         const { mid: srcMid, authCred: srcAuth } = resolveCredentialAndMid(
-            mcdevrc, mcdevAuth, params.sourceCred, params.sourceBu
+            mcdevrc,
+            mcdevAuth,
+            params.sourceCred,
+            params.sourceBu,
         );
-        srcSdk = new SDK(buildSdkAuthObject(srcAuth, srcMid), { requestAttempts: 3 });
+        srcSdk = new SDK(buildSdkAuthObject(srcAuth, srcMid), buildSdkOptions(logger));
     }
 
     // Optional pre-import backup of target BU data
@@ -133,8 +139,13 @@ export async function crossBuImport(params) {
         const doBackup = await offerPreExportBackup({ targets, deKeys, stdin, stdout });
         if (doBackup) {
             for (const { credential, bu } of targets) {
-                const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
-                const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
+                const { mid, authCred } = resolveCredentialAndMid(
+                    mcdevrc,
+                    mcdevAuth,
+                    credential,
+                    bu,
+                );
+                const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
                 for (const deKey of deKeys) {
                     const { path: outPath, rowCount } = await exportDataExtensionToFile(tgtSdk, {
                         projectRoot,
@@ -158,22 +169,37 @@ export async function crossBuImport(params) {
 
     // Load rows once per DE then fan out to every target
     for (const deKey of deKeys) {
-        const rows = isFileBased
-            ? await readRowsFromFile(fileByDeKey.get(deKey), format)
-            : await fetchAllRowObjects(srcSdk, deKey);
+        let rows;
+        if (isFileBased) {
+            const filePath = fileByDeKey.get(deKey);
+            const detectedFormat = formatFromExtension(filePath);
+            if (!detectedFormat) {
+                throw new Error(
+                    `Cannot determine format for file: ${filePath}. Use .csv, .tsv, or .json extension.`,
+                );
+            }
+            rows = await readRowsFromFile(filePath, detectedFormat);
+        } else {
+            rows = await fetchAllRowObjects(srcSdk, deKey);
+        }
 
         // Clear targets before import (rows already confirmed above)
         if (clearBeforeImport) {
             for (const { credential, bu } of targets) {
-                const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
-                const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
+                const { mid, authCred } = resolveCredentialAndMid(
+                    mcdevrc,
+                    mcdevAuth,
+                    credential,
+                    bu,
+                );
+                const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
                 await clearDataExtensionRows(tgtSdk.soap, deKey);
             }
         }
 
         for (const { credential, bu } of targets) {
             const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
-            const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
+            const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
 
             // Write a snapshot file in the target BU's data directory.
             const dir = dataDirectoryForBu(projectRoot, credential, bu);

package/lib/debug-logger.mjs

@@ -0,0 +1,42 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * @typedef {object} DebugLogger
+ * @property {string} logPath - Absolute path to the log file
+ * @property {(text: string) => void} write - Append a line to the log file
+ */
+
+/**
+ * Initialize a debug logger that writes API interactions to a timestamped log file.
+ *
+ * @param {string} projectRoot - mcdev project root directory
+ * @param {string} version - mcdata version string
+ * @param {string[]} argv - Full process.argv array
+ * @returns {DebugLogger}
+ */
+export function initDebugLogger(projectRoot, version, argv) {
+    const logsDir = path.join(projectRoot, 'logs', 'data');
+    fs.mkdirSync(logsDir, { recursive: true });
+
+    // Timestamp with dots instead of colons for Windows filesystem compatibility
+    const ts = new Date().toISOString().replaceAll(':', '.');
+    const logPath = path.join(logsDir, `${ts}.log`);
+
+    // Reconstruct command line for header, quoting args with spaces
+    const command =
+        'mcdata ' +
+        argv
+            .slice(2)
+            .map((arg) => (arg.includes(' ') ? `"${arg}"` : arg))
+            .join(' ');
+
+    // Write header
+    const header = `mcdata v${version}\nRan command: ${command}\n---\n`;
+    fs.writeFileSync(logPath, header, 'utf8');
+
+    return {
+        logPath,
+        write: (text) => fs.appendFileSync(logPath, text + '\n', 'utf8'),
+    };
+}

package/lib/export-de.mjs

@@ -6,13 +6,22 @@ import { buildExportBasename, filesystemSafeTimestamp } from './filename.mjs';
 import { dataDirectoryForBu } from './paths.mjs';
 
 /**
- * @param {{ rest: { getBulk: (path: string, pageSize?: number) => Promise<any> } }} sdk
+ * @param {{rest: {getBulk: (path: string, pageSize?: number) => Promise.<any>}}} sdk
  * @param {string} deKey
- * @returns {Promise<object[]>}
+ * @returns {Promise.<object[]>}
  */
 export async function fetchAllRowObjects(sdk, deKey) {
     const basePath = rowsetGetPath(deKey);
-    const data = await sdk.rest.getBulk(basePath, 2500);
+    let data;
+    try {
+        data = await sdk.rest.getBulk(basePath, 2500);
+    } catch (ex) {
+        // this api endpoint won't return "items" if the dataExtension is empty
+        if (ex.message !== 'Could not find an array to iterate over') {
+            throw ex;
+        }
+        data = { items: [] };
+    }
     const items = data.items ?? [];
     const rows = [];
     for (const item of items) {
@@ -35,14 +44,14 @@ export function serializeRows(rows, format, jsonPretty) {
     const delimiter = format === 'tsv' ? '\t' : ',';
     return stringify(rows, {
         header: true,
-        quoted: true,
+        quoted: format === 'csv',
         bom: true,
         delimiter,
     });
 }
 
 /**
- * @param {{ rest: { getBulk: (path: string, pageSize?: number) => Promise<any> } }} sdk
+ * @param {{rest: {getBulk: (path: string, pageSize?: number) => Promise.<any>}}} sdk
  * @param {object} params
  * @param {string} params.projectRoot
  * @param {string} params.credentialName
@@ -51,10 +60,18 @@ export function serializeRows(rows, format, jsonPretty) {
  * @param {'csv'|'tsv'|'json'} params.format
  * @param {boolean} [params.jsonPretty]
  * @param {boolean} [params.useGit]
- * @returns {Promise<{ path: string, rowCount: number }>}
+ * @returns {Promise.<{path: string, rowCount: number}>}
  */
 export async function exportDataExtensionToFile(sdk, params) {
-    const { projectRoot, credentialName, buName, deKey, format, jsonPretty = false, useGit = false } = params;
+    const {
+        projectRoot,
+        credentialName,
+        buName,
+        deKey,
+        format,
+        jsonPretty = false,
+        useGit = false,
+    } = params;
     const rows = await fetchAllRowObjects(sdk, deKey);
     const dir = dataDirectoryForBu(projectRoot, credentialName, buName);
     await fs.mkdir(dir, { recursive: true });

package/lib/file-resolve.mjs

@@ -2,13 +2,37 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 import { parseExportBasename } from './filename.mjs';
 
+/** Supported import/export file extensions */
+const SUPPORTED_EXTENSIONS = ['csv', 'tsv', 'json'];
+
+/**
+ * Derive format from file extension.
+ *
+ * @param {string} filePath
+ * @returns {'csv'|'tsv'|'json'|null}
+ */
+export function formatFromExtension(filePath) {
+    const ext = path.extname(filePath).toLowerCase().slice(1);
+    if (ext === 'csv') {
+        return 'csv';
+    }
+    if (ext === 'tsv') {
+        return 'tsv';
+    }
+    if (ext === 'json') {
+        return 'json';
+    }
+    return null;
+}
+
 /**
  * Find export files under data dir matching the DE customer key and extension.
+ * When format is omitted, searches all supported extensions (csv, tsv, json).
  *
  * @param {string} dataDir
  * @param {string} customerKey
- * @param {'csv'|'tsv'|'json'} format
- * @returns {Promise<string[]>} full paths
+ * @param {'csv'|'tsv'|'json'} [format] - optional; if omitted, searches all extensions
+ * @returns {Promise.<string[]>} full paths
  */
 export async function findImportCandidates(dataDir, customerKey, format) {
     let entries;
@@ -17,14 +41,15 @@ export async function findImportCandidates(dataDir, customerKey, format) {
     } catch {
         return [];
     }
-    const ext = format;
+    const extensions = format ? [format] : SUPPORTED_EXTENSIONS;
     const matches = [];
     for (const ent of entries) {
         if (!ent.isFile()) {
             continue;
         }
         const name = ent.name;
-        if (!name.endsWith(`.${ext}`)) {
+        const fileExt = path.extname(name).toLowerCase().slice(1);
+        if (!extensions.includes(fileExt)) {
             continue;
         }
         try {
@@ -41,7 +66,7 @@ export async function findImportCandidates(dataDir, customerKey, format) {
 
 /**
  * @param {string[]} paths
- * @returns {Promise<string>} path with newest mtime
+ * @returns {Promise.<string>} path with newest mtime
  */
 export async function pickLatestByMtime(paths) {
     if (paths.length === 0) {

package/lib/filename.mjs

@@ -1,6 +1,7 @@
 /**
  * Mirrors sfmc-devtools `File.filterIllegalFilenames` / `reverseFilterIllegalFilenames`
  * so export filenames stay consistent with mcdev retrieve-style paths.
+ *
  * @see https://github.com/Accenture/sfmc-devtools (lib/util/file.js)
  */
 
@@ -12,22 +13,20 @@ export const MCDATA_SEGMENT = '.mcdata.';
  * @returns {string}
  */
 export function filterIllegalFilenames(filename) {
-    return (
-        encodeURIComponent(filename)
-            .replaceAll(/[*]/g, '_STAR_')
-            .split('%20')
-            .join(' ')
-            .split('%7B')
-            .join('{')
-            .split('%7D')
-            .join('}')
-            .split('%5B')
-            .join('[')
-            .split('%5D')
-            .join(']')
-            .split('%40')
-            .join('@')
-    );
+    return encodeURIComponent(filename)
+        .replaceAll(/[*]/g, '_STAR_')
+        .split('%20')
+        .join(' ')
+        .split('%7B')
+        .join('{')
+        .split('%7D')
+        .join('}')
+        .split('%5B')
+        .join('[')
+        .split('%5D')
+        .join(']')
+        .split('%40')
+        .join('@');
 }
 
 /**
@@ -90,5 +89,7 @@ export function parseExportBasename(basename) {
         };
     }
 
-    throw new Error(`Filename must contain ".mcdata." or end with ".mcdata" before the extension: ${basename}`);
+    throw new Error(
+        `Filename must contain ".mcdata." or end with ".mcdata" before the extension: ${basename}`,
+    );
 }

package/lib/import-de.mjs

@@ -1,4 +1,5 @@
 import { chunkItemsForPayload } from './batch.mjs';
+import { formatFromExtension } from './file-resolve.mjs';
 import { resolveImportRoute } from './import-routes.mjs';
 import { withRetry429 } from './retry.mjs';
 import { readRowsFromFile } from './read-rows.mjs';
@@ -9,7 +10,7 @@ import { readRowsFromFile } from './read-rows.mjs';
  * @param {string} params.deKey
  * @param {object[]} params.rows
  * @param {'upsert'|'insert'} params.mode
- * @returns {Promise<number>} number of rows imported
+ * @returns {Promise.<number>} number of rows imported
  */
 export async function importRowsForDe(sdk, params) {
     const { deKey, rows, mode } = params;
@@ -19,9 +20,7 @@ export async function importRowsForDe(sdk, params) {
         const p = route.path(deKey);
         const body = { items: chunk };
         await withRetry429(() =>
-            route.method === 'PUT'
-                ? sdk.rest.put(p, body)
-                : sdk.rest.post(p, body)
+            route.method === 'PUT' ? sdk.rest.put(p, body) : sdk.rest.post(p, body),
         );
     }
     return rows.length;
@@ -32,12 +31,18 @@ export async function importRowsForDe(sdk, params) {
  * @param {object} params
  * @param {string} params.filePath
  * @param {string} params.deKey - target DE customer key for API
- * @param {'csv'|'tsv'|'json'} params.format
+ * @param {'csv'|'tsv'|'json'} [params.format] - optional; auto-detected from file extension if omitted
  * @param {'upsert'|'insert'} params.mode
- * @returns {Promise<number>} number of rows imported
+ * @returns {Promise.<number>} number of rows imported
  */
 export async function importFromFile(sdk, params) {
-    const rows = await readRowsFromFile(params.filePath, params.format);
+    const format = params.format || formatFromExtension(params.filePath);
+    if (!format) {
+        throw new Error(
+            `Cannot determine format for file: ${params.filePath}. Use .csv, .tsv, or .json extension.`,
+        );
+    }
+    const rows = await readRowsFromFile(params.filePath, format);
     return importRowsForDe(sdk, {
         deKey: params.deKey,
         rows,

package/lib/import-routes.mjs

@@ -13,6 +13,7 @@ export function rowsetGetPath(deKey) {
 
 /**
  * Async bulk row writes: POST insert, PUT upsert (same URL).
+ *
  * @param {string} deKey
  * @returns {string}
  */

package/lib/index.mjs

@@ -1,8 +1,17 @@
 export { main } from './cli.mjs';
-export { filterIllegalFilenames, reverseFilterIllegalFilenames, parseExportBasename } from './filename.mjs';
+export {
+    filterIllegalFilenames,
+    reverseFilterIllegalFilenames,
+    parseExportBasename,
+} from './filename.mjs';
 export { chunkItemsForPayload, DEFAULT_MAX_BODY_BYTES, MAX_OBJECTS_PER_BATCH } from './batch.mjs';
 export { resolveImportRoute, rowsetGetPath, asyncDataExtensionRowsPath } from './import-routes.mjs';
-export { loadMcdevProject, parseCredBu, resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
+export {
+    loadMcdevProject,
+    parseCredBu,
+    resolveCredentialAndMid,
+    buildSdkAuthObject,
+} from './config.mjs';
 export { multiBuExport } from './multi-bu-export.mjs';
 export { crossBuImport } from './cross-bu-import.mjs';
 export { projectRelativePosix } from './paths.mjs';

package/lib/multi-bu-export.mjs

@@ -1,5 +1,5 @@
 import SDK from 'sfmc-sdk';
-import { resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
+import { resolveCredentialAndMid, buildSdkAuthObject, buildSdkOptions } from './config.mjs';
 import { exportDataExtensionToFile } from './export-de.mjs';
 import { projectRelativePosix } from './paths.mjs';
 
@@ -21,14 +21,25 @@ import { projectRelativePosix } from './paths.mjs';
  * @param {'csv'|'tsv'|'json'} params.format
  * @param {boolean} [params.jsonPretty]
  * @param {boolean} [params.useGit]
- * @returns {Promise<string[]>} Paths of all written files
+ * @param {import('./config.mjs').DebugLogger|null} [params.logger]
+ * @returns {Promise.<string[]>} Paths of all written files
  */
-export async function multiBuExport({ projectRoot, mcdevrc, mcdevAuth, sources, deKeys, format, jsonPretty = false, useGit = false }) {
+export async function multiBuExport({
+    projectRoot,
+    mcdevrc,
+    mcdevAuth,
+    sources,
+    deKeys,
+    format,
+    jsonPretty = false,
+    useGit = false,
+    logger = null,
+}) {
     /** @type {string[]} */
     const exported = [];
     for (const { credential, bu } of sources) {
         const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
-        const sdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
+        const sdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
         for (const deKey of deKeys) {
             const { path: outPath, rowCount } = await exportDataExtensionToFile(sdk, {
                 projectRoot,

package/lib/read-rows.mjs

@@ -4,7 +4,7 @@ import csv from 'csv-parser';
 /**
  * @param {string} filePath
  * @param {'csv'|'tsv'|'json'} format
- * @returns {Promise<object[]>}
+ * @returns {Promise.<object[]>}
  */
 export async function readRowsFromFile(filePath, format) {
     if (format === 'json') {
@@ -26,7 +26,26 @@ export async function readRowsFromFile(filePath, format) {
                 csv({
                     separator: delimiter,
                     bom: true,
-                })
+                    mapHeaders: ({ header }) => {
+                        let h = header;
+                        // Strip BOM if present (backup in case bom:true misses it)
+                        if (h.codePointAt(0) === 0xFEFF) {
+                            h = h.slice(1);
+                        }
+                        // Strip surrounding quotes if present (non-standard quoted TSV)
+                        if (h.startsWith('"') && h.endsWith('"') && h.length >= 2) {
+                            h = h.slice(1, -1);
+                        }
+                        return h;
+                    },
+                    mapValues: ({ value }) => {
+                        // Strip surrounding quotes from values if present
+                        if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
+                            return value.slice(1, -1);
+                        }
+                        return value;
+                    },
+                }),
             )
             .on('data', (row) => rows.push(row))
             .on('end', () => resolve(rows))

package/lib/retry.mjs

@@ -1,10 +1,10 @@
 import { RestError } from 'sfmc-sdk/util';
 
 /**
- * @param {() => Promise<any>} fn
+ * @param {() => Promise.<any>} fn
  * @param {object} [opts]
  * @param {number} [opts.maxAttempts] default 5
- * @returns {Promise<any>}
+ * @returns {Promise.<any>}
  */
 export async function withRetry429(fn, opts = {}) {
     const maxAttempts = opts.maxAttempts ?? 5;
@@ -14,26 +14,27 @@ export async function withRetry429(fn, opts = {}) {
         attempt++;
         try {
             return await fn();
-        } catch (e) {
-            const status = e instanceof RestError ? e.response?.status : undefined;
-            const retryAfter = e instanceof RestError ? e.response?.headers?.['retry-after'] : undefined;
+        } catch (ex) {
+            const status = ex instanceof RestError ? ex.response?.status : undefined;
+            const retryAfter =
+                ex instanceof RestError ? ex.response?.headers?.['retry-after'] : undefined;
             if (status === 429 && attempt < maxAttempts) {
                 const wait =
-                    retryAfter !== undefined
-                        ? Number.parseInt(String(retryAfter), 10) * 1000 || delayMs
-                        : delayMs;
+                    retryAfter === undefined
+                        ? delayMs
+                        : Number.parseInt(String(retryAfter), 10) * 1000 || delayMs;
                 await sleep(wait);
                 delayMs = Math.min(delayMs * 2, 60_000);
                 continue;
             }
-            throw e;
+            throw ex;
         }
     }
 }
 
 /**
  * @param {number} ms
- * @returns {Promise<void>}
+ * @returns {Promise.<void>}
  */
 function sleep(ms) {
     return new Promise((r) => setTimeout(r, ms));

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sfmc-dataloader",
-    "version": "2.0.2",
+    "version": "2.1.0",
     "description": "SFMC Data Loader CLI (mcdata) to export and import Marketing Cloud Data Extension rows using mcdev project config and sfmc-sdk",
     "author": "Jörn Berkefeld <joern.berkefeld@gmail.com>",
     "license": "MIT",
@@ -44,6 +44,20 @@
         "sfmc-sdk": "3.0.3"
     },
     "scripts": {
-        "test": "node --test test/**/*.test.js"
+        "test": "node --test test/**/*.test.js",
+        "lint": "eslint .",
+        "lint:fix": "eslint --fix .",
+        "format": "prettier --write .",
+        "format:check": "prettier --check ."
+    },
+    "devDependencies": {
+        "@eslint/js": "^10.0.1",
+        "eslint": "^10.1.0",
+        "eslint-config-prettier": "^10.1.8",
+        "eslint-plugin-jsdoc": "^62.0.0",
+        "eslint-plugin-prettier": "^5.5.0",
+        "eslint-plugin-unicorn": "^64.0.0",
+        "globals": "^17.4.0",
+        "prettier": "^3.8.1"
     }
 }