sfmc-dataloader 2.2.0 → 2.4.0

package/README.md

@@ -44,7 +44,7 @@ Creates one file per BU/DE combination using the same `.mcdata.` naming rules.
 mcdata import MyCred/MyBU --de MyDE_CustomerKey --mode upsert
 ```
 
-Imports use the **asynchronous** bulk row API only: `POST` for `--mode insert`, `PUT` for `--mode upsert` (same endpoint path).
+Imports use the **asynchronous** bulk row API only: `POST` for `--mode insert`, `PUT` for `--mode upsert` (same endpoint path). After uploading, `mcdata` polls the async status endpoint every 5 seconds until the job reaches `Complete` or `Error`. On `Error`, per-row error messages are printed to the log so you can see exactly which records failed. The process exits with code 1 if any import job fails, which also surfaces as an error notification in the VS Code extension.
 
 Resolves the latest matching export file under `./data/MyCred/MyBU/` for that DE key. The file format is detected automatically from the file extension (`.csv`, `.tsv`, `.json`).
 

package/lib/async-status.mjs

@@ -0,0 +1,89 @@
+import { asyncRequestResultsPath, asyncRequestStatusPath } from './import-routes.mjs';
+
+const POLL_INTERVAL_MS = 5000;
+const PENDING_STATUSES = new Set(['Pending', 'Executing']);
+
+/**
+ * @param {number} ms
+ * @returns {Promise.<void>}
+ */
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Polls the async import status API for each requestId until the job reaches
+ * a terminal state ("Complete" or "Error").  On error, fetches per-row results
+ * and logs each item message so the user can see what failed.
+ *
+ * @param {{ rest: { get: Function } }} sdk
+ * @param {(string|null|undefined)[]} requestIds - one per imported chunk
+ * @returns {Promise.<boolean>} true if at least one chunk job returned "Error"
+ */
+export async function pollAsyncImportCompletion(sdk, requestIds) {
+    let hasError = false;
+
+    for (const requestId of requestIds) {
+        if (!requestId) {
+            console.error('Async import: no requestId returned for chunk — skipping status check.');
+            continue;
+        }
+
+        console.error(`Waiting for async import to complete (requestId: ${requestId})…`);
+        await sleep(POLL_INTERVAL_MS);
+
+        let requestStatus;
+        do {
+            let statusResult;
+            try {
+                statusResult = await sdk.rest.get(asyncRequestStatusPath(requestId));
+            } catch (ex) {
+                console.error(
+                    `Async import: status check failed for requestId ${requestId}: ${ex.message}`,
+                );
+                break;
+            }
+
+            requestStatus = statusResult?.status?.requestStatus;
+
+            if (PENDING_STATUSES.has(requestStatus)) {
+                console.error(
+                    `Async import still in progress (requestId: ${requestId}) — retrying in 5 s…`,
+                );
+                await sleep(POLL_INTERVAL_MS);
+            }
+        } while (PENDING_STATUSES.has(requestStatus));
+
+        if (requestStatus === 'Complete') {
+            console.error(`Async import completed successfully (requestId: ${requestId}).`);
+        } else if (requestStatus === 'Error') {
+            console.error(`Async import job failed (requestId: ${requestId}).`);
+            hasError = true;
+
+            let resultsResult;
+            try {
+                resultsResult = await sdk.rest.get(asyncRequestResultsPath(requestId));
+            } catch (ex) {
+                console.error(
+                    `Async import: could not retrieve error details for requestId ${requestId}: ${ex.message}`,
+                );
+                continue;
+            }
+
+            const items = resultsResult?.items ?? [];
+            if (items.length === 0) {
+                console.error('Async import: no item-level error details returned by API.');
+            } else {
+                for (const item of items) {
+                    console.error(`Import error: ${item.message}`);
+                }
+            }
+        } else if (requestStatus !== undefined) {
+            console.error(
+                `Async import: unexpected status "${requestStatus}" for requestId ${requestId}.`,
+            );
+        }
+    }
+
+    return hasError;
+}

package/lib/cli.mjs

@@ -16,8 +16,10 @@ import { exportDataExtensionToFile } from './export-de.mjs';
 import { findImportCandidates, pickLatestByMtime } from './file-resolve.mjs';
 import { parseExportBasename } from './filename.mjs';
 import { importFromFile } from './import-de.mjs';
+import { pollAsyncImportCompletion } from './async-status.mjs';
 import { clearDataExtensionRows } from './clear-de.mjs';
 import { confirmClearBeforeImport } from './confirm-clear.mjs';
+import { getDeRowCount } from './row-count.mjs';
 import { multiBuExport } from './multi-bu-export.mjs';
 import { crossBuImport } from './cross-bu-import.mjs';
 import { initDebugLogger } from './debug-logger.mjs';
@@ -266,7 +268,7 @@ export async function main(argv) {
                 return 1;
             }
             const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
-            await crossBuImport({
+            const crossBuHadError = await crossBuImport({
                 projectRoot,
                 mcdevrc,
                 mcdevAuth,
@@ -281,7 +283,7 @@ export async function main(argv) {
                 useGit,
                 logger,
             });
-            return 0;
+            return crossBuHadError ? 1 : 0;
         }
 
         // ── Cross-BU import (API mode): --from + --to + --de ────────────────
@@ -331,7 +333,7 @@ export async function main(argv) {
                 return 1;
             }
             const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
-            await crossBuImport({
+            const crossBuApiHadError = await crossBuImport({
                 projectRoot,
                 mcdevrc,
                 mcdevAuth,
@@ -348,7 +350,7 @@ export async function main(argv) {
                 useGit,
                 logger,
             });
-            return 0;
+            return crossBuApiHadError ? 1 : 0;
         }
 
         // ── Single-BU import (original behavior) ────────────────────────────
@@ -398,10 +400,8 @@ export async function main(argv) {
                     acceptRiskFlag: acceptRisk,
                     isTTY: process.stdin.isTTY === true,
                 });
-                for (const deKey of deKeys) {
-                    await clearDataExtensionRows(sdk.soap, deKey);
-                }
             }
+            let anyError = false;
             for (const deKey of deKeys) {
                 const candidates = await findImportCandidates(dataDir, deKey);
                 if (candidates.length === 0) {
@@ -412,15 +412,54 @@ export async function main(argv) {
                 }
                 const filePath =
                     candidates.length === 1 ? candidates[0] : await pickLatestByMtime(candidates);
-                const n = await importFromFile(sdk, {
+
+                const countBefore = await getDeRowCount(sdk, deKey);
+                console.error(
+                    `Row count before import: ${countBefore ?? '(unavailable)'} (DE "${deKey}")`,
+                );
+
+                if (clear) {
+                    if (countBefore === 0) {
+                        console.error(
+                            `Skipping clear-data for DE "${deKey}" — DE is already empty.`,
+                        );
+                    } else {
+                        await clearDataExtensionRows(sdk.soap, deKey);
+                        console.warn(`Cleared data: DE "${deKey}"`);
+                    }
+                }
+
+                const { count: n, requestIds } = await importFromFile(sdk, {
                     filePath,
                     deKey,
                     mode: /** @type {'upsert'|'insert'} */ (mode),
                 });
                 const rel = projectRelativePosix(projectRoot, filePath);
                 console.error(`Imported: ${rel} (${n} rows) -> DE ${deKey}`);
+
+                const importHadError = await pollAsyncImportCompletion(sdk, requestIds);
+                if (importHadError) {
+                    anyError = true;
+                }
+
+                const countAfter = await getDeRowCount(sdk, deKey);
+                console.error(
+                    `Row count after import: ${countAfter ?? '(unavailable)'} (DE "${deKey}")`,
+                );
+                if (countAfter === null) {
+                    console.error(`Could not verify import result for DE "${deKey}".`);
+                } else if (countBefore !== null) {
+                    // Insert: expect countBefore + n; upsert on empty: same; upsert on non-empty: expect >= n
+                    const expected =
+                        mode === 'insert' || countBefore === 0 ? (countBefore ?? 0) + n : n;
+                    if (countAfter < expected) {
+                        console.error(
+                            `Import result for DE "${deKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}.`,
+                        );
+                    }
+                }
             }
-            return 0;
+            return anyError ? 1 : 0;
         }
 
         const fileList = values.file ?? [];
@@ -448,22 +487,58 @@ export async function main(argv) {
                 acceptRiskFlag: acceptRisk,
                 isTTY: process.stdin.isTTY === true,
             });
-            for (const deKey of keysFromFiles) {
-                await clearDataExtensionRows(sdk.soap, deKey);
-            }
         }
+        let anyFileError = false;
         for (const filePath of fileList) {
             const base = path.basename(filePath);
             const { customerKey } = parseExportBasename(base);
-            const n = await importFromFile(sdk, {
+
+            const countBefore = await getDeRowCount(sdk, customerKey);
+            console.error(
+                `Row count before import: ${countBefore ?? '(unavailable)'} (DE "${customerKey}")`,
+            );
+
+            if (clear) {
+                if (countBefore === 0) {
+                    console.error(
+                        `Skipping clear-data for DE "${customerKey}" — DE is already empty.`,
+                    );
+                } else {
+                    await clearDataExtensionRows(sdk.soap, customerKey);
+                    console.warn(`Cleared data: DE "${customerKey}"`);
+                }
+            }
+
+            const { count: n, requestIds } = await importFromFile(sdk, {
                 filePath,
                 deKey: customerKey,
                 mode: /** @type {'upsert'|'insert'} */ (mode),
             });
             const rel = projectRelativePosix(projectRoot, filePath);
             console.error(`Imported: ${rel} (${n} rows)`);
+
+            const importHadError = await pollAsyncImportCompletion(sdk, requestIds);
+            if (importHadError) {
+                anyFileError = true;
+            }
+
+            const countAfter = await getDeRowCount(sdk, customerKey);
+            console.error(
+                `Row count after import: ${countAfter ?? '(unavailable)'} (DE "${customerKey}")`,
+            );
+            if (countAfter === null) {
+                console.error(`Could not verify import result for DE "${customerKey}".`);
+            } else {
+                const expected =
+                    mode === 'insert' || countBefore === 0 ? (countBefore ?? 0) + n : n;
+                if (countAfter < expected) {
+                    console.error(
+                        `Import result for DE "${customerKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}.`,
+                    );
+                }
+            }
         }
-        return 0;
+        return anyFileError ? 1 : 0;
     }
 
     console.error(`Unknown command: ${sub}`);

package/lib/cross-bu-import.mjs

@@ -7,11 +7,13 @@ import { resolveCredentialAndMid, buildSdkAuthObject, buildSdkOptions } from './
 import { fetchAllRowObjects, serializeRows, exportDataExtensionToFile } from './export-de.mjs';
 import { formatFromExtension } from './file-resolve.mjs';
 import { importRowsForDe } from './import-de.mjs';
+import { pollAsyncImportCompletion } from './async-status.mjs';
 import { readRowsFromFile } from './read-rows.mjs';
 import { clearDataExtensionRows } from './clear-de.mjs';
 import { confirmClearBeforeImport } from './confirm-clear.mjs';
 import { dataDirectoryForBu, projectRelativePosix } from './paths.mjs';
 import { buildExportBasename, filesystemSafeTimestamp, parseExportBasename } from './filename.mjs';
+import { getDeRowCount } from './row-count.mjs';
 
 /**
  * @typedef {{ credential: string, bu: string }} CredBuTarget
@@ -81,7 +83,7 @@ async function offerPreExportBackup({ targets, deKeys, stdin: stdinStream, stdou
  * @param {import('./config.mjs').DebugLogger|null} [params.logger] - debug logger for API requests/responses
  * @param {NodeJS.ReadableStream} [params.stdin] Override for testing
  * @param {NodeJS.WritableStream} [params.stdout] Override for testing
- * @returns {Promise.<void>}
+ * @returns {Promise.<boolean>} true if at least one import job returned an error
  */
 export async function crossBuImport(params) {
     const {
@@ -168,6 +170,8 @@ export async function crossBuImport(params) {
         await confirmClearBeforeImport({ deKeys, targets, acceptRiskFlag, isTTY, stdin, stdout });
     }
 
+    let hasError = false;
+
     // Load rows once per DE then fan out to every target
     for (const deKey of deKeys) {
         let rows;
@@ -184,24 +188,27 @@ export async function crossBuImport(params) {
             rows = await fetchAllRowObjects(srcSdk, deKey);
         }
 
-        // Clear targets before import (rows already confirmed above)
-        if (clearBeforeImport) {
-            for (const { credential, bu } of targets) {
-                const { mid, authCred } = resolveCredentialAndMid(
-                    mcdevrc,
-                    mcdevAuth,
-                    credential,
-                    bu,
-                );
-                const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
-                await clearDataExtensionRows(tgtSdk.soap, deKey);
-            }
-        }
-
         for (const { credential, bu } of targets) {
             const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
             const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
 
+            const countBefore = await getDeRowCount(tgtSdk, deKey);
+            console.error(
+                `Row count before import: ${countBefore ?? '(unavailable)'} (${credential}/${bu} DE "${deKey}")`,
+            );
+
+            // Clear target before import (already confirmed above); skip if DE is empty
+            if (clearBeforeImport) {
+                if (countBefore === 0) {
+                    console.error(
+                        `Skipping clear-data for ${credential}/${bu} DE "${deKey}" — DE is already empty.`,
+                    );
+                } else {
+                    await clearDataExtensionRows(tgtSdk.soap, deKey);
+                    console.warn(`Cleared data: ${credential}/${bu} DE "${deKey}"`);
+                }
+            }
+
             // Write a snapshot file in the target BU's data directory.
             const dir = dataDirectoryForBu(projectRoot, credential, bu);
             await fs.mkdir(dir, { recursive: true });
@@ -212,8 +219,39 @@ export async function crossBuImport(params) {
             const snapRel = projectRelativePosix(projectRoot, snapshotPath);
             console.error(`Download stored: ${snapRel} (${rows.length} rows)`);
 
-            const imported = await importRowsForDe(tgtSdk, { deKey, rows, mode });
+            const { count: imported, requestIds } = await importRowsForDe(tgtSdk, {
+                deKey,
+                rows,
+                mode,
+            });
             console.error(`Imported: ${credential}/${bu} DE ${deKey} (${imported} rows)`);
+
+            const importHadError = await pollAsyncImportCompletion(tgtSdk, requestIds);
+            if (importHadError) {
+                hasError = true;
+            }
+
+            const countAfter = await getDeRowCount(tgtSdk, deKey);
+            console.error(
+                `Row count after import: ${countAfter ?? '(unavailable)'} (${credential}/${bu} DE "${deKey}")`,
+            );
+            if (countAfter === null) {
+                console.error(
+                    `Could not verify import result for ${credential}/${bu} DE "${deKey}".`,
+                );
+            } else {
+                const expected =
+                    mode === 'insert' || countBefore === 0
+                        ? (countBefore ?? 0) + imported
+                        : imported;
+                if (countAfter < expected) {
+                    console.error(
+                        `Import result for ${credential}/${bu} DE "${deKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}.`,
+                    );
+                }
+            }
         }
     }
+
+    return hasError;
 }

package/lib/import-de.mjs

@@ -10,20 +10,22 @@ import { readRowsFromFile } from './read-rows.mjs';
  * @param {string} params.deKey
  * @param {object[]} params.rows
  * @param {'upsert'|'insert'} params.mode
- * @returns {Promise.<number>} number of rows imported
+ * @returns {Promise.<{ count: number, requestIds: (string|null)[] }>}
  */
 export async function importRowsForDe(sdk, params) {
     const { deKey, rows, mode } = params;
     const route = resolveImportRoute(mode);
     const chunks = chunkItemsForPayload(rows);
+    const requestIds = [];
     for (const chunk of chunks) {
         const p = route.path(deKey);
         const body = { items: chunk };
-        await withRetry429(() =>
+        const resp = await withRetry429(() =>
             route.method === 'PUT' ? sdk.rest.put(p, body) : sdk.rest.post(p, body),
         );
+        requestIds.push(resp?.requestId ?? null);
     }
-    return rows.length;
+    return { count: rows.length, requestIds };
 }
 
 /**
@@ -33,7 +35,7 @@ export async function importRowsForDe(sdk, params) {
  * @param {string} params.deKey - target DE customer key for API
  * @param {'csv'|'tsv'|'json'} [params.format] - optional; auto-detected from file extension if omitted
  * @param {'upsert'|'insert'} params.mode
- * @returns {Promise.<number>} number of rows imported
+ * @returns {Promise.<{ count: number, requestIds: (string|null)[] }>}
  */
 export async function importFromFile(sdk, params) {
     const format = params.format || formatFromExtension(params.filePath);

package/lib/import-routes.mjs

@@ -21,6 +21,26 @@ export function asyncDataExtensionRowsPath(deKey) {
     return `/data/v1/async/dataextensions/key:${encodeURIComponent(deKey)}/rows`;
 }
 
+/**
+ * Status endpoint for a previously submitted async request.
+ *
+ * @param {string} requestId
+ * @returns {string}
+ */
+export function asyncRequestStatusPath(requestId) {
+    return `/data/v1/async/${encodeURIComponent(requestId)}/status`;
+}
+
+/**
+ * Results endpoint for a previously submitted async request.
+ *
+ * @param {string} requestId
+ * @returns {string}
+ */
+export function asyncRequestResultsPath(requestId) {
+    return `/data/v1/async/${encodeURIComponent(requestId)}/results`;
+}
+
 /**
  * @param {'upsert'|'insert'} mode
  * @returns {{ method: 'PUT'|'POST', path: (de: string) => string }}

package/lib/index.mjs

@@ -5,7 +5,14 @@ export {
     parseExportBasename,
 } from './filename.mjs';
 export { chunkItemsForPayload, DEFAULT_MAX_BODY_BYTES, MAX_OBJECTS_PER_BATCH } from './batch.mjs';
-export { resolveImportRoute, rowsetGetPath, asyncDataExtensionRowsPath } from './import-routes.mjs';
+export {
+    resolveImportRoute,
+    rowsetGetPath,
+    asyncDataExtensionRowsPath,
+    asyncRequestStatusPath,
+    asyncRequestResultsPath,
+} from './import-routes.mjs';
+export { pollAsyncImportCompletion } from './async-status.mjs';
 export {
     loadMcdevProject,
     parseCredBu,
@@ -14,4 +21,5 @@ export {
 } from './config.mjs';
 export { multiBuExport } from './multi-bu-export.mjs';
 export { crossBuImport } from './cross-bu-import.mjs';
+export { getDeRowCount } from './row-count.mjs';
 export { projectRelativePosix } from './paths.mjs';

package/lib/row-count.mjs

@@ -0,0 +1,20 @@
+import { rowsetGetPath } from './import-routes.mjs';
+
+/**
+ * Fetch the total row count for a Data Extension without downloading all rows.
+ * Uses a pagesize=1 request so only the `count` metadata is transferred.
+ * Returns `null` if the count cannot be determined (e.g. permissions error).
+ *
+ * @param {{ rest: { get: (path: string) => Promise.<any> } }} sdk
+ * @param {string} deKey - DE external key
+ * @returns {Promise.<number|null>}
+ */
+export async function getDeRowCount(sdk, deKey) {
+    try {
+        const result = await sdk.rest.get(`${rowsetGetPath(deKey)}?$page=1&$pagesize=1`);
+        return result?.count ?? 0;
+    } catch (ex) {
+        console.error(`Could not retrieve row count for DE "${deKey}": ${ex.message}`);
+        return null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sfmc-dataloader",
-    "version": "2.2.0",
+    "version": "2.4.0",
     "description": "SFMC Data Loader CLI (mcdata) to export and import Marketing Cloud Data Extension rows using mcdev project config and sfmc-sdk",
     "author": "Jörn Berkefeld <joern.berkefeld@gmail.com>",
     "license": "MIT",
@@ -40,13 +40,14 @@
     },
     "dependencies": {
         "csv-parser": "3.2.0",
-        "csv-stringify": "6.5.2",
+        "csv-stringify": "6.7.0",
         "sfmc-sdk": "3.0.3"
     },
     "scripts": {
         "test": "node --test test/*.test.js",
         "lint": "eslint .",
         "lint:fix": "eslint --fix .",
+        "prepare": "husky || true",
         "format": "prettier --write .",
         "format:check": "prettier --check ."
     },
@@ -58,6 +59,7 @@
         "eslint-plugin-prettier": "^5.5.0",
         "eslint-plugin-unicorn": "^64.0.0",
         "globals": "^17.4.0",
+        "husky": "^9.1.7",
         "prettier": "^3.8.1"
     }
 }