sfmc-dataloader 2.3.0 → 2.4.0

package/README.md

@@ -44,7 +44,7 @@ Creates one file per BU/DE combination using the same `.mcdata.` naming rules.
 mcdata import MyCred/MyBU --de MyDE_CustomerKey --mode upsert
 ```
 
-Imports use the **asynchronous** bulk row API only: `POST` for `--mode insert`, `PUT` for `--mode upsert` (same endpoint path).
+Imports use the **asynchronous** bulk row API only: `POST` for `--mode insert`, `PUT` for `--mode upsert` (same endpoint path). After uploading, `mcdata` polls the async status endpoint every 5 seconds until the job reaches `Complete` or `Error`. On `Error`, per-row error messages are printed to the log so you can see exactly which records failed. The process exits with code 1 if any import job fails, which also surfaces as an error notification in the VS Code extension.
 
 Resolves the latest matching export file under `./data/MyCred/MyBU/` for that DE key. The file format is detected automatically from the file extension (`.csv`, `.tsv`, `.json`).
 

package/lib/async-status.mjs

@@ -0,0 +1,89 @@
+import { asyncRequestResultsPath, asyncRequestStatusPath } from './import-routes.mjs';
+
+const POLL_INTERVAL_MS = 5000;
+const PENDING_STATUSES = new Set(['Pending', 'Executing']);
+
+/**
+ * @param {number} ms
+ * @returns {Promise.<void>}
+ */
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Polls the async import status API for each requestId until the job reaches
+ * a terminal state ("Complete" or "Error").  On error, fetches per-row results
+ * and logs each item message so the user can see what failed.
+ *
+ * @param {{ rest: { get: Function } }} sdk
+ * @param {(string|null|undefined)[]} requestIds - one per imported chunk
+ * @returns {Promise.<boolean>} true if at least one chunk job returned "Error"
+ */
+export async function pollAsyncImportCompletion(sdk, requestIds) {
+    let hasError = false;
+
+    for (const requestId of requestIds) {
+        if (!requestId) {
+            console.error('Async import: no requestId returned for chunk — skipping status check.');
+            continue;
+        }
+
+        console.error(`Waiting for async import to complete (requestId: ${requestId})…`);
+        await sleep(POLL_INTERVAL_MS);
+
+        let requestStatus;
+        do {
+            let statusResult;
+            try {
+                statusResult = await sdk.rest.get(asyncRequestStatusPath(requestId));
+            } catch (ex) {
+                console.error(
+                    `Async import: status check failed for requestId ${requestId}: ${ex.message}`,
+                );
+                break;
+            }
+
+            requestStatus = statusResult?.status?.requestStatus;
+
+            if (PENDING_STATUSES.has(requestStatus)) {
+                console.error(
+                    `Async import still in progress (requestId: ${requestId}) — retrying in 5 s…`,
+                );
+                await sleep(POLL_INTERVAL_MS);
+            }
+        } while (PENDING_STATUSES.has(requestStatus));
+
+        if (requestStatus === 'Complete') {
+            console.error(`Async import completed successfully (requestId: ${requestId}).`);
+        } else if (requestStatus === 'Error') {
+            console.error(`Async import job failed (requestId: ${requestId}).`);
+            hasError = true;
+
+            let resultsResult;
+            try {
+                resultsResult = await sdk.rest.get(asyncRequestResultsPath(requestId));
+            } catch (ex) {
+                console.error(
+                    `Async import: could not retrieve error details for requestId ${requestId}: ${ex.message}`,
+                );
+                continue;
+            }
+
+            const items = resultsResult?.items ?? [];
+            if (items.length === 0) {
+                console.error('Async import: no item-level error details returned by API.');
+            } else {
+                for (const item of items) {
+                    console.error(`Import error: ${item.message}`);
+                }
+            }
+        } else if (requestStatus !== undefined) {
+            console.error(
+                `Async import: unexpected status "${requestStatus}" for requestId ${requestId}.`,
+            );
+        }
+    }
+
+    return hasError;
+}

package/lib/cli.mjs

@@ -16,6 +16,7 @@ import { exportDataExtensionToFile } from './export-de.mjs';
 import { findImportCandidates, pickLatestByMtime } from './file-resolve.mjs';
 import { parseExportBasename } from './filename.mjs';
 import { importFromFile } from './import-de.mjs';
+import { pollAsyncImportCompletion } from './async-status.mjs';
 import { clearDataExtensionRows } from './clear-de.mjs';
 import { confirmClearBeforeImport } from './confirm-clear.mjs';
 import { getDeRowCount } from './row-count.mjs';
@@ -267,7 +268,7 @@ export async function main(argv) {
                 return 1;
             }
             const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
-            await crossBuImport({
+            const crossBuHadError = await crossBuImport({
                 projectRoot,
                 mcdevrc,
                 mcdevAuth,
@@ -282,7 +283,7 @@ export async function main(argv) {
                 useGit,
                 logger,
             });
-            return 0;
+            return crossBuHadError ? 1 : 0;
         }
 
         // ── Cross-BU import (API mode): --from + --to + --de ────────────────
@@ -332,7 +333,7 @@ export async function main(argv) {
                 return 1;
             }
             const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
-            await crossBuImport({
+            const crossBuApiHadError = await crossBuImport({
                 projectRoot,
                 mcdevrc,
                 mcdevAuth,
@@ -349,7 +350,7 @@ export async function main(argv) {
                 useGit,
                 logger,
             });
-            return 0;
+            return crossBuApiHadError ? 1 : 0;
         }
 
         // ── Single-BU import (original behavior) ────────────────────────────
@@ -400,6 +401,7 @@ export async function main(argv) {
                     isTTY: process.stdin.isTTY === true,
                 });
             }
+            let anyError = false;
             for (const deKey of deKeys) {
                 const candidates = await findImportCandidates(dataDir, deKey);
                 if (candidates.length === 0) {
@@ -427,7 +429,7 @@ export async function main(argv) {
                     }
                 }
 
-                const n = await importFromFile(sdk, {
+                const { count: n, requestIds } = await importFromFile(sdk, {
                     filePath,
                     deKey,
                     mode: /** @type {'upsert'|'insert'} */ (mode),
@@ -435,24 +437,29 @@ export async function main(argv) {
                 const rel = projectRelativePosix(projectRoot, filePath);
                 console.error(`Imported: ${rel} (${n} rows) -> DE ${deKey}`);
 
+                const importHadError = await pollAsyncImportCompletion(sdk, requestIds);
+                if (importHadError) {
+                    anyError = true;
+                }
+
                 const countAfter = await getDeRowCount(sdk, deKey);
                 console.error(
                     `Row count after import: ${countAfter ?? '(unavailable)'} (DE "${deKey}")`,
                 );
                 if (countAfter === null) {
                     console.error(`Could not verify import result for DE "${deKey}".`);
-                } else if (countBefore !== null && countAfter < (countBefore ?? 0) + n) {
+                } else if (countBefore !== null) {
                     // Insert: expect countBefore + n; upsert on empty: same; upsert on non-empty: expect >= n
                     const expected =
                         mode === 'insert' || countBefore === 0 ? (countBefore ?? 0) + n : n;
                     if (countAfter < expected) {
                         console.error(
-                            `Import result for DE "${deKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}. Note: the async API may not have committed all rows yet.`,
+                            `Import result for DE "${deKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}.`,
                         );
                     }
                 }
             }
-            return 0;
+            return anyError ? 1 : 0;
         }
 
         const fileList = values.file ?? [];
@@ -481,6 +488,7 @@ export async function main(argv) {
                 isTTY: process.stdin.isTTY === true,
             });
         }
+        let anyFileError = false;
         for (const filePath of fileList) {
             const base = path.basename(filePath);
             const { customerKey } = parseExportBasename(base);
@@ -501,7 +509,7 @@ export async function main(argv) {
                 }
             }
 
-            const n = await importFromFile(sdk, {
+            const { count: n, requestIds } = await importFromFile(sdk, {
                 filePath,
                 deKey: customerKey,
                 mode: /** @type {'upsert'|'insert'} */ (mode),
@@ -509,6 +517,11 @@ export async function main(argv) {
             const rel = projectRelativePosix(projectRoot, filePath);
             console.error(`Imported: ${rel} (${n} rows)`);
 
+            const importHadError = await pollAsyncImportCompletion(sdk, requestIds);
+            if (importHadError) {
+                anyFileError = true;
+            }
+
             const countAfter = await getDeRowCount(sdk, customerKey);
             console.error(
                 `Row count after import: ${countAfter ?? '(unavailable)'} (DE "${customerKey}")`,
@@ -520,12 +533,12 @@ export async function main(argv) {
                     mode === 'insert' || countBefore === 0 ? (countBefore ?? 0) + n : n;
                 if (countAfter < expected) {
                     console.error(
-                        `Import result for DE "${customerKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}. Note: the async API may not have committed all rows yet.`,
+                        `Import result for DE "${customerKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}.`,
                     );
                 }
             }
         }
-        return 0;
+        return anyFileError ? 1 : 0;
     }
 
     console.error(`Unknown command: ${sub}`);

package/lib/cross-bu-import.mjs

@@ -7,6 +7,7 @@ import { resolveCredentialAndMid, buildSdkAuthObject, buildSdkOptions } from './
 import { fetchAllRowObjects, serializeRows, exportDataExtensionToFile } from './export-de.mjs';
 import { formatFromExtension } from './file-resolve.mjs';
 import { importRowsForDe } from './import-de.mjs';
+import { pollAsyncImportCompletion } from './async-status.mjs';
 import { readRowsFromFile } from './read-rows.mjs';
 import { clearDataExtensionRows } from './clear-de.mjs';
 import { confirmClearBeforeImport } from './confirm-clear.mjs';
@@ -82,7 +83,7 @@ async function offerPreExportBackup({ targets, deKeys, stdin: stdinStream, stdou
  * @param {import('./config.mjs').DebugLogger|null} [params.logger] - debug logger for API requests/responses
  * @param {NodeJS.ReadableStream} [params.stdin] Override for testing
  * @param {NodeJS.WritableStream} [params.stdout] Override for testing
- * @returns {Promise.<void>}
+ * @returns {Promise.<boolean>} true if at least one import job returned an error
  */
 export async function crossBuImport(params) {
     const {
@@ -169,6 +170,8 @@ export async function crossBuImport(params) {
         await confirmClearBeforeImport({ deKeys, targets, acceptRiskFlag, isTTY, stdin, stdout });
     }
 
+    let hasError = false;
+
     // Load rows once per DE then fan out to every target
     for (const deKey of deKeys) {
         let rows;
@@ -216,9 +219,18 @@ export async function crossBuImport(params) {
             const snapRel = projectRelativePosix(projectRoot, snapshotPath);
             console.error(`Download stored: ${snapRel} (${rows.length} rows)`);
 
-            const imported = await importRowsForDe(tgtSdk, { deKey, rows, mode });
+            const { count: imported, requestIds } = await importRowsForDe(tgtSdk, {
+                deKey,
+                rows,
+                mode,
+            });
             console.error(`Imported: ${credential}/${bu} DE ${deKey} (${imported} rows)`);
 
+            const importHadError = await pollAsyncImportCompletion(tgtSdk, requestIds);
+            if (importHadError) {
+                hasError = true;
+            }
+
             const countAfter = await getDeRowCount(tgtSdk, deKey);
             console.error(
                 `Row count after import: ${countAfter ?? '(unavailable)'} (${credential}/${bu} DE "${deKey}")`,
@@ -229,13 +241,17 @@ export async function crossBuImport(params) {
                 );
             } else {
                 const expected =
-                    mode === 'insert' || countBefore === 0 ? (countBefore ?? 0) + imported : imported;
+                    mode === 'insert' || countBefore === 0
+                        ? (countBefore ?? 0) + imported
+                        : imported;
                 if (countAfter < expected) {
                     console.error(
-                        `Import result for ${credential}/${bu} DE "${deKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}. Note: the async API may not have committed all rows yet.`,
+                        `Import result for ${credential}/${bu} DE "${deKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}.`,
                     );
                 }
             }
         }
     }
+
+    return hasError;
 }

package/lib/import-de.mjs

@@ -10,20 +10,22 @@ import { readRowsFromFile } from './read-rows.mjs';
  * @param {string} params.deKey
  * @param {object[]} params.rows
  * @param {'upsert'|'insert'} params.mode
- * @returns {Promise.<number>} number of rows imported
+ * @returns {Promise.<{ count: number, requestIds: (string|null)[] }>}
  */
 export async function importRowsForDe(sdk, params) {
     const { deKey, rows, mode } = params;
     const route = resolveImportRoute(mode);
     const chunks = chunkItemsForPayload(rows);
+    const requestIds = [];
     for (const chunk of chunks) {
         const p = route.path(deKey);
         const body = { items: chunk };
-        await withRetry429(() =>
+        const resp = await withRetry429(() =>
             route.method === 'PUT' ? sdk.rest.put(p, body) : sdk.rest.post(p, body),
         );
+        requestIds.push(resp?.requestId ?? null);
     }
-    return rows.length;
+    return { count: rows.length, requestIds };
 }
 
 /**
@@ -33,7 +35,7 @@ export async function importRowsForDe(sdk, params) {
  * @param {string} params.deKey - target DE customer key for API
  * @param {'csv'|'tsv'|'json'} [params.format] - optional; auto-detected from file extension if omitted
  * @param {'upsert'|'insert'} params.mode
- * @returns {Promise.<number>} number of rows imported
+ * @returns {Promise.<{ count: number, requestIds: (string|null)[] }>}
  */
 export async function importFromFile(sdk, params) {
     const format = params.format || formatFromExtension(params.filePath);

package/lib/import-routes.mjs

@@ -21,6 +21,26 @@ export function asyncDataExtensionRowsPath(deKey) {
     return `/data/v1/async/dataextensions/key:${encodeURIComponent(deKey)}/rows`;
 }
 
+/**
+ * Status endpoint for a previously submitted async request.
+ *
+ * @param {string} requestId
+ * @returns {string}
+ */
+export function asyncRequestStatusPath(requestId) {
+    return `/data/v1/async/${encodeURIComponent(requestId)}/status`;
+}
+
+/**
+ * Results endpoint for a previously submitted async request.
+ *
+ * @param {string} requestId
+ * @returns {string}
+ */
+export function asyncRequestResultsPath(requestId) {
+    return `/data/v1/async/${encodeURIComponent(requestId)}/results`;
+}
+
 /**
  * @param {'upsert'|'insert'} mode
  * @returns {{ method: 'PUT'|'POST', path: (de: string) => string }}

package/lib/index.mjs

@@ -5,7 +5,14 @@ export {
     parseExportBasename,
 } from './filename.mjs';
 export { chunkItemsForPayload, DEFAULT_MAX_BODY_BYTES, MAX_OBJECTS_PER_BATCH } from './batch.mjs';
-export { resolveImportRoute, rowsetGetPath, asyncDataExtensionRowsPath } from './import-routes.mjs';
+export {
+    resolveImportRoute,
+    rowsetGetPath,
+    asyncDataExtensionRowsPath,
+    asyncRequestStatusPath,
+    asyncRequestResultsPath,
+} from './import-routes.mjs';
+export { pollAsyncImportCompletion } from './async-status.mjs';
 export {
     loadMcdevProject,
     parseCredBu,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sfmc-dataloader",
-    "version": "2.3.0",
+    "version": "2.4.0",
     "description": "SFMC Data Loader CLI (mcdata) to export and import Marketing Cloud Data Extension rows using mcdev project config and sfmc-sdk",
     "author": "Jörn Berkefeld <joern.berkefeld@gmail.com>",
     "license": "MIT",
@@ -40,13 +40,14 @@
     },
     "dependencies": {
         "csv-parser": "3.2.0",
-        "csv-stringify": "6.5.2",
+        "csv-stringify": "6.7.0",
         "sfmc-sdk": "3.0.3"
     },
     "scripts": {
         "test": "node --test test/*.test.js",
         "lint": "eslint .",
         "lint:fix": "eslint --fix .",
+        "prepare": "husky || true",
         "format": "prettier --write .",
         "format:check": "prettier --check ."
     },
@@ -58,6 +59,7 @@
         "eslint-plugin-prettier": "^5.5.0",
         "eslint-plugin-unicorn": "^64.0.0",
         "globals": "^17.4.0",
+        "husky": "^9.1.7",
         "prettier": "^3.8.1"
     }
 }