sfmc-dataloader 2.2.0 → 2.3.0

package/lib/cli.mjs

@@ -18,6 +18,7 @@ import { parseExportBasename } from './filename.mjs';
 import { importFromFile } from './import-de.mjs';
 import { clearDataExtensionRows } from './clear-de.mjs';
 import { confirmClearBeforeImport } from './confirm-clear.mjs';
+import { getDeRowCount } from './row-count.mjs';
 import { multiBuExport } from './multi-bu-export.mjs';
 import { crossBuImport } from './cross-bu-import.mjs';
 import { initDebugLogger } from './debug-logger.mjs';
@@ -398,9 +399,6 @@ export async function main(argv) {
                     acceptRiskFlag: acceptRisk,
                     isTTY: process.stdin.isTTY === true,
                 });
-                for (const deKey of deKeys) {
-                    await clearDataExtensionRows(sdk.soap, deKey);
-                }
             }
             for (const deKey of deKeys) {
                 const candidates = await findImportCandidates(dataDir, deKey);
@@ -412,6 +410,23 @@ export async function main(argv) {
                 }
                 const filePath =
                     candidates.length === 1 ? candidates[0] : await pickLatestByMtime(candidates);
+
+                const countBefore = await getDeRowCount(sdk, deKey);
+                console.error(
+                    `Row count before import: ${countBefore ?? '(unavailable)'} (DE "${deKey}")`,
+                );
+
+                if (clear) {
+                    if (countBefore === 0) {
+                        console.error(
+                            `Skipping clear-data for DE "${deKey}" — DE is already empty.`,
+                        );
+                    } else {
+                        await clearDataExtensionRows(sdk.soap, deKey);
+                        console.warn(`Cleared data: DE "${deKey}"`);
+                    }
+                }
+
                 const n = await importFromFile(sdk, {
                     filePath,
                     deKey,
@@ -419,6 +434,23 @@ export async function main(argv) {
                 });
                 const rel = projectRelativePosix(projectRoot, filePath);
                 console.error(`Imported: ${rel} (${n} rows) -> DE ${deKey}`);
+
+                const countAfter = await getDeRowCount(sdk, deKey);
+                console.error(
+                    `Row count after import: ${countAfter ?? '(unavailable)'} (DE "${deKey}")`,
+                );
+                if (countAfter === null) {
+                    console.error(`Could not verify import result for DE "${deKey}".`);
+                } else if (countBefore !== null && countAfter < (countBefore ?? 0) + n) {
+                    // Insert: expect countBefore + n; upsert on empty: same; upsert on non-empty: expect >= n
+                    const expected =
+                        mode === 'insert' || countBefore === 0 ? (countBefore ?? 0) + n : n;
+                    if (countAfter < expected) {
+                        console.error(
+                            `Import result for DE "${deKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}. Note: the async API may not have committed all rows yet.`,
+                        );
+                    }
+                }
             }
             return 0;
         }
@@ -448,13 +480,27 @@ export async function main(argv) {
                 acceptRiskFlag: acceptRisk,
                 isTTY: process.stdin.isTTY === true,
             });
-            for (const deKey of keysFromFiles) {
-                await clearDataExtensionRows(sdk.soap, deKey);
-            }
         }
         for (const filePath of fileList) {
             const base = path.basename(filePath);
             const { customerKey } = parseExportBasename(base);
+
+            const countBefore = await getDeRowCount(sdk, customerKey);
+            console.error(
+                `Row count before import: ${countBefore ?? '(unavailable)'} (DE "${customerKey}")`,
+            );
+
+            if (clear) {
+                if (countBefore === 0) {
+                    console.error(
+                        `Skipping clear-data for DE "${customerKey}" — DE is already empty.`,
+                    );
+                } else {
+                    await clearDataExtensionRows(sdk.soap, customerKey);
+                    console.warn(`Cleared data: DE "${customerKey}"`);
+                }
+            }
+
             const n = await importFromFile(sdk, {
                 filePath,
                 deKey: customerKey,
@@ -462,6 +508,22 @@ export async function main(argv) {
             });
             const rel = projectRelativePosix(projectRoot, filePath);
             console.error(`Imported: ${rel} (${n} rows)`);
+
+            const countAfter = await getDeRowCount(sdk, customerKey);
+            console.error(
+                `Row count after import: ${countAfter ?? '(unavailable)'} (DE "${customerKey}")`,
+            );
+            if (countAfter === null) {
+                console.error(`Could not verify import result for DE "${customerKey}".`);
+            } else {
+                const expected =
+                    mode === 'insert' || countBefore === 0 ? (countBefore ?? 0) + n : n;
+                if (countAfter < expected) {
+                    console.error(
+                        `Import result for DE "${customerKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}. Note: the async API may not have committed all rows yet.`,
+                    );
+                }
+            }
         }
         return 0;
     }

package/lib/cross-bu-import.mjs

@@ -12,6 +12,7 @@ import { clearDataExtensionRows } from './clear-de.mjs';
 import { confirmClearBeforeImport } from './confirm-clear.mjs';
 import { dataDirectoryForBu, projectRelativePosix } from './paths.mjs';
 import { buildExportBasename, filesystemSafeTimestamp, parseExportBasename } from './filename.mjs';
+import { getDeRowCount } from './row-count.mjs';
 
 /**
  * @typedef {{ credential: string, bu: string }} CredBuTarget
@@ -184,24 +185,27 @@ export async function crossBuImport(params) {
             rows = await fetchAllRowObjects(srcSdk, deKey);
         }
 
-        // Clear targets before import (rows already confirmed above)
-        if (clearBeforeImport) {
-            for (const { credential, bu } of targets) {
-                const { mid, authCred } = resolveCredentialAndMid(
-                    mcdevrc,
-                    mcdevAuth,
-                    credential,
-                    bu,
-                );
-                const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
-                await clearDataExtensionRows(tgtSdk.soap, deKey);
-            }
-        }
-
         for (const { credential, bu } of targets) {
             const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
             const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
 
+            const countBefore = await getDeRowCount(tgtSdk, deKey);
+            console.error(
+                `Row count before import: ${countBefore ?? '(unavailable)'} (${credential}/${bu} DE "${deKey}")`,
+            );
+
+            // Clear target before import (already confirmed above); skip if DE is empty
+            if (clearBeforeImport) {
+                if (countBefore === 0) {
+                    console.error(
+                        `Skipping clear-data for ${credential}/${bu} DE "${deKey}" — DE is already empty.`,
+                    );
+                } else {
+                    await clearDataExtensionRows(tgtSdk.soap, deKey);
+                    console.warn(`Cleared data: ${credential}/${bu} DE "${deKey}"`);
+                }
+            }
+
             // Write a snapshot file in the target BU's data directory.
             const dir = dataDirectoryForBu(projectRoot, credential, bu);
             await fs.mkdir(dir, { recursive: true });
@@ -214,6 +218,24 @@ export async function crossBuImport(params) {
 
             const imported = await importRowsForDe(tgtSdk, { deKey, rows, mode });
             console.error(`Imported: ${credential}/${bu} DE ${deKey} (${imported} rows)`);
+
+            const countAfter = await getDeRowCount(tgtSdk, deKey);
+            console.error(
+                `Row count after import: ${countAfter ?? '(unavailable)'} (${credential}/${bu} DE "${deKey}")`,
+            );
+            if (countAfter === null) {
+                console.error(
+                    `Could not verify import result for ${credential}/${bu} DE "${deKey}".`,
+                );
+            } else {
+                const expected =
+                    mode === 'insert' || countBefore === 0 ? (countBefore ?? 0) + imported : imported;
+                if (countAfter < expected) {
+                    console.error(
+                        `Import result for ${credential}/${bu} DE "${deKey}" looks unexpected: expected at least ${expected} rows, got ${countAfter}. Note: the async API may not have committed all rows yet.`,
+                    );
+                }
+            }
         }
     }
 }

package/lib/index.mjs

@@ -14,4 +14,5 @@ export {
 } from './config.mjs';
 export { multiBuExport } from './multi-bu-export.mjs';
 export { crossBuImport } from './cross-bu-import.mjs';
+export { getDeRowCount } from './row-count.mjs';
 export { projectRelativePosix } from './paths.mjs';

package/lib/row-count.mjs

@@ -0,0 +1,20 @@
+import { rowsetGetPath } from './import-routes.mjs';
+
+/**
+ * Fetch the total row count for a Data Extension without downloading all rows.
+ * Uses a pagesize=1 request so only the `count` metadata is transferred.
+ * Returns `null` if the count cannot be determined (e.g. permissions error).
+ *
+ * @param {{ rest: { get: (path: string) => Promise.<any> } }} sdk
+ * @param {string} deKey - DE external key
+ * @returns {Promise.<number|null>}
+ */
+export async function getDeRowCount(sdk, deKey) {
+    try {
+        const result = await sdk.rest.get(`${rowsetGetPath(deKey)}?$page=1&$pagesize=1`);
+        return result?.count ?? 0;
+    } catch (ex) {
+        console.error(`Could not retrieve row count for DE "${deKey}": ${ex.message}`);
+        return null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sfmc-dataloader",
-    "version": "2.2.0",
+    "version": "2.3.0",
     "description": "SFMC Data Loader CLI (mcdata) to export and import Marketing Cloud Data Extension rows using mcdev project config and sfmc-sdk",
     "author": "Jörn Berkefeld <joern.berkefeld@gmail.com>",
     "license": "MIT",