sfmc-dataloader 2.6.1 → 2.7.0

package/lib/filename.mjs

@@ -42,14 +42,20 @@ export function reverseFilterIllegalFilenames(filename) {
  * @param {string} safeTs - filesystem-safe UTC timestamp (ignored when useGit is true)
  * @param {'csv'|'tsv'|'json'} ext
  * @param {boolean} [useGit] - stable `key.mcdata.ext` without timestamp
+ * @param {number} [partNumber] - when set, inserts `partN` before the timestamp (or before the extension in --git mode)
  * @returns {string} basename without directory
  */
-export function buildExportBasename(customerKey, safeTs, ext, useGit = false) {
+export function buildExportBasename(customerKey, safeTs, ext, useGit = false, partNumber) {
     const enc = filterIllegalFilenames(customerKey);
+    const partInfix =
+        partNumber !== undefined && partNumber !== null ? `part${Number(partNumber)}.` : '';
     if (useGit) {
+        if (partNumber !== undefined && partNumber !== null) {
+            return `${enc}.mcdata.part${Number(partNumber)}.${ext}`;
+        }
         return `${enc}.mcdata.${ext}`;
     }
-    return `${enc}${MCDATA_SEGMENT}${safeTs}.${ext}`;
+    return `${enc}${MCDATA_SEGMENT}${partInfix}${safeTs}.${ext}`;
 }
 
 /**
@@ -62,7 +68,7 @@ export function filesystemSafeTimestamp(d = new Date()) {
 
 /**
  * @param {string} basename - e.g. `encodedKey.mcdata.2026-04-06T15-00-00.000Z.csv` or `encodedKey.mcdata.csv`
- * @returns {{ customerKey: string, timestampPart: string, ext: string }}
+ * @returns {{ customerKey: string, timestampPart: string, ext: string, partNumber?: number }}
  */
 export function parseExportBasename(basename) {
     const lastDot = basename.lastIndexOf('.');
@@ -72,11 +78,25 @@ export function parseExportBasename(basename) {
     const idx = stem.indexOf(MCDATA_SEGMENT);
     if (idx !== -1) {
         const encodedKey = stem.slice(0, idx);
-        const timestampPart = stem.slice(idx + MCDATA_SEGMENT.length);
+        let rest = stem.slice(idx + MCDATA_SEGMENT.length);
+        /** @type {number|undefined} */
+        let partNumber;
+        const partTs = rest.match(/^part(\d+)\.(.+)$/);
+        if (partTs) {
+            partNumber = Number(partTs[1]);
+            rest = partTs[2];
+        } else {
+            const partOnly = rest.match(/^part(\d+)$/);
+            if (partOnly) {
+                partNumber = Number(partOnly[1]);
+                rest = '';
+            }
+        }
         return {
             customerKey: reverseFilterIllegalFilenames(encodedKey),
-            timestampPart,
+            timestampPart: rest,
             ext,
+            ...(typeof partNumber === 'number' ? { partNumber } : {}),
         };
     }
 

package/lib/import-de.mjs

@@ -1,9 +1,101 @@
 import { RestError } from 'sfmc-sdk/util';
-import { chunkItemsForPayload } from './batch.mjs';
+import { chunkItemsForPayload, MAX_OBJECTS_PER_BATCH } from './batch.mjs';
 import { formatFromExtension } from './file-resolve.mjs';
 import { resolveImportRoute } from './import-routes.mjs';
 import { withRetry429 } from './retry.mjs';
-import { readRowsFromFile } from './read-rows.mjs';
+import { countDataRowsFromImportPaths, streamRowsFromFile } from './read-rows.mjs';
+import { log } from './log.mjs';
+
+/**
+ * @param {number} rowCount
+ * @param {string} [emptySourceLabel]
+ */
+export function assertNonEmptyImportRowCount(rowCount, emptySourceLabel) {
+    if (rowCount > 0) {
+        return;
+    }
+    const prefix = emptySourceLabel
+        ? `Import file contains no data rows: "${emptySourceLabel}". `
+        : 'Import file contains no data rows. ';
+    throw new Error(
+        prefix +
+            'The file may be empty, contain only a BOM, or contain only a header row. ' +
+            'Export the DE first to obtain a template with column names, then add rows.',
+    );
+}
+
+/**
+ * Logs a warning when the post-import row count cannot be verified or looks unexpectedly low.
+ * Skips the "unexpected" check when `countBefore` is unavailable (cannot compute expected).
+ *
+ * @param {object} opts
+ * @param {number|null} opts.countBefore - row count before any clear/import
+ * @param {boolean} opts.cleared - whether `clearDataExtensionRows` ran (DE emptied before upload)
+ * @param {number|null} opts.countAfter - row count after the import
+ * @param {number} opts.imported - number of rows sent to the API
+ * @param {'upsert'|'insert'} opts.mode
+ * @param {string} opts.label - text after "for" in messages, e.g. `DE "MyKey"` or `cred/bu DE "MyKey"`
+ */
+export function warnIfImportCountUnexpected({
+    countBefore,
+    cleared,
+    countAfter,
+    imported,
+    mode,
+    label,
+}) {
+    if (countAfter === null) {
+        log.warn(`Could not verify import result for ${label}.`);
+        return;
+    }
+    if (countBefore === null) {
+        return;
+    }
+    const effectiveCountBefore = cleared ? 0 : countBefore;
+    const expected =
+        mode === 'insert' || effectiveCountBefore === 0
+            ? effectiveCountBefore + imported
+            : imported;
+    if (countAfter < expected) {
+        log.warn(
+            `Import result for ${label} looks unexpected: expected at least ${expected} rows, got ${countAfter}.`,
+        );
+    }
+}
+
+/**
+ * @param {{ rest: { put: Function, post: Function } }} sdk
+ * @param {string} deKey
+ * @param {'upsert'|'insert'} mode
+ * @param {object[]} chunk
+ * @returns {Promise.<string|null>}
+ */
+async function postImportPayloadChunk(sdk, deKey, mode, chunk) {
+    const route = resolveImportRoute(mode);
+    const p = route.path(deKey);
+    const body = { items: chunk };
+    try {
+        const resp = await withRetry429(() =>
+            route.method === 'PUT' ? sdk.rest.put(p, body) : sdk.rest.post(p, body),
+        );
+        return resp?.requestId ?? null;
+    } catch (ex) {
+        const msgs =
+            ex instanceof RestError &&
+            ex.response?.status === 400 &&
+            Array.isArray(ex.response?.data?.resultMessages) &&
+            ex.response.data.resultMessages.length > 0
+                ? ex.response.data.resultMessages
+                : null;
+        if (msgs) {
+            const summary = msgs.map((m) => m.message ?? String(m)).join('; ');
+            throw new Error(`Import failed for DE "${deKey}" (HTTP 400): ${summary}`, {
+                cause: ex,
+            });
+        }
+        throw ex;
+    }
+}
 
 /**
  * @param {{ rest: { put: Function, post: Function } }} sdk
@@ -15,38 +107,71 @@ import { readRowsFromFile } from './read-rows.mjs';
  */
 export async function importRowsForDe(sdk, params) {
     const { deKey, rows, mode } = params;
-    const route = resolveImportRoute(mode);
     const chunks = chunkItemsForPayload(rows);
     const requestIds = [];
     for (const chunk of chunks) {
-        const p = route.path(deKey);
-        const body = { items: chunk };
-        let resp;
-        try {
-            resp = await withRetry429(() =>
-                route.method === 'PUT' ? sdk.rest.put(p, body) : sdk.rest.post(p, body),
-            );
-        } catch (ex) {
-            const msgs =
-                ex instanceof RestError &&
-                ex.response?.status === 400 &&
-                Array.isArray(ex.response?.data?.resultMessages) &&
-                ex.response.data.resultMessages.length > 0
-                    ? ex.response.data.resultMessages
-                    : null;
-            if (msgs) {
-                const summary = msgs.map((m) => m.message ?? String(m)).join('; ');
-                throw new Error(`Import failed for DE "${deKey}" (HTTP 400): ${summary}`, {
-                    cause: ex,
-                });
-            }
-            throw ex;
-        }
-        requestIds.push(resp?.requestId ?? null);
+        requestIds.push(await postImportPayloadChunk(sdk, deKey, mode, chunk));
     }
     return { count: rows.length, requestIds };
 }
 
+/**
+ * Streams row objects from disk in memory windows, then chunks for HTTP payload limits.
+ *
+ * @param {{ rest: { put: Function, post: Function } }} sdk
+ * @param {object} params
+ * @param {string} params.deKey
+ * @param {AsyncIterable<object>} params.rowSource
+ * @param {'upsert'|'insert'} params.mode
+ * @param {number} [params.maxRowsPerBatch]
+ * @param {number} params.totalMemoryBatches - upload windows (ceil(rowCount / maxRowsPerBatch))
+ * @param {string} [params.emptySourceLabel] - included in error when zero rows (e.g. file path)
+ * @returns {Promise.<{ count: number, requestIds: (string|null)[] }>}
+ */
+export async function importRowsStreamingForDe(sdk, params) {
+    const { deKey, rowSource, mode, totalMemoryBatches } = params;
+    const maxRowsPerBatch = params.maxRowsPerBatch ?? MAX_OBJECTS_PER_BATCH;
+    const emptySourceLabel = params.emptySourceLabel;
+
+    const requestIds = [];
+    let totalCount = 0;
+    let memoryBatchIndex = 0;
+    /** @type {object[]} */
+    let buffer = [];
+
+    /**
+     * @param {object[]} rows
+     * @returns {Promise.<void>}
+     */
+    async function flushMemoryBatch(rows) {
+        if (rows.length === 0) {
+            return;
+        }
+        memoryBatchIndex += 1;
+        log.info(`Uploading batch ${memoryBatchIndex} of ${totalMemoryBatches}`);
+        const httpChunks = chunkItemsForPayload(rows);
+        for (const chunk of httpChunks) {
+            requestIds.push(await postImportPayloadChunk(sdk, deKey, mode, chunk));
+        }
+    }
+
+    for await (const row of rowSource) {
+        buffer.push(row);
+        totalCount += 1;
+        if (buffer.length >= maxRowsPerBatch) {
+            await flushMemoryBatch(buffer);
+            buffer = [];
+        }
+    }
+    await flushMemoryBatch(buffer);
+
+    if (totalCount === 0) {
+        assertNonEmptyImportRowCount(0, emptySourceLabel);
+    }
+
+    return { count: totalCount, requestIds };
+}
+
 /**
  * @param {{ rest: { put: Function, post: Function } }} sdk
  * @param {object} params
@@ -63,17 +188,15 @@ export async function importFromFile(sdk, params) {
             `Cannot determine format for file: ${params.filePath}. Use .csv, .tsv, or .json extension.`,
         );
     }
-    const rows = await readRowsFromFile(params.filePath, format);
-    if (rows.length === 0) {
-        throw new Error(
-            `Import file contains no data rows: "${params.filePath}". ` +
-                `The file may be empty, contain only a BOM, or contain only a header row. ` +
-                `Export the DE first to obtain a template with column names, then add rows.`,
-        );
-    }
-    return importRowsForDe(sdk, {
+    const rowCount = await countDataRowsFromImportPaths([params.filePath], format);
+    assertNonEmptyImportRowCount(rowCount, params.filePath);
+    const totalMemoryBatches = Math.max(1, Math.ceil(rowCount / MAX_OBJECTS_PER_BATCH));
+    const rowSource = streamRowsFromFile(params.filePath, format);
+    return importRowsStreamingForDe(sdk, {
         deKey: params.deKey,
-        rows,
+        rowSource,
         mode: params.mode,
+        totalMemoryBatches,
+        emptySourceLabel: params.filePath,
     });
 }

package/lib/init-project.mjs

@@ -4,6 +4,7 @@ import readline from 'node:readline/promises';
 import { stdin as input, stdout as output } from 'node:process';
 import { FILE_MCDEV_RC, FILE_MCDEV_AUTH, FILE_MCDATA_RC, FILE_MCDATA_AUTH } from './config.mjs';
 import { fetchBusinessUnits } from './business-units.mjs';
+import { log } from './log.mjs';
 
 /**
  * @typedef {object} InitOptions
@@ -68,7 +69,7 @@ function ensureGitignore(projectRoot) {
 export async function runMcdataInit(opts) {
     const { projectRoot, isTTY, yes = false, _buFetcher } = opts;
     const out = opts.stdout ?? ((msg) => console.log(msg));
-    const err = opts.stderr ?? ((msg) => console.error(msg));
+    const err = opts.stderr ?? ((msg) => log.error(msg));
 
     // Guard: do not init over an existing mcdev project (both files must be present)
     if (

package/lib/log.mjs

@@ -0,0 +1,56 @@
+/**
+ * Timestamped logging for mcdata (local timezone via `Date`).
+ * Operational messages use info / warn / error levels.
+ * info goes to stdout; warn and error go to stderr.
+ * When a debug logger is registered via setDebugLogger(), every log.* call
+ * also appends the formatted line to the debug log file.
+ */
+
+/** @type {import('./debug-logger.mjs').DebugLogger|null} */
+let _debugLogger = null;
+
+/**
+ * @returns {string}
+ */
+export function formatTime() {
+    return new Date().toLocaleTimeString([], {
+        hour: '2-digit',
+        minute: '2-digit',
+        second: '2-digit',
+        hour12: false,
+    });
+}
+
+/**
+ * Register the debug file logger. Once set, every log.* call also appends to the log file.
+ *
+ * @param {import('./debug-logger.mjs').DebugLogger} logger
+ */
+export function setDebugLogger(logger) {
+    _debugLogger = logger;
+}
+
+/**
+ * @param {'info'|'warn'|'error'} level
+ * @param {string} message
+ */
+function write(level, message) {
+    const line = `${formatTime()} ${level}: ${message}`;
+    _debugLogger?.write(line);
+    if (level === 'error') {
+        console.error(line);
+    } else if (level === 'warn') {
+        console.warn(line);
+    } else {
+        console.log(line);
+    }
+}
+
+export const log = {
+    /** @param {string} message */
+    info: (message) => write('info', message),
+    /** @param {string} message */
+    warn: (message) => write('warn', message),
+    /** @param {string} message */
+    error: (message) => write('error', message),
+};

package/lib/multi-bu-export.mjs

@@ -2,6 +2,7 @@ import path from 'node:path';
 import SDK from 'sfmc-sdk';
 import { resolveCredentialAndMid, buildSdkAuthObject, buildSdkOptions } from './config.mjs';
 import { exportDataExtensionToFile } from './export-de.mjs';
+import { log } from './log.mjs';
 
 /**
  * @typedef {{ credential: string, bu: string }} CredBuSource
@@ -21,6 +22,7 @@ import { exportDataExtensionToFile } from './export-de.mjs';
  * @param {'csv'|'tsv'|'json'} params.format
  * @param {boolean} [params.jsonPretty]
  * @param {boolean} [params.useGit]
+ * @param {number} [params.maxRowsPerFile]
  * @param {import('./config.mjs').DebugLogger|null} [params.logger]
  * @returns {Promise.<string[]>} Paths of all written files
  */
@@ -33,6 +35,7 @@ export async function multiBuExport({
     format,
     jsonPretty = false,
     useGit = false,
+    maxRowsPerFile,
     logger = null,
 }) {
     /** @type {string[]} */
@@ -41,7 +44,7 @@ export async function multiBuExport({
         const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
         const sdk = new SDK(buildSdkAuthObject(authCred, mid), buildSdkOptions(logger));
         for (const deKey of deKeys) {
-            const { path: outPath, rowCount } = await exportDataExtensionToFile(sdk, {
+            const { paths: outPaths, rowCount } = await exportDataExtensionToFile(sdk, {
                 projectRoot,
                 credentialName: credential,
                 buName: bu,
@@ -49,9 +52,11 @@ export async function multiBuExport({
                 format,
                 jsonPretty,
                 useGit,
+                maxRowsPerFile,
             });
-            console.error(`Exported: "${path.resolve(outPath)}" (${rowCount} rows)`);
-            exported.push(outPath);
+            const label = outPaths.map((p) => `"${path.resolve(p)}"`).join(', ');
+            log.info(`Exported: ${label} (${rowCount} rows)`);
+            exported.push(...outPaths);
         }
     }
     return exported;

package/lib/read-rows.mjs

@@ -1,12 +1,49 @@
 import { createReadStream, promises as fsPromises } from 'node:fs';
 import csv from 'csv-parser';
 
+/**
+ * @param {'csv'|'tsv'} format
+ * @param {string[]} [columnHeaders]
+ * @returns {object}
+ */
+function buildCsvParserOptions(format, columnHeaders) {
+    const delimiter = format === 'tsv' ? '\t' : ',';
+    /** @type {object} */
+    const parserOptions = {
+        separator: delimiter,
+        bom: true,
+        mapHeaders: ({ header }) => {
+            let h = header;
+            if (h.startsWith('\uFEFF')) {
+                h = h.slice(1);
+            }
+            if (h.startsWith('"') && h.endsWith('"') && h.length >= 2) {
+                h = h.slice(1, -1);
+            }
+            return h;
+        },
+        mapValues: ({ value }) => {
+            if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
+                return value.slice(1, -1);
+            }
+            return value;
+        },
+    };
+    if (columnHeaders && columnHeaders.length > 0) {
+        parserOptions.headers = columnHeaders;
+    }
+    return parserOptions;
+}
+
 /**
  * @param {string} filePath
  * @param {'csv'|'tsv'|'json'} format
+ * @param {object} [options]
+ * @param {string[]} [options.columnHeaders] - when set (CSV/TSV), the file has no header row; map columns by position
  * @returns {Promise.<object[]>}
  */
-export async function readRowsFromFile(filePath, format) {
+export async function readRowsFromFile(filePath, format, options = {}) {
+    const { columnHeaders } = options;
     if (format === 'json') {
         const raw = await fsPromises.readFile(filePath, 'utf8');
         const parsed = JSON.parse(raw);
@@ -18,37 +55,129 @@ export async function readRowsFromFile(filePath, format) {
         }
         throw new Error('JSON import must be an array of row objects or { "items": [...] }');
     }
-    const delimiter = format === 'tsv' ? '\t' : ',';
+    const parserOptions = buildCsvParserOptions(format, columnHeaders);
     return new Promise((resolve, reject) => {
         const rows = [];
         createReadStream(filePath)
-            .pipe(
-                csv({
-                    separator: delimiter,
-                    bom: true,
-                    mapHeaders: ({ header }) => {
-                        let h = header;
-                        // Strip BOM if present (backup in case bom:true misses it)
-                        if (h.startsWith('\uFEFF')) {
-                            h = h.slice(1);
-                        }
-                        // Strip surrounding quotes if present (non-standard quoted TSV)
-                        if (h.startsWith('"') && h.endsWith('"') && h.length >= 2) {
-                            h = h.slice(1, -1);
-                        }
-                        return h;
-                    },
-                    mapValues: ({ value }) => {
-                        // Strip surrounding quotes from values if present
-                        if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
-                            return value.slice(1, -1);
-                        }
-                        return value;
-                    },
-                }),
-            )
+            .pipe(csv(parserOptions))
             .on('data', (row) => rows.push(row))
             .on('end', () => resolve(rows))
             .on('error', reject);
     });
 }
+
+/**
+ * Streams CSV/TSV rows from disk (one row at a time). JSON reads the full file then yields each row.
+ *
+ * @param {string} filePath
+ * @param {'csv'|'tsv'|'json'} format
+ * @param {object} [options]
+ * @param {string[]} [options.columnHeaders]
+ * @yields {object} one row object per iteration
+ * @returns {AsyncGenerator<object, void, void>}
+ */
+export async function* streamRowsFromFile(filePath, format, options = {}) {
+    const { columnHeaders } = options;
+    if (format === 'json') {
+        const rows = await readRowsFromFile(filePath, 'json');
+        for (const row of rows) {
+            yield row;
+        }
+        return;
+    }
+    const parserOptions = buildCsvParserOptions(format, columnHeaders);
+    const stream = createReadStream(filePath).pipe(csv(parserOptions));
+    for await (const row of stream) {
+        yield row;
+    }
+}
+
+/**
+ * Reads one or more export parts in order. CSV/TSV multi-part exports from mcdata include a header
+ * row only in the first file; continuation files are data-only and require inferred column names.
+ *
+ * @param {string[]} filePaths
+ * @param {'csv'|'tsv'|'json'} format
+ * @returns {Promise.<object[]>}
+ */
+export async function readRowsFromImportPaths(filePaths, format) {
+    if (filePaths.length === 0) {
+        throw new Error('readRowsFromImportPaths requires at least one file path');
+    }
+    if (format === 'json') {
+        const all = [];
+        for (const fp of filePaths) {
+            all.push(...(await readRowsFromFile(fp, 'json')));
+        }
+        return all;
+    }
+    const firstRows = await readRowsFromFile(filePaths[0], format);
+    if (filePaths.length === 1) {
+        return firstRows;
+    }
+    const headers = firstRows.length > 0 ? Object.keys(firstRows[0]) : [];
+    if (headers.length === 0) {
+        throw new Error(
+            `Cannot import multi-part ${format}: the first file has no data rows to infer columns from.`,
+        );
+    }
+    const rest = [];
+    for (let i = 1; i < filePaths.length; i++) {
+        rest.push(
+            ...(await readRowsFromFile(filePaths[i], format, {
+                columnHeaders: headers,
+            })),
+        );
+    }
+    return [...firstRows, ...rest];
+}
+
+/**
+ * Counts data rows without retaining row objects (two passes vs import: count then stream).
+ *
+ * @param {string[]} filePaths
+ * @param {'csv'|'tsv'|'json'} format
+ * @returns {Promise.<number>}
+ */
+export async function countDataRowsFromImportPaths(filePaths, format) {
+    let n = 0;
+    for await (const _row of streamRowsFromImportPaths(filePaths, format)) {
+        n += 1;
+    }
+    return n;
+}
+
+export async function* streamRowsFromImportPaths(filePaths, format) {
+    if (filePaths.length === 0) {
+        throw new Error('streamRowsFromImportPaths requires at least one file path');
+    }
+    if (format === 'json') {
+        for (const fp of filePaths) {
+            yield* streamRowsFromFile(fp, 'json');
+        }
+        return;
+    }
+    /** @type {string[]|null} */
+    let headers = null;
+    for (let i = 0; i < filePaths.length; i++) {
+        if (i === 0) {
+            let sawRow = false;
+            for await (const row of streamRowsFromFile(filePaths[0], format)) {
+                if (!sawRow) {
+                    headers = Object.keys(row);
+                    sawRow = true;
+                }
+                yield row;
+            }
+            if (filePaths.length > 1 && (!headers || headers.length === 0)) {
+                throw new Error(
+                    `Cannot import multi-part ${format}: the first file has no data rows to infer columns from.`,
+                );
+            }
+        } else {
+            yield* streamRowsFromFile(filePaths[i], format, {
+                columnHeaders: /** @type {string[]} */ (headers),
+            });
+        }
+    }
+}

package/lib/row-count.mjs

@@ -1,4 +1,5 @@
 import { rowsetGetPath } from './import-routes.mjs';
+import { log } from './log.mjs';
 
 /**
  * Fetch the total row count for a Data Extension without downloading all rows.
@@ -14,7 +15,7 @@ export async function getDeRowCount(sdk, deKey) {
         const result = await sdk.rest.get(`${rowsetGetPath(deKey)}?$page=1&$pagesize=1`);
         return result?.count ?? 0;
     } catch (ex) {
-        console.error(`Could not retrieve row count for DE "${deKey}": ${ex.message}`);
+        log.warn(`Could not retrieve row count for DE "${deKey}": ${ex.message}`);
         return null;
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sfmc-dataloader",
-    "version": "2.6.1",
+    "version": "2.7.0",
     "description": "SFMC Data Loader CLI (mcdata) — standalone export/import of Marketing Cloud Data Extension rows; optional mcdev integration",
     "author": "Jörn Berkefeld <joern.berkefeld@gmail.com>",
     "license": "MIT",
@@ -38,7 +38,7 @@
     "dependencies": {
         "csv-parser": "3.2.0",
         "csv-stringify": "6.7.0",
-        "sfmc-sdk": "3.0.3"
+        "sfmc-sdk": "^3.2.1"
     },
     "scripts": {
         "test": "node --test test/*.test.js",