sfmc-dataloader 1.0.0

package/README.md

@@ -0,0 +1,66 @@
+# sfmc-dataloader
+
+Command-line tool **`mcdata`** to export and import Salesforce Marketing Cloud Data Extension rows using the same project files as [mcdev](https://github.com/Accenture/sfmc-devtools) (`.mcdevrc.json`, `.mcdev-auth.json`) and [sfmc-sdk](https://www.npmjs.com/package/sfmc-sdk) for REST/SOAP.
+
+## Requirements
+
+- Node.js `^20.19.0 || ^22.13.0 || >=24` (aligned with `sfmc-sdk`)
+- A mcdev-style project with credentials on disk
+- Peer: `mcdev` `>=7` (declare alongside your project tooling)
+
+## Install
+
+```bash
+npm install -g sfmc-dataloader
+```
+
+## Usage
+
+Run from your mcdev project root (where `.mcdevrc.json` lives).
+
+### Export
+
+```bash
+mcdata export MyCred/MyBU --de MyDE_CustomerKey --format csv
+```
+
+Writes to `./data/MyCred/MyBU/<encodedKey>+MCDATA+<timestamp>.csv` (TSV/JSON with `--format`).
+
+### Import
+
+```bash
+mcdata import MyCred/MyBU --de MyDE_CustomerKey --format csv --api async --mode upsert
+```
+
+Resolves the latest matching export file under `./data/MyCred/MyBU/` for that DE key.
+
+Import from explicit paths (DE key is recovered from the `+MCDATA+` filename):
+
+```bash
+mcdata import MyCred/MyBU --file ./data/MyCred/MyBU/encoded%2Bkey+MCDATA+2026-04-06T12-00-00.000Z.csv
+```
+
+### Clear all rows before import
+
+**Dangerous:** removes every row in the target Data Extension before uploading.
+
+```bash
+mcdata import MyCred/MyBU --de MyKey --clear-before-import
+```
+
+Interactive: type `YES` when prompted. In CI, add `--i-accept-clear-data-risk` after reviewing the risk.
+
+## Options
+
+| Option | Description |
+|--------|-------------|
+| `-p, --project` | Project root (default: cwd) |
+| `--format` | `csv` (default), `tsv`, or `json` |
+| `--api` | `async` (default) or `sync` |
+| `--mode` | `upsert` (default), `insert` and `update` require `--api sync` |
+| `--clear-before-import` | SOAP `ClearData` before REST import |
+| `--i-accept-clear-data-risk` | Non-interactive consent for clear |
+
+## License
+
+MIT — Author: Jörn Berkefeld

package/bin/mcdata.mjs

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+import { main } from '../lib/cli.mjs';
+
+main(process.argv)
+    .then((code) => process.exit(typeof code === 'number' ? code : 0))
+    .catch((err) => {
+        console.error(err);
+        process.exit(1);
+    });

package/lib/batch.mjs

@@ -0,0 +1,42 @@
+/** Default max UTF-8 bytes per request body (Data API family; margin below 5.9 MB). */
+export const DEFAULT_MAX_BODY_BYTES = 5_500_000;
+
+/** Salesforce hard cap on objects per batch. */
+export const MAX_OBJECTS_PER_BATCH = 5_000;
+
+/**
+ * Split rows into chunks that respect both max row count and serialized JSON body size.
+ * Uses JSON.stringify on `{ items: chunk }` to estimate bytes (same shape as REST body).
+ *
+ * @param {object[]} rows - row objects (flat field map)
+ * @param {object} [opts]
+ * @param {number} [opts.maxBytes]
+ * @param {number} [opts.maxObjects]
+ * @returns {object[][]}
+ */
+export function chunkItemsForPayload(rows, opts = {}) {
+    const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BODY_BYTES;
+    const maxObjects = opts.maxObjects ?? MAX_OBJECTS_PER_BATCH;
+    const out = [];
+    let i = 0;
+    while (i < rows.length) {
+        const chunk = [];
+        while (i < rows.length && chunk.length < maxObjects) {
+            const next = rows[i];
+            const trial = [...chunk, next];
+            const bytes = Buffer.byteLength(JSON.stringify({ items: trial }), 'utf8');
+            if (bytes > maxBytes) {
+                if (chunk.length > 0) {
+                    break;
+                }
+                chunk.push(next);
+                i++;
+                break;
+            }
+            chunk.push(next);
+            i++;
+        }
+        out.push(chunk);
+    }
+    return out;
+}

package/lib/clear-de.mjs

@@ -0,0 +1,13 @@
+/**
+ * Remove all rows from a Data Extension via SOAP Perform (ClearData).
+ * Requires appropriate tenant and user rights; may fail for shared DEs from non-owner BUs.
+ *
+ * @param {*} soap - SDK soap instance
+ * @param {string} customerKey - DE external key
+ * @returns {Promise<any>}
+ */
+export async function clearDataExtensionRows(soap, customerKey) {
+    return soap.perform('DataExtension', 'ClearData', {
+        CustomerKey: customerKey,
+    });
+}

package/lib/cli.mjs

@@ -0,0 +1,216 @@
+import { parseArgs } from 'node:util';
+import process from 'node:process';
+import path from 'node:path';
+import SDK from 'sfmc-sdk';
+import {
+    loadMcdevProject,
+    parseCredBu,
+    resolveCredentialAndMid,
+    buildSdkAuthObject,
+} from './config.mjs';
+import { dataDirectoryForBu } from './paths.mjs';
+import { exportDataExtensionToFile } from './export-de.mjs';
+import { findImportCandidates, pickLatestByMtime } from './file-resolve.mjs';
+import { parseExportBasename } from './filename.mjs';
+import { importFromFile } from './import-de.mjs';
+import { clearDataExtensionRows } from './clear-de.mjs';
+import { confirmClearBeforeImport } from './confirm-clear.mjs';
+
+function printHelp() {
+    console.log(`mcdata — SFMC Data Extension export/import (mcdev project)
+
+Usage:
+  mcdata export <credential>/<bu> --de <key> [--de <key> ...] [options]
+  mcdata import <credential>/<bu> (--de <key> ... | --file <path> ...) [options]
+
+Options:
+  -p, --project <dir>     mcdev project root (default: cwd)
+  --format <csv|tsv|json> File format (default: csv)
+  --json-pretty           Pretty-print JSON on export
+
+Import options:
+  --api <async|sync>      REST row API family (default: async)
+  --mode <upsert|insert|update>  (default: upsert; insert/update require --api sync)
+  --clear-before-import   SOAP ClearData before import (destructive; see below)
+  --i-accept-clear-data-risk  Non-interactive acknowledgement for --clear-before-import
+
+Notes:
+  Exports are written under ./data/<credential>/<bu>/ with "+MCDATA+" in the filename.
+  Import with --de resolves the latest matching file in that folder (by mtime).
+  Import with --file parses the DE key from the basename (+MCDATA+ prefix).
+
+Clear data warning:
+  --clear-before-import deletes ALL existing rows in the target DE(s) before upload.
+  Interactive: type YES when prompted. CI: also pass --i-accept-clear-data-risk.
+`);
+}
+
+/**
+ * @param {string[]} argv
+ * @returns {Promise<number>} exit code
+ */
+export async function main(argv) {
+    let values;
+    let positionals;
+    try {
+        const parsed = parseArgs({
+            args: argv.slice(2),
+            allowPositionals: true,
+            strict: true,
+            options: {
+                project: { type: 'string', short: 'p' },
+                format: { type: 'string' },
+                de: { type: 'string', multiple: true },
+                file: { type: 'string', multiple: true },
+                api: { type: 'string' },
+                mode: { type: 'string' },
+                'clear-before-import': { type: 'boolean', default: false },
+                'i-accept-clear-data-risk': { type: 'boolean', default: false },
+                'json-pretty': { type: 'boolean', default: false },
+                help: { type: 'boolean', short: 'h', default: false },
+            },
+        });
+        values = parsed.values;
+        positionals = parsed.positionals;
+    } catch (e) {
+        console.error(e.message);
+        printHelp();
+        return 1;
+    }
+
+    if (values.help) {
+        printHelp();
+        return 0;
+    }
+    if (positionals.length === 0) {
+        printHelp();
+        return 1;
+    }
+
+    const sub = positionals[0];
+    const credBuRaw = positionals[1];
+    if (!credBuRaw) {
+        console.error('Missing <credential>/<businessUnit>.');
+        printHelp();
+        return 1;
+    }
+
+    const projectRoot = path.resolve(values.project ?? process.cwd());
+    const fmt = values.format ?? 'csv';
+    if (!['csv', 'tsv', 'json'].includes(fmt)) {
+        console.error(`Invalid --format: ${fmt}`);
+        return 1;
+    }
+
+    const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
+    const { credential, bu } = parseCredBu(credBuRaw);
+    const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
+    const authObj = buildSdkAuthObject(authCred, mid);
+    const sdk = new SDK(authObj, { requestAttempts: 3 });
+
+    if (sub === 'export') {
+        const des = [].concat(values.de ?? []);
+        if (des.length === 0) {
+            console.error('export requires at least one --de <customerKey>');
+            return 1;
+        }
+        for (const deKey of des) {
+            const out = await exportDataExtensionToFile(sdk, {
+                projectRoot,
+                credentialName: credential,
+                buName: bu,
+                deKey,
+                format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
+                jsonPretty: values['json-pretty'],
+            });
+            console.error(`Exported: ${out}`);
+        }
+        return 0;
+    }
+
+    if (sub === 'import') {
+        const api = values.api ?? 'async';
+        const mode = values.mode ?? 'upsert';
+        if (!['async', 'sync'].includes(api)) {
+            console.error(`Invalid --api: ${api}`);
+            return 1;
+        }
+        if (!['upsert', 'insert', 'update'].includes(mode)) {
+            console.error(`Invalid --mode: ${mode}`);
+            return 1;
+        }
+
+        const hasDe = values.de?.length > 0;
+        const hasFile = values.file?.length > 0;
+        if (hasDe === hasFile) {
+            console.error('import requires exactly one of: repeated --de <key> OR repeated --file <path>');
+            return 1;
+        }
+
+        const clear = values['clear-before-import'];
+        const acceptRisk = values['i-accept-clear-data-risk'];
+
+        if (hasDe) {
+            const deKeys = [].concat(values.de ?? []);
+            const dataDir = dataDirectoryForBu(projectRoot, credential, bu);
+            if (clear) {
+                await confirmClearBeforeImport({
+                    deKeys,
+                    acceptRiskFlag: acceptRisk,
+                    isTTY: process.stdin.isTTY === true,
+                });
+                for (const deKey of deKeys) {
+                    await clearDataExtensionRows(sdk.soap, deKey);
+                }
+            }
+            for (const deKey of deKeys) {
+                const candidates = await findImportCandidates(dataDir, deKey, fmt);
+                if (candidates.length === 0) {
+                    console.error(`No ${fmt} file found for DE "${deKey}" under ${dataDir}`);
+                    return 1;
+                }
+                const filePath =
+                    candidates.length === 1 ? candidates[0] : await pickLatestByMtime(candidates);
+                await importFromFile(sdk, {
+                    filePath,
+                    deKey,
+                    format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
+                    api: /** @type {'async'|'sync'} */ (api),
+                    mode: /** @type {'upsert'|'insert'|'update'} */ (mode),
+                });
+                console.error(`Imported ${filePath} -> DE ${deKey}`);
+            }
+            return 0;
+        }
+
+        const fileList = values.file ?? [];
+        const keysFromFiles = fileList.map((fp) => parseExportBasename(path.basename(fp)).customerKey);
+        if (clear) {
+            await confirmClearBeforeImport({
+                deKeys: keysFromFiles,
+                acceptRiskFlag: acceptRisk,
+                isTTY: process.stdin.isTTY === true,
+            });
+            for (const deKey of keysFromFiles) {
+                await clearDataExtensionRows(sdk.soap, deKey);
+            }
+        }
+        for (const filePath of fileList) {
+            const base = path.basename(filePath);
+            const { customerKey } = parseExportBasename(base);
+            await importFromFile(sdk, {
+                filePath,
+                deKey: customerKey,
+                format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
+                api: /** @type {'async'|'sync'} */ (api),
+                mode: /** @type {'upsert'|'insert'|'update'} */ (mode),
+            });
+            console.error(`Imported ${filePath}`);
+        }
+        return 0;
+    }
+
+    console.error(`Unknown command: ${sub}`);
+    printHelp();
+    return 1;
+}

package/lib/config.mjs

@@ -0,0 +1,95 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * @typedef {object} McdevrcCredentials
+ * @property {Record<string, Record<string, number|string>>} businessUnits
+ */
+
+/**
+ * @typedef {object} Mcdevrc
+ * @property {Record<string, McdevrcCredentials>} credentials
+ */
+
+/**
+ * @typedef {object} AuthCredential
+ * @property {string} client_id
+ * @property {string} client_secret
+ * @property {string} auth_url
+ */
+
+/**
+ * @param {string} projectRoot
+ * @returns {{ mcdevrc: Mcdevrc, mcdevAuth: Record<string, AuthCredential> }}
+ */
+export function loadMcdevProject(projectRoot) {
+    const rcPath = path.join(projectRoot, '.mcdevrc.json');
+    const authPath = path.join(projectRoot, '.mcdev-auth.json');
+    if (!fs.existsSync(rcPath)) {
+        throw new Error(`Missing ${rcPath}`);
+    }
+    if (!fs.existsSync(authPath)) {
+        throw new Error(`Missing ${authPath}`);
+    }
+    const mcdevrc = JSON.parse(fs.readFileSync(rcPath, 'utf8'));
+    const mcdevAuth = JSON.parse(fs.readFileSync(authPath, 'utf8'));
+    return { mcdevrc, mcdevAuth };
+}
+
+/**
+ * @param {string} credBu - `CredentialName/BUName`
+ * @returns {{ credential: string, bu: string }}
+ */
+export function parseCredBu(credBu) {
+    const slash = credBu.indexOf('/');
+    if (slash <= 0 || slash === credBu.length - 1) {
+        throw new Error(`Expected <credential>/<businessUnit>, got: ${credBu}`);
+    }
+    return {
+        credential: credBu.slice(0, slash),
+        bu: credBu.slice(slash + 1),
+    };
+}
+
+/**
+ * @param {Mcdevrc} mcdevrc
+ * @param {Record<string, AuthCredential>} mcdevAuth
+ * @param {string} credentialName
+ * @param {string} buName
+ * @returns {{ mid: number, authCred: AuthCredential }}
+ */
+export function resolveCredentialAndMid(mcdevrc, mcdevAuth, credentialName, buName) {
+    const credBlock = mcdevrc.credentials?.[credentialName];
+    if (!credBlock) {
+        throw new Error(`Unknown credential "${credentialName}" in .mcdevrc.json`);
+    }
+    const midRaw = credBlock.businessUnits?.[buName];
+    if (midRaw === undefined || midRaw === null) {
+        throw new Error(`Unknown business unit "${buName}" under credential "${credentialName}"`);
+    }
+    const mid =
+        typeof midRaw === 'number' ? midRaw : Number.parseInt(String(midRaw), 10);
+    if (!Number.isInteger(mid)) {
+        throw new Error(`Invalid MID for ${credentialName}/${buName}: ${midRaw}`);
+    }
+    const authCred = mcdevAuth[credentialName];
+    if (!authCred?.client_id || !authCred?.client_secret || !authCred?.auth_url) {
+        throw new Error(`Missing auth fields for credential "${credentialName}" in .mcdev-auth.json`);
+    }
+    return { mid, authCred };
+}
+
+/**
+ * Auth object for sfmc-sdk `Auth` / `SDK` constructor.
+ * @param {AuthCredential} authCred
+ * @param {number} mid
+ * @returns {import('sfmc-sdk').AuthObject}
+ */
+export function buildSdkAuthObject(authCred, mid) {
+    return {
+        client_id: authCred.client_id,
+        client_secret: authCred.client_secret,
+        auth_url: authCred.auth_url,
+        account_id: mid,
+    };
+}

package/lib/confirm-clear.mjs

@@ -0,0 +1,42 @@
+import readline from 'node:readline/promises';
+import { stdin as input, stdout as output } from 'node:process';
+
+/**
+ * @param {object} opts
+ * @param {string[]} opts.deKeys
+ * @param {boolean} opts.acceptRiskFlag
+ * @param {boolean} opts.isTTY
+ * @param {NodeJS.ReadableStream} [opts.stdin]
+ * @param {NodeJS.WritableStream} [opts.stdout]
+ * @returns {Promise<void>}
+ */
+export async function confirmClearBeforeImport(opts) {
+    const { deKeys, acceptRiskFlag, isTTY } = opts;
+    const stdin = opts.stdin ?? input;
+    const stdout = opts.stdout ?? output;
+    if (acceptRiskFlag) {
+        return;
+    }
+    if (!isTTY) {
+        throw new Error(
+            'Refusing to clear data in non-interactive mode without --i-accept-clear-data-risk. ' +
+                'All rows in the target Data Extension(s) would be permanently deleted.'
+        );
+    }
+    const msg =
+        '\n*** DANGER: CLEAR DATA ***\n' +
+        'This will permanently DELETE ALL ROWS in:\n' +
+        deKeys.map((k) => `  - ${k}\n`).join('') +
+        'This cannot be undone. Enterprise 2.0 / admin / shared-DE rules may apply.\n' +
+        'Type YES to continue, anything else to abort: ';
+    stdout.write(msg);
+    const rl = readline.createInterface({ input: stdin, output: stdout });
+    try {
+        const line = await rl.question('');
+        if (line.trim() !== 'YES') {
+            throw new Error('Aborted by user (clear not confirmed).');
+        }
+    } finally {
+        rl.close();
+    }
+}

package/lib/export-de.mjs

@@ -0,0 +1,86 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { stringify } from 'csv-stringify/sync';
+import { rowsetGetPath } from './import-routes.mjs';
+import { buildExportBasename, filesystemSafeTimestamp } from './filename.mjs';
+import { dataDirectoryForBu } from './paths.mjs';
+
+/**
+ * @param {{ rest: { get: (path: string) => Promise<any> } }} sdk
+ * @param {string} deKey
+ * @returns {Promise<object[]>}
+ */
+export async function fetchAllRowObjects(sdk, deKey) {
+    const pageSize = 2500;
+    let page = 1;
+    const rows = [];
+    let hasMore = true;
+    while (hasMore) {
+        const qs = new URLSearchParams({
+            page: String(page),
+            pageSize: String(pageSize),
+        });
+        const urlPath = `${rowsetGetPath(deKey)}?${qs.toString()}`;
+        const data = await sdk.rest.get(urlPath);
+        const items = data.items ?? [];
+        for (const item of items) {
+            rows.push({ ...item.keys, ...item.values });
+        }
+        if (items.length === 0) {
+            hasMore = false;
+        } else if (data.hasMoreRows === false) {
+            hasMore = false;
+        } else if (data.hasMoreRows === true) {
+            hasMore = true;
+            page++;
+        } else {
+            hasMore = items.length === pageSize;
+            page++;
+        }
+    }
+    return rows;
+}
+
+/**
+ * @param {object[]} rows
+ * @param {'csv'|'tsv'|'json'} format
+ * @param {boolean} jsonPretty
+ * @returns {string}
+ */
+export function serializeRows(rows, format, jsonPretty) {
+    if (format === 'json') {
+        const space = jsonPretty ? 2 : undefined;
+        return JSON.stringify(rows, null, space) + '\n';
+    }
+    const delimiter = format === 'tsv' ? '\t' : ',';
+    return stringify(rows, {
+        header: true,
+        quoted: true,
+        bom: true,
+        delimiter,
+    });
+}
+
+/**
+ * @param {{ rest: { get: (path: string) => Promise<any> } }} sdk
+ * @param {object} params
+ * @param {string} params.projectRoot
+ * @param {string} params.credentialName
+ * @param {string} params.buName
+ * @param {string} params.deKey
+ * @param {'csv'|'tsv'|'json'} params.format
+ * @param {boolean} [params.jsonPretty]
+ * @returns {Promise<string>} written file path
+ */
+export async function exportDataExtensionToFile(sdk, params) {
+    const { projectRoot, credentialName, buName, deKey, format, jsonPretty = false } = params;
+    const rows = await fetchAllRowObjects(sdk, deKey);
+    const dir = dataDirectoryForBu(projectRoot, credentialName, buName);
+    await fs.mkdir(dir, { recursive: true });
+    const ts = filesystemSafeTimestamp();
+    const basename = buildExportBasename(deKey, ts, format);
+    const outPath = path.join(dir, basename);
+    const body = serializeRows(rows, format, jsonPretty);
+    await fs.writeFile(outPath, body, 'utf8');
+    return outPath;
+}

package/lib/file-resolve.mjs

@@ -0,0 +1,57 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { filterIllegalFilenames, MCDATA_SENTINEL } from './filename.mjs';
+
+/**
+ * Find export files under data dir matching encoded DE key prefix and extension.
+ *
+ * @param {string} dataDir
+ * @param {string} customerKey
+ * @param {'csv'|'tsv'|'json'} format
+ * @returns {Promise<string[]>} full paths
+ */
+export async function findImportCandidates(dataDir, customerKey, format) {
+    const prefix = filterIllegalFilenames(customerKey) + MCDATA_SENTINEL;
+    let entries;
+    try {
+        entries = await fs.readdir(dataDir, { withFileTypes: true });
+    } catch {
+        return [];
+    }
+    const ext = format;
+    const matches = [];
+    for (const ent of entries) {
+        if (!ent.isFile()) {
+            continue;
+        }
+        const name = ent.name;
+        if (!name.endsWith(`.${ext}`)) {
+            continue;
+        }
+        if (name.startsWith(prefix)) {
+            matches.push(path.join(dataDir, name));
+        }
+    }
+    return matches;
+}
+
+/**
+ * @param {string[]} paths
+ * @returns {Promise<string>} path with newest mtime
+ */
+export async function pickLatestByMtime(paths) {
+    if (paths.length === 0) {
+        throw new Error('No candidate files');
+    }
+    let best = paths[0];
+    let bestTime = 0;
+    for (const p of paths) {
+        const st = await fs.stat(p);
+        const t = st.mtimeMs;
+        if (t >= bestTime) {
+            bestTime = t;
+            best = p;
+        }
+    }
+    return best;
+}

package/lib/filename.mjs

@@ -0,0 +1,80 @@
+/**
+ * Mirrors sfmc-devtools `File.filterIllegalFilenames` / `reverseFilterIllegalFilenames`
+ * so export filenames stay consistent with mcdev retrieve-style paths.
+ * @see https://github.com/Accenture/sfmc-devtools (lib/util/file.js)
+ */
+
+/**
+ * @param {string} filename
+ * @returns {string}
+ */
+export function filterIllegalFilenames(filename) {
+    return (
+        encodeURIComponent(filename)
+            .replaceAll(/[*]/g, '_STAR_')
+            .split('%20')
+            .join(' ')
+            .split('%7B')
+            .join('{')
+            .split('%7D')
+            .join('}')
+            .split('%5B')
+            .join('[')
+            .split('%5D')
+            .join(']')
+            .split('%40')
+            .join('@')
+    );
+}
+
+/**
+ * @param {string} filename
+ * @returns {string}
+ */
+export function reverseFilterIllegalFilenames(filename) {
+    return decodeURIComponent(filename).split('_STAR_').join('*');
+}
+
+/** Sentinel between encoded DE key and timestamp in export basenames; cannot appear in the key segment after encoding. */
+export const MCDATA_SENTINEL = '+MCDATA+';
+
+/**
+ * @param {string} customerKey
+ * @param {string} safeTs - filesystem-safe UTC timestamp
+ * @param {'csv'|'tsv'|'json'} ext
+ * @returns {string} basename without directory
+ */
+export function buildExportBasename(customerKey, safeTs, ext) {
+    return `${filterIllegalFilenames(customerKey)}${MCDATA_SENTINEL}${safeTs}.${ext}`;
+}
+
+/**
+ * @param {Date} [d]
+ * @returns {string} e.g. 2026-04-06T15-48-30Z
+ */
+export function filesystemSafeTimestamp(d = new Date()) {
+    return d.toISOString().replaceAll(':', '-');
+}
+
+/**
+ * @param {string} basename - e.g. `encodedKey+MCDATA+2026-04-06T15-00-00.000Z.csv`
+ * @returns {{ customerKey: string, timestampPart: string, ext: string }}
+ */
+export function parseExportBasename(basename) {
+    const lastDot = basename.lastIndexOf('.');
+    const stem = lastDot === -1 ? basename : basename.slice(0, lastDot);
+    const ext = lastDot === -1 ? '' : basename.slice(lastDot + 1);
+    const idx = stem.indexOf(MCDATA_SENTINEL);
+    if (idx === -1) {
+        throw new Error(
+            `Filename must contain "${MCDATA_SENTINEL}" between encoded key and timestamp: ${basename}`
+        );
+    }
+    const encodedKey = stem.slice(0, idx);
+    const timestampPart = stem.slice(idx + MCDATA_SENTINEL.length);
+    return {
+        customerKey: reverseFilterIllegalFilenames(encodedKey),
+        timestampPart,
+        ext: ext.toLowerCase(),
+    };
+}

package/lib/import-de.mjs

@@ -0,0 +1,48 @@
+import { chunkItemsForPayload } from './batch.mjs';
+import { resolveImportRoute } from './import-routes.mjs';
+import { withRetry429 } from './retry.mjs';
+import { readRowsFromFile } from './read-rows.mjs';
+
+/**
+ * @param {{ rest: { put: Function, post: Function } }} sdk
+ * @param {object} params
+ * @param {string} params.deKey
+ * @param {object[]} params.rows
+ * @param {'async'|'sync'} params.api
+ * @param {'upsert'|'insert'|'update'} params.mode
+ * @returns {Promise<void>}
+ */
+export async function importRowsForDe(sdk, params) {
+    const { deKey, rows, api, mode } = params;
+    const route = resolveImportRoute(api, mode);
+    const chunks = chunkItemsForPayload(rows);
+    for (const chunk of chunks) {
+        const path = route.path(deKey);
+        const body = { items: chunk };
+        await withRetry429(() =>
+            route.method === 'PUT'
+                ? sdk.rest.put(path, body)
+                : sdk.rest.post(path, body)
+        );
+    }
+}
+
+/**
+ * @param {{ rest: { put: Function, post: Function } }} sdk
+ * @param {object} params
+ * @param {string} params.filePath
+ * @param {string} params.deKey - target DE customer key for API
+ * @param {'csv'|'tsv'|'json'} params.format
+ * @param {'async'|'sync'} params.api
+ * @param {'upsert'|'insert'|'update'} params.mode
+ * @returns {Promise<void>}
+ */
+export async function importFromFile(sdk, params) {
+    const rows = await readRowsFromFile(params.filePath, params.format);
+    await importRowsForDe(sdk, {
+        deKey: params.deKey,
+        rows,
+        api: params.api,
+        mode: params.mode,
+    });
+}

package/lib/import-routes.mjs

@@ -0,0 +1,67 @@
+/**
+ * REST paths for Data Extension row writes (relative to REST base URL).
+ * Confirm against current Salesforce reference hubs when adjusting behavior.
+ */
+
+/**
+ * @param {string} deKey
+ * @returns {string} path starting with /
+ */
+export function rowsetGetPath(deKey) {
+    return `/data/v1/customobjectdata/key/${encodeURIComponent(deKey)}/rowset`;
+}
+
+/**
+ * Async bulk upsert (default for `--api async`).
+ * @param {string} deKey
+ * @returns {string}
+ */
+export function asyncUpsertPath(deKey) {
+    return `/data/v1/async/dataextensions/key:${encodeURIComponent(deKey)}/rows`;
+}
+
+/**
+ * Synchronous upsert row set.
+ * @param {string} deKey
+ * @returns {string}
+ */
+export function syncUpsertPath(deKey) {
+    return `/data/v1/customobjectdata/key/${encodeURIComponent(deKey)}/rows`;
+}
+
+/**
+ * Synchronous insert row set (POST).
+ * @param {string} deKey
+ * @returns {string}
+ */
+export function syncInsertPath(deKey) {
+    return `/data/v1/customobjectdata/key/${encodeURIComponent(deKey)}/rows`;
+}
+
+/**
+ * @param {'async'|'sync'} api
+ * @param {'upsert'|'insert'|'update'} mode
+ * @returns {{ method: 'PUT'|'POST', path: (de: string) => string }}
+ */
+export function resolveImportRoute(api, mode) {
+    if (api === 'async') {
+        if (mode !== 'upsert') {
+            throw new Error(
+                `Import mode "${mode}" is not supported with --api async (use --api sync or --mode upsert).`
+            );
+        }
+        return { method: 'PUT', path: asyncUpsertPath };
+    }
+    if (api === 'sync') {
+        if (mode === 'upsert') {
+            return { method: 'PUT', path: syncUpsertPath };
+        }
+        if (mode === 'insert') {
+            return { method: 'POST', path: syncInsertPath };
+        }
+        if (mode === 'update') {
+            return { method: 'PUT', path: syncUpsertPath };
+        }
+    }
+    throw new Error(`Unsupported --api / --mode combination: ${api} + ${mode}`);
+}

package/lib/index.mjs

@@ -0,0 +1,5 @@
+export { main } from './cli.mjs';
+export { filterIllegalFilenames, reverseFilterIllegalFilenames, parseExportBasename } from './filename.mjs';
+export { chunkItemsForPayload, DEFAULT_MAX_BODY_BYTES, MAX_OBJECTS_PER_BATCH } from './batch.mjs';
+export { resolveImportRoute, rowsetGetPath, asyncUpsertPath } from './import-routes.mjs';
+export { loadMcdevProject, parseCredBu, resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';

package/lib/paths.mjs

@@ -0,0 +1,11 @@
+import path from 'node:path';
+
+/**
+ * @param {string} projectRoot
+ * @param {string} credentialName
+ * @param {string} buName
+ * @returns {string} absolute path ./data/<cred>/<bu>/
+ */
+export function dataDirectoryForBu(projectRoot, credentialName, buName) {
+    return path.join(projectRoot, 'data', credentialName, buName);
+}

package/lib/read-rows.mjs

@@ -0,0 +1,35 @@
+import { createReadStream, promises as fsPromises } from 'node:fs';
+import csv from 'csv-parser';
+
+/**
+ * @param {string} filePath
+ * @param {'csv'|'tsv'|'json'} format
+ * @returns {Promise<object[]>}
+ */
+export async function readRowsFromFile(filePath, format) {
+    if (format === 'json') {
+        const raw = await fsPromises.readFile(filePath, 'utf8');
+        const parsed = JSON.parse(raw);
+        if (Array.isArray(parsed)) {
+            return parsed;
+        }
+        if (parsed && typeof parsed === 'object' && Array.isArray(parsed.items)) {
+            return parsed.items;
+        }
+        throw new Error('JSON import must be an array of row objects or { "items": [...] }');
+    }
+    const delimiter = format === 'tsv' ? '\t' : ',';
+    return new Promise((resolve, reject) => {
+        const rows = [];
+        createReadStream(filePath)
+            .pipe(
+                csv({
+                    separator: delimiter,
+                    bom: true,
+                })
+            )
+            .on('data', (row) => rows.push(row))
+            .on('end', () => resolve(rows))
+            .on('error', reject);
+    });
+}

package/lib/retry.mjs

@@ -0,0 +1,40 @@
+import { RestError } from 'sfmc-sdk/util';
+
+/**
+ * @param {() => Promise<any>} fn
+ * @param {object} [opts]
+ * @param {number} [opts.maxAttempts] default 5
+ * @returns {Promise<any>}
+ */
+export async function withRetry429(fn, opts = {}) {
+    const maxAttempts = opts.maxAttempts ?? 5;
+    let attempt = 0;
+    let delayMs = 1000;
+    while (true) {
+        attempt++;
+        try {
+            return await fn();
+        } catch (e) {
+            const status = e instanceof RestError ? e.response?.status : undefined;
+            const retryAfter = e instanceof RestError ? e.response?.headers?.['retry-after'] : undefined;
+            if (status === 429 && attempt < maxAttempts) {
+                const wait =
+                    retryAfter !== undefined
+                        ? Number.parseInt(String(retryAfter), 10) * 1000 || delayMs
+                        : delayMs;
+                await sleep(wait);
+                delayMs = Math.min(delayMs * 2, 60_000);
+                continue;
+            }
+            throw e;
+        }
+    }
+}
+
+/**
+ * @param {number} ms
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}

package/package.json

@@ -0,0 +1,49 @@
+{
+    "name": "sfmc-dataloader",
+    "version": "1.0.0",
+    "description": "CLI (mcdata) to export and import Marketing Cloud Data Extension rows using mcdev project config and sfmc-sdk",
+    "author": "Jörn Berkefeld <joern.berkefeld@gmail.com>",
+    "license": "MIT",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/JoernBerkefeld/sfmc-dataloader.git"
+    },
+    "bugs": {
+        "url": "https://github.com/JoernBerkefeld/sfmc-dataloader/issues"
+    },
+    "keywords": [
+        "sfmc",
+        "marketing-cloud",
+        "data-extension",
+        "mcdev",
+        "cli",
+        "exacttarget"
+    ],
+    "type": "module",
+    "bin": {
+        "mcdata": "./bin/mcdata.mjs"
+    },
+    "main": "./lib/index.mjs",
+    "exports": {
+        ".": "./lib/index.mjs"
+    },
+    "files": [
+        "bin",
+        "lib",
+        "README.md"
+    ],
+    "engines": {
+        "node": "^20.19.0 || ^22.13.0 || >=24"
+    },
+    "peerDependencies": {
+        "mcdev": ">=7"
+    },
+    "dependencies": {
+        "csv-parser": "3.2.0",
+        "csv-stringify": "6.5.2",
+        "sfmc-sdk": "3.0.3"
+    },
+    "scripts": {
+        "test": "node --test test/**/*.test.js"
+    }
+}