sfmc-dataloader 1.1.0 → 1.2.0

package/README.md

@@ -57,9 +57,9 @@ Import from explicit paths (DE key is recovered from the `+MCDATA+` filename):
 mcdata import MyCred/MyBU --file ./data/MyCred/MyBU/encoded%2Bkey+MCDATA+2026-04-06T12-00-00.000Z.csv
 ```
 
-### Import — one source BU into multiple target BUs
+### Import — one source BU into multiple target BUs (API mode)
 
-Use `--from` (one source) and `--to` (repeatable targets) for a cross-BU import:
+Use `--from` (one source) and `--to` (repeatable targets) for a cross-BU import where rows are fetched live from the source BU:
 
 ```bash
 mcdata import --from MyCred/Dev --to MyCred/QA --to MyCred/Prod --de Contact_DE
@@ -67,6 +67,25 @@ mcdata import --from MyCred/Dev --to MyCred/QA --to MyCred/Prod --de Contact_DE
 
 Before the import starts you will be offered the option to export the current data from each target BU as a timestamped backup. A timestamped download file is also written to each target BU's data directory so there is a traceable record of exactly what was imported.
 
+### Import — local export files into multiple target BUs (file mode)
+
+Use `--to` (repeatable targets) and `--file` (repeatable file paths) to push previously exported data files to multiple BUs without connecting to a source BU. The DE customer key is derived from each filename automatically:
+
+```bash
+mcdata import --to MyCred/QA --to MyCred/Prod \
+  --file ./data/MyCred/Dev/Contact_DE+MCDATA+2026-04-08T10-00-00.000Z.csv
+```
+
+Multiple files can be supplied to push several DEs in one command:
+
+```bash
+mcdata import --to MyCred/QA --to MyCred/Prod \
+  --file ./data/MyCred/Dev/Contact_DE+MCDATA+2026-04-08T10-00-00.000Z.csv \
+  --file ./data/MyCred/Dev/Order_DE+MCDATA+2026-04-08T10-00-00.000Z.csv
+```
+
+A timestamped download file is written to each target BU's data directory, giving a traceable snapshot of exactly what was imported.
+
 ### Clear all rows before import
 
 **Dangerous:** removes every row in the target Data Extension(s) before uploading.
@@ -92,8 +111,8 @@ Interactive: type `YES` when prompted. In CI, add `--i-accept-clear-data-risk` a
 | `--format` | `csv` (default), `tsv`, or `json` |
 | `--api` | `async` (default) or `sync` |
 | `--mode` | `upsert` (default), `insert` and `update` require `--api sync` |
-| `--from <cred>/<bu>` | Export: source BU (repeatable). Import: single source BU (use with `--to`) |
-| `--to <cred>/<bu>` | Import: target BU (repeatable for multiple targets) |
+| `--from <cred>/<bu>` | Export: source BU (repeatable). Import API mode: single source BU (use with `--to` and `--de`) |
+| `--to <cred>/<bu>` | Import: target BU (repeatable). API mode: use with `--from`/`--de`. File mode: use with `--file` (no `--from` needed) |
 | `--clear-before-import` | SOAP `ClearData` before REST import |
 | `--i-accept-clear-data-risk` | Non-interactive consent for clear |
 

package/lib/cli.mjs

@@ -26,6 +26,7 @@ Usage:
   mcdata export --from <cred>/<bu> [--from <cred>/<bu> ...] --de <key> [--de <key> ...] [options]
   mcdata import <credential>/<bu> (--de <key> ... | --file <path> ...) [options]
   mcdata import --from <cred>/<bu> --to <cred>/<bu> [--to <cred>/<bu> ...] --de <key> ... [options]
+  mcdata import --to <cred>/<bu> [--to <cred>/<bu> ...] --file <path> [--file <path> ...] [options]
 
 Options:
   -p, --project <dir>     mcdev project root (default: cwd)
@@ -40,8 +41,9 @@ Import options:
 
 Multi-BU options:
   --from <cred>/<bu>      Export: source BU (repeatable for multiple sources)
-                          Import: single source BU (use with --to)
+                          Import (API mode): single source BU (use with --to and --de)
   --to <cred>/<bu>        Import: target BU (repeatable for multiple targets)
+                          Import (file mode): use with --file only (no --from needed)
 
 Notes:
   Exports are written under ./data/<credential>/<bu>/ with "+MCDATA+" in the filename.
@@ -196,7 +198,43 @@ export async function main(argv) {
         const clear = values['clear-before-import'];
         const acceptRisk = values['i-accept-clear-data-risk'];
 
-        // ── Cross-BU import: --from + --to ──────────────────────────────────
+        // ── File-to-multi-BU import: --to + --file (no --from) ─────────────
+        // Rows are read from local export files; no source BU auth needed.
+        if (hasTo && !hasFrom && values.file?.length > 0) {
+            if (hasPositional) {
+                console.error('Cannot mix a positional <credential>/<bu> with --to/--file. Use one or the other.');
+                return 1;
+            }
+            if (values.de?.length > 0) {
+                console.error('Cannot mix --de with --file in multi-target import. Use --file only.');
+                return 1;
+            }
+            const filePaths = values.file;
+            let targets;
+            try {
+                targets = toFlags.map(parseCredBu);
+            } catch (e) {
+                console.error(e.message);
+                return 1;
+            }
+            const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
+            await crossBuImport({
+                projectRoot,
+                mcdevrc,
+                mcdevAuth,
+                filePaths,
+                targets,
+                format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
+                api: /** @type {'async'|'sync'} */ (api),
+                mode: /** @type {'upsert'|'insert'|'update'} */ (mode),
+                clearBeforeImport: clear,
+                acceptRiskFlag: acceptRisk,
+                isTTY: process.stdin.isTTY === true,
+            });
+            return 0;
+        }
+
+        // ── Cross-BU import (API mode): --from + --to + --de ────────────────
         if (hasFrom || hasTo) {
             if (hasPositional) {
                 console.error('Cannot mix a positional <credential>/<bu> with --from/--to. Use one or the other.');
@@ -215,7 +253,7 @@ export async function main(argv) {
                 return 1;
             }
             if (values.file?.length > 0) {
-                console.error('--file cannot be combined with --from/--to. Use --de instead.');
+                console.error('--file cannot be combined with --from/--to/--de. For file-based multi-target import use --to + --file (without --from).');
                 return 1;
             }
             const deKeys = [].concat(values.de ?? []);

package/lib/cross-bu-import.mjs

@@ -7,10 +7,11 @@ import { resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
 import { fetchAllRowObjects, serializeRows } from './export-de.mjs';
 import { exportDataExtensionToFile } from './export-de.mjs';
 import { importRowsForDe } from './import-de.mjs';
+import { readRowsFromFile } from './read-rows.mjs';
 import { clearDataExtensionRows } from './clear-de.mjs';
 import { confirmClearBeforeImport } from './confirm-clear.mjs';
 import { dataDirectoryForBu } from './paths.mjs';
-import { buildExportBasename, filesystemSafeTimestamp } from './filename.mjs';
+import { buildExportBasename, filesystemSafeTimestamp, parseExportBasename } from './filename.mjs';
 
 /**
  * @typedef {{ credential: string, bu: string }} CredBuTarget
@@ -51,6 +52,16 @@ async function offerPreExportBackup({ targets, deKeys, stdin: stdinStream, stdou
  * Imports Data Extension rows from a single source BU into one or more target
  * BUs.
  *
+ * Two source modes are supported:
+ *
+ * **API mode** (default): rows are fetched live from the source BU via the
+ * SFMC REST API.  Requires `sourceCred`, `sourceBu`, and `deKeys`.
+ *
+ * **File mode**: rows are read from local export files (e.g. previously
+ * created by `mcdata export`).  Requires `filePaths`; `deKeys` and
+ * `sourceCred`/`sourceBu` must be omitted.  The DE customer key is derived
+ * from each filename via `parseExportBasename`.
+ *
  * Before the import each target BU:
  * 1. Optionally exports its current DE data as a timestamped backup (TTY only).
  * 2. Optionally clears all existing rows (with danger warning covering every target).
@@ -62,10 +73,11 @@ async function offerPreExportBackup({ targets, deKeys, stdin: stdinStream, stdou
  * @param {string} params.projectRoot
  * @param {import('./config.mjs').Mcdevrc} params.mcdevrc
  * @param {Record<string, import('./config.mjs').AuthCredential>} params.mcdevAuth
- * @param {string} params.sourceCred
- * @param {string} params.sourceBu
+ * @param {string} [params.sourceCred] - API mode only
+ * @param {string} [params.sourceBu]   - API mode only
+ * @param {string[]} [params.deKeys]   - API mode only
+ * @param {string[]} [params.filePaths] - File mode only; mutually exclusive with sourceCred/sourceBu/deKeys
  * @param {CredBuTarget[]} params.targets
- * @param {string[]} params.deKeys
  * @param {'csv'|'tsv'|'json'} params.format
  * @param {'async'|'sync'} params.api
  * @param {'upsert'|'insert'|'update'} params.mode
@@ -81,10 +93,7 @@ export async function crossBuImport(params) {
         projectRoot,
         mcdevrc,
         mcdevAuth,
-        sourceCred,
-        sourceBu,
         targets,
-        deKeys,
         format,
         api,
         mode,
@@ -95,14 +104,37 @@ export async function crossBuImport(params) {
     const stdin = params.stdin;
     const stdout = params.stdout;
 
-    // Validate all BU configurations upfront before making any API calls
-    const { mid: srcMid, authCred: srcAuth } = resolveCredentialAndMid(mcdevrc, mcdevAuth, sourceCred, sourceBu);
+    // Determine source mode
+    const filePaths = params.filePaths ?? null;
+    const isFileBased = filePaths !== null && filePaths.length > 0;
+
+    // Derive DE keys: from explicit list (API mode) or from filenames (file mode)
+    const deKeys = isFileBased
+        ? filePaths.map((fp) => parseExportBasename(path.basename(fp)).customerKey)
+        : (params.deKeys ?? []);
+
+    // Build a lookup map from deKey → filePath for file mode
+    /** @type {Map<string, string>} */
+    const fileByDeKey = new Map();
+    if (isFileBased) {
+        for (const fp of filePaths) {
+            fileByDeKey.set(parseExportBasename(path.basename(fp)).customerKey, fp);
+        }
+    }
+
+    // Validate all target BU configurations upfront
     for (const { credential, bu } of targets) {
         resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
     }
 
-    // Connect to source BU
-    const srcSdk = new SDK(buildSdkAuthObject(srcAuth, srcMid), { requestAttempts: 3 });
+    // Connect to source BU (API mode only)
+    let srcSdk = null;
+    if (!isFileBased) {
+        const { mid: srcMid, authCred: srcAuth } = resolveCredentialAndMid(
+            mcdevrc, mcdevAuth, params.sourceCred, params.sourceBu
+        );
+        srcSdk = new SDK(buildSdkAuthObject(srcAuth, srcMid), { requestAttempts: 3 });
+    }
 
     // Optional pre-import backup of target BU data
     if (isTTY) {
@@ -130,9 +162,11 @@ export async function crossBuImport(params) {
         await confirmClearBeforeImport({ deKeys, targets, acceptRiskFlag, isTTY, stdin, stdout });
     }
 
-    // Fetch rows once per DE from the source, then fan out to every target
+    // Load rows once per DE then fan out to every target
     for (const deKey of deKeys) {
-        const rows = await fetchAllRowObjects(srcSdk, deKey);
+        const rows = isFileBased
+            ? await readRowsFromFile(fileByDeKey.get(deKey), format)
+            : await fetchAllRowObjects(srcSdk, deKey);
 
         // Clear targets before import (rows already confirmed above)
         if (clearBeforeImport) {
@@ -154,9 +188,9 @@ export async function crossBuImport(params) {
             await fs.mkdir(dir, { recursive: true });
             const ts = filesystemSafeTimestamp();
             const basename = buildExportBasename(deKey, ts, format);
-            const filePath = path.join(dir, basename);
-            await fs.writeFile(filePath, serializeRows(rows, format, false), 'utf8');
-            console.error(`Download stored: ${filePath}`);
+            const snapshotPath = path.join(dir, basename);
+            await fs.writeFile(snapshotPath, serializeRows(rows, format, false), 'utf8');
+            console.error(`Download stored: ${snapshotPath}`);
 
             await importRowsForDe(tgtSdk, { deKey, rows, api, mode });
             console.error(`Imported -> ${credential}/${bu} DE ${deKey}`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sfmc-dataloader",
-    "version": "1.1.0",
+    "version": "1.2.0",
     "description": "CLI (mcdata) to export and import Marketing Cloud Data Extension rows using mcdev project config and sfmc-sdk",
     "author": "Jörn Berkefeld <joern.berkefeld@gmail.com>",
     "license": "MIT",