sfmc-dataloader 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -3
- package/lib/cli.mjs +171 -13
- package/lib/confirm-clear.mjs +27 -7
- package/lib/cross-bu-import.mjs +199 -0
- package/lib/index.mjs +2 -0
- package/lib/multi-bu-export.mjs +44 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -18,7 +18,7 @@ npm install -g sfmc-dataloader
|
|
|
18
18
|
|
|
19
19
|
Run from your mcdev project root (where `.mcdevrc.json` lives).
|
|
20
20
|
|
|
21
|
-
### Export
|
|
21
|
+
### Export — single BU
|
|
22
22
|
|
|
23
23
|
```bash
|
|
24
24
|
mcdata export MyCred/MyBU --de MyDE_CustomerKey --format csv
|
|
@@ -26,7 +26,24 @@ mcdata export MyCred/MyBU --de MyDE_CustomerKey --format csv
|
|
|
26
26
|
|
|
27
27
|
Writes to `./data/MyCred/MyBU/<encodedKey>+MCDATA+<timestamp>.csv` (TSV/JSON with `--format`).
|
|
28
28
|
|
|
29
|
-
###
|
|
29
|
+
### Export — multiple BUs at once
|
|
30
|
+
|
|
31
|
+
Use `--from` (repeatable) instead of the positional argument to export the same DE(s) from several BUs in one command:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
mcdata export --from MyCred/Dev --from MyCred/QA --de Contact_DE --de Order_DE
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Creates one timestamped file per BU/DE combination:
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
./data/MyCred/Dev/Contact_DE+MCDATA+<timestamp>.csv
|
|
41
|
+
./data/MyCred/Dev/Order_DE+MCDATA+<timestamp>.csv
|
|
42
|
+
./data/MyCred/QA/Contact_DE+MCDATA+<timestamp>.csv
|
|
43
|
+
./data/MyCred/QA/Order_DE+MCDATA+<timestamp>.csv
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
### Import — single BU
|
|
30
47
|
|
|
31
48
|
```bash
|
|
32
49
|
mcdata import MyCred/MyBU --de MyDE_CustomerKey --format csv --api async --mode upsert
|
|
@@ -40,14 +57,50 @@ Import from explicit paths (DE key is recovered from the `+MCDATA+` filename):
|
|
|
40
57
|
mcdata import MyCred/MyBU --file ./data/MyCred/MyBU/encoded%2Bkey+MCDATA+2026-04-06T12-00-00.000Z.csv
|
|
41
58
|
```
|
|
42
59
|
|
|
60
|
+
### Import — one source BU into multiple target BUs (API mode)
|
|
61
|
+
|
|
62
|
+
Use `--from` (one source) and `--to` (repeatable targets) for a cross-BU import where rows are fetched live from the source BU:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
mcdata import --from MyCred/Dev --to MyCred/QA --to MyCred/Prod --de Contact_DE
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Before the import starts you will be offered the option to export the current data from each target BU as a timestamped backup. A timestamped download file is also written to each target BU's data directory so there is a traceable record of exactly what was imported.
|
|
69
|
+
|
|
70
|
+
### Import — local export files into multiple target BUs (file mode)
|
|
71
|
+
|
|
72
|
+
Use `--to` (repeatable targets) and `--file` (repeatable file paths) to push previously exported data files to multiple BUs without connecting to a source BU. The DE customer key is derived from each filename automatically:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
mcdata import --to MyCred/QA --to MyCred/Prod \
|
|
76
|
+
--file ./data/MyCred/Dev/Contact_DE+MCDATA+2026-04-08T10-00-00.000Z.csv
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Multiple files can be supplied to push several DEs in one command:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
mcdata import --to MyCred/QA --to MyCred/Prod \
|
|
83
|
+
--file ./data/MyCred/Dev/Contact_DE+MCDATA+2026-04-08T10-00-00.000Z.csv \
|
|
84
|
+
--file ./data/MyCred/Dev/Order_DE+MCDATA+2026-04-08T10-00-00.000Z.csv
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
A timestamped download file is written to each target BU's data directory, giving a traceable snapshot of exactly what was imported.
|
|
88
|
+
|
|
43
89
|
### Clear all rows before import
|
|
44
90
|
|
|
45
|
-
**Dangerous:** removes every row in the target Data Extension before uploading.
|
|
91
|
+
**Dangerous:** removes every row in the target Data Extension(s) before uploading.
|
|
46
92
|
|
|
93
|
+
Single-BU:
|
|
47
94
|
```bash
|
|
48
95
|
mcdata import MyCred/MyBU --de MyKey --clear-before-import
|
|
49
96
|
```
|
|
50
97
|
|
|
98
|
+
Cross-BU (warning lists every affected BU):
|
|
99
|
+
```bash
|
|
100
|
+
mcdata import --from MyCred/Dev --to MyCred/QA --to MyCred/Prod \
|
|
101
|
+
--de Contact_DE --clear-before-import
|
|
102
|
+
```
|
|
103
|
+
|
|
51
104
|
Interactive: type `YES` when prompted. In CI, add `--i-accept-clear-data-risk` after reviewing the risk.
|
|
52
105
|
|
|
53
106
|
## Options
|
|
@@ -58,6 +111,8 @@ Interactive: type `YES` when prompted. In CI, add `--i-accept-clear-data-risk` a
|
|
|
58
111
|
| `--format` | `csv` (default), `tsv`, or `json` |
|
|
59
112
|
| `--api` | `async` (default) or `sync` |
|
|
60
113
|
| `--mode` | `upsert` (default), `insert` and `update` require `--api sync` |
|
|
114
|
+
| `--from <cred>/<bu>` | Export: source BU (repeatable). Import API mode: single source BU (use with `--to` and `--de`) |
|
|
115
|
+
| `--to <cred>/<bu>` | Import: target BU (repeatable). API mode: use with `--from`/`--de`. File mode: use with `--file` (no `--from` needed) |
|
|
61
116
|
| `--clear-before-import` | SOAP `ClearData` before REST import |
|
|
62
117
|
| `--i-accept-clear-data-risk` | Non-interactive consent for clear |
|
|
63
118
|
|
package/lib/cli.mjs
CHANGED
|
@@ -15,13 +15,18 @@ import { parseExportBasename } from './filename.mjs';
|
|
|
15
15
|
import { importFromFile } from './import-de.mjs';
|
|
16
16
|
import { clearDataExtensionRows } from './clear-de.mjs';
|
|
17
17
|
import { confirmClearBeforeImport } from './confirm-clear.mjs';
|
|
18
|
+
import { multiBuExport } from './multi-bu-export.mjs';
|
|
19
|
+
import { crossBuImport } from './cross-bu-import.mjs';
|
|
18
20
|
|
|
19
21
|
function printHelp() {
|
|
20
22
|
console.log(`mcdata — SFMC Data Extension export/import (mcdev project)
|
|
21
23
|
|
|
22
24
|
Usage:
|
|
23
25
|
mcdata export <credential>/<bu> --de <key> [--de <key> ...] [options]
|
|
26
|
+
mcdata export --from <cred>/<bu> [--from <cred>/<bu> ...] --de <key> [--de <key> ...] [options]
|
|
24
27
|
mcdata import <credential>/<bu> (--de <key> ... | --file <path> ...) [options]
|
|
28
|
+
mcdata import --from <cred>/<bu> --to <cred>/<bu> [--to <cred>/<bu> ...] --de <key> ... [options]
|
|
29
|
+
mcdata import --to <cred>/<bu> [--to <cred>/<bu> ...] --file <path> [--file <path> ...] [options]
|
|
25
30
|
|
|
26
31
|
Options:
|
|
27
32
|
-p, --project <dir> mcdev project root (default: cwd)
|
|
@@ -34,10 +39,17 @@ Import options:
|
|
|
34
39
|
--clear-before-import SOAP ClearData before import (destructive; see below)
|
|
35
40
|
--i-accept-clear-data-risk Non-interactive acknowledgement for --clear-before-import
|
|
36
41
|
|
|
42
|
+
Multi-BU options:
|
|
43
|
+
--from <cred>/<bu> Export: source BU (repeatable for multiple sources)
|
|
44
|
+
Import (API mode): single source BU (use with --to and --de)
|
|
45
|
+
--to <cred>/<bu> Import: target BU (repeatable for multiple targets)
|
|
46
|
+
Import (file mode): use with --file only (no --from needed)
|
|
47
|
+
|
|
37
48
|
Notes:
|
|
38
49
|
Exports are written under ./data/<credential>/<bu>/ with "+MCDATA+" in the filename.
|
|
39
50
|
Import with --de resolves the latest matching file in that folder (by mtime).
|
|
40
51
|
Import with --file parses the DE key from the basename (+MCDATA+ prefix).
|
|
52
|
+
Cross-BU import stores a timestamped download file in each target BU's data directory.
|
|
41
53
|
|
|
42
54
|
Clear data warning:
|
|
43
55
|
--clear-before-import deletes ALL existing rows in the target DE(s) before upload.
|
|
@@ -64,6 +76,8 @@ export async function main(argv) {
|
|
|
64
76
|
file: { type: 'string', multiple: true },
|
|
65
77
|
api: { type: 'string' },
|
|
66
78
|
mode: { type: 'string' },
|
|
79
|
+
from: { type: 'string', multiple: true },
|
|
80
|
+
to: { type: 'string', multiple: true },
|
|
67
81
|
'clear-before-import': { type: 'boolean', default: false },
|
|
68
82
|
'i-accept-clear-data-risk': { type: 'boolean', default: false },
|
|
69
83
|
'json-pretty': { type: 'boolean', default: false },
|
|
@@ -88,12 +102,7 @@ export async function main(argv) {
|
|
|
88
102
|
}
|
|
89
103
|
|
|
90
104
|
const sub = positionals[0];
|
|
91
|
-
const credBuRaw = positionals[1];
|
|
92
|
-
if (!credBuRaw) {
|
|
93
|
-
console.error('Missing <credential>/<businessUnit>.');
|
|
94
|
-
printHelp();
|
|
95
|
-
return 1;
|
|
96
|
-
}
|
|
105
|
+
const credBuRaw = positionals[1]; // May be undefined when --from/--to flags are used
|
|
97
106
|
|
|
98
107
|
const projectRoot = path.resolve(values.project ?? process.cwd());
|
|
99
108
|
const fmt = values.format ?? 'csv';
|
|
@@ -102,18 +111,63 @@ export async function main(argv) {
|
|
|
102
111
|
return 1;
|
|
103
112
|
}
|
|
104
113
|
|
|
105
|
-
const
|
|
106
|
-
const
|
|
107
|
-
const
|
|
108
|
-
const
|
|
109
|
-
const
|
|
114
|
+
const fromFlags = values.from ?? [];
|
|
115
|
+
const toFlags = values.to ?? [];
|
|
116
|
+
const hasFrom = fromFlags.length > 0;
|
|
117
|
+
const hasTo = toFlags.length > 0;
|
|
118
|
+
const hasPositional = !!credBuRaw;
|
|
110
119
|
|
|
120
|
+
// ── export ──────────────────────────────────────────────────────────────
|
|
111
121
|
if (sub === 'export') {
|
|
122
|
+
if (hasTo) {
|
|
123
|
+
console.error('--to is not valid for export. Did you mean import?');
|
|
124
|
+
return 1;
|
|
125
|
+
}
|
|
126
|
+
|
|
112
127
|
const des = [].concat(values.de ?? []);
|
|
113
128
|
if (des.length === 0) {
|
|
114
129
|
console.error('export requires at least one --de <customerKey>');
|
|
115
130
|
return 1;
|
|
116
131
|
}
|
|
132
|
+
|
|
133
|
+
if (hasFrom && hasPositional) {
|
|
134
|
+
console.error('Cannot mix a positional <credential>/<bu> with --from. Use one or the other.');
|
|
135
|
+
return 1;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
if (!hasFrom && !hasPositional) {
|
|
139
|
+
console.error('export requires either a positional <credential>/<bu> or at least one --from <cred>/<bu>.');
|
|
140
|
+
printHelp();
|
|
141
|
+
return 1;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (hasFrom) {
|
|
145
|
+
// Multi-BU export
|
|
146
|
+
let sources;
|
|
147
|
+
try {
|
|
148
|
+
sources = fromFlags.map(parseCredBu);
|
|
149
|
+
} catch (e) {
|
|
150
|
+
console.error(e.message);
|
|
151
|
+
return 1;
|
|
152
|
+
}
|
|
153
|
+
const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
|
|
154
|
+
await multiBuExport({
|
|
155
|
+
projectRoot,
|
|
156
|
+
mcdevrc,
|
|
157
|
+
mcdevAuth,
|
|
158
|
+
sources,
|
|
159
|
+
deKeys: des,
|
|
160
|
+
format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
|
|
161
|
+
jsonPretty: values['json-pretty'],
|
|
162
|
+
});
|
|
163
|
+
return 0;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Single-BU export (original behavior)
|
|
167
|
+
const { credential, bu } = parseCredBu(credBuRaw);
|
|
168
|
+
const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
|
|
169
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
170
|
+
const sdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
117
171
|
for (const deKey of des) {
|
|
118
172
|
const out = await exportDataExtensionToFile(sdk, {
|
|
119
173
|
projectRoot,
|
|
@@ -128,6 +182,7 @@ export async function main(argv) {
|
|
|
128
182
|
return 0;
|
|
129
183
|
}
|
|
130
184
|
|
|
185
|
+
// ── import ──────────────────────────────────────────────────────────────
|
|
131
186
|
if (sub === 'import') {
|
|
132
187
|
const api = values.api ?? 'async';
|
|
133
188
|
const mode = values.mode ?? 'upsert';
|
|
@@ -140,6 +195,107 @@ export async function main(argv) {
|
|
|
140
195
|
return 1;
|
|
141
196
|
}
|
|
142
197
|
|
|
198
|
+
const clear = values['clear-before-import'];
|
|
199
|
+
const acceptRisk = values['i-accept-clear-data-risk'];
|
|
200
|
+
|
|
201
|
+
// ── File-to-multi-BU import: --to + --file (no --from) ─────────────
|
|
202
|
+
// Rows are read from local export files; no source BU auth needed.
|
|
203
|
+
if (hasTo && !hasFrom && values.file?.length > 0) {
|
|
204
|
+
if (hasPositional) {
|
|
205
|
+
console.error('Cannot mix a positional <credential>/<bu> with --to/--file. Use one or the other.');
|
|
206
|
+
return 1;
|
|
207
|
+
}
|
|
208
|
+
if (values.de?.length > 0) {
|
|
209
|
+
console.error('Cannot mix --de with --file in multi-target import. Use --file only.');
|
|
210
|
+
return 1;
|
|
211
|
+
}
|
|
212
|
+
const filePaths = values.file;
|
|
213
|
+
let targets;
|
|
214
|
+
try {
|
|
215
|
+
targets = toFlags.map(parseCredBu);
|
|
216
|
+
} catch (e) {
|
|
217
|
+
console.error(e.message);
|
|
218
|
+
return 1;
|
|
219
|
+
}
|
|
220
|
+
const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
|
|
221
|
+
await crossBuImport({
|
|
222
|
+
projectRoot,
|
|
223
|
+
mcdevrc,
|
|
224
|
+
mcdevAuth,
|
|
225
|
+
filePaths,
|
|
226
|
+
targets,
|
|
227
|
+
format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
|
|
228
|
+
api: /** @type {'async'|'sync'} */ (api),
|
|
229
|
+
mode: /** @type {'upsert'|'insert'|'update'} */ (mode),
|
|
230
|
+
clearBeforeImport: clear,
|
|
231
|
+
acceptRiskFlag: acceptRisk,
|
|
232
|
+
isTTY: process.stdin.isTTY === true,
|
|
233
|
+
});
|
|
234
|
+
return 0;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// ── Cross-BU import (API mode): --from + --to + --de ────────────────
|
|
238
|
+
if (hasFrom || hasTo) {
|
|
239
|
+
if (hasPositional) {
|
|
240
|
+
console.error('Cannot mix a positional <credential>/<bu> with --from/--to. Use one or the other.');
|
|
241
|
+
return 1;
|
|
242
|
+
}
|
|
243
|
+
if (!hasFrom) {
|
|
244
|
+
console.error('--to requires --from <cred>/<bu> to specify the source Business Unit.');
|
|
245
|
+
return 1;
|
|
246
|
+
}
|
|
247
|
+
if (!hasTo) {
|
|
248
|
+
console.error('--from requires at least one --to <cred>/<bu> to specify target Business Unit(s).');
|
|
249
|
+
return 1;
|
|
250
|
+
}
|
|
251
|
+
if (fromFlags.length > 1) {
|
|
252
|
+
console.error('import accepts exactly one --from <cred>/<bu> (use multiple --to for multiple targets).');
|
|
253
|
+
return 1;
|
|
254
|
+
}
|
|
255
|
+
if (values.file?.length > 0) {
|
|
256
|
+
console.error('--file cannot be combined with --from/--to/--de. For file-based multi-target import use --to + --file (without --from).');
|
|
257
|
+
return 1;
|
|
258
|
+
}
|
|
259
|
+
const deKeys = [].concat(values.de ?? []);
|
|
260
|
+
if (deKeys.length === 0) {
|
|
261
|
+
console.error('Cross-BU import requires at least one --de <customerKey>.');
|
|
262
|
+
return 1;
|
|
263
|
+
}
|
|
264
|
+
let sourceParsed;
|
|
265
|
+
let targets;
|
|
266
|
+
try {
|
|
267
|
+
sourceParsed = parseCredBu(fromFlags[0]);
|
|
268
|
+
targets = toFlags.map(parseCredBu);
|
|
269
|
+
} catch (e) {
|
|
270
|
+
console.error(e.message);
|
|
271
|
+
return 1;
|
|
272
|
+
}
|
|
273
|
+
const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
|
|
274
|
+
await crossBuImport({
|
|
275
|
+
projectRoot,
|
|
276
|
+
mcdevrc,
|
|
277
|
+
mcdevAuth,
|
|
278
|
+
sourceCred: sourceParsed.credential,
|
|
279
|
+
sourceBu: sourceParsed.bu,
|
|
280
|
+
targets,
|
|
281
|
+
deKeys,
|
|
282
|
+
format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
|
|
283
|
+
api: /** @type {'async'|'sync'} */ (api),
|
|
284
|
+
mode: /** @type {'upsert'|'insert'|'update'} */ (mode),
|
|
285
|
+
clearBeforeImport: clear,
|
|
286
|
+
acceptRiskFlag: acceptRisk,
|
|
287
|
+
isTTY: process.stdin.isTTY === true,
|
|
288
|
+
});
|
|
289
|
+
return 0;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// ── Single-BU import (original behavior) ────────────────────────────
|
|
293
|
+
if (!hasPositional) {
|
|
294
|
+
console.error('import requires either a positional <credential>/<bu> or --from/--to flags.');
|
|
295
|
+
printHelp();
|
|
296
|
+
return 1;
|
|
297
|
+
}
|
|
298
|
+
|
|
143
299
|
const hasDe = values.de?.length > 0;
|
|
144
300
|
const hasFile = values.file?.length > 0;
|
|
145
301
|
if (hasDe === hasFile) {
|
|
@@ -147,8 +303,10 @@ export async function main(argv) {
|
|
|
147
303
|
return 1;
|
|
148
304
|
}
|
|
149
305
|
|
|
150
|
-
const
|
|
151
|
-
const
|
|
306
|
+
const { credential, bu } = parseCredBu(credBuRaw);
|
|
307
|
+
const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
|
|
308
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
309
|
+
const sdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
152
310
|
|
|
153
311
|
if (hasDe) {
|
|
154
312
|
const deKeys = [].concat(values.de ?? []);
|
package/lib/confirm-clear.mjs
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
import readline from 'node:readline/promises';
|
|
2
2
|
import { stdin as input, stdout as output } from 'node:process';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* @typedef {{ credential: string, bu: string }} CredBuTarget
|
|
6
|
+
*/
|
|
7
|
+
|
|
4
8
|
/**
|
|
5
9
|
* @param {object} opts
|
|
6
10
|
* @param {string[]} opts.deKeys
|
|
7
11
|
* @param {boolean} opts.acceptRiskFlag
|
|
8
12
|
* @param {boolean} opts.isTTY
|
|
13
|
+
* @param {CredBuTarget[]} [opts.targets] When present, renders a per-BU breakdown.
|
|
9
14
|
* @param {NodeJS.ReadableStream} [opts.stdin]
|
|
10
15
|
* @param {NodeJS.WritableStream} [opts.stdout]
|
|
11
16
|
* @returns {Promise<void>}
|
|
12
17
|
*/
|
|
13
18
|
export async function confirmClearBeforeImport(opts) {
|
|
14
|
-
const { deKeys, acceptRiskFlag, isTTY } = opts;
|
|
19
|
+
const { deKeys, targets, acceptRiskFlag, isTTY } = opts;
|
|
15
20
|
const stdin = opts.stdin ?? input;
|
|
16
21
|
const stdout = opts.stdout ?? output;
|
|
17
22
|
if (acceptRiskFlag) {
|
|
@@ -23,12 +28,27 @@ export async function confirmClearBeforeImport(opts) {
|
|
|
23
28
|
'All rows in the target Data Extension(s) would be permanently deleted.'
|
|
24
29
|
);
|
|
25
30
|
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
let msg;
|
|
32
|
+
if (targets && targets.length > 0) {
|
|
33
|
+
msg =
|
|
34
|
+
'\n*** DANGER: CLEAR DATA ***\n' +
|
|
35
|
+
`This will permanently DELETE ALL ROWS across ${targets.length} Business Unit(s):\n\n` +
|
|
36
|
+
targets
|
|
37
|
+
.map(
|
|
38
|
+
({ credential, bu }) =>
|
|
39
|
+
` ${credential}/${bu}:\n` + deKeys.map((k) => ` - ${k}\n`).join('')
|
|
40
|
+
)
|
|
41
|
+
.join('') +
|
|
42
|
+
'\nThis cannot be undone. Enterprise 2.0 / admin / shared-DE rules may apply.\n' +
|
|
43
|
+
'Type YES to continue, anything else to abort: ';
|
|
44
|
+
} else {
|
|
45
|
+
msg =
|
|
46
|
+
'\n*** DANGER: CLEAR DATA ***\n' +
|
|
47
|
+
'This will permanently DELETE ALL ROWS in:\n' +
|
|
48
|
+
deKeys.map((k) => ` - ${k}\n`).join('') +
|
|
49
|
+
'This cannot be undone. Enterprise 2.0 / admin / shared-DE rules may apply.\n' +
|
|
50
|
+
'Type YES to continue, anything else to abort: ';
|
|
51
|
+
}
|
|
32
52
|
stdout.write(msg);
|
|
33
53
|
const rl = readline.createInterface({ input: stdin, output: stdout });
|
|
34
54
|
try {
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import readline from 'node:readline/promises';
|
|
4
|
+
import { stdin as input, stdout as output } from 'node:process';
|
|
5
|
+
import SDK from 'sfmc-sdk';
|
|
6
|
+
import { resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
|
|
7
|
+
import { fetchAllRowObjects, serializeRows } from './export-de.mjs';
|
|
8
|
+
import { exportDataExtensionToFile } from './export-de.mjs';
|
|
9
|
+
import { importRowsForDe } from './import-de.mjs';
|
|
10
|
+
import { readRowsFromFile } from './read-rows.mjs';
|
|
11
|
+
import { clearDataExtensionRows } from './clear-de.mjs';
|
|
12
|
+
import { confirmClearBeforeImport } from './confirm-clear.mjs';
|
|
13
|
+
import { dataDirectoryForBu } from './paths.mjs';
|
|
14
|
+
import { buildExportBasename, filesystemSafeTimestamp, parseExportBasename } from './filename.mjs';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* @typedef {{ credential: string, bu: string }} CredBuTarget
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* In TTY mode, asks the user whether to export target DE data as backup
|
|
22
|
+
* before importing. Returns true when the user answers YES.
|
|
23
|
+
*
|
|
24
|
+
* @param {object} opts
|
|
25
|
+
* @param {CredBuTarget[]} opts.targets
|
|
26
|
+
* @param {string[]} opts.deKeys
|
|
27
|
+
* @param {NodeJS.ReadableStream} [opts.stdin]
|
|
28
|
+
* @param {NodeJS.WritableStream} [opts.stdout]
|
|
29
|
+
* @returns {Promise<boolean>}
|
|
30
|
+
*/
|
|
31
|
+
async function offerPreExportBackup({ targets, deKeys, stdin: stdinStream, stdout: stdoutStream }) {
|
|
32
|
+
const stdinSrc = stdinStream ?? input;
|
|
33
|
+
const stdoutSrc = stdoutStream ?? output;
|
|
34
|
+
const targetList = targets.map(({ credential, bu }) => `${credential}/${bu}`).join(', ');
|
|
35
|
+
const msg =
|
|
36
|
+
'\nBefore importing, would you like to export the current data from target BU(s) as a backup?\n' +
|
|
37
|
+
'This creates timestamped files that will not be overwritten by the following import.\n\n' +
|
|
38
|
+
` Target(s): ${targetList}\n` +
|
|
39
|
+
` Data Extensions: ${deKeys.join(', ')}\n\n` +
|
|
40
|
+
'Type YES to export first, or press Enter to skip: ';
|
|
41
|
+
stdoutSrc.write(msg);
|
|
42
|
+
const rl = readline.createInterface({ input: stdinSrc, output: stdoutSrc });
|
|
43
|
+
try {
|
|
44
|
+
const line = await rl.question('');
|
|
45
|
+
return line.trim() === 'YES';
|
|
46
|
+
} finally {
|
|
47
|
+
rl.close();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Imports Data Extension rows from a single source BU into one or more target
|
|
53
|
+
* BUs.
|
|
54
|
+
*
|
|
55
|
+
* Two source modes are supported:
|
|
56
|
+
*
|
|
57
|
+
* **API mode** (default): rows are fetched live from the source BU via the
|
|
58
|
+
* SFMC REST API. Requires `sourceCred`, `sourceBu`, and `deKeys`.
|
|
59
|
+
*
|
|
60
|
+
* **File mode**: rows are read from local export files (e.g. previously
|
|
61
|
+
* created by `mcdata export`). Requires `filePaths`; `deKeys` and
|
|
62
|
+
* `sourceCred`/`sourceBu` must be omitted. The DE customer key is derived
|
|
63
|
+
* from each filename via `parseExportBasename`.
|
|
64
|
+
*
|
|
65
|
+
* Before the import each target BU:
|
|
66
|
+
* 1. Optionally exports its current DE data as a timestamped backup (TTY only).
|
|
67
|
+
* 2. Optionally clears all existing rows (with danger warning covering every target).
|
|
68
|
+
* 3. Receives the source rows written to a timestamped "download" file in its
|
|
69
|
+
* own `./data/<credential>/<bu>/` directory (mirroring mcdev retrieve).
|
|
70
|
+
* 4. Has the rows imported via the REST API.
|
|
71
|
+
*
|
|
72
|
+
* @param {object} params
|
|
73
|
+
* @param {string} params.projectRoot
|
|
74
|
+
* @param {import('./config.mjs').Mcdevrc} params.mcdevrc
|
|
75
|
+
* @param {Record<string, import('./config.mjs').AuthCredential>} params.mcdevAuth
|
|
76
|
+
* @param {string} [params.sourceCred] - API mode only
|
|
77
|
+
* @param {string} [params.sourceBu] - API mode only
|
|
78
|
+
* @param {string[]} [params.deKeys] - API mode only
|
|
79
|
+
* @param {string[]} [params.filePaths] - File mode only; mutually exclusive with sourceCred/sourceBu/deKeys
|
|
80
|
+
* @param {CredBuTarget[]} params.targets
|
|
81
|
+
* @param {'csv'|'tsv'|'json'} params.format
|
|
82
|
+
* @param {'async'|'sync'} params.api
|
|
83
|
+
* @param {'upsert'|'insert'|'update'} params.mode
|
|
84
|
+
* @param {boolean} params.clearBeforeImport
|
|
85
|
+
* @param {boolean} params.acceptRiskFlag
|
|
86
|
+
* @param {boolean} params.isTTY
|
|
87
|
+
* @param {NodeJS.ReadableStream} [params.stdin] Override for testing
|
|
88
|
+
* @param {NodeJS.WritableStream} [params.stdout] Override for testing
|
|
89
|
+
* @returns {Promise<void>}
|
|
90
|
+
*/
|
|
91
|
+
export async function crossBuImport(params) {
|
|
92
|
+
const {
|
|
93
|
+
projectRoot,
|
|
94
|
+
mcdevrc,
|
|
95
|
+
mcdevAuth,
|
|
96
|
+
targets,
|
|
97
|
+
format,
|
|
98
|
+
api,
|
|
99
|
+
mode,
|
|
100
|
+
clearBeforeImport,
|
|
101
|
+
acceptRiskFlag,
|
|
102
|
+
isTTY,
|
|
103
|
+
} = params;
|
|
104
|
+
const stdin = params.stdin;
|
|
105
|
+
const stdout = params.stdout;
|
|
106
|
+
|
|
107
|
+
// Determine source mode
|
|
108
|
+
const filePaths = params.filePaths ?? null;
|
|
109
|
+
const isFileBased = filePaths !== null && filePaths.length > 0;
|
|
110
|
+
|
|
111
|
+
// Derive DE keys: from explicit list (API mode) or from filenames (file mode)
|
|
112
|
+
const deKeys = isFileBased
|
|
113
|
+
? filePaths.map((fp) => parseExportBasename(path.basename(fp)).customerKey)
|
|
114
|
+
: (params.deKeys ?? []);
|
|
115
|
+
|
|
116
|
+
// Build a lookup map from deKey → filePath for file mode
|
|
117
|
+
/** @type {Map<string, string>} */
|
|
118
|
+
const fileByDeKey = new Map();
|
|
119
|
+
if (isFileBased) {
|
|
120
|
+
for (const fp of filePaths) {
|
|
121
|
+
fileByDeKey.set(parseExportBasename(path.basename(fp)).customerKey, fp);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Validate all target BU configurations upfront
|
|
126
|
+
for (const { credential, bu } of targets) {
|
|
127
|
+
resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Connect to source BU (API mode only)
|
|
131
|
+
let srcSdk = null;
|
|
132
|
+
if (!isFileBased) {
|
|
133
|
+
const { mid: srcMid, authCred: srcAuth } = resolveCredentialAndMid(
|
|
134
|
+
mcdevrc, mcdevAuth, params.sourceCred, params.sourceBu
|
|
135
|
+
);
|
|
136
|
+
srcSdk = new SDK(buildSdkAuthObject(srcAuth, srcMid), { requestAttempts: 3 });
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Optional pre-import backup of target BU data
|
|
140
|
+
if (isTTY) {
|
|
141
|
+
const doBackup = await offerPreExportBackup({ targets, deKeys, stdin, stdout });
|
|
142
|
+
if (doBackup) {
|
|
143
|
+
for (const { credential, bu } of targets) {
|
|
144
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
145
|
+
const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
146
|
+
for (const deKey of deKeys) {
|
|
147
|
+
const outPath = await exportDataExtensionToFile(tgtSdk, {
|
|
148
|
+
projectRoot,
|
|
149
|
+
credentialName: credential,
|
|
150
|
+
buName: bu,
|
|
151
|
+
deKey,
|
|
152
|
+
format,
|
|
153
|
+
});
|
|
154
|
+
console.error(`Backup export: ${outPath}`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Single up-front clear confirmation covering all targets + all DE keys
|
|
161
|
+
if (clearBeforeImport) {
|
|
162
|
+
await confirmClearBeforeImport({ deKeys, targets, acceptRiskFlag, isTTY, stdin, stdout });
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Load rows once per DE then fan out to every target
|
|
166
|
+
for (const deKey of deKeys) {
|
|
167
|
+
const rows = isFileBased
|
|
168
|
+
? await readRowsFromFile(fileByDeKey.get(deKey), format)
|
|
169
|
+
: await fetchAllRowObjects(srcSdk, deKey);
|
|
170
|
+
|
|
171
|
+
// Clear targets before import (rows already confirmed above)
|
|
172
|
+
if (clearBeforeImport) {
|
|
173
|
+
for (const { credential, bu } of targets) {
|
|
174
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
175
|
+
const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
176
|
+
await clearDataExtensionRows(tgtSdk.soap, deKey);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
for (const { credential, bu } of targets) {
|
|
181
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
182
|
+
const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
183
|
+
|
|
184
|
+
// Write a timestamped "download" file in the target BU's data directory.
|
|
185
|
+
// This mirrors what mcdev retrieve / mcdata export produces, giving a
|
|
186
|
+
// traceable snapshot of exactly what was imported.
|
|
187
|
+
const dir = dataDirectoryForBu(projectRoot, credential, bu);
|
|
188
|
+
await fs.mkdir(dir, { recursive: true });
|
|
189
|
+
const ts = filesystemSafeTimestamp();
|
|
190
|
+
const basename = buildExportBasename(deKey, ts, format);
|
|
191
|
+
const snapshotPath = path.join(dir, basename);
|
|
192
|
+
await fs.writeFile(snapshotPath, serializeRows(rows, format, false), 'utf8');
|
|
193
|
+
console.error(`Download stored: ${snapshotPath}`);
|
|
194
|
+
|
|
195
|
+
await importRowsForDe(tgtSdk, { deKey, rows, api, mode });
|
|
196
|
+
console.error(`Imported -> ${credential}/${bu} DE ${deKey}`);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
package/lib/index.mjs
CHANGED
|
@@ -3,3 +3,5 @@ export { filterIllegalFilenames, reverseFilterIllegalFilenames, parseExportBasen
|
|
|
3
3
|
export { chunkItemsForPayload, DEFAULT_MAX_BODY_BYTES, MAX_OBJECTS_PER_BATCH } from './batch.mjs';
|
|
4
4
|
export { resolveImportRoute, rowsetGetPath, asyncUpsertPath } from './import-routes.mjs';
|
|
5
5
|
export { loadMcdevProject, parseCredBu, resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
|
|
6
|
+
export { multiBuExport } from './multi-bu-export.mjs';
|
|
7
|
+
export { crossBuImport } from './cross-bu-import.mjs';
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import SDK from 'sfmc-sdk';
|
|
2
|
+
import { resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
|
|
3
|
+
import { exportDataExtensionToFile } from './export-de.mjs';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @typedef {{ credential: string, bu: string }} CredBuSource
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Exports one or more Data Extensions from multiple source BUs in a single
|
|
11
|
+
* pass. Each source BU gets its own timestamped file per DE key under
|
|
12
|
+
* `./data/<credential>/<bu>/`.
|
|
13
|
+
*
|
|
14
|
+
* @param {object} params
|
|
15
|
+
* @param {string} params.projectRoot
|
|
16
|
+
* @param {import('./config.mjs').Mcdevrc} params.mcdevrc
|
|
17
|
+
* @param {Record<string, import('./config.mjs').AuthCredential>} params.mcdevAuth
|
|
18
|
+
* @param {CredBuSource[]} params.sources
|
|
19
|
+
* @param {string[]} params.deKeys
|
|
20
|
+
* @param {'csv'|'tsv'|'json'} params.format
|
|
21
|
+
* @param {boolean} [params.jsonPretty]
|
|
22
|
+
* @returns {Promise<string[]>} Paths of all written files
|
|
23
|
+
*/
|
|
24
|
+
export async function multiBuExport({ projectRoot, mcdevrc, mcdevAuth, sources, deKeys, format, jsonPretty = false }) {
|
|
25
|
+
/** @type {string[]} */
|
|
26
|
+
const exported = [];
|
|
27
|
+
for (const { credential, bu } of sources) {
|
|
28
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
29
|
+
const sdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
30
|
+
for (const deKey of deKeys) {
|
|
31
|
+
const outPath = await exportDataExtensionToFile(sdk, {
|
|
32
|
+
projectRoot,
|
|
33
|
+
credentialName: credential,
|
|
34
|
+
buName: bu,
|
|
35
|
+
deKey,
|
|
36
|
+
format,
|
|
37
|
+
jsonPretty,
|
|
38
|
+
});
|
|
39
|
+
console.error(`Exported: ${outPath}`);
|
|
40
|
+
exported.push(outPath);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return exported;
|
|
44
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sfmc-dataloader",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "CLI (mcdata) to export and import Marketing Cloud Data Extension rows using mcdev project config and sfmc-sdk",
|
|
5
5
|
"author": "Jörn Berkefeld <joern.berkefeld@gmail.com>",
|
|
6
6
|
"license": "MIT",
|