sfmc-dataloader 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -3
- package/lib/cli.mjs +133 -13
- package/lib/confirm-clear.mjs +27 -7
- package/lib/cross-bu-import.mjs +165 -0
- package/lib/index.mjs +2 -0
- package/lib/multi-bu-export.mjs +44 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -18,7 +18,7 @@ npm install -g sfmc-dataloader
|
|
|
18
18
|
|
|
19
19
|
Run from your mcdev project root (where `.mcdevrc.json` lives).
|
|
20
20
|
|
|
21
|
-
### Export
|
|
21
|
+
### Export — single BU
|
|
22
22
|
|
|
23
23
|
```bash
|
|
24
24
|
mcdata export MyCred/MyBU --de MyDE_CustomerKey --format csv
|
|
@@ -26,7 +26,24 @@ mcdata export MyCred/MyBU --de MyDE_CustomerKey --format csv
|
|
|
26
26
|
|
|
27
27
|
Writes to `./data/MyCred/MyBU/<encodedKey>+MCDATA+<timestamp>.csv` (TSV/JSON with `--format`).
|
|
28
28
|
|
|
29
|
-
###
|
|
29
|
+
### Export — multiple BUs at once
|
|
30
|
+
|
|
31
|
+
Use `--from` (repeatable) instead of the positional argument to export the same DE(s) from several BUs in one command:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
mcdata export --from MyCred/Dev --from MyCred/QA --de Contact_DE --de Order_DE
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Creates one timestamped file per BU/DE combination:
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
./data/MyCred/Dev/Contact_DE+MCDATA+<timestamp>.csv
|
|
41
|
+
./data/MyCred/Dev/Order_DE+MCDATA+<timestamp>.csv
|
|
42
|
+
./data/MyCred/QA/Contact_DE+MCDATA+<timestamp>.csv
|
|
43
|
+
./data/MyCred/QA/Order_DE+MCDATA+<timestamp>.csv
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
### Import — single BU
|
|
30
47
|
|
|
31
48
|
```bash
|
|
32
49
|
mcdata import MyCred/MyBU --de MyDE_CustomerKey --format csv --api async --mode upsert
|
|
@@ -40,14 +57,31 @@ Import from explicit paths (DE key is recovered from the `+MCDATA+` filename):
|
|
|
40
57
|
mcdata import MyCred/MyBU --file ./data/MyCred/MyBU/encoded%2Bkey+MCDATA+2026-04-06T12-00-00.000Z.csv
|
|
41
58
|
```
|
|
42
59
|
|
|
60
|
+
### Import — one source BU into multiple target BUs
|
|
61
|
+
|
|
62
|
+
Use `--from` (one source) and `--to` (repeatable targets) for a cross-BU import:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
mcdata import --from MyCred/Dev --to MyCred/QA --to MyCred/Prod --de Contact_DE
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Before the import starts you will be offered the option to export the current data from each target BU as a timestamped backup. A timestamped download file is also written to each target BU's data directory so there is a traceable record of exactly what was imported.
|
|
69
|
+
|
|
43
70
|
### Clear all rows before import
|
|
44
71
|
|
|
45
|
-
**Dangerous:** removes every row in the target Data Extension before uploading.
|
|
72
|
+
**Dangerous:** removes every row in the target Data Extension(s) before uploading.
|
|
46
73
|
|
|
74
|
+
Single-BU:
|
|
47
75
|
```bash
|
|
48
76
|
mcdata import MyCred/MyBU --de MyKey --clear-before-import
|
|
49
77
|
```
|
|
50
78
|
|
|
79
|
+
Cross-BU (warning lists every affected BU):
|
|
80
|
+
```bash
|
|
81
|
+
mcdata import --from MyCred/Dev --to MyCred/QA --to MyCred/Prod \
|
|
82
|
+
--de Contact_DE --clear-before-import
|
|
83
|
+
```
|
|
84
|
+
|
|
51
85
|
Interactive: type `YES` when prompted. In CI, add `--i-accept-clear-data-risk` after reviewing the risk.
|
|
52
86
|
|
|
53
87
|
## Options
|
|
@@ -58,6 +92,8 @@ Interactive: type `YES` when prompted. In CI, add `--i-accept-clear-data-risk` a
|
|
|
58
92
|
| `--format` | `csv` (default), `tsv`, or `json` |
|
|
59
93
|
| `--api` | `async` (default) or `sync` |
|
|
60
94
|
| `--mode` | `upsert` (default), `insert` and `update` require `--api sync` |
|
|
95
|
+
| `--from <cred>/<bu>` | Export: source BU (repeatable). Import: single source BU (use with `--to`) |
|
|
96
|
+
| `--to <cred>/<bu>` | Import: target BU (repeatable for multiple targets) |
|
|
61
97
|
| `--clear-before-import` | SOAP `ClearData` before REST import |
|
|
62
98
|
| `--i-accept-clear-data-risk` | Non-interactive consent for clear |
|
|
63
99
|
|
package/lib/cli.mjs
CHANGED
|
@@ -15,13 +15,17 @@ import { parseExportBasename } from './filename.mjs';
|
|
|
15
15
|
import { importFromFile } from './import-de.mjs';
|
|
16
16
|
import { clearDataExtensionRows } from './clear-de.mjs';
|
|
17
17
|
import { confirmClearBeforeImport } from './confirm-clear.mjs';
|
|
18
|
+
import { multiBuExport } from './multi-bu-export.mjs';
|
|
19
|
+
import { crossBuImport } from './cross-bu-import.mjs';
|
|
18
20
|
|
|
19
21
|
function printHelp() {
|
|
20
22
|
console.log(`mcdata — SFMC Data Extension export/import (mcdev project)
|
|
21
23
|
|
|
22
24
|
Usage:
|
|
23
25
|
mcdata export <credential>/<bu> --de <key> [--de <key> ...] [options]
|
|
26
|
+
mcdata export --from <cred>/<bu> [--from <cred>/<bu> ...] --de <key> [--de <key> ...] [options]
|
|
24
27
|
mcdata import <credential>/<bu> (--de <key> ... | --file <path> ...) [options]
|
|
28
|
+
mcdata import --from <cred>/<bu> --to <cred>/<bu> [--to <cred>/<bu> ...] --de <key> ... [options]
|
|
25
29
|
|
|
26
30
|
Options:
|
|
27
31
|
-p, --project <dir> mcdev project root (default: cwd)
|
|
@@ -34,10 +38,16 @@ Import options:
|
|
|
34
38
|
--clear-before-import SOAP ClearData before import (destructive; see below)
|
|
35
39
|
--i-accept-clear-data-risk Non-interactive acknowledgement for --clear-before-import
|
|
36
40
|
|
|
41
|
+
Multi-BU options:
|
|
42
|
+
--from <cred>/<bu> Export: source BU (repeatable for multiple sources)
|
|
43
|
+
Import: single source BU (use with --to)
|
|
44
|
+
--to <cred>/<bu> Import: target BU (repeatable for multiple targets)
|
|
45
|
+
|
|
37
46
|
Notes:
|
|
38
47
|
Exports are written under ./data/<credential>/<bu>/ with "+MCDATA+" in the filename.
|
|
39
48
|
Import with --de resolves the latest matching file in that folder (by mtime).
|
|
40
49
|
Import with --file parses the DE key from the basename (+MCDATA+ prefix).
|
|
50
|
+
Cross-BU import stores a timestamped download file in each target BU's data directory.
|
|
41
51
|
|
|
42
52
|
Clear data warning:
|
|
43
53
|
--clear-before-import deletes ALL existing rows in the target DE(s) before upload.
|
|
@@ -64,6 +74,8 @@ export async function main(argv) {
|
|
|
64
74
|
file: { type: 'string', multiple: true },
|
|
65
75
|
api: { type: 'string' },
|
|
66
76
|
mode: { type: 'string' },
|
|
77
|
+
from: { type: 'string', multiple: true },
|
|
78
|
+
to: { type: 'string', multiple: true },
|
|
67
79
|
'clear-before-import': { type: 'boolean', default: false },
|
|
68
80
|
'i-accept-clear-data-risk': { type: 'boolean', default: false },
|
|
69
81
|
'json-pretty': { type: 'boolean', default: false },
|
|
@@ -88,12 +100,7 @@ export async function main(argv) {
|
|
|
88
100
|
}
|
|
89
101
|
|
|
90
102
|
const sub = positionals[0];
|
|
91
|
-
const credBuRaw = positionals[1];
|
|
92
|
-
if (!credBuRaw) {
|
|
93
|
-
console.error('Missing <credential>/<businessUnit>.');
|
|
94
|
-
printHelp();
|
|
95
|
-
return 1;
|
|
96
|
-
}
|
|
103
|
+
const credBuRaw = positionals[1]; // May be undefined when --from/--to flags are used
|
|
97
104
|
|
|
98
105
|
const projectRoot = path.resolve(values.project ?? process.cwd());
|
|
99
106
|
const fmt = values.format ?? 'csv';
|
|
@@ -102,18 +109,63 @@ export async function main(argv) {
|
|
|
102
109
|
return 1;
|
|
103
110
|
}
|
|
104
111
|
|
|
105
|
-
const
|
|
106
|
-
const
|
|
107
|
-
const
|
|
108
|
-
const
|
|
109
|
-
const
|
|
112
|
+
const fromFlags = values.from ?? [];
|
|
113
|
+
const toFlags = values.to ?? [];
|
|
114
|
+
const hasFrom = fromFlags.length > 0;
|
|
115
|
+
const hasTo = toFlags.length > 0;
|
|
116
|
+
const hasPositional = !!credBuRaw;
|
|
110
117
|
|
|
118
|
+
// ── export ──────────────────────────────────────────────────────────────
|
|
111
119
|
if (sub === 'export') {
|
|
120
|
+
if (hasTo) {
|
|
121
|
+
console.error('--to is not valid for export. Did you mean import?');
|
|
122
|
+
return 1;
|
|
123
|
+
}
|
|
124
|
+
|
|
112
125
|
const des = [].concat(values.de ?? []);
|
|
113
126
|
if (des.length === 0) {
|
|
114
127
|
console.error('export requires at least one --de <customerKey>');
|
|
115
128
|
return 1;
|
|
116
129
|
}
|
|
130
|
+
|
|
131
|
+
if (hasFrom && hasPositional) {
|
|
132
|
+
console.error('Cannot mix a positional <credential>/<bu> with --from. Use one or the other.');
|
|
133
|
+
return 1;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (!hasFrom && !hasPositional) {
|
|
137
|
+
console.error('export requires either a positional <credential>/<bu> or at least one --from <cred>/<bu>.');
|
|
138
|
+
printHelp();
|
|
139
|
+
return 1;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
if (hasFrom) {
|
|
143
|
+
// Multi-BU export
|
|
144
|
+
let sources;
|
|
145
|
+
try {
|
|
146
|
+
sources = fromFlags.map(parseCredBu);
|
|
147
|
+
} catch (e) {
|
|
148
|
+
console.error(e.message);
|
|
149
|
+
return 1;
|
|
150
|
+
}
|
|
151
|
+
const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
|
|
152
|
+
await multiBuExport({
|
|
153
|
+
projectRoot,
|
|
154
|
+
mcdevrc,
|
|
155
|
+
mcdevAuth,
|
|
156
|
+
sources,
|
|
157
|
+
deKeys: des,
|
|
158
|
+
format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
|
|
159
|
+
jsonPretty: values['json-pretty'],
|
|
160
|
+
});
|
|
161
|
+
return 0;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Single-BU export (original behavior)
|
|
165
|
+
const { credential, bu } = parseCredBu(credBuRaw);
|
|
166
|
+
const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
|
|
167
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
168
|
+
const sdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
117
169
|
for (const deKey of des) {
|
|
118
170
|
const out = await exportDataExtensionToFile(sdk, {
|
|
119
171
|
projectRoot,
|
|
@@ -128,6 +180,7 @@ export async function main(argv) {
|
|
|
128
180
|
return 0;
|
|
129
181
|
}
|
|
130
182
|
|
|
183
|
+
// ── import ──────────────────────────────────────────────────────────────
|
|
131
184
|
if (sub === 'import') {
|
|
132
185
|
const api = values.api ?? 'async';
|
|
133
186
|
const mode = values.mode ?? 'upsert';
|
|
@@ -140,6 +193,71 @@ export async function main(argv) {
|
|
|
140
193
|
return 1;
|
|
141
194
|
}
|
|
142
195
|
|
|
196
|
+
const clear = values['clear-before-import'];
|
|
197
|
+
const acceptRisk = values['i-accept-clear-data-risk'];
|
|
198
|
+
|
|
199
|
+
// ── Cross-BU import: --from + --to ──────────────────────────────────
|
|
200
|
+
if (hasFrom || hasTo) {
|
|
201
|
+
if (hasPositional) {
|
|
202
|
+
console.error('Cannot mix a positional <credential>/<bu> with --from/--to. Use one or the other.');
|
|
203
|
+
return 1;
|
|
204
|
+
}
|
|
205
|
+
if (!hasFrom) {
|
|
206
|
+
console.error('--to requires --from <cred>/<bu> to specify the source Business Unit.');
|
|
207
|
+
return 1;
|
|
208
|
+
}
|
|
209
|
+
if (!hasTo) {
|
|
210
|
+
console.error('--from requires at least one --to <cred>/<bu> to specify target Business Unit(s).');
|
|
211
|
+
return 1;
|
|
212
|
+
}
|
|
213
|
+
if (fromFlags.length > 1) {
|
|
214
|
+
console.error('import accepts exactly one --from <cred>/<bu> (use multiple --to for multiple targets).');
|
|
215
|
+
return 1;
|
|
216
|
+
}
|
|
217
|
+
if (values.file?.length > 0) {
|
|
218
|
+
console.error('--file cannot be combined with --from/--to. Use --de instead.');
|
|
219
|
+
return 1;
|
|
220
|
+
}
|
|
221
|
+
const deKeys = [].concat(values.de ?? []);
|
|
222
|
+
if (deKeys.length === 0) {
|
|
223
|
+
console.error('Cross-BU import requires at least one --de <customerKey>.');
|
|
224
|
+
return 1;
|
|
225
|
+
}
|
|
226
|
+
let sourceParsed;
|
|
227
|
+
let targets;
|
|
228
|
+
try {
|
|
229
|
+
sourceParsed = parseCredBu(fromFlags[0]);
|
|
230
|
+
targets = toFlags.map(parseCredBu);
|
|
231
|
+
} catch (e) {
|
|
232
|
+
console.error(e.message);
|
|
233
|
+
return 1;
|
|
234
|
+
}
|
|
235
|
+
const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
|
|
236
|
+
await crossBuImport({
|
|
237
|
+
projectRoot,
|
|
238
|
+
mcdevrc,
|
|
239
|
+
mcdevAuth,
|
|
240
|
+
sourceCred: sourceParsed.credential,
|
|
241
|
+
sourceBu: sourceParsed.bu,
|
|
242
|
+
targets,
|
|
243
|
+
deKeys,
|
|
244
|
+
format: /** @type {'csv'|'tsv'|'json'} */ (fmt),
|
|
245
|
+
api: /** @type {'async'|'sync'} */ (api),
|
|
246
|
+
mode: /** @type {'upsert'|'insert'|'update'} */ (mode),
|
|
247
|
+
clearBeforeImport: clear,
|
|
248
|
+
acceptRiskFlag: acceptRisk,
|
|
249
|
+
isTTY: process.stdin.isTTY === true,
|
|
250
|
+
});
|
|
251
|
+
return 0;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// ── Single-BU import (original behavior) ────────────────────────────
|
|
255
|
+
if (!hasPositional) {
|
|
256
|
+
console.error('import requires either a positional <credential>/<bu> or --from/--to flags.');
|
|
257
|
+
printHelp();
|
|
258
|
+
return 1;
|
|
259
|
+
}
|
|
260
|
+
|
|
143
261
|
const hasDe = values.de?.length > 0;
|
|
144
262
|
const hasFile = values.file?.length > 0;
|
|
145
263
|
if (hasDe === hasFile) {
|
|
@@ -147,8 +265,10 @@ export async function main(argv) {
|
|
|
147
265
|
return 1;
|
|
148
266
|
}
|
|
149
267
|
|
|
150
|
-
const
|
|
151
|
-
const
|
|
268
|
+
const { credential, bu } = parseCredBu(credBuRaw);
|
|
269
|
+
const { mcdevrc, mcdevAuth } = loadMcdevProject(projectRoot);
|
|
270
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
271
|
+
const sdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
152
272
|
|
|
153
273
|
if (hasDe) {
|
|
154
274
|
const deKeys = [].concat(values.de ?? []);
|
package/lib/confirm-clear.mjs
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
import readline from 'node:readline/promises';
|
|
2
2
|
import { stdin as input, stdout as output } from 'node:process';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* @typedef {{ credential: string, bu: string }} CredBuTarget
|
|
6
|
+
*/
|
|
7
|
+
|
|
4
8
|
/**
|
|
5
9
|
* @param {object} opts
|
|
6
10
|
* @param {string[]} opts.deKeys
|
|
7
11
|
* @param {boolean} opts.acceptRiskFlag
|
|
8
12
|
* @param {boolean} opts.isTTY
|
|
13
|
+
* @param {CredBuTarget[]} [opts.targets] When present, renders a per-BU breakdown.
|
|
9
14
|
* @param {NodeJS.ReadableStream} [opts.stdin]
|
|
10
15
|
* @param {NodeJS.WritableStream} [opts.stdout]
|
|
11
16
|
* @returns {Promise<void>}
|
|
12
17
|
*/
|
|
13
18
|
export async function confirmClearBeforeImport(opts) {
|
|
14
|
-
const { deKeys, acceptRiskFlag, isTTY } = opts;
|
|
19
|
+
const { deKeys, targets, acceptRiskFlag, isTTY } = opts;
|
|
15
20
|
const stdin = opts.stdin ?? input;
|
|
16
21
|
const stdout = opts.stdout ?? output;
|
|
17
22
|
if (acceptRiskFlag) {
|
|
@@ -23,12 +28,27 @@ export async function confirmClearBeforeImport(opts) {
|
|
|
23
28
|
'All rows in the target Data Extension(s) would be permanently deleted.'
|
|
24
29
|
);
|
|
25
30
|
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
let msg;
|
|
32
|
+
if (targets && targets.length > 0) {
|
|
33
|
+
msg =
|
|
34
|
+
'\n*** DANGER: CLEAR DATA ***\n' +
|
|
35
|
+
`This will permanently DELETE ALL ROWS across ${targets.length} Business Unit(s):\n\n` +
|
|
36
|
+
targets
|
|
37
|
+
.map(
|
|
38
|
+
({ credential, bu }) =>
|
|
39
|
+
` ${credential}/${bu}:\n` + deKeys.map((k) => ` - ${k}\n`).join('')
|
|
40
|
+
)
|
|
41
|
+
.join('') +
|
|
42
|
+
'\nThis cannot be undone. Enterprise 2.0 / admin / shared-DE rules may apply.\n' +
|
|
43
|
+
'Type YES to continue, anything else to abort: ';
|
|
44
|
+
} else {
|
|
45
|
+
msg =
|
|
46
|
+
'\n*** DANGER: CLEAR DATA ***\n' +
|
|
47
|
+
'This will permanently DELETE ALL ROWS in:\n' +
|
|
48
|
+
deKeys.map((k) => ` - ${k}\n`).join('') +
|
|
49
|
+
'This cannot be undone. Enterprise 2.0 / admin / shared-DE rules may apply.\n' +
|
|
50
|
+
'Type YES to continue, anything else to abort: ';
|
|
51
|
+
}
|
|
32
52
|
stdout.write(msg);
|
|
33
53
|
const rl = readline.createInterface({ input: stdin, output: stdout });
|
|
34
54
|
try {
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import readline from 'node:readline/promises';
|
|
4
|
+
import { stdin as input, stdout as output } from 'node:process';
|
|
5
|
+
import SDK from 'sfmc-sdk';
|
|
6
|
+
import { resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
|
|
7
|
+
import { fetchAllRowObjects, serializeRows } from './export-de.mjs';
|
|
8
|
+
import { exportDataExtensionToFile } from './export-de.mjs';
|
|
9
|
+
import { importRowsForDe } from './import-de.mjs';
|
|
10
|
+
import { clearDataExtensionRows } from './clear-de.mjs';
|
|
11
|
+
import { confirmClearBeforeImport } from './confirm-clear.mjs';
|
|
12
|
+
import { dataDirectoryForBu } from './paths.mjs';
|
|
13
|
+
import { buildExportBasename, filesystemSafeTimestamp } from './filename.mjs';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @typedef {{ credential: string, bu: string }} CredBuTarget
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* In TTY mode, asks the user whether to export target DE data as backup
|
|
21
|
+
* before importing. Returns true when the user answers YES.
|
|
22
|
+
*
|
|
23
|
+
* @param {object} opts
|
|
24
|
+
* @param {CredBuTarget[]} opts.targets
|
|
25
|
+
* @param {string[]} opts.deKeys
|
|
26
|
+
* @param {NodeJS.ReadableStream} [opts.stdin]
|
|
27
|
+
* @param {NodeJS.WritableStream} [opts.stdout]
|
|
28
|
+
* @returns {Promise<boolean>}
|
|
29
|
+
*/
|
|
30
|
+
async function offerPreExportBackup({ targets, deKeys, stdin: stdinStream, stdout: stdoutStream }) {
|
|
31
|
+
const stdinSrc = stdinStream ?? input;
|
|
32
|
+
const stdoutSrc = stdoutStream ?? output;
|
|
33
|
+
const targetList = targets.map(({ credential, bu }) => `${credential}/${bu}`).join(', ');
|
|
34
|
+
const msg =
|
|
35
|
+
'\nBefore importing, would you like to export the current data from target BU(s) as a backup?\n' +
|
|
36
|
+
'This creates timestamped files that will not be overwritten by the following import.\n\n' +
|
|
37
|
+
` Target(s): ${targetList}\n` +
|
|
38
|
+
` Data Extensions: ${deKeys.join(', ')}\n\n` +
|
|
39
|
+
'Type YES to export first, or press Enter to skip: ';
|
|
40
|
+
stdoutSrc.write(msg);
|
|
41
|
+
const rl = readline.createInterface({ input: stdinSrc, output: stdoutSrc });
|
|
42
|
+
try {
|
|
43
|
+
const line = await rl.question('');
|
|
44
|
+
return line.trim() === 'YES';
|
|
45
|
+
} finally {
|
|
46
|
+
rl.close();
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Imports Data Extension rows from a single source BU into one or more target
|
|
52
|
+
* BUs.
|
|
53
|
+
*
|
|
54
|
+
* Before the import each target BU:
|
|
55
|
+
* 1. Optionally exports its current DE data as a timestamped backup (TTY only).
|
|
56
|
+
* 2. Optionally clears all existing rows (with danger warning covering every target).
|
|
57
|
+
* 3. Receives the source rows written to a timestamped "download" file in its
|
|
58
|
+
* own `./data/<credential>/<bu>/` directory (mirroring mcdev retrieve).
|
|
59
|
+
* 4. Has the rows imported via the REST API.
|
|
60
|
+
*
|
|
61
|
+
* @param {object} params
|
|
62
|
+
* @param {string} params.projectRoot
|
|
63
|
+
* @param {import('./config.mjs').Mcdevrc} params.mcdevrc
|
|
64
|
+
* @param {Record<string, import('./config.mjs').AuthCredential>} params.mcdevAuth
|
|
65
|
+
* @param {string} params.sourceCred
|
|
66
|
+
* @param {string} params.sourceBu
|
|
67
|
+
* @param {CredBuTarget[]} params.targets
|
|
68
|
+
* @param {string[]} params.deKeys
|
|
69
|
+
* @param {'csv'|'tsv'|'json'} params.format
|
|
70
|
+
* @param {'async'|'sync'} params.api
|
|
71
|
+
* @param {'upsert'|'insert'|'update'} params.mode
|
|
72
|
+
* @param {boolean} params.clearBeforeImport
|
|
73
|
+
* @param {boolean} params.acceptRiskFlag
|
|
74
|
+
* @param {boolean} params.isTTY
|
|
75
|
+
* @param {NodeJS.ReadableStream} [params.stdin] Override for testing
|
|
76
|
+
* @param {NodeJS.WritableStream} [params.stdout] Override for testing
|
|
77
|
+
* @returns {Promise<void>}
|
|
78
|
+
*/
|
|
79
|
+
export async function crossBuImport(params) {
|
|
80
|
+
const {
|
|
81
|
+
projectRoot,
|
|
82
|
+
mcdevrc,
|
|
83
|
+
mcdevAuth,
|
|
84
|
+
sourceCred,
|
|
85
|
+
sourceBu,
|
|
86
|
+
targets,
|
|
87
|
+
deKeys,
|
|
88
|
+
format,
|
|
89
|
+
api,
|
|
90
|
+
mode,
|
|
91
|
+
clearBeforeImport,
|
|
92
|
+
acceptRiskFlag,
|
|
93
|
+
isTTY,
|
|
94
|
+
} = params;
|
|
95
|
+
const stdin = params.stdin;
|
|
96
|
+
const stdout = params.stdout;
|
|
97
|
+
|
|
98
|
+
// Validate all BU configurations upfront before making any API calls
|
|
99
|
+
const { mid: srcMid, authCred: srcAuth } = resolveCredentialAndMid(mcdevrc, mcdevAuth, sourceCred, sourceBu);
|
|
100
|
+
for (const { credential, bu } of targets) {
|
|
101
|
+
resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Connect to source BU
|
|
105
|
+
const srcSdk = new SDK(buildSdkAuthObject(srcAuth, srcMid), { requestAttempts: 3 });
|
|
106
|
+
|
|
107
|
+
// Optional pre-import backup of target BU data
|
|
108
|
+
if (isTTY) {
|
|
109
|
+
const doBackup = await offerPreExportBackup({ targets, deKeys, stdin, stdout });
|
|
110
|
+
if (doBackup) {
|
|
111
|
+
for (const { credential, bu } of targets) {
|
|
112
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
113
|
+
const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
114
|
+
for (const deKey of deKeys) {
|
|
115
|
+
const outPath = await exportDataExtensionToFile(tgtSdk, {
|
|
116
|
+
projectRoot,
|
|
117
|
+
credentialName: credential,
|
|
118
|
+
buName: bu,
|
|
119
|
+
deKey,
|
|
120
|
+
format,
|
|
121
|
+
});
|
|
122
|
+
console.error(`Backup export: ${outPath}`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Single up-front clear confirmation covering all targets + all DE keys
|
|
129
|
+
if (clearBeforeImport) {
|
|
130
|
+
await confirmClearBeforeImport({ deKeys, targets, acceptRiskFlag, isTTY, stdin, stdout });
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Fetch rows once per DE from the source, then fan out to every target
|
|
134
|
+
for (const deKey of deKeys) {
|
|
135
|
+
const rows = await fetchAllRowObjects(srcSdk, deKey);
|
|
136
|
+
|
|
137
|
+
// Clear targets before import (rows already confirmed above)
|
|
138
|
+
if (clearBeforeImport) {
|
|
139
|
+
for (const { credential, bu } of targets) {
|
|
140
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
141
|
+
const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
142
|
+
await clearDataExtensionRows(tgtSdk.soap, deKey);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
for (const { credential, bu } of targets) {
|
|
147
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
148
|
+
const tgtSdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
149
|
+
|
|
150
|
+
// Write a timestamped "download" file in the target BU's data directory.
|
|
151
|
+
// This mirrors what mcdev retrieve / mcdata export produces, giving a
|
|
152
|
+
// traceable snapshot of exactly what was imported.
|
|
153
|
+
const dir = dataDirectoryForBu(projectRoot, credential, bu);
|
|
154
|
+
await fs.mkdir(dir, { recursive: true });
|
|
155
|
+
const ts = filesystemSafeTimestamp();
|
|
156
|
+
const basename = buildExportBasename(deKey, ts, format);
|
|
157
|
+
const filePath = path.join(dir, basename);
|
|
158
|
+
await fs.writeFile(filePath, serializeRows(rows, format, false), 'utf8');
|
|
159
|
+
console.error(`Download stored: ${filePath}`);
|
|
160
|
+
|
|
161
|
+
await importRowsForDe(tgtSdk, { deKey, rows, api, mode });
|
|
162
|
+
console.error(`Imported -> ${credential}/${bu} DE ${deKey}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
package/lib/index.mjs
CHANGED
|
@@ -3,3 +3,5 @@ export { filterIllegalFilenames, reverseFilterIllegalFilenames, parseExportBasen
|
|
|
3
3
|
export { chunkItemsForPayload, DEFAULT_MAX_BODY_BYTES, MAX_OBJECTS_PER_BATCH } from './batch.mjs';
|
|
4
4
|
export { resolveImportRoute, rowsetGetPath, asyncUpsertPath } from './import-routes.mjs';
|
|
5
5
|
export { loadMcdevProject, parseCredBu, resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
|
|
6
|
+
export { multiBuExport } from './multi-bu-export.mjs';
|
|
7
|
+
export { crossBuImport } from './cross-bu-import.mjs';
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import SDK from 'sfmc-sdk';
|
|
2
|
+
import { resolveCredentialAndMid, buildSdkAuthObject } from './config.mjs';
|
|
3
|
+
import { exportDataExtensionToFile } from './export-de.mjs';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @typedef {{ credential: string, bu: string }} CredBuSource
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Exports one or more Data Extensions from multiple source BUs in a single
|
|
11
|
+
* pass. Each source BU gets its own timestamped file per DE key under
|
|
12
|
+
* `./data/<credential>/<bu>/`.
|
|
13
|
+
*
|
|
14
|
+
* @param {object} params
|
|
15
|
+
* @param {string} params.projectRoot
|
|
16
|
+
* @param {import('./config.mjs').Mcdevrc} params.mcdevrc
|
|
17
|
+
* @param {Record<string, import('./config.mjs').AuthCredential>} params.mcdevAuth
|
|
18
|
+
* @param {CredBuSource[]} params.sources
|
|
19
|
+
* @param {string[]} params.deKeys
|
|
20
|
+
* @param {'csv'|'tsv'|'json'} params.format
|
|
21
|
+
* @param {boolean} [params.jsonPretty]
|
|
22
|
+
* @returns {Promise<string[]>} Paths of all written files
|
|
23
|
+
*/
|
|
24
|
+
export async function multiBuExport({ projectRoot, mcdevrc, mcdevAuth, sources, deKeys, format, jsonPretty = false }) {
|
|
25
|
+
/** @type {string[]} */
|
|
26
|
+
const exported = [];
|
|
27
|
+
for (const { credential, bu } of sources) {
|
|
28
|
+
const { mid, authCred } = resolveCredentialAndMid(mcdevrc, mcdevAuth, credential, bu);
|
|
29
|
+
const sdk = new SDK(buildSdkAuthObject(authCred, mid), { requestAttempts: 3 });
|
|
30
|
+
for (const deKey of deKeys) {
|
|
31
|
+
const outPath = await exportDataExtensionToFile(sdk, {
|
|
32
|
+
projectRoot,
|
|
33
|
+
credentialName: credential,
|
|
34
|
+
buName: bu,
|
|
35
|
+
deKey,
|
|
36
|
+
format,
|
|
37
|
+
jsonPretty,
|
|
38
|
+
});
|
|
39
|
+
console.error(`Exported: ${outPath}`);
|
|
40
|
+
exported.push(outPath);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return exported;
|
|
44
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sfmc-dataloader",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "CLI (mcdata) to export and import Marketing Cloud Data Extension rows using mcdev project config and sfmc-sdk",
|
|
5
5
|
"author": "Jörn Berkefeld <joern.berkefeld@gmail.com>",
|
|
6
6
|
"license": "MIT",
|