pgpm 5.19.1 → 5.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/cache.js +1 -1
- package/commands/diff.d.ts +3 -0
- package/commands/diff.js +359 -0
- package/commands/dump.js +3 -3
- package/commands/export.js +24 -5
- package/commands/import.d.ts +3 -0
- package/commands/import.js +235 -0
- package/commands/init/boilerplate.js +1 -1
- package/commands/init/index.js +3 -3
- package/commands/package.js +0 -1
- package/commands/slice.js +1 -1
- package/commands/test-packages.js +1 -1
- package/commands/transform.d.ts +13 -0
- package/commands/transform.js +317 -0
- package/commands/update.js +1 -1
- package/commands.js +7 -0
- package/esm/commands/cache.js +1 -1
- package/esm/commands/diff.js +324 -0
- package/esm/commands/dump.js +3 -3
- package/esm/commands/export.js +22 -3
- package/esm/commands/import.js +200 -0
- package/esm/commands/init/boilerplate.js +1 -1
- package/esm/commands/init/index.js +3 -3
- package/esm/commands/package.js +0 -1
- package/esm/commands/slice.js +1 -1
- package/esm/commands/test-packages.js +1 -1
- package/esm/commands/transform.js +278 -0
- package/esm/commands/update.js +1 -1
- package/esm/commands.js +7 -0
- package/esm/utils/display.js +3 -0
- package/esm/utils/driver.js +1 -1
- package/esm/utils/emit-package.js +1 -0
- package/esm/utils/engine.js +1 -1
- package/esm/utils/module-projections.js +82 -0
- package/esm/utils/scratch-db.js +61 -0
- package/package.json +14 -12
- package/utils/display.d.ts +1 -1
- package/utils/display.js +3 -0
- package/utils/driver.js +1 -1
- package/utils/emit-package.d.ts +9 -0
- package/utils/emit-package.js +7 -0
- package/utils/engine.js +1 -1
- package/utils/module-projections.d.ts +36 -0
- package/utils/module-projections.js +123 -0
- package/utils/scratch-db.d.ts +16 -0
- package/utils/scratch-db.js +66 -0
package/commands.js
CHANGED
|
@@ -13,12 +13,14 @@ const analyze_1 = __importDefault(require("./commands/analyze"));
|
|
|
13
13
|
const cache_1 = __importDefault(require("./commands/cache"));
|
|
14
14
|
const clear_1 = __importDefault(require("./commands/clear"));
|
|
15
15
|
const deploy_1 = __importDefault(require("./commands/deploy"));
|
|
16
|
+
const diff_1 = __importDefault(require("./commands/diff"));
|
|
16
17
|
const docker_1 = __importDefault(require("./commands/docker"));
|
|
17
18
|
const doctor_1 = __importDefault(require("./commands/doctor"));
|
|
18
19
|
const dump_1 = __importDefault(require("./commands/dump"));
|
|
19
20
|
const env_1 = __importDefault(require("./commands/env"));
|
|
20
21
|
const export_1 = __importDefault(require("./commands/export"));
|
|
21
22
|
const extension_1 = __importDefault(require("./commands/extension"));
|
|
23
|
+
const import_1 = __importDefault(require("./commands/import"));
|
|
22
24
|
const init_1 = __importDefault(require("./commands/init"));
|
|
23
25
|
const install_1 = __importDefault(require("./commands/install"));
|
|
24
26
|
const kill_1 = __importDefault(require("./commands/kill"));
|
|
@@ -34,6 +36,7 @@ const slice_1 = __importDefault(require("./commands/slice"));
|
|
|
34
36
|
const sync_versions_1 = __importDefault(require("./commands/sync-versions"));
|
|
35
37
|
const tag_1 = __importDefault(require("./commands/tag"));
|
|
36
38
|
const test_packages_1 = __importDefault(require("./commands/test-packages"));
|
|
39
|
+
const transform_1 = __importDefault(require("./commands/transform"));
|
|
37
40
|
const tune_1 = __importDefault(require("./commands/tune"));
|
|
38
41
|
const update_1 = __importDefault(require("./commands/update"));
|
|
39
42
|
const upgrade_1 = __importDefault(require("./commands/upgrade"));
|
|
@@ -53,6 +56,7 @@ const ENGINE_EXEMPT_COMMANDS = new Set([
|
|
|
53
56
|
'doctor',
|
|
54
57
|
'env',
|
|
55
58
|
'extension',
|
|
59
|
+
'import',
|
|
56
60
|
'init',
|
|
57
61
|
'install',
|
|
58
62
|
'package',
|
|
@@ -85,6 +89,7 @@ const createPgpmCommandMap = (skipPgTeardown = false) => {
|
|
|
85
89
|
'admin-users': pgt(admin_users_1.default),
|
|
86
90
|
clear: pgt(clear_1.default),
|
|
87
91
|
deploy: pgt(deploy_1.default),
|
|
92
|
+
diff: pgt(diff_1.default),
|
|
88
93
|
docker: docker_1.default,
|
|
89
94
|
doctor: doctor_1.default,
|
|
90
95
|
dump: pgt(dump_1.default),
|
|
@@ -97,6 +102,7 @@ const createPgpmCommandMap = (skipPgTeardown = false) => {
|
|
|
97
102
|
plan: pgt(plan_1.default),
|
|
98
103
|
regen: regen_1.default,
|
|
99
104
|
export: pgt(export_1.default),
|
|
105
|
+
import: pgt(import_1.default),
|
|
100
106
|
package: pgt(package_1.default),
|
|
101
107
|
tag: pgt(tag_1.default),
|
|
102
108
|
kill: pgt(kill_1.default),
|
|
@@ -108,6 +114,7 @@ const createPgpmCommandMap = (skipPgTeardown = false) => {
|
|
|
108
114
|
slice: slice_1.default,
|
|
109
115
|
'sync-versions': sync_versions_1.default,
|
|
110
116
|
'test-packages': pgt(test_packages_1.default),
|
|
117
|
+
transform: pgt(transform_1.default),
|
|
111
118
|
tune: pgt(tune_1.default),
|
|
112
119
|
upgrade: pgt(upgrade_1.default),
|
|
113
120
|
up: pgt(upgrade_1.default),
|
package/esm/commands/cache.js
CHANGED
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
import { PgpmMigrate } from '@pgpmjs/core';
|
|
2
|
+
import { deltaChangesToRows, loadDiffSideFromDisk, sqlToDiffChanges } from '@pgpmjs/diff';
|
|
3
|
+
import { Logger } from '@pgpmjs/logger';
|
|
4
|
+
import { appendModule, CHANGE_GRANULARITIES, diffChangeSets, EXPORT_GRANULARITIES, isChangeGranularity, isExportGranularity, loadModule, withoutColumnOrder, writeModule } from '@pgpmjs/transform';
|
|
5
|
+
import { spawn } from 'child_process';
|
|
6
|
+
import * as fs from 'fs';
|
|
7
|
+
import { cliExitWithError, extractFirst } from 'inquirerer';
|
|
8
|
+
import * as os from 'os';
|
|
9
|
+
import * as path from 'path';
|
|
10
|
+
import { getPgPool } from 'pg-cache';
|
|
11
|
+
import { getPgEnvOptions, getSpawnEnvWithPg } from 'pg-env';
|
|
12
|
+
import { emitModuleBundle, emitModuleSql, STDOUT_TARGET } from '../utils/module-projections';
|
|
13
|
+
import { catalogDifferences, withScratchDatabases } from '../utils/scratch-db';
|
|
14
|
+
const log = new Logger('diff');
|
|
15
|
+
const diffUsageText = `
|
|
16
|
+
Diff Command:
|
|
17
|
+
|
|
18
|
+
pgpm diff <A> <B> [OPTIONS]
|
|
19
|
+
|
|
20
|
+
Identity-keyed semantic diff between two schema sources. Each side is
|
|
21
|
+
normalized to an object set keyed by identity (kind/schema/name), so the
|
|
22
|
+
same schema authored at different granularity, naming, or partitioning
|
|
23
|
+
diffs as empty. Tables are compared column-by-column and
|
|
24
|
+
constraint-by-constraint, so table changes emit ALTER TABLE, not a rebuild.
|
|
25
|
+
|
|
26
|
+
Sides:
|
|
27
|
+
<A> / <B> may each be:
|
|
28
|
+
- a pgpm module directory (flattened in plan order)
|
|
29
|
+
- a raw .sql file
|
|
30
|
+
- a live database: a postgres:// connection string, or db:<name>
|
|
31
|
+
(uses PG* env for host/port/user; schema is read via pg_dump)
|
|
32
|
+
|
|
33
|
+
The delta is one model; every --emit-* flag is a projection of it, so they
|
|
34
|
+
compose (a single run can emit a module, a linear SQL file, and a bundle).
|
|
35
|
+
|
|
36
|
+
Options:
|
|
37
|
+
--help, -h Show this help message
|
|
38
|
+
--emit-migration <dir> Write the delta as a pgpm module (deploy/revert/
|
|
39
|
+
verify per change, spec-derived paths, graph-derived
|
|
40
|
+
requires) into <dir>/<pkg>
|
|
41
|
+
--emit-module <dir> Alias of --emit-migration
|
|
42
|
+
--append-module <dir> Append the delta into an EXISTING pgpm module at
|
|
43
|
+
<dir> (new changes only; existing changes, scripts,
|
|
44
|
+
and .control are left untouched). Standalone: not
|
|
45
|
+
combinable with --emit-*/--verify.
|
|
46
|
+
--emit-sql <file|-> Also project the delta to a single consolidated SQL
|
|
47
|
+
file (deparsed in plan order); - writes to stdout
|
|
48
|
+
--emit-bundle <file> Also project the delta to a content-addressed
|
|
49
|
+
.bundle.tar.gz archive
|
|
50
|
+
--pkg <name> Emitted migration package name (default: diff-migration)
|
|
51
|
+
--granularity <level> Granularity for emitted changes: atomic | object |
|
|
52
|
+
consolidated (default: object)
|
|
53
|
+
--change-granularity <level>
|
|
54
|
+
Change-level distribution for emitted changes:
|
|
55
|
+
alteration | object | single (default: object)
|
|
56
|
+
--naming <style> Change path naming style: directory | flat (default: directory)
|
|
57
|
+
--json Machine-readable output
|
|
58
|
+
--verify Oracle mode: deploy A plus the emitted migration into
|
|
59
|
+
a scratch database and assert catalog equivalence
|
|
60
|
+
with B deployed fresh
|
|
61
|
+
--cwd <directory> Working directory (default: current directory)
|
|
62
|
+
|
|
63
|
+
Exit status: 0 when the sides are identical or the diff succeeds; non-zero on
|
|
64
|
+
error or when --verify finds the migrated catalog differs from B.
|
|
65
|
+
|
|
66
|
+
Examples:
|
|
67
|
+
pgpm diff ./v1-module ./v2-module
|
|
68
|
+
pgpm diff ./v1-module ./schema-v2.sql --json
|
|
69
|
+
pgpm diff db:app_v1 db:app_v2 --emit-migration ./out --verify
|
|
70
|
+
`;
|
|
71
|
+
const NAMING_STYLES = ['directory', 'flat'];
|
|
72
|
+
/** pg_dump a side's schema: `db:<name>` uses PG* env; DSNs pass through. */
|
|
73
|
+
const dumpDatabase = async (spec) => {
|
|
74
|
+
const args = ['--schema-only', '--no-owner'];
|
|
75
|
+
let env = process.env;
|
|
76
|
+
if (spec.startsWith('db:')) {
|
|
77
|
+
const config = getPgEnvOptions({ database: spec.slice(3) });
|
|
78
|
+
env = getSpawnEnvWithPg(config);
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
args.push('--dbname', spec);
|
|
82
|
+
}
|
|
83
|
+
return new Promise((resolve, reject) => {
|
|
84
|
+
const child = spawn('pg_dump', args, { env, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
85
|
+
let out = '';
|
|
86
|
+
let err = '';
|
|
87
|
+
child.stdout.on('data', chunk => { out += chunk; });
|
|
88
|
+
child.stderr.on('data', chunk => { err += chunk; });
|
|
89
|
+
child.on('error', (e) => {
|
|
90
|
+
if (e.code === 'ENOENT') {
|
|
91
|
+
reject(new Error('pg_dump not found; ensure PostgreSQL client tools are installed and in PATH'));
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
reject(e);
|
|
95
|
+
});
|
|
96
|
+
child.on('close', code => {
|
|
97
|
+
if (code === 0)
|
|
98
|
+
resolve(out);
|
|
99
|
+
else
|
|
100
|
+
reject(new Error(`pg_dump exited with code ${code}: ${err.trim()}`));
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
};
|
|
104
|
+
/** Resolve one side spec into diff input changes. */
|
|
105
|
+
const loadSide = async (spec, cwd) => {
|
|
106
|
+
if (resolveDiffSideKindSafe(spec) === 'database') {
|
|
107
|
+
const label = spec.startsWith('db:') ? spec.slice(3) : spec;
|
|
108
|
+
return {
|
|
109
|
+
kind: 'database',
|
|
110
|
+
label,
|
|
111
|
+
changes: sqlToDiffChanges(await dumpDatabase(spec), label),
|
|
112
|
+
warnings: []
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
return loadDiffSideFromDisk(path.resolve(cwd, spec));
|
|
116
|
+
};
|
|
117
|
+
/** `resolveDiffSideKind` without throwing on relative on-disk paths. */
|
|
118
|
+
const resolveDiffSideKindSafe = (spec) => /^postgres(ql)?:\/\//.test(spec) || spec.startsWith('db:') ? 'database' : 'disk';
|
|
119
|
+
/** Apply one diff side into a scratch database (module deploy or raw SQL). */
|
|
120
|
+
const applySide = async (config, side, spec, cwd) => {
|
|
121
|
+
if (side.kind === 'module') {
|
|
122
|
+
const client = new PgpmMigrate(config);
|
|
123
|
+
const result = await client.deploy({ modulePath: path.resolve(cwd, spec) });
|
|
124
|
+
if (result.failed) {
|
|
125
|
+
throw new Error(`deploy failed at change ${result.failed} (module ${spec})`);
|
|
126
|
+
}
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
const pool = getPgPool(config);
|
|
130
|
+
const sql = side.changes.map(c => c.deploy).filter(Boolean).join('\n\n');
|
|
131
|
+
if (sql.trim())
|
|
132
|
+
await pool.query(sql);
|
|
133
|
+
};
|
|
134
|
+
/**
|
|
135
|
+
* Oracle mode: deploy A into a scratch database, deploy the emitted migration
|
|
136
|
+
* module on top (when there is one), and assert catalog equivalence with B
|
|
137
|
+
* deployed fresh into a second scratch database. Column order is physical — a
|
|
138
|
+
* drop+add migration cannot reproduce a fresh deploy's ordinals — so
|
|
139
|
+
* equivalence is checked order-insensitively via `withoutColumnOrder`.
|
|
140
|
+
*/
|
|
141
|
+
const runVerify = async (sideA, sideB, specA, specB, migrationDir, cwd) => {
|
|
142
|
+
const config = getPgEnvOptions();
|
|
143
|
+
const stamp = Date.now();
|
|
144
|
+
const names = [`pgpm_diff_verify_a_${stamp}`, `pgpm_diff_verify_b_${stamp}`];
|
|
145
|
+
return withScratchDatabases(config, names, async ([cfgMigrated, cfgTarget]) => {
|
|
146
|
+
await applySide(cfgMigrated, sideA, specA, cwd);
|
|
147
|
+
if (migrationDir) {
|
|
148
|
+
const result = await new PgpmMigrate(cfgMigrated).deploy({ modulePath: migrationDir });
|
|
149
|
+
if (result.failed) {
|
|
150
|
+
throw new Error(`migration deploy failed at change ${result.failed}`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
await applySide(cfgTarget, sideB, specB, cwd);
|
|
154
|
+
return catalogDifferences(cfgMigrated, cfgTarget, withoutColumnOrder);
|
|
155
|
+
});
|
|
156
|
+
};
|
|
157
|
+
const printSummary = (result, labelA, labelB) => {
|
|
158
|
+
if (result.identical) {
|
|
159
|
+
console.log(`No differences: ${labelA} and ${labelB} describe the same objects.`);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const byDelta = (delta) => result.objects.filter(o => o.delta === delta);
|
|
163
|
+
const section = (title, objects) => {
|
|
164
|
+
if (objects.length === 0)
|
|
165
|
+
return;
|
|
166
|
+
console.log(`${title} (${objects.length}):`);
|
|
167
|
+
for (const obj of objects) {
|
|
168
|
+
const detail = [];
|
|
169
|
+
if (obj.columnsAdded?.length)
|
|
170
|
+
detail.push(`+cols: ${obj.columnsAdded.join(', ')}`);
|
|
171
|
+
if (obj.columnsRemoved?.length)
|
|
172
|
+
detail.push(`-cols: ${obj.columnsRemoved.join(', ')}`);
|
|
173
|
+
if (obj.columnsModified?.length)
|
|
174
|
+
detail.push(`~cols: ${obj.columnsModified.join(', ')}`);
|
|
175
|
+
console.log(` ${obj.identity.kind} ${obj.path}${detail.length ? ` (${detail.join('; ')})` : ''}`);
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
section('Added', byDelta('added'));
|
|
179
|
+
section('Removed', byDelta('removed'));
|
|
180
|
+
section('Changed', byDelta('modified'));
|
|
181
|
+
console.log(`${result.changes.length} migration change(s) derivable.`);
|
|
182
|
+
};
|
|
183
|
+
export default async (argv, prompter, _options) => {
|
|
184
|
+
if (argv.help || argv.h) {
|
|
185
|
+
console.log(diffUsageText);
|
|
186
|
+
process.exit(0);
|
|
187
|
+
}
|
|
188
|
+
const { first: specA, newArgv } = extractFirst(argv);
|
|
189
|
+
const { first: specB, newArgv: restArgv } = extractFirst(newArgv);
|
|
190
|
+
argv = restArgv;
|
|
191
|
+
if (!specA || !specB) {
|
|
192
|
+
await cliExitWithError('pgpm diff requires two sides: pgpm diff <A> <B> (module dir, .sql file, db:<name>, or connection string).');
|
|
193
|
+
}
|
|
194
|
+
const granularityRaw = argv.granularity ?? 'object';
|
|
195
|
+
if (!isExportGranularity(granularityRaw)) {
|
|
196
|
+
await cliExitWithError(`Invalid --granularity "${granularityRaw}". Expected one of: ${EXPORT_GRANULARITIES.join(', ')}.`);
|
|
197
|
+
}
|
|
198
|
+
const granularity = granularityRaw;
|
|
199
|
+
const changeGranularityRaw = argv['change-granularity'] ?? argv.changeGranularity ?? 'object';
|
|
200
|
+
if (!isChangeGranularity(changeGranularityRaw)) {
|
|
201
|
+
await cliExitWithError(`Invalid --change-granularity "${changeGranularityRaw}". Expected one of: ${CHANGE_GRANULARITIES.join(', ')}.`);
|
|
202
|
+
}
|
|
203
|
+
const changeGranularity = changeGranularityRaw;
|
|
204
|
+
const namingRaw = argv.naming ?? 'directory';
|
|
205
|
+
if (!NAMING_STYLES.includes(namingRaw)) {
|
|
206
|
+
await cliExitWithError(`Invalid --naming "${namingRaw}". Expected one of: ${NAMING_STYLES.join(', ')}.`);
|
|
207
|
+
}
|
|
208
|
+
const naming = namingRaw;
|
|
209
|
+
const cwd = argv.cwd || process.cwd();
|
|
210
|
+
const json = Boolean(argv.json);
|
|
211
|
+
const verify = Boolean(argv.verify);
|
|
212
|
+
const emitModuleRaw = argv['emit-migration'] ?? argv.emitMigration ?? argv['emit-module'] ?? argv.emitModule;
|
|
213
|
+
const emitModuleDir = typeof emitModuleRaw === 'string' && emitModuleRaw
|
|
214
|
+
? path.resolve(cwd, emitModuleRaw)
|
|
215
|
+
: undefined;
|
|
216
|
+
const appendModuleRaw = argv['append-module'] ?? argv.appendModule;
|
|
217
|
+
const appendModuleDir = typeof appendModuleRaw === 'string' && appendModuleRaw
|
|
218
|
+
? path.resolve(cwd, appendModuleRaw)
|
|
219
|
+
: undefined;
|
|
220
|
+
const emitSqlRaw = argv['emit-sql'] ?? argv.emitSql;
|
|
221
|
+
const emitSql = typeof emitSqlRaw === 'string' && emitSqlRaw
|
|
222
|
+
? (emitSqlRaw === STDOUT_TARGET ? STDOUT_TARGET : path.resolve(cwd, emitSqlRaw))
|
|
223
|
+
: undefined;
|
|
224
|
+
const emitBundleRaw = argv['emit-bundle'] ?? argv.emitBundle;
|
|
225
|
+
const emitBundle = typeof emitBundleRaw === 'string' && emitBundleRaw
|
|
226
|
+
? path.resolve(cwd, emitBundleRaw)
|
|
227
|
+
: undefined;
|
|
228
|
+
const sqlToStdout = emitSql === STDOUT_TARGET;
|
|
229
|
+
const pkgName = argv.pkg || 'diff-migration';
|
|
230
|
+
if (appendModuleDir && (emitModuleDir || emitSql || emitBundle || verify)) {
|
|
231
|
+
await cliExitWithError('--append-module is standalone; it cannot be combined with --emit-migration/--emit-module/--emit-sql/--emit-bundle/--verify.');
|
|
232
|
+
}
|
|
233
|
+
await loadModule();
|
|
234
|
+
let sideA;
|
|
235
|
+
let sideB;
|
|
236
|
+
try {
|
|
237
|
+
sideA = await loadSide(specA, cwd);
|
|
238
|
+
sideB = await loadSide(specB, cwd);
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
await cliExitWithError(err instanceof Error ? err.message : String(err));
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
const result = diffChangeSets(sideA.changes, sideB.changes, { granularity, changeGranularity, style: naming });
|
|
245
|
+
const warnings = [
|
|
246
|
+
...sideA.warnings.map(w => `${sideA.label}: ${w}`),
|
|
247
|
+
...sideB.warnings.map(w => `${sideB.label}: ${w}`),
|
|
248
|
+
...result.warnings
|
|
249
|
+
];
|
|
250
|
+
if (json) {
|
|
251
|
+
console.log(JSON.stringify({
|
|
252
|
+
identical: result.identical,
|
|
253
|
+
objects: result.objects,
|
|
254
|
+
changes: result.changes.map(c => ({
|
|
255
|
+
name: c.name,
|
|
256
|
+
dependencies: c.dependencies,
|
|
257
|
+
deploy: c.deploy,
|
|
258
|
+
revert: c.revert,
|
|
259
|
+
verify: c.verify
|
|
260
|
+
})),
|
|
261
|
+
warnings
|
|
262
|
+
}, null, 2));
|
|
263
|
+
}
|
|
264
|
+
else {
|
|
265
|
+
for (const warning of warnings)
|
|
266
|
+
console.warn(`diff: ${warning}`);
|
|
267
|
+
// Keep stdout clean when the SQL projection is piped there.
|
|
268
|
+
if (!sqlToStdout)
|
|
269
|
+
printSummary(result, sideA.label, sideB.label);
|
|
270
|
+
}
|
|
271
|
+
if (appendModuleDir) {
|
|
272
|
+
const rows = deltaChangesToRows(result.changes);
|
|
273
|
+
if (rows.length === 0) {
|
|
274
|
+
log.info('no migration changes to append (sides are identical).');
|
|
275
|
+
}
|
|
276
|
+
else {
|
|
277
|
+
const appended = appendModule(appendModuleDir, rows);
|
|
278
|
+
for (const w of appended.warnings)
|
|
279
|
+
console.warn(`diff: ${w}`);
|
|
280
|
+
log.success(`appended ${appended.added.length} change(s) to ${appended.dir}` +
|
|
281
|
+
(appended.skipped.length ? ` (${appended.skipped.length} skipped)` : ''));
|
|
282
|
+
}
|
|
283
|
+
prompter.close();
|
|
284
|
+
return argv;
|
|
285
|
+
}
|
|
286
|
+
let migrationDir;
|
|
287
|
+
const needModule = Boolean(emitModuleDir || emitSql || emitBundle) || (verify && !result.identical);
|
|
288
|
+
if (needModule) {
|
|
289
|
+
const outBase = emitModuleDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'pgpm-diff-'));
|
|
290
|
+
const rows = deltaChangesToRows(result.changes);
|
|
291
|
+
if (rows.length === 0) {
|
|
292
|
+
if (!sqlToStdout)
|
|
293
|
+
log.info('no migration changes to emit (sides are identical).');
|
|
294
|
+
}
|
|
295
|
+
else {
|
|
296
|
+
migrationDir = writeModule(outBase, { name: pkgName, requires: [], rows });
|
|
297
|
+
if (emitModuleDir && !sqlToStdout)
|
|
298
|
+
log.success(`wrote ${rows.length} migration change(s) to ${migrationDir}`);
|
|
299
|
+
if (emitSql) {
|
|
300
|
+
await emitModuleSql(migrationDir, emitSql);
|
|
301
|
+
if (!sqlToStdout)
|
|
302
|
+
log.success(`wrote linear SQL to ${emitSql}`);
|
|
303
|
+
}
|
|
304
|
+
if (emitBundle) {
|
|
305
|
+
await emitModuleBundle(migrationDir, emitBundle);
|
|
306
|
+
if (!sqlToStdout)
|
|
307
|
+
log.success(`wrote bundle to ${emitBundle}`);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
if (verify) {
|
|
312
|
+
log.info('running --verify: deploying A plus the migration and B into scratch databases...');
|
|
313
|
+
const diffs = await runVerify(sideA, sideB, specA, specB, migrationDir, cwd);
|
|
314
|
+
if (diffs.length) {
|
|
315
|
+
console.error(`--verify failed: catalogs differ (${diffs.length} differences):`);
|
|
316
|
+
for (const diff of diffs)
|
|
317
|
+
console.error(` ${diff}`);
|
|
318
|
+
await cliExitWithError('Migrated catalog does not match the target.');
|
|
319
|
+
}
|
|
320
|
+
log.success('--verify passed: migrated catalog is structurally equivalent to the target.');
|
|
321
|
+
}
|
|
322
|
+
prompter.close();
|
|
323
|
+
return argv;
|
|
324
|
+
};
|
package/esm/commands/dump.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Logger } from '@pgpmjs/logger';
|
|
2
|
-
import {
|
|
3
|
-
import { getPgPool } from 'pg-cache';
|
|
2
|
+
import { QuoteUtils } from '@pgsql/quotes';
|
|
4
3
|
import { spawn } from 'child_process';
|
|
5
4
|
import fs from 'fs';
|
|
6
5
|
import path from 'path';
|
|
7
|
-
import {
|
|
6
|
+
import { getPgPool } from 'pg-cache';
|
|
7
|
+
import { getPgEnvOptions, getSpawnEnvWithPg } from 'pg-env';
|
|
8
8
|
import { getTargetDatabase } from '../utils';
|
|
9
9
|
const log = new Logger('dump');
|
|
10
10
|
const dumpUsageText = `
|
package/esm/commands/export.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { PgpmPackage } from '@pgpmjs/core';
|
|
2
|
-
import { exportMigrations, exportGraphQL, GraphQLClient, graphqlRowToPostgresRow, isExportGranularity, EXPORT_GRANULARITIES } from '@pgpmjs/export';
|
|
3
2
|
import { getEnvOptions } from '@pgpmjs/env';
|
|
3
|
+
import { exportGraphQL, exportMigrations, GraphQLClient, graphqlRowToPostgresRow } from '@pgpmjs/export';
|
|
4
|
+
import { EXPORT_GRANULARITIES, isExportGranularity, parsePartitionConfig } from '@pgpmjs/transform';
|
|
4
5
|
import { getGitConfigInfo } from '@pgpmjs/types';
|
|
5
6
|
import { resolve } from 'path';
|
|
6
7
|
import { getPgPool } from 'pg-cache';
|
|
@@ -28,12 +29,16 @@ Options:
|
|
|
28
29
|
Change paths are derived from the naming spec and
|
|
29
30
|
requires from the statement graph. When omitted,
|
|
30
31
|
sql_actions rows are exported unchanged.
|
|
32
|
+
--partition <file> Partition config (JSON: rules/defaultPackage/splitRiders)
|
|
33
|
+
splitting the exported database module into multiple
|
|
34
|
+
pgpm packages with derived cross-package requires.
|
|
31
35
|
--cwd <directory> Working directory (default: current directory)
|
|
32
36
|
|
|
33
37
|
Examples:
|
|
34
38
|
pgpm export Export migrations from selected database (SQL mode)
|
|
35
39
|
pgpm export --exclude-categories security,permissions,auth,memberships
|
|
36
40
|
pgpm export --granularity consolidated
|
|
41
|
+
pgpm export --granularity object --partition partition.json
|
|
37
42
|
pgpm export --graphql-endpoint 'http://[::1]:3002/graphql' --migrate-endpoint 'http://[::1]:3000/graphql' --migrate-host db_migrate.localhost:3000
|
|
38
43
|
`;
|
|
39
44
|
export default async (argv, prompter, _options) => {
|
|
@@ -65,6 +70,18 @@ export default async (argv, prompter, _options) => {
|
|
|
65
70
|
return;
|
|
66
71
|
}
|
|
67
72
|
const granularity = granularityRaw;
|
|
73
|
+
const partitionRaw = argv.partition;
|
|
74
|
+
let partition;
|
|
75
|
+
if (typeof partitionRaw === 'string' && partitionRaw) {
|
|
76
|
+
try {
|
|
77
|
+
partition = parsePartitionConfig(resolve(cwd, partitionRaw));
|
|
78
|
+
}
|
|
79
|
+
catch (err) {
|
|
80
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
81
|
+
prompter.close();
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
68
85
|
if (graphqlEndpoint) {
|
|
69
86
|
// =========================================================================
|
|
70
87
|
// GraphQL export mode
|
|
@@ -162,7 +179,8 @@ export default async (argv, prompter, _options) => {
|
|
|
162
179
|
argv,
|
|
163
180
|
username,
|
|
164
181
|
excludeCategories,
|
|
165
|
-
granularity
|
|
182
|
+
granularity,
|
|
183
|
+
partition
|
|
166
184
|
});
|
|
167
185
|
}
|
|
168
186
|
else {
|
|
@@ -283,7 +301,8 @@ export default async (argv, prompter, _options) => {
|
|
|
283
301
|
argv,
|
|
284
302
|
username,
|
|
285
303
|
excludeCategories,
|
|
286
|
-
granularity
|
|
304
|
+
granularity,
|
|
305
|
+
partition
|
|
287
306
|
});
|
|
288
307
|
}
|
|
289
308
|
prompter.close();
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { dumpCompatibilityWarnings, importDumpRows, linkTextualDeps, loadDumpSource } from '@pgpmjs/import';
|
|
2
|
+
import { Logger } from '@pgpmjs/logger';
|
|
3
|
+
import { CHANGE_GRANULARITIES, EXPORT_GRANULARITIES, isChangeGranularity, isExportGranularity, parsePartitionConfig, PartitionCycleError, partitionExportRows } from '@pgpmjs/transform';
|
|
4
|
+
import { cliExitWithError } from 'inquirerer';
|
|
5
|
+
import * as path from 'path';
|
|
6
|
+
import { checkOverwrite, writePackage } from '../utils/emit-package';
|
|
7
|
+
import { hasEmitProjection, parseEmitProjectionTargets, projectModule, STDOUT_TARGET } from '../utils/module-projections';
|
|
8
|
+
const log = new Logger('import');
|
|
9
|
+
const importUsageText = `
|
|
10
|
+
Import Command:
|
|
11
|
+
|
|
12
|
+
pgpm import <dump.sql | dir-of-sql-files> --pkg <module-name> [OPTIONS]
|
|
13
|
+
|
|
14
|
+
pgpm-itize an arbitrary SQL dump (e.g. pg_dump --schema-only output): parse
|
|
15
|
+
it, classify every statement, and emit a complete deployable pgpm module —
|
|
16
|
+
module dir, pgpm.plan, and deploy/revert/verify trees with spec-derived
|
|
17
|
+
change paths, graph-derived requires, and generated revert/verify scripts.
|
|
18
|
+
|
|
19
|
+
Directory inputs concatenate their .sql files in sorted (lexicographic)
|
|
20
|
+
filename order.
|
|
21
|
+
|
|
22
|
+
Dump handling:
|
|
23
|
+
- pg_dump preamble noise (SET ..., SELECT set_config(...), psql backslash
|
|
24
|
+
commands) is skipped; COMMENT ON and GRANT statements are kept, attached
|
|
25
|
+
to their host object's change.
|
|
26
|
+
- CREATE EXTENSION statements become .control requires (pgpm owns
|
|
27
|
+
extension creation at deploy time).
|
|
28
|
+
- COPY ... FROM stdin blocks and INSERTs are skipped with a warning unless
|
|
29
|
+
--with-data is passed, which emits them as seed fixture changes (COPY
|
|
30
|
+
data is converted to INSERTs).
|
|
31
|
+
- Statements that classify to no object land in a misc/statements change
|
|
32
|
+
with a warning — never dropped silently.
|
|
33
|
+
|
|
34
|
+
Options:
|
|
35
|
+
--help, -h Show this help message
|
|
36
|
+
--pkg <name> Module name for the generated package (required)
|
|
37
|
+
--granularity <level> Granularity dial: atomic | object | consolidated
|
|
38
|
+
(default: object)
|
|
39
|
+
--change-granularity <level>
|
|
40
|
+
Change-level distribution: alteration | object | single
|
|
41
|
+
(default: object)
|
|
42
|
+
--naming <style> Change path naming style: directory | flat (default: directory)
|
|
43
|
+
--out <dir> Output base directory (default: current directory);
|
|
44
|
+
the module is written to <out>/<pkg>
|
|
45
|
+
--with-data Import COPY/INSERT data as seed fixture changes
|
|
46
|
+
--partition <file> Partition config (JSON: rules/defaultPackage/splitRiders)
|
|
47
|
+
splitting the import into multiple pgpm packages with
|
|
48
|
+
derived cross-package requires
|
|
49
|
+
--write Allow overwriting an existing module directory
|
|
50
|
+
--emit-sql <file|-> Also project the imported module into a single linear
|
|
51
|
+
SQL script (- for stdout). Requires a single output
|
|
52
|
+
package (not supported with --partition).
|
|
53
|
+
--emit-bundle <file> Also project the imported module into a
|
|
54
|
+
content-addressed .bundle.tar.gz. Same single-package
|
|
55
|
+
requirement as --emit-sql.
|
|
56
|
+
--cwd <directory> Working directory (default: current directory)
|
|
57
|
+
--dry-run Print the resulting plan/paths without writing
|
|
58
|
+
|
|
59
|
+
The imported module is the canonical artifact; --emit-sql and --emit-bundle are
|
|
60
|
+
pure projections of it (the same machinery pgpm package and pgpm diff use).
|
|
61
|
+
|
|
62
|
+
Examples:
|
|
63
|
+
pgpm import dump.sql --pkg my-app
|
|
64
|
+
pgpm import dump.sql --pkg my-app --granularity consolidated --naming flat
|
|
65
|
+
pgpm import ./sql-files --pkg my-app --out ./packages --with-data
|
|
66
|
+
pgpm import dump.sql --pkg my-app --partition partition.json
|
|
67
|
+
pgpm import dump.sql --pkg my-app --emit-sql my-app.sql
|
|
68
|
+
`;
|
|
69
|
+
const NAMING_STYLES = ['directory', 'flat'];
|
|
70
|
+
export default async (argv, prompter, _options) => {
|
|
71
|
+
if (argv.help || argv.h) {
|
|
72
|
+
console.log(importUsageText);
|
|
73
|
+
process.exit(0);
|
|
74
|
+
}
|
|
75
|
+
const cwd = argv.cwd || process.cwd();
|
|
76
|
+
const input = argv._?.filter(Boolean)[0];
|
|
77
|
+
if (!input) {
|
|
78
|
+
await cliExitWithError('Missing input: pgpm import <dump.sql | dir-of-sql-files> --pkg <name>');
|
|
79
|
+
}
|
|
80
|
+
const pkgName = argv.pkg;
|
|
81
|
+
if (!pkgName || typeof pkgName !== 'string') {
|
|
82
|
+
await cliExitWithError('--pkg <module-name> is required.');
|
|
83
|
+
}
|
|
84
|
+
const granularityRaw = argv.granularity ?? 'object';
|
|
85
|
+
if (!isExportGranularity(granularityRaw)) {
|
|
86
|
+
await cliExitWithError(`Invalid --granularity "${granularityRaw}". Expected one of: ${EXPORT_GRANULARITIES.join(', ')}.`);
|
|
87
|
+
}
|
|
88
|
+
const granularity = granularityRaw;
|
|
89
|
+
const changeGranularityRaw = argv['change-granularity'] ?? argv.changeGranularity ?? 'object';
|
|
90
|
+
if (!isChangeGranularity(changeGranularityRaw)) {
|
|
91
|
+
await cliExitWithError(`Invalid --change-granularity "${changeGranularityRaw}". Expected one of: ${CHANGE_GRANULARITIES.join(', ')}.`);
|
|
92
|
+
}
|
|
93
|
+
const changeGranularity = changeGranularityRaw;
|
|
94
|
+
const namingRaw = argv.naming ?? 'directory';
|
|
95
|
+
if (!NAMING_STYLES.includes(namingRaw)) {
|
|
96
|
+
await cliExitWithError(`Invalid --naming "${namingRaw}". Expected one of: ${NAMING_STYLES.join(', ')}.`);
|
|
97
|
+
}
|
|
98
|
+
const naming = namingRaw;
|
|
99
|
+
let partition;
|
|
100
|
+
if (typeof argv.partition === 'string' && argv.partition) {
|
|
101
|
+
try {
|
|
102
|
+
partition = parsePartitionConfig(path.resolve(cwd, argv.partition));
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
await cliExitWithError(err instanceof Error ? err.message : String(err));
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
const outBase = typeof argv.out === 'string' && argv.out ? path.resolve(cwd, argv.out) : path.resolve(cwd);
|
|
109
|
+
const withData = Boolean(argv['with-data'] ?? argv.withData);
|
|
110
|
+
const write = Boolean(argv.write);
|
|
111
|
+
const dryRun = Boolean(argv['dry-run'] ?? argv.dryRun);
|
|
112
|
+
const emit = parseEmitProjectionTargets(argv, cwd);
|
|
113
|
+
const emitRequested = hasEmitProjection(emit);
|
|
114
|
+
const sqlToStdout = emit.emitSql === STDOUT_TARGET;
|
|
115
|
+
if (emitRequested && dryRun) {
|
|
116
|
+
await cliExitWithError('--emit-sql/--emit-bundle cannot be combined with --dry-run.');
|
|
117
|
+
}
|
|
118
|
+
let source;
|
|
119
|
+
try {
|
|
120
|
+
source = loadDumpSource(path.resolve(cwd, input));
|
|
121
|
+
}
|
|
122
|
+
catch (err) {
|
|
123
|
+
await cliExitWithError(err instanceof Error ? err.message : String(err));
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
if (source.files.length > 1 && !sqlToStdout) {
|
|
127
|
+
log.info(`concatenated ${source.files.length} .sql files in sorted order`);
|
|
128
|
+
}
|
|
129
|
+
for (const warning of dumpCompatibilityWarnings(source)) {
|
|
130
|
+
console.warn(`\nWARNING: ${warning}\n`);
|
|
131
|
+
}
|
|
132
|
+
const result = await importDumpRows(source, { granularity, changeGranularity, naming, withData });
|
|
133
|
+
let packages;
|
|
134
|
+
if (partition) {
|
|
135
|
+
try {
|
|
136
|
+
const partitioned = await partitionExportRows(result.rows, partition);
|
|
137
|
+
result.warnings.push(...partitioned.warnings.map(w => `partition: ${w}`));
|
|
138
|
+
packages = partitioned.packages.map(pkg => ({
|
|
139
|
+
name: pkg.name,
|
|
140
|
+
requires: [...result.controlRequires, ...pkg.requires],
|
|
141
|
+
rows: linkTextualDeps(pkg.rows)
|
|
142
|
+
}));
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
if (err instanceof PartitionCycleError) {
|
|
146
|
+
await cliExitWithError(`Partition failed: ${err.message}`);
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
throw err;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
else {
|
|
153
|
+
packages = [{ name: pkgName, requires: result.controlRequires, rows: result.rows }];
|
|
154
|
+
}
|
|
155
|
+
for (const warning of result.warnings) {
|
|
156
|
+
console.warn(`import: ${warning}`);
|
|
157
|
+
}
|
|
158
|
+
if (dryRun) {
|
|
159
|
+
for (const pkg of packages) {
|
|
160
|
+
console.log(`package ${pkg.name} -> ${path.join(outBase, pkg.name)}`);
|
|
161
|
+
if (pkg.requires.length) {
|
|
162
|
+
console.log(` requires: ${pkg.requires.join(', ')}`);
|
|
163
|
+
}
|
|
164
|
+
for (const row of pkg.rows) {
|
|
165
|
+
const deps = row.deps?.length ? ` [${row.deps.join(' ')}]` : '';
|
|
166
|
+
console.log(` ${row.deploy}${deps}`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
if (emitRequested && packages.length !== 1) {
|
|
172
|
+
await cliExitWithError('--emit-sql/--emit-bundle require a single output package; a --partition import emits multiple packages.');
|
|
173
|
+
}
|
|
174
|
+
for (const pkg of packages) {
|
|
175
|
+
const guard = checkOverwrite(path.join(outBase, pkg.name), source.files[0], write);
|
|
176
|
+
if (guard) {
|
|
177
|
+
await cliExitWithError(guard);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
let firstDir;
|
|
181
|
+
for (const pkg of packages) {
|
|
182
|
+
const dir = writePackage(outBase, pkg);
|
|
183
|
+
firstDir ??= dir;
|
|
184
|
+
if (!sqlToStdout) {
|
|
185
|
+
log.success(`wrote ${pkg.rows.length} changes to ${dir}`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (emitRequested && firstDir) {
|
|
189
|
+
await projectModule(firstDir, emit, sqlToStdout ? undefined : msg => log.success(msg));
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
const { summary } = result;
|
|
193
|
+
if (!sqlToStdout) {
|
|
194
|
+
log.success(`import: ${summary.statements} statement(s) -> ${summary.changes} change(s), ` +
|
|
195
|
+
`${summary.skippedPreamble} preamble skipped, ${summary.skippedData} data skipped, ` +
|
|
196
|
+
`${summary.misc} in misc, ${result.warnings.length} warning(s)`);
|
|
197
|
+
}
|
|
198
|
+
prompter.close();
|
|
199
|
+
return argv;
|
|
200
|
+
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import { DEFAULT_TEMPLATE_REPO, PgpmPackage, TEMPLATE_REPOS } from '@pgpmjs/core';
|
|
1
2
|
import fs from 'fs';
|
|
2
3
|
import path from 'path';
|
|
3
|
-
import { DEFAULT_TEMPLATE_REPO, PgpmPackage, TEMPLATE_REPOS } from '@pgpmjs/core';
|
|
4
4
|
/**
|
|
5
5
|
* Resolve the template repo for an `init` invocation from its flags.
|
|
6
6
|
*
|