pgpm 5.19.1 → 5.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/commands/cache.js +1 -1
  2. package/commands/diff.d.ts +3 -0
  3. package/commands/diff.js +359 -0
  4. package/commands/dump.js +3 -3
  5. package/commands/export.js +24 -5
  6. package/commands/import.d.ts +3 -0
  7. package/commands/import.js +235 -0
  8. package/commands/init/boilerplate.js +1 -1
  9. package/commands/init/index.js +3 -3
  10. package/commands/package.js +0 -1
  11. package/commands/slice.js +1 -1
  12. package/commands/test-packages.js +1 -1
  13. package/commands/transform.d.ts +13 -0
  14. package/commands/transform.js +317 -0
  15. package/commands/update.js +1 -1
  16. package/commands.js +7 -0
  17. package/esm/commands/cache.js +1 -1
  18. package/esm/commands/diff.js +324 -0
  19. package/esm/commands/dump.js +3 -3
  20. package/esm/commands/export.js +22 -3
  21. package/esm/commands/import.js +200 -0
  22. package/esm/commands/init/boilerplate.js +1 -1
  23. package/esm/commands/init/index.js +3 -3
  24. package/esm/commands/package.js +0 -1
  25. package/esm/commands/slice.js +1 -1
  26. package/esm/commands/test-packages.js +1 -1
  27. package/esm/commands/transform.js +278 -0
  28. package/esm/commands/update.js +1 -1
  29. package/esm/commands.js +7 -0
  30. package/esm/utils/display.js +3 -0
  31. package/esm/utils/driver.js +1 -1
  32. package/esm/utils/emit-package.js +1 -0
  33. package/esm/utils/engine.js +1 -1
  34. package/esm/utils/module-projections.js +82 -0
  35. package/esm/utils/scratch-db.js +61 -0
  36. package/package.json +14 -12
  37. package/utils/display.d.ts +1 -1
  38. package/utils/display.js +3 -0
  39. package/utils/driver.js +1 -1
  40. package/utils/emit-package.d.ts +9 -0
  41. package/utils/emit-package.js +7 -0
  42. package/utils/engine.js +1 -1
  43. package/utils/module-projections.d.ts +36 -0
  44. package/utils/module-projections.js +123 -0
  45. package/utils/scratch-db.d.ts +16 -0
  46. package/utils/scratch-db.js +66 -0
package/commands.js CHANGED
@@ -13,12 +13,14 @@ const analyze_1 = __importDefault(require("./commands/analyze"));
13
13
  const cache_1 = __importDefault(require("./commands/cache"));
14
14
  const clear_1 = __importDefault(require("./commands/clear"));
15
15
  const deploy_1 = __importDefault(require("./commands/deploy"));
16
+ const diff_1 = __importDefault(require("./commands/diff"));
16
17
  const docker_1 = __importDefault(require("./commands/docker"));
17
18
  const doctor_1 = __importDefault(require("./commands/doctor"));
18
19
  const dump_1 = __importDefault(require("./commands/dump"));
19
20
  const env_1 = __importDefault(require("./commands/env"));
20
21
  const export_1 = __importDefault(require("./commands/export"));
21
22
  const extension_1 = __importDefault(require("./commands/extension"));
23
+ const import_1 = __importDefault(require("./commands/import"));
22
24
  const init_1 = __importDefault(require("./commands/init"));
23
25
  const install_1 = __importDefault(require("./commands/install"));
24
26
  const kill_1 = __importDefault(require("./commands/kill"));
@@ -34,6 +36,7 @@ const slice_1 = __importDefault(require("./commands/slice"));
34
36
  const sync_versions_1 = __importDefault(require("./commands/sync-versions"));
35
37
  const tag_1 = __importDefault(require("./commands/tag"));
36
38
  const test_packages_1 = __importDefault(require("./commands/test-packages"));
39
+ const transform_1 = __importDefault(require("./commands/transform"));
37
40
  const tune_1 = __importDefault(require("./commands/tune"));
38
41
  const update_1 = __importDefault(require("./commands/update"));
39
42
  const upgrade_1 = __importDefault(require("./commands/upgrade"));
@@ -53,6 +56,7 @@ const ENGINE_EXEMPT_COMMANDS = new Set([
53
56
  'doctor',
54
57
  'env',
55
58
  'extension',
59
+ 'import',
56
60
  'init',
57
61
  'install',
58
62
  'package',
@@ -85,6 +89,7 @@ const createPgpmCommandMap = (skipPgTeardown = false) => {
85
89
  'admin-users': pgt(admin_users_1.default),
86
90
  clear: pgt(clear_1.default),
87
91
  deploy: pgt(deploy_1.default),
92
+ diff: pgt(diff_1.default),
88
93
  docker: docker_1.default,
89
94
  doctor: doctor_1.default,
90
95
  dump: pgt(dump_1.default),
@@ -97,6 +102,7 @@ const createPgpmCommandMap = (skipPgTeardown = false) => {
97
102
  plan: pgt(plan_1.default),
98
103
  regen: regen_1.default,
99
104
  export: pgt(export_1.default),
105
+ import: pgt(import_1.default),
100
106
  package: pgt(package_1.default),
101
107
  tag: pgt(tag_1.default),
102
108
  kill: pgt(kill_1.default),
@@ -108,6 +114,7 @@ const createPgpmCommandMap = (skipPgTeardown = false) => {
108
114
  slice: slice_1.default,
109
115
  'sync-versions': sync_versions_1.default,
110
116
  'test-packages': pgt(test_packages_1.default),
117
+ transform: pgt(transform_1.default),
111
118
  tune: pgt(tune_1.default),
112
119
  upgrade: pgt(upgrade_1.default),
113
120
  up: pgt(upgrade_1.default),
@@ -1,5 +1,5 @@
1
- import { cliExitWithError } from 'inquirerer';
2
1
  import { CacheManager } from 'genomic';
2
+ import { cliExitWithError } from 'inquirerer';
3
3
  const cacheUsageText = `
4
4
  Cache Command:
5
5
 
@@ -0,0 +1,324 @@
1
+ import { PgpmMigrate } from '@pgpmjs/core';
2
+ import { deltaChangesToRows, loadDiffSideFromDisk, sqlToDiffChanges } from '@pgpmjs/diff';
3
+ import { Logger } from '@pgpmjs/logger';
4
+ import { appendModule, CHANGE_GRANULARITIES, diffChangeSets, EXPORT_GRANULARITIES, isChangeGranularity, isExportGranularity, loadModule, withoutColumnOrder, writeModule } from '@pgpmjs/transform';
5
+ import { spawn } from 'child_process';
6
+ import * as fs from 'fs';
7
+ import { cliExitWithError, extractFirst } from 'inquirerer';
8
+ import * as os from 'os';
9
+ import * as path from 'path';
10
+ import { getPgPool } from 'pg-cache';
11
+ import { getPgEnvOptions, getSpawnEnvWithPg } from 'pg-env';
12
+ import { emitModuleBundle, emitModuleSql, STDOUT_TARGET } from '../utils/module-projections';
13
+ import { catalogDifferences, withScratchDatabases } from '../utils/scratch-db';
14
+ const log = new Logger('diff');
15
+ const diffUsageText = `
16
+ Diff Command:
17
+
18
+ pgpm diff <A> <B> [OPTIONS]
19
+
20
+ Identity-keyed semantic diff between two schema sources. Each side is
21
+ normalized to an object set keyed by identity (kind/schema/name), so the
22
+ same schema authored at different granularity, naming, or partitioning
23
+ diffs as empty. Tables are compared column-by-column and
24
+ constraint-by-constraint, so table changes emit ALTER TABLE, not a rebuild.
25
+
26
+ Sides:
27
+ <A> / <B> may each be:
28
+ - a pgpm module directory (flattened in plan order)
29
+ - a raw .sql file
30
+ - a live database: a postgres:// connection string, or db:<name>
31
+ (uses PG* env for host/port/user; schema is read via pg_dump)
32
+
33
+ The delta is one model; every --emit-* flag is a projection of it, so they
34
+ compose (a single run can emit a module, a linear SQL file, and a bundle).
35
+
36
+ Options:
37
+ --help, -h Show this help message
38
+ --emit-migration <dir> Write the delta as a pgpm module (deploy/revert/
39
+ verify per change, spec-derived paths, graph-derived
40
+ requires) into <dir>/<pkg>
41
+ --emit-module <dir> Alias of --emit-migration
42
+ --append-module <dir> Append the delta into an EXISTING pgpm module at
43
+ <dir> (new changes only; existing changes, scripts,
44
+ and .control are left untouched). Standalone: not
45
+ combinable with --emit-*/--verify.
46
+ --emit-sql <file|-> Also project the delta to a single consolidated SQL
47
+ file (deparsed in plan order); - writes to stdout
48
+ --emit-bundle <file> Also project the delta to a content-addressed
49
+ .bundle.tar.gz archive
50
+ --pkg <name> Emitted migration package name (default: diff-migration)
51
+ --granularity <level> Granularity for emitted changes: atomic | object |
52
+ consolidated (default: object)
53
+ --change-granularity <level>
54
+ Change-level distribution for emitted changes:
55
+ alteration | object | single (default: object)
56
+ --naming <style> Change path naming style: directory | flat (default: directory)
57
+ --json Machine-readable output
58
+ --verify Oracle mode: deploy A plus the emitted migration into
59
+ a scratch database and assert catalog equivalence
60
+ with B deployed fresh
61
+ --cwd <directory> Working directory (default: current directory)
62
+
63
+ Exit status: 0 when the sides are identical or the diff succeeds; non-zero on
64
+ error or when --verify finds the migrated catalog differs from B.
65
+
66
+ Examples:
67
+ pgpm diff ./v1-module ./v2-module
68
+ pgpm diff ./v1-module ./schema-v2.sql --json
69
+ pgpm diff db:app_v1 db:app_v2 --emit-migration ./out --verify
70
+ `;
71
+ const NAMING_STYLES = ['directory', 'flat'];
72
+ /** pg_dump a side's schema: `db:<name>` uses PG* env; DSNs pass through. */
73
+ const dumpDatabase = async (spec) => {
74
+ const args = ['--schema-only', '--no-owner'];
75
+ let env = process.env;
76
+ if (spec.startsWith('db:')) {
77
+ const config = getPgEnvOptions({ database: spec.slice(3) });
78
+ env = getSpawnEnvWithPg(config);
79
+ }
80
+ else {
81
+ args.push('--dbname', spec);
82
+ }
83
+ return new Promise((resolve, reject) => {
84
+ const child = spawn('pg_dump', args, { env, stdio: ['ignore', 'pipe', 'pipe'] });
85
+ let out = '';
86
+ let err = '';
87
+ child.stdout.on('data', chunk => { out += chunk; });
88
+ child.stderr.on('data', chunk => { err += chunk; });
89
+ child.on('error', (e) => {
90
+ if (e.code === 'ENOENT') {
91
+ reject(new Error('pg_dump not found; ensure PostgreSQL client tools are installed and in PATH'));
92
+ return;
93
+ }
94
+ reject(e);
95
+ });
96
+ child.on('close', code => {
97
+ if (code === 0)
98
+ resolve(out);
99
+ else
100
+ reject(new Error(`pg_dump exited with code ${code}: ${err.trim()}`));
101
+ });
102
+ });
103
+ };
104
+ /** Resolve one side spec into diff input changes. */
105
+ const loadSide = async (spec, cwd) => {
106
+ if (resolveDiffSideKindSafe(spec) === 'database') {
107
+ const label = spec.startsWith('db:') ? spec.slice(3) : spec;
108
+ return {
109
+ kind: 'database',
110
+ label,
111
+ changes: sqlToDiffChanges(await dumpDatabase(spec), label),
112
+ warnings: []
113
+ };
114
+ }
115
+ return loadDiffSideFromDisk(path.resolve(cwd, spec));
116
+ };
117
+ /** `resolveDiffSideKind` without throwing on relative on-disk paths. */
118
+ const resolveDiffSideKindSafe = (spec) => /^postgres(ql)?:\/\//.test(spec) || spec.startsWith('db:') ? 'database' : 'disk';
119
+ /** Apply one diff side into a scratch database (module deploy or raw SQL). */
120
+ const applySide = async (config, side, spec, cwd) => {
121
+ if (side.kind === 'module') {
122
+ const client = new PgpmMigrate(config);
123
+ const result = await client.deploy({ modulePath: path.resolve(cwd, spec) });
124
+ if (result.failed) {
125
+ throw new Error(`deploy failed at change ${result.failed} (module ${spec})`);
126
+ }
127
+ return;
128
+ }
129
+ const pool = getPgPool(config);
130
+ const sql = side.changes.map(c => c.deploy).filter(Boolean).join('\n\n');
131
+ if (sql.trim())
132
+ await pool.query(sql);
133
+ };
134
+ /**
135
+ * Oracle mode: deploy A into a scratch database, deploy the emitted migration
136
+ * module on top (when there is one), and assert catalog equivalence with B
137
+ * deployed fresh into a second scratch database. Column order is physical — a
138
+ * drop+add migration cannot reproduce a fresh deploy's ordinals — so
139
+ * equivalence is checked order-insensitively via `withoutColumnOrder`.
140
+ */
141
+ const runVerify = async (sideA, sideB, specA, specB, migrationDir, cwd) => {
142
+ const config = getPgEnvOptions();
143
+ const stamp = Date.now();
144
+ const names = [`pgpm_diff_verify_a_${stamp}`, `pgpm_diff_verify_b_${stamp}`];
145
+ return withScratchDatabases(config, names, async ([cfgMigrated, cfgTarget]) => {
146
+ await applySide(cfgMigrated, sideA, specA, cwd);
147
+ if (migrationDir) {
148
+ const result = await new PgpmMigrate(cfgMigrated).deploy({ modulePath: migrationDir });
149
+ if (result.failed) {
150
+ throw new Error(`migration deploy failed at change ${result.failed}`);
151
+ }
152
+ }
153
+ await applySide(cfgTarget, sideB, specB, cwd);
154
+ return catalogDifferences(cfgMigrated, cfgTarget, withoutColumnOrder);
155
+ });
156
+ };
157
+ const printSummary = (result, labelA, labelB) => {
158
+ if (result.identical) {
159
+ console.log(`No differences: ${labelA} and ${labelB} describe the same objects.`);
160
+ return;
161
+ }
162
+ const byDelta = (delta) => result.objects.filter(o => o.delta === delta);
163
+ const section = (title, objects) => {
164
+ if (objects.length === 0)
165
+ return;
166
+ console.log(`${title} (${objects.length}):`);
167
+ for (const obj of objects) {
168
+ const detail = [];
169
+ if (obj.columnsAdded?.length)
170
+ detail.push(`+cols: ${obj.columnsAdded.join(', ')}`);
171
+ if (obj.columnsRemoved?.length)
172
+ detail.push(`-cols: ${obj.columnsRemoved.join(', ')}`);
173
+ if (obj.columnsModified?.length)
174
+ detail.push(`~cols: ${obj.columnsModified.join(', ')}`);
175
+ console.log(` ${obj.identity.kind} ${obj.path}${detail.length ? ` (${detail.join('; ')})` : ''}`);
176
+ }
177
+ };
178
+ section('Added', byDelta('added'));
179
+ section('Removed', byDelta('removed'));
180
+ section('Changed', byDelta('modified'));
181
+ console.log(`${result.changes.length} migration change(s) derivable.`);
182
+ };
183
+ export default async (argv, prompter, _options) => {
184
+ if (argv.help || argv.h) {
185
+ console.log(diffUsageText);
186
+ process.exit(0);
187
+ }
188
+ const { first: specA, newArgv } = extractFirst(argv);
189
+ const { first: specB, newArgv: restArgv } = extractFirst(newArgv);
190
+ argv = restArgv;
191
+ if (!specA || !specB) {
192
+ await cliExitWithError('pgpm diff requires two sides: pgpm diff <A> <B> (module dir, .sql file, db:<name>, or connection string).');
193
+ }
194
+ const granularityRaw = argv.granularity ?? 'object';
195
+ if (!isExportGranularity(granularityRaw)) {
196
+ await cliExitWithError(`Invalid --granularity "${granularityRaw}". Expected one of: ${EXPORT_GRANULARITIES.join(', ')}.`);
197
+ }
198
+ const granularity = granularityRaw;
199
+ const changeGranularityRaw = argv['change-granularity'] ?? argv.changeGranularity ?? 'object';
200
+ if (!isChangeGranularity(changeGranularityRaw)) {
201
+ await cliExitWithError(`Invalid --change-granularity "${changeGranularityRaw}". Expected one of: ${CHANGE_GRANULARITIES.join(', ')}.`);
202
+ }
203
+ const changeGranularity = changeGranularityRaw;
204
+ const namingRaw = argv.naming ?? 'directory';
205
+ if (!NAMING_STYLES.includes(namingRaw)) {
206
+ await cliExitWithError(`Invalid --naming "${namingRaw}". Expected one of: ${NAMING_STYLES.join(', ')}.`);
207
+ }
208
+ const naming = namingRaw;
209
+ const cwd = argv.cwd || process.cwd();
210
+ const json = Boolean(argv.json);
211
+ const verify = Boolean(argv.verify);
212
+ const emitModuleRaw = argv['emit-migration'] ?? argv.emitMigration ?? argv['emit-module'] ?? argv.emitModule;
213
+ const emitModuleDir = typeof emitModuleRaw === 'string' && emitModuleRaw
214
+ ? path.resolve(cwd, emitModuleRaw)
215
+ : undefined;
216
+ const appendModuleRaw = argv['append-module'] ?? argv.appendModule;
217
+ const appendModuleDir = typeof appendModuleRaw === 'string' && appendModuleRaw
218
+ ? path.resolve(cwd, appendModuleRaw)
219
+ : undefined;
220
+ const emitSqlRaw = argv['emit-sql'] ?? argv.emitSql;
221
+ const emitSql = typeof emitSqlRaw === 'string' && emitSqlRaw
222
+ ? (emitSqlRaw === STDOUT_TARGET ? STDOUT_TARGET : path.resolve(cwd, emitSqlRaw))
223
+ : undefined;
224
+ const emitBundleRaw = argv['emit-bundle'] ?? argv.emitBundle;
225
+ const emitBundle = typeof emitBundleRaw === 'string' && emitBundleRaw
226
+ ? path.resolve(cwd, emitBundleRaw)
227
+ : undefined;
228
+ const sqlToStdout = emitSql === STDOUT_TARGET;
229
+ const pkgName = argv.pkg || 'diff-migration';
230
+ if (appendModuleDir && (emitModuleDir || emitSql || emitBundle || verify)) {
231
+ await cliExitWithError('--append-module is standalone; it cannot be combined with --emit-migration/--emit-module/--emit-sql/--emit-bundle/--verify.');
232
+ }
233
+ await loadModule();
234
+ let sideA;
235
+ let sideB;
236
+ try {
237
+ sideA = await loadSide(specA, cwd);
238
+ sideB = await loadSide(specB, cwd);
239
+ }
240
+ catch (err) {
241
+ await cliExitWithError(err instanceof Error ? err.message : String(err));
242
+ return;
243
+ }
244
+ const result = diffChangeSets(sideA.changes, sideB.changes, { granularity, changeGranularity, style: naming });
245
+ const warnings = [
246
+ ...sideA.warnings.map(w => `${sideA.label}: ${w}`),
247
+ ...sideB.warnings.map(w => `${sideB.label}: ${w}`),
248
+ ...result.warnings
249
+ ];
250
+ if (json) {
251
+ console.log(JSON.stringify({
252
+ identical: result.identical,
253
+ objects: result.objects,
254
+ changes: result.changes.map(c => ({
255
+ name: c.name,
256
+ dependencies: c.dependencies,
257
+ deploy: c.deploy,
258
+ revert: c.revert,
259
+ verify: c.verify
260
+ })),
261
+ warnings
262
+ }, null, 2));
263
+ }
264
+ else {
265
+ for (const warning of warnings)
266
+ console.warn(`diff: ${warning}`);
267
+ // Keep stdout clean when the SQL projection is piped there.
268
+ if (!sqlToStdout)
269
+ printSummary(result, sideA.label, sideB.label);
270
+ }
271
+ if (appendModuleDir) {
272
+ const rows = deltaChangesToRows(result.changes);
273
+ if (rows.length === 0) {
274
+ log.info('no migration changes to append (sides are identical).');
275
+ }
276
+ else {
277
+ const appended = appendModule(appendModuleDir, rows);
278
+ for (const w of appended.warnings)
279
+ console.warn(`diff: ${w}`);
280
+ log.success(`appended ${appended.added.length} change(s) to ${appended.dir}` +
281
+ (appended.skipped.length ? ` (${appended.skipped.length} skipped)` : ''));
282
+ }
283
+ prompter.close();
284
+ return argv;
285
+ }
286
+ let migrationDir;
287
+ const needModule = Boolean(emitModuleDir || emitSql || emitBundle) || (verify && !result.identical);
288
+ if (needModule) {
289
+ const outBase = emitModuleDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'pgpm-diff-'));
290
+ const rows = deltaChangesToRows(result.changes);
291
+ if (rows.length === 0) {
292
+ if (!sqlToStdout)
293
+ log.info('no migration changes to emit (sides are identical).');
294
+ }
295
+ else {
296
+ migrationDir = writeModule(outBase, { name: pkgName, requires: [], rows });
297
+ if (emitModuleDir && !sqlToStdout)
298
+ log.success(`wrote ${rows.length} migration change(s) to ${migrationDir}`);
299
+ if (emitSql) {
300
+ await emitModuleSql(migrationDir, emitSql);
301
+ if (!sqlToStdout)
302
+ log.success(`wrote linear SQL to ${emitSql}`);
303
+ }
304
+ if (emitBundle) {
305
+ await emitModuleBundle(migrationDir, emitBundle);
306
+ if (!sqlToStdout)
307
+ log.success(`wrote bundle to ${emitBundle}`);
308
+ }
309
+ }
310
+ }
311
+ if (verify) {
312
+ log.info('running --verify: deploying A plus the migration and B into scratch databases...');
313
+ const diffs = await runVerify(sideA, sideB, specA, specB, migrationDir, cwd);
314
+ if (diffs.length) {
315
+ console.error(`--verify failed: catalogs differ (${diffs.length} differences):`);
316
+ for (const diff of diffs)
317
+ console.error(` ${diff}`);
318
+ await cliExitWithError('Migrated catalog does not match the target.');
319
+ }
320
+ log.success('--verify passed: migrated catalog is structurally equivalent to the target.');
321
+ }
322
+ prompter.close();
323
+ return argv;
324
+ };
@@ -1,10 +1,10 @@
1
1
  import { Logger } from '@pgpmjs/logger';
2
- import { getPgEnvOptions, getSpawnEnvWithPg } from 'pg-env';
3
- import { getPgPool } from 'pg-cache';
2
+ import { QuoteUtils } from '@pgsql/quotes';
4
3
  import { spawn } from 'child_process';
5
4
  import fs from 'fs';
6
5
  import path from 'path';
7
- import { QuoteUtils } from '@pgsql/quotes';
6
+ import { getPgPool } from 'pg-cache';
7
+ import { getPgEnvOptions, getSpawnEnvWithPg } from 'pg-env';
8
8
  import { getTargetDatabase } from '../utils';
9
9
  const log = new Logger('dump');
10
10
  const dumpUsageText = `
@@ -1,6 +1,7 @@
1
1
  import { PgpmPackage } from '@pgpmjs/core';
2
- import { exportMigrations, exportGraphQL, GraphQLClient, graphqlRowToPostgresRow, isExportGranularity, EXPORT_GRANULARITIES } from '@pgpmjs/export';
3
2
  import { getEnvOptions } from '@pgpmjs/env';
3
+ import { exportGraphQL, exportMigrations, GraphQLClient, graphqlRowToPostgresRow } from '@pgpmjs/export';
4
+ import { EXPORT_GRANULARITIES, isExportGranularity, parsePartitionConfig } from '@pgpmjs/transform';
4
5
  import { getGitConfigInfo } from '@pgpmjs/types';
5
6
  import { resolve } from 'path';
6
7
  import { getPgPool } from 'pg-cache';
@@ -28,12 +29,16 @@ Options:
28
29
  Change paths are derived from the naming spec and
29
30
  requires from the statement graph. When omitted,
30
31
  sql_actions rows are exported unchanged.
32
+ --partition <file> Partition config (JSON: rules/defaultPackage/splitRiders)
33
+ splitting the exported database module into multiple
34
+ pgpm packages with derived cross-package requires.
31
35
  --cwd <directory> Working directory (default: current directory)
32
36
 
33
37
  Examples:
34
38
  pgpm export Export migrations from selected database (SQL mode)
35
39
  pgpm export --exclude-categories security,permissions,auth,memberships
36
40
  pgpm export --granularity consolidated
41
+ pgpm export --granularity object --partition partition.json
37
42
  pgpm export --graphql-endpoint 'http://[::1]:3002/graphql' --migrate-endpoint 'http://[::1]:3000/graphql' --migrate-host db_migrate.localhost:3000
38
43
  `;
39
44
  export default async (argv, prompter, _options) => {
@@ -65,6 +70,18 @@ export default async (argv, prompter, _options) => {
65
70
  return;
66
71
  }
67
72
  const granularity = granularityRaw;
73
+ const partitionRaw = argv.partition;
74
+ let partition;
75
+ if (typeof partitionRaw === 'string' && partitionRaw) {
76
+ try {
77
+ partition = parsePartitionConfig(resolve(cwd, partitionRaw));
78
+ }
79
+ catch (err) {
80
+ console.error(err instanceof Error ? err.message : String(err));
81
+ prompter.close();
82
+ return;
83
+ }
84
+ }
68
85
  if (graphqlEndpoint) {
69
86
  // =========================================================================
70
87
  // GraphQL export mode
@@ -162,7 +179,8 @@ export default async (argv, prompter, _options) => {
162
179
  argv,
163
180
  username,
164
181
  excludeCategories,
165
- granularity
182
+ granularity,
183
+ partition
166
184
  });
167
185
  }
168
186
  else {
@@ -283,7 +301,8 @@ export default async (argv, prompter, _options) => {
283
301
  argv,
284
302
  username,
285
303
  excludeCategories,
286
- granularity
304
+ granularity,
305
+ partition
287
306
  });
288
307
  }
289
308
  prompter.close();
@@ -0,0 +1,200 @@
1
+ import { dumpCompatibilityWarnings, importDumpRows, linkTextualDeps, loadDumpSource } from '@pgpmjs/import';
2
+ import { Logger } from '@pgpmjs/logger';
3
+ import { CHANGE_GRANULARITIES, EXPORT_GRANULARITIES, isChangeGranularity, isExportGranularity, parsePartitionConfig, PartitionCycleError, partitionExportRows } from '@pgpmjs/transform';
4
+ import { cliExitWithError } from 'inquirerer';
5
+ import * as path from 'path';
6
+ import { checkOverwrite, writePackage } from '../utils/emit-package';
7
+ import { hasEmitProjection, parseEmitProjectionTargets, projectModule, STDOUT_TARGET } from '../utils/module-projections';
8
+ const log = new Logger('import');
9
+ const importUsageText = `
10
+ Import Command:
11
+
12
+ pgpm import <dump.sql | dir-of-sql-files> --pkg <module-name> [OPTIONS]
13
+
14
+ pgpm-itize an arbitrary SQL dump (e.g. pg_dump --schema-only output): parse
15
+ it, classify every statement, and emit a complete deployable pgpm module —
16
+ module dir, pgpm.plan, and deploy/revert/verify trees with spec-derived
17
+ change paths, graph-derived requires, and generated revert/verify scripts.
18
+
19
+ Directory inputs concatenate their .sql files in sorted (lexicographic)
20
+ filename order.
21
+
22
+ Dump handling:
23
+ - pg_dump preamble noise (SET ..., SELECT set_config(...), psql backslash
24
+ commands) is skipped; COMMENT ON and GRANT statements are kept, attached
25
+ to their host object's change.
26
+ - CREATE EXTENSION statements become .control requires (pgpm owns
27
+ extension creation at deploy time).
28
+ - COPY ... FROM stdin blocks and INSERTs are skipped with a warning unless
29
+ --with-data is passed, which emits them as seed fixture changes (COPY
30
+ data is converted to INSERTs).
31
+ - Statements that classify to no object land in a misc/statements change
32
+ with a warning — never dropped silently.
33
+
34
+ Options:
35
+ --help, -h Show this help message
36
+ --pkg <name> Module name for the generated package (required)
37
+ --granularity <level> Granularity dial: atomic | object | consolidated
38
+ (default: object)
39
+ --change-granularity <level>
40
+ Change-level distribution: alteration | object | single
41
+ (default: object)
42
+ --naming <style> Change path naming style: directory | flat (default: directory)
43
+ --out <dir> Output base directory (default: current directory);
44
+ the module is written to <out>/<pkg>
45
+ --with-data Import COPY/INSERT data as seed fixture changes
46
+ --partition <file> Partition config (JSON: rules/defaultPackage/splitRiders)
47
+ splitting the import into multiple pgpm packages with
48
+ derived cross-package requires
49
+ --write Allow overwriting an existing module directory
50
+ --emit-sql <file|-> Also project the imported module into a single linear
51
+ SQL script (- for stdout). Requires a single output
52
+ package (not supported with --partition).
53
+ --emit-bundle <file> Also project the imported module into a
54
+ content-addressed .bundle.tar.gz. Same single-package
55
+ requirement as --emit-sql.
56
+ --cwd <directory> Working directory (default: current directory)
57
+ --dry-run Print the resulting plan/paths without writing
58
+
59
+ The imported module is the canonical artifact; --emit-sql and --emit-bundle are
60
+ pure projections of it (the same machinery pgpm package and pgpm diff use).
61
+
62
+ Examples:
63
+ pgpm import dump.sql --pkg my-app
64
+ pgpm import dump.sql --pkg my-app --granularity consolidated --naming flat
65
+ pgpm import ./sql-files --pkg my-app --out ./packages --with-data
66
+ pgpm import dump.sql --pkg my-app --partition partition.json
67
+ pgpm import dump.sql --pkg my-app --emit-sql my-app.sql
68
+ `;
69
+ const NAMING_STYLES = ['directory', 'flat'];
70
+ export default async (argv, prompter, _options) => {
71
+ if (argv.help || argv.h) {
72
+ console.log(importUsageText);
73
+ process.exit(0);
74
+ }
75
+ const cwd = argv.cwd || process.cwd();
76
+ const input = argv._?.filter(Boolean)[0];
77
+ if (!input) {
78
+ await cliExitWithError('Missing input: pgpm import <dump.sql | dir-of-sql-files> --pkg <name>');
79
+ }
80
+ const pkgName = argv.pkg;
81
+ if (!pkgName || typeof pkgName !== 'string') {
82
+ await cliExitWithError('--pkg <module-name> is required.');
83
+ }
84
+ const granularityRaw = argv.granularity ?? 'object';
85
+ if (!isExportGranularity(granularityRaw)) {
86
+ await cliExitWithError(`Invalid --granularity "${granularityRaw}". Expected one of: ${EXPORT_GRANULARITIES.join(', ')}.`);
87
+ }
88
+ const granularity = granularityRaw;
89
+ const changeGranularityRaw = argv['change-granularity'] ?? argv.changeGranularity ?? 'object';
90
+ if (!isChangeGranularity(changeGranularityRaw)) {
91
+ await cliExitWithError(`Invalid --change-granularity "${changeGranularityRaw}". Expected one of: ${CHANGE_GRANULARITIES.join(', ')}.`);
92
+ }
93
+ const changeGranularity = changeGranularityRaw;
94
+ const namingRaw = argv.naming ?? 'directory';
95
+ if (!NAMING_STYLES.includes(namingRaw)) {
96
+ await cliExitWithError(`Invalid --naming "${namingRaw}". Expected one of: ${NAMING_STYLES.join(', ')}.`);
97
+ }
98
+ const naming = namingRaw;
99
+ let partition;
100
+ if (typeof argv.partition === 'string' && argv.partition) {
101
+ try {
102
+ partition = parsePartitionConfig(path.resolve(cwd, argv.partition));
103
+ }
104
+ catch (err) {
105
+ await cliExitWithError(err instanceof Error ? err.message : String(err));
106
+ }
107
+ }
108
+ const outBase = typeof argv.out === 'string' && argv.out ? path.resolve(cwd, argv.out) : path.resolve(cwd);
109
+ const withData = Boolean(argv['with-data'] ?? argv.withData);
110
+ const write = Boolean(argv.write);
111
+ const dryRun = Boolean(argv['dry-run'] ?? argv.dryRun);
112
+ const emit = parseEmitProjectionTargets(argv, cwd);
113
+ const emitRequested = hasEmitProjection(emit);
114
+ const sqlToStdout = emit.emitSql === STDOUT_TARGET;
115
+ if (emitRequested && dryRun) {
116
+ await cliExitWithError('--emit-sql/--emit-bundle cannot be combined with --dry-run.');
117
+ }
118
+ let source;
119
+ try {
120
+ source = loadDumpSource(path.resolve(cwd, input));
121
+ }
122
+ catch (err) {
123
+ await cliExitWithError(err instanceof Error ? err.message : String(err));
124
+ return;
125
+ }
126
+ if (source.files.length > 1 && !sqlToStdout) {
127
+ log.info(`concatenated ${source.files.length} .sql files in sorted order`);
128
+ }
129
+ for (const warning of dumpCompatibilityWarnings(source)) {
130
+ console.warn(`\nWARNING: ${warning}\n`);
131
+ }
132
+ const result = await importDumpRows(source, { granularity, changeGranularity, naming, withData });
133
+ let packages;
134
+ if (partition) {
135
+ try {
136
+ const partitioned = await partitionExportRows(result.rows, partition);
137
+ result.warnings.push(...partitioned.warnings.map(w => `partition: ${w}`));
138
+ packages = partitioned.packages.map(pkg => ({
139
+ name: pkg.name,
140
+ requires: [...result.controlRequires, ...pkg.requires],
141
+ rows: linkTextualDeps(pkg.rows)
142
+ }));
143
+ }
144
+ catch (err) {
145
+ if (err instanceof PartitionCycleError) {
146
+ await cliExitWithError(`Partition failed: ${err.message}`);
147
+ return;
148
+ }
149
+ throw err;
150
+ }
151
+ }
152
+ else {
153
+ packages = [{ name: pkgName, requires: result.controlRequires, rows: result.rows }];
154
+ }
155
+ for (const warning of result.warnings) {
156
+ console.warn(`import: ${warning}`);
157
+ }
158
+ if (dryRun) {
159
+ for (const pkg of packages) {
160
+ console.log(`package ${pkg.name} -> ${path.join(outBase, pkg.name)}`);
161
+ if (pkg.requires.length) {
162
+ console.log(` requires: ${pkg.requires.join(', ')}`);
163
+ }
164
+ for (const row of pkg.rows) {
165
+ const deps = row.deps?.length ? ` [${row.deps.join(' ')}]` : '';
166
+ console.log(` ${row.deploy}${deps}`);
167
+ }
168
+ }
169
+ }
170
+ else {
171
+ if (emitRequested && packages.length !== 1) {
172
+ await cliExitWithError('--emit-sql/--emit-bundle require a single output package; a --partition import emits multiple packages.');
173
+ }
174
+ for (const pkg of packages) {
175
+ const guard = checkOverwrite(path.join(outBase, pkg.name), source.files[0], write);
176
+ if (guard) {
177
+ await cliExitWithError(guard);
178
+ }
179
+ }
180
+ let firstDir;
181
+ for (const pkg of packages) {
182
+ const dir = writePackage(outBase, pkg);
183
+ firstDir ??= dir;
184
+ if (!sqlToStdout) {
185
+ log.success(`wrote ${pkg.rows.length} changes to ${dir}`);
186
+ }
187
+ }
188
+ if (emitRequested && firstDir) {
189
+ await projectModule(firstDir, emit, sqlToStdout ? undefined : msg => log.success(msg));
190
+ }
191
+ }
192
+ const { summary } = result;
193
+ if (!sqlToStdout) {
194
+ log.success(`import: ${summary.statements} statement(s) -> ${summary.changes} change(s), ` +
195
+ `${summary.skippedPreamble} preamble skipped, ${summary.skippedData} data skipped, ` +
196
+ `${summary.misc} in misc, ${result.warnings.length} warning(s)`);
197
+ }
198
+ prompter.close();
199
+ return argv;
200
+ };
@@ -1,6 +1,6 @@
1
+ import { DEFAULT_TEMPLATE_REPO, PgpmPackage, TEMPLATE_REPOS } from '@pgpmjs/core';
1
2
  import fs from 'fs';
2
3
  import path from 'path';
3
- import { DEFAULT_TEMPLATE_REPO, PgpmPackage, TEMPLATE_REPOS } from '@pgpmjs/core';
4
4
  /**
5
5
  * Resolve the template repo for an `init` invocation from its flags.
6
6
  *