pgdevkit 0.4.0__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/PKG-INFO +90 -24
  2. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/README.md +89 -23
  3. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/docs/database-layout.md +11 -3
  4. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/cli.py +85 -14
  5. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/dialect.py +10 -0
  6. pgdevkit-0.6.0/pgdevkit/envtag.py +54 -0
  7. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/migrate.py +27 -53
  8. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/parser.py +9 -2
  9. pgdevkit-0.6.0/pgdevkit/schemas.py +118 -0
  10. pgdevkit-0.6.0/pgdevkit/sql_text.py +56 -0
  11. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/api.py +56 -7
  12. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/mssql/api.py +36 -7
  13. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/schema.py +72 -24
  14. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pyproject.toml +1 -1
  15. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/skills/pgdevkit/SKILL.md +2 -2
  16. pgdevkit-0.6.0/tests/test_envtag.py +56 -0
  17. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_migrate.py +2 -2
  18. pgdevkit-0.6.0/tests/test_migrate_env.py +44 -0
  19. pgdevkit-0.6.0/tests/test_migrate_schemas.py +52 -0
  20. pgdevkit-0.6.0/tests/test_parser_schemas.py +98 -0
  21. pgdevkit-0.6.0/tests/test_schemas.py +156 -0
  22. pgdevkit-0.6.0/tests/testdb/fixtures/database/app/tables/prod_only.prod.sql +3 -0
  23. pgdevkit-0.6.0/tests/testdb/fixtures/database/app/tables/widget.init.sql +1 -0
  24. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_schema.py +37 -0
  25. pgdevkit-0.6.0/tests/testdb/test_schema_filtering.py +115 -0
  26. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/uv.lock +1 -1
  27. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/.github/workflows/auto-release.yml +0 -0
  28. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/.github/workflows/python-publish.yml +0 -0
  29. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/.github/workflows/python-test.yml +0 -0
  30. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/.gitignore +0 -0
  31. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/.python-version +0 -0
  32. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/__init__.py +0 -0
  33. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/areas.py +0 -0
  34. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/backends/__init__.py +0 -0
  35. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/backends/base.py +0 -0
  36. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/backends/mssql.py +0 -0
  37. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/backends/postgres.py +0 -0
  38. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/connection.py +0 -0
  39. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/db/__init__.py +0 -0
  40. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/db/complex_types.py +0 -0
  41. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/db/connection.py +0 -0
  42. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/db/crud.py +0 -0
  43. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/db/loader.py +0 -0
  44. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/db/model.py +0 -0
  45. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/db/mssql_crud.py +0 -0
  46. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/db/mssql_sql.py +0 -0
  47. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/diff.py +0 -0
  48. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/fetch_missing.py +0 -0
  49. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/introspect.py +0 -0
  50. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/lakebase.py +0 -0
  51. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/models.py +0 -0
  52. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/mssql_introspect.py +0 -0
  53. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/__init__.py +0 -0
  54. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/_docker.py +0 -0
  55. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/config.py +0 -0
  56. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/constants.py +0 -0
  57. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/container.py +0 -0
  58. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/mssql/__init__.py +0 -0
  59. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/mssql/constants.py +0 -0
  60. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/mssql/container.py +0 -0
  61. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/naming.py +0 -0
  62. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/pgdevkit/testdb/query.py +0 -0
  63. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/skills/pgdevkit/references/dynamic-sql.md +0 -0
  64. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/skills/pgdevkit/references/temporal-tables.md +0 -0
  65. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/__init__.py +0 -0
  66. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/conftest.py +0 -0
  67. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/db/__init__.py +0 -0
  68. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/db/test_complex_types.py +0 -0
  69. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/db/test_connection.py +0 -0
  70. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/db/test_crud.py +0 -0
  71. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/db/test_loader.py +0 -0
  72. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/db/test_mssql_crud_live.py +0 -0
  73. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/db/test_mssql_crud_sql.py +0 -0
  74. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/fixtures/01_schema.sql +0 -0
  75. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/fixtures/02_types.sql +0 -0
  76. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/fixtures/03_tables.sql +0 -0
  77. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/fixtures/04_views.sql +0 -0
  78. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/fixtures/05_functions.sql +0 -0
  79. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/fixtures/06_indexes.sql +0 -0
  80. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_areas.py +0 -0
  81. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_cli_compare.py +0 -0
  82. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_compare.py +0 -0
  83. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_compare_mssql_live.py +0 -0
  84. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_connection.py +0 -0
  85. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_dialect.py +0 -0
  86. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_diff_mssql.py +0 -0
  87. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_fetch_missing.py +0 -0
  88. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_fetch_missing_cli.py +0 -0
  89. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_lakebase.py +0 -0
  90. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_migrate_areas.py +0 -0
  91. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_mssql_introspect.py +0 -0
  92. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_parser_areas.py +0 -0
  93. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/test_parser_mssql.py +0 -0
  94. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/__init__.py +0 -0
  95. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/conftest.py +0 -0
  96. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/migrations/001_add_gadget_note.sql +0 -0
  97. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/tables/gadget.sql +0 -0
  98. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/tables/gadget.test_data.json +0 -0
  99. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/tables/widget.sql +0 -0
  100. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/tables/widget.test_data.json +0 -0
  101. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/tables/widget_part.sql +0 -0
  102. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/tables/widget_part_detail.sql +0 -0
  103. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/types/dimensions.sql +0 -0
  104. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/types/mood.sql +0 -0
  105. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/views/a_wrapper_view.sql +0 -0
  106. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/app/views/b_base_view.sql +0 -0
  107. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database/schema/app.sql +0 -0
  108. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database_mssql/app/tables/widget.sql +0 -0
  109. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database_mssql/app/tables/widget.test_data.json +0 -0
  110. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database_mssql/app/views/a_wrapper_view.sql +0 -0
  111. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database_mssql/app/views/b_base_view.sql +0 -0
  112. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/fixtures/database_mssql/schema/app.sql +0 -0
  113. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_api.py +0 -0
  114. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_api_mssql.py +0 -0
  115. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_api_mssql_live.py +0 -0
  116. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_cli.py +0 -0
  117. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_config.py +0 -0
  118. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_config_mssql.py +0 -0
  119. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_constants.py +0 -0
  120. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_container.py +0 -0
  121. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_mssql_constants.py +0 -0
  122. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_naming.py +0 -0
  123. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_query.py +0 -0
  124. {pgdevkit-0.4.0 → pgdevkit-0.6.0}/tests/testdb/test_schema_mssql.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: pgdevkit
3
- Version: 0.4.0
3
+ Version: 0.6.0
4
4
  Summary: A helper for developing with Postgres
5
5
  Requires-Python: >=3.14
6
6
  Requires-Dist: docker>=7.1.0
@@ -79,7 +79,13 @@ which parses/introspects/diffs like any other column type; see
79
79
  handled on the CRUD side (write-side serialization only, no auto-parsing on
80
80
  read — `mssql-python` doesn't distinguish `json` columns from `nvarchar`).
81
81
 
82
- ### Area tagging and filtering
82
+ ### Area and schema filtering
83
+
84
+ Two independent, composable ways to narrow which files a command touches:
85
+ **area** is an explicit opt-in tag; **schema** is derived automatically from
86
+ each file's own SQL.
87
+
88
+ #### Area tagging
83
89
 
84
90
  Any migration file or `database/` code file can declare one or more areas by
85
91
  starting with a `-- area:` comment:
@@ -97,43 +103,95 @@ first real statement); a `-- area:` comment later in the file doesn't count.
97
103
  A file with no directive is untagged, and untagged files are treated as
98
104
  shared/common.
99
105
 
100
- `pgdb compare`, `pgdb migrate check`, and `pgdb migrate apply` all accept:
106
+ #### Schema filtering
107
+
108
+ No tag needed — schema membership is parsed straight out of the SQL itself:
109
+ every schema-qualified (or default-schema, when unqualified) table/view/
110
+ function/index reference across every statement in the file, DDL or DML
111
+ alike, plus any `CREATE SCHEMA name`. A file whose schema(s) can't be
112
+ determined (unparseable content, or no table/schema reference in it at all)
113
+ is treated the same as an untagged file — always kept.
114
+
115
+ #### Options
116
+
117
+ `pgdb compare`, `pgdb migrate check`, `pgdb migrate apply`, `pgdb testdb up`,
118
+ and `pgdb testdb reset` all accept:
101
119
 
102
120
  - `--area NAME` (repeatable) — restrict to files declaring one of the given
103
121
  areas, **plus every untagged file** (untagged files always stay in scope).
104
122
  - `--exclude-area NAME` (repeatable) — drop files declaring one of the given
105
123
  areas; untagged files are never dropped by this.
124
+ - `--schema NAME` (repeatable) — restrict to files referencing one of the
125
+ given schemas, **plus every file with no detectable schema reference**.
126
+ - `--exclude-schema NAME` (repeatable) — drop files referencing one of the
127
+ given schemas; files with no detectable reference are never dropped.
106
128
 
107
- Both can be combined; a file matching both an included and an excluded area
108
- is excluded. Passing neither option applies no filtering (the default,
109
- unchanged behavior).
129
+ All four can be combined — a file must pass every filter it's subject to (an
130
+ area match doesn't excuse a schema mismatch, and vice versa), and a file
131
+ matching both an included and an excluded value on the same axis is
132
+ excluded. Passing none of them applies no filtering (the default, unchanged
133
+ behavior).
110
134
 
111
135
  ```bash
112
136
  pgdb migrate apply path/to/database/_migration_scripts --url ... --area billing
113
137
  pgdb compare path/to/database/ --url ... --exclude-area reporting
138
+ pgdb migrate check path/to/database/_migration_scripts --url ... --schema billing --exclude-schema reporting
139
+ pgdb testdb up --schema billing
114
140
  ```
115
141
 
116
142
  `compare`'s default report (no `--report-extra-db`) only checks that the
117
143
  filtered scripts exist correctly in the DB, so it composes safely with area
118
- filtering. Passing `--report-extra-db` together with an area filter also
119
- reports every DB object outside the filtered area(s) as "missing in
120
- scripts" — since the live database has no concept of areas, only the
121
- scripts side is filtered — so treat that combination's "missing in scripts"
122
- results with that in mind (the CLI prints a warning when you combine them).
123
-
124
- `pgdb fetch-missing` deliberately has **no** `--area`/`--exclude-area`: it
125
- diffs the full database against scripts to find genuinely untracked
126
- objects, so narrowing the scripts side by area would make every object
127
- tracked only under a different area look "missing" too — and `--write`
128
- would then reconstruct a duplicate file for something that already exists.
129
-
130
- `pgdevkit.areas` exposes the same logic for scripting:
144
+ and schema filtering. Passing `--report-extra-db` together with either kind
145
+ of filter also reports every DB object outside the filtered area(s)/
146
+ schema(s) as "missing in scripts" — since the live database has no concept
147
+ of areas, and isn't itself filtered by `--schema` either — only the scripts
148
+ side is filtered — so treat that combination's "missing in scripts" results
149
+ with that in mind (the CLI prints a warning when you combine them).
150
+
151
+ `pgdb fetch-missing` deliberately has **no** `--area`/`--exclude-area` (or
152
+ `--schema`/`--exclude-schema`): it diffs the full database against scripts
153
+ to find genuinely untracked objects, so narrowing the scripts side would
154
+ make every object tracked under a different area/schema look "missing" too
155
+ — and `--write` would then reconstruct a duplicate file for something that
156
+ already exists.
157
+
158
+ `pgdevkit.areas` exposes the tag-filtering logic for scripting:
131
159
  `parse_areas`/`file_areas` read a file's declared areas, and
132
160
  `area_allowed`/`filter_by_area` apply the `only`/`exclude` semantics above.
133
- `pgdevkit.migrate.list_migration_files`/`pending_migrations` and
134
- `pgdevkit.parser.parse_directory` take the same `areas`/`exclude_areas`
135
- keyword arguments (`pgdevkit.fetch_missing.find_missing_objects` doesn't,
136
- for the reason above).
161
+ `pgdevkit.schemas` exposes the equivalent for schema filtering:
162
+ `sql_schemas`/`file_schemas` detect a file's referenced schemas, and
163
+ `schema_allowed`/`filter_by_schema` apply the same `only`/`exclude`
164
+ semantics. `pgdevkit.migrate.list_migration_files`/`pending_migrations` and
165
+ `pgdevkit.parser.parse_directory` take both pairs of keyword arguments
166
+ (`areas`/`exclude_areas` and `schemas`/`exclude_schemas`);
167
+ `pgdevkit.fetch_missing.find_missing_objects` takes neither, for the reason
168
+ above.
169
+
170
+ ## Environment-tagged files (`<name>.<env>.sql`)
171
+
172
+ A file whose name ends `.<env>.sql` (e.g. `grants.prod.sql`,
173
+ `seed.staging.sql`) is only in scope when targeting that environment; a
174
+ plain `<name>.sql` file is untagged and always in scope, regardless of
175
+ environment. `.init.sql` (see `docs/database-layout.md`) is reserved and is
176
+ never treated as an environment tag.
177
+
178
+ - `pgdb testdb up`/`pgdb testdb reset` accept `--env` (default
179
+ `local_test`) — so an untagged `grants.sql` always applies, but
180
+ `grants.prod.sql` is skipped unless run with `--env prod`.
181
+ - `pgdb migrate check`/`pgdb migrate apply` accept `--env` too, but it's
182
+ optional with **no** default: omit it and every file is a candidate
183
+ regardless of its tag (unchanged, today's behavior); pass it to restrict
184
+ to files tagged for that environment plus untagged ones.
185
+
186
+ ```bash
187
+ pgdb testdb up --env prod # apply prod-tagged files too, against the local test container
188
+ pgdb migrate apply path/to/database/_migration_scripts --url ... --env prod
189
+ ```
190
+
191
+ `pgdevkit.envtag` exposes the same logic for scripting: `file_env` reads a
192
+ file's tag, `env_allowed` applies the filtering semantics above, and
193
+ `strip_env_suffix` returns a tagged file's logical name (e.g.
194
+ `grants.prod.sql` -> `"grants"`).
137
195
 
138
196
  ## `pgdb testdb`
139
197
 
@@ -163,7 +221,15 @@ def ensure_test_postgres():
163
221
  os.environ[k] = v
164
222
  ```
165
223
 
166
- CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`.
224
+ CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`. `up`/`reset` accept
225
+ `--env` (default `local_test`) — see "Environment-tagged files" above.
226
+
227
+ `up`/`reset` accept `--area`/`--exclude-area` and `--schema`/`--exclude-schema`
228
+ (see "Area and schema filtering" above) to scope which `database/` files get
229
+ applied — e.g. `pgdb testdb up --schema billing` for a test DB with only the
230
+ `billing` schema's tables/views/functions, without waiting on the rest of the
231
+ project's schema to apply. `ensure_testdb`/`reset_testdb` take the same
232
+ keyword arguments when called from Python (e.g. from a pytest fixture).
167
233
 
168
234
  Container connection defaults (`localhost:54322`, `postgres`/`testpwd`) can
169
235
  be overridden with `PGDEVKIT_TESTDB_HOST`, `PGDEVKIT_TESTDB_PORT`,
@@ -58,7 +58,13 @@ which parses/introspects/diffs like any other column type; see
58
58
  handled on the CRUD side (write-side serialization only, no auto-parsing on
59
59
  read — `mssql-python` doesn't distinguish `json` columns from `nvarchar`).
60
60
 
61
- ### Area tagging and filtering
61
+ ### Area and schema filtering
62
+
63
+ Two independent, composable ways to narrow which files a command touches:
64
+ **area** is an explicit opt-in tag; **schema** is derived automatically from
65
+ each file's own SQL.
66
+
67
+ #### Area tagging
62
68
 
63
69
  Any migration file or `database/` code file can declare one or more areas by
64
70
  starting with a `-- area:` comment:
@@ -76,43 +82,95 @@ first real statement); a `-- area:` comment later in the file doesn't count.
76
82
  A file with no directive is untagged, and untagged files are treated as
77
83
  shared/common.
78
84
 
79
- `pgdb compare`, `pgdb migrate check`, and `pgdb migrate apply` all accept:
85
+ #### Schema filtering
86
+
87
+ No tag needed — schema membership is parsed straight out of the SQL itself:
88
+ every schema-qualified (or default-schema, when unqualified) table/view/
89
+ function/index reference across every statement in the file, DDL or DML
90
+ alike, plus any `CREATE SCHEMA name`. A file whose schema(s) can't be
91
+ determined (unparseable content, or no table/schema reference in it at all)
92
+ is treated the same as an untagged file — always kept.
93
+
94
+ #### Options
95
+
96
+ `pgdb compare`, `pgdb migrate check`, `pgdb migrate apply`, `pgdb testdb up`,
97
+ and `pgdb testdb reset` all accept:
80
98
 
81
99
  - `--area NAME` (repeatable) — restrict to files declaring one of the given
82
100
  areas, **plus every untagged file** (untagged files always stay in scope).
83
101
  - `--exclude-area NAME` (repeatable) — drop files declaring one of the given
84
102
  areas; untagged files are never dropped by this.
103
+ - `--schema NAME` (repeatable) — restrict to files referencing one of the
104
+ given schemas, **plus every file with no detectable schema reference**.
105
+ - `--exclude-schema NAME` (repeatable) — drop files referencing one of the
106
+ given schemas; files with no detectable reference are never dropped.
85
107
 
86
- Both can be combined; a file matching both an included and an excluded area
87
- is excluded. Passing neither option applies no filtering (the default,
88
- unchanged behavior).
108
+ All four can be combined — a file must pass every filter it's subject to (an
109
+ area match doesn't excuse a schema mismatch, and vice versa), and a file
110
+ matching both an included and an excluded value on the same axis is
111
+ excluded. Passing none of them applies no filtering (the default, unchanged
112
+ behavior).
89
113
 
90
114
  ```bash
91
115
  pgdb migrate apply path/to/database/_migration_scripts --url ... --area billing
92
116
  pgdb compare path/to/database/ --url ... --exclude-area reporting
117
+ pgdb migrate check path/to/database/_migration_scripts --url ... --schema billing --exclude-schema reporting
118
+ pgdb testdb up --schema billing
93
119
  ```
94
120
 
95
121
  `compare`'s default report (no `--report-extra-db`) only checks that the
96
122
  filtered scripts exist correctly in the DB, so it composes safely with area
97
- filtering. Passing `--report-extra-db` together with an area filter also
98
- reports every DB object outside the filtered area(s) as "missing in
99
- scripts" — since the live database has no concept of areas, only the
100
- scripts side is filtered — so treat that combination's "missing in scripts"
101
- results with that in mind (the CLI prints a warning when you combine them).
102
-
103
- `pgdb fetch-missing` deliberately has **no** `--area`/`--exclude-area`: it
104
- diffs the full database against scripts to find genuinely untracked
105
- objects, so narrowing the scripts side by area would make every object
106
- tracked only under a different area look "missing" too — and `--write`
107
- would then reconstruct a duplicate file for something that already exists.
108
-
109
- `pgdevkit.areas` exposes the same logic for scripting:
123
+ and schema filtering. Passing `--report-extra-db` together with either kind
124
+ of filter also reports every DB object outside the filtered area(s)/
125
+ schema(s) as "missing in scripts" — since the live database has no concept
126
+ of areas, and isn't itself filtered by `--schema` either — only the scripts
127
+ side is filtered — so treat that combination's "missing in scripts" results
128
+ with that in mind (the CLI prints a warning when you combine them).
129
+
130
+ `pgdb fetch-missing` deliberately has **no** `--area`/`--exclude-area` (or
131
+ `--schema`/`--exclude-schema`): it diffs the full database against scripts
132
+ to find genuinely untracked objects, so narrowing the scripts side would
133
+ make every object tracked under a different area/schema look "missing" too
134
+ — and `--write` would then reconstruct a duplicate file for something that
135
+ already exists.
136
+
137
+ `pgdevkit.areas` exposes the tag-filtering logic for scripting:
110
138
  `parse_areas`/`file_areas` read a file's declared areas, and
111
139
  `area_allowed`/`filter_by_area` apply the `only`/`exclude` semantics above.
112
- `pgdevkit.migrate.list_migration_files`/`pending_migrations` and
113
- `pgdevkit.parser.parse_directory` take the same `areas`/`exclude_areas`
114
- keyword arguments (`pgdevkit.fetch_missing.find_missing_objects` doesn't,
115
- for the reason above).
140
+ `pgdevkit.schemas` exposes the equivalent for schema filtering:
141
+ `sql_schemas`/`file_schemas` detect a file's referenced schemas, and
142
+ `schema_allowed`/`filter_by_schema` apply the same `only`/`exclude`
143
+ semantics. `pgdevkit.migrate.list_migration_files`/`pending_migrations` and
144
+ `pgdevkit.parser.parse_directory` take both pairs of keyword arguments
145
+ (`areas`/`exclude_areas` and `schemas`/`exclude_schemas`);
146
+ `pgdevkit.fetch_missing.find_missing_objects` takes neither, for the reason
147
+ above.
148
+
149
+ ## Environment-tagged files (`<name>.<env>.sql`)
150
+
151
+ A file whose name ends `.<env>.sql` (e.g. `grants.prod.sql`,
152
+ `seed.staging.sql`) is only in scope when targeting that environment; a
153
+ plain `<name>.sql` file is untagged and always in scope, regardless of
154
+ environment. `.init.sql` (see `docs/database-layout.md`) is reserved and is
155
+ never treated as an environment tag.
156
+
157
+ - `pgdb testdb up`/`pgdb testdb reset` accept `--env` (default
158
+ `local_test`) — so an untagged `grants.sql` always applies, but
159
+ `grants.prod.sql` is skipped unless run with `--env prod`.
160
+ - `pgdb migrate check`/`pgdb migrate apply` accept `--env` too, but it's
161
+ optional with **no** default: omit it and every file is a candidate
162
+ regardless of its tag (unchanged, today's behavior); pass it to restrict
163
+ to files tagged for that environment plus untagged ones.
164
+
165
+ ```bash
166
+ pgdb testdb up --env prod # apply prod-tagged files too, against the local test container
167
+ pgdb migrate apply path/to/database/_migration_scripts --url ... --env prod
168
+ ```
169
+
170
+ `pgdevkit.envtag` exposes the same logic for scripting: `file_env` reads a
171
+ file's tag, `env_allowed` applies the filtering semantics above, and
172
+ `strip_env_suffix` returns a tagged file's logical name (e.g.
173
+ `grants.prod.sql` -> `"grants"`).
116
174
 
117
175
  ## `pgdb testdb`
118
176
 
@@ -142,7 +200,15 @@ def ensure_test_postgres():
142
200
  os.environ[k] = v
143
201
  ```
144
202
 
145
- CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`.
203
+ CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`. `up`/`reset` accept
204
+ `--env` (default `local_test`) — see "Environment-tagged files" above.
205
+
206
+ `up`/`reset` accept `--area`/`--exclude-area` and `--schema`/`--exclude-schema`
207
+ (see "Area and schema filtering" above) to scope which `database/` files get
208
+ applied — e.g. `pgdb testdb up --schema billing` for a test DB with only the
209
+ `billing` schema's tables/views/functions, without waiting on the rest of the
210
+ project's schema to apply. `ensure_testdb`/`reset_testdb` take the same
211
+ keyword arguments when called from Python (e.g. from a pytest fixture).
146
212
 
147
213
  Container connection defaults (`localhost:54322`, `postgres`/`testpwd`) can
148
214
  be overridden with `PGDEVKIT_TESTDB_HOST`, `PGDEVKIT_TESTDB_PORT`,
@@ -71,10 +71,18 @@ One object per file: `tables/user.sql`, `views/all_edits.sql`,
71
71
  |---|---|
72
72
  | `<name>.sql` | The object's live definition (`CREATE TABLE`, `CREATE OR REPLACE VIEW`, ...) |
73
73
  | `<name>.test_data.json` | Seed rows for a table — a JSON array of row objects, loaded after the table is created |
74
- | `<name>.init.sql` | One-time setup for an object (e.g. a backfill), run once, kept separate from the reusable definition |
75
- | `<name>.prod.sql` / `.prod` anywhere in the name | Production-only (real permission grants, real user accounts) — skipped by `pgdb testdb` |
74
+ | `<name>.init.sql` | One-time setup for an object (e.g. a backfill), run once, kept separate from the reusable definition — `init` is reserved and is never treated as an environment tag |
75
+ | `<name>.<env>.sql` | Only applied when targeting environment `<env>` (any name you like — `prod`, `staging`, ...); a file with no such suffix is untagged and always applies, regardless of environment |
76
76
  | `all.sql` | Generated concatenation of the whole tree — not hand-edited, not committed |
77
77
 
78
+ `pgdb testdb up`/`pgdb testdb reset` apply the `--env` they're given (default
79
+ `local_test`) — so an untagged `grants.sql` always applies, but
80
+ `grants.prod.sql` is skipped unless you pass `--env prod`. `pgdb migrate
81
+ check`/`pgdb migrate apply` accept the same `--env`, but it's optional with no
82
+ default: omit it and every file is a candidate regardless of its tag (today's
83
+ behavior); pass it to restrict to files tagged for that environment plus
84
+ untagged ones.
85
+
78
86
  ---
79
87
 
80
88
  ## Migrations
@@ -156,5 +164,5 @@ leading sort number.
156
164
  - [ ] Object-type folder (`tables`, `views`, ...) matches the apply-order table above — that's what governs ordering, not the layer's leading number
157
165
  - [ ] One-off changes go in `migrations/`, dated, never edited after applying
158
166
  - [ ] The live `.sql` file is updated in the same change as any migration touching that object
159
- - [ ] `.prod` files are production-only and skipped by `pgdb testdb`
167
+ - [ ] `.<env>.sql` files (e.g. `.prod.sql`) are skipped by `pgdb testdb` unless it's run with a matching `--env`
160
168
  - [ ] Every table (and non-obvious column) has a `COMMENT ON`, placed in the object's own `.sql` file
@@ -29,9 +29,32 @@ _AREA_OPTION = typer.Option(
29
29
  _EXCLUDE_AREA_OPTION = typer.Option(
30
30
  [], "--exclude-area", help="Skip files declaring this area (repeatable); untagged files are never excluded"
31
31
  )
32
+ _SCHEMA_OPTION = typer.Option(
33
+ [],
34
+ "--schema",
35
+ help="Restrict to files referencing this DB schema (repeatable); "
36
+ "files with no detectable schema reference always stay in scope",
37
+ )
38
+ _EXCLUDE_SCHEMA_OPTION = typer.Option(
39
+ [],
40
+ "--exclude-schema",
41
+ help="Skip files referencing this DB schema (repeatable); "
42
+ "files with no detectable schema reference are never excluded",
43
+ )
44
+ _TESTDB_ENV_OPTION = typer.Option(
45
+ "local_test",
46
+ "--env",
47
+ help="Environment to apply: skips any <name>.<other-env>.sql file (e.g. grants.prod.sql); untagged files always apply",
48
+ )
49
+ _MIGRATE_ENV_OPTION = typer.Option(
50
+ None,
51
+ "--env",
52
+ help="Restrict to files tagged for this environment (e.g. <name>.prod.sql); untagged files always apply; "
53
+ "omit to apply every file regardless of its env tag",
54
+ )
32
55
 
33
56
 
34
- def _as_area_set(values: list[str]) -> frozenset[str] | None:
57
+ def _as_set(values: list[str]) -> frozenset[str] | None:
35
58
  return frozenset(values) if values else None
36
59
 
37
60
  testdb_app = typer.Typer(name="testdb", help="Manage the shared local Postgres test container")
@@ -59,17 +82,20 @@ def compare(
59
82
  dialect: str = typer.Option("postgres", "--dialect", help="postgres (default) or mssql"),
60
83
  area: list[str] = _AREA_OPTION,
61
84
  exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
85
+ schema: list[str] = _SCHEMA_OPTION,
86
+ exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
62
87
  scripts_dir: Path = typer.Argument(..., help="Directory containing SQL scripts"),
63
88
  ) -> None:
64
89
  """Compare SQL scripts to a live database and report differences."""
65
90
  if not scripts_dir.is_dir():
66
91
  err_console.print(f"[red]Error:[/red] {scripts_dir} is not a directory")
67
92
  raise typer.Exit(2)
68
- if report_extra_db and (area or exclude_area):
93
+ if report_extra_db and (area or exclude_area or schema or exclude_schema):
69
94
  console.print(
70
- "[yellow]⚠[/yellow] --report-extra-db with --area/--exclude-area will report every DB object "
71
- "outside the filtered area(s) as \"missing in scripts\", since the live database has no concept "
72
- "of areas — only the scripts side is filtered."
95
+ "[yellow]⚠[/yellow] --report-extra-db with --area/--exclude-area/--schema/--exclude-schema will "
96
+ "report every DB object outside the filtered area(s)/schema(s) as \"missing in scripts\", since the "
97
+ "live database has no concept of areas — and isn't itself filtered by --schema either — only the "
98
+ "scripts side is filtered."
73
99
  )
74
100
 
75
101
  try:
@@ -91,7 +117,12 @@ def compare(
91
117
 
92
118
  with console.status("Parsing SQL scripts..."):
93
119
  scripts_schema = parse_directory(
94
- scripts_dir, dialect=backend.dialect, areas=_as_area_set(area), exclude_areas=_as_area_set(exclude_area)
120
+ scripts_dir,
121
+ dialect=backend.dialect,
122
+ areas=_as_set(area),
123
+ exclude_areas=_as_set(exclude_area),
124
+ schemas=_as_set(schema),
125
+ exclude_schemas=_as_set(exclude_schema),
95
126
  )
96
127
 
97
128
  with console.status("Introspecting database..."):
@@ -188,17 +219,35 @@ def fetch_missing(
188
219
 
189
220
 
190
221
  @testdb_app.command("up")
191
- def testdb_up() -> None:
222
+ def testdb_up(
223
+ env: str = _TESTDB_ENV_OPTION,
224
+ area: list[str] = _AREA_OPTION,
225
+ exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
226
+ schema: list[str] = _SCHEMA_OPTION,
227
+ exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
228
+ ) -> None:
192
229
  """Ensure the container is running, the workspace DB exists, and schema is applied."""
193
- testdb.ensure_testdb()
230
+ testdb.ensure_testdb(
231
+ env=env, areas=_as_set(area), exclude_areas=_as_set(exclude_area), schemas=_as_set(schema),
232
+ exclude_schemas=_as_set(exclude_schema),
233
+ )
194
234
  info = testdb.status()
195
235
  console.print(f"[green]Test DB ready:[/green] {info['database']} ({info['dsn']})")
196
236
 
197
237
 
198
238
  @testdb_app.command("reset")
199
- def testdb_reset() -> None:
239
+ def testdb_reset(
240
+ env: str = _TESTDB_ENV_OPTION,
241
+ area: list[str] = _AREA_OPTION,
242
+ exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
243
+ schema: list[str] = _SCHEMA_OPTION,
244
+ exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
245
+ ) -> None:
200
246
  """Drop and recreate only this workspace's database, then reapply schema + seed data."""
201
- testdb.reset_testdb()
247
+ testdb.reset_testdb(
248
+ env=env, areas=_as_set(area), exclude_areas=_as_set(exclude_area), schemas=_as_set(schema),
249
+ exclude_schemas=_as_set(exclude_schema),
250
+ )
202
251
  info = testdb.status()
203
252
  console.print(f"[green]Test DB reset:[/green] {info['database']}")
204
253
 
@@ -272,6 +321,9 @@ def migrate_check(
272
321
  ),
273
322
  area: list[str] = _AREA_OPTION,
274
323
  exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
324
+ schema: list[str] = _SCHEMA_OPTION,
325
+ exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
326
+ env: str | None = _MIGRATE_ENV_OPTION,
275
327
  ) -> None:
276
328
  """List which migration files under migrations_dir are applied vs. pending."""
277
329
  if not migrations_dir.is_dir():
@@ -281,7 +333,12 @@ def migrate_check(
281
333
  conninfo = build_conninfo(url, entra_user)
282
334
  tracking_table = tracking_table or migrate.default_tracking_table(migrations_dir)
283
335
  local_files = migrate.list_migration_files(
284
- migrations_dir, areas=_as_area_set(area), exclude_areas=_as_area_set(exclude_area)
336
+ migrations_dir,
337
+ areas=_as_set(area),
338
+ exclude_areas=_as_set(exclude_area),
339
+ schemas=_as_set(schema),
340
+ exclude_schemas=_as_set(exclude_schema),
341
+ env=env,
285
342
  )
286
343
  try:
287
344
  applied = migrate.applied_migrations(conninfo, tracking_table)
@@ -323,6 +380,9 @@ def migrate_apply(
323
380
  yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm-target prompt"),
324
381
  area: list[str] = _AREA_OPTION,
325
382
  exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
383
+ schema: list[str] = _SCHEMA_OPTION,
384
+ exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
385
+ env: str | None = _MIGRATE_ENV_OPTION,
326
386
  ) -> None:
327
387
  """Apply pending migration files, in filename order, tracking each in tracking_table."""
328
388
  if not migrations_dir.is_dir():
@@ -331,7 +391,8 @@ def migrate_apply(
331
391
 
332
392
  conninfo = build_conninfo(url, entra_user)
333
393
  tracking_table = tracking_table or migrate.default_tracking_table(migrations_dir)
334
- areas, exclude_areas = _as_area_set(area), _as_area_set(exclude_area)
394
+ areas, exclude_areas = _as_set(area), _as_set(exclude_area)
395
+ schemas, exclude_schemas = _as_set(schema), _as_set(exclude_schema)
335
396
  target_desc = url.rsplit("@", 1)[-1] if "@" in url else url
336
397
  if not yes:
337
398
  typer.confirm(f"About to run migrations against {target_desc}. Continue?", abort=True)
@@ -341,13 +402,23 @@ def migrate_apply(
341
402
  else:
342
403
  try:
343
404
  targets = migrate.pending_migrations(
344
- migrations_dir, conninfo, tracking_table, areas=areas, exclude_areas=exclude_areas
405
+ migrations_dir,
406
+ conninfo,
407
+ tracking_table,
408
+ areas=areas,
409
+ exclude_areas=exclude_areas,
410
+ schemas=schemas,
411
+ exclude_schemas=exclude_schemas,
412
+ env=env,
345
413
  )
346
414
  except migrate.TrackingTableMissing:
347
415
  err_console.print(
348
416
  f"[yellow]⚠[/yellow] {tracking_table} not found — treating every migration as pending"
349
417
  )
350
- targets = migrate.list_migration_files(migrations_dir, areas=areas, exclude_areas=exclude_areas)
418
+ targets = migrate.list_migration_files(
419
+ migrations_dir, areas=areas, exclude_areas=exclude_areas, schemas=schemas,
420
+ exclude_schemas=exclude_schemas, env=env,
421
+ )
351
422
 
352
423
  if not targets:
353
424
  console.print("No pending migrations.")
@@ -43,6 +43,16 @@ _MSSQL_TYPE_SYNONYMS = {
43
43
  }
44
44
 
45
45
 
46
+ # Schemas that hold system catalog views/tables, never a file any project
47
+ # using pgdevkit manages -- a reference to one (e.g. an idempotency guard
48
+ # querying it, or a `SELECT ... FROM information_schema/pg_catalog/sys ...`)
49
+ # is never a real schema-membership or cross-file-dependency signal. Shared
50
+ # by `schemas.py` (schema-reference filtering) and `testdb/schema.py`
51
+ # (dependency-safe apply ordering), which both walk the same sqlglot Table
52
+ # nodes for a related-but-different purpose.
53
+ SYSTEM_SCHEMAS = {"pg_catalog", "information_schema", "sys"}
54
+
55
+
46
56
  @dataclass(frozen=True)
47
57
  class Dialect:
48
58
  """A thin wrapper around a sqlglot dialect name plus the handful of
@@ -0,0 +1,54 @@
1
+ """Optional `<name>.<env>.sql` filename convention: a file whose dot-segment
2
+ immediately before `.sql` names a deployment environment (e.g.
3
+ `grants.prod.sql`, `seed.staging.sql`) is only in scope when the caller is
4
+ targeting that same environment. A plain `<name>.sql` file (no such segment)
5
+ is untagged/common and is always in scope, regardless of which environment is
6
+ requested — mirroring the untagged-file rule for `-- area:` tags in
7
+ areas.py.
8
+
9
+ This generalizes the older, hardcoded `.prod.sql` convention (still the usual
10
+ name for a production-only file — grants, real user accounts — that
11
+ `pgdb testdb` should never touch); any string can now be used as an
12
+ environment name.
13
+
14
+ `.init.sql` (one-time setup, see docs/database-layout.md) is reserved and is
15
+ never interpreted as an environment tag.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from pathlib import Path
21
+
22
+ _RESERVED_SQL_SUFFIXES = {"init"}
23
+
24
+
25
+ def file_env(path: Path) -> str | None:
26
+ """The environment tag from `path`'s name, or None if it's untagged (or
27
+ the suffix is a reserved, non-env one like `.init.sql`). Only `.sql`
28
+ files can carry a tag."""
29
+ if path.suffix != ".sql":
30
+ return None
31
+ stem = path.stem
32
+ base, dot, suffix = stem.rpartition(".")
33
+ if not dot or suffix in _RESERVED_SQL_SUFFIXES:
34
+ return None
35
+ return suffix
36
+
37
+
38
+ def env_allowed(path: Path, env: str | None) -> bool:
39
+ """Whether `path` is in scope for `env`. `env=None` means no environment
40
+ filtering was requested, so every file (tagged or not) is in scope."""
41
+ if env is None:
42
+ return True
43
+ tag = file_env(path)
44
+ return tag is None or tag == env
45
+
46
+
47
+ def strip_env_suffix(path: Path) -> str:
48
+ """`path.stem` with a trailing `.<tag>` removed, so a tagged file
49
+ resolves to the same logical name as its untagged counterpart would
50
+ (e.g. `grants.prod.sql` -> "grants", same as `grants.sql`)."""
51
+ tag = file_env(path)
52
+ if tag is None:
53
+ return path.stem
54
+ return path.stem[: -(len(tag) + 1)]