pgdevkit 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pgdevkit/cli.py +68 -14
- pgdevkit/dialect.py +10 -0
- pgdevkit/migrate.py +13 -53
- pgdevkit/parser.py +9 -2
- pgdevkit/schemas.py +118 -0
- pgdevkit/sql_text.py +56 -0
- pgdevkit/testdb/api.py +49 -7
- pgdevkit/testdb/mssql/api.py +28 -4
- pgdevkit/testdb/schema.py +52 -17
- {pgdevkit-0.4.0.dist-info → pgdevkit-0.5.0.dist-info}/METADATA +62 -23
- {pgdevkit-0.4.0.dist-info → pgdevkit-0.5.0.dist-info}/RECORD +13 -11
- {pgdevkit-0.4.0.dist-info → pgdevkit-0.5.0.dist-info}/WHEEL +0 -0
- {pgdevkit-0.4.0.dist-info → pgdevkit-0.5.0.dist-info}/entry_points.txt +0 -0
pgdevkit/cli.py
CHANGED
|
@@ -29,9 +29,21 @@ _AREA_OPTION = typer.Option(
|
|
|
29
29
|
_EXCLUDE_AREA_OPTION = typer.Option(
|
|
30
30
|
[], "--exclude-area", help="Skip files declaring this area (repeatable); untagged files are never excluded"
|
|
31
31
|
)
|
|
32
|
+
_SCHEMA_OPTION = typer.Option(
|
|
33
|
+
[],
|
|
34
|
+
"--schema",
|
|
35
|
+
help="Restrict to files referencing this DB schema (repeatable); "
|
|
36
|
+
"files with no detectable schema reference always stay in scope",
|
|
37
|
+
)
|
|
38
|
+
_EXCLUDE_SCHEMA_OPTION = typer.Option(
|
|
39
|
+
[],
|
|
40
|
+
"--exclude-schema",
|
|
41
|
+
help="Skip files referencing this DB schema (repeatable); "
|
|
42
|
+
"files with no detectable schema reference are never excluded",
|
|
43
|
+
)
|
|
32
44
|
|
|
33
45
|
|
|
34
|
-
def
|
|
46
|
+
def _as_set(values: list[str]) -> frozenset[str] | None:
|
|
35
47
|
return frozenset(values) if values else None
|
|
36
48
|
|
|
37
49
|
testdb_app = typer.Typer(name="testdb", help="Manage the shared local Postgres test container")
|
|
@@ -59,17 +71,20 @@ def compare(
|
|
|
59
71
|
dialect: str = typer.Option("postgres", "--dialect", help="postgres (default) or mssql"),
|
|
60
72
|
area: list[str] = _AREA_OPTION,
|
|
61
73
|
exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
|
|
74
|
+
schema: list[str] = _SCHEMA_OPTION,
|
|
75
|
+
exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
|
|
62
76
|
scripts_dir: Path = typer.Argument(..., help="Directory containing SQL scripts"),
|
|
63
77
|
) -> None:
|
|
64
78
|
"""Compare SQL scripts to a live database and report differences."""
|
|
65
79
|
if not scripts_dir.is_dir():
|
|
66
80
|
err_console.print(f"[red]Error:[/red] {scripts_dir} is not a directory")
|
|
67
81
|
raise typer.Exit(2)
|
|
68
|
-
if report_extra_db and (area or exclude_area):
|
|
82
|
+
if report_extra_db and (area or exclude_area or schema or exclude_schema):
|
|
69
83
|
console.print(
|
|
70
|
-
"[yellow]⚠[/yellow] --report-extra-db with --area/--exclude-area will
|
|
71
|
-
"outside the filtered area(s) as \"missing in scripts\", since the
|
|
72
|
-
"of areas —
|
|
84
|
+
"[yellow]⚠[/yellow] --report-extra-db with --area/--exclude-area/--schema/--exclude-schema will "
|
|
85
|
+
"report every DB object outside the filtered area(s)/schema(s) as \"missing in scripts\", since the "
|
|
86
|
+
"live database has no concept of areas — and isn't itself filtered by --schema either — only the "
|
|
87
|
+
"scripts side is filtered."
|
|
73
88
|
)
|
|
74
89
|
|
|
75
90
|
try:
|
|
@@ -91,7 +106,12 @@ def compare(
|
|
|
91
106
|
|
|
92
107
|
with console.status("Parsing SQL scripts..."):
|
|
93
108
|
scripts_schema = parse_directory(
|
|
94
|
-
scripts_dir,
|
|
109
|
+
scripts_dir,
|
|
110
|
+
dialect=backend.dialect,
|
|
111
|
+
areas=_as_set(area),
|
|
112
|
+
exclude_areas=_as_set(exclude_area),
|
|
113
|
+
schemas=_as_set(schema),
|
|
114
|
+
exclude_schemas=_as_set(exclude_schema),
|
|
95
115
|
)
|
|
96
116
|
|
|
97
117
|
with console.status("Introspecting database..."):
|
|
@@ -188,17 +208,33 @@ def fetch_missing(
|
|
|
188
208
|
|
|
189
209
|
|
|
190
210
|
@testdb_app.command("up")
|
|
191
|
-
def testdb_up(
|
|
211
|
+
def testdb_up(
|
|
212
|
+
area: list[str] = _AREA_OPTION,
|
|
213
|
+
exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
|
|
214
|
+
schema: list[str] = _SCHEMA_OPTION,
|
|
215
|
+
exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
|
|
216
|
+
) -> None:
|
|
192
217
|
"""Ensure the container is running, the workspace DB exists, and schema is applied."""
|
|
193
|
-
testdb.ensure_testdb(
|
|
218
|
+
testdb.ensure_testdb(
|
|
219
|
+
areas=_as_set(area), exclude_areas=_as_set(exclude_area), schemas=_as_set(schema),
|
|
220
|
+
exclude_schemas=_as_set(exclude_schema),
|
|
221
|
+
)
|
|
194
222
|
info = testdb.status()
|
|
195
223
|
console.print(f"[green]Test DB ready:[/green] {info['database']} ({info['dsn']})")
|
|
196
224
|
|
|
197
225
|
|
|
198
226
|
@testdb_app.command("reset")
|
|
199
|
-
def testdb_reset(
|
|
227
|
+
def testdb_reset(
|
|
228
|
+
area: list[str] = _AREA_OPTION,
|
|
229
|
+
exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
|
|
230
|
+
schema: list[str] = _SCHEMA_OPTION,
|
|
231
|
+
exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
|
|
232
|
+
) -> None:
|
|
200
233
|
"""Drop and recreate only this workspace's database, then reapply schema + seed data."""
|
|
201
|
-
testdb.reset_testdb(
|
|
234
|
+
testdb.reset_testdb(
|
|
235
|
+
areas=_as_set(area), exclude_areas=_as_set(exclude_area), schemas=_as_set(schema),
|
|
236
|
+
exclude_schemas=_as_set(exclude_schema),
|
|
237
|
+
)
|
|
202
238
|
info = testdb.status()
|
|
203
239
|
console.print(f"[green]Test DB reset:[/green] {info['database']}")
|
|
204
240
|
|
|
@@ -272,6 +308,8 @@ def migrate_check(
|
|
|
272
308
|
),
|
|
273
309
|
area: list[str] = _AREA_OPTION,
|
|
274
310
|
exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
|
|
311
|
+
schema: list[str] = _SCHEMA_OPTION,
|
|
312
|
+
exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
|
|
275
313
|
) -> None:
|
|
276
314
|
"""List which migration files under migrations_dir are applied vs. pending."""
|
|
277
315
|
if not migrations_dir.is_dir():
|
|
@@ -281,7 +319,11 @@ def migrate_check(
|
|
|
281
319
|
conninfo = build_conninfo(url, entra_user)
|
|
282
320
|
tracking_table = tracking_table or migrate.default_tracking_table(migrations_dir)
|
|
283
321
|
local_files = migrate.list_migration_files(
|
|
284
|
-
migrations_dir,
|
|
322
|
+
migrations_dir,
|
|
323
|
+
areas=_as_set(area),
|
|
324
|
+
exclude_areas=_as_set(exclude_area),
|
|
325
|
+
schemas=_as_set(schema),
|
|
326
|
+
exclude_schemas=_as_set(exclude_schema),
|
|
285
327
|
)
|
|
286
328
|
try:
|
|
287
329
|
applied = migrate.applied_migrations(conninfo, tracking_table)
|
|
@@ -323,6 +365,8 @@ def migrate_apply(
|
|
|
323
365
|
yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm-target prompt"),
|
|
324
366
|
area: list[str] = _AREA_OPTION,
|
|
325
367
|
exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
|
|
368
|
+
schema: list[str] = _SCHEMA_OPTION,
|
|
369
|
+
exclude_schema: list[str] = _EXCLUDE_SCHEMA_OPTION,
|
|
326
370
|
) -> None:
|
|
327
371
|
"""Apply pending migration files, in filename order, tracking each in tracking_table."""
|
|
328
372
|
if not migrations_dir.is_dir():
|
|
@@ -331,7 +375,8 @@ def migrate_apply(
|
|
|
331
375
|
|
|
332
376
|
conninfo = build_conninfo(url, entra_user)
|
|
333
377
|
tracking_table = tracking_table or migrate.default_tracking_table(migrations_dir)
|
|
334
|
-
areas, exclude_areas =
|
|
378
|
+
areas, exclude_areas = _as_set(area), _as_set(exclude_area)
|
|
379
|
+
schemas, exclude_schemas = _as_set(schema), _as_set(exclude_schema)
|
|
335
380
|
target_desc = url.rsplit("@", 1)[-1] if "@" in url else url
|
|
336
381
|
if not yes:
|
|
337
382
|
typer.confirm(f"About to run migrations against {target_desc}. Continue?", abort=True)
|
|
@@ -341,13 +386,22 @@ def migrate_apply(
|
|
|
341
386
|
else:
|
|
342
387
|
try:
|
|
343
388
|
targets = migrate.pending_migrations(
|
|
344
|
-
migrations_dir,
|
|
389
|
+
migrations_dir,
|
|
390
|
+
conninfo,
|
|
391
|
+
tracking_table,
|
|
392
|
+
areas=areas,
|
|
393
|
+
exclude_areas=exclude_areas,
|
|
394
|
+
schemas=schemas,
|
|
395
|
+
exclude_schemas=exclude_schemas,
|
|
345
396
|
)
|
|
346
397
|
except migrate.TrackingTableMissing:
|
|
347
398
|
err_console.print(
|
|
348
399
|
f"[yellow]⚠[/yellow] {tracking_table} not found — treating every migration as pending"
|
|
349
400
|
)
|
|
350
|
-
targets = migrate.list_migration_files(
|
|
401
|
+
targets = migrate.list_migration_files(
|
|
402
|
+
migrations_dir, areas=areas, exclude_areas=exclude_areas, schemas=schemas,
|
|
403
|
+
exclude_schemas=exclude_schemas,
|
|
404
|
+
)
|
|
351
405
|
|
|
352
406
|
if not targets:
|
|
353
407
|
console.print("No pending migrations.")
|
pgdevkit/dialect.py
CHANGED
|
@@ -43,6 +43,16 @@ _MSSQL_TYPE_SYNONYMS = {
|
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
|
|
46
|
+
# Schemas that hold system catalog views/tables, never a file any project
|
|
47
|
+
# using pgdevkit manages -- a reference to one (e.g. an idempotency guard
|
|
48
|
+
# querying it, or a `SELECT ... FROM information_schema/pg_catalog/sys ...`)
|
|
49
|
+
# is never a real schema-membership or cross-file-dependency signal. Shared
|
|
50
|
+
# by `schemas.py` (schema-reference filtering) and `testdb/schema.py`
|
|
51
|
+
# (dependency-safe apply ordering), which both walk the same sqlglot Table
|
|
52
|
+
# nodes for a related-but-different purpose.
|
|
53
|
+
SYSTEM_SCHEMAS = {"pg_catalog", "information_schema", "sys"}
|
|
54
|
+
|
|
55
|
+
|
|
46
56
|
@dataclass(frozen=True)
|
|
47
57
|
class Dialect:
|
|
48
58
|
"""A thin wrapper around a sqlglot dialect name plus the handful of
|
pgdevkit/migrate.py
CHANGED
|
@@ -19,6 +19,8 @@ from psycopg import errors as pg_errors
|
|
|
19
19
|
from psycopg import sql as pg_sql
|
|
20
20
|
|
|
21
21
|
from .areas import filter_by_area
|
|
22
|
+
from .schemas import filter_by_schema
|
|
23
|
+
from .sql_text import strip_line_comments
|
|
22
24
|
|
|
23
25
|
_IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_]*"
|
|
24
26
|
_DEFAULT_TRACKING_TABLE = "public.schema_migrations"
|
|
@@ -124,61 +126,12 @@ def _split_sql(sql: str) -> list[str]:
|
|
|
124
126
|
return stmts
|
|
125
127
|
|
|
126
128
|
|
|
127
|
-
def _strip_line_comments(sql: str) -> str:
|
|
128
|
-
"""Drop '--' line comments, respecting string literals and $$...$$ blocks."""
|
|
129
|
-
buf: list[str] = []
|
|
130
|
-
i = 0
|
|
131
|
-
in_string = False
|
|
132
|
-
in_line_comment = False
|
|
133
|
-
dollar_tag: str | None = None
|
|
134
|
-
|
|
135
|
-
while i < len(sql):
|
|
136
|
-
c = sql[i]
|
|
137
|
-
if in_line_comment:
|
|
138
|
-
if c == "\n":
|
|
139
|
-
in_line_comment = False
|
|
140
|
-
buf.append(c)
|
|
141
|
-
elif dollar_tag is not None:
|
|
142
|
-
buf.append(c)
|
|
143
|
-
if c == "$" and sql[i:i + len(dollar_tag)] == dollar_tag:
|
|
144
|
-
buf.extend(list(dollar_tag[1:]))
|
|
145
|
-
i += len(dollar_tag)
|
|
146
|
-
dollar_tag = None
|
|
147
|
-
continue
|
|
148
|
-
elif in_string:
|
|
149
|
-
if c == "'" and i + 1 < len(sql) and sql[i + 1] == "'":
|
|
150
|
-
buf.append(c)
|
|
151
|
-
buf.append(sql[i + 1])
|
|
152
|
-
i += 2
|
|
153
|
-
continue
|
|
154
|
-
elif c == "'":
|
|
155
|
-
in_string = False
|
|
156
|
-
buf.append(c)
|
|
157
|
-
elif c == "-" and i + 1 < len(sql) and sql[i + 1] == "-":
|
|
158
|
-
in_line_comment = True
|
|
159
|
-
elif c == "$":
|
|
160
|
-
m = re.match(r"\$([A-Za-z0-9_]*)\$", sql[i:])
|
|
161
|
-
if m:
|
|
162
|
-
dollar_tag = m.group(0)
|
|
163
|
-
buf.extend(list(dollar_tag))
|
|
164
|
-
i += len(dollar_tag)
|
|
165
|
-
continue
|
|
166
|
-
buf.append(c)
|
|
167
|
-
elif c == "'":
|
|
168
|
-
in_string = True
|
|
169
|
-
buf.append(c)
|
|
170
|
-
else:
|
|
171
|
-
buf.append(c)
|
|
172
|
-
i += 1
|
|
173
|
-
return "".join(buf)
|
|
174
|
-
|
|
175
|
-
|
|
176
129
|
def _created_table_names(stmts: list[str]) -> list[str]:
|
|
177
130
|
"""Names of tables any CREATE TABLE statement targets, parsed via sqlglot (falls back to
|
|
178
131
|
regex on comment-stripped text for statements sqlglot's postgres dialect can't parse)."""
|
|
179
132
|
names: list[str] = []
|
|
180
133
|
for stmt in stmts:
|
|
181
|
-
stripped =
|
|
134
|
+
stripped = strip_line_comments(stmt)
|
|
182
135
|
if not re.search(r"CREATE\s+TABLE", stripped, re.IGNORECASE):
|
|
183
136
|
continue # skip sqlglot entirely for statements that can't be a CREATE TABLE
|
|
184
137
|
try:
|
|
@@ -220,7 +173,7 @@ def _idempotent_target(stmt: str) -> tuple[str, ...] | None:
|
|
|
220
173
|
COLUMN. None if the statement isn't one of these shapes — including any CREATE OR
|
|
221
174
|
REPLACE, which is never safe to treat as a no-op just because the object exists, since
|
|
222
175
|
the migration could be replacing it with different content."""
|
|
223
|
-
stripped =
|
|
176
|
+
stripped = strip_line_comments(stmt).strip()
|
|
224
177
|
if re.search(r"\bOR\s+REPLACE\b", stripped, re.IGNORECASE):
|
|
225
178
|
return None
|
|
226
179
|
|
|
@@ -281,9 +234,12 @@ def list_migration_files(
|
|
|
281
234
|
*,
|
|
282
235
|
areas: frozenset[str] | None = None,
|
|
283
236
|
exclude_areas: frozenset[str] | None = None,
|
|
237
|
+
schemas: frozenset[str] | None = None,
|
|
238
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
284
239
|
) -> list[Path]:
|
|
285
240
|
files = sorted(migrations_dir.glob("*.sql"))
|
|
286
|
-
|
|
241
|
+
files = filter_by_area(files, only=areas, exclude=exclude_areas)
|
|
242
|
+
return filter_by_schema(files, only=schemas, exclude=exclude_schemas)
|
|
287
243
|
|
|
288
244
|
|
|
289
245
|
def applied_migrations(conninfo: str, tracking_table: str) -> dict[str, tuple[datetime, str]]:
|
|
@@ -306,9 +262,13 @@ def pending_migrations(
|
|
|
306
262
|
*,
|
|
307
263
|
areas: frozenset[str] | None = None,
|
|
308
264
|
exclude_areas: frozenset[str] | None = None,
|
|
265
|
+
schemas: frozenset[str] | None = None,
|
|
266
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
309
267
|
) -> list[Path]:
|
|
310
268
|
applied = applied_migrations(conninfo, tracking_table)
|
|
311
|
-
files = list_migration_files(
|
|
269
|
+
files = list_migration_files(
|
|
270
|
+
migrations_dir, areas=areas, exclude_areas=exclude_areas, schemas=schemas, exclude_schemas=exclude_schemas
|
|
271
|
+
)
|
|
312
272
|
return [p for p in files if p.name not in applied]
|
|
313
273
|
|
|
314
274
|
|
pgdevkit/parser.py
CHANGED
|
@@ -10,6 +10,7 @@ import sqlglot.expressions as exp
|
|
|
10
10
|
|
|
11
11
|
from .areas import area_allowed, parse_areas
|
|
12
12
|
from .dialect import Dialect, POSTGRES, resolve_dialect
|
|
13
|
+
from .schemas import schema_allowed, sql_schemas
|
|
13
14
|
from .models import (
|
|
14
15
|
ColumnDef, ConstraintDef, CompositeTypeDef, DatabaseSchema,
|
|
15
16
|
EnumDef, FunctionDef, IndexDef, TableDef, ViewDef,
|
|
@@ -55,15 +56,21 @@ def parse_directory(
|
|
|
55
56
|
dialect: str | Dialect = "postgres",
|
|
56
57
|
areas: frozenset[str] | None = None,
|
|
57
58
|
exclude_areas: frozenset[str] | None = None,
|
|
59
|
+
schemas: frozenset[str] | None = None,
|
|
60
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
58
61
|
) -> DatabaseSchema:
|
|
59
62
|
resolved = resolve_dialect(dialect)
|
|
60
63
|
db_schema = DatabaseSchema()
|
|
61
64
|
for sql_file in sorted(_iter_sql_files(scripts_dir)):
|
|
62
|
-
# Read once and reuse for
|
|
63
|
-
# filtering the file list up front (which would need its own read).
|
|
65
|
+
# Read once and reuse for the area/schema checks and parsing, rather
|
|
66
|
+
# than filtering the file list up front (which would need its own read).
|
|
64
67
|
content = sql_file.read_text(encoding="utf-8")
|
|
65
68
|
if (areas or exclude_areas) and not area_allowed(parse_areas(content), only=areas, exclude=exclude_areas):
|
|
66
69
|
continue
|
|
70
|
+
if (schemas or exclude_schemas) and not schema_allowed(
|
|
71
|
+
sql_schemas(content, resolved), only=schemas, exclude=exclude_schemas
|
|
72
|
+
):
|
|
73
|
+
continue
|
|
67
74
|
_parse_file(sql_file, db_schema, resolved, content=content)
|
|
68
75
|
return db_schema
|
|
69
76
|
|
pgdevkit/schemas.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Schema-membership filtering, composable with (but independent of) the
|
|
2
|
+
`-- area:` tag filtering in `areas.py`.
|
|
3
|
+
|
|
4
|
+
Unlike area, which is an explicit opt-in tag, a file's schema membership is
|
|
5
|
+
derived by parsing its SQL: every schema-qualified (or default-schema, when
|
|
6
|
+
unqualified) table/view/function/index/schema reference across every
|
|
7
|
+
statement in the file, DDL or DML alike.
|
|
8
|
+
|
|
9
|
+
A file whose schema(s) can't be determined -- content sqlglot can't parse at
|
|
10
|
+
all, or with no table/schema reference in it (e.g. a DO block touching no
|
|
11
|
+
table) -- is treated the same as an untagged file for `-- area:`: `only`
|
|
12
|
+
filters always keep it, `exclude` filters never drop it. Being unable to
|
|
13
|
+
prove a file belongs to an excluded schema is not the same as proving it
|
|
14
|
+
doesn't, so this errs toward keeping the file in scope rather than silently
|
|
15
|
+
dropping it.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import re
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
import sqlglot
|
|
24
|
+
import sqlglot.expressions as exp
|
|
25
|
+
|
|
26
|
+
from .dialect import Dialect, POSTGRES, SYSTEM_SCHEMAS
|
|
27
|
+
from .sql_text import strip_line_comments
|
|
28
|
+
|
|
29
|
+
_CREATE_SCHEMA_RE = re.compile(
|
|
30
|
+
r"CREATE\s+SCHEMA\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:AUTHORIZATION\s+)?\"?(\w+)\"?", re.IGNORECASE
|
|
31
|
+
)
|
|
32
|
+
_QUALIFIED_REF_RE = re.compile(r"\b(\w+)\.\w+")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _regex_fallback(sql: str) -> frozenset[str]:
|
|
36
|
+
"""Crude schema scan used when sqlglot can't parse `sql` at all, or found
|
|
37
|
+
no reference. Errs toward over-matching (any `schema.name`-shaped token)
|
|
38
|
+
rather than under-matching -- a false positive here only keeps a file in
|
|
39
|
+
scope for one extra schema filter, where a false negative would silently
|
|
40
|
+
drop it from an `only` filter. Comments are stripped first so a schema
|
|
41
|
+
merely *mentioned* in one (e.g. "-- see billing.invoice for context")
|
|
42
|
+
isn't mistaken for a real reference; string literals are left as-is
|
|
43
|
+
(matching one inside a literal is the same over-matching tradeoff as
|
|
44
|
+
above, not worth the complexity of also blanking them out)."""
|
|
45
|
+
stripped = strip_line_comments(sql)
|
|
46
|
+
schemas = set(_CREATE_SCHEMA_RE.findall(stripped))
|
|
47
|
+
schemas.update(m.group(1) for m in _QUALIFIED_REF_RE.finditer(stripped))
|
|
48
|
+
return frozenset(s for s in schemas if s.lower() not in SYSTEM_SCHEMAS)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def sql_schemas(content: str, dialect: Dialect = POSTGRES) -> frozenset[str]:
|
|
52
|
+
"""Schema names referenced anywhere in `content`: every table/view/
|
|
53
|
+
function/index reference's schema (its `dialect.default_schema` when
|
|
54
|
+
unqualified), plus any `CREATE SCHEMA name`. Falls back to a regex scan
|
|
55
|
+
when sqlglot can't parse the content, or finds no reference at all."""
|
|
56
|
+
try:
|
|
57
|
+
exprs = sqlglot.parse(content, dialect=dialect.sqlglot_name, error_level=sqlglot.ErrorLevel.IGNORE)
|
|
58
|
+
except Exception: # noqa: BLE001
|
|
59
|
+
return _regex_fallback(content)
|
|
60
|
+
|
|
61
|
+
schemas: set[str] = set()
|
|
62
|
+
for e in exprs:
|
|
63
|
+
if e is None:
|
|
64
|
+
continue
|
|
65
|
+
for t in e.find_all(exp.Table):
|
|
66
|
+
# T-SQL's `EXEC('...dynamic sql...')` parses as a Table subquery
|
|
67
|
+
# whose "name" is the whole literal string, not a real
|
|
68
|
+
# identifier -- e.g. a quoted "CREATE SCHEMA app". Guard against
|
|
69
|
+
# treating that as a schema-qualified (or default-schema) table
|
|
70
|
+
# reference by rejecting a name containing whitespace -- a real
|
|
71
|
+
# identifier, quoted or not (even one with hyphens or other
|
|
72
|
+
# punctuation), never has any, while embedded SQL text always
|
|
73
|
+
# does. Still accepts an empty name, the legitimate shape
|
|
74
|
+
# sqlglot gives a bare `CREATE SCHEMA x` (whose schema name ends
|
|
75
|
+
# up in `db`, not `this`).
|
|
76
|
+
if t.name and re.search(r"\s", t.name):
|
|
77
|
+
continue
|
|
78
|
+
db_node = t.args.get("db")
|
|
79
|
+
name = db_node.name if db_node else dialect.default_schema
|
|
80
|
+
if name.lower() not in SYSTEM_SCHEMAS:
|
|
81
|
+
schemas.add(name)
|
|
82
|
+
|
|
83
|
+
if not schemas:
|
|
84
|
+
return _regex_fallback(content)
|
|
85
|
+
return frozenset(schemas)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def file_schemas(path: Path, dialect: Dialect = POSTGRES) -> frozenset[str]:
|
|
89
|
+
"""Schema names referenced in the file at `path`."""
|
|
90
|
+
return sql_schemas(path.read_text(encoding="utf-8"), dialect)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def schema_allowed(
|
|
94
|
+
schemas: frozenset[str],
|
|
95
|
+
*,
|
|
96
|
+
only: frozenset[str] | None = None,
|
|
97
|
+
exclude: frozenset[str] | None = None,
|
|
98
|
+
) -> bool:
|
|
99
|
+
"""Whether a file that references `schemas` passes an `only`/`exclude` filter."""
|
|
100
|
+
if exclude and schemas & exclude:
|
|
101
|
+
return False
|
|
102
|
+
if only and schemas and not (schemas & only):
|
|
103
|
+
return False
|
|
104
|
+
return True
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def filter_by_schema(
|
|
108
|
+
paths: list[Path],
|
|
109
|
+
*,
|
|
110
|
+
only: frozenset[str] | None = None,
|
|
111
|
+
exclude: frozenset[str] | None = None,
|
|
112
|
+
dialect: Dialect = POSTGRES,
|
|
113
|
+
) -> list[Path]:
|
|
114
|
+
"""`paths` restricted by an `only`/`exclude` schema filter. Returns `paths`
|
|
115
|
+
unchanged (no file reads) when neither filter is set."""
|
|
116
|
+
if not only and not exclude:
|
|
117
|
+
return paths
|
|
118
|
+
return [p for p in paths if schema_allowed(file_schemas(p, dialect), only=only, exclude=exclude)]
|
pgdevkit/sql_text.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Small text-level SQL helpers shared by modules that can't import each
|
|
2
|
+
other directly (`migrate.py` imports `schemas.py`, so `schemas.py` can't
|
|
3
|
+
import back from `migrate.py`)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def strip_line_comments(sql: str) -> str:
|
|
11
|
+
"""Drop '--' line comments, respecting string literals and $$...$$ blocks."""
|
|
12
|
+
buf: list[str] = []
|
|
13
|
+
i = 0
|
|
14
|
+
in_string = False
|
|
15
|
+
in_line_comment = False
|
|
16
|
+
dollar_tag: str | None = None
|
|
17
|
+
|
|
18
|
+
while i < len(sql):
|
|
19
|
+
c = sql[i]
|
|
20
|
+
if in_line_comment:
|
|
21
|
+
if c == "\n":
|
|
22
|
+
in_line_comment = False
|
|
23
|
+
buf.append(c)
|
|
24
|
+
elif dollar_tag is not None:
|
|
25
|
+
buf.append(c)
|
|
26
|
+
if c == "$" and sql[i:i + len(dollar_tag)] == dollar_tag:
|
|
27
|
+
buf.extend(list(dollar_tag[1:]))
|
|
28
|
+
i += len(dollar_tag)
|
|
29
|
+
dollar_tag = None
|
|
30
|
+
continue
|
|
31
|
+
elif in_string:
|
|
32
|
+
if c == "'" and i + 1 < len(sql) and sql[i + 1] == "'":
|
|
33
|
+
buf.append(c)
|
|
34
|
+
buf.append(sql[i + 1])
|
|
35
|
+
i += 2
|
|
36
|
+
continue
|
|
37
|
+
elif c == "'":
|
|
38
|
+
in_string = False
|
|
39
|
+
buf.append(c)
|
|
40
|
+
elif c == "-" and i + 1 < len(sql) and sql[i + 1] == "-":
|
|
41
|
+
in_line_comment = True
|
|
42
|
+
elif c == "$":
|
|
43
|
+
m = re.match(r"\$([A-Za-z0-9_]*)\$", sql[i:])
|
|
44
|
+
if m:
|
|
45
|
+
dollar_tag = m.group(0)
|
|
46
|
+
buf.extend(list(dollar_tag))
|
|
47
|
+
i += len(dollar_tag)
|
|
48
|
+
continue
|
|
49
|
+
buf.append(c)
|
|
50
|
+
elif c == "'":
|
|
51
|
+
in_string = True
|
|
52
|
+
buf.append(c)
|
|
53
|
+
else:
|
|
54
|
+
buf.append(c)
|
|
55
|
+
i += 1
|
|
56
|
+
return "".join(buf)
|
pgdevkit/testdb/api.py
CHANGED
|
@@ -68,24 +68,53 @@ async def _drop_database(db_name: str) -> None:
|
|
|
68
68
|
await con.execute(SQL("DROP DATABASE IF EXISTS {}").format(Identifier(db_name)))
|
|
69
69
|
|
|
70
70
|
|
|
71
|
-
async def _apply(
|
|
71
|
+
async def _apply(
|
|
72
|
+
config: ProjectConfig,
|
|
73
|
+
db_name: str,
|
|
74
|
+
force_reset: bool,
|
|
75
|
+
*,
|
|
76
|
+
areas: frozenset[str] | None = None,
|
|
77
|
+
exclude_areas: frozenset[str] | None = None,
|
|
78
|
+
schemas: frozenset[str] | None = None,
|
|
79
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
80
|
+
) -> None:
|
|
72
81
|
async with await psycopg.AsyncConnection.connect(_db_dsn(db_name), autocommit=True) as con:
|
|
73
82
|
await apply_schema(
|
|
74
83
|
con,
|
|
75
84
|
config.root / config.database_dir,
|
|
76
85
|
extensions=config.extensions,
|
|
77
86
|
force_reset=force_reset,
|
|
87
|
+
areas=areas,
|
|
88
|
+
exclude_areas=exclude_areas,
|
|
89
|
+
schemas=schemas,
|
|
90
|
+
exclude_schemas=exclude_schemas,
|
|
78
91
|
)
|
|
79
92
|
|
|
80
93
|
|
|
81
|
-
def ensure_testdb(
|
|
94
|
+
def ensure_testdb(
|
|
95
|
+
project_root: Path | None = None,
|
|
96
|
+
force_reset: bool = False,
|
|
97
|
+
*,
|
|
98
|
+
areas: frozenset[str] | None = None,
|
|
99
|
+
exclude_areas: frozenset[str] | None = None,
|
|
100
|
+
schemas: frozenset[str] | None = None,
|
|
101
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
102
|
+
) -> dict[str, str]:
|
|
82
103
|
"""Ensure the shared container is running, this workspace's database
|
|
83
104
|
exists, and its schema is applied. Returns the {PREFIX}POSTGRES_* env
|
|
84
105
|
vars for this workspace (or the mssql equivalent's env vars, per
|
|
85
|
-
`config.engine`).
|
|
106
|
+
`config.engine`).
|
|
107
|
+
|
|
108
|
+
`areas`/`exclude_areas` and `schemas`/`exclude_schemas` restrict which
|
|
109
|
+
database/ files get applied -- e.g. for a test DB scoped to one area or
|
|
110
|
+
schema. Neither filters what gets *dropped* by force_reset/clean, only
|
|
111
|
+
what gets (re)applied."""
|
|
86
112
|
config, db_name = _resolve(project_root)
|
|
87
113
|
if config.engine == "mssql":
|
|
88
|
-
return _mssql_api().ensure_testdb(
|
|
114
|
+
return _mssql_api().ensure_testdb(
|
|
115
|
+
config, db_name, force_reset,
|
|
116
|
+
areas=areas, exclude_areas=exclude_areas, schemas=schemas, exclude_schemas=exclude_schemas,
|
|
117
|
+
)
|
|
89
118
|
|
|
90
119
|
ensure_container()
|
|
91
120
|
|
|
@@ -93,16 +122,29 @@ def ensure_testdb(project_root: Path | None = None, force_reset: bool = False) -
|
|
|
93
122
|
if force_reset:
|
|
94
123
|
await _drop_database(db_name)
|
|
95
124
|
await _ensure_database(db_name)
|
|
96
|
-
await _apply(
|
|
125
|
+
await _apply(
|
|
126
|
+
config, db_name, force_reset,
|
|
127
|
+
areas=areas, exclude_areas=exclude_areas, schemas=schemas, exclude_schemas=exclude_schemas,
|
|
128
|
+
)
|
|
97
129
|
|
|
98
130
|
asyncio.run(_run())
|
|
99
131
|
return _env_for(config, db_name)
|
|
100
132
|
|
|
101
133
|
|
|
102
|
-
def reset_testdb(
|
|
134
|
+
def reset_testdb(
|
|
135
|
+
project_root: Path | None = None,
|
|
136
|
+
*,
|
|
137
|
+
areas: frozenset[str] | None = None,
|
|
138
|
+
exclude_areas: frozenset[str] | None = None,
|
|
139
|
+
schemas: frozenset[str] | None = None,
|
|
140
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
141
|
+
) -> dict[str, str]:
|
|
103
142
|
"""Drop and recreate only this workspace's database, then reapply
|
|
104
143
|
schema and seed data."""
|
|
105
|
-
return ensure_testdb(
|
|
144
|
+
return ensure_testdb(
|
|
145
|
+
project_root, force_reset=True,
|
|
146
|
+
areas=areas, exclude_areas=exclude_areas, schemas=schemas, exclude_schemas=exclude_schemas,
|
|
147
|
+
)
|
|
106
148
|
|
|
107
149
|
|
|
108
150
|
def clean_testdb(project_root: Path | None = None, all: bool = False) -> None:
|
pgdevkit/testdb/mssql/api.py
CHANGED
|
@@ -101,7 +101,16 @@ async def _insert_test_data(json_file: Path, table: str, force_reset: bool, conn
|
|
|
101
101
|
await asyncio.to_thread(_run)
|
|
102
102
|
|
|
103
103
|
|
|
104
|
-
async def _apply(
|
|
104
|
+
async def _apply(
|
|
105
|
+
config: ProjectConfig,
|
|
106
|
+
db_name: str,
|
|
107
|
+
force_reset: bool,
|
|
108
|
+
*,
|
|
109
|
+
areas: frozenset[str] | None = None,
|
|
110
|
+
exclude_areas: frozenset[str] | None = None,
|
|
111
|
+
schemas: frozenset[str] | None = None,
|
|
112
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
113
|
+
) -> None:
|
|
105
114
|
def _connect() -> Any:
|
|
106
115
|
return mssql_python.connect(_db_dsn(db_name), autocommit=True)
|
|
107
116
|
|
|
@@ -110,7 +119,10 @@ async def _apply(config: ProjectConfig, db_name: str, force_reset: bool) -> None
|
|
|
110
119
|
database_dir = config.root / config.database_dir
|
|
111
120
|
if not database_dir.is_dir():
|
|
112
121
|
return
|
|
113
|
-
for file, sql in _iter_sql_files(
|
|
122
|
+
for file, sql in _iter_sql_files(
|
|
123
|
+
database_dir, MSSQL,
|
|
124
|
+
areas=areas, exclude_areas=exclude_areas, schemas=schemas, exclude_schemas=exclude_schemas,
|
|
125
|
+
):
|
|
114
126
|
for batch in query.split_tsql_batches(sql):
|
|
115
127
|
|
|
116
128
|
def _exec(batch: str = batch) -> None:
|
|
@@ -126,14 +138,26 @@ async def _apply(config: ProjectConfig, db_name: str, force_reset: bool) -> None
|
|
|
126
138
|
await asyncio.to_thread(conn.close)
|
|
127
139
|
|
|
128
140
|
|
|
129
|
-
def ensure_testdb(
|
|
141
|
+
def ensure_testdb(
|
|
142
|
+
config: ProjectConfig,
|
|
143
|
+
db_name: str,
|
|
144
|
+
force_reset: bool,
|
|
145
|
+
*,
|
|
146
|
+
areas: frozenset[str] | None = None,
|
|
147
|
+
exclude_areas: frozenset[str] | None = None,
|
|
148
|
+
schemas: frozenset[str] | None = None,
|
|
149
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
150
|
+
) -> dict[str, str]:
|
|
130
151
|
ensure_mssql_container()
|
|
131
152
|
|
|
132
153
|
async def _run() -> None:
|
|
133
154
|
if force_reset:
|
|
134
155
|
await _drop_database(db_name)
|
|
135
156
|
await _ensure_database(db_name)
|
|
136
|
-
await _apply(
|
|
157
|
+
await _apply(
|
|
158
|
+
config, db_name, force_reset,
|
|
159
|
+
areas=areas, exclude_areas=exclude_areas, schemas=schemas, exclude_schemas=exclude_schemas,
|
|
160
|
+
)
|
|
137
161
|
|
|
138
162
|
asyncio.run(_run())
|
|
139
163
|
return _env_for(config, db_name)
|
pgdevkit/testdb/schema.py
CHANGED
|
@@ -13,9 +13,11 @@ import sqlglot.expressions as exp
|
|
|
13
13
|
from psycopg.rows import dict_row
|
|
14
14
|
from psycopg.sql import SQL, Identifier, Placeholder
|
|
15
15
|
|
|
16
|
+
from ..areas import area_allowed, parse_areas
|
|
16
17
|
from ..db.complex_types import ComplexHelper
|
|
17
|
-
from ..dialect import Dialect, POSTGRES
|
|
18
|
+
from ..dialect import Dialect, POSTGRES, SYSTEM_SCHEMAS
|
|
18
19
|
from ..parser import IGNORED_DIR_NAMES
|
|
20
|
+
from ..schemas import schema_allowed, sql_schemas
|
|
19
21
|
|
|
20
22
|
logger = logging.getLogger(__name__)
|
|
21
23
|
logging.getLogger("sqlglot").setLevel(logging.ERROR)
|
|
@@ -65,16 +67,14 @@ _SCHEMA_QUALIFIED_TYPES = {
|
|
|
65
67
|
}
|
|
66
68
|
|
|
67
69
|
|
|
68
|
-
#
|
|
69
|
-
#
|
|
70
|
-
#
|
|
71
|
-
#
|
|
72
|
-
#
|
|
73
|
-
#
|
|
74
|
-
#
|
|
75
|
-
#
|
|
76
|
-
# reverse-order resolution can then apply files out of their intended order.
|
|
77
|
-
_SYSTEM_SCHEMAS = {"pg_catalog", "information_schema", "sys"}
|
|
70
|
+
# SYSTEM_SCHEMAS (see dialect.py) matters most here for T-SQL, where
|
|
71
|
+
# "IF NOT EXISTS (SELECT ... FROM sys.schemas/sys.tables/sys.objects ...)
|
|
72
|
+
# BEGIN CREATE ... END" is the idiomatic idempotency-guard pattern (T-SQL has
|
|
73
|
+
# no native "CREATE TABLE IF NOT EXISTS"/"CREATE SCHEMA IF NOT EXISTS"), so
|
|
74
|
+
# without this exclusion nearly every T-SQL file would pick up a spurious,
|
|
75
|
+
# never-resolvable dependency on "sys.*" and get shuffled into the
|
|
76
|
+
# delayed-retry path, whose reverse-order resolution can then apply files
|
|
77
|
+
# out of their intended order.
|
|
78
78
|
|
|
79
79
|
_DECLARE_RE = re.compile(
|
|
80
80
|
r"CREATE\s+(?:OR\s+REPLACE\s+)?(?:TABLE|VIEW|FUNCTION|PROCEDURE|TYPE|SCHEMA)\s+(\w+\.\w+)", re.IGNORECASE
|
|
@@ -89,7 +89,7 @@ def _get_sql_deps_regex_fallback(sql: str) -> set[str]:
|
|
|
89
89
|
is fine."""
|
|
90
90
|
declares = set(_DECLARE_RE.findall(sql))
|
|
91
91
|
deps = set(_DEPEND_RE.findall(sql))
|
|
92
|
-
deps = {d for d in deps if d.split(".", 1)[0].lower() not in
|
|
92
|
+
deps = {d for d in deps if d.split(".", 1)[0].lower() not in SYSTEM_SCHEMAS}
|
|
93
93
|
return deps - declares
|
|
94
94
|
|
|
95
95
|
|
|
@@ -109,7 +109,7 @@ def _get_sql_deps(sql: str, dialect: Dialect = POSTGRES) -> set[str]:
|
|
|
109
109
|
for t in e.find_all(exp.Table):
|
|
110
110
|
db_node = t.args.get("db")
|
|
111
111
|
if t.args.get("this") is not None and db_node is not None:
|
|
112
|
-
if db_node.name.lower() in
|
|
112
|
+
if db_node.name.lower() in SYSTEM_SCHEMAS:
|
|
113
113
|
continue
|
|
114
114
|
# exp.table_name(), not str(t): str() includes " AS alias" for
|
|
115
115
|
# an aliased reference (e.g. "FROM editing.visit v"), which
|
|
@@ -119,8 +119,21 @@ def _get_sql_deps(sql: str, dialect: Dialect = POSTGRES) -> set[str]:
|
|
|
119
119
|
return deps
|
|
120
120
|
|
|
121
121
|
|
|
122
|
-
def _iter_sql_files(
|
|
123
|
-
|
|
122
|
+
def _iter_sql_files(
|
|
123
|
+
database_dir: Path,
|
|
124
|
+
dialect: Dialect = POSTGRES,
|
|
125
|
+
*,
|
|
126
|
+
areas: frozenset[str] | None = None,
|
|
127
|
+
exclude_areas: frozenset[str] | None = None,
|
|
128
|
+
schemas: frozenset[str] | None = None,
|
|
129
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
130
|
+
):
|
|
131
|
+
"""Yield (Path, sql_content) pairs in dependency-safe execution order.
|
|
132
|
+
|
|
133
|
+
A file dropped by the area/schema filter is skipped entirely -- as if it
|
|
134
|
+
didn't exist -- so it never delivers a dependency another (in-scope)
|
|
135
|
+
file waits on; that's the same tradeoff `list_migration_files`/
|
|
136
|
+
`parse_directory` make for migrations/database code files."""
|
|
124
137
|
files: list[Path] = []
|
|
125
138
|
for root, dirs, dbfiles in os.walk(database_dir):
|
|
126
139
|
dirs[:] = [d for d in dirs if d not in IGNORED_DIR_NAMES]
|
|
@@ -138,6 +151,12 @@ def _iter_sql_files(database_dir: Path, dialect: Dialect = POSTGRES):
|
|
|
138
151
|
|
|
139
152
|
for file in sorted(files, key=lambda p: (_get_type_order(p), p.name)):
|
|
140
153
|
content = file.read_text(encoding="utf-8")
|
|
154
|
+
if (areas or exclude_areas) and not area_allowed(parse_areas(content), only=areas, exclude=exclude_areas):
|
|
155
|
+
continue
|
|
156
|
+
if (schemas or exclude_schemas) and not schema_allowed(
|
|
157
|
+
sql_schemas(content, dialect), only=schemas, exclude=exclude_schemas
|
|
158
|
+
):
|
|
159
|
+
continue
|
|
141
160
|
deps = _get_sql_deps(content, dialect)
|
|
142
161
|
if file.parent.name in _SCHEMA_QUALIFIED_TYPES:
|
|
143
162
|
schema = _strip_layer_prefix(file.parent.parent.name)
|
|
@@ -242,6 +261,10 @@ async def apply_schema(
|
|
|
242
261
|
force_reset: bool = False,
|
|
243
262
|
*,
|
|
244
263
|
dialect: Dialect = POSTGRES,
|
|
264
|
+
areas: frozenset[str] | None = None,
|
|
265
|
+
exclude_areas: frozenset[str] | None = None,
|
|
266
|
+
schemas: frozenset[str] | None = None,
|
|
267
|
+
exclude_schemas: frozenset[str] | None = None,
|
|
245
268
|
) -> None:
|
|
246
269
|
"""Apply every .sql file under database_dir (in dependency-safe order)
|
|
247
270
|
and seed any matching .test_data.json files. Safe to call repeatedly.
|
|
@@ -249,7 +272,12 @@ async def apply_schema(
|
|
|
249
272
|
`migrations/` subdirectories are never applied here — they're for
|
|
250
273
|
one-time manual application against real (already-provisioned)
|
|
251
274
|
databases, not for building a fresh schema. The base object files under
|
|
252
|
-
`database_dir` must reflect the current, final schema on their own.
|
|
275
|
+
`database_dir` must reflect the current, final schema on their own.
|
|
276
|
+
|
|
277
|
+
`areas`/`exclude_areas` and `schemas`/`exclude_schemas` restrict which
|
|
278
|
+
files get applied, the same as `pgdevkit migrate`/`compare` — handy for
|
|
279
|
+
standing up a test DB scoped to one area or schema instead of the whole
|
|
280
|
+
project."""
|
|
253
281
|
await con.set_autocommit(True)
|
|
254
282
|
for extension in extensions:
|
|
255
283
|
await con.execute(SQL("CREATE EXTENSION IF NOT EXISTS {e}").format(e=Identifier(extension)))
|
|
@@ -265,7 +293,14 @@ async def apply_schema(
|
|
|
265
293
|
await _insert_test_data(json_file, f"{schema_name}.{table_stem}", force_reset, con, complex_helper)
|
|
266
294
|
|
|
267
295
|
failures: list[tuple[Path, str]] = []
|
|
268
|
-
for file, sql in _iter_sql_files(
|
|
296
|
+
for file, sql in _iter_sql_files(
|
|
297
|
+
database_dir,
|
|
298
|
+
dialect,
|
|
299
|
+
areas=areas,
|
|
300
|
+
exclude_areas=exclude_areas,
|
|
301
|
+
schemas=schemas,
|
|
302
|
+
exclude_schemas=exclude_schemas,
|
|
303
|
+
):
|
|
269
304
|
try:
|
|
270
305
|
await _apply(file, sql)
|
|
271
306
|
except Exception as e: # noqa: BLE001
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: pgdevkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A helper for developing with Postgres
|
|
5
5
|
Requires-Python: >=3.14
|
|
6
6
|
Requires-Dist: docker>=7.1.0
|
|
@@ -79,7 +79,13 @@ which parses/introspects/diffs like any other column type; see
|
|
|
79
79
|
handled on the CRUD side (write-side serialization only, no auto-parsing on
|
|
80
80
|
read — `mssql-python` doesn't distinguish `json` columns from `nvarchar`).
|
|
81
81
|
|
|
82
|
-
### Area
|
|
82
|
+
### Area and schema filtering
|
|
83
|
+
|
|
84
|
+
Two independent, composable ways to narrow which files a command touches:
|
|
85
|
+
**area** is an explicit opt-in tag; **schema** is derived automatically from
|
|
86
|
+
each file's own SQL.
|
|
87
|
+
|
|
88
|
+
#### Area tagging
|
|
83
89
|
|
|
84
90
|
Any migration file or `database/` code file can declare one or more areas by
|
|
85
91
|
starting with a `-- area:` comment:
|
|
@@ -97,43 +103,69 @@ first real statement); a `-- area:` comment later in the file doesn't count.
|
|
|
97
103
|
A file with no directive is untagged, and untagged files are treated as
|
|
98
104
|
shared/common.
|
|
99
105
|
|
|
100
|
-
|
|
106
|
+
#### Schema filtering
|
|
107
|
+
|
|
108
|
+
No tag needed — schema membership is parsed straight out of the SQL itself:
|
|
109
|
+
every schema-qualified (or default-schema, when unqualified) table/view/
|
|
110
|
+
function/index reference across every statement in the file, DDL or DML
|
|
111
|
+
alike, plus any `CREATE SCHEMA name`. A file whose schema(s) can't be
|
|
112
|
+
determined (unparseable content, or no table/schema reference in it at all)
|
|
113
|
+
is treated the same as an untagged file — always kept.
|
|
114
|
+
|
|
115
|
+
#### Options
|
|
116
|
+
|
|
117
|
+
`pgdb compare`, `pgdb migrate check`, `pgdb migrate apply`, `pgdb testdb up`,
|
|
118
|
+
and `pgdb testdb reset` all accept:
|
|
101
119
|
|
|
102
120
|
- `--area NAME` (repeatable) — restrict to files declaring one of the given
|
|
103
121
|
areas, **plus every untagged file** (untagged files always stay in scope).
|
|
104
122
|
- `--exclude-area NAME` (repeatable) — drop files declaring one of the given
|
|
105
123
|
areas; untagged files are never dropped by this.
|
|
124
|
+
- `--schema NAME` (repeatable) — restrict to files referencing one of the
|
|
125
|
+
given schemas, **plus every file with no detectable schema reference**.
|
|
126
|
+
- `--exclude-schema NAME` (repeatable) — drop files referencing one of the
|
|
127
|
+
given schemas; files with no detectable reference are never dropped.
|
|
106
128
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
129
|
+
All four can be combined — a file must pass every filter it's subject to (an
|
|
130
|
+
area match doesn't excuse a schema mismatch, and vice versa), and a file
|
|
131
|
+
matching both an included and an excluded value on the same axis is
|
|
132
|
+
excluded. Passing none of them applies no filtering (the default, unchanged
|
|
133
|
+
behavior).
|
|
110
134
|
|
|
111
135
|
```bash
|
|
112
136
|
pgdb migrate apply path/to/database/_migration_scripts --url ... --area billing
|
|
113
137
|
pgdb compare path/to/database/ --url ... --exclude-area reporting
|
|
138
|
+
pgdb migrate check path/to/database/_migration_scripts --url ... --schema billing --exclude-schema reporting
|
|
139
|
+
pgdb testdb up --schema billing
|
|
114
140
|
```
|
|
115
141
|
|
|
116
142
|
`compare`'s default report (no `--report-extra-db`) only checks that the
|
|
117
143
|
filtered scripts exist correctly in the DB, so it composes safely with area
|
|
118
|
-
filtering. Passing `--report-extra-db` together with
|
|
119
|
-
reports every DB object outside the filtered area(s)
|
|
120
|
-
scripts" — since the live database has no concept
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
144
|
+
and schema filtering. Passing `--report-extra-db` together with either kind
|
|
145
|
+
of filter also reports every DB object outside the filtered area(s)/
|
|
146
|
+
schema(s) as "missing in scripts" — since the live database has no concept
|
|
147
|
+
of areas, and isn't itself filtered by `--schema` either — only the scripts
|
|
148
|
+
side is filtered — so treat that combination's "missing in scripts" results
|
|
149
|
+
with that in mind (the CLI prints a warning when you combine them).
|
|
150
|
+
|
|
151
|
+
`pgdb fetch-missing` deliberately has **no** `--area`/`--exclude-area` (or
|
|
152
|
+
`--schema`/`--exclude-schema`): it diffs the full database against scripts
|
|
153
|
+
to find genuinely untracked objects, so narrowing the scripts side would
|
|
154
|
+
make every object tracked under a different area/schema look "missing" too
|
|
155
|
+
— and `--write` would then reconstruct a duplicate file for something that
|
|
156
|
+
already exists.
|
|
157
|
+
|
|
158
|
+
`pgdevkit.areas` exposes the tag-filtering logic for scripting:
|
|
131
159
|
`parse_areas`/`file_areas` read a file's declared areas, and
|
|
132
160
|
`area_allowed`/`filter_by_area` apply the `only`/`exclude` semantics above.
|
|
133
|
-
`pgdevkit.
|
|
134
|
-
`
|
|
135
|
-
|
|
136
|
-
|
|
161
|
+
`pgdevkit.schemas` exposes the equivalent for schema filtering:
|
|
162
|
+
`sql_schemas`/`file_schemas` detect a file's referenced schemas, and
|
|
163
|
+
`schema_allowed`/`filter_by_schema` apply the same `only`/`exclude`
|
|
164
|
+
semantics. `pgdevkit.migrate.list_migration_files`/`pending_migrations` and
|
|
165
|
+
`pgdevkit.parser.parse_directory` take both pairs of keyword arguments
|
|
166
|
+
(`areas`/`exclude_areas` and `schemas`/`exclude_schemas`);
|
|
167
|
+
`pgdevkit.fetch_missing.find_missing_objects` takes neither, for the reason
|
|
168
|
+
above.
|
|
137
169
|
|
|
138
170
|
## `pgdb testdb`
|
|
139
171
|
|
|
@@ -165,6 +197,13 @@ def ensure_test_postgres():
|
|
|
165
197
|
|
|
166
198
|
CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`.
|
|
167
199
|
|
|
200
|
+
`up`/`reset` accept `--area`/`--exclude-area` and `--schema`/`--exclude-schema`
|
|
201
|
+
(see "Area and schema filtering" above) to scope which `database/` files get
|
|
202
|
+
applied — e.g. `pgdb testdb up --schema billing` for a test DB with only the
|
|
203
|
+
`billing` schema's tables/views/functions, without waiting on the rest of the
|
|
204
|
+
project's schema to apply. `ensure_testdb`/`reset_testdb` take the same
|
|
205
|
+
keyword arguments when called from Python (e.g. from a pytest fixture).
|
|
206
|
+
|
|
168
207
|
Container connection defaults (`localhost:54322`, `postgres`/`testpwd`) can
|
|
169
208
|
be overridden with `PGDEVKIT_TESTDB_HOST`, `PGDEVKIT_TESTDB_PORT`,
|
|
170
209
|
`PGDEVKIT_TESTDB_USER`, `PGDEVKIT_TESTDB_PASSWORD`. Before touching the
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
pgdevkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
2
|
pgdevkit/areas.py,sha256=iEmOObYxhZyDFl2iQVf1OQJN4JtZ-H3EL-qMfxYZdSQ,2203
|
|
3
|
-
pgdevkit/cli.py,sha256=
|
|
3
|
+
pgdevkit/cli.py,sha256=vVUg4Feln7Mc0DbZIFGPNkOVRsl2AL3ZOXM30hRoWLc,21179
|
|
4
4
|
pgdevkit/connection.py,sha256=x49aXKsJfyOsDTlqURs9idQUWaD-RO-XdtAqsG3guBI,4000
|
|
5
|
-
pgdevkit/dialect.py,sha256=
|
|
5
|
+
pgdevkit/dialect.py,sha256=gZwn4UNZ8PYBPC4CnYnfA95C7ppv7sWmpZSF8pjSPVw,3659
|
|
6
6
|
pgdevkit/diff.py,sha256=Y2TbjwpFHYik0wZDhoI--9EAlIfp4AUg8BUubkOXyUg,9295
|
|
7
7
|
pgdevkit/fetch_missing.py,sha256=vb8hXWBKMWgqkxHso5SuhX9l6rxNXfaybAEsajPLYow,8495
|
|
8
8
|
pgdevkit/introspect.py,sha256=QYIlRuZk4lBHGxaW7zb_kFRIuNfi3AxH0Q2F_GKzlVI,7870
|
|
9
9
|
pgdevkit/lakebase.py,sha256=0PABOk4p-2klLTY8QAUibQf9QeyNHKywUWEWGdkUY8k,3165
|
|
10
|
-
pgdevkit/migrate.py,sha256=
|
|
10
|
+
pgdevkit/migrate.py,sha256=UvR5hrkkM7yO65eUL3ZPwJ5405e6nJWMJDyIwu78ugA,12945
|
|
11
11
|
pgdevkit/models.py,sha256=dAZ8f7pgDsxoLVrDWsM4h0qwJ0pOzXDdWMGsnjrW80E,2103
|
|
12
12
|
pgdevkit/mssql_introspect.py,sha256=Fhv1N4zHYEDZ0PEy0OIpawUmoIFeoi-Lm97pmEyDmYY,12031
|
|
13
|
-
pgdevkit/parser.py,sha256=
|
|
13
|
+
pgdevkit/parser.py,sha256=FmXL6-eCFzP_0FNyTrg4wF3aPzPeq7re2i0xvrSJIcs,16885
|
|
14
|
+
pgdevkit/schemas.py,sha256=zZsg3m9FtI7IUG8e8HfRruCh6-O6bdpbA7Kjnk3AYJg,5053
|
|
15
|
+
pgdevkit/sql_text.py,sha256=ednfdfxGBTWYqQ5Q3l3Zv9vrjjFmLiRlGKyq3tcozco,1744
|
|
14
16
|
pgdevkit/backends/__init__.py,sha256=ys4VBglMImRtnudJBlTPVBNevDMRippuwAJOz0yRKiY,610
|
|
15
17
|
pgdevkit/backends/base.py,sha256=0i0KLCYVWlpVFnUoq2t0LCRE_abyWPPIhw1bMxmW9KQ,1048
|
|
16
18
|
pgdevkit/backends/mssql.py,sha256=pocBdoQ14ZnYLutXEOStVmJvmpclXGs3gaxHN0uJCMk,1068
|
|
@@ -25,22 +27,22 @@ pgdevkit/db/mssql_crud.py,sha256=yN4izZ0fsFwAs14008JECHH4dqjNbdHUHGEG8kidyNs,113
|
|
|
25
27
|
pgdevkit/db/mssql_sql.py,sha256=DO3UMsTFN9pg0tNdaPGb0wf6lWlHGcqwqzkrGVhJF2g,2292
|
|
26
28
|
pgdevkit/testdb/__init__.py,sha256=LpdC2XMBxqn9JrGmXhIho3sK8O1aNo_aD7leKyBr-Lo,204
|
|
27
29
|
pgdevkit/testdb/_docker.py,sha256=kj_0xrGO_mxHlK83ilsB-yE0ELX1kpH22ruYrS-N3iU,1467
|
|
28
|
-
pgdevkit/testdb/api.py,sha256=
|
|
30
|
+
pgdevkit/testdb/api.py,sha256=QE0UmtUNI1dc1MG4MdyatSbswHj4mAc-aWSjtL35x0w,7407
|
|
29
31
|
pgdevkit/testdb/config.py,sha256=vjcDr3jH3-5VakNAEMqQu_ZHPXiYIvM1_BSETSJW5QU,2164
|
|
30
32
|
pgdevkit/testdb/constants.py,sha256=xi8uIoEgN9LrRkr41j3mmWB9PqqU0r8aCUb2JJXBAs0,1165
|
|
31
33
|
pgdevkit/testdb/container.py,sha256=kELTb6qaECAO9a5Bwvx1btv6i5QKRFAa9PX6auWymNo,2807
|
|
32
34
|
pgdevkit/testdb/naming.py,sha256=Kyk5qEfxUkbggUG5bD7U7CJ_kAET332VPQNB-O5kvTY,1318
|
|
33
35
|
pgdevkit/testdb/query.py,sha256=iYpy_EoYIq5USfwr6pNktmDjcGi_yQx0vHvjHZNGRr0,3854
|
|
34
|
-
pgdevkit/testdb/schema.py,sha256=
|
|
36
|
+
pgdevkit/testdb/schema.py,sha256=fuW5juQWoMW9J2mzPQxLZDlF4ZExEPCVhbQ5DCUMhNg,13349
|
|
35
37
|
pgdevkit/testdb/mssql/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
|
|
36
|
-
pgdevkit/testdb/mssql/api.py,sha256=
|
|
38
|
+
pgdevkit/testdb/mssql/api.py,sha256=Q5Gt8t4LbDtDkMXXSQdd2_Ud7nhelF2UpD8KoUjI3qg,8203
|
|
37
39
|
pgdevkit/testdb/mssql/constants.py,sha256=7n_NcYuhabbtb18CqG_H_rRbI7Wb_qu_l4nKk3281vI,2847
|
|
38
40
|
pgdevkit/testdb/mssql/container.py,sha256=_rG0AMDXrNwok_E60Xai0DZdRiZ6Ba1j3GaLUnSAueg,3364
|
|
39
41
|
pgdevkit/docs/database-layout.md,sha256=4GBwv4PA2cKHl2BxwLd-awLOe9he9ie4X6CpQWjIEyI,6820
|
|
40
42
|
pgdevkit/skills/pgdevkit/SKILL.md,sha256=6uxe6BQwOLexGiBpf00VTFTHOiWJo5k2Qqzu172rb4I,12156
|
|
41
43
|
pgdevkit/skills/pgdevkit/references/dynamic-sql.md,sha256=aTR1sx81moiTwaMaiOecaqk-9rT8y5Zxz7W1BoC3vK0,2774
|
|
42
44
|
pgdevkit/skills/pgdevkit/references/temporal-tables.md,sha256=LwZnItvQWI7bB0oRHxIfUc0ynnq7N0V5Mnv-QHvFN3Q,1364
|
|
43
|
-
pgdevkit-0.
|
|
44
|
-
pgdevkit-0.
|
|
45
|
-
pgdevkit-0.
|
|
46
|
-
pgdevkit-0.
|
|
45
|
+
pgdevkit-0.5.0.dist-info/METADATA,sha256=1RXl13YLU2NmB8E4YXpLGc3pNr4EUp1C4INsK0pLJ-s,17629
|
|
46
|
+
pgdevkit-0.5.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
47
|
+
pgdevkit-0.5.0.dist-info/entry_points.txt,sha256=FK2P55lf0WaGlOZJIINOGgCt8WB7ep0OnPsgnrp80bg,42
|
|
48
|
+
pgdevkit-0.5.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|