sqlpush 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sqlpush-0.1.0 → sqlpush-0.3.0}/PKG-INFO +1 -2
- {sqlpush-0.1.0 → sqlpush-0.3.0}/README.md +0 -1
- {sqlpush-0.1.0 → sqlpush-0.3.0}/pyproject.toml +1 -1
- {sqlpush-0.1.0 → sqlpush-0.3.0}/pyproject.toml.orig +1 -1
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/core/classify.py +7 -3
- sqlpush-0.3.0/src/sqlpush/core/diff.py +409 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/directives/timescale.py +15 -5
- sqlpush-0.1.0/src/sqlpush/core/diff.py +0 -205
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/__init__.py +0 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/annotations.py +0 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/api.py +0 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/apply/__init__.py +0 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/apply/executor.py +0 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/cli.py +0 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/core/__init__.py +0 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/core/render.py +0 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/directives/__init__.py +0 -0
- {sqlpush-0.1.0 → sqlpush-0.3.0}/src/sqlpush/types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sqlpush
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models
|
|
5
5
|
Keywords: sqlalchemy,alembic,postgresql,timescaledb,prisma,schema,migrations,database,drift,cli
|
|
6
6
|
Author: Juan Miguel Contreras
|
|
@@ -29,7 +29,6 @@ Description-Content-Type: text/markdown
|
|
|
29
29
|
|
|
30
30
|
# sqlpush
|
|
31
31
|
|
|
32
|
-
[](https://github.com/juanmicl/sqlpush/actions/workflows/ci.yml)
|
|
33
32
|
[](https://pypi.org/project/sqlpush/)
|
|
34
33
|
[](https://pypi.org/project/sqlpush/)
|
|
35
34
|
[](LICENSE)
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# sqlpush
|
|
2
2
|
|
|
3
|
-
[](https://github.com/juanmicl/sqlpush/actions/workflows/ci.yml)
|
|
4
3
|
[](https://pypi.org/project/sqlpush/)
|
|
5
4
|
[](https://pypi.org/project/sqlpush/)
|
|
6
5
|
[](LICENSE)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sqlpush"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sqlpush"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
@@ -8,9 +8,13 @@ DESTRUCTIVE = frozenset({"drop_column", "drop_table", "drop_index", "drop_constr
|
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def classify(op_type: str) -> RiskClass:
|
|
11
|
-
"""
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
"""add_index renders standalone: on alembic 1.19.1 even plain
|
|
12
|
+
declared indexes of NEW tables arrive standalone
|
|
13
|
+
(CreateTableOp.from_table captures columns+constraints, not
|
|
14
|
+
indexes; only instrumentation-embedded ones ride inside the
|
|
15
|
+
add_table render, and the diff dedups those away). What survives
|
|
16
|
+
runs CREATE INDEX alone — a SHARE lock that blocks writes, hence
|
|
17
|
+
risky."""
|
|
14
18
|
if op_type in DESTRUCTIVE:
|
|
15
19
|
return RiskClass.DESTRUCTIVE
|
|
16
20
|
if op_type in SAFE:
|
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
"""The ONLY module in sqlpush that imports alembic."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import fnmatch
|
|
6
|
+
import io
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
|
|
9
|
+
from alembic.autogenerate import produce_migrations
|
|
10
|
+
from alembic.migration import MigrationContext
|
|
11
|
+
from alembic.operations import Operations
|
|
12
|
+
from alembic.operations.ops import AlterColumnOp, OpContainer
|
|
13
|
+
from sqlalchemy import MetaData, text
|
|
14
|
+
from sqlalchemy.engine import Engine
|
|
15
|
+
|
|
16
|
+
from sqlpush.core.classify import classify
|
|
17
|
+
from sqlpush.types import Plan, PlannedOperation
|
|
18
|
+
|
|
19
|
+
_SYSTEM_SCHEMAS = ("_timescaledb%", "information_schema", "pg_%")
|
|
20
|
+
_SYSTEM_TABLES = ("alembic_version", "spatial_ref_sys")
|
|
21
|
+
|
|
22
|
+
# Leaf op class -> sqlpush op type. Class names follow alembic 1.19.1
|
|
23
|
+
# autogen output as observed in docs/notes/alembic-notes.md and probes:
|
|
24
|
+
# table create is CreateTableOp (not "AddTableOp"), column modify is
|
|
25
|
+
# AlterColumnOp (not "ModifyColumnOp"), index create is CreateIndexOp,
|
|
26
|
+
# and constraint creates are the concrete Create*ConstraintOp subclasses
|
|
27
|
+
# The AddConstraintOp base class is never emitted as a leaf op.
|
|
28
|
+
# AlterColumnOp is absent ON PURPOSE: its label is derived per-op (see
|
|
29
|
+
# _alter_column_label) because default-only, nullable-only and type
|
|
30
|
+
# changes all arrive as the same class.
|
|
31
|
+
_OP_TYPE = {
|
|
32
|
+
"CreateTableOp": "add_table",
|
|
33
|
+
"DropTableOp": "drop_table",
|
|
34
|
+
"AddColumnOp": "add_column",
|
|
35
|
+
"DropColumnOp": "drop_column",
|
|
36
|
+
"CreateIndexOp": "add_index",
|
|
37
|
+
"DropIndexOp": "drop_index",
|
|
38
|
+
"CreateUniqueConstraintOp": "add_constraint",
|
|
39
|
+
"CreateForeignKeyOp": "add_constraint",
|
|
40
|
+
"CreatePrimaryKeyOp": "add_constraint",
|
|
41
|
+
"CreateCheckConstraintOp": "add_constraint",
|
|
42
|
+
"DropConstraintOp": "drop_constraint",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _alter_column_label(op: AlterColumnOp) -> str:
|
|
47
|
+
"""Precise op type for AlterColumnOp (alembic-notes Pattern C).
|
|
48
|
+
|
|
49
|
+
Sentinel semantics: ``False`` and ``None`` both mean "leave
|
|
50
|
+
unchanged"; any other value is the new setting. Default-only and
|
|
51
|
+
nullable-only drift arrive as AlterColumnOp just like type changes,
|
|
52
|
+
so disambiguate on the attributes.
|
|
53
|
+
"""
|
|
54
|
+
if op.modify_server_default not in (False, None):
|
|
55
|
+
return "modify_default"
|
|
56
|
+
if op.modify_nullable not in (False, None):
|
|
57
|
+
return "modify_nullable"
|
|
58
|
+
return "modify_type"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _is_system_schema(schema: str) -> bool:
|
|
62
|
+
return any(fnmatch.fnmatch(schema, pat) for pat in _SYSTEM_SCHEMAS)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _extension_owned_schemas(conn) -> frozenset[str]:
|
|
66
|
+
"""Namespaces owned by an installed extension (live server truth).
|
|
67
|
+
|
|
68
|
+
Extensions that install into their own namespace (e.g.
|
|
69
|
+
postgis_topology -> ``topology``) manage it: its objects are
|
|
70
|
+
extension state, not user metadata. NB: extensions relocated into
|
|
71
|
+
the default schema (timescaledb, postgis, pg_trgm all live in
|
|
72
|
+
``public``) do NOT own it — the default schema stays user scope.
|
|
73
|
+
"""
|
|
74
|
+
rows = conn.execute(
|
|
75
|
+
text("SELECT n.nspname FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace")
|
|
76
|
+
).scalars()
|
|
77
|
+
return frozenset(rows)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _search_path_schemas(conn, default_schema: str, extension_schemas: frozenset[str]) -> list[str]:
|
|
81
|
+
"""Derive the diff scope from the live search_path.
|
|
82
|
+
|
|
83
|
+
Same parsing as before (``"$user"`` dropped, whitespace stripped)
|
|
84
|
+
with one exclusion: extension-owned non-default schemas never enter
|
|
85
|
+
the scope. They usually appear on the search_path because their
|
|
86
|
+
extension ``ALTER DATABASE``d it there at install time
|
|
87
|
+
(postgis_topology does exactly that), not because the user scoped
|
|
88
|
+
them in. A schema the caller passes explicitly via ``schemas=`` is
|
|
89
|
+
never filtered here — explicit user intent wins.
|
|
90
|
+
"""
|
|
91
|
+
# a live PG session always reports a search_path
|
|
92
|
+
search_path = conn.execute(text("SHOW search_path")).scalar()
|
|
93
|
+
assert search_path is not None
|
|
94
|
+
candidates = [s.strip() for s in search_path.split(",") if s.strip() and s.strip() != '"$user"']
|
|
95
|
+
return [s for s in candidates if s == default_schema or s not in extension_schemas]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _timescale_auto_indexes(conn) -> dict[str, set[str]]:
|
|
99
|
+
"""``{schema.index_name}`` -> set of ``{schema.hypertable}`` owners.
|
|
100
|
+
|
|
101
|
+
TimescaleDB's implicit per-hypertable time-column index
|
|
102
|
+
(``<hypertable>_<dimension>_idx``). Set-valued: distinct hypertables
|
|
103
|
+
can produce the SAME index name (table ``a`` dim ``b_c`` vs table
|
|
104
|
+
``a_b`` dim ``c`` both yield ``a_b_c_idx``) — a str value would
|
|
105
|
+
last-win and leak the other as false drift. Qualified keys carry
|
|
106
|
+
cross-schema hypertables. Empty when the extension is absent.
|
|
107
|
+
"""
|
|
108
|
+
installed = conn.execute(
|
|
109
|
+
text("SELECT 1 FROM pg_extension WHERE extname = 'timescaledb'")
|
|
110
|
+
).first()
|
|
111
|
+
if installed is None:
|
|
112
|
+
return {}
|
|
113
|
+
rows = conn.execute(
|
|
114
|
+
text(
|
|
115
|
+
"SELECT hypertable_schema, hypertable_name, primary_dimension "
|
|
116
|
+
"FROM timescaledb_information.hypertables"
|
|
117
|
+
)
|
|
118
|
+
).all()
|
|
119
|
+
out: dict[str, set[str]] = {}
|
|
120
|
+
for schema, hypertable, dimension in rows:
|
|
121
|
+
out.setdefault(f"{schema}.{hypertable}_{dimension}_idx", set()).add(
|
|
122
|
+
f"{schema}.{hypertable}"
|
|
123
|
+
)
|
|
124
|
+
return out
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _spatial_auto_indexes(conn) -> frozenset[str]:
|
|
128
|
+
"""Qualified names of geoalchemy2-style implicit spatial indexes.
|
|
129
|
+
|
|
130
|
+
Single-column indexes named ``<table>_<col>_idx`` on a geometry or
|
|
131
|
+
geography column — what geoalchemy2 (<0.18 or ``spatial_index=True``)
|
|
132
|
+
creates at table-create time and no model ever declares. Catalog-driven
|
|
133
|
+
(pg_type), so detection never depends on geoalchemy2 being importable
|
|
134
|
+
in this process.
|
|
135
|
+
"""
|
|
136
|
+
rows = conn.execute(
|
|
137
|
+
text(
|
|
138
|
+
"SELECT n.nspname, i2.relname "
|
|
139
|
+
"FROM pg_index i "
|
|
140
|
+
"JOIN pg_class t ON t.oid = i.indrelid "
|
|
141
|
+
"JOIN pg_namespace n ON n.oid = t.relnamespace "
|
|
142
|
+
"JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(i.indkey) "
|
|
143
|
+
"JOIN pg_class i2 ON i2.oid = i.indexrelid "
|
|
144
|
+
"JOIN pg_type ty ON ty.oid = a.atttypid "
|
|
145
|
+
"WHERE i.indnatts = 1 "
|
|
146
|
+
"AND ty.typname IN ('geometry', 'geography') "
|
|
147
|
+
"AND i2.relname = format('%s_%s_idx', t.relname, a.attname)"
|
|
148
|
+
)
|
|
149
|
+
).all()
|
|
150
|
+
return frozenset(f"{nsp}.{idx}" for nsp, idx in rows)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _restrict_search_path(conn, default_schema: str, schemas: frozenset[str]) -> str | None:
|
|
154
|
+
"""Confine the default-schema reflection pass to the default schema.
|
|
155
|
+
|
|
156
|
+
Alembic's unqualified (None-schema) reflection pass resolves table
|
|
157
|
+
names via ``pg_table_is_visible``, i.e. the SESSION search_path —
|
|
158
|
+
which ambient database-level settings can stretch beyond the
|
|
159
|
+
declared scope (postgis_topology ``ALTER DATABASE``s its own
|
|
160
|
+
namespace onto the search_path; its tables then surface both
|
|
161
|
+
unqualified and schema-qualified — one object, two drop ops).
|
|
162
|
+
``pg_table_is_visible`` cannot deliver "reflection sees exactly
|
|
163
|
+
what the scope says" for a multi-schema scope: pinning the full
|
|
164
|
+
scope list would let the None-pass resolve NON-default tables as
|
|
165
|
+
unqualified default-schema tables — false destructive drops plus
|
|
166
|
+
duplicate unqualified drops in mixed scopes. So pin to the default
|
|
167
|
+
schema alone, and only when it is a scope member; otherwise the
|
|
168
|
+
None-pass is filtered out by ``_make_include_name`` before
|
|
169
|
+
reflection runs, so there is nothing visibility-based to confine
|
|
170
|
+
and no pin (nor restore) happens. Schema-qualified reflection is
|
|
171
|
+
unaffected either way. Returns the original setting for
|
|
172
|
+
:func:`_restore_search_path`, or ``None`` when nothing was pinned.
|
|
173
|
+
"""
|
|
174
|
+
if default_schema not in schemas:
|
|
175
|
+
return None
|
|
176
|
+
original = conn.execute(text("SHOW search_path")).scalar()
|
|
177
|
+
assert original is not None
|
|
178
|
+
preparer = conn.dialect.identifier_preparer
|
|
179
|
+
conn.exec_driver_sql(f"SET search_path TO {preparer.quote(default_schema)}")
|
|
180
|
+
return original
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _restore_search_path(conn, original: str | None) -> None:
|
|
184
|
+
# original None => nothing was pinned, nothing to restore.
|
|
185
|
+
if original is None:
|
|
186
|
+
return
|
|
187
|
+
# SET is session-level and survives ROLLBACK: a pooled connection
|
|
188
|
+
# must never hand the restricted path to its next borrower. The
|
|
189
|
+
# value is SHOW's own output — already a valid identifier list —
|
|
190
|
+
# so round-tripping it verbatim is safe.
|
|
191
|
+
try:
|
|
192
|
+
conn.exec_driver_sql(f"SET search_path TO {original}")
|
|
193
|
+
except Exception: # noqa: BLE001
|
|
194
|
+
# A dead connection cannot be restored, and a restore failure must
|
|
195
|
+
# NEVER mask the root error from produce_migrations. A LIVE pooled
|
|
196
|
+
# connection whose restore failed must be discarded, not returned
|
|
197
|
+
# to the pool still carrying the restricted path (SET survives
|
|
198
|
+
# ROLLBACK) — hence invalidate(), which forces the pool to drop
|
|
199
|
+
# it transparently. Server-side session teardown drops the
|
|
200
|
+
# restricted path anyway.
|
|
201
|
+
conn.invalidate()
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _make_include_name(schemas: frozenset[str], default_schema: str):
|
|
205
|
+
def include_name(name, type_, parent_names):
|
|
206
|
+
# Prune schemas (and everything inside them) BEFORE reflection:
|
|
207
|
+
# system catalogs (timescale et al.) are then never reflected at
|
|
208
|
+
# all. Schema filtering routes through include_name: alembic
|
|
209
|
+
# 1.19 never calls include_object with type_ == "schema", and
|
|
210
|
+
# its return value must be a real bool (falsy = exclude).
|
|
211
|
+
# NB: alembic substitutes None for the default schema name in
|
|
212
|
+
# the schema pass (compare/schema.py), so map None back.
|
|
213
|
+
if type_ == "schema":
|
|
214
|
+
schema = default_schema if name is None else name
|
|
215
|
+
else:
|
|
216
|
+
schema = parent_names.get("schema_name") or default_schema
|
|
217
|
+
return schema in schemas and not _is_system_schema(schema)
|
|
218
|
+
|
|
219
|
+
return include_name
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _make_include(
|
|
223
|
+
schemas: frozenset[str],
|
|
224
|
+
default_schema: str,
|
|
225
|
+
ts_auto_indexes: dict[str, set[str]],
|
|
226
|
+
spatial_auto_indexes: frozenset[str],
|
|
227
|
+
):
|
|
228
|
+
def include_object(obj, name, type_, reflected, compare_to):
|
|
229
|
+
# Bound the diff to the target schemas on every branch: derive
|
|
230
|
+
# the object's effective schema and require membership, for
|
|
231
|
+
# reflected-only, metadata-only and both-present objects alike.
|
|
232
|
+
# Column/Index/Constraint objects have no `.schema` of their own
|
|
233
|
+
# (SQLAlchemy 2.0.52): they resolve it through the parent table.
|
|
234
|
+
schema = (
|
|
235
|
+
getattr(obj, "schema", None)
|
|
236
|
+
or getattr(getattr(obj, "table", None), "schema", None)
|
|
237
|
+
or default_schema
|
|
238
|
+
)
|
|
239
|
+
if schema not in schemas or _is_system_schema(schema):
|
|
240
|
+
return False
|
|
241
|
+
if reflected and compare_to is None:
|
|
242
|
+
# DB-only object: skip system tables. NB: alembic also
|
|
243
|
+
# auto-excludes its own version table from autogen; the
|
|
244
|
+
# name check here is defense in depth.
|
|
245
|
+
if name in _SYSTEM_TABLES:
|
|
246
|
+
return False
|
|
247
|
+
# Skip TimescaleDB's implicit time-column index: extension
|
|
248
|
+
# state no model declares. Qualified name AND owner set must
|
|
249
|
+
# both match, so a same-named index on another table stays
|
|
250
|
+
# real drift, as does one the metadata actually declares
|
|
251
|
+
# (that arrives in the both-present branch above).
|
|
252
|
+
parent = getattr(obj, "table", None)
|
|
253
|
+
parent_table = getattr(parent, "name", None)
|
|
254
|
+
parent_schema = getattr(parent, "schema", None) or default_schema
|
|
255
|
+
qualified_index = f"{parent_schema}.{name}"
|
|
256
|
+
qualified_parent = f"{parent_schema}.{parent_table}"
|
|
257
|
+
if type_ == "index" and qualified_index in spatial_auto_indexes:
|
|
258
|
+
return False
|
|
259
|
+
owners = ts_auto_indexes.get(qualified_index, frozenset())
|
|
260
|
+
return not (type_ == "index" and qualified_parent in owners)
|
|
261
|
+
return True
|
|
262
|
+
|
|
263
|
+
return include_object
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _flatten(ops):
|
|
267
|
+
# D1 (alembic-notes): ops targeting existing tables arrive wrapped in
|
|
268
|
+
# ModifyTableOps containers; Operations.invoke crashes on containers,
|
|
269
|
+
# so recurse into anything that is an OpContainer before invoking.
|
|
270
|
+
for op in ops:
|
|
271
|
+
if isinstance(op, OpContainer):
|
|
272
|
+
yield from _flatten(op.ops)
|
|
273
|
+
else:
|
|
274
|
+
yield op
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _dedup_embedded_indexes(ops: list[PlannedOperation]) -> list[PlannedOperation]:
|
|
278
|
+
"""Drop standalone ``add_index`` ops already embedded in an ``add_table`` render.
|
|
279
|
+
|
|
280
|
+
On alembic 1.19.1 ``CreateTableOp.from_table`` captures columns and
|
|
281
|
+
constraints only, NOT indexes: a plain declared ``Index(...)`` on a
|
|
282
|
+
new table never reaches the create render — it arrives
|
|
283
|
+
standalone-only and is untouched here. The embedding this dedup
|
|
284
|
+
targets happens when the table carries instrumentation-appended
|
|
285
|
+
indexes (geoalchemy2-style listeners attaching at Table
|
|
286
|
+
construction): ``to_table()`` reconstruction re-fires the
|
|
287
|
+
attachment, the rebuilt table carries the index again, the offline
|
|
288
|
+
create render embeds it, and autogen ALSO emits the standalone
|
|
289
|
+
CreateIndexOp — executing both is a guaranteed duplicate-object
|
|
290
|
+
failure (push fire-test F1/F2: the renders are byte-identical and
|
|
291
|
+
the second execution collides with 42P07). Suppression side: the
|
|
292
|
+
standalone op is the redundant one — its statement already runs
|
|
293
|
+
inside the add_table op, whose render embeds it verbatim; the
|
|
294
|
+
embedded copy has no other carrier op. Exact-statement containment
|
|
295
|
+
is safe: both renders come from the same renderer over the same
|
|
296
|
+
Index objects, so an embedded index matches its standalone op
|
|
297
|
+
byte-for-byte while a different index's statement cannot be a
|
|
298
|
+
substring of the create-table render (statement text runs to its
|
|
299
|
+
own terminator). Known limitation: the keys are bare table names,
|
|
300
|
+
so two NEW tables sharing a bare name across schemas under-dedup
|
|
301
|
+
(last-wins in the dict) — containment is SQL-qualified either way,
|
|
302
|
+
so no wrong suppression is possible.
|
|
303
|
+
"""
|
|
304
|
+
table_renders = {op.table: op.sql for op in ops if op.type == "add_table"}
|
|
305
|
+
return [
|
|
306
|
+
op
|
|
307
|
+
for op in ops
|
|
308
|
+
if not (
|
|
309
|
+
op.type == "add_index"
|
|
310
|
+
and op.table in table_renders
|
|
311
|
+
and op.sql.strip() in table_renders[op.table]
|
|
312
|
+
)
|
|
313
|
+
]
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _render_op_sql(op, engine: Engine) -> str:
|
|
317
|
+
buf = io.StringIO()
|
|
318
|
+
offline = MigrationContext.configure(
|
|
319
|
+
dialect=engine.dialect, opts={"as_sql": True, "output_buffer": buf}
|
|
320
|
+
)
|
|
321
|
+
operations = Operations(offline)
|
|
322
|
+
operations.invoke(op)
|
|
323
|
+
# offline render terminates each op with ";\n\n"; normalize so each
|
|
324
|
+
# PlannedOperation.sql is a single clean statement
|
|
325
|
+
return buf.getvalue().strip().rstrip(";")
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
class DiffEngine:
|
|
329
|
+
def plan(
|
|
330
|
+
self,
|
|
331
|
+
metadata: MetaData,
|
|
332
|
+
engine: Engine,
|
|
333
|
+
*,
|
|
334
|
+
schemas: Sequence[str] | None = None,
|
|
335
|
+
exclude: Sequence[str] = (),
|
|
336
|
+
) -> Plan:
|
|
337
|
+
exclude = tuple(exclude)
|
|
338
|
+
# Typed `str | None` by SQLAlchemy; a dialect without a default
|
|
339
|
+
# schema is meaningless for this PostgreSQL-only tool; "public"
|
|
340
|
+
# matches every supported server config.
|
|
341
|
+
default_schema = engine.dialect.default_schema_name or "public"
|
|
342
|
+
|
|
343
|
+
# One connection for the whole plan: the scope derivation, the
|
|
344
|
+
# catalog probes and the reflection must all see the same
|
|
345
|
+
# session — the search_path pinning below would be meaningless
|
|
346
|
+
# on a second connection.
|
|
347
|
+
with engine.connect() as conn:
|
|
348
|
+
if schemas is None:
|
|
349
|
+
schemas = _search_path_schemas(conn, default_schema, _extension_owned_schemas(conn))
|
|
350
|
+
# New binding rather than a param reassignment: frozenset is
|
|
351
|
+
# a Set, not a Sequence, and the helpers below declare the
|
|
352
|
+
# exact type they take.
|
|
353
|
+
schema_set = frozenset(schemas)
|
|
354
|
+
ts_auto_indexes = _timescale_auto_indexes(conn)
|
|
355
|
+
spatial_auto_indexes = _spatial_auto_indexes(conn)
|
|
356
|
+
|
|
357
|
+
opts = {
|
|
358
|
+
"compare_type": True,
|
|
359
|
+
"compare_server_default": True,
|
|
360
|
+
"include_name": _make_include_name(schema_set, default_schema),
|
|
361
|
+
"include_object": _make_include(
|
|
362
|
+
schema_set, default_schema, ts_auto_indexes, spatial_auto_indexes
|
|
363
|
+
),
|
|
364
|
+
"include_schemas": True,
|
|
365
|
+
}
|
|
366
|
+
original_search_path = _restrict_search_path(conn, default_schema, schema_set)
|
|
367
|
+
try:
|
|
368
|
+
ctx = MigrationContext.configure(conn, opts=opts)
|
|
369
|
+
script = produce_migrations(ctx, metadata)
|
|
370
|
+
finally:
|
|
371
|
+
_restore_search_path(conn, original_search_path)
|
|
372
|
+
|
|
373
|
+
# produce_migrations always builds the upgrade bundle
|
|
374
|
+
assert script.upgrade_ops is not None
|
|
375
|
+
ops: list[PlannedOperation] = []
|
|
376
|
+
for op in _flatten(script.upgrade_ops.ops):
|
|
377
|
+
ops.extend(self._translate(op, engine, exclude))
|
|
378
|
+
ops = _dedup_embedded_indexes(ops)
|
|
379
|
+
return Plan(operations=tuple(ops))
|
|
380
|
+
|
|
381
|
+
def _translate(self, op, engine: Engine, exclude: tuple[str, ...]) -> list[PlannedOperation]:
|
|
382
|
+
op_type = _OP_TYPE.get(type(op).__name__, "raw_sql")
|
|
383
|
+
if isinstance(op, AlterColumnOp):
|
|
384
|
+
op_type = _alter_column_label(op)
|
|
385
|
+
sql_text = _render_op_sql(op, engine)
|
|
386
|
+
table = getattr(getattr(op, "table", None), "name", None) or getattr(op, "table_name", None)
|
|
387
|
+
# AddColumnOp carries a real Column in `.column`; DropColumnOp and
|
|
388
|
+
# AlterColumnOp only expose the plain string `column_name`
|
|
389
|
+
# (alembic-notes op reference).
|
|
390
|
+
column = getattr(getattr(op, "column", None), "name", None) or getattr(
|
|
391
|
+
op, "column_name", None
|
|
392
|
+
)
|
|
393
|
+
# table-level patterns reach column-level ops too; a qualified
|
|
394
|
+
# table.column pattern suppresses only that specific column op
|
|
395
|
+
if table and any(fnmatch.fnmatch(table, pat) for pat in exclude):
|
|
396
|
+
return []
|
|
397
|
+
if table and column:
|
|
398
|
+
full = f"{table}.{column}"
|
|
399
|
+
if any(fnmatch.fnmatch(full, pat) for pat in exclude):
|
|
400
|
+
return []
|
|
401
|
+
return [
|
|
402
|
+
PlannedOperation(
|
|
403
|
+
type=op_type,
|
|
404
|
+
risk=classify(op_type),
|
|
405
|
+
sql=sql_text,
|
|
406
|
+
table=table,
|
|
407
|
+
column=column,
|
|
408
|
+
)
|
|
409
|
+
]
|
|
@@ -9,15 +9,15 @@ from sqlpush.annotations import HYPERTABLE_KEY
|
|
|
9
9
|
from sqlpush.types import PlannedOperation, RiskClass
|
|
10
10
|
|
|
11
11
|
|
|
12
|
-
def _is_hypertable(conn: Connection, table_name: str) -> bool:
|
|
12
|
+
def _is_hypertable(conn: Connection, schema: str, table_name: str) -> bool:
|
|
13
13
|
try:
|
|
14
14
|
return bool(
|
|
15
15
|
conn.execute(
|
|
16
16
|
text(
|
|
17
17
|
"SELECT 1 FROM timescaledb_information.hypertables "
|
|
18
|
-
"WHERE hypertable_name = :name"
|
|
18
|
+
"WHERE hypertable_schema = :schema AND hypertable_name = :name"
|
|
19
19
|
),
|
|
20
|
-
{"name": table_name},
|
|
20
|
+
{"schema": schema, "name": table_name},
|
|
21
21
|
).scalar()
|
|
22
22
|
)
|
|
23
23
|
except ProgrammingError:
|
|
@@ -50,12 +50,22 @@ def hypertable_operations(
|
|
|
50
50
|
table for table in metadata.tables.values() if table.info.get(HYPERTABLE_KEY) is not None
|
|
51
51
|
]
|
|
52
52
|
if engine is not None and pending:
|
|
53
|
+
default_schema = engine.dialect.default_schema_name or "public"
|
|
53
54
|
with engine.connect() as conn:
|
|
54
|
-
pending = [
|
|
55
|
+
pending = [
|
|
56
|
+
t for t in pending if not _is_hypertable(conn, t.schema or default_schema, t.name)
|
|
57
|
+
]
|
|
55
58
|
ops: list[PlannedOperation] = []
|
|
56
59
|
for table in pending:
|
|
57
60
|
info = table.info[HYPERTABLE_KEY]
|
|
58
|
-
|
|
61
|
+
# Schema-qualified relation: create_hypertable resolves an
|
|
62
|
+
# unqualified name via the session search_path, so a table in a
|
|
63
|
+
# non-default schema MUST carry its schema or the op lands on
|
|
64
|
+
# public.<name> (UndefinedTable). Schema-less tables keep the
|
|
65
|
+
# bare name: they live in the default schema, which the
|
|
66
|
+
# search_path already resolves.
|
|
67
|
+
relation = table.name if table.schema is None else f"{table.schema}.{table.name}"
|
|
68
|
+
name = _lit(relation)
|
|
59
69
|
time_column = _lit(info.time_column)
|
|
60
70
|
parts = [f"SELECT create_hypertable('{name}', '{time_column}'"]
|
|
61
71
|
if info.chunk_time_interval:
|
|
@@ -1,205 +0,0 @@
|
|
|
1
|
-
"""The ONLY module in sqlpush that imports alembic."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import fnmatch
|
|
6
|
-
import io
|
|
7
|
-
from collections.abc import Sequence
|
|
8
|
-
|
|
9
|
-
from alembic.autogenerate import produce_migrations
|
|
10
|
-
from alembic.migration import MigrationContext
|
|
11
|
-
from alembic.operations import Operations
|
|
12
|
-
from alembic.operations.ops import AlterColumnOp, OpContainer
|
|
13
|
-
from sqlalchemy import MetaData, text
|
|
14
|
-
from sqlalchemy.engine import Engine
|
|
15
|
-
|
|
16
|
-
from sqlpush.core.classify import classify
|
|
17
|
-
from sqlpush.types import Plan, PlannedOperation
|
|
18
|
-
|
|
19
|
-
_SYSTEM_SCHEMAS = ("_timescaledb%", "information_schema", "pg_%")
|
|
20
|
-
_SYSTEM_TABLES = ("alembic_version", "spatial_ref_sys")
|
|
21
|
-
|
|
22
|
-
# Leaf op class -> sqlpush op type. Class names follow alembic 1.19.1
|
|
23
|
-
# autogen output as observed in docs/notes/alembic-notes.md and probes:
|
|
24
|
-
# table create is CreateTableOp (not "AddTableOp"), column modify is
|
|
25
|
-
# AlterColumnOp (not "ModifyColumnOp"), index create is CreateIndexOp,
|
|
26
|
-
# and constraint creates are the concrete Create*ConstraintOp subclasses
|
|
27
|
-
# The AddConstraintOp base class is never emitted as a leaf op.
|
|
28
|
-
# AlterColumnOp is absent ON PURPOSE: its label is derived per-op (see
|
|
29
|
-
# _alter_column_label) because default-only, nullable-only and type
|
|
30
|
-
# changes all arrive as the same class.
|
|
31
|
-
_OP_TYPE = {
|
|
32
|
-
"CreateTableOp": "add_table",
|
|
33
|
-
"DropTableOp": "drop_table",
|
|
34
|
-
"AddColumnOp": "add_column",
|
|
35
|
-
"DropColumnOp": "drop_column",
|
|
36
|
-
"CreateIndexOp": "add_index",
|
|
37
|
-
"DropIndexOp": "drop_index",
|
|
38
|
-
"CreateUniqueConstraintOp": "add_constraint",
|
|
39
|
-
"CreateForeignKeyOp": "add_constraint",
|
|
40
|
-
"CreatePrimaryKeyOp": "add_constraint",
|
|
41
|
-
"CreateCheckConstraintOp": "add_constraint",
|
|
42
|
-
"DropConstraintOp": "drop_constraint",
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def _alter_column_label(op: AlterColumnOp) -> str:
|
|
47
|
-
"""Precise op type for AlterColumnOp (alembic-notes Pattern C).
|
|
48
|
-
|
|
49
|
-
Sentinel semantics: ``False`` and ``None`` both mean "leave
|
|
50
|
-
unchanged"; any other value is the new setting. Default-only and
|
|
51
|
-
nullable-only drift arrive as AlterColumnOp just like type changes,
|
|
52
|
-
so disambiguate on the attributes.
|
|
53
|
-
"""
|
|
54
|
-
if op.modify_server_default not in (False, None):
|
|
55
|
-
return "modify_default"
|
|
56
|
-
if op.modify_nullable not in (False, None):
|
|
57
|
-
return "modify_nullable"
|
|
58
|
-
return "modify_type"
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def _is_system_schema(schema: str) -> bool:
|
|
62
|
-
return any(fnmatch.fnmatch(schema, pat) for pat in _SYSTEM_SCHEMAS)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def _make_include_name(schemas: frozenset[str], default_schema: str):
|
|
66
|
-
def include_name(name, type_, parent_names):
|
|
67
|
-
# Prune schemas (and everything inside them) BEFORE reflection:
|
|
68
|
-
# system catalogs (timescale et al.) are then never reflected at
|
|
69
|
-
# all. Schema filtering routes through include_name: alembic
|
|
70
|
-
# 1.19 never calls include_object with type_ == "schema", and
|
|
71
|
-
# its return value must be a real bool (falsy = exclude).
|
|
72
|
-
# NB: alembic substitutes None for the default schema name in
|
|
73
|
-
# the schema pass (compare/schema.py), so map None back.
|
|
74
|
-
if type_ == "schema":
|
|
75
|
-
schema = default_schema if name is None else name
|
|
76
|
-
else:
|
|
77
|
-
schema = parent_names.get("schema_name") or default_schema
|
|
78
|
-
return schema in schemas and not _is_system_schema(schema)
|
|
79
|
-
|
|
80
|
-
return include_name
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def _make_include(schemas: frozenset[str], default_schema: str):
|
|
84
|
-
def include_object(obj, name, type_, reflected, compare_to):
|
|
85
|
-
# Bound the diff to the target schemas on every branch: derive
|
|
86
|
-
# the object's effective schema and require membership, for
|
|
87
|
-
# reflected-only, metadata-only and both-present objects alike.
|
|
88
|
-
# Column/Index/Constraint objects have no `.schema` of their own
|
|
89
|
-
# (SQLAlchemy 2.0.52): they resolve it through the parent table.
|
|
90
|
-
schema = (
|
|
91
|
-
getattr(obj, "schema", None)
|
|
92
|
-
or getattr(getattr(obj, "table", None), "schema", None)
|
|
93
|
-
or default_schema
|
|
94
|
-
)
|
|
95
|
-
if schema not in schemas or _is_system_schema(schema):
|
|
96
|
-
return False
|
|
97
|
-
if reflected and compare_to is None:
|
|
98
|
-
# DB-only object: skip system tables. NB: alembic also
|
|
99
|
-
# auto-excludes its own version table from autogen; the
|
|
100
|
-
# name check here is defense in depth.
|
|
101
|
-
return name not in _SYSTEM_TABLES
|
|
102
|
-
return True
|
|
103
|
-
|
|
104
|
-
return include_object
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def _flatten(ops):
|
|
108
|
-
# D1 (alembic-notes): ops targeting existing tables arrive wrapped in
|
|
109
|
-
# ModifyTableOps containers; Operations.invoke crashes on containers,
|
|
110
|
-
# so recurse into anything that is an OpContainer before invoking.
|
|
111
|
-
for op in ops:
|
|
112
|
-
if isinstance(op, OpContainer):
|
|
113
|
-
yield from _flatten(op.ops)
|
|
114
|
-
else:
|
|
115
|
-
yield op
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def _render_op_sql(op, engine: Engine) -> str:
|
|
119
|
-
buf = io.StringIO()
|
|
120
|
-
offline = MigrationContext.configure(
|
|
121
|
-
dialect=engine.dialect, opts={"as_sql": True, "output_buffer": buf}
|
|
122
|
-
)
|
|
123
|
-
operations = Operations(offline)
|
|
124
|
-
operations.invoke(op)
|
|
125
|
-
# offline render terminates each op with ";\n\n"; normalize so each
|
|
126
|
-
# PlannedOperation.sql is a single clean statement
|
|
127
|
-
return buf.getvalue().strip().rstrip(";")
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
class DiffEngine:
|
|
131
|
-
def plan(
|
|
132
|
-
self,
|
|
133
|
-
metadata: MetaData,
|
|
134
|
-
engine: Engine,
|
|
135
|
-
*,
|
|
136
|
-
schemas: Sequence[str] | None = None,
|
|
137
|
-
exclude: Sequence[str] = (),
|
|
138
|
-
) -> Plan:
|
|
139
|
-
if schemas is None:
|
|
140
|
-
with engine.connect() as conn:
|
|
141
|
-
search_path = conn.execute(text("SHOW search_path")).scalar()
|
|
142
|
-
# a live PG session always reports a search_path
|
|
143
|
-
assert search_path is not None
|
|
144
|
-
schemas = [
|
|
145
|
-
s.strip()
|
|
146
|
-
for s in search_path.split(",")
|
|
147
|
-
if s.strip() and s.strip() != '"$user"'
|
|
148
|
-
]
|
|
149
|
-
# New binding rather than a param reassignment: frozenset is a
|
|
150
|
-
# Set, not a Sequence, and the helpers below declare the exact
|
|
151
|
-
# type they take.
|
|
152
|
-
schema_set = frozenset(schemas)
|
|
153
|
-
exclude = tuple(exclude)
|
|
154
|
-
# Typed `str | None` by SQLAlchemy; a dialect without a default
|
|
155
|
-
# schema is meaningless for this PostgreSQL-only tool; "public"
|
|
156
|
-
# matches every supported server config.
|
|
157
|
-
default_schema = engine.dialect.default_schema_name or "public"
|
|
158
|
-
|
|
159
|
-
opts = {
|
|
160
|
-
"compare_type": True,
|
|
161
|
-
"compare_server_default": True,
|
|
162
|
-
"include_name": _make_include_name(schema_set, default_schema),
|
|
163
|
-
"include_object": _make_include(schema_set, default_schema),
|
|
164
|
-
"include_schemas": True,
|
|
165
|
-
}
|
|
166
|
-
with engine.connect() as conn:
|
|
167
|
-
ctx = MigrationContext.configure(conn, opts=opts)
|
|
168
|
-
script = produce_migrations(ctx, metadata)
|
|
169
|
-
|
|
170
|
-
# produce_migrations always builds the upgrade bundle
|
|
171
|
-
assert script.upgrade_ops is not None
|
|
172
|
-
ops: list[PlannedOperation] = []
|
|
173
|
-
for op in _flatten(script.upgrade_ops.ops):
|
|
174
|
-
ops.extend(self._translate(op, engine, exclude))
|
|
175
|
-
return Plan(operations=tuple(ops))
|
|
176
|
-
|
|
177
|
-
def _translate(self, op, engine: Engine, exclude: tuple[str, ...]) -> list[PlannedOperation]:
|
|
178
|
-
op_type = _OP_TYPE.get(type(op).__name__, "raw_sql")
|
|
179
|
-
if isinstance(op, AlterColumnOp):
|
|
180
|
-
op_type = _alter_column_label(op)
|
|
181
|
-
sql_text = _render_op_sql(op, engine)
|
|
182
|
-
table = getattr(getattr(op, "table", None), "name", None) or getattr(op, "table_name", None)
|
|
183
|
-
# AddColumnOp carries a real Column in `.column`; DropColumnOp and
|
|
184
|
-
# AlterColumnOp only expose the plain string `column_name`
|
|
185
|
-
# (alembic-notes op reference).
|
|
186
|
-
column = getattr(getattr(op, "column", None), "name", None) or getattr(
|
|
187
|
-
op, "column_name", None
|
|
188
|
-
)
|
|
189
|
-
# table-level patterns reach column-level ops too; a qualified
|
|
190
|
-
# table.column pattern suppresses only that specific column op
|
|
191
|
-
if table and any(fnmatch.fnmatch(table, pat) for pat in exclude):
|
|
192
|
-
return []
|
|
193
|
-
if table and column:
|
|
194
|
-
full = f"{table}.{column}"
|
|
195
|
-
if any(fnmatch.fnmatch(full, pat) for pat in exclude):
|
|
196
|
-
return []
|
|
197
|
-
return [
|
|
198
|
-
PlannedOperation(
|
|
199
|
-
type=op_type,
|
|
200
|
-
risk=classify(op_type),
|
|
201
|
-
sql=sql_text,
|
|
202
|
-
table=table,
|
|
203
|
-
column=column,
|
|
204
|
-
)
|
|
205
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|