sqlpush 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlpush/annotations.py +70 -4
- sqlpush/api.py +53 -12
- sqlpush/chain/migrate.py +65 -14
- sqlpush/cli.py +13 -3
- sqlpush/core/diff.py +143 -6
- sqlpush/directives/timescale.py +5 -28
- {sqlpush-0.4.0.dist-info → sqlpush-0.4.2.dist-info}/METADATA +1 -1
- {sqlpush-0.4.0.dist-info → sqlpush-0.4.2.dist-info}/RECORD +10 -10
- {sqlpush-0.4.0.dist-info → sqlpush-0.4.2.dist-info}/WHEEL +0 -0
- {sqlpush-0.4.0.dist-info → sqlpush-0.4.2.dist-info}/entry_points.txt +0 -0
sqlpush/annotations.py
CHANGED
|
@@ -12,12 +12,78 @@ class HypertableInfo:
|
|
|
12
12
|
chunk_time_interval: str | None = None
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
def _lit(value: str) -> str:
|
|
16
|
+
# single-quoted SQL literal escaping: ' -> '' (defense in depth
|
|
17
|
+
# against quote break-out via table/column names from user metadata)
|
|
18
|
+
return value.replace("'", "''")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def create_hypertable_sql(table_name: str, schema: str | None, info: HypertableInfo) -> str:
|
|
22
|
+
"""Render the idempotent ``create_hypertable`` statement for an annotated table.
|
|
23
|
+
|
|
24
|
+
Single source of truth shared by the directive planner
|
|
25
|
+
(``directives/timescale.py`` — push/revision) and the decorator's
|
|
26
|
+
``after_create`` listener (``create_all``): both mechanisms must
|
|
27
|
+
always emit identical SQL.
|
|
28
|
+
"""
|
|
29
|
+
# Schema-qualified relation: create_hypertable resolves an
|
|
30
|
+
# unqualified name via the session search_path, so a table in a
|
|
31
|
+
# non-default schema MUST carry its schema or the op lands on
|
|
32
|
+
# public.<name> (UndefinedTable). Schema-less tables keep the
|
|
33
|
+
# bare name: they live in the default schema, which the
|
|
34
|
+
# search_path already resolves.
|
|
35
|
+
relation = table_name if schema is None else f"{schema}.{table_name}"
|
|
36
|
+
parts = [f"SELECT create_hypertable('{_lit(relation)}', '{_lit(info.time_column)}'"]
|
|
37
|
+
if info.chunk_time_interval:
|
|
38
|
+
parts.append(f", chunk_time_interval => INTERVAL '{_lit(info.chunk_time_interval)}'")
|
|
39
|
+
# if_not_exists: race insurance between state checks and apply —
|
|
40
|
+
# and what makes listener (create_all) + directive (push) coexistence
|
|
41
|
+
# duplicate-free. migrate_data: required when the directive runs on an
|
|
42
|
+
# existing populated table; a no-op on the empty table the listener
|
|
43
|
+
# just created. create_default_indexes=false: timescale's implicit
|
|
44
|
+
# time-column index is invisible to metadata, so the default would
|
|
45
|
+
# drift forever (destructive drop_index) on a fully synced schema;
|
|
46
|
+
# users declare wanted indexes in the Table instead (metadata is the
|
|
47
|
+
# source of truth).
|
|
48
|
+
parts.append(", migrate_data => true, if_not_exists => true, create_default_indexes => false)")
|
|
49
|
+
return "".join(parts)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _listen_after_create(table, info: HypertableInfo) -> None:
|
|
53
|
+
# Dual-mechanism (0.4.1): besides the info entry (planned from
|
|
54
|
+
# metadata on push/revision paths), register an after_create
|
|
55
|
+
# listener so create_all paths register the hypertable too —
|
|
56
|
+
# the same idempotent SQL the directive plans, executed via
|
|
57
|
+
# text() exactly like the executor runs planned ops. With a
|
|
58
|
+
# declared index on the time column, create_hypertable sees the
|
|
59
|
+
# column indexed and create_default_indexes => false adds nothing
|
|
60
|
+
# (see tests/test_annotations_createall.py).
|
|
61
|
+
# Imports stay inside the function: this module must remain free of
|
|
62
|
+
# heavy imports at module level (pinned by test_annotations.py).
|
|
63
|
+
from sqlalchemy import event, text
|
|
64
|
+
|
|
65
|
+
sql = create_hypertable_sql(table.name, table.schema, info)
|
|
66
|
+
|
|
67
|
+
def _after_create(target, connection, **kw):
|
|
68
|
+
connection.execute(text(sql))
|
|
69
|
+
|
|
70
|
+
event.listen(table, "after_create", _after_create)
|
|
71
|
+
|
|
72
|
+
|
|
15
73
|
def hypertable(*, time_column: str, chunk_time_interval: str | None = None):
|
|
16
|
-
"""Record hypertable intent on the model's Table (
|
|
74
|
+
"""Record hypertable intent on the model's Table (dual-mechanism).
|
|
17
75
|
|
|
18
76
|
Works with SQLModel, Flask-SQLAlchemy and plain declarative, anything
|
|
19
77
|
whose class already carries a built ``__table__`` when decorated.
|
|
20
78
|
|
|
79
|
+
Mechanism 1 (metadata): the annotation is recorded on ``Table.info``
|
|
80
|
+
and planned as a ``create_hypertable`` op by push/revision.
|
|
81
|
+
|
|
82
|
+
Mechanism 2 (create_all): an ``after_create`` listener runs the same
|
|
83
|
+
idempotent ``create_hypertable``, so ``MetaData.create_all`` paths
|
|
84
|
+
register the hypertable too; coexistence with a later push is
|
|
85
|
+
duplicate-free (``if_not_exists``) and ``check()`` stays clean.
|
|
86
|
+
|
|
21
87
|
The generated ``create_hypertable`` runs with
|
|
22
88
|
``create_default_indexes => false``: timescale's implicit time-column
|
|
23
89
|
index is invisible to metadata and would drift forever. Declare it
|
|
@@ -29,9 +95,9 @@ def hypertable(*, time_column: str, chunk_time_interval: str | None = None):
|
|
|
29
95
|
table = getattr(cls, "__table__", None)
|
|
30
96
|
if table is None:
|
|
31
97
|
raise TypeError(f"{cls.__name__} has no __table__; decorate a mapped class")
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
)
|
|
98
|
+
info = HypertableInfo(time_column=time_column, chunk_time_interval=chunk_time_interval)
|
|
99
|
+
table.info[HYPERTABLE_KEY] = info
|
|
100
|
+
_listen_after_create(table, info)
|
|
35
101
|
return cls
|
|
36
102
|
|
|
37
103
|
return decorator
|
sqlpush/api.py
CHANGED
|
@@ -139,7 +139,12 @@ def revision(
|
|
|
139
139
|
risk = max((op.risk for op in p.operations), key=lambda r: RISK_RANK[r])
|
|
140
140
|
ops = [(f"[{op.risk.name}] {op.type} {op.table or '?'}", op.sql) for op in p.operations]
|
|
141
141
|
out = Path(out_dir)
|
|
142
|
-
|
|
142
|
+
try:
|
|
143
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
144
|
+
except OSError as exc:
|
|
145
|
+
# same typing as the write below: only SqlpushError subclasses
|
|
146
|
+
# escape the API surface (e.g. out_dir under a file)
|
|
147
|
+
raise SqlpushError(f"cannot create migrations directory {out}: {exc}") from exc
|
|
143
148
|
rev_id = next_revision_id(out)
|
|
144
149
|
slug = re.sub(r"[^a-z0-9_]+", "_", (message or "migration").lower())[:40]
|
|
145
150
|
path = out / f"{rev_id}_{slug}.sql"
|
|
@@ -163,16 +168,32 @@ def revision(
|
|
|
163
168
|
return path
|
|
164
169
|
|
|
165
170
|
|
|
166
|
-
def migrate(
|
|
171
|
+
def migrate(
|
|
172
|
+
target,
|
|
173
|
+
*,
|
|
174
|
+
chain_dir="migrations/versions",
|
|
175
|
+
allow_destructive=False,
|
|
176
|
+
advisory_wait=30.0,
|
|
177
|
+
lock_timeout=5.0,
|
|
178
|
+
) -> MigrateReport:
|
|
167
179
|
"""Replay annotated-SQL chain files with gates + same-txn bookkeeping.
|
|
168
180
|
|
|
169
181
|
``target`` is a DSN string, sync ``Engine`` or ``AsyncEngine`` (resolved
|
|
170
|
-
via ``_sync_engine_from``; engines created here are disposed).
|
|
182
|
+
via ``_sync_engine_from``; engines created here are disposed).
|
|
183
|
+
``advisory_wait`` bounds the advisory-lock wait and ``lock_timeout``
|
|
184
|
+
bounds each per-file transaction's lock wait (seconds; 0 = fail
|
|
185
|
+
immediately — same contract as ``push``). See
|
|
171
186
|
``chain.migrate.run_migrate`` for the execution contract.
|
|
172
187
|
"""
|
|
173
188
|
engine, dispose = _sync_engine_from(target)
|
|
174
189
|
try:
|
|
175
|
-
return run_migrate(
|
|
190
|
+
return run_migrate(
|
|
191
|
+
engine,
|
|
192
|
+
chain_dir=chain_dir,
|
|
193
|
+
allow_destructive=allow_destructive,
|
|
194
|
+
advisory_wait=advisory_wait,
|
|
195
|
+
lock_timeout=lock_timeout,
|
|
196
|
+
)
|
|
176
197
|
except SQLAlchemyError as exc:
|
|
177
198
|
# MigrationFileError/SqlpushError (typed) pass through untouched
|
|
178
199
|
_raise_typed(exc)
|
|
@@ -181,16 +202,19 @@ def migrate(target, *, chain_dir="migrations/versions", allow_destructive=False)
|
|
|
181
202
|
engine.dispose()
|
|
182
203
|
|
|
183
204
|
|
|
184
|
-
def stamp(target, *, chain_dir="migrations/versions") -> MigrateReport:
|
|
205
|
+
def stamp(target, *, chain_dir="migrations/versions", force: bool = False) -> MigrateReport:
|
|
185
206
|
"""Bootstrap: register chain files as applied WITHOUT executing SQL.
|
|
186
207
|
|
|
187
|
-
For adopting a DB whose schema already reflects the chain.
|
|
188
|
-
|
|
189
|
-
``
|
|
208
|
+
For adopting a DB whose schema already reflects the chain. A file
|
|
209
|
+
already registered with a different checksum (edited after apply)
|
|
210
|
+
is refused unless ``force`` is set — edit detection survives
|
|
211
|
+
re-stamping. ``target`` resolution and error typing match
|
|
212
|
+
:func:`migrate`. See ``chain.migrate.run_stamp`` for the report
|
|
213
|
+
convention.
|
|
190
214
|
"""
|
|
191
215
|
engine, dispose = _sync_engine_from(target)
|
|
192
216
|
try:
|
|
193
|
-
return run_stamp(engine, chain_dir=chain_dir)
|
|
217
|
+
return run_stamp(engine, chain_dir=chain_dir, force=force)
|
|
194
218
|
except SQLAlchemyError as exc:
|
|
195
219
|
# MigrationFileError/SqlpushError (typed) pass through untouched
|
|
196
220
|
_raise_typed(exc)
|
|
@@ -199,18 +223,35 @@ def stamp(target, *, chain_dir="migrations/versions") -> MigrateReport:
|
|
|
199
223
|
engine.dispose()
|
|
200
224
|
|
|
201
225
|
|
|
226
|
+
def _translate_asyncpg(url):
|
|
227
|
+
"""Map a ``postgresql+asyncpg`` URL onto the sync psycopg driver.
|
|
228
|
+
|
|
229
|
+
An asyncpg URL cannot back a SYNC engine (the dialect is
|
|
230
|
+
async-only), but psycopg is a runtime dependency — the translated
|
|
231
|
+
engine actually connects. asyncpg itself is never required in this
|
|
232
|
+
process. Everything but the driver (host/db/credentials/query
|
|
233
|
+
options) is preserved via the URL API, no string surgery.
|
|
234
|
+
"""
|
|
235
|
+
if url.drivername == "postgresql+asyncpg":
|
|
236
|
+
return url.set(drivername="postgresql+psycopg")
|
|
237
|
+
return url
|
|
238
|
+
|
|
239
|
+
|
|
202
240
|
def _sync_engine_from(target):
|
|
203
241
|
if isinstance(target, AsyncEngine):
|
|
204
242
|
from sqlalchemy import create_engine
|
|
205
243
|
from sqlalchemy.pool import NullPool
|
|
206
244
|
|
|
207
|
-
dsn = target.url
|
|
208
|
-
return create_engine(dsn, poolclass=NullPool), True
|
|
245
|
+
dsn = _translate_asyncpg(target.url)
|
|
246
|
+
return create_engine(dsn.render_as_string(hide_password=False), poolclass=NullPool), True
|
|
209
247
|
if isinstance(target, str):
|
|
210
248
|
from sqlalchemy import create_engine
|
|
249
|
+
from sqlalchemy.engine import make_url
|
|
211
250
|
from sqlalchemy.pool import NullPool
|
|
212
251
|
|
|
213
|
-
|
|
252
|
+
# make_url raises the same ArgumentError a bad DSN would raise
|
|
253
|
+
# inside create_engine — error typing downstream is unchanged
|
|
254
|
+
return create_engine(_translate_asyncpg(make_url(target)), poolclass=NullPool), True
|
|
214
255
|
return target, False
|
|
215
256
|
|
|
216
257
|
|
sqlpush/chain/migrate.py
CHANGED
|
@@ -11,13 +11,15 @@ let a crash between apply and registry re-apply the file forever
|
|
|
11
11
|
|
|
12
12
|
Fail-loud ordering: any blocked file (parse error, checksum mismatch,
|
|
13
13
|
destructive gate, SQL failure) stops the chain — nothing later runs (R4).
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
The advisory lock wait is BOUNDED (--advisory-wait, default 30s): an
|
|
15
|
+
exhausted budget raises a typed error instead of hanging on a stuck
|
|
16
|
+
holder — diagnose the holder via pg_locks / pg_stat_activity.
|
|
16
17
|
"""
|
|
17
18
|
|
|
18
19
|
from __future__ import annotations
|
|
19
20
|
|
|
20
21
|
import contextlib
|
|
22
|
+
import time
|
|
21
23
|
from collections.abc import Iterator
|
|
22
24
|
from pathlib import Path
|
|
23
25
|
|
|
@@ -26,7 +28,7 @@ from sqlalchemy.engine import Connection, Engine
|
|
|
26
28
|
|
|
27
29
|
from sqlpush.apply.executor import advisory_key
|
|
28
30
|
from sqlpush.chain.format import MigrationFileError, checksum, parse_migration_file
|
|
29
|
-
from sqlpush.types import MigrateReport, RiskClass
|
|
31
|
+
from sqlpush.types import MigrateReport, RiskClass, SqlpushError
|
|
30
32
|
|
|
31
33
|
_VERSIONS_DDL = (
|
|
32
34
|
"CREATE TABLE IF NOT EXISTS public.sqlpush_versions ("
|
|
@@ -45,18 +47,33 @@ def _chain_files(chain_dir: str | Path) -> list[Path]:
|
|
|
45
47
|
|
|
46
48
|
|
|
47
49
|
@contextlib.contextmanager
|
|
48
|
-
def _chain_session(engine: Engine) -> Iterator[Connection]:
|
|
50
|
+
def _chain_session(engine: Engine, *, advisory_wait: float = 30.0) -> Iterator[Connection]:
|
|
49
51
|
"""Session-scoped advisory lock + versions table, shared by every verb.
|
|
50
52
|
|
|
51
53
|
Same key derivation as push (fnv1a_32(b"sqlpush") ^ db oid): serializes
|
|
52
54
|
concurrent chain workers and excludes push on the same database. The
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
55
|
+
wait is BOUNDED, mirroring executor.with_advisory_lock:
|
|
56
|
+
``pg_try_advisory_lock`` polled every 0.5 s against a
|
|
57
|
+
``time.monotonic()`` deadline; an exhausted budget raises
|
|
58
|
+
:class:`SqlpushError` — a blocking ``pg_advisory_lock`` would hang
|
|
59
|
+
forever on a stuck holder. The txn opened by the key/probe queries
|
|
60
|
+
is committed right away — session advisory locks survive
|
|
61
|
+
COMMIT/ROLLBACK, and an idle-in-transaction session is exposed to
|
|
62
|
+
idle_in_transaction_session_timeout (executor.py note).
|
|
56
63
|
"""
|
|
64
|
+
if advisory_wait < 0:
|
|
65
|
+
raise SqlpushError(f"advisory_wait must be >= 0, got {advisory_wait}")
|
|
57
66
|
with engine.connect() as conn:
|
|
58
67
|
key = advisory_key(conn)
|
|
59
|
-
|
|
68
|
+
deadline = time.monotonic() + advisory_wait
|
|
69
|
+
locked = conn.execute(text("SELECT pg_try_advisory_lock(:k)"), {"k": key}).scalar()
|
|
70
|
+
while not locked and time.monotonic() < deadline:
|
|
71
|
+
time.sleep(0.5)
|
|
72
|
+
locked = conn.execute(text("SELECT pg_try_advisory_lock(:k)"), {"k": key}).scalar()
|
|
73
|
+
if not locked:
|
|
74
|
+
raise SqlpushError(
|
|
75
|
+
f"another sqlpush worker holds the advisory lock after {advisory_wait}s"
|
|
76
|
+
)
|
|
60
77
|
conn.commit()
|
|
61
78
|
try:
|
|
62
79
|
conn.execute(text(_VERSIONS_DDL))
|
|
@@ -73,14 +90,26 @@ def _chain_session(engine: Engine) -> Iterator[Connection]:
|
|
|
73
90
|
conn.commit()
|
|
74
91
|
|
|
75
92
|
|
|
76
|
-
def run_migrate(
|
|
93
|
+
def run_migrate(
|
|
94
|
+
engine: Engine,
|
|
95
|
+
*,
|
|
96
|
+
chain_dir: str | Path,
|
|
97
|
+
allow_destructive: bool,
|
|
98
|
+
advisory_wait: float = 30.0,
|
|
99
|
+
lock_timeout: float = 5.0,
|
|
100
|
+
) -> MigrateReport:
|
|
101
|
+
if lock_timeout < 0:
|
|
102
|
+
# same contract as push (executor.with_advisory_lock): budgets
|
|
103
|
+
# are typed floats, never user input, and a negative one must
|
|
104
|
+
# fail before any file or connection work
|
|
105
|
+
raise SqlpushError(f"lock_timeout must be >= 0, got {lock_timeout}")
|
|
77
106
|
applied: list[str] = []
|
|
78
107
|
skipped: list[str] = []
|
|
79
108
|
blocked: list[str] = []
|
|
80
109
|
notes: list[str] = []
|
|
81
110
|
partial = False
|
|
82
111
|
chain = _chain_files(chain_dir)
|
|
83
|
-
with _chain_session(engine) as conn:
|
|
112
|
+
with _chain_session(engine, advisory_wait=advisory_wait) as conn:
|
|
84
113
|
recorded = {
|
|
85
114
|
row[0]: row[1]
|
|
86
115
|
for row in conn.execute(text("SELECT name, sha256 FROM public.sqlpush_versions"))
|
|
@@ -107,6 +136,14 @@ def run_migrate(engine: Engine, *, chain_dir: str | Path, allow_destructive: boo
|
|
|
107
136
|
break
|
|
108
137
|
try:
|
|
109
138
|
with conn.begin():
|
|
139
|
+
# txn-scoped: SET LOCAL dies with the per-file txn
|
|
140
|
+
# (style: push's transactional segment, executor.py).
|
|
141
|
+
# NOTE: PostgreSQL does not accept bind parameters for
|
|
142
|
+
# SET (utility statement), so the int is inlined;
|
|
143
|
+
# lock_timeout is a typed float parameter, not user
|
|
144
|
+
# input. A chain file blocked behind another
|
|
145
|
+
# transaction's lock fails fast instead of queuing.
|
|
146
|
+
conn.execute(text(f"SET LOCAL lock_timeout = {int(lock_timeout * 1000)}"))
|
|
110
147
|
# whole-file replay: exec_driver_sql bypasses text()'s
|
|
111
148
|
# bind-param parsing entirely — ":casts" and ":=" in
|
|
112
149
|
# hand-edited SQL must reach the server verbatim
|
|
@@ -130,14 +167,18 @@ def run_migrate(engine: Engine, *, chain_dir: str | Path, allow_destructive: boo
|
|
|
130
167
|
)
|
|
131
168
|
|
|
132
169
|
|
|
133
|
-
def run_stamp(engine: Engine, *, chain_dir: str | Path) -> MigrateReport:
|
|
170
|
+
def run_stamp(engine: Engine, *, chain_dir: str | Path, force: bool = False) -> MigrateReport:
|
|
134
171
|
"""Register every parseable chain file WITHOUT executing any SQL.
|
|
135
172
|
|
|
136
173
|
Bootstrap seam (spec §5 R7): adopt a DB whose schema already reflects
|
|
137
174
|
the chain. Only the header must parse (fail-loud on THAT) — invalid SQL
|
|
138
|
-
in a body still registers, because stamp never executes anything.
|
|
139
|
-
|
|
140
|
-
|
|
175
|
+
in a body still registers, because stamp never executes anything. A
|
|
176
|
+
file already recorded with a DIFFERENT checksum is refused — it was
|
|
177
|
+
edited after apply/stamp, and silently refreshing would wipe the
|
|
178
|
+
edit-detection integrity — unless ``force`` is set; the first mismatch
|
|
179
|
+
raises and nothing after it registers (strict order, same as migrate).
|
|
180
|
+
Unrecorded files, unchanged re-stamps and forced re-stamps upsert
|
|
181
|
+
idempotently (``ON CONFLICT (name) DO UPDATE``).
|
|
141
182
|
|
|
142
183
|
Report convention: registered files are listed in ``skipped`` (stamp
|
|
143
184
|
never populates ``applied`` and never sets ``partial_failure``); a
|
|
@@ -149,6 +190,11 @@ def run_stamp(engine: Engine, *, chain_dir: str | Path) -> MigrateReport:
|
|
|
149
190
|
notes: list[str] = []
|
|
150
191
|
chain = _chain_files(chain_dir)
|
|
151
192
|
with _chain_session(engine) as conn:
|
|
193
|
+
recorded = {
|
|
194
|
+
row[0]: row[1]
|
|
195
|
+
for row in conn.execute(text("SELECT name, sha256 FROM public.sqlpush_versions"))
|
|
196
|
+
}
|
|
197
|
+
conn.commit()
|
|
152
198
|
for f in chain:
|
|
153
199
|
raw = f.read_text()
|
|
154
200
|
try:
|
|
@@ -157,6 +203,11 @@ def run_stamp(engine: Engine, *, chain_dir: str | Path) -> MigrateReport:
|
|
|
157
203
|
blocked.append(f.name)
|
|
158
204
|
notes.append(f"{f.name}: {exc}")
|
|
159
205
|
break # orden estricto: nada posterior se registra
|
|
206
|
+
if f.name in recorded and recorded[f.name] != checksum(raw) and not force:
|
|
207
|
+
raise SqlpushError(
|
|
208
|
+
f"{f.name}: checksum mismatch (file edited after apply?); "
|
|
209
|
+
"pass force=True/--force to accept the new content"
|
|
210
|
+
)
|
|
160
211
|
with conn.begin():
|
|
161
212
|
conn.execute(
|
|
162
213
|
text(
|
sqlpush/cli.py
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
Exit codes: diff always 0; check 0 clean / 2 drift / 3 destructive drift;
|
|
5
5
|
push 0 applied / 1 destructive blocked / 2 error (incl. partial failure);
|
|
6
6
|
revision 0 written / 1 error (empty drift refuses); migrate 0 clean /
|
|
7
|
-
1 blocked or partial failure
|
|
7
|
+
1 blocked or partial failure; stamp 0 registered / 1 blocked or refused
|
|
8
|
+
(an edited file without --force raises a typed error: exit 1).
|
|
8
9
|
"""
|
|
9
10
|
|
|
10
11
|
from __future__ import annotations
|
|
@@ -250,12 +251,20 @@ def revision(
|
|
|
250
251
|
def migrate(
|
|
251
252
|
dsn: str | None = typer.Option(None),
|
|
252
253
|
allow_destructive: bool = typer.Option(False, "--allow-destructive"),
|
|
254
|
+
advisory_wait: float = typer.Option(30.0, "--advisory-wait"),
|
|
255
|
+
lock_timeout: float = typer.Option(5.0, "--lock-timeout"),
|
|
253
256
|
out_dir: DirOpt = Path("migrations/versions"),
|
|
254
257
|
):
|
|
255
258
|
"""Replay pending migration files (gates + checksum bookkeeping)."""
|
|
256
259
|
engine = _engine(dsn)
|
|
257
260
|
try:
|
|
258
|
-
report = api.migrate(
|
|
261
|
+
report = api.migrate(
|
|
262
|
+
engine,
|
|
263
|
+
chain_dir=out_dir,
|
|
264
|
+
allow_destructive=allow_destructive,
|
|
265
|
+
advisory_wait=advisory_wait,
|
|
266
|
+
lock_timeout=lock_timeout,
|
|
267
|
+
)
|
|
259
268
|
finally:
|
|
260
269
|
engine.dispose()
|
|
261
270
|
typer.echo(
|
|
@@ -272,12 +281,13 @@ def migrate(
|
|
|
272
281
|
@app.command()
|
|
273
282
|
def stamp(
|
|
274
283
|
dsn: str | None = typer.Option(None),
|
|
284
|
+
force: bool = typer.Option(False, "--force"),
|
|
275
285
|
out_dir: DirOpt = Path("migrations/versions"),
|
|
276
286
|
):
|
|
277
287
|
"""Adopt an existing DB: register chain files without executing SQL."""
|
|
278
288
|
engine = _engine(dsn)
|
|
279
289
|
try:
|
|
280
|
-
report = api.stamp(engine, chain_dir=out_dir)
|
|
290
|
+
report = api.stamp(engine, chain_dir=out_dir, force=force)
|
|
281
291
|
finally:
|
|
282
292
|
engine.dispose()
|
|
283
293
|
typer.echo(
|
sqlpush/core/diff.py
CHANGED
|
@@ -107,6 +107,12 @@ def _timescale_auto_indexes(conn) -> dict[str, set[str]]:
|
|
|
107
107
|
``a_b`` dim ``c`` both yield ``a_b_c_idx``) — a str value would
|
|
108
108
|
last-win and leak the other as false drift. Qualified keys carry
|
|
109
109
|
cross-schema hypertables. Empty when the extension is absent.
|
|
110
|
+
|
|
111
|
+
Consumed twice in ``_make_include``: DB-only prunes (the reflected
|
|
112
|
+
auto index with no metadata counterpart) and the S1 both-present
|
|
113
|
+
sibling (a metadata-declared index with the same qualified name and
|
|
114
|
+
column sequence compares EQUAL — the default's born-DESC ordering is
|
|
115
|
+
extension state, not drift).
|
|
110
116
|
"""
|
|
111
117
|
installed = conn.execute(
|
|
112
118
|
text("SELECT 1 FROM pg_extension WHERE extname = 'timescaledb'")
|
|
@@ -222,6 +228,24 @@ def _make_include_name(schemas: frozenset[str], default_schema: str):
|
|
|
222
228
|
return include_name
|
|
223
229
|
|
|
224
230
|
|
|
231
|
+
def _expr_name(e) -> str | None:
|
|
232
|
+
"""Best-effort column name of an Index expression, order-insensitive.
|
|
233
|
+
|
|
234
|
+
Metadata-side expressions may be bare strings (``Index(..., "col")``)
|
|
235
|
+
or Column objects; reflected-side ones arrive wrapped in
|
|
236
|
+
``UnaryExpression`` (the born-DESC/ASC ordering modifier). Unwrap
|
|
237
|
+
wrapper elements (UnaryExpression/Label carry ``.element``), resolve
|
|
238
|
+
strings and Columns to their names. Functions fall back to ``.name``
|
|
239
|
+
— a sufficient equality proxy for the suppression check below.
|
|
240
|
+
"""
|
|
241
|
+
if isinstance(e, str):
|
|
242
|
+
return e
|
|
243
|
+
inner = getattr(e, "element", None)
|
|
244
|
+
if inner is not None:
|
|
245
|
+
return _expr_name(inner)
|
|
246
|
+
return getattr(e, "name", None)
|
|
247
|
+
|
|
248
|
+
|
|
225
249
|
def _make_include(
|
|
226
250
|
schemas: frozenset[str],
|
|
227
251
|
default_schema: str,
|
|
@@ -241,6 +265,11 @@ def _make_include(
|
|
|
241
265
|
)
|
|
242
266
|
if schema not in schemas or _is_system_schema(schema):
|
|
243
267
|
return False
|
|
268
|
+
parent = getattr(obj, "table", None)
|
|
269
|
+
parent_table = getattr(parent, "name", None)
|
|
270
|
+
parent_schema = getattr(parent, "schema", None) or default_schema
|
|
271
|
+
qualified_index = f"{parent_schema}.{name}"
|
|
272
|
+
qualified_parent = f"{parent_schema}.{parent_table}"
|
|
244
273
|
if reflected and compare_to is None:
|
|
245
274
|
# DB-only object: skip system tables. NB: alembic also
|
|
246
275
|
# auto-excludes its own version table from autogen; the
|
|
@@ -251,16 +280,29 @@ def _make_include(
|
|
|
251
280
|
# state no model declares. Qualified name AND owner set must
|
|
252
281
|
# both match, so a same-named index on another table stays
|
|
253
282
|
# real drift, as does one the metadata actually declares
|
|
254
|
-
# (that arrives in the both-present
|
|
255
|
-
parent = getattr(obj, "table", None)
|
|
256
|
-
parent_table = getattr(parent, "name", None)
|
|
257
|
-
parent_schema = getattr(parent, "schema", None) or default_schema
|
|
258
|
-
qualified_index = f"{parent_schema}.{name}"
|
|
259
|
-
qualified_parent = f"{parent_schema}.{parent_table}"
|
|
283
|
+
# (that arrives in the both-present branches below).
|
|
260
284
|
if type_ == "index" and qualified_index in spatial_auto_indexes:
|
|
261
285
|
return False
|
|
262
286
|
owners = ts_auto_indexes.get(qualified_index, frozenset())
|
|
263
287
|
return not (type_ == "index" and qualified_parent in owners)
|
|
288
|
+
if type_ == "index" and not reflected and compare_to is not None:
|
|
289
|
+
# S1 both-present sibling (atlas cycle-6): alembic gates the
|
|
290
|
+
# drop+add PAIR for a "changed" index behind exactly ONE
|
|
291
|
+
# include_object call (metadata index, reflected=False,
|
|
292
|
+
# compare_to=reflected index). TimescaleDB's default time
|
|
293
|
+
# index is born DESC (pg_index.indoption DESC bit); a
|
|
294
|
+
# metadata-declared index with the same name, same owning
|
|
295
|
+
# hypertable and the SAME column sequence is its metadata
|
|
296
|
+
# stand-in — the sort-order difference is extension-birth
|
|
297
|
+
# state, not drift. Column names compared ORDER-SENSITIVELY:
|
|
298
|
+
# a genuinely different declaration (reorder, extra column)
|
|
299
|
+
# still reports the pair.
|
|
300
|
+
owners = ts_auto_indexes.get(qualified_index, frozenset())
|
|
301
|
+
if qualified_parent in owners:
|
|
302
|
+
m_cols = [_expr_name(e) for e in obj.expressions]
|
|
303
|
+
r_cols = [_expr_name(e) for e in compare_to.expressions]
|
|
304
|
+
if m_cols == r_cols:
|
|
305
|
+
return False
|
|
264
306
|
return True
|
|
265
307
|
|
|
266
308
|
return include_object
|
|
@@ -316,6 +358,100 @@ def _dedup_embedded_indexes(ops: list[PlannedOperation]) -> list[PlannedOperatio
|
|
|
316
358
|
]
|
|
317
359
|
|
|
318
360
|
|
|
361
|
+
def _split_statements(sql: str) -> list[str]:
|
|
362
|
+
"""Split a multi-statement op render on top-level ``;`` only.
|
|
363
|
+
|
|
364
|
+
Quote-aware: ``;`` inside single-quoted SQL literals (enum labels
|
|
365
|
+
can carry them) never splits; ``''`` escapes are consumed in pairs.
|
|
366
|
+
"""
|
|
367
|
+
parts: list[str] = []
|
|
368
|
+
buf: list[str] = []
|
|
369
|
+
in_quote = False
|
|
370
|
+
i = 0
|
|
371
|
+
while i < len(sql):
|
|
372
|
+
ch = sql[i]
|
|
373
|
+
if ch == "'":
|
|
374
|
+
if in_quote and i + 1 < len(sql) and sql[i + 1] == "'":
|
|
375
|
+
buf.append("''")
|
|
376
|
+
i += 2
|
|
377
|
+
continue
|
|
378
|
+
in_quote = not in_quote
|
|
379
|
+
buf.append(ch)
|
|
380
|
+
elif ch == ";" and not in_quote:
|
|
381
|
+
parts.append("".join(buf))
|
|
382
|
+
buf = []
|
|
383
|
+
else:
|
|
384
|
+
buf.append(ch)
|
|
385
|
+
i += 1
|
|
386
|
+
if "".join(buf).strip():
|
|
387
|
+
parts.append("".join(buf))
|
|
388
|
+
return parts
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _dedup_enum_types(ops: list[PlannedOperation]) -> list[PlannedOperation]:
|
|
392
|
+
"""Emit each native enum's ``CREATE/DROP TYPE`` once across the plan (S2).
|
|
393
|
+
|
|
394
|
+
SQLAlchemy's offline render embeds a native enum's DDL inside EVERY
|
|
395
|
+
table op that references it (per-invoke create/drop memos — unlike
|
|
396
|
+
metadata-level ``create_all``, which memoizes per metadata), so two
|
|
397
|
+
tables sharing a Python enum each carry a verbatim copy of the
|
|
398
|
+
``CREATE TYPE`` in their add_table render; executing both is a
|
|
399
|
+
guaranteed DuplicateObject (atlas cycle-6 finding S2 — the enum
|
|
400
|
+
sibling of F1/F2's embedded-index dedup, but statement-level and
|
|
401
|
+
cross-table).
|
|
402
|
+
|
|
403
|
+
Statement-level: only VERBATIM duplicates (whitespace-normalized)
|
|
404
|
+
are dropped — later byte-identical copies leave their op, the first
|
|
405
|
+
copy stays embedded in its original add_table render. A metadata
|
|
406
|
+
that defines one type name two DIFFERENT ways keeps both statements
|
|
407
|
+
and still fails loudly at apply time: silent first-win would mask a
|
|
408
|
+
genuine contradiction. Ops whose statements all survive pass
|
|
409
|
+
through as the ORIGINAL object (identity) — a re-joined copy of
|
|
410
|
+
unchanged SQL buys nothing.
|
|
411
|
+
"""
|
|
412
|
+
seen: set[str] = set()
|
|
413
|
+
|
|
414
|
+
def _is_type_stmt(norm: str) -> bool:
|
|
415
|
+
low = norm.lower()
|
|
416
|
+
return low.startswith(("create type ", "drop type "))
|
|
417
|
+
|
|
418
|
+
out: list[PlannedOperation] = []
|
|
419
|
+
for op in ops:
|
|
420
|
+
if "type" not in op.sql.lower() or ";" not in op.sql:
|
|
421
|
+
out.append(op)
|
|
422
|
+
continue
|
|
423
|
+
stmts = _split_statements(op.sql)
|
|
424
|
+
kept: list[str] = []
|
|
425
|
+
dropped = False
|
|
426
|
+
for stmt in stmts:
|
|
427
|
+
norm = " ".join(stmt.split())
|
|
428
|
+
if _is_type_stmt(norm):
|
|
429
|
+
if norm in seen:
|
|
430
|
+
dropped = True
|
|
431
|
+
continue
|
|
432
|
+
seen.add(norm)
|
|
433
|
+
if norm:
|
|
434
|
+
kept.append(stmt)
|
|
435
|
+
if not kept:
|
|
436
|
+
# unreachable for table ops (their CREATE/DROP TABLE always
|
|
437
|
+
# survives); guard anyway — an empty op.sql is worse than
|
|
438
|
+
# the duplicate it replaced
|
|
439
|
+
continue
|
|
440
|
+
if not dropped:
|
|
441
|
+
out.append(op)
|
|
442
|
+
continue
|
|
443
|
+
out.append(
|
|
444
|
+
PlannedOperation(
|
|
445
|
+
type=op.type,
|
|
446
|
+
risk=op.risk,
|
|
447
|
+
sql=";\n\n".join(stmt.strip() for stmt in kept if stmt.strip()),
|
|
448
|
+
table=op.table,
|
|
449
|
+
column=op.column,
|
|
450
|
+
)
|
|
451
|
+
)
|
|
452
|
+
return out
|
|
453
|
+
|
|
454
|
+
|
|
319
455
|
def _render_op_sql(op, engine: Engine) -> str:
|
|
320
456
|
buf = io.StringIO()
|
|
321
457
|
offline = MigrationContext.configure(
|
|
@@ -379,6 +515,7 @@ class DiffEngine:
|
|
|
379
515
|
for op in _flatten(script.upgrade_ops.ops):
|
|
380
516
|
ops.extend(self._translate(op, engine, exclude))
|
|
381
517
|
ops = _dedup_embedded_indexes(ops)
|
|
518
|
+
ops = _dedup_enum_types(ops)
|
|
382
519
|
return Plan(operations=tuple(ops))
|
|
383
520
|
|
|
384
521
|
def _translate(self, op, engine: Engine, exclude: tuple[str, ...]) -> list[PlannedOperation]:
|
sqlpush/directives/timescale.py
CHANGED
|
@@ -5,7 +5,7 @@ from sqlalchemy import MetaData, text
|
|
|
5
5
|
from sqlalchemy.engine import Connection, Engine
|
|
6
6
|
from sqlalchemy.exc import ProgrammingError
|
|
7
7
|
|
|
8
|
-
from sqlpush.annotations import HYPERTABLE_KEY
|
|
8
|
+
from sqlpush.annotations import HYPERTABLE_KEY, create_hypertable_sql
|
|
9
9
|
from sqlpush.types import PlannedOperation, RiskClass
|
|
10
10
|
|
|
11
11
|
|
|
@@ -29,12 +29,6 @@ def _is_hypertable(conn: Connection, schema: str, table_name: str) -> bool:
|
|
|
29
29
|
return False
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
def _lit(value: str) -> str:
|
|
33
|
-
# single-quoted SQL literal escaping: ' -> '' (defense in depth
|
|
34
|
-
# against quote break-out via table/column names from user metadata)
|
|
35
|
-
return value.replace("'", "''")
|
|
36
|
-
|
|
37
|
-
|
|
38
32
|
def hypertable_operations(
|
|
39
33
|
metadata: MetaData, engine: Engine | None = None
|
|
40
34
|
) -> list[PlannedOperation]:
|
|
@@ -58,31 +52,14 @@ def hypertable_operations(
|
|
|
58
52
|
ops: list[PlannedOperation] = []
|
|
59
53
|
for table in pending:
|
|
60
54
|
info = table.info[HYPERTABLE_KEY]
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
# public.<name> (UndefinedTable). Schema-less tables keep the
|
|
65
|
-
# bare name: they live in the default schema, which the
|
|
66
|
-
# search_path already resolves.
|
|
67
|
-
relation = table.name if table.schema is None else f"{table.schema}.{table.name}"
|
|
68
|
-
name = _lit(relation)
|
|
69
|
-
time_column = _lit(info.time_column)
|
|
70
|
-
parts = [f"SELECT create_hypertable('{name}', '{time_column}'"]
|
|
71
|
-
if info.chunk_time_interval:
|
|
72
|
-
parts.append(f", chunk_time_interval => INTERVAL '{_lit(info.chunk_time_interval)}'")
|
|
73
|
-
parts.append(", migrate_data => true")
|
|
74
|
-
# if_not_exists: race insurance between the state check above
|
|
75
|
-
# and the apply. create_default_indexes=false: timescale's
|
|
76
|
-
# implicit time-column index is invisible to metadata, so the
|
|
77
|
-
# default would drift forever (destructive drop_index) on a
|
|
78
|
-
# fully synced schema; users declare wanted indexes in the
|
|
79
|
-
# Table instead (metadata is the source of truth).
|
|
80
|
-
parts.append(", if_not_exists => true, create_default_indexes => false)")
|
|
55
|
+
# SQL parity with the decorator's after_create listener is by
|
|
56
|
+
# construction: both render through create_hypertable_sql (the
|
|
57
|
+
# directive's render adds the statement terminator).
|
|
81
58
|
ops.append(
|
|
82
59
|
PlannedOperation(
|
|
83
60
|
type="create_hypertable",
|
|
84
61
|
risk=RiskClass.SAFE,
|
|
85
|
-
sql=
|
|
62
|
+
sql=create_hypertable_sql(table.name, table.schema, info) + ";",
|
|
86
63
|
table=table.name,
|
|
87
64
|
)
|
|
88
65
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sqlpush
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models
|
|
5
5
|
Keywords: sqlalchemy,alembic,postgresql,timescaledb,prisma,schema,migrations,database,drift,cli
|
|
6
6
|
Author: Juan Miguel Contreras
|
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
sqlpush/__init__.py,sha256=Px0jJFHbwwM1-UHtDwD3xQlRifEMW4StSXQs5DD9z6w,1697
|
|
2
|
-
sqlpush/annotations.py,sha256=
|
|
3
|
-
sqlpush/api.py,sha256=
|
|
2
|
+
sqlpush/annotations.py,sha256=xe5EoYPvraLJy-WvMkFK_zOIKquB3-VsrAzy6HE_hiA,4652
|
|
3
|
+
sqlpush/api.py,sha256=7fHsy1AHd2ffca-wtQDowfpt70P7c6L0_s720k8gRcU,10166
|
|
4
4
|
sqlpush/apply/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
sqlpush/apply/executor.py,sha256=-HgmscUO6M_xQ5HPBQhJkUo6XwEPPqBig7_G9aWQJAU,7242
|
|
6
6
|
sqlpush/chain/__init__.py,sha256=AJ96P3I63mX6-uMuK0paCpKtaRexufnCqjUuElbyAsM,293
|
|
7
7
|
sqlpush/chain/format.py,sha256=uj0iMQfFWa9HpKhrokfJu_5B_FesiD4qCZxAGDJ3bfQ,4938
|
|
8
|
-
sqlpush/chain/migrate.py,sha256=
|
|
9
|
-
sqlpush/cli.py,sha256=
|
|
8
|
+
sqlpush/chain/migrate.py,sha256=P7I07Zv3xRmRYgFTTLAqkrUdbjsGOGNQZG2j-eP5Hgk,10055
|
|
9
|
+
sqlpush/cli.py,sha256=7TBKMxmvkpwSH6gyqs2fDQ5Tyec1yMOLGJCUJb-2Af0,10599
|
|
10
10
|
sqlpush/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
11
|
sqlpush/core/classify.py,sha256=-HNPQ1Kfnr0MMYLoP26_VC14JjcQHravrbT7lPGqlIU,927
|
|
12
|
-
sqlpush/core/diff.py,sha256=
|
|
12
|
+
sqlpush/core/diff.py,sha256=q0f7NtiKQ8N33UbI40ttxgiNbsAsBpqa0XJ15oTLSBY,24147
|
|
13
13
|
sqlpush/core/render.py,sha256=9SNEyHOGmMxKUfIjvo3x0MNVQ6N2o2fl3noWT1EWO3M,609
|
|
14
14
|
sqlpush/directives/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
-
sqlpush/directives/timescale.py,sha256=
|
|
15
|
+
sqlpush/directives/timescale.py,sha256=W_Cd_stdhKZlzlMum7Bdzq99VfXOulYxq0B-sukz-xE,2645
|
|
16
16
|
sqlpush/types.py,sha256=IMA1HRLIumat7tpt8Yscv-9g2U5U_--xxnfcf_pfkyo,2938
|
|
17
|
-
sqlpush-0.4.
|
|
18
|
-
sqlpush-0.4.
|
|
19
|
-
sqlpush-0.4.
|
|
20
|
-
sqlpush-0.4.
|
|
17
|
+
sqlpush-0.4.2.dist-info/WHEEL,sha256=y6e-a5KI2W-qDAJKfh9Xr81bin8NgUdfWgz3VSXXpe4,80
|
|
18
|
+
sqlpush-0.4.2.dist-info/entry_points.txt,sha256=VYha3PJSvklRgpcDAZYJdE66EXM79MSKQf0z8igyAOI,46
|
|
19
|
+
sqlpush-0.4.2.dist-info/METADATA,sha256=KwaHMTxd1Upr_Mf2GbulnbnY3MY58NfBLNBacSNQjWE,8373
|
|
20
|
+
sqlpush-0.4.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|