sqlpush 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sqlpush-0.2.0 → sqlpush-0.4.0}/PKG-INFO +1 -1
- {sqlpush-0.2.0 → sqlpush-0.4.0}/pyproject.toml +1 -1
- {sqlpush-0.2.0 → sqlpush-0.4.0}/pyproject.toml.orig +1 -1
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/api.py +83 -0
- sqlpush-0.4.0/src/sqlpush/chain/__init__.py +6 -0
- sqlpush-0.4.0/src/sqlpush/chain/format.py +142 -0
- sqlpush-0.4.0/src/sqlpush/chain/migrate.py +175 -0
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/cli.py +82 -1
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/core/classify.py +7 -3
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/core/diff.py +44 -1
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/directives/timescale.py +15 -5
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/types.py +12 -0
- {sqlpush-0.2.0 → sqlpush-0.4.0}/README.md +0 -0
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/__init__.py +0 -0
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/annotations.py +0 -0
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/apply/__init__.py +0 -0
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/apply/executor.py +0 -0
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/core/__init__.py +0 -0
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/core/render.py +0 -0
- {sqlpush-0.2.0 → sqlpush-0.4.0}/src/sqlpush/directives/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sqlpush
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models
|
|
5
5
|
Keywords: sqlalchemy,alembic,postgresql,timescaledb,prisma,schema,migrations,database,drift,cli
|
|
6
6
|
Author: Juan Miguel Contreras
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sqlpush"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
description = "Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sqlpush"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
description = "Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import asyncio
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
5
7
|
from typing import NoReturn
|
|
6
8
|
|
|
7
9
|
from sqlalchemy import MetaData
|
|
@@ -14,11 +16,18 @@ from sqlalchemy.exc import (
|
|
|
14
16
|
from sqlalchemy.ext.asyncio import AsyncEngine
|
|
15
17
|
|
|
16
18
|
from sqlpush.apply.executor import apply_plan, with_advisory_lock
|
|
19
|
+
from sqlpush.chain.format import (
|
|
20
|
+
RISK_RANK,
|
|
21
|
+
next_revision_id,
|
|
22
|
+
render_migration_file,
|
|
23
|
+
)
|
|
24
|
+
from sqlpush.chain.migrate import run_migrate, run_stamp
|
|
17
25
|
from sqlpush.core.diff import DiffEngine
|
|
18
26
|
from sqlpush.directives.timescale import hypertable_operations
|
|
19
27
|
from sqlpush.types import (
|
|
20
28
|
CheckResult,
|
|
21
29
|
ConnectFailed,
|
|
30
|
+
MigrateReport,
|
|
22
31
|
Plan,
|
|
23
32
|
Report,
|
|
24
33
|
SqlpushError,
|
|
@@ -116,6 +125,80 @@ def check(metadata, engine, *, schemas=None, exclude=()) -> CheckResult:
|
|
|
116
125
|
)
|
|
117
126
|
|
|
118
127
|
|
|
128
|
+
def revision(
|
|
129
|
+
metadata, ref_engine, *, out_dir="migrations/versions", message=None, schemas=None, exclude=()
|
|
130
|
+
) -> Path:
|
|
131
|
+
"""Generate the next annotated-SQL migration file from models-vs-ref drift.
|
|
132
|
+
|
|
133
|
+
The reference DB must sit at the chain head (caller-provided — sqlpush
|
|
134
|
+
stays docker-free). Empty drift refuses loudly: no empty files.
|
|
135
|
+
"""
|
|
136
|
+
p = plan(metadata, ref_engine, schemas=schemas, exclude=exclude)
|
|
137
|
+
if not p.operations:
|
|
138
|
+
raise SqlpushError("no drift between models and reference DB — nothing to revise")
|
|
139
|
+
risk = max((op.risk for op in p.operations), key=lambda r: RISK_RANK[r])
|
|
140
|
+
ops = [(f"[{op.risk.name}] {op.type} {op.table or '?'}", op.sql) for op in p.operations]
|
|
141
|
+
out = Path(out_dir)
|
|
142
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
143
|
+
rev_id = next_revision_id(out)
|
|
144
|
+
slug = re.sub(r"[^a-z0-9_]+", "_", (message or "migration").lower())[:40]
|
|
145
|
+
path = out / f"{rev_id}_{slug}.sql"
|
|
146
|
+
if path.exists():
|
|
147
|
+
# NNNN+slug collision (e.g. same message re-run without the file being
|
|
148
|
+
# consumed) must be loud, never a silent overwrite of chain history
|
|
149
|
+
raise SqlpushError(f"refusing to overwrite existing migration file {path}")
|
|
150
|
+
prev_n = int(rev_id) - 1
|
|
151
|
+
try:
|
|
152
|
+
path.write_text(
|
|
153
|
+
render_migration_file(
|
|
154
|
+
ops=ops,
|
|
155
|
+
revision_id=rev_id,
|
|
156
|
+
risk=risk,
|
|
157
|
+
message=message,
|
|
158
|
+
parent=f"{prev_n:04d}" if prev_n >= 1 else None,
|
|
159
|
+
)
|
|
160
|
+
)
|
|
161
|
+
except OSError as exc:
|
|
162
|
+
raise SqlpushError(f"cannot write migration file {path}: {exc}") from exc
|
|
163
|
+
return path
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def migrate(target, *, chain_dir="migrations/versions", allow_destructive=False) -> MigrateReport:
|
|
167
|
+
"""Replay annotated-SQL chain files with gates + same-txn bookkeeping.
|
|
168
|
+
|
|
169
|
+
``target`` is a DSN string, sync ``Engine`` or ``AsyncEngine`` (resolved
|
|
170
|
+
via ``_sync_engine_from``; engines created here are disposed). See
|
|
171
|
+
``chain.migrate.run_migrate`` for the execution contract.
|
|
172
|
+
"""
|
|
173
|
+
engine, dispose = _sync_engine_from(target)
|
|
174
|
+
try:
|
|
175
|
+
return run_migrate(engine, chain_dir=chain_dir, allow_destructive=allow_destructive)
|
|
176
|
+
except SQLAlchemyError as exc:
|
|
177
|
+
# MigrationFileError/SqlpushError (typed) pass through untouched
|
|
178
|
+
_raise_typed(exc)
|
|
179
|
+
finally:
|
|
180
|
+
if dispose:
|
|
181
|
+
engine.dispose()
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def stamp(target, *, chain_dir="migrations/versions") -> MigrateReport:
|
|
185
|
+
"""Bootstrap: register chain files as applied WITHOUT executing SQL.
|
|
186
|
+
|
|
187
|
+
For adopting a DB whose schema already reflects the chain. ``target``
|
|
188
|
+
resolution and error typing match :func:`migrate`. See
|
|
189
|
+
``chain.migrate.run_stamp`` for the report convention.
|
|
190
|
+
"""
|
|
191
|
+
engine, dispose = _sync_engine_from(target)
|
|
192
|
+
try:
|
|
193
|
+
return run_stamp(engine, chain_dir=chain_dir)
|
|
194
|
+
except SQLAlchemyError as exc:
|
|
195
|
+
# MigrationFileError/SqlpushError (typed) pass through untouched
|
|
196
|
+
_raise_typed(exc)
|
|
197
|
+
finally:
|
|
198
|
+
if dispose:
|
|
199
|
+
engine.dispose()
|
|
200
|
+
|
|
201
|
+
|
|
119
202
|
def _sync_engine_from(target):
|
|
120
203
|
if isinstance(target, AsyncEngine):
|
|
121
204
|
from sqlalchemy import create_engine
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# re-export for downstream consumers (stamp verb): canonical home is
|
|
2
|
+
# chain.format. NOT re-exported from sqlpush.types — types is imported BY
|
|
3
|
+
# chain.format, so a types-level re-export would be circular.
|
|
4
|
+
from sqlpush.chain.format import MigrationFileError
|
|
5
|
+
|
|
6
|
+
__all__ = ["MigrationFileError"]
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Annotated-SQL migration files: the chain format (spec 2026-09-02 §4).
|
|
2
|
+
|
|
3
|
+
Files are plain editable SQL with a structured header line that carries
|
|
4
|
+
the risk classification. Labels are legal SQL comments, so generated
|
|
5
|
+
files are directly psql-runnable. Parsing is FAIL-LOUD: a missing or
|
|
6
|
+
malformed header refuses the file — never assume-safe.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import re
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from sqlpush.types import RiskClass, SqlpushError
|
|
17
|
+
|
|
18
|
+
RISK_RANK: dict[RiskClass, int] = {
|
|
19
|
+
RiskClass.SAFE: 0,
|
|
20
|
+
RiskClass.RISKY: 1,
|
|
21
|
+
RiskClass.DESTRUCTIVE: 2,
|
|
22
|
+
}
|
|
23
|
+
"""Explicit ordering — RiskClass is a plain str-Enum with no < (spec P5)."""
|
|
24
|
+
|
|
25
|
+
# Header line: `-- sqlpush: revision=0007 risk=DESTRUCTIVE ops=3`. Keys other
|
|
26
|
+
# than revision/risk (e.g. ops=, parent=, generated=) are tolerated — the
|
|
27
|
+
# writer may add informative keys, the parser only requires the fail-loud two.
|
|
28
|
+
_HEADER_LINE_RE = re.compile(r"^--\s*sqlpush:\s*(?P<body>.*)$")
|
|
29
|
+
_OP_RE = re.compile(r"^--\s*op\s+\d+\s+\[(?P<label>[^\]]*)\]\s*(?P<desc>.*)$")
|
|
30
|
+
_REV_PREFIX_RE = re.compile(r"^(\d+)_")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class MigrationFileError(SqlpushError):
|
|
34
|
+
"""A chain file is missing/malformed — fail-loud, never assume-safe."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class MigrationFile:
|
|
39
|
+
name: str
|
|
40
|
+
revision_id: str
|
|
41
|
+
risk: RiskClass
|
|
42
|
+
ops: list[tuple[str, str]] = field(default_factory=list)
|
|
43
|
+
text: str = ""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def checksum(text: str) -> str:
|
|
47
|
+
normalized = text.replace("\r\n", "\n").rstrip()
|
|
48
|
+
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _parse_header(text: str, name: str) -> tuple[str, RiskClass]:
|
|
52
|
+
for line in text.splitlines():
|
|
53
|
+
m = _HEADER_LINE_RE.match(line.strip())
|
|
54
|
+
if m:
|
|
55
|
+
break
|
|
56
|
+
else:
|
|
57
|
+
raise MigrationFileError(f"{name}: missing -- sqlpush: header")
|
|
58
|
+
body = m.group("body").strip()
|
|
59
|
+
if not body:
|
|
60
|
+
raise MigrationFileError(f"{name}: malformed -- sqlpush: header (empty)")
|
|
61
|
+
pairs: dict[str, str] = {}
|
|
62
|
+
for token in body.split():
|
|
63
|
+
key, sep, value = token.partition("=")
|
|
64
|
+
if not sep or not key or not value:
|
|
65
|
+
raise MigrationFileError(f"{name}: malformed -- sqlpush: header token {token!r}")
|
|
66
|
+
pairs[key] = value
|
|
67
|
+
revision = pairs.get("revision", "")
|
|
68
|
+
if not revision.isdigit():
|
|
69
|
+
raise MigrationFileError(f"{name}: malformed -- sqlpush: header: revision=NNNN required")
|
|
70
|
+
risk_name = pairs.get("risk")
|
|
71
|
+
if risk_name is None:
|
|
72
|
+
raise MigrationFileError(
|
|
73
|
+
f"{name}: malformed -- sqlpush: header: risk= missing (expected SAFE|RISKY|DESTRUCTIVE)"
|
|
74
|
+
)
|
|
75
|
+
try:
|
|
76
|
+
risk = RiskClass[risk_name]
|
|
77
|
+
except KeyError:
|
|
78
|
+
raise MigrationFileError(
|
|
79
|
+
f"{name}: malformed -- sqlpush: header: unknown risk={risk_name!r} "
|
|
80
|
+
f"(expected SAFE|RISKY|DESTRUCTIVE)"
|
|
81
|
+
) from None
|
|
82
|
+
return revision, risk
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def render_migration_file(
|
|
86
|
+
*,
|
|
87
|
+
ops: list[tuple[str, str]],
|
|
88
|
+
revision_id: str,
|
|
89
|
+
risk: RiskClass,
|
|
90
|
+
message: str | None = None,
|
|
91
|
+
parent: str | None = None,
|
|
92
|
+
) -> str:
|
|
93
|
+
lines = [f"-- sqlpush: revision={revision_id} risk={risk.name}"]
|
|
94
|
+
# parent= is informative-only: the parser never requires it (hand-written
|
|
95
|
+
# files omit it freely), but the writer always records the slot.
|
|
96
|
+
lines.append(f"-- parent={parent or ''}")
|
|
97
|
+
if message:
|
|
98
|
+
lines.append(f"-- {message}")
|
|
99
|
+
lines.append("")
|
|
100
|
+
for i, (label, sql) in enumerate(ops, start=1):
|
|
101
|
+
lines.append(f"-- op {i} {label}")
|
|
102
|
+
lines.append(sql.rstrip(";") + ";")
|
|
103
|
+
lines.append("")
|
|
104
|
+
return "\n".join(lines).rstrip() + "\n"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def parse_migration_file(text: str, *, name: str) -> MigrationFile:
|
|
108
|
+
revision, risk = _parse_header(text, name)
|
|
109
|
+
|
|
110
|
+
ops: list[tuple[str, str]] = []
|
|
111
|
+
sql_buf: list[str] = []
|
|
112
|
+
current_label = ""
|
|
113
|
+
for line in text.splitlines():
|
|
114
|
+
stripped = line.strip()
|
|
115
|
+
if stripped.startswith("-- sqlpush:"):
|
|
116
|
+
continue
|
|
117
|
+
op_m = _OP_RE.match(stripped)
|
|
118
|
+
if op_m:
|
|
119
|
+
if sql_buf:
|
|
120
|
+
ops.append((current_label, "\n".join(sql_buf).strip()))
|
|
121
|
+
sql_buf = []
|
|
122
|
+
current_label = f"[{op_m.group('label')}] {op_m.group('desc')}".strip()
|
|
123
|
+
continue
|
|
124
|
+
if stripped.startswith("--"):
|
|
125
|
+
continue # comentarios libres (message, parent, generated...) — ignorable
|
|
126
|
+
if stripped:
|
|
127
|
+
sql_buf.append(line)
|
|
128
|
+
if sql_buf:
|
|
129
|
+
ops.append((current_label, "\n".join(sql_buf).strip()))
|
|
130
|
+
if not ops:
|
|
131
|
+
raise MigrationFileError(f"{name}: file has no SQL")
|
|
132
|
+
|
|
133
|
+
return MigrationFile(name=name, revision_id=revision, risk=risk, ops=ops, text=text)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def next_revision_id(chain_dir: Path | str) -> str:
|
|
137
|
+
max_n = 0
|
|
138
|
+
for f in sorted(Path(chain_dir).glob("*.sql")):
|
|
139
|
+
m = _REV_PREFIX_RE.match(f.name)
|
|
140
|
+
if m:
|
|
141
|
+
max_n = max(max_n, int(m.group(1)))
|
|
142
|
+
return f"{max_n + 1:04d}"
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""migrate/stamp: chain replay + bootstrap with gates and bookkeeping.
|
|
2
|
+
|
|
3
|
+
migrate execution contract (spec 2026-09-02 §5): each file's WHOLE TEXT is
|
|
4
|
+
replayed in ONE ``exec_driver_sql()`` call inside a per-file transaction —
|
|
5
|
+
the parsed ``ops`` are for gating/display only, NEVER reconstructed for
|
|
6
|
+
execution (psycopg3 accepts multi-statement strings; nothing is tokenized,
|
|
7
|
+
so dollar-quoted bodies are safe). The checksum row is inserted INSIDE the
|
|
8
|
+
same transaction as the file's SQL — bookkeeping in a separate txn would
|
|
9
|
+
let a crash between apply and registry re-apply the file forever
|
|
10
|
+
(crash-loop over existing objects).
|
|
11
|
+
|
|
12
|
+
Fail-loud ordering: any blocked file (parse error, checksum mismatch,
|
|
13
|
+
destructive gate, SQL failure) stops the chain — nothing later runs (R4).
|
|
14
|
+
A hung migrate (stuck waiting on the advisory lock) is diagnosed via
|
|
15
|
+
pg_locks / pg_stat_activity.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import contextlib
|
|
21
|
+
from collections.abc import Iterator
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
from sqlalchemy import text
|
|
25
|
+
from sqlalchemy.engine import Connection, Engine
|
|
26
|
+
|
|
27
|
+
from sqlpush.apply.executor import advisory_key
|
|
28
|
+
from sqlpush.chain.format import MigrationFileError, checksum, parse_migration_file
|
|
29
|
+
from sqlpush.types import MigrateReport, RiskClass
|
|
30
|
+
|
|
31
|
+
_VERSIONS_DDL = (
|
|
32
|
+
"CREATE TABLE IF NOT EXISTS public.sqlpush_versions ("
|
|
33
|
+
"name TEXT PRIMARY KEY, sha256 TEXT NOT NULL, "
|
|
34
|
+
"applied_at timestamptz NOT NULL DEFAULT now())"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _chain_files(chain_dir: str | Path) -> list[Path]:
|
|
39
|
+
chain_path = Path(chain_dir)
|
|
40
|
+
if not chain_path.is_dir():
|
|
41
|
+
# a typo'd --dir silently no-op'ing is the footgun; an empty-but-
|
|
42
|
+
# EXISTING dir is a legitimate idle run (versions table still ensured)
|
|
43
|
+
raise MigrationFileError(f"chain dir not found: {chain_path}")
|
|
44
|
+
return sorted(chain_path.glob("*.sql"))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@contextlib.contextmanager
|
|
48
|
+
def _chain_session(engine: Engine) -> Iterator[Connection]:
|
|
49
|
+
"""Session-scoped advisory lock + versions table, shared by every verb.
|
|
50
|
+
|
|
51
|
+
Same key derivation as push (fnv1a_32(b"sqlpush") ^ db oid): serializes
|
|
52
|
+
concurrent chain workers and excludes push on the same database. The
|
|
53
|
+
txn opened by the key query is committed right away — session advisory
|
|
54
|
+
locks survive COMMIT/ROLLBACK, and an idle-in-transaction session is
|
|
55
|
+
exposed to idle_in_transaction_session_timeout (executor.py note).
|
|
56
|
+
"""
|
|
57
|
+
with engine.connect() as conn:
|
|
58
|
+
key = advisory_key(conn)
|
|
59
|
+
conn.execute(text("SELECT pg_advisory_lock(:k)"), {"k": key})
|
|
60
|
+
conn.commit()
|
|
61
|
+
try:
|
|
62
|
+
conn.execute(text(_VERSIONS_DDL))
|
|
63
|
+
conn.commit()
|
|
64
|
+
yield conn
|
|
65
|
+
finally:
|
|
66
|
+
# best-effort unlock (executor pattern): a secondary failure here
|
|
67
|
+
# must never mask the primary outcome; the session lock dies with
|
|
68
|
+
# the connection even if this fails
|
|
69
|
+
if not conn.closed:
|
|
70
|
+
with contextlib.suppress(Exception):
|
|
71
|
+
conn.rollback()
|
|
72
|
+
conn.execute(text("SELECT pg_advisory_unlock(:k)"), {"k": key})
|
|
73
|
+
conn.commit()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def run_migrate(engine: Engine, *, chain_dir: str | Path, allow_destructive: bool) -> MigrateReport:
|
|
77
|
+
applied: list[str] = []
|
|
78
|
+
skipped: list[str] = []
|
|
79
|
+
blocked: list[str] = []
|
|
80
|
+
notes: list[str] = []
|
|
81
|
+
partial = False
|
|
82
|
+
chain = _chain_files(chain_dir)
|
|
83
|
+
with _chain_session(engine) as conn:
|
|
84
|
+
recorded = {
|
|
85
|
+
row[0]: row[1]
|
|
86
|
+
for row in conn.execute(text("SELECT name, sha256 FROM public.sqlpush_versions"))
|
|
87
|
+
}
|
|
88
|
+
conn.commit()
|
|
89
|
+
for f in chain:
|
|
90
|
+
raw = f.read_text()
|
|
91
|
+
try:
|
|
92
|
+
mf = parse_migration_file(raw, name=f.name)
|
|
93
|
+
except MigrationFileError as exc:
|
|
94
|
+
blocked.append(f.name)
|
|
95
|
+
notes.append(f"{f.name}: {exc}")
|
|
96
|
+
break # orden estricto: nada posterior corre
|
|
97
|
+
if f.name in recorded:
|
|
98
|
+
if recorded[f.name] != checksum(raw):
|
|
99
|
+
blocked.append(f.name)
|
|
100
|
+
notes.append(f"{f.name}: checksum mismatch (edited after apply?)")
|
|
101
|
+
break
|
|
102
|
+
skipped.append(f.name)
|
|
103
|
+
continue
|
|
104
|
+
if mf.risk is RiskClass.DESTRUCTIVE and not allow_destructive:
|
|
105
|
+
blocked.append(f.name)
|
|
106
|
+
notes.append(f"{f.name}: DESTRUCTIVE requires --allow-destructive")
|
|
107
|
+
break
|
|
108
|
+
try:
|
|
109
|
+
with conn.begin():
|
|
110
|
+
# whole-file replay: exec_driver_sql bypasses text()'s
|
|
111
|
+
# bind-param parsing entirely — ":casts" and ":=" in
|
|
112
|
+
# hand-edited SQL must reach the server verbatim
|
|
113
|
+
conn.exec_driver_sql(raw)
|
|
114
|
+
conn.execute(
|
|
115
|
+
text("INSERT INTO public.sqlpush_versions (name, sha256) VALUES (:n, :s)"),
|
|
116
|
+
{"n": f.name, "s": checksum(raw)},
|
|
117
|
+
)
|
|
118
|
+
applied.append(f.name)
|
|
119
|
+
except Exception as exc: # noqa: BLE001 — report, no mask
|
|
120
|
+
blocked.append(f.name)
|
|
121
|
+
notes.append(f"{f.name}: {exc}")
|
|
122
|
+
partial = True
|
|
123
|
+
break
|
|
124
|
+
return MigrateReport(
|
|
125
|
+
applied=tuple(applied),
|
|
126
|
+
skipped=tuple(skipped),
|
|
127
|
+
blocked=tuple(blocked),
|
|
128
|
+
partial_failure=partial,
|
|
129
|
+
notes=tuple(notes),
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def run_stamp(engine: Engine, *, chain_dir: str | Path) -> MigrateReport:
|
|
134
|
+
"""Register every parseable chain file WITHOUT executing any SQL.
|
|
135
|
+
|
|
136
|
+
Bootstrap seam (spec §5 R7): adopt a DB whose schema already reflects
|
|
137
|
+
the chain. Only the header must parse (fail-loud on THAT) — invalid SQL
|
|
138
|
+
in a body still registers, because stamp never executes anything. Each
|
|
139
|
+
registration is an idempotent upsert (``ON CONFLICT (name) DO UPDATE``),
|
|
140
|
+
so re-stamping refreshes checksums instead of failing.
|
|
141
|
+
|
|
142
|
+
Report convention: registered files are listed in ``skipped`` (stamp
|
|
143
|
+
never populates ``applied`` and never sets ``partial_failure``); a
|
|
144
|
+
header that fails to parse goes to ``blocked`` + ``notes`` and stops
|
|
145
|
+
the walk (strict order, same as migrate).
|
|
146
|
+
"""
|
|
147
|
+
skipped: list[str] = []
|
|
148
|
+
blocked: list[str] = []
|
|
149
|
+
notes: list[str] = []
|
|
150
|
+
chain = _chain_files(chain_dir)
|
|
151
|
+
with _chain_session(engine) as conn:
|
|
152
|
+
for f in chain:
|
|
153
|
+
raw = f.read_text()
|
|
154
|
+
try:
|
|
155
|
+
parse_migration_file(raw, name=f.name)
|
|
156
|
+
except MigrationFileError as exc:
|
|
157
|
+
blocked.append(f.name)
|
|
158
|
+
notes.append(f"{f.name}: {exc}")
|
|
159
|
+
break # orden estricto: nada posterior se registra
|
|
160
|
+
with conn.begin():
|
|
161
|
+
conn.execute(
|
|
162
|
+
text(
|
|
163
|
+
"INSERT INTO public.sqlpush_versions (name, sha256) VALUES (:n, :s) "
|
|
164
|
+
"ON CONFLICT (name) DO UPDATE SET sha256 = EXCLUDED.sha256"
|
|
165
|
+
),
|
|
166
|
+
{"n": f.name, "s": checksum(raw)},
|
|
167
|
+
)
|
|
168
|
+
skipped.append(f.name)
|
|
169
|
+
return MigrateReport(
|
|
170
|
+
applied=(),
|
|
171
|
+
skipped=tuple(skipped),
|
|
172
|
+
blocked=tuple(blocked),
|
|
173
|
+
partial_failure=False,
|
|
174
|
+
notes=tuple(notes),
|
|
175
|
+
)
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
"""sqlpush: diff/push/check PostgreSQL schema drift from SQLAlchemy models.
|
|
3
3
|
|
|
4
4
|
Exit codes: diff always 0; check 0 clean / 2 drift / 3 destructive drift;
|
|
5
|
-
push 0 applied / 1 destructive blocked / 2 error (incl. partial failure)
|
|
5
|
+
push 0 applied / 1 destructive blocked / 2 error (incl. partial failure);
|
|
6
|
+
revision 0 written / 1 error (empty drift refuses); migrate 0 clean /
|
|
7
|
+
1 blocked or partial failure.
|
|
6
8
|
"""
|
|
7
9
|
|
|
8
10
|
from __future__ import annotations
|
|
@@ -11,6 +13,7 @@ import importlib
|
|
|
11
13
|
import json
|
|
12
14
|
import os
|
|
13
15
|
import sys
|
|
16
|
+
from pathlib import Path
|
|
14
17
|
from typing import Annotated
|
|
15
18
|
|
|
16
19
|
import typer
|
|
@@ -33,6 +36,9 @@ app = typer.Typer(add_completion=False, help=__doc__ or "")
|
|
|
33
36
|
# default, no mutable default) while staying the idiomatic typer style.
|
|
34
37
|
SchemaOpt = Annotated[list[str] | None, typer.Option("--schema")]
|
|
35
38
|
ExcludeOpt = Annotated[list[str] | None, typer.Option("--exclude")]
|
|
39
|
+
# Path-typed options need the Annotated form: B008 (call in default) only
|
|
40
|
+
# exempts typer.Option for non-Path annotations
|
|
41
|
+
DirOpt = Annotated[Path, typer.Option("--dir")]
|
|
36
42
|
|
|
37
43
|
|
|
38
44
|
def _load_metadata(spec: str):
|
|
@@ -210,6 +216,81 @@ def push(
|
|
|
210
216
|
raise typer.Exit(code=0)
|
|
211
217
|
|
|
212
218
|
|
|
219
|
+
@app.command()
|
|
220
|
+
def revision(
|
|
221
|
+
metadata_spec: str = typer.Argument(..., help="module:metadata"),
|
|
222
|
+
ref_dsn: str = typer.Option(..., "--ref-dsn"),
|
|
223
|
+
message: str | None = typer.Option(None, "--message", "-m"),
|
|
224
|
+
out_dir: DirOpt = Path("migrations/versions"),
|
|
225
|
+
schema: SchemaOpt = None,
|
|
226
|
+
exclude: ExcludeOpt = None,
|
|
227
|
+
):
|
|
228
|
+
"""Generate the next migration file from models vs the reference DB."""
|
|
229
|
+
md = _load_metadata(metadata_spec)
|
|
230
|
+
# required --ref-dsn (no DATABASE_URL fallback): the reference DB is a
|
|
231
|
+
# different database from the push target — conflating them silently
|
|
232
|
+
# would chain against the wrong head
|
|
233
|
+
engine = _engine(ref_dsn)
|
|
234
|
+
try:
|
|
235
|
+
path = api.revision(
|
|
236
|
+
md,
|
|
237
|
+
engine,
|
|
238
|
+
out_dir=out_dir,
|
|
239
|
+
message=message,
|
|
240
|
+
schemas=schema,
|
|
241
|
+
exclude=exclude or (),
|
|
242
|
+
)
|
|
243
|
+
finally:
|
|
244
|
+
engine.dispose()
|
|
245
|
+
typer.echo(str(path))
|
|
246
|
+
raise typer.Exit(code=0)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
@app.command()
|
|
250
|
+
def migrate(
|
|
251
|
+
dsn: str | None = typer.Option(None),
|
|
252
|
+
allow_destructive: bool = typer.Option(False, "--allow-destructive"),
|
|
253
|
+
out_dir: DirOpt = Path("migrations/versions"),
|
|
254
|
+
):
|
|
255
|
+
"""Replay pending migration files (gates + checksum bookkeeping)."""
|
|
256
|
+
engine = _engine(dsn)
|
|
257
|
+
try:
|
|
258
|
+
report = api.migrate(engine, chain_dir=out_dir, allow_destructive=allow_destructive)
|
|
259
|
+
finally:
|
|
260
|
+
engine.dispose()
|
|
261
|
+
typer.echo(
|
|
262
|
+
f"applied: {len(report.applied)}, skipped: {len(report.skipped)}, "
|
|
263
|
+
f"blocked: {len(report.blocked)}, partial_failure: {report.partial_failure}"
|
|
264
|
+
)
|
|
265
|
+
for note in report.notes:
|
|
266
|
+
typer.secho(note, fg="yellow", err=True)
|
|
267
|
+
if report.blocked or report.partial_failure:
|
|
268
|
+
raise typer.Exit(code=1)
|
|
269
|
+
raise typer.Exit(code=0)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
@app.command()
|
|
273
|
+
def stamp(
|
|
274
|
+
dsn: str | None = typer.Option(None),
|
|
275
|
+
out_dir: DirOpt = Path("migrations/versions"),
|
|
276
|
+
):
|
|
277
|
+
"""Adopt an existing DB: register chain files without executing SQL."""
|
|
278
|
+
engine = _engine(dsn)
|
|
279
|
+
try:
|
|
280
|
+
report = api.stamp(engine, chain_dir=out_dir)
|
|
281
|
+
finally:
|
|
282
|
+
engine.dispose()
|
|
283
|
+
typer.echo(
|
|
284
|
+
f"applied: {len(report.applied)}, skipped (registered): {len(report.skipped)}, "
|
|
285
|
+
f"blocked: {len(report.blocked)}, partial_failure: {report.partial_failure}"
|
|
286
|
+
)
|
|
287
|
+
for note in report.notes:
|
|
288
|
+
typer.secho(note, fg="yellow", err=True)
|
|
289
|
+
if report.blocked or report.partial_failure:
|
|
290
|
+
raise typer.Exit(code=1)
|
|
291
|
+
raise typer.Exit(code=0)
|
|
292
|
+
|
|
293
|
+
|
|
213
294
|
def main() -> None: # [project.scripts] entry point
|
|
214
295
|
try:
|
|
215
296
|
app()
|
|
@@ -8,9 +8,13 @@ DESTRUCTIVE = frozenset({"drop_column", "drop_table", "drop_index", "drop_constr
|
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def classify(op_type: str) -> RiskClass:
|
|
11
|
-
"""
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
"""add_index renders standalone: on alembic 1.19.1 even plain
|
|
12
|
+
declared indexes of NEW tables arrive standalone
|
|
13
|
+
(CreateTableOp.from_table captures columns+constraints, not
|
|
14
|
+
indexes; only instrumentation-embedded ones ride inside the
|
|
15
|
+
add_table render, and the diff dedups those away). What survives
|
|
16
|
+
runs CREATE INDEX alone — a SHARE lock that blocks writes, hence
|
|
17
|
+
risky."""
|
|
14
18
|
if op_type in DESTRUCTIVE:
|
|
15
19
|
return RiskClass.DESTRUCTIVE
|
|
16
20
|
if op_type in SAFE:
|
|
@@ -17,7 +17,10 @@ from sqlpush.core.classify import classify
|
|
|
17
17
|
from sqlpush.types import Plan, PlannedOperation
|
|
18
18
|
|
|
19
19
|
_SYSTEM_SCHEMAS = ("_timescaledb%", "information_schema", "pg_%")
|
|
20
|
-
|
|
20
|
+
# sqlpush_versions: the chain engine's own bookkeeping (migrate/stamp) —
|
|
21
|
+
# same treatment as alembic_version (C1): a public-scoped post-migrate
|
|
22
|
+
# check must be clean, not report its own registry as destructive drift
|
|
23
|
+
_SYSTEM_TABLES = ("alembic_version", "spatial_ref_sys", "sqlpush_versions")
|
|
21
24
|
|
|
22
25
|
# Leaf op class -> sqlpush op type. Class names follow alembic 1.19.1
|
|
23
26
|
# autogen output as observed in docs/notes/alembic-notes.md and probes:
|
|
@@ -274,6 +277,45 @@ def _flatten(ops):
|
|
|
274
277
|
yield op
|
|
275
278
|
|
|
276
279
|
|
|
280
|
+
def _dedup_embedded_indexes(ops: list[PlannedOperation]) -> list[PlannedOperation]:
|
|
281
|
+
"""Drop standalone ``add_index`` ops already embedded in an ``add_table`` render.
|
|
282
|
+
|
|
283
|
+
On alembic 1.19.1 ``CreateTableOp.from_table`` captures columns and
|
|
284
|
+
constraints only, NOT indexes: a plain declared ``Index(...)`` on a
|
|
285
|
+
new table never reaches the create render — it arrives
|
|
286
|
+
standalone-only and is untouched here. The embedding this dedup
|
|
287
|
+
targets happens when the table carries instrumentation-appended
|
|
288
|
+
indexes (geoalchemy2-style listeners attaching at Table
|
|
289
|
+
construction): ``to_table()`` reconstruction re-fires the
|
|
290
|
+
attachment, the rebuilt table carries the index again, the offline
|
|
291
|
+
create render embeds it, and autogen ALSO emits the standalone
|
|
292
|
+
CreateIndexOp — executing both is a guaranteed duplicate-object
|
|
293
|
+
failure (push fire-test F1/F2: the renders are byte-identical and
|
|
294
|
+
the second execution collides with 42P07). Suppression side: the
|
|
295
|
+
standalone op is the redundant one — its statement already runs
|
|
296
|
+
inside the add_table op, whose render embeds it verbatim; the
|
|
297
|
+
embedded copy has no other carrier op. Exact-statement containment
|
|
298
|
+
is safe: both renders come from the same renderer over the same
|
|
299
|
+
Index objects, so an embedded index matches its standalone op
|
|
300
|
+
byte-for-byte while a different index's statement cannot be a
|
|
301
|
+
substring of the create-table render (statement text runs to its
|
|
302
|
+
own terminator). Known limitation: the keys are bare table names,
|
|
303
|
+
so two NEW tables sharing a bare name across schemas under-dedup
|
|
304
|
+
(last-wins in the dict) — containment is SQL-qualified either way,
|
|
305
|
+
so no wrong suppression is possible.
|
|
306
|
+
"""
|
|
307
|
+
table_renders = {op.table: op.sql for op in ops if op.type == "add_table"}
|
|
308
|
+
return [
|
|
309
|
+
op
|
|
310
|
+
for op in ops
|
|
311
|
+
if not (
|
|
312
|
+
op.type == "add_index"
|
|
313
|
+
and op.table in table_renders
|
|
314
|
+
and op.sql.strip() in table_renders[op.table]
|
|
315
|
+
)
|
|
316
|
+
]
|
|
317
|
+
|
|
318
|
+
|
|
277
319
|
def _render_op_sql(op, engine: Engine) -> str:
|
|
278
320
|
buf = io.StringIO()
|
|
279
321
|
offline = MigrationContext.configure(
|
|
@@ -336,6 +378,7 @@ class DiffEngine:
|
|
|
336
378
|
ops: list[PlannedOperation] = []
|
|
337
379
|
for op in _flatten(script.upgrade_ops.ops):
|
|
338
380
|
ops.extend(self._translate(op, engine, exclude))
|
|
381
|
+
ops = _dedup_embedded_indexes(ops)
|
|
339
382
|
return Plan(operations=tuple(ops))
|
|
340
383
|
|
|
341
384
|
def _translate(self, op, engine: Engine, exclude: tuple[str, ...]) -> list[PlannedOperation]:
|
|
@@ -9,15 +9,15 @@ from sqlpush.annotations import HYPERTABLE_KEY
|
|
|
9
9
|
from sqlpush.types import PlannedOperation, RiskClass
|
|
10
10
|
|
|
11
11
|
|
|
12
|
-
def _is_hypertable(conn: Connection, table_name: str) -> bool:
|
|
12
|
+
def _is_hypertable(conn: Connection, schema: str, table_name: str) -> bool:
|
|
13
13
|
try:
|
|
14
14
|
return bool(
|
|
15
15
|
conn.execute(
|
|
16
16
|
text(
|
|
17
17
|
"SELECT 1 FROM timescaledb_information.hypertables "
|
|
18
|
-
"WHERE hypertable_name = :name"
|
|
18
|
+
"WHERE hypertable_schema = :schema AND hypertable_name = :name"
|
|
19
19
|
),
|
|
20
|
-
{"name": table_name},
|
|
20
|
+
{"schema": schema, "name": table_name},
|
|
21
21
|
).scalar()
|
|
22
22
|
)
|
|
23
23
|
except ProgrammingError:
|
|
@@ -50,12 +50,22 @@ def hypertable_operations(
|
|
|
50
50
|
table for table in metadata.tables.values() if table.info.get(HYPERTABLE_KEY) is not None
|
|
51
51
|
]
|
|
52
52
|
if engine is not None and pending:
|
|
53
|
+
default_schema = engine.dialect.default_schema_name or "public"
|
|
53
54
|
with engine.connect() as conn:
|
|
54
|
-
pending = [
|
|
55
|
+
pending = [
|
|
56
|
+
t for t in pending if not _is_hypertable(conn, t.schema or default_schema, t.name)
|
|
57
|
+
]
|
|
55
58
|
ops: list[PlannedOperation] = []
|
|
56
59
|
for table in pending:
|
|
57
60
|
info = table.info[HYPERTABLE_KEY]
|
|
58
|
-
|
|
61
|
+
# Schema-qualified relation: create_hypertable resolves an
|
|
62
|
+
# unqualified name via the session search_path, so a table in a
|
|
63
|
+
# non-default schema MUST carry its schema or the op lands on
|
|
64
|
+
# public.<name> (UndefinedTable). Schema-less tables keep the
|
|
65
|
+
# bare name: they live in the default schema, which the
|
|
66
|
+
# search_path already resolves.
|
|
67
|
+
relation = table.name if table.schema is None else f"{table.schema}.{table.name}"
|
|
68
|
+
name = _lit(relation)
|
|
59
69
|
time_column = _lit(info.time_column)
|
|
60
70
|
parts = [f"SELECT create_hypertable('{name}', '{time_column}'"]
|
|
61
71
|
if info.chunk_time_interval:
|
|
@@ -83,6 +83,18 @@ class Report:
|
|
|
83
83
|
duration: float = 0.0
|
|
84
84
|
|
|
85
85
|
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class MigrateReport:
|
|
88
|
+
# applied/skipped/blocked carry BARE filenames so callers (tests, CI)
|
|
89
|
+
# can do exact membership checks; the human-readable why (gate reason,
|
|
90
|
+
# checksum mismatch, SQL error) lives in `notes` aligned by prefix.
|
|
91
|
+
applied: tuple[str, ...] = ()
|
|
92
|
+
skipped: tuple[str, ...] = ()
|
|
93
|
+
blocked: tuple[str, ...] = ()
|
|
94
|
+
partial_failure: bool = False
|
|
95
|
+
notes: tuple[str, ...] = ()
|
|
96
|
+
|
|
97
|
+
|
|
86
98
|
class SqlpushError(Exception):
|
|
87
99
|
pass
|
|
88
100
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|