Pyzzel 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyzzel/__init__.py +46 -0
- pyzzel/core/__init__.py +26 -0
- pyzzel/core/base.py +465 -0
- pyzzel/core/column.py +222 -0
- pyzzel/core/query.py +365 -0
- pyzzel/core/types.py +95 -0
- pyzzel/errors.py +49 -0
- pyzzel/migrate/__init__.py +61 -0
- pyzzel/migrate/__main__.py +118 -0
- pyzzel/migrate/apply.py +158 -0
- pyzzel/migrate/compare.py +58 -0
- pyzzel/migrate/diff.py +180 -0
- pyzzel/migrate/introspect.py +133 -0
- pyzzel/migrate/schema.py +137 -0
- pyzzel/migrate/sqlgen.py +79 -0
- pyzzel/pool.py +108 -0
- pyzzel/sql/__init__.py +50 -0
- pyzzel/sql/builder.py +357 -0
- pyzzel/sql/conditions.py +195 -0
- pyzzel-0.1.1.dist-info/METADATA +9 -0
- pyzzel-0.1.1.dist-info/RECORD +24 -0
- pyzzel-0.1.1.dist-info/WHEEL +5 -0
- pyzzel-0.1.1.dist-info/licenses/LICENSE +21 -0
- pyzzel-0.1.1.dist-info/top_level.txt +1 -0
pyzzel/errors.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Pyzzel exception types."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class PyzzelError(Exception):
|
|
5
|
+
"""Base exception for Pyzzel errors."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ModelDefinitionError(PyzzelError):
|
|
9
|
+
"""Raised when a model class is defined incorrectly."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class NotConnectedError(PyzzelError):
|
|
13
|
+
"""Raised when a database operation is attempted without a connection pool."""
|
|
14
|
+
|
|
15
|
+
def __init__(self):
|
|
16
|
+
super().__init__("Not connected. Call pyzzel.connect(url) first.")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class AlreadyConnectedError(PyzzelError):
|
|
20
|
+
"""Raised when connect is called while a pool is already active."""
|
|
21
|
+
|
|
22
|
+
def __init__(self):
|
|
23
|
+
super().__init__("Already connected. Call pyzzel.disconnect() first.")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class NotFound(PyzzelError):
|
|
27
|
+
"""Raised when a model row no longer exists."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, table, pk_value):
|
|
30
|
+
super().__init__(f"No row found in table {table!r} with primary key {pk_value!r}.")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class VersionConflict(PyzzelError):
|
|
34
|
+
"""Raised when a model row has a newer version than the loaded object."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, table, pk_value):
|
|
37
|
+
super().__init__(f"Row in table {table!r} with primary key {pk_value!r} was changed by someone else.")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class QueryError(PyzzelError):
|
|
41
|
+
"""Raised when a query is invalid for its model."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class MigrationError(PyzzelError):
|
|
45
|
+
"""Raised when a migration cannot be generated safely."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class TooManyRowsError(QueryError):
|
|
49
|
+
"""Raised when an unbounded query exceeds its configured row limit."""
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Offline migration snapshot and SQL generation."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from pyzzel.errors import MigrationError
|
|
7
|
+
from .diff import diff, describe
|
|
8
|
+
from .schema import load_snapshot, save_snapshot, schema_from_models
|
|
9
|
+
from .sqlgen import render
|
|
10
|
+
from .apply import MigrationStatus, apply, status
|
|
11
|
+
from .introspect import fetch_catalog, build_snapshot, schema_from_database
|
|
12
|
+
from .compare import PullResult, CheckReport, pull, check
|
|
13
|
+
|
|
14
|
+
_NAME = re.compile(r"^[a-z][a-z0-9_]{0,50}$")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _validate_name(name):
|
|
18
|
+
if not isinstance(name, str) or _NAME.fullmatch(name) is None:
|
|
19
|
+
raise MigrationError("Migration name must match ^[a-z][a-z0-9_]{0,50}$")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _next_path(name, directory):
|
|
23
|
+
_validate_name(name)
|
|
24
|
+
directory = Path(directory)
|
|
25
|
+
highest = 0
|
|
26
|
+
if directory.exists():
|
|
27
|
+
for path in directory.iterdir():
|
|
28
|
+
match = re.match(r"^(\d{4})_.*\.sql$", path.name)
|
|
29
|
+
if match:
|
|
30
|
+
highest = max(highest, int(match[1]))
|
|
31
|
+
path = directory / f"{highest + 1:04d}_{name}.sql"
|
|
32
|
+
if path.exists():
|
|
33
|
+
raise MigrationError(f"Migration file already exists: {path}")
|
|
34
|
+
return path
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def generate(name, directory="migrations"):
|
|
38
|
+
path = _next_path(name, directory)
|
|
39
|
+
snapshot_path = path.parent / "snapshot.json"
|
|
40
|
+
old = load_snapshot(snapshot_path)
|
|
41
|
+
new = schema_from_models()
|
|
42
|
+
ops = diff(old, new)
|
|
43
|
+
if not ops:
|
|
44
|
+
return None
|
|
45
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
content = f"-- pyzzel migration {path.stem}\n\n{render(ops)}\n"
|
|
47
|
+
path.write_text(content, encoding="utf-8", newline="\n")
|
|
48
|
+
save_snapshot(snapshot_path, new)
|
|
49
|
+
return path
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def create(name, directory="migrations"):
|
|
53
|
+
path = _next_path(name, directory)
|
|
54
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
55
|
+
path.write_text(f"-- pyzzel migration {path.stem}\n", encoding="utf-8", newline="\n")
|
|
56
|
+
return path
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
__all__ = ["generate", "create", "apply", "status", "MigrationStatus", "MigrationError",
|
|
60
|
+
"diff", "describe", "fetch_catalog", "build_snapshot", "schema_from_database",
|
|
61
|
+
"PullResult", "CheckReport", "pull", "check"]
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Command line interface for migration generation and application."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import importlib
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from pyzzel import connect, disconnect
|
|
10
|
+
from pyzzel.errors import MigrationError, PyzzelError
|
|
11
|
+
from . import apply, create, generate, status, check, pull
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _parser():
|
|
15
|
+
parser = argparse.ArgumentParser(prog="python -m pyzzel.migrate")
|
|
16
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
17
|
+
for name in ("generate", "create", "apply", "status", "check", "pull"):
|
|
18
|
+
command = commands.add_parser(name)
|
|
19
|
+
command.add_argument("--dir", default="migrations")
|
|
20
|
+
command.add_argument("--models", action="append", default=[])
|
|
21
|
+
command.add_argument("--history-table", default="pyzzel_migrations")
|
|
22
|
+
if name in ("generate", "create"):
|
|
23
|
+
command.add_argument("name")
|
|
24
|
+
if name in ("apply", "status", "check", "pull"):
|
|
25
|
+
command.add_argument("--url", default=None)
|
|
26
|
+
if name == "check":
|
|
27
|
+
command.add_argument("--ignore", action="append", default=[])
|
|
28
|
+
if name == "pull":
|
|
29
|
+
command.add_argument("--ignore", action="append", default=[])
|
|
30
|
+
command.add_argument("--write", action="store_true")
|
|
31
|
+
if name == "apply":
|
|
32
|
+
command.add_argument("--allow-destructive", action="store_true")
|
|
33
|
+
command.add_argument("--dry-run", action="store_true")
|
|
34
|
+
return parser
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main(argv=None):
|
|
38
|
+
args = _parser().parse_args(argv)
|
|
39
|
+
cwd = str(Path.cwd())
|
|
40
|
+
if cwd not in sys.path:
|
|
41
|
+
sys.path.insert(0, cwd)
|
|
42
|
+
try:
|
|
43
|
+
if args.command == "check" and not args.models:
|
|
44
|
+
print("check requires at least one --models module.", file=sys.stderr)
|
|
45
|
+
return 1
|
|
46
|
+
for module in args.models:
|
|
47
|
+
try:
|
|
48
|
+
importlib.import_module(module)
|
|
49
|
+
except ImportError as error:
|
|
50
|
+
print(f"could not import {module}: {error}", file=sys.stderr)
|
|
51
|
+
return 1
|
|
52
|
+
if args.command == "generate":
|
|
53
|
+
path = generate(args.name, args.dir)
|
|
54
|
+
print(path if path is not None else "No changes.")
|
|
55
|
+
return 0
|
|
56
|
+
if args.command == "create":
|
|
57
|
+
print(create(args.name, args.dir))
|
|
58
|
+
return 0
|
|
59
|
+
url = args.url or os.environ.get("PYZZEL_DATABASE_URL")
|
|
60
|
+
if not url:
|
|
61
|
+
print("Database URL required: pass --url or set PYZZEL_DATABASE_URL.", file=sys.stderr)
|
|
62
|
+
return 1
|
|
63
|
+
try:
|
|
64
|
+
connect(url)
|
|
65
|
+
if args.command == "check":
|
|
66
|
+
report = check(args.dir, ignore=args.ignore, history_table=args.history_table)
|
|
67
|
+
sections = [
|
|
68
|
+
("Pending model changes (run generate):", report.pending_model_changes),
|
|
69
|
+
("Migration files:", report.migration_problems),
|
|
70
|
+
("Drift (snapshot vs database):", report.drift),
|
|
71
|
+
("Warnings:", report.warnings),
|
|
72
|
+
]
|
|
73
|
+
shown = False
|
|
74
|
+
for title, items in sections:
|
|
75
|
+
if items:
|
|
76
|
+
print(title)
|
|
77
|
+
for item in items: print(f" - {item}")
|
|
78
|
+
shown = True
|
|
79
|
+
if not shown:
|
|
80
|
+
print("OK: models, migrations and database agree.")
|
|
81
|
+
return 0 if report.ok else 1
|
|
82
|
+
if args.command == "pull":
|
|
83
|
+
result = pull(args.dir, write=args.write, ignore=args.ignore,
|
|
84
|
+
history_table=args.history_table)
|
|
85
|
+
if result.changes:
|
|
86
|
+
print("Differences between the snapshot and the database:")
|
|
87
|
+
for item in result.changes: print(f" - {item}")
|
|
88
|
+
else:
|
|
89
|
+
print("Snapshot already matches the database.")
|
|
90
|
+
if result.warnings:
|
|
91
|
+
print("Warnings:")
|
|
92
|
+
for item in result.warnings: print(f" - {item}")
|
|
93
|
+
print("snapshot.json updated" if args.write else "run again with --write to replace the snapshot")
|
|
94
|
+
return 0
|
|
95
|
+
if args.command == "apply":
|
|
96
|
+
names = apply(
|
|
97
|
+
args.dir, allow_destructive=args.allow_destructive,
|
|
98
|
+
dry_run=args.dry_run, history_table=args.history_table,
|
|
99
|
+
)
|
|
100
|
+
if names:
|
|
101
|
+
verb = "would apply" if args.dry_run else "applied"
|
|
102
|
+
for name in names:
|
|
103
|
+
print(f"{verb} {name}")
|
|
104
|
+
else:
|
|
105
|
+
print("Nothing to apply.")
|
|
106
|
+
else:
|
|
107
|
+
for item in status(args.dir, history_table=args.history_table):
|
|
108
|
+
print(f"{item.state:<8} {item.name}")
|
|
109
|
+
return 0
|
|
110
|
+
finally:
|
|
111
|
+
disconnect()
|
|
112
|
+
except (MigrationError, PyzzelError) as error:
|
|
113
|
+
print(str(error), file=sys.stderr)
|
|
114
|
+
return 1
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
if __name__ == "__main__":
|
|
118
|
+
sys.exit(main())
|
pyzzel/migrate/apply.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Apply numbered migration files and inspect migration history."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
import hashlib
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import re
|
|
7
|
+
from typing import NamedTuple
|
|
8
|
+
|
|
9
|
+
from pyzzel.errors import MigrationError
|
|
10
|
+
from pyzzel.pool import connection
|
|
11
|
+
from pyzzel.sql import quote_ident
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MigrationStatus(NamedTuple):
|
|
15
|
+
name: str
|
|
16
|
+
state: str
|
|
17
|
+
applied_at: datetime | None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
_FILE = re.compile(r"^\d{4}_[a-z][a-z0-9_]*\.sql$")
|
|
21
|
+
_LOCK_ID = 7_894_215_087_123 # Shared transaction lock for serialized migration runs.
|
|
22
|
+
_HISTORY_DDL = (
|
|
23
|
+
'CREATE TABLE IF NOT EXISTS {table} ('
|
|
24
|
+
'"id" INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, '
|
|
25
|
+
'"name" TEXT NOT NULL UNIQUE, "checksum" TEXT NOT NULL, '
|
|
26
|
+
'"applied_at" TIMESTAMPTZ NOT NULL DEFAULT now())'
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _files(directory):
|
|
31
|
+
found = []
|
|
32
|
+
for path in sorted(Path(directory).iterdir() if Path(directory).exists() else ()):
|
|
33
|
+
if not _FILE.fullmatch(path.name) or not path.is_file():
|
|
34
|
+
continue
|
|
35
|
+
number = path.name[:4]
|
|
36
|
+
found.append((number, path.name, path))
|
|
37
|
+
numbers = set()
|
|
38
|
+
files = []
|
|
39
|
+
for number, name, path in found:
|
|
40
|
+
if number in numbers:
|
|
41
|
+
raise MigrationError(f"Duplicate migration number {number}: {name}")
|
|
42
|
+
numbers.add(number)
|
|
43
|
+
try:
|
|
44
|
+
raw = path.read_bytes()
|
|
45
|
+
text = raw.decode("utf-8")
|
|
46
|
+
except (OSError, UnicodeError) as error:
|
|
47
|
+
raise MigrationError(f"Could not read migration {name}: {error}") from error
|
|
48
|
+
normalized = raw.replace(b"\r\n", b"\n")
|
|
49
|
+
files.append((name, text, hashlib.sha256(normalized).hexdigest()))
|
|
50
|
+
return files
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _table_name(history_table):
|
|
54
|
+
return quote_ident(history_table)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _history(conn, table_sql):
|
|
58
|
+
exists = conn.execute("SELECT to_regclass(%s)", (table_sql,)).fetchone()[0]
|
|
59
|
+
if exists is None:
|
|
60
|
+
return None
|
|
61
|
+
rows = conn.execute(
|
|
62
|
+
f"SELECT \"name\", \"checksum\", \"applied_at\" FROM {table_sql}"
|
|
63
|
+
).fetchall()
|
|
64
|
+
return {name: (checksum, applied_at) for name, checksum, applied_at in rows}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def status(directory="migrations", *, history_table="pyzzel_migrations"):
|
|
68
|
+
table_sql = _table_name(history_table)
|
|
69
|
+
files = _files(directory)
|
|
70
|
+
with connection() as conn:
|
|
71
|
+
history = _history(conn, table_sql)
|
|
72
|
+
history = history or {}
|
|
73
|
+
result = []
|
|
74
|
+
names = set()
|
|
75
|
+
for name, _text, checksum in files:
|
|
76
|
+
names.add(name)
|
|
77
|
+
stored = history.get(name)
|
|
78
|
+
if stored is None:
|
|
79
|
+
result.append(MigrationStatus(name, "pending", None))
|
|
80
|
+
elif stored[0] == checksum:
|
|
81
|
+
result.append(MigrationStatus(name, "applied", stored[1]))
|
|
82
|
+
else:
|
|
83
|
+
result.append(MigrationStatus(name, "edited", stored[1]))
|
|
84
|
+
result.extend(
|
|
85
|
+
MigrationStatus(name, "missing", applied_at)
|
|
86
|
+
for name, (_checksum, applied_at) in history.items() if name not in names
|
|
87
|
+
)
|
|
88
|
+
return sorted(result, key=lambda item: item.name)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _check_history(files, history):
|
|
92
|
+
by_name = {name: checksum for name, _text, checksum in files}
|
|
93
|
+
for name, (_checksum, _applied_at) in history.items():
|
|
94
|
+
if name not in by_name:
|
|
95
|
+
raise MigrationError(f"Applied migration {name} is missing from disk")
|
|
96
|
+
if by_name[name] != _checksum:
|
|
97
|
+
raise MigrationError(f"Migration {name} was edited after it was applied")
|
|
98
|
+
applied = sorted(history)
|
|
99
|
+
latest = applied[-1] if applied else None
|
|
100
|
+
pending = [item for item in files if item[0] not in history]
|
|
101
|
+
if latest is not None:
|
|
102
|
+
for name, _text, _checksum in pending:
|
|
103
|
+
if name < latest:
|
|
104
|
+
raise MigrationError(
|
|
105
|
+
f"Migration {name} is older than applied migration {latest}; renumber it"
|
|
106
|
+
)
|
|
107
|
+
return pending
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def apply(directory="migrations", *, allow_destructive=False, dry_run=False,
|
|
111
|
+
history_table="pyzzel_migrations"):
|
|
112
|
+
table_sql = _table_name(history_table)
|
|
113
|
+
files = _files(directory)
|
|
114
|
+
with connection() as conn:
|
|
115
|
+
if dry_run:
|
|
116
|
+
history = _history(conn, table_sql) or {}
|
|
117
|
+
pending = _check_history(files, history)
|
|
118
|
+
return [name for name, _text, _checksum in pending]
|
|
119
|
+
|
|
120
|
+
applied = []
|
|
121
|
+
while True:
|
|
122
|
+
name = "<migration history>"
|
|
123
|
+
try:
|
|
124
|
+
with conn.transaction():
|
|
125
|
+
# Transaction-scoped locks work through transaction poolers,
|
|
126
|
+
# unlike session-scoped locks whose backend may be reassigned.
|
|
127
|
+
conn.execute("SELECT pg_advisory_xact_lock(%s)", (_LOCK_ID,))
|
|
128
|
+
history = _history(conn, table_sql) or {}
|
|
129
|
+
pending = _check_history(files, history)
|
|
130
|
+
if not pending:
|
|
131
|
+
conn.execute(_HISTORY_DDL.format(table=table_sql))
|
|
132
|
+
return applied
|
|
133
|
+
destructive = [
|
|
134
|
+
migration_name
|
|
135
|
+
for migration_name, migration_text, _checksum in pending
|
|
136
|
+
if any(
|
|
137
|
+
line.strip().startswith("-- pyzzel:destructive")
|
|
138
|
+
for line in migration_text.splitlines()
|
|
139
|
+
)
|
|
140
|
+
]
|
|
141
|
+
if destructive and not allow_destructive:
|
|
142
|
+
names = ", ".join(destructive)
|
|
143
|
+
raise MigrationError(
|
|
144
|
+
f"Destructive migrations require allow_destructive=True "
|
|
145
|
+
f"(CLI: --allow-destructive): {names}"
|
|
146
|
+
)
|
|
147
|
+
conn.execute(_HISTORY_DDL.format(table=table_sql))
|
|
148
|
+
name, text, checksum = pending[0]
|
|
149
|
+
conn.execute(text)
|
|
150
|
+
conn.execute(
|
|
151
|
+
f"INSERT INTO {table_sql} (\"name\", \"checksum\") VALUES (%s, %s)",
|
|
152
|
+
(name, checksum),
|
|
153
|
+
)
|
|
154
|
+
except MigrationError:
|
|
155
|
+
raise
|
|
156
|
+
except Exception as error:
|
|
157
|
+
raise MigrationError(f"Migration {name} failed: {error}") from error
|
|
158
|
+
applied.append(name)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Compare models, migration files, snapshots, and a PostgreSQL schema."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import NamedTuple
|
|
5
|
+
|
|
6
|
+
from pyzzel.core.base import Base
|
|
7
|
+
from pyzzel.errors import MigrationError
|
|
8
|
+
from .diff import describe, diff
|
|
9
|
+
from .introspect import schema_from_database
|
|
10
|
+
from .schema import load_snapshot, save_snapshot, schema_from_models
|
|
11
|
+
from .apply import status
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class PullResult(NamedTuple):
|
|
15
|
+
changes: list[str]
|
|
16
|
+
warnings: list[str]
|
|
17
|
+
written: bool
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class CheckReport(NamedTuple):
|
|
21
|
+
pending_model_changes: list[str]
|
|
22
|
+
migration_problems: list[str]
|
|
23
|
+
drift: list[str]
|
|
24
|
+
warnings: list[str]
|
|
25
|
+
drift_checked: bool
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def ok(self):
|
|
29
|
+
return not (self.pending_model_changes or self.migration_problems or self.drift)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def pull(directory="migrations", *, write=False, ignore=(), history_table="pyzzel_migrations"):
|
|
33
|
+
directory = Path(directory)
|
|
34
|
+
current = load_snapshot(directory / "snapshot.json")
|
|
35
|
+
database, warnings = schema_from_database(ignore=ignore, exclude=(history_table,))
|
|
36
|
+
changes = [describe(op, "snapshot", "database") for op in diff(current, database)]
|
|
37
|
+
if write:
|
|
38
|
+
save_snapshot(directory / "snapshot.json", database)
|
|
39
|
+
return PullResult(changes, warnings, bool(write))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def check(directory="migrations", *, ignore=(), history_table="pyzzel_migrations"):
|
|
43
|
+
if not Base._registry:
|
|
44
|
+
raise MigrationError("No models are registered; import your models first (for the command line use --models).")
|
|
45
|
+
directory = Path(directory)
|
|
46
|
+
snapshot = load_snapshot(directory / "snapshot.json")
|
|
47
|
+
models = schema_from_models()
|
|
48
|
+
pending = [describe(op, "snapshot", "models") for op in diff(snapshot, models)]
|
|
49
|
+
problems = []
|
|
50
|
+
for item in status(directory, history_table=history_table):
|
|
51
|
+
if item.state == "pending": problems.append(f"{item.name}: not applied yet")
|
|
52
|
+
elif item.state == "edited": problems.append(f"{item.name}: edited after it was applied")
|
|
53
|
+
elif item.state == "missing": problems.append(f"{item.name}: applied but the file is missing")
|
|
54
|
+
if problems:
|
|
55
|
+
return CheckReport(pending, problems, [], ["drift not checked: apply or fix the migration files first"], False)
|
|
56
|
+
database, warnings = schema_from_database(ignore=ignore, exclude=(history_table,))
|
|
57
|
+
drift = [describe(op, "snapshot", "database") for op in diff(snapshot, database)]
|
|
58
|
+
return CheckReport(pending, problems, drift, warnings, True)
|
pyzzel/migrate/diff.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Deterministic offline schema differences."""
|
|
2
|
+
|
|
3
|
+
from typing import NamedTuple
|
|
4
|
+
|
|
5
|
+
from pyzzel.errors import MigrationError
|
|
6
|
+
from .schema import canonical_type, normalize_default
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class CreateTable(NamedTuple):
|
|
10
|
+
table: str
|
|
11
|
+
columns: dict
|
|
12
|
+
indexes: list
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class DropTable(NamedTuple):
|
|
16
|
+
table: str
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class AddColumn(NamedTuple):
|
|
20
|
+
table: str
|
|
21
|
+
name: str
|
|
22
|
+
column: dict
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DropColumn(NamedTuple):
|
|
26
|
+
table: str
|
|
27
|
+
name: str
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class AlterType(NamedTuple):
|
|
31
|
+
table: str
|
|
32
|
+
name: str
|
|
33
|
+
old: str
|
|
34
|
+
new: str
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class SetNotNull(NamedTuple):
|
|
38
|
+
table: str
|
|
39
|
+
name: str
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class DropNotNull(NamedTuple):
|
|
43
|
+
table: str
|
|
44
|
+
name: str
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class SetDefault(NamedTuple):
|
|
48
|
+
table: str
|
|
49
|
+
name: str
|
|
50
|
+
default: str
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class DropDefault(NamedTuple):
|
|
54
|
+
table: str
|
|
55
|
+
name: str
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class AddUnique(NamedTuple):
|
|
59
|
+
table: str
|
|
60
|
+
name: str
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class DropUnique(NamedTuple):
|
|
64
|
+
table: str
|
|
65
|
+
name: str
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class AddForeignKey(NamedTuple):
|
|
69
|
+
table: str
|
|
70
|
+
name: str
|
|
71
|
+
references: list
|
|
72
|
+
on_delete: str
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class DropForeignKey(NamedTuple):
|
|
76
|
+
table: str
|
|
77
|
+
name: str
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class AddIndex(NamedTuple):
|
|
81
|
+
table: str
|
|
82
|
+
name: str
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class DropIndex(NamedTuple):
|
|
86
|
+
table: str
|
|
87
|
+
name: str
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def describe(op, old_name, new_name):
|
|
91
|
+
"""Return one readable line describing a schema operation."""
|
|
92
|
+
if isinstance(op, CreateTable): return f"table {op.table}: only in {new_name}"
|
|
93
|
+
if isinstance(op, DropTable): return f"table {op.table}: only in {old_name}"
|
|
94
|
+
if isinstance(op, AddColumn): return f"column {op.table}.{op.name}: only in {new_name}"
|
|
95
|
+
if isinstance(op, DropColumn): return f"column {op.table}.{op.name}: only in {old_name}"
|
|
96
|
+
if isinstance(op, AlterType): return f"column {op.table}.{op.name}: type differs ({old_name} {op.old}, {new_name} {op.new})"
|
|
97
|
+
if isinstance(op, SetNotNull): return f"column {op.table}.{op.name}: NOT NULL in {new_name} but nullable in {old_name}"
|
|
98
|
+
if isinstance(op, DropNotNull): return f"column {op.table}.{op.name}: nullable in {new_name} but NOT NULL in {old_name}"
|
|
99
|
+
if isinstance(op, SetDefault): return f"column {op.table}.{op.name}: default differs ({new_name} has {op.default})"
|
|
100
|
+
if isinstance(op, DropDefault): return f"column {op.table}.{op.name}: default only in {old_name}"
|
|
101
|
+
if isinstance(op, AddUnique): return f"column {op.table}.{op.name}: unique only in {new_name}"
|
|
102
|
+
if isinstance(op, DropUnique): return f"column {op.table}.{op.name}: unique only in {old_name}"
|
|
103
|
+
if isinstance(op, AddForeignKey):
|
|
104
|
+
return f"column {op.table}.{op.name}: foreign key to {op.references[0]}.{op.references[1]} (ON DELETE {op.on_delete}) only in {new_name}"
|
|
105
|
+
if isinstance(op, DropForeignKey): return f"column {op.table}.{op.name}: foreign key only in {old_name}"
|
|
106
|
+
if isinstance(op, AddIndex): return f"column {op.table}.{op.name}: index only in {new_name}"
|
|
107
|
+
if isinstance(op, DropIndex): return f"column {op.table}.{op.name}: index only in {old_name}"
|
|
108
|
+
raise TypeError(f"Unsupported schema operation: {type(op).__name__}")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _order_tables(tables, snapshots, reverse=False):
|
|
112
|
+
result, visiting, visited = [], set(), set()
|
|
113
|
+
|
|
114
|
+
def visit(table):
|
|
115
|
+
if table in visited:
|
|
116
|
+
return
|
|
117
|
+
if table in visiting:
|
|
118
|
+
raise MigrationError("Circular foreign key dependency between new tables")
|
|
119
|
+
visiting.add(table)
|
|
120
|
+
for column in snapshots[table]["columns"].values():
|
|
121
|
+
ref = column.get("references")
|
|
122
|
+
if ref and ref[0] in tables and ref[0] != table:
|
|
123
|
+
visit(ref[0])
|
|
124
|
+
visiting.remove(table); visited.add(table); result.append(table)
|
|
125
|
+
|
|
126
|
+
for table in tables:
|
|
127
|
+
visit(table)
|
|
128
|
+
return list(reversed(result)) if reverse else result
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def diff(old, new):
|
|
132
|
+
before, after = old["tables"], new["tables"]
|
|
133
|
+
drop_fk, drop_unique, drop_index, drop_columns = [], [], [], []
|
|
134
|
+
creates, add_columns, types, defaults, nulls = [], [], [], [], []
|
|
135
|
+
add_unique, add_fk, add_index, drop_tables = [], [], [], []
|
|
136
|
+
for table, previous in before.items():
|
|
137
|
+
if table not in after:
|
|
138
|
+
continue
|
|
139
|
+
current = after[table]
|
|
140
|
+
old_cols, new_cols = previous["columns"], current["columns"]
|
|
141
|
+
for name, column in old_cols.items():
|
|
142
|
+
if name not in new_cols:
|
|
143
|
+
drop_columns.append(DropColumn(table, name)); continue
|
|
144
|
+
updated = new_cols[name]
|
|
145
|
+
if column.get("primary_key") != updated.get("primary_key") or column.get("identity") != updated.get("identity"):
|
|
146
|
+
raise MigrationError(f"changing the primary key or identity of {table}.{name} is not supported; write the migration by hand")
|
|
147
|
+
if column.get("references") != updated.get("references") or column.get("on_delete") != updated.get("on_delete"):
|
|
148
|
+
if column.get("references"):
|
|
149
|
+
drop_fk.append(DropForeignKey(table, name))
|
|
150
|
+
if updated.get("references"):
|
|
151
|
+
add_fk.append(AddForeignKey(table, name, updated["references"], updated.get("on_delete") or "NO ACTION"))
|
|
152
|
+
if column.get("unique") != updated.get("unique"):
|
|
153
|
+
(add_unique if updated.get("unique") else drop_unique).append((AddUnique if updated.get("unique") else DropUnique)(table, name))
|
|
154
|
+
old_index, new_index = name in previous.get("indexes", []), name in current.get("indexes", [])
|
|
155
|
+
if old_index != new_index:
|
|
156
|
+
(add_index if new_index else drop_index).append((AddIndex if new_index else DropIndex)(table, name))
|
|
157
|
+
if canonical_type(column["type"]) != canonical_type(updated["type"]):
|
|
158
|
+
types.append(AlterType(table, name, column["type"], updated["type"]))
|
|
159
|
+
if normalize_default(column.get("default")) != normalize_default(updated.get("default")):
|
|
160
|
+
if updated.get("default") is None: defaults.append(DropDefault(table, name))
|
|
161
|
+
else: defaults.append(SetDefault(table, name, updated["default"]))
|
|
162
|
+
if column.get("nullable", True) != updated.get("nullable", True):
|
|
163
|
+
nulls.append((DropNotNull if updated.get("nullable") else SetNotNull)(table, name))
|
|
164
|
+
for name, column in new_cols.items():
|
|
165
|
+
if name not in old_cols:
|
|
166
|
+
if column.get("primary_key"):
|
|
167
|
+
raise MigrationError(f"adding primary key column {table}.{name} is not supported")
|
|
168
|
+
add_columns.append(AddColumn(table, name, column))
|
|
169
|
+
if name in current.get("indexes", []):
|
|
170
|
+
add_index.append(AddIndex(table, name))
|
|
171
|
+
for table, value in after.items():
|
|
172
|
+
if table not in before:
|
|
173
|
+
creates.append(table)
|
|
174
|
+
for name in value.get("indexes", []):
|
|
175
|
+
add_index.append(AddIndex(table, name))
|
|
176
|
+
create_ops = [CreateTable(t, after[t]["columns"], after[t].get("indexes", [])) for t in _order_tables(creates, after)]
|
|
177
|
+
for table in _order_tables([t for t in before if t not in after], before, reverse=True):
|
|
178
|
+
drop_tables.append(DropTable(table))
|
|
179
|
+
return (drop_fk + drop_unique + drop_index + drop_columns + create_ops + add_columns
|
|
180
|
+
+ types + defaults + nulls + add_unique + add_fk + add_index + drop_tables)
|