delembic 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- delembic/__init__.py +4 -0
- delembic/alembic_compat.py +65 -0
- delembic/cli.py +199 -0
- delembic/config.py +38 -0
- delembic/dag.py +37 -0
- delembic/db.py +75 -0
- delembic/executor.py +115 -0
- delembic/migration.py +14 -0
- delembic/registry.py +34 -0
- delembic-0.2.0.dist-info/METADATA +155 -0
- delembic-0.2.0.dist-info/RECORD +13 -0
- delembic-0.2.0.dist-info/WHEEL +4 -0
- delembic-0.2.0.dist-info/entry_points.txt +2 -0
delembic/__init__.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import sqlalchemy as sa
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class AlembicDepsError(Exception):
|
|
7
|
+
pass
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_alembic_applied_revisions(conn: sa.Connection, alembic_ini: Path) -> set[str]:
|
|
11
|
+
"""
|
|
12
|
+
Walk Alembic script history from current heads back to base.
|
|
13
|
+
Returns all revision IDs that have been applied (i.e. are ancestors of current heads).
|
|
14
|
+
Requires alembic to be installed.
|
|
15
|
+
"""
|
|
16
|
+
try:
|
|
17
|
+
from alembic.config import Config as AlembicConfig
|
|
18
|
+
from alembic.runtime.migration import MigrationContext
|
|
19
|
+
from alembic.script import ScriptDirectory
|
|
20
|
+
except ImportError:
|
|
21
|
+
raise AlembicDepsError(
|
|
22
|
+
"alembic package not installed. Run: pip install alembic"
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
alembic_cfg = AlembicConfig(str(alembic_ini))
|
|
26
|
+
script = ScriptDirectory.from_config(alembic_cfg)
|
|
27
|
+
ctx = MigrationContext.configure(conn)
|
|
28
|
+
current_heads = ctx.get_current_heads()
|
|
29
|
+
|
|
30
|
+
applied: set[str] = set()
|
|
31
|
+
for head in current_heads:
|
|
32
|
+
for rev in script.iterate_revisions(head, "base"):
|
|
33
|
+
applied.add(rev.revision)
|
|
34
|
+
|
|
35
|
+
return applied
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def get_current_heads(conn: sa.Connection) -> list[str]:
|
|
39
|
+
"""Return the current Alembic head revision IDs from alembic_version table."""
|
|
40
|
+
try:
|
|
41
|
+
from alembic.runtime.migration import MigrationContext
|
|
42
|
+
except ImportError:
|
|
43
|
+
raise AlembicDepsError(
|
|
44
|
+
"alembic package not installed. Run: pip install alembic"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
ctx = MigrationContext.configure(conn)
|
|
48
|
+
return list(ctx.get_current_heads())
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def check_alembic_deps(
|
|
52
|
+
conn: sa.Connection,
|
|
53
|
+
alembic_ini: Path,
|
|
54
|
+
required: list[str],
|
|
55
|
+
migration_revision: str,
|
|
56
|
+
) -> None:
|
|
57
|
+
"""Raise AlembicDepsError if any required Alembic revision has not been applied."""
|
|
58
|
+
applied = get_alembic_applied_revisions(conn, alembic_ini)
|
|
59
|
+
missing = [rev for rev in required if rev not in applied]
|
|
60
|
+
if missing:
|
|
61
|
+
raise AlembicDepsError(
|
|
62
|
+
f"Migration {migration_revision} requires Alembic revision(s) "
|
|
63
|
+
f"{', '.join(missing)} to be applied first. "
|
|
64
|
+
f"Run 'alembic upgrade head' (or to the required revision) before retrying."
|
|
65
|
+
)
|
delembic/cli.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import string
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
import sqlalchemy as sa
|
|
7
|
+
|
|
8
|
+
from delembic.config import Config, find_config
|
|
9
|
+
from delembic.db import ensure_tables, get_applied, history_table, version_table
|
|
10
|
+
from delembic.executor import run_upgrade
|
|
11
|
+
from delembic.registry import load_migrations
|
|
12
|
+
|
|
13
|
+
_INI_TEMPLATE = """\
|
|
14
|
+
[delembic]
|
|
15
|
+
script_location = delembic
|
|
16
|
+
sqlalchemy.url = postgresql+psycopg://user:pass@localhost/mydb
|
|
17
|
+
alembic_config = alembic.ini
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
_ENV_TEMPLATE = """\
|
|
21
|
+
from sqlalchemy import create_engine
|
|
22
|
+
|
|
23
|
+
# Configure your database URL here or load from environment variables
|
|
24
|
+
DATABASE_URL = "postgresql+psycopg://user:pass@localhost/mydb"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_engine():
|
|
28
|
+
return create_engine(DATABASE_URL)
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
_MIGRATION_TEMPLATE = """\
|
|
32
|
+
from delembic import DataMigration
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class $class_name(DataMigration):
|
|
36
|
+
|
|
37
|
+
revision = "$revision"
|
|
38
|
+
|
|
39
|
+
depends_on = $depends_on
|
|
40
|
+
|
|
41
|
+
description = "$description"
|
|
42
|
+
|
|
43
|
+
def upgrade(self, conn):
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
def validate(self, conn):
|
|
47
|
+
pass
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@click.group()
|
|
52
|
+
def cli() -> None:
|
|
53
|
+
"""Delembic — data migration framework."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@cli.command()
|
|
57
|
+
@click.argument("directory", default="delembic")
|
|
58
|
+
def init(directory: str) -> None:
|
|
59
|
+
"""Initialize a new delembic project in the current directory.
|
|
60
|
+
|
|
61
|
+
DIRECTORY is the folder name for migration scripts (default: delembic).
|
|
62
|
+
"""
|
|
63
|
+
cwd = Path.cwd()
|
|
64
|
+
ini_path = cwd / "delembic.ini"
|
|
65
|
+
script_dir = cwd / directory
|
|
66
|
+
versions_dir = script_dir / "versions"
|
|
67
|
+
|
|
68
|
+
if ini_path.exists():
|
|
69
|
+
raise click.ClickException("delembic.ini already exists.")
|
|
70
|
+
|
|
71
|
+
ini_content = _INI_TEMPLATE.replace("script_location = delembic", f"script_location = {directory}")
|
|
72
|
+
|
|
73
|
+
versions_dir.mkdir(parents=True, exist_ok=True)
|
|
74
|
+
ini_path.write_text(ini_content)
|
|
75
|
+
(script_dir / "env.py").write_text(_ENV_TEMPLATE)
|
|
76
|
+
(versions_dir / ".gitkeep").touch()
|
|
77
|
+
|
|
78
|
+
click.echo(f"Created {ini_path}")
|
|
79
|
+
click.echo(f"Created {script_dir / 'env.py'}")
|
|
80
|
+
click.echo(f"Created {versions_dir}/")
|
|
81
|
+
click.echo("\nEdit delembic.ini and set sqlalchemy.url before running migrations.")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@cli.command()
|
|
85
|
+
@click.option("-m", "--message", required=True, help="Short description of this migration.")
|
|
86
|
+
def revision(message: str) -> None:
|
|
87
|
+
"""Generate a new migration file."""
|
|
88
|
+
from delembic.alembic_compat import AlembicDepsError, get_current_heads
|
|
89
|
+
|
|
90
|
+
cfg = find_config()
|
|
91
|
+
cfg.versions_dir.mkdir(parents=True, exist_ok=True)
|
|
92
|
+
|
|
93
|
+
alembic_heads: list[str] = []
|
|
94
|
+
if cfg.alembic_config:
|
|
95
|
+
try:
|
|
96
|
+
engine = cfg.engine()
|
|
97
|
+
with engine.connect() as conn:
|
|
98
|
+
alembic_heads = get_current_heads(conn)
|
|
99
|
+
except AlembicDepsError as e:
|
|
100
|
+
click.echo(f"Warning: could not read Alembic heads: {e}", err=True)
|
|
101
|
+
except Exception as e:
|
|
102
|
+
click.echo(f"Warning: could not connect to DB for Alembic heads: {e}", err=True)
|
|
103
|
+
|
|
104
|
+
next_id = _next_revision_id(cfg.versions_dir)
|
|
105
|
+
slug = re.sub(r"[^a-z0-9]+", "_", message.lower()).strip("_")
|
|
106
|
+
filename = f"{next_id}_{slug}.py"
|
|
107
|
+
class_name = "".join(word.title() for word in slug.split("_"))
|
|
108
|
+
content = string.Template(_MIGRATION_TEMPLATE).substitute(
|
|
109
|
+
class_name=class_name,
|
|
110
|
+
revision=next_id,
|
|
111
|
+
description=message,
|
|
112
|
+
depends_on=repr(alembic_heads),
|
|
113
|
+
)
|
|
114
|
+
path = cfg.versions_dir / filename
|
|
115
|
+
path.write_text(content)
|
|
116
|
+
if alembic_heads:
|
|
117
|
+
click.echo(f" Alembic heads captured: {alembic_heads}")
|
|
118
|
+
click.echo(f"Created {path}")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@cli.command()
|
|
122
|
+
@click.argument("target", default="head")
|
|
123
|
+
def upgrade(target: str) -> None:
|
|
124
|
+
"""Run unapplied migrations up to TARGET (default: head)."""
|
|
125
|
+
cfg = find_config()
|
|
126
|
+
engine = cfg.engine()
|
|
127
|
+
run_upgrade(engine, cfg.versions_dir, target, alembic_ini=cfg.alembic_config)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@cli.command()
|
|
131
|
+
def current() -> None:
|
|
132
|
+
"""Show the most recently applied revision."""
|
|
133
|
+
cfg = find_config()
|
|
134
|
+
engine = cfg.engine()
|
|
135
|
+
with engine.connect() as conn:
|
|
136
|
+
try:
|
|
137
|
+
ensure_tables(conn)
|
|
138
|
+
conn.commit()
|
|
139
|
+
row = conn.execute(
|
|
140
|
+
sa.select(version_table.c.revision, version_table.c.applied_at)
|
|
141
|
+
.where(version_table.c.status == "success")
|
|
142
|
+
.order_by(version_table.c.applied_at.desc())
|
|
143
|
+
.limit(1)
|
|
144
|
+
).fetchone()
|
|
145
|
+
except Exception as e:
|
|
146
|
+
raise click.ClickException(str(e))
|
|
147
|
+
|
|
148
|
+
if row:
|
|
149
|
+
click.echo(f"{row[0]} (applied {row[1]})")
|
|
150
|
+
else:
|
|
151
|
+
click.echo("No migrations applied yet.")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@cli.command()
|
|
155
|
+
def history() -> None:
|
|
156
|
+
"""List all migrations and their status."""
|
|
157
|
+
cfg = find_config()
|
|
158
|
+
migrations = load_migrations(cfg.versions_dir)
|
|
159
|
+
|
|
160
|
+
engine = cfg.engine()
|
|
161
|
+
with engine.connect() as conn:
|
|
162
|
+
try:
|
|
163
|
+
ensure_tables(conn)
|
|
164
|
+
conn.commit()
|
|
165
|
+
applied = get_applied(conn)
|
|
166
|
+
failed_rows = conn.execute(
|
|
167
|
+
sa.select(version_table.c.revision)
|
|
168
|
+
.where(version_table.c.status == "failed")
|
|
169
|
+
)
|
|
170
|
+
failed = {row[0] for row in failed_rows}
|
|
171
|
+
except Exception as e:
|
|
172
|
+
raise click.ClickException(str(e))
|
|
173
|
+
|
|
174
|
+
if not migrations:
|
|
175
|
+
click.echo("No migrations found.")
|
|
176
|
+
return
|
|
177
|
+
|
|
178
|
+
from delembic.dag import topological_sort
|
|
179
|
+
order = topological_sort(migrations)
|
|
180
|
+
|
|
181
|
+
for rev in order:
|
|
182
|
+
cls = migrations[rev]
|
|
183
|
+
if rev in applied:
|
|
184
|
+
status = "applied "
|
|
185
|
+
elif rev in failed:
|
|
186
|
+
status = "failed "
|
|
187
|
+
else:
|
|
188
|
+
status = "pending "
|
|
189
|
+
click.echo(f"{status} {rev} {cls.description}")
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _next_revision_id(versions_dir: Path) -> str:
|
|
193
|
+
existing = list(versions_dir.glob("D*.py"))
|
|
194
|
+
max_n = 0
|
|
195
|
+
for p in existing:
|
|
196
|
+
m = re.match(r"D(\d+)", p.stem)
|
|
197
|
+
if m:
|
|
198
|
+
max_n = max(max_n, int(m.group(1)))
|
|
199
|
+
return f"D{max_n + 1:03d}"
|
delembic/config.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import configparser
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import sqlalchemy as sa
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Config:
|
|
8
|
+
def __init__(self, ini_path: Path):
|
|
9
|
+
self.ini_path = ini_path.resolve()
|
|
10
|
+
cp = configparser.ConfigParser()
|
|
11
|
+
cp.read(self.ini_path)
|
|
12
|
+
self.script_location = Path(cp.get("delembic", "script_location", fallback="delembic"))
|
|
13
|
+
self.url = cp.get("delembic", "sqlalchemy.url", fallback="")
|
|
14
|
+
_alembic_raw = cp.get("delembic", "alembic_config", fallback="")
|
|
15
|
+
self.alembic_config: Path | None = (
|
|
16
|
+
(self.ini_path.parent / _alembic_raw) if _alembic_raw else None
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def versions_dir(self) -> Path:
|
|
21
|
+
return self.ini_path.parent / self.script_location / "versions"
|
|
22
|
+
|
|
23
|
+
def engine(self) -> sa.Engine:
|
|
24
|
+
if not self.url:
|
|
25
|
+
raise RuntimeError("sqlalchemy.url not set in delembic.ini")
|
|
26
|
+
return sa.create_engine(self.url)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def find_config() -> Config:
|
|
30
|
+
"""Walk up from cwd looking for delembic.ini."""
|
|
31
|
+
cwd = Path.cwd()
|
|
32
|
+
for directory in [cwd, *cwd.parents]:
|
|
33
|
+
candidate = directory / "delembic.ini"
|
|
34
|
+
if candidate.exists():
|
|
35
|
+
return Config(candidate)
|
|
36
|
+
raise FileNotFoundError(
|
|
37
|
+
"delembic.ini not found. Run 'delembic init' first."
|
|
38
|
+
)
|
delembic/dag.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from typing import Type
|
|
2
|
+
|
|
3
|
+
from delembic.migration import DataMigration
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class CycleError(Exception):
|
|
7
|
+
pass
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def topological_sort(migrations: dict[str, Type[DataMigration]]) -> list[str]:
|
|
11
|
+
"""Return revision IDs in dependency-safe execution order (Kahn's algorithm)."""
|
|
12
|
+
in_degree: dict[str, int] = {rev: 0 for rev in migrations}
|
|
13
|
+
dependents: dict[str, list[str]] = {rev: [] for rev in migrations}
|
|
14
|
+
|
|
15
|
+
for rev, cls in migrations.items():
|
|
16
|
+
for dep in cls.depends_on:
|
|
17
|
+
if dep not in migrations:
|
|
18
|
+
# External dep (e.g. Alembic revision) — skip in sort, validate at runtime
|
|
19
|
+
continue
|
|
20
|
+
in_degree[rev] += 1
|
|
21
|
+
dependents[dep].append(rev)
|
|
22
|
+
|
|
23
|
+
queue = sorted(rev for rev, deg in in_degree.items() if deg == 0)
|
|
24
|
+
order: list[str] = []
|
|
25
|
+
|
|
26
|
+
while queue:
|
|
27
|
+
node = queue.pop(0)
|
|
28
|
+
order.append(node)
|
|
29
|
+
for dependent in sorted(dependents.get(node, [])):
|
|
30
|
+
in_degree[dependent] -= 1
|
|
31
|
+
if in_degree[dependent] == 0:
|
|
32
|
+
queue.append(dependent)
|
|
33
|
+
|
|
34
|
+
if len(order) != len(migrations):
|
|
35
|
+
raise CycleError("Cycle detected in migration dependency graph")
|
|
36
|
+
|
|
37
|
+
return order
|
delembic/db.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from datetime import datetime, timezone
|
|
2
|
+
|
|
3
|
+
import sqlalchemy as sa
|
|
4
|
+
|
|
5
|
+
_metadata = sa.MetaData()
|
|
6
|
+
|
|
7
|
+
version_table = sa.Table(
|
|
8
|
+
"delembic_version",
|
|
9
|
+
_metadata,
|
|
10
|
+
sa.Column("revision", sa.Text, primary_key=True),
|
|
11
|
+
sa.Column("applied_at", sa.DateTime(timezone=True)),
|
|
12
|
+
sa.Column("status", sa.Text),
|
|
13
|
+
sa.Column("execution_time_seconds", sa.Float),
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
history_table = sa.Table(
|
|
17
|
+
"delembic_run_history",
|
|
18
|
+
_metadata,
|
|
19
|
+
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
|
|
20
|
+
sa.Column("revision", sa.Text),
|
|
21
|
+
sa.Column("started_at", sa.DateTime(timezone=True)),
|
|
22
|
+
sa.Column("ended_at", sa.DateTime(timezone=True)),
|
|
23
|
+
sa.Column("duration_seconds", sa.Float),
|
|
24
|
+
sa.Column("status", sa.Text),
|
|
25
|
+
sa.Column("exception", sa.Text),
|
|
26
|
+
sa.Column("username", sa.Text),
|
|
27
|
+
sa.Column("hostname", sa.Text),
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def ensure_tables(conn: sa.Connection) -> None:
|
|
32
|
+
_metadata.create_all(conn, checkfirst=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_applied(conn: sa.Connection) -> set[str]:
|
|
36
|
+
rows = conn.execute(
|
|
37
|
+
sa.select(version_table.c.revision).where(version_table.c.status == "success")
|
|
38
|
+
)
|
|
39
|
+
return {row[0] for row in rows}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def record_result(
|
|
43
|
+
conn: sa.Connection,
|
|
44
|
+
revision: str,
|
|
45
|
+
started_at: datetime,
|
|
46
|
+
ended_at: datetime,
|
|
47
|
+
duration: float,
|
|
48
|
+
status: str,
|
|
49
|
+
exception: str | None = None,
|
|
50
|
+
username: str | None = None,
|
|
51
|
+
hostname: str | None = None,
|
|
52
|
+
) -> None:
|
|
53
|
+
conn.execute(
|
|
54
|
+
sa.delete(version_table).where(version_table.c.revision == revision)
|
|
55
|
+
)
|
|
56
|
+
conn.execute(
|
|
57
|
+
sa.insert(version_table).values(
|
|
58
|
+
revision=revision,
|
|
59
|
+
applied_at=ended_at,
|
|
60
|
+
status=status,
|
|
61
|
+
execution_time_seconds=duration,
|
|
62
|
+
)
|
|
63
|
+
)
|
|
64
|
+
conn.execute(
|
|
65
|
+
sa.insert(history_table).values(
|
|
66
|
+
revision=revision,
|
|
67
|
+
started_at=started_at,
|
|
68
|
+
ended_at=ended_at,
|
|
69
|
+
duration_seconds=duration,
|
|
70
|
+
status=status,
|
|
71
|
+
exception=exception,
|
|
72
|
+
username=username,
|
|
73
|
+
hostname=hostname,
|
|
74
|
+
)
|
|
75
|
+
)
|
delembic/executor.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import getpass
|
|
2
|
+
import socket
|
|
3
|
+
import time
|
|
4
|
+
import traceback
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Type
|
|
8
|
+
|
|
9
|
+
import sqlalchemy as sa
|
|
10
|
+
|
|
11
|
+
from delembic.alembic_compat import AlembicDepsError, check_alembic_deps
|
|
12
|
+
from delembic.dag import topological_sort
|
|
13
|
+
from delembic.db import ensure_tables, get_applied, record_result
|
|
14
|
+
from delembic.migration import DataMigration
|
|
15
|
+
from delembic.registry import load_migrations
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def run_upgrade(
|
|
19
|
+
engine: sa.Engine,
|
|
20
|
+
versions_dir: Path,
|
|
21
|
+
target: str = "head",
|
|
22
|
+
alembic_ini: Path | None = None,
|
|
23
|
+
) -> None:
|
|
24
|
+
migrations = load_migrations(versions_dir)
|
|
25
|
+
if not migrations:
|
|
26
|
+
print("No migrations found.")
|
|
27
|
+
return
|
|
28
|
+
|
|
29
|
+
order = topological_sort(migrations)
|
|
30
|
+
|
|
31
|
+
username = _safe(getpass.getuser)
|
|
32
|
+
hostname = _safe(socket.gethostname)
|
|
33
|
+
|
|
34
|
+
# meta_conn records delembic_version / run_history — always committed independently
|
|
35
|
+
# work_conn runs migration SQL — rolled back on failure without touching meta_conn
|
|
36
|
+
with engine.connect() as meta_conn:
|
|
37
|
+
ensure_tables(meta_conn)
|
|
38
|
+
meta_conn.commit()
|
|
39
|
+
applied = get_applied(meta_conn)
|
|
40
|
+
|
|
41
|
+
pending = [rev for rev in order if rev not in applied]
|
|
42
|
+
|
|
43
|
+
if target != "head":
|
|
44
|
+
if target not in order:
|
|
45
|
+
raise ValueError(f"Unknown target revision: {target}")
|
|
46
|
+
cutoff = order.index(target)
|
|
47
|
+
pending = [rev for rev in pending if order.index(rev) <= cutoff]
|
|
48
|
+
|
|
49
|
+
if not pending:
|
|
50
|
+
print("Already up to date.")
|
|
51
|
+
return
|
|
52
|
+
|
|
53
|
+
for revision in pending:
|
|
54
|
+
cls = migrations[revision]
|
|
55
|
+
instance = cls()
|
|
56
|
+
|
|
57
|
+
# Validate Alembic deps before attempting to run
|
|
58
|
+
alembic_deps = _external_deps(cls, migrations)
|
|
59
|
+
if alembic_deps:
|
|
60
|
+
if alembic_ini is None:
|
|
61
|
+
raise SystemExit(
|
|
62
|
+
f"Migration {revision} depends on Alembic revisions "
|
|
63
|
+
f"{alembic_deps} but alembic_config is not set in delembic.ini."
|
|
64
|
+
)
|
|
65
|
+
try:
|
|
66
|
+
check_alembic_deps(meta_conn, alembic_ini, alembic_deps, revision)
|
|
67
|
+
except AlembicDepsError as e:
|
|
68
|
+
print(f" BLOCKED: {e}")
|
|
69
|
+
raise SystemExit(1)
|
|
70
|
+
|
|
71
|
+
print(f"Running {revision}: {cls.description}")
|
|
72
|
+
started_at = datetime.now(timezone.utc)
|
|
73
|
+
t0 = time.monotonic()
|
|
74
|
+
|
|
75
|
+
with engine.connect() as work_conn:
|
|
76
|
+
try:
|
|
77
|
+
instance.upgrade(work_conn)
|
|
78
|
+
instance.validate(work_conn)
|
|
79
|
+
ended_at = datetime.now(timezone.utc)
|
|
80
|
+
duration = time.monotonic() - t0
|
|
81
|
+
work_conn.commit()
|
|
82
|
+
record_result(
|
|
83
|
+
meta_conn, revision, started_at, ended_at, duration,
|
|
84
|
+
"success", username=username, hostname=hostname,
|
|
85
|
+
)
|
|
86
|
+
meta_conn.commit()
|
|
87
|
+
print(f" OK ({duration:.2f}s)")
|
|
88
|
+
|
|
89
|
+
except Exception as e:
|
|
90
|
+
ended_at = datetime.now(timezone.utc)
|
|
91
|
+
duration = time.monotonic() - t0
|
|
92
|
+
tb = traceback.format_exc()
|
|
93
|
+
work_conn.rollback()
|
|
94
|
+
record_result(
|
|
95
|
+
meta_conn, revision, started_at, ended_at, duration,
|
|
96
|
+
"failed", exception=tb, username=username, hostname=hostname,
|
|
97
|
+
)
|
|
98
|
+
meta_conn.commit()
|
|
99
|
+
print(f" FAILED: {e}")
|
|
100
|
+
raise SystemExit(1)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _external_deps(
|
|
104
|
+
cls: Type[DataMigration],
|
|
105
|
+
migrations: dict[str, Type[DataMigration]],
|
|
106
|
+
) -> list[str]:
|
|
107
|
+
"""Return deps that are not Delembic revisions (i.e. Alembic or other external)."""
|
|
108
|
+
return [dep for dep in cls.depends_on if dep not in migrations]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _safe(fn):
|
|
112
|
+
try:
|
|
113
|
+
return fn()
|
|
114
|
+
except Exception:
|
|
115
|
+
return None
|
delembic/migration.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class DataMigration(ABC):
|
|
6
|
+
revision: str
|
|
7
|
+
depends_on: list[str] = []
|
|
8
|
+
description: str = ""
|
|
9
|
+
|
|
10
|
+
@abstractmethod
|
|
11
|
+
def upgrade(self, conn: Any) -> None: ...
|
|
12
|
+
|
|
13
|
+
def validate(self, conn: Any) -> None:
|
|
14
|
+
pass
|
delembic/registry.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import importlib.util
|
|
2
|
+
import sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Type
|
|
5
|
+
|
|
6
|
+
from delembic.migration import DataMigration
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def load_migrations(versions_dir: Path) -> dict[str, Type[DataMigration]]:
|
|
10
|
+
migrations: dict[str, Type[DataMigration]] = {}
|
|
11
|
+
|
|
12
|
+
for path in sorted(versions_dir.glob("*.py")):
|
|
13
|
+
if path.name.startswith("_"):
|
|
14
|
+
continue
|
|
15
|
+
|
|
16
|
+
spec = importlib.util.spec_from_file_location(path.stem, path)
|
|
17
|
+
if spec is None or spec.loader is None:
|
|
18
|
+
continue
|
|
19
|
+
|
|
20
|
+
module = importlib.util.module_from_spec(spec)
|
|
21
|
+
sys.modules[path.stem] = module
|
|
22
|
+
spec.loader.exec_module(module) # type: ignore[union-attr]
|
|
23
|
+
|
|
24
|
+
for name in dir(module):
|
|
25
|
+
obj = getattr(module, name)
|
|
26
|
+
if (
|
|
27
|
+
isinstance(obj, type)
|
|
28
|
+
and issubclass(obj, DataMigration)
|
|
29
|
+
and obj is not DataMigration
|
|
30
|
+
and hasattr(obj, "revision")
|
|
31
|
+
):
|
|
32
|
+
migrations[obj.revision] = obj
|
|
33
|
+
|
|
34
|
+
return migrations
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: delembic
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Data migration framework for versioning ETL and data operations alongside Alembic
|
|
5
|
+
Author-email: htshpradhan5@gmail.com
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Requires-Dist: click>=8.0
|
|
9
|
+
Requires-Dist: sqlalchemy>=2.0
|
|
10
|
+
Provides-Extra: dev
|
|
11
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
12
|
+
Provides-Extra: docs
|
|
13
|
+
Requires-Dist: furo>=2024.0; extra == 'docs'
|
|
14
|
+
Requires-Dist: myst-parser>=3.0; extra == 'docs'
|
|
15
|
+
Requires-Dist: sphinx-copybutton>=0.5; extra == 'docs'
|
|
16
|
+
Requires-Dist: sphinx>=7.0; extra == 'docs'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# Delembic
|
|
20
|
+
|
|
21
|
+
Data migration framework for Python — Alembic for ETL and data operations.
|
|
22
|
+
|
|
23
|
+
Alembic versions schema changes. Delembic versions **data** changes: vocabulary loads, ETL runs, reference data inserts, corrections. Together they describe the complete database state.
|
|
24
|
+
|
|
25
|
+
## Installation
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install delembic
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Quick Start
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
# 1. Initialize in your project
|
|
35
|
+
delembic init
|
|
36
|
+
|
|
37
|
+
# 2. Edit delembic.ini — set your database URL
|
|
38
|
+
# sqlalchemy.url = postgresql+psycopg://user:pass@host/dbname
|
|
39
|
+
|
|
40
|
+
# 3. Create a migration
|
|
41
|
+
delembic revision -m "load vocabulary"
|
|
42
|
+
|
|
43
|
+
# 4. Fill in the generated file, then run
|
|
44
|
+
delembic upgrade head
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Configuration
|
|
48
|
+
|
|
49
|
+
`delembic init` creates:
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
your-project/
|
|
53
|
+
├── delembic.ini # config — commit this
|
|
54
|
+
└── delembic/
|
|
55
|
+
├── env.py # optional connection helpers
|
|
56
|
+
└── versions/ # migration files live here
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
**`delembic.ini`:**
|
|
60
|
+
|
|
61
|
+
```ini
|
|
62
|
+
[delembic]
|
|
63
|
+
script_location = delembic
|
|
64
|
+
sqlalchemy.url = postgresql+psycopg://user:pass@localhost/mydb
|
|
65
|
+
alembic_config = alembic.ini
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
You can name the script folder anything:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
delembic init data-migrations
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Writing Migrations
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from delembic import DataMigration
|
|
78
|
+
|
|
79
|
+
class LoadVocabulary(DataMigration):
|
|
80
|
+
|
|
81
|
+
revision = "D001"
|
|
82
|
+
depends_on = []
|
|
83
|
+
description = "Load OMOP vocabulary tables"
|
|
84
|
+
|
|
85
|
+
def upgrade(self, conn):
|
|
86
|
+
conn.execute(...)
|
|
87
|
+
|
|
88
|
+
def validate(self, conn):
|
|
89
|
+
count = conn.execute("SELECT COUNT(*) FROM concept").scalar()
|
|
90
|
+
assert count > 0, "vocabulary load produced no rows"
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
`conn` is a SQLAlchemy `Connection`. `validate` is optional — migration is marked failed if it raises.
|
|
94
|
+
|
|
95
|
+
## Dependency Tracking
|
|
96
|
+
|
|
97
|
+
Migrations declare explicit dependencies. Delembic builds a DAG and runs them in topological order.
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
class LoadPerson(DataMigration):
|
|
101
|
+
revision = "D002"
|
|
102
|
+
depends_on = ["D001"] # waits for D001
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Alembic Integration
|
|
106
|
+
|
|
107
|
+
Point `alembic_config` at your `alembic.ini`:
|
|
108
|
+
|
|
109
|
+
```ini
|
|
110
|
+
[delembic]
|
|
111
|
+
alembic_config = alembic.ini
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
When you create a migration, Delembic automatically captures the current Alembic head:
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
delembic revision -m "load person"
|
|
118
|
+
# → depends_on = ['3d1e3e6abc12'] (current alembic head)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
At upgrade time, Delembic verifies the required Alembic revision has been applied before running:
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
BLOCKED: Migration D002 requires Alembic revision(s) 3d1e3e6abc12 to be applied first.
|
|
125
|
+
Run 'alembic upgrade head' before retrying.
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## CLI Reference
|
|
129
|
+
|
|
130
|
+
| Command | Description |
|
|
131
|
+
|---|---|
|
|
132
|
+
| `delembic init [DIR]` | Initialize project. DIR defaults to `delembic` |
|
|
133
|
+
| `delembic revision -m "msg"` | Generate new migration file |
|
|
134
|
+
| `delembic upgrade head` | Run all pending migrations |
|
|
135
|
+
| `delembic upgrade D003` | Run migrations up to D003 |
|
|
136
|
+
| `delembic current` | Show last applied revision |
|
|
137
|
+
| `delembic history` | List all migrations with status |
|
|
138
|
+
|
|
139
|
+
## Metadata Tables
|
|
140
|
+
|
|
141
|
+
Delembic creates two tables:
|
|
142
|
+
|
|
143
|
+
```sql
|
|
144
|
+
delembic_version -- current status per revision
|
|
145
|
+
delembic_run_history -- full audit log (start/end time, duration, exception, user, host)
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Failed migration work is rolled back. The failure record is always committed — audit trail survives transaction failures.
|
|
149
|
+
|
|
150
|
+
## Development
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
pip install -e ".[dev]"
|
|
154
|
+
pytest
|
|
155
|
+
```
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
delembic/__init__.py,sha256=eJsFgs5kAEKFOA0SaCRyWlLdMTqYJIn4nN1fZD1GAu8,96
|
|
2
|
+
delembic/alembic_compat.py,sha256=0zNE9D2o6JJlnju3FrEChcEWdty-gCdT4DSxCIMGses,2152
|
|
3
|
+
delembic/cli.py,sha256=8cDSZFhh_XTS93FQaBBRcWRM6FG6g-rabt5_fHnLTE0,5844
|
|
4
|
+
delembic/config.py,sha256=oTa_dKnijU6HimIFBWLmcvOmeJbeG3eHfy6Nr7dCN28,1267
|
|
5
|
+
delembic/dag.py,sha256=V5frlaQ765TzqwpEgi_4YPFygABGiihdVjFu17fUo_c,1193
|
|
6
|
+
delembic/db.py,sha256=qFxKEqULtfSq8PPDbNLG_3GPLNj5r1ia5C8ea1rGKsA,2048
|
|
7
|
+
delembic/executor.py,sha256=M8s31fO3y0OJUlfM7yA7t0kJGle1GSEiVNRb_csP1g8,4019
|
|
8
|
+
delembic/migration.py,sha256=Vyt1cFQXtS8JseZ9kSQzUAyBkuOFEe1W7OYEkxSya6w,286
|
|
9
|
+
delembic/registry.py,sha256=k4bCp61w3oPhEZEEVM6i_vuYw7C-8xlzA1kty-oPNWU,1028
|
|
10
|
+
delembic-0.2.0.dist-info/METADATA,sha256=euid0K4SstknuoX8pBx3TsErkjFlQzq7YnHp5RKfh_I,3881
|
|
11
|
+
delembic-0.2.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
12
|
+
delembic-0.2.0.dist-info/entry_points.txt,sha256=R_iUBPB80NsK7iYsnUoyCKpwD3qbH8I8V6XtdQWB_Q4,46
|
|
13
|
+
delembic-0.2.0.dist-info/RECORD,,
|